From 46783a6036f4c8f3315b2684d182bb75f378b62f Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 04:23:41 +0200 Subject: [PATCH 01/11] refactor(interpreter): per-engine module trees + a DuckDB engine stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the existing engine under src/datafusion/ and hoist what is engine-agnostic to the crate root: Value marshalling (value.rs), the Base/FieldType/Schema vocabulary (types.rs), InterpError (error.rs). schema.rs/lookup.rs were already engine-free. datafusion::{expr,plan, types} re-export the moved names so engine-internal paths still resolve. src/duckdb/ adds DuckDBInferFn: same Python API as InferFn minus the transformer callout. Stub only — the constructor parses in the DuckDB dialect, infer() raises NotImplementedError. Co-Authored-By: Claude Opus 5 --- sql_transform/_interpreter.pyi | 23 + src/duckdb/mod.rs | 56 ++ src/error.rs | 20 + src/expr.rs | 869 --------------------- src/expr_build.rs | 278 ------- src/lib.rs | 362 +-------- src/lookup.rs | 4 +- src/plan.rs | 1253 ------------------------------ src/schema.rs | 2 +- src/types.rs | 315 +------- src/value.rs | 224 ++++++ tests/test_duckdb_interpreter.py | 25 + 12 files changed, 375 insertions(+), 3056 deletions(-) create mode 100644 src/duckdb/mod.rs create mode 100644 src/error.rs delete mode 100644 src/expr.rs delete mode 100644 src/expr_build.rs delete mode 100644 src/plan.rs create mode 100644 src/value.rs create mode 100644 tests/test_duckdb_interpreter.py diff --git a/sql_transform/_interpreter.pyi b/sql_transform/_interpreter.pyi index 725d546..699af87 100644 --- a/sql_transform/_interpreter.pyi +++ b/sql_transform/_interpreter.pyi @@ -19,3 +19,26 @@ class InferFn: tables: dict[str, list[Any]] | None = None, **kwargs: list[Any], ) -> list[BaseModel]: ... + +class DuckDBInferFn: + """DuckDB-semantics interpreter: the same API as `InferFn` without the + transformer callout. + + Stub -- the SQL is parsed in the DuckDB dialect and nothing else; no output + model is derived and `infer()` raises `NotImplementedError`. + """ + + output_model: type[BaseModel] | None + + def __init__( + self, + sql: str, + row_tables: dict[str, type[BaseModel]], + static_tables: dict[str, pa.Table], + output_model: type[BaseModel] | None = None, + ) -> None: ... + def infer( + self, + tables: dict[str, list[Any]] | None = None, + **kwargs: list[Any], + ) -> list[BaseModel]: ... diff --git a/src/duckdb/mod.rs b/src/duckdb/mod.rs new file mode 100644 index 0000000..bcdf907 --- /dev/null +++ b/src/duckdb/mod.rs @@ -0,0 +1,56 @@ +//! The DuckDB-semantics interpreter: `DuckDBInferFn`. +//! +//! Same Python API as `InferFn` minus the transformer callout. Stub: the SQL +//! is parsed in the DuckDB dialect (so the wiring is real) and nothing else +//! happens — `infer()` raises rather than returning results that would +//! silently carry DataFusion semantics. +//! +//! Plan building, expression evaluation and type inference get their own +//! modules here as they land; none of them exist yet. + +use std::collections::HashMap; + +use pyo3::exceptions::PyNotImplementedError; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use sqlparser::dialect::DuckDbDialect; +use sqlparser::parser::Parser; + +use crate::error::InterpError; + +#[pyclass] +pub struct DuckDBInferFn { + /// No output model is derived yet, so this is whatever the caller declared + /// (or `None`) rather than a synthesized one. + #[pyo3(get)] + output_model: Option>, +} + +#[pymethods] +impl DuckDBInferFn { + #[new] + #[pyo3(signature = (sql, row_tables, static_tables, output_model=None))] + fn new( + sql: String, + row_tables: HashMap>, + static_tables: HashMap>, + output_model: Option>, + ) -> PyResult { + let _ = (row_tables, static_tables); + Parser::parse_sql(&DuckDbDialect {}, &sql) + .map_err(|e| InterpError::Build(format!("SQL parse error: {e}")))?; + Ok(DuckDBInferFn { output_model }) + } + + #[pyo3(signature = (tables=None, **kwargs))] + fn infer( + &self, + tables: Option>>>, + kwargs: Option>, + ) -> PyResult>> { + let _ = (tables, kwargs); + Err(PyNotImplementedError::new_err( + "the DuckDB interpreter is a stub; use InferFn", + )) + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..0f3899a --- /dev/null +++ b/src/error.rs @@ -0,0 +1,20 @@ +//! The one error type every interpreter engine reports through. + +use pyo3::exceptions::{PyKeyError, PyValueError}; +use pyo3::PyErr; + +pub enum InterpError { + Build(String), + MissingKey(String), + Eval(String), +} + +impl From for PyErr { + fn from(e: InterpError) -> PyErr { + match e { + InterpError::Build(msg) => PyValueError::new_err(msg), + InterpError::MissingKey(msg) => PyKeyError::new_err(msg), + InterpError::Eval(msg) => PyValueError::new_err(msg), + } + } +} diff --git a/src/expr.rs b/src/expr.rs deleted file mode 100644 index a08298b..0000000 --- a/src/expr.rs +++ /dev/null @@ -1,869 +0,0 @@ -use pyo3::prelude::*; -use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString}; - -use std::sync::Arc; - -use crate::types::FieldType; - -#[derive(Debug)] -pub enum Value { - Int(i64), - Float(f64), - Str(String), - Bool(bool), - Null, - /// Opaque passthrough for row values that aren't a SQL primitive - /// (e.g. a nested dict). Round-trips unchanged through column refs; - /// arithmetic/comparison on it is a runtime error. - Object(Py), - /// Ordered field list (name, value). Field order is significant for - /// equality/hash, mirroring `Base::Struct`. - Struct(Vec<(String, Value)>), - /// Ordered element list. - List(Vec), -} - -impl Clone for Value { - // Py isn't Clone (cloning it requires a GIL token to bump the - // refcount safely), so this can't be derived; Python::attach supplies - // the token for the Object case. - fn clone(&self) -> Self { - match self { - Value::Int(i) => Value::Int(*i), - Value::Float(f) => Value::Float(*f), - Value::Str(s) => Value::Str(s.clone()), - Value::Bool(b) => Value::Bool(*b), - Value::Null => Value::Null, - Value::Object(o) => Python::attach(|py| Value::Object(o.clone_ref(py))), - Value::Struct(fields) => { - Value::Struct(fields.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) - } - Value::List(items) => Value::List(items.iter().map(|v| v.clone()).collect()), - } - } -} - -impl PartialEq for Value { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Value::Int(a), Value::Int(b)) => a == b, - (Value::Float(a), Value::Float(b)) => a == b, - (Value::Str(a), Value::Str(b)) => a == b, - (Value::Bool(a), Value::Bool(b)) => a == b, - (Value::Null, Value::Null) => true, - (Value::Object(a), Value::Object(b)) => a.as_ptr() == b.as_ptr(), - (Value::Struct(a), Value::Struct(b)) => a == b, - (Value::List(a), Value::List(b)) => a == b, - _ => false, - } - } -} - -impl Eq for Value {} - -impl std::hash::Hash for Value { - fn hash(&self, state: &mut H) { - match self { - Value::Int(i) => { - 0u8.hash(state); - i.hash(state); - } - Value::Float(f) => { - 1u8.hash(state); - f.to_bits().hash(state); - } - Value::Str(s) => { - 2u8.hash(state); - s.hash(state); - } - Value::Bool(b) => { - 3u8.hash(state); - b.hash(state); - } - Value::Null => 4u8.hash(state), - Value::Object(o) => { - 5u8.hash(state); - (o.as_ptr() as usize).hash(state); - } - Value::Struct(fields) => { - 6u8.hash(state); - fields.hash(state); - } - Value::List(items) => { - 7u8.hash(state); - items.hash(state); - } - } - } -} - -/// Human-readable type name for error messages. -pub fn type_name(v: &Value) -> &'static str { - match v { - Value::Int(_) => "int", - Value::Float(_) => "float", - Value::Str(_) => "string", - Value::Bool(_) => "bool", - Value::Null => "null", - Value::Object(_) => "object", - Value::Struct(_) => "struct", - Value::List(_) => "list", - } -} - -/// String form used by CONCAT and CAST(.. AS VARCHAR). -pub fn display_value(v: &Value) -> String { - match v { - Value::Int(i) => i.to_string(), - // `{:?}` matches Arrow/DataFusion's Float64->Utf8 rendering (1.0 -> "1.0", - // 1e300 -> "1e300"), unlike `to_string` (1.0 -> "1", 1e300 -> 300 digits) - // -- EXCEPT |x| in [1e-5, 1e-4): there `{:?}` uses exponential ('1e-5') - // but DataFusion stays fixed-decimal ('0.00001'). `{}` is fixed-point - // shortest-round-trip and matches the oracle for exactly that band (the - // only magnitude where the two disagree -- see TASK-7 probe). - Value::Float(f) => { - if (1e-5..1e-4).contains(&f.abs()) { - format!("{f}") - } else { - format!("{f:?}") - } - } - Value::Str(s) => s.clone(), - Value::Bool(b) => b.to_string(), - Value::Null => String::new(), - Value::Object(_) => "".to_string(), - Value::Struct(fields) => { - let inner = fields - .iter() - .map(|(k, v)| format!("{k}: {}", display_value(v))) - .collect::>() - .join(", "); - format!("{{{inner}}}") - } - Value::List(items) => { - let inner = items - .iter() - .map(display_value) - .collect::>() - .join(", "); - format!("[{inner}]") - } - } -} - -impl Value { - pub fn from_pyobject(obj: &Bound<'_, PyAny>) -> PyResult { - if obj.is_none() { - return Ok(Value::Null); - } - if let Ok(b) = obj.cast::() { - return Ok(Value::Bool(b.is_true())); - } - if let Ok(i) = obj.cast::() { - return Ok(Value::Int(i.extract::()?)); - } - if let Ok(f) = obj.cast::() { - return Ok(Value::Float(f.extract::()?)); - } - if let Ok(s) = obj.cast::() { - return Ok(Value::Str(s.extract::()?)); - } - Ok(Value::Object(obj.clone().unbind())) - } - - /// Schema-driven read: converts a Python value into a `Value` per the - /// field's declared `Base`, recursing into `Struct`/`List` so a nested - /// dict/list is marshalled by its declared shape rather than falling - /// through to an opaque `Value::Object`. Scalars behave exactly like - /// `from_pyobject`. `obj` may be a raw `dict`/`list` OR (when the row - /// model declares a nested pydantic submodel / `list[X]`) an already- - /// validated nested `BaseModel` instance / `list` — struct field access - /// falls back from dict indexing to attribute access to cover both. - pub fn from_pyobject_typed( - obj: &Bound<'_, PyAny>, - base: &crate::types::Base, - ) -> PyResult { - use crate::types::Base; - if obj.is_none() { - return Ok(Value::Null); - } - match base { - Base::Struct(fields) => { - // Accept a dict (read by key) or a pydantic-model-like object - // (read by attr) as struct-shaped input; anything else (e.g. - // a bare scalar) is a genuine type mismatch and must error, - // not silently marshal into an all-null struct. - let dict = obj.cast::().ok(); - // Probe `model_fields` on the CLASS, not the instance: instance - // access is deprecated in Pydantic 2.11 (PydanticDeprecatedSince211) - // and removed in 3.0, where the instance probe would return false - // and misclassify a struct value. Class access stays valid. - if dict.is_none() && !obj.get_type().hasattr("model_fields").unwrap_or(false) { - return Err(pyo3::exceptions::PyValueError::new_err(format!( - "Expected a struct/dict value for a struct-typed field: got {}", - obj.get_type().name()? - ))); - } - let mut out = Vec::with_capacity(fields.len()); - for (name, field_ft) in fields { - let field_val = if let Some(dict) = &dict { - dict.get_item(name)? - } else { - obj.getattr(name.as_str()).ok() - }; - let v = match field_val { - Some(item) => Value::from_pyobject_typed(&item, &field_ft.base)?, - None => Value::Null, - }; - out.push((name.clone(), v)); - } - Ok(Value::Struct(out)) - } - Base::List(inner) => { - let list = obj.cast::().map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!( - "Expected a list value for a list-typed field: {e}" - )) - })?; - let mut out = Vec::with_capacity(list.len()); - for item in list.iter() { - out.push(Value::from_pyobject_typed(&item, &inner.base)?); - } - Ok(Value::List(out)) - } - _ => Value::from_pyobject(obj), - } - } - - pub fn to_pyobject(&self, py: Python<'_>) -> PyResult> { - Ok(match self { - Value::Int(i) => i.into_pyobject(py)?.into_any().unbind(), - Value::Float(f) => f.into_pyobject(py)?.into_any().unbind(), - Value::Str(s) => s.into_pyobject(py)?.into_any().unbind(), - Value::Bool(b) => b.into_pyobject(py)?.to_owned().into_any().unbind(), - Value::Null => py.None(), - Value::Object(o) => o.clone_ref(py), - Value::Struct(fields) => { - let dict = PyDict::new(py); - for (k, v) in fields { - dict.set_item(k, v.to_pyobject(py)?)?; - } - dict.into_any().unbind() - } - Value::List(items) => { - let elements = items - .iter() - .map(|v| v.to_pyobject(py)) - .collect::>>()?; - PyList::new(py, elements)?.into_any().unbind() - } - }) - } -} - -#[derive(Clone)] -pub enum Expr { - Column { - table: Option, - name: String, - }, - Literal(Value), - BinaryOp { - op: BinOp, - left: Box, - right: Box, - }, - Not(Box), - Function { - name: String, - args: Vec, - }, - Cast { - expr: Box, - target: CastType, - }, - Struct(Vec<(String, Expr)>), - List(Vec), - FieldAccess { - base: Box, - field: String, - }, - /// Callout to an opaque, already-fitted Python transformer. `obj` is the - /// fitted sklearn object; `input_features` is its `feature_names_in_` - /// (the struct arg is reordered to this before `.transform`); - /// `output_fields` is the caller-declared output schema (marshalling - /// target, order significant); `arg` evaluates to the input `Value::Struct`. - Transform { - obj: Arc>, - input_features: Vec, - output_fields: Vec<(String, FieldType)>, - arg: Box, - }, - /// SQL CASE. `arms` are (condition, result) pairs evaluated left to right; - /// the first arm whose condition is `Bool(true)` wins. `default` is the ELSE - /// result, or `None` when there is no ELSE (an unmatched row yields NULL). - Case { - arms: Vec<(Expr, Expr)>, - default: Option>, - }, -} - -#[derive(Clone, Copy)] -pub enum CastType { - Str, - Int, - Float, - Bool, -} - -#[derive(Clone, Copy, PartialEq)] -pub enum BinOp { - Add, - Sub, - Mul, - Div, - Mod, - Eq, - NotEq, - Lt, - Gt, - LtEq, - GtEq, - And, - Or, - Concat, -} - -pub fn eval(expr: &Expr, row: &crate::plan::Row) -> Result { - match expr { - Expr::Column { table, name } => resolve_column(row, table.as_deref(), name), - Expr::Literal(v) => Ok(v.clone()), - Expr::BinaryOp { op, left, right } => { - let l = eval(left, row)?; - let r = eval(right, row)?; - eval_binary_op(*op, l, r) - } - Expr::Not(inner) => { - let v = eval(inner, row)?; - match as_tribool(&v)? { - Some(b) => Ok(Value::Bool(!b)), - None => Ok(Value::Null), - } - } - Expr::Function { name, args } => { - let values: Vec = args - .iter() - .map(|a| eval(a, row)) - .collect::>()?; - eval_builtin(name, values) - } - Expr::Cast { expr, target } => { - let v = eval(expr, row)?; - eval_cast(v, *target) - } - Expr::Struct(fields) => { - let values = fields - .iter() - .map(|(k, e)| Ok((k.clone(), eval(e, row)?))) - .collect::>()?; - Ok(Value::Struct(values)) - } - Expr::List(items) => { - let values = items - .iter() - .map(|e| eval(e, row)) - .collect::>()?; - Ok(Value::List(values)) - } - Expr::FieldAccess { base, field } => { - let v = eval(base, row)?; - match v { - Value::Null => Ok(Value::Null), - Value::Struct(fields) => fields - .into_iter() - .find(|(name, _)| name == field) - .map(|(_, v)| v) - .ok_or_else(|| { - crate::plan::InterpError::Eval(format!("Unknown struct field: {field}")) - }), - other => Err(crate::plan::InterpError::Eval(format!( - "Cannot access field '{field}' on a {} value", - type_name(&other) - ))), - } - } - Expr::Transform { - obj, - input_features, - output_fields, - arg, - } => { - let fields = match eval(arg, row)? { - Value::Struct(f) => f, - Value::Null => return Ok(Value::Null), - other => { - return Err(crate::plan::InterpError::Eval(format!( - "transformer argument must be a struct, got a {} value", - type_name(&other) - ))) - } - }; - // infer() already holds the GIL; attach is cheap and re-entrant, so - // eval() stays a pure-Rust signature (no py token threaded through). - Python::attach(|py| -> Result { - // Reorder the struct's fields to feature_names_in_ order, then - // build a 1-row Python list-of-lists. sklearn's check_array - // coerces it -- no numpy on the Rust side. - let mut ordered: Vec> = Vec::with_capacity(input_features.len()); - for feat in input_features { - let value = fields - .iter() - .find(|(n, _)| n == feat) - .map(|(_, v)| v) - .ok_or_else(|| { - crate::plan::InterpError::Eval(format!( - "transformer input struct is missing field '{feat}'" - )) - })?; - let py_val = value.to_pyobject(py).map_err(|e| { - crate::plan::InterpError::Eval(format!( - "marshalling transformer input failed: {e}" - )) - })?; - ordered.push(py_val); - } - let row_list = PyList::new(py, ordered).map_err(|e| { - crate::plan::InterpError::Eval(format!( - "building transformer input row failed: {e}" - )) - })?; - let x = PyList::new(py, [row_list]).map_err(|e| { - crate::plan::InterpError::Eval(format!( - "building transformer input matrix failed: {e}" - )) - })?; - let y = obj.bind(py).call_method1("transform", (x,)).map_err(|e| { - crate::plan::InterpError::Eval(format!("transformer.transform failed: {e}")) - })?; - // .tolist() turns numpy scalars into Python builtins so - // from_pyobject sees float/int/str, not opaque numpy objects. - let y_list = y.call_method0("tolist").map_err(|e| { - crate::plan::InterpError::Eval(format!( - "transformer output .tolist() failed: {e}" - )) - })?; - let y0 = y_list.get_item(0).map_err(|e| { - crate::plan::InterpError::Eval(format!( - "transformer produced no output row: {e}" - )) - })?; - // Marshal each output position through the declared field - // type. Parity invariant: the declared output dtype must equal - // the transform's natural dtype -- here it drives pydantic - // coercion at model-validate time, and the DataFusion oracle - // reaches the same type via a pyarrow cast; the two agree only - // when no real coercion is needed (see _transformer_udf.py). - let mut out = Vec::with_capacity(output_fields.len()); - for (i, (fname, ft)) in output_fields.iter().enumerate() { - let elem = y0.get_item(i).map_err(|e| { - crate::plan::InterpError::Eval(format!( - "transformer output missing position {i} for field '{fname}': {e}" - )) - })?; - let val = Value::from_pyobject_typed(&elem, &ft.base).map_err(|e| { - crate::plan::InterpError::Eval(format!( - "marshalling transformer output field '{fname}' failed: {e}" - )) - })?; - out.push((fname.clone(), val)); - } - Ok(Value::Struct(out)) - }) - } - Expr::Case { arms, default } => { - // Short-circuit: evaluate conditions left to right, stop at the first - // Bool(true), and evaluate ONLY that arm's result. A non-matching - // arm's result is never touched, so CASE WHEN x>0 THEN 1/x ELSE 0 END - // does not divide by zero at x=0 -- matching the oracle. - for (cond, result) in arms { - if let Some(true) = as_tribool(&eval(cond, row)?)? { - return eval(result, row); - } - } - match default { - Some(d) => eval(d, row), - None => Ok(Value::Null), - } - } - } -} - -fn resolve_column( - row: &crate::plan::Row, - table: Option<&str>, - name: &str, -) -> Result { - use crate::plan::InterpError; - - if let Some(t) = table { - return row - .get(t) - .and_then(|cols| cols.get(name)) - .cloned() - .ok_or_else(|| InterpError::Build(format!("Unknown column: {t}.{name}"))); - } - let mut found: Option<&Value> = None; - for cols in row.values() { - if let Some(v) = cols.get(name) { - if found.is_some() { - return Err(InterpError::Build(format!( - "Ambiguous column reference: {name}" - ))); - } - found = Some(v); - } - } - found - .cloned() - .ok_or_else(|| InterpError::Build(format!("Unknown column: {name}"))) -} - -fn eval_binary_op(op: BinOp, l: Value, r: Value) -> Result { - match op { - BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div | BinOp::Mod => arithmetic(op, l, r), - BinOp::Eq | BinOp::NotEq | BinOp::Lt | BinOp::Gt | BinOp::LtEq | BinOp::GtEq => { - comparison(op, l, r) - } - BinOp::And | BinOp::Or => logic(op, l, r), - BinOp::Concat => concat_op(l, r), - } -} - -/// `||` string concatenation. NULL-propagating (SQL/DataFusion semantics: any -/// NULL operand yields NULL, unlike CONCAT which skips NULLs). -fn concat_op(l: Value, r: Value) -> Result { - if matches!(l, Value::Null) || matches!(r, Value::Null) { - return Ok(Value::Null); - } - Ok(Value::Str(format!( - "{}{}", - display_value(&l), - display_value(&r) - ))) -} - -fn arithmetic(op: BinOp, l: Value, r: Value) -> Result { - if matches!(l, Value::Null) || matches!(r, Value::Null) { - return Ok(Value::Null); - } - match (l, r) { - (Value::Int(a), Value::Int(b)) => { - if matches!(op, BinOp::Div | BinOp::Mod) && b == 0 { - return Err(crate::plan::InterpError::Eval( - "division by zero".to_string(), - )); - } - Ok(match op { - BinOp::Add => Value::Int(a + b), - BinOp::Sub => Value::Int(a - b), - BinOp::Mul => Value::Int(a * b), - BinOp::Div => Value::Int(a / b), - BinOp::Mod => Value::Int(a % b), - _ => unreachable!(), - }) - } - (a, b) => { - let af = as_f64(&a)?; - let bf = as_f64(&b)?; - Ok(match op { - BinOp::Add => Value::Float(af + bf), - BinOp::Sub => Value::Float(af - bf), - BinOp::Mul => Value::Float(af * bf), - BinOp::Div => Value::Float(af / bf), - BinOp::Mod => Value::Float(af % bf), - _ => unreachable!(), - }) - } - } -} - -fn as_f64(v: &Value) -> Result { - match v { - Value::Int(i) => Ok(*i as f64), - Value::Float(f) => Ok(*f), - other => Err(crate::plan::InterpError::Eval(format!( - "Cannot use a {} value in an arithmetic expression", - type_name(other) - ))), - } -} - -fn comparison(op: BinOp, l: Value, r: Value) -> Result { - if matches!(l, Value::Null) || matches!(r, Value::Null) { - return Ok(Value::Null); - } - // Structs/lists support deep structural equality (DataFusion parity) but - // not ordering; route Eq/NotEq through the existing structural `PartialEq` - // on `Value` before falling into `compare_values`, which only handles - // scalars and would error on a container. - if matches!(op, BinOp::Eq | BinOp::NotEq) - && matches!( - (&l, &r), - (Value::Struct(_), _) - | (Value::List(_), _) - | (_, Value::Struct(_)) - | (_, Value::List(_)) - ) - { - let eq = l == r; - return Ok(Value::Bool(if op == BinOp::Eq { eq } else { !eq })); - } - let ordering = compare_values(&l, &r)?; - Ok(Value::Bool(match op { - BinOp::Eq => ordering == std::cmp::Ordering::Equal, - BinOp::NotEq => ordering != std::cmp::Ordering::Equal, - BinOp::Lt => ordering == std::cmp::Ordering::Less, - BinOp::Gt => ordering == std::cmp::Ordering::Greater, - BinOp::LtEq => ordering != std::cmp::Ordering::Greater, - BinOp::GtEq => ordering != std::cmp::Ordering::Less, - _ => unreachable!(), - })) -} - -fn compare_values(l: &Value, r: &Value) -> Result { - match (l, r) { - (Value::Int(a), Value::Int(b)) => Ok(a.cmp(b)), - (Value::Str(a), Value::Str(b)) => Ok(a.cmp(b)), - (Value::Bool(a), Value::Bool(b)) => Ok(a.cmp(b)), - (a, b) => { - let af = as_f64(a)?; - let bf = as_f64(b)?; - // DataFusion float ordering: NaN == NaN and NaN sorts greatest. - // Keep partial_cmp for the normal path (so 0.0 == -0.0) and only - // special-case NaN, which partial_cmp reports as incomparable. - Ok(af - .partial_cmp(&bf) - .unwrap_or_else(|| match (af.is_nan(), bf.is_nan()) { - (true, true) => std::cmp::Ordering::Equal, - (true, false) => std::cmp::Ordering::Greater, - (false, true) => std::cmp::Ordering::Less, - (false, false) => unreachable!("partial_cmp only fails on NaN"), - })) - } - } -} - -fn logic(op: BinOp, l: Value, r: Value) -> Result { - let lb = as_tribool(&l)?; - let rb = as_tribool(&r)?; - Ok(match op { - BinOp::And => match (lb, rb) { - (Some(false), _) | (_, Some(false)) => Value::Bool(false), - (Some(true), Some(true)) => Value::Bool(true), - _ => Value::Null, - }, - BinOp::Or => match (lb, rb) { - (Some(true), _) | (_, Some(true)) => Value::Bool(true), - (Some(false), Some(false)) => Value::Bool(false), - _ => Value::Null, - }, - _ => unreachable!(), - }) -} - -fn as_tribool(v: &Value) -> Result, crate::plan::InterpError> { - match v { - Value::Bool(b) => Ok(Some(*b)), - Value::Null => Ok(None), - other => Err(crate::plan::InterpError::Eval(format!( - "Expected a boolean expression, got a {} value", - type_name(other) - ))), - } -} - -fn eval_builtin(name: &str, args: Vec) -> Result { - use crate::plan::InterpError; - - if matches!(name, "upper" | "lower" | "trim" | "substr" | "substring") - && args.iter().any(|a| matches!(a, Value::Null)) - { - return Ok(Value::Null); - } - - match name { - "upper" => Ok(Value::Str(as_str(&args, 0)?.to_uppercase())), - "lower" => Ok(Value::Str(as_str(&args, 0)?.to_lowercase())), - "trim" => Ok(Value::Str(as_str(&args, 0)?.trim().to_string())), - "concat" => { - let mut s = String::new(); - for a in &args { - if !matches!(a, Value::Null) { - s.push_str(&display_value(a)); - } - } - Ok(Value::Str(s)) - } - "abs" => match &args[0] { - Value::Int(i) => Ok(Value::Int(i.abs())), - Value::Float(f) => Ok(Value::Float(f.abs())), - Value::Null => Ok(Value::Null), - other => Err(InterpError::Eval(format!( - "ABS expects a number, got a {} value", - type_name(other) - ))), - }, - "round" => match &args[0] { - Value::Float(f) => Ok(Value::Float(f.round())), - // DataFusion's ROUND returns Float64 even for an integer arg. - Value::Int(i) => Ok(Value::Float(*i as f64)), - Value::Null => Ok(Value::Null), - other => Err(InterpError::Eval(format!( - "ROUND expects a number, got a {} value", - type_name(other) - ))), - }, - "substr" | "substring" => { - let s = as_str(&args, 0)?; - let start = as_i64(&args, 1)?; - let length = if args.len() > 2 { - Some(as_i64(&args, 2)?) - } else { - None - }; - Ok(Value::Str(substr(s, start, length))) - } - "coalesce" => Ok(args - .into_iter() - .find(|v| !matches!(v, Value::Null)) - .unwrap_or(Value::Null)), - "nullif" => { - if args.len() != 2 { - return Err(InterpError::Eval("NULLIF expects 2 arguments".to_string())); - } - // DataFusion coerces numerically (NULLIF(1, 1.0) -> NULL), so - // compare through the numeric-aware comparison, not Value's - // variant-tagged PartialEq. - let equal = matches!( - comparison(BinOp::Eq, args[0].clone(), args[1].clone())?, - Value::Bool(true) - ); - if equal { - Ok(Value::Null) - } else { - Ok(args[0].clone()) - } - } - other => Err(InterpError::Eval(format!("Unknown function: {other}"))), - } -} - -fn as_str(args: &[Value], idx: usize) -> Result<&str, crate::plan::InterpError> { - match args.get(idx) { - Some(Value::Str(s)) => Ok(s.as_str()), - other => Err(crate::plan::InterpError::Eval(format!( - "Expected a string argument at position {idx}, got {:?}", - other.map(type_name) - ))), - } -} - -fn as_i64(args: &[Value], idx: usize) -> Result { - match args.get(idx) { - Some(Value::Int(i)) => Ok(*i), - other => Err(crate::plan::InterpError::Eval(format!( - "Expected an integer argument at position {idx}, got {:?}", - other.map(type_name) - ))), - } -} - -fn substr(s: &str, start: i64, length: Option) -> String { - // Postgres/DataFusion windowing: the 1-based window is [start, start+length); - // positions < 1 are consumed by the length. SUBSTR('hello',0,3) -> 'he', - // SUBSTR('hello',-2,5) -> 'he'. `start` clamps up to 1; the end is - // start+length-1 (0-based exclusive), both clamped into [0, len]. - let chars: Vec = s.chars().collect(); - let len = chars.len() as i64; - let begin = (start.max(1) - 1).clamp(0, len) as usize; - let end = match length { - Some(l) => (start + l - 1).clamp(0, len) as usize, - None => len as usize, - }; - let end = end.max(begin); - chars[begin..end].iter().collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn struct_and_list_value_equality() { - let a = Value::Struct(vec![("x".into(), Value::Int(1))]); - let b = Value::Struct(vec![("x".into(), Value::Int(1))]); - let c = Value::List(vec![Value::Int(1), Value::Int(2)]); - assert_eq!(a, b); - assert_ne!(a, c); - assert_eq!(c.clone(), c); - } -} - -fn eval_cast(v: Value, target: CastType) -> Result { - use crate::plan::InterpError; - - if matches!(v, Value::Null) { - return Ok(Value::Null); - } - Ok(match target { - CastType::Str => Value::Str(display_value(&v)), - CastType::Int => match v { - Value::Int(i) => Value::Int(i), - Value::Float(f) => Value::Int(f.trunc() as i64), - // No trim: DataFusion's Utf8->Int cast rejects surrounding - // whitespace (CAST(' 42 ' AS BIGINT) errors), so we must too. - Value::Str(s) => Value::Int( - s.parse::() - .map_err(|_| InterpError::Eval(format!("Cannot cast '{s}' to INT")))?, - ), - Value::Bool(b) => Value::Int(b as i64), - Value::Null | Value::Object(_) | Value::Struct(_) | Value::List(_) => { - return Err(InterpError::Eval( - "Cannot cast this value to INT".to_string(), - )) - } - }, - CastType::Float => match v { - Value::Int(i) => Value::Float(i as f64), - Value::Float(f) => Value::Float(f), - // No trim: match DataFusion, which rejects surrounding whitespace. - Value::Str(s) => Value::Float( - s.parse::() - .map_err(|_| InterpError::Eval(format!("Cannot cast '{s}' to FLOAT")))?, - ), - Value::Bool(b) => Value::Float(if b { 1.0 } else { 0.0 }), - Value::Null | Value::Object(_) | Value::Struct(_) | Value::List(_) => { - return Err(InterpError::Eval( - "Cannot cast this value to FLOAT".to_string(), - )) - } - }, - CastType::Bool => match v { - Value::Bool(b) => Value::Bool(b), - Value::Int(i) => Value::Bool(i != 0), - Value::Float(f) => Value::Bool(f != 0.0), - // DataFusion/Arrow's Utf8->Boolean accept set (case-insensitive); - // anything else errors, unlike the old "only 'true'" behavior. - Value::Str(s) => match s.to_ascii_lowercase().as_str() { - "true" | "t" | "yes" | "y" | "1" | "on" => Value::Bool(true), - "false" | "f" | "no" | "n" | "0" | "off" => Value::Bool(false), - _ => return Err(InterpError::Eval(format!("Cannot cast '{s}' to BOOLEAN"))), - }, - Value::Null | Value::Object(_) | Value::Struct(_) | Value::List(_) => { - return Err(InterpError::Eval( - "Cannot cast this value to BOOLEAN".to_string(), - )) - } - }, - }) -} diff --git a/src/expr_build.rs b/src/expr_build.rs deleted file mode 100644 index 899835f..0000000 --- a/src/expr_build.rs +++ /dev/null @@ -1,278 +0,0 @@ -use sqlparser::ast::{ - Array, BinaryOperator, DataType, Expr as SqlExpr, Function, FunctionArg, FunctionArgExpr, - FunctionArguments, Ident, UnaryOperator, Value as SqlValue, -}; - -use crate::expr::{BinOp, CastType, Expr, Value}; -use crate::plan::InterpError; - -/// DataFusion identifier folding: an unquoted identifier folds to lowercase, a -/// double-quoted one stays case-exact. sqlparser keeps the unquoted text in -/// `.value` regardless, and the quoting in `.quote_style` -- so the fold key is -/// purely whether a quote style was present. Matches the oracle so `SELECT Age` -/// on a column named `Age` folds to `age` and misses. -pub fn fold_ident(ident: &Ident) -> String { - match ident.quote_style { - Some(_) => ident.value.clone(), - None => ident.value.to_lowercase(), - } -} - -pub fn convert_expr(e: &SqlExpr) -> Result { - match e { - SqlExpr::Identifier(ident) => Ok(Expr::Column { - table: None, - name: fold_ident(ident), - }), - // `a.b` is ambiguous at parse time -- it's either `table.column` or - // `struct_col.field`, and we don't know the relation-alias set here - // (that only exists once the row/static tables are supplied, in - // plan::validate_columns). Parse the first two parts as - // Column{table,name} (today's `table.column` shape) and layer any - // further dotted parts on top as FieldAccess (so `s.a.b` parses as - // field `b` of field `a` of column-or-table `s`). validate_expr - // rewrites the Column node into a FieldAccess when its `table` part - // turns out not to be a relation alias -- see plan.rs. - SqlExpr::CompoundIdentifier(parts) if parts.len() >= 2 => { - // Fold the column/field parts (`parts[1..]`); leave the leading - // qualifier (`parts[0]`) raw -- it names a relation (`__THIS__`/ - // generated), never a data column. ponytail: a real CamelCase table - // or struct-column qualifier won't fold like DataFusion here; not - // reachable today (qualifiers are always library-internal). Widen to - // fold parts[0] too if user-named CamelCase relations ever appear. - let mut expr = Expr::Column { - table: Some(parts[0].value.clone()), - name: fold_ident(&parts[1]), - }; - for part in &parts[2..] { - expr = Expr::FieldAccess { - base: Box::new(expr), - field: fold_ident(part), - }; - } - Ok(expr) - } - SqlExpr::Value(vws) => Ok(Expr::Literal(convert_literal(&vws.value)?)), - SqlExpr::Nested(inner) => convert_expr(inner), - SqlExpr::UnaryOp { - op: UnaryOperator::Not, - expr, - } => Ok(Expr::Not(Box::new(convert_expr(expr)?))), - // Unary minus: lower to `0 - x`, reusing Sub's numeric promotion - // (int stays int, float stays float) to match DataFusion. - SqlExpr::UnaryOp { - op: UnaryOperator::Minus, - expr, - } => Ok(Expr::BinaryOp { - op: BinOp::Sub, - left: Box::new(Expr::Literal(Value::Int(0))), - right: Box::new(convert_expr(expr)?), - }), - SqlExpr::BinaryOp { left, op, right } => { - let bin_op = convert_binary_operator(op)?; - Ok(Expr::BinaryOp { - op: bin_op, - left: Box::new(convert_expr(left)?), - right: Box::new(convert_expr(right)?), - }) - } - SqlExpr::Function(func) => convert_function(func), - SqlExpr::Array(Array { elem, .. }) => Ok(Expr::List( - elem.iter().map(convert_expr).collect::>()?, - )), - SqlExpr::Cast { - expr, data_type, .. - } => Ok(Expr::Cast { - expr: Box::new(convert_expr(expr)?), - target: convert_cast_type(data_type)?, - }), - // sqlparser 0.62 parses SUBSTR/SUBSTRING into a dedicated AST node - // rather than a generic Function call; normalize it to - // Expr::Function("substr", ...) so eval_builtin's dispatch handles - // both call syntaxes uniformly. - SqlExpr::Substring { - expr, - substring_from, - substring_for, - .. - } => { - if substring_from.is_none() && substring_for.is_none() { - return Err(InterpError::Build( - "SUBSTRING requires FROM and/or FOR".to_string(), - )); - } - let mut args = vec![convert_expr(expr)?]; - match substring_from { - Some(from) => args.push(convert_expr(from)?), - // SQL-92: SUBSTRING(expr FOR n) with no FROM means "the - // first n characters", equivalent to - // SUBSTRING(expr FROM 1 FOR n). - None => args.push(Expr::Literal(Value::Int(1))), - } - if let Some(for_) = substring_for { - args.push(convert_expr(for_)?); - } - Ok(Expr::Function { - name: "substr".to_string(), - args, - }) - } - // Likewise TRIM(expr) is a dedicated AST node. Only the plain form - // (no BOTH/LEADING/TRAILING side, no explicit trim characters) maps - // onto eval_builtin's "trim" (Rust's str::trim, whitespace only). - SqlExpr::Trim { - expr, - trim_where: None, - trim_what: None, - trim_characters: None, - .. - } => Ok(Expr::Function { - name: "trim".to_string(), - args: vec![convert_expr(expr)?], - }), - SqlExpr::Case { - operand, - conditions, - else_result, - .. - } => { - let mut arms = Vec::with_capacity(conditions.len()); - for when in conditions { - // Simple form: `CASE WHEN ...` normalizes each arm's - // condition to `operand = v`, matching DataFusion/codegen. A NULL - // operand/value makes the `=` NULL, so the arm doesn't match. - let cond = match operand { - Some(op) => Expr::BinaryOp { - op: BinOp::Eq, - left: Box::new(convert_expr(op)?), - right: Box::new(convert_expr(&when.condition)?), - }, - None => convert_expr(&when.condition)?, - }; - arms.push((cond, convert_expr(&when.result)?)); - } - let default = match else_result { - Some(e) => Some(Box::new(convert_expr(e)?)), - None => None, - }; - Ok(Expr::Case { arms, default }) - } - _ => Err(InterpError::Build(format!("Unsupported expression: {e}"))), - } -} - -fn convert_function(func: &Function) -> Result { - let name = func.name.to_string().to_lowercase(); - let args = match &func.args { - FunctionArguments::List(list) => list - .args - .iter() - .map(convert_function_arg) - .collect::, _>>()?, - FunctionArguments::None => Vec::new(), - FunctionArguments::Subquery(_) => { - return Err(InterpError::Build(format!( - "Subquery arguments are not supported in function: {name}" - ))) - } - }; - if name == "named_struct" { - if args.len() % 2 != 0 { - return Err(InterpError::Build( - "named_struct expects an even number of arguments (key, value, ...)".to_string(), - )); - } - let mut fields = Vec::with_capacity(args.len() / 2); - let mut it = args.into_iter(); - while let (Some(key), Some(value)) = (it.next(), it.next()) { - let Expr::Literal(Value::Str(key)) = key else { - return Err(InterpError::Build( - "named_struct field names must be string literals".to_string(), - )); - }; - fields.push((key, value)); - } - return Ok(Expr::Struct(fields)); - } - if name == "struct" { - let fields = args - .into_iter() - .enumerate() - .map(|(i, e)| (format!("c{i}"), e)) - .collect(); - return Ok(Expr::Struct(fields)); - } - Ok(Expr::Function { name, args }) -} - -fn convert_function_arg(arg: &FunctionArg) -> Result { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => convert_expr(e), - _ => Err(InterpError::Build( - "Only plain positional function arguments are supported".to_string(), - )), - } -} - -fn convert_cast_type(dt: &DataType) -> Result { - let name = dt.to_string().to_uppercase(); - if name.starts_with("VARCHAR") - || name.starts_with("TEXT") - || name.starts_with("STRING") - || name.starts_with("CHAR") - { - Ok(CastType::Str) - } else if name.starts_with("BIGINT") || name.starts_with("INT") { - Ok(CastType::Int) - } else if name.starts_with("DOUBLE") || name.starts_with("FLOAT") || name.starts_with("REAL") { - Ok(CastType::Float) - } else if name.starts_with("BOOL") { - Ok(CastType::Bool) - } else { - Err(InterpError::Build(format!( - "Unsupported CAST target type: {name}" - ))) - } -} - -fn convert_literal(v: &SqlValue) -> Result { - match v { - SqlValue::Null => Ok(Value::Null), - SqlValue::Boolean(b) => Ok(Value::Bool(*b)), - SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => { - Ok(Value::Str(s.clone())) - } - SqlValue::Number(text, _) => { - if text.contains('.') || text.to_lowercase().contains('e') { - text.parse::() - .map(Value::Float) - .map_err(|_| InterpError::Build(format!("Invalid numeric literal: {text}"))) - } else { - text.parse::() - .map(Value::Int) - .map_err(|_| InterpError::Build(format!("Invalid numeric literal: {text}"))) - } - } - other => Err(InterpError::Build(format!("Unsupported literal: {other}"))), - } -} - -fn convert_binary_operator(op: &BinaryOperator) -> Result { - match op { - BinaryOperator::Plus => Ok(BinOp::Add), - BinaryOperator::Minus => Ok(BinOp::Sub), - BinaryOperator::Multiply => Ok(BinOp::Mul), - BinaryOperator::Divide => Ok(BinOp::Div), - BinaryOperator::Modulo => Ok(BinOp::Mod), - BinaryOperator::Eq => Ok(BinOp::Eq), - BinaryOperator::NotEq => Ok(BinOp::NotEq), - BinaryOperator::Lt => Ok(BinOp::Lt), - BinaryOperator::Gt => Ok(BinOp::Gt), - BinaryOperator::LtEq => Ok(BinOp::LtEq), - BinaryOperator::GtEq => Ok(BinOp::GtEq), - BinaryOperator::And => Ok(BinOp::And), - BinaryOperator::Or => Ok(BinOp::Or), - BinaryOperator::StringConcat => Ok(BinOp::Concat), - other => Err(InterpError::Build(format!("Unsupported operator: {other}"))), - } -} diff --git a/src/lib.rs b/src/lib.rs index 96d9ff9..3aac431 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,356 +1,28 @@ -use std::collections::{HashMap, HashSet}; +//! Native row-at-a-time SQL interpreters. +//! +//! One extension module, one engine per submodule: +//! * `datafusion` — `InferFn`, the engine differentially tested against +//! DataFusion. Full expression surface, transformer callouts, joins. +//! * `duckdb` — `DuckDBInferFn`, DuckDB semantics. Build-only so far. +//! +//! Each engine owns its own plan/expression/type-inference modules so their +//! semantics can diverge freely. What they share sits at the crate root and is +//! deliberately semantics-free: the value representation, the type vocabulary, +//! the error type, and the Python schema/model marshalling. use pyo3::prelude::*; -use pyo3::types::PyDict; -mod expr; -mod expr_build; +mod datafusion; +mod duckdb; +mod error; mod lookup; -mod plan; mod schema; mod types; - -use expr::Expr; -use expr::Value; -use lookup::LookupIndex; -use plan::Plan; - -#[pyclass] -struct InferFn { - plan: Plan, - lookups: HashMap, - row_table_columns: HashMap>, - row_schemas: HashMap, - #[pyo3(get)] - output_model: Py, -} - -fn synthesize_output_model( - py: Python<'_>, - projection: &[(String, Expr)], - schemas: &HashMap, -) -> PyResult> { - let pydantic = PyModule::import(py, "pydantic")?; - let create_model = pydantic.getattr("create_model")?; - let builtins = PyModule::import(py, "builtins")?; - let ellipsis = builtins.getattr("Ellipsis")?; - - let kwargs = PyDict::new(py); - for (alias, expr) in projection { - let ft = types::infer_type(expr, schemas)?; - let py_type = schema::field_type_to_python(py, ft)?; - kwargs.set_item(alias, (py_type, &ellipsis))?; - } - let model = create_model.call(("OutputRow",), Some(&kwargs))?; - Ok(model.unbind()) -} - -/// Validates a caller-supplied `output_model` against the query's inferred -/// output shape. Only rejects what can be *proven* wrong at build time: a -/// missing/extra field vs. the projection's aliases, or a base-type mismatch -/// `types::compatible` can't excuse. Nullability mismatches are never a -/// build-time error — see `types::compatible`'s docs and Task 2's -/// deliberately conservative `infer_type`. -fn validate_output_model( - py: Python<'_>, - model: &Py, - projection: &[(String, Expr)], - schemas: &HashMap, -) -> PyResult<()> { - let declared_schema = schema::from_pydantic_model(py, model)?; - - let mut projected_aliases: HashSet = HashSet::new(); - for (alias, expr) in projection { - projected_aliases.insert(alias.clone()); - let declared = declared_schema.get(alias).ok_or_else(|| { - plan::InterpError::Build(format!( - "output_model is missing field '{alias}' produced by the query" - )) - })?; - let inferred = types::infer_type(expr, schemas)?; - if !types::compatible(&inferred.base, &declared.base) { - return Err(plan::InterpError::Build(format!( - "output_model field '{alias}' is declared as a type incompatible with the \ - query's inferred output ({:?} vs declared {:?})", - inferred.base, declared.base - )) - .into()); - } - } - - let declared_fields: HashSet = declared_schema.keys().cloned().collect(); - let extra: Vec<&String> = declared_fields.difference(&projected_aliases).collect(); - if !extra.is_empty() { - return Err(plan::InterpError::Build(format!( - "output_model declares fields not produced by the query: {extra:?}" - )) - .into()); - } - Ok(()) -} - -/// A `transformers` registry entry resolved at build time: the fitted object, -/// its `feature_names_in_` (input alignment order), and the caller-declared -/// output field list (marshalling target, order significant). -struct ResolvedTransformer { - obj: std::sync::Arc>, - input_features: Vec, - output_fields: Vec<(String, types::FieldType)>, -} - -/// Reads `obj.feature_names_in_.tolist()`. Absence is a clear build error: -/// the object was fit on bare arrays, so we cannot align inputs by name. -fn read_feature_names_in(py: Python<'_>, obj: &Py) -> Result, plan::InterpError> { - let bound = obj.bind(py); - let attr = bound.getattr("feature_names_in_").map_err(|_| { - plan::InterpError::Build( - "transformer has no `feature_names_in_`; fit it on named data \ - (e.g. a pandas DataFrame) so input columns can be aligned by name" - .to_string(), - ) - })?; - attr.call_method0("tolist") - .and_then(|l| l.extract::>()) - .map_err(|e| plan::InterpError::Build(format!("could not read feature_names_in_: {e}"))) -} - -/// Rewrites every `Expr::Function` whose name is a registered transformer into -/// an `Expr::Transform`, recursing through the whole expression tree so a -/// transformer call nested inside arithmetic is still resolved. A transformer -/// call must have exactly one argument. -fn resolve_transformers( - expr: Expr, - resolved: &HashMap, -) -> Result { - match expr { - Expr::Function { name, args } => { - let mut new_args = Vec::with_capacity(args.len()); - for a in args { - new_args.push(resolve_transformers(a, resolved)?); - } - if let Some(rt) = resolved.get(&name) { - if new_args.len() != 1 { - return Err(plan::InterpError::Build(format!( - "transformer '{name}' takes exactly one argument, got {}", - new_args.len() - ))); - } - let arg = new_args.into_iter().next().unwrap(); - return Ok(Expr::Transform { - obj: rt.obj.clone(), - input_features: rt.input_features.clone(), - output_fields: rt.output_fields.clone(), - arg: Box::new(arg), - }); - } - Ok(Expr::Function { name, args: new_args }) - } - Expr::BinaryOp { op, left, right } => Ok(Expr::BinaryOp { - op, - left: Box::new(resolve_transformers(*left, resolved)?), - right: Box::new(resolve_transformers(*right, resolved)?), - }), - Expr::Not(inner) => Ok(Expr::Not(Box::new(resolve_transformers(*inner, resolved)?))), - Expr::Cast { expr, target } => Ok(Expr::Cast { - expr: Box::new(resolve_transformers(*expr, resolved)?), - target, - }), - Expr::Struct(fields) => { - let mut out = Vec::with_capacity(fields.len()); - for (k, v) in fields { - out.push((k, resolve_transformers(v, resolved)?)); - } - Ok(Expr::Struct(out)) - } - Expr::List(items) => { - let mut out = Vec::with_capacity(items.len()); - for e in items { - out.push(resolve_transformers(e, resolved)?); - } - Ok(Expr::List(out)) - } - Expr::FieldAccess { base, field } => Ok(Expr::FieldAccess { - base: Box::new(resolve_transformers(*base, resolved)?), - field, - }), - Expr::Case { arms, default } => { - let mut new_arms = Vec::with_capacity(arms.len()); - for (cond, result) in arms { - new_arms.push(( - resolve_transformers(cond, resolved)?, - resolve_transformers(result, resolved)?, - )); - } - let default = match default { - Some(d) => Some(Box::new(resolve_transformers(*d, resolved)?)), - None => None, - }; - Ok(Expr::Case { arms: new_arms, default }) - } - // Column, Literal, and an already-built Transform pass through. - other => Ok(other), - } -} - -#[pymethods] -impl InferFn { - #[new] - #[pyo3(signature = (sql, row_tables, static_tables, output_model=None, transformers=None))] - fn new( - py: Python<'_>, - sql: String, - row_tables: HashMap>, - static_tables: HashMap>, - output_model: Option>, - transformers: Option, Py)>>, - ) -> PyResult { - let raw_plan = plan::build_plan(&sql)?; - let row_table_names: HashSet = row_tables.keys().cloned().collect(); - let static_table_names: HashSet = static_tables.keys().cloned().collect(); - let (mut optimized_plan, specs) = plan::optimize(raw_plan, &static_table_names)?; - - let mut row_schemas = HashMap::new(); - for (name, model_class) in &row_tables { - row_schemas.insert(name.clone(), schema::from_pydantic_model(py, model_class)?); - } - let mut static_schemas = HashMap::new(); - for (name, table_obj) in &static_tables { - static_schemas.insert(name.clone(), schema::from_arrow_table(py, table_obj)?); - } - - let column_validation = plan::validate_columns( - &mut optimized_plan, - &row_table_names, - &row_schemas, - &static_schemas, - )?; - - // Resolve registered transformers AFTER column validation (so - // validate_columns sees the plain Expr::Function and its named_struct - // arg -- no Transform arm needed there) and BEFORE output-model - // synthesis (so infer_type sees Expr::Transform and returns the - // declared output struct type). Reads feature_names_in_ here (with py). - let transformers = transformers.unwrap_or_default(); - if !transformers.is_empty() { - let mut resolved: HashMap = HashMap::new(); - for (name, (obj, out_schema_obj)) in &transformers { - let input_features = read_feature_names_in(py, obj)?; - let output_fields = schema::arrow_schema_to_ordered_fields(py, out_schema_obj)?; - resolved.insert( - name.clone(), - ResolvedTransformer { - obj: std::sync::Arc::new(obj.clone_ref(py)), - input_features, - output_fields, - }, - ); - } - let projection = std::mem::take(&mut optimized_plan.projection); - let mut new_projection = Vec::with_capacity(projection.len()); - for (alias, expr) in projection { - new_projection.push((alias, resolve_transformers(expr, &resolved)?)); - } - optimized_plan.projection = new_projection; - } - - let output_model = match output_model { - Some(supplied) => { - validate_output_model( - py, - &supplied, - &optimized_plan.projection, - &column_validation.effective_schemas, - )?; - supplied - } - None => synthesize_output_model( - py, - &optimized_plan.projection, - &column_validation.effective_schemas, - )?, - }; - - let mut lookups = HashMap::new(); - for spec in specs { - let table_obj = static_tables.get(&spec.static_table).ok_or_else(|| { - plan::InterpError::Build(format!( - "SQL references static table '{}' that was not provided", - spec.static_table - )) - })?; - let index = lookup::build_index(py, table_obj, &spec.key_columns)?; - lookups.insert(spec.static_table, index); - } - - Ok(InferFn { - plan: optimized_plan, - lookups, - row_table_columns: column_validation.row_table_columns, - row_schemas, - output_model, - }) - } - - #[pyo3(signature = (tables=None, **kwargs))] - fn infer( - &self, - py: Python<'_>, - tables: Option>>>, - kwargs: Option>, - ) -> PyResult>> { - let mut merged: HashMap>> = tables.unwrap_or_default(); - if let Some(kwargs) = kwargs { - for (k, v) in kwargs.iter() { - let key: String = k.extract()?; - let rows: Vec> = v.extract()?; - merged.insert(key, rows); - } - } - - let empty: Vec = Vec::new(); - let mut value_tables: HashMap>> = HashMap::new(); - for (table, rows) in &merged { - let columns = self.row_table_columns.get(table).unwrap_or(&empty); - let schema = self.row_schemas.get(table); - let mut out_rows = Vec::with_capacity(rows.len()); - for row_obj in rows { - let bound = row_obj.bind(py); - let mut row: HashMap = HashMap::new(); - for col in columns { - let attr = bound.getattr(col.as_str()).map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!( - "Row for table '{table}' is missing attribute '{col}': {e}" - )) - })?; - let value = match schema.and_then(|s| s.get(col)) { - Some(ft) => Value::from_pyobject_typed(&attr, &ft.base)?, - None => Value::from_pyobject(&attr)?, - }; - row.insert(col.clone(), value); - } - out_rows.push(row); - } - value_tables.insert(table.clone(), out_rows); - } - - let result_rows = plan::execute(&self.plan, &value_tables, &self.lookups)?; - - let output_model = self.output_model.bind(py); - let mut out = Vec::with_capacity(result_rows.len()); - for row in &result_rows { - let dict = PyDict::new(py); - for (k, v) in row { - dict.set_item(k, v.to_pyobject(py)?)?; - } - let instance = output_model.call_method1("model_validate", (dict,))?; - out.push(instance.unbind()); - } - Ok(out) - } -} +mod value; #[pymodule] fn _interpreter(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/src/lookup.rs b/src/lookup.rs index 7fae8bb..9d6f76b 100644 --- a/src/lookup.rs +++ b/src/lookup.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use pyo3::prelude::*; use pyo3::types::PyDict; -use crate::expr::Value; -use crate::plan::InterpError; +use crate::value::Value; +use crate::error::InterpError; pub struct LookupIndex { pub index: HashMap, HashMap>, diff --git a/src/plan.rs b/src/plan.rs deleted file mode 100644 index 027ddcd..0000000 --- a/src/plan.rs +++ /dev/null @@ -1,1253 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use pyo3::exceptions::{PyKeyError, PyValueError}; -use pyo3::PyErr; -use sqlparser::ast::{ - BinaryOperator, Expr as SqlExpr, Join, JoinConstraint, JoinOperator, SelectItem, SetExpr, - Statement, TableFactor, TableWithJoins, -}; -use sqlparser::dialect::GenericDialect; -use sqlparser::parser::Parser; - -use crate::expr::{Expr, Value}; -use crate::types::Schema; - -pub type Row = HashMap>; - -pub enum InterpError { - Build(String), - MissingKey(String), - Eval(String), -} - -impl From for PyErr { - fn from(e: InterpError) -> PyErr { - match e { - InterpError::Build(msg) => PyValueError::new_err(msg), - InterpError::MissingKey(msg) => PyKeyError::new_err(msg), - InterpError::Eval(msg) => PyValueError::new_err(msg), - } - } -} - -pub enum RelNode { - TableScan { - table: String, - }, - Filter { - input: Box, - predicate: Expr, - }, - CrossJoin { - left: Box, - right: Box, - }, - Join { - left: Box, - right: Box, - on: Vec<(Expr, Expr)>, - outer: bool, - }, - SubqueryAlias { - input: Box, - alias: String, - }, - LookupJoin { - input: Box, - table: String, - keys: Vec, - outer: bool, - }, - /// Row-multiplying `unnest(list)`: evaluates `list_expr` per input row and - /// emits one output row per list element, binding it to `output_col`. A - /// NULL or empty list emits zero rows (matches DataFusion). - Unnest { - input: Box, - list_expr: Expr, - output_col: String, - }, -} - -/// Synthetic outer key under which `RelNode::Unnest` binds its emitted element -/// column — in the runtime `Row` and in the validation-time effective-schema -/// map. The NUL byte can never collide with a real SQL table/alias identifier. -const UNNEST_KEY: &str = "\0unnest"; - -pub struct Plan { - pub projection: Vec<(String, Expr)>, - pub input: RelNode, -} - -pub struct LookupSpec { - pub static_table: String, - pub key_columns: Vec, -} - -pub fn build_plan(sql: &str) -> Result { - let dialect = GenericDialect {}; - let statements = Parser::parse_sql(&dialect, sql) - .map_err(|e| InterpError::Build(format!("SQL parse error: {e}")))?; - - if statements.len() != 1 { - return Err(InterpError::Build( - "Expected exactly one SQL statement".to_string(), - )); - } - - let select = match &statements[0] { - Statement::Query(query) => match query.body.as_ref() { - SetExpr::Select(select) => select.as_ref(), - _ => { - return Err(InterpError::Build( - "Only SELECT queries are supported".to_string(), - )) - } - }, - _ => { - return Err(InterpError::Build( - "Only SELECT queries are supported".to_string(), - )) - } - }; - - let mut input = build_from(&select.from)?; - if let Some(predicate) = &select.selection { - input = RelNode::Filter { - input: Box::new(input), - predicate: crate::expr_build::convert_expr(predicate)?, - }; - } - let projection = build_projection(&select.projection)?; - - Ok(Plan { projection, input }) -} - -fn build_from(from: &[TableWithJoins]) -> Result { - if from.is_empty() { - return Err(InterpError::Build("FROM clause is required".to_string())); - } - let mut seen_tables: HashSet = HashSet::new(); - let mut node = build_table_with_joins(&from[0], &mut seen_tables)?; - for twj in &from[1..] { - let right = build_table_with_joins(twj, &mut seen_tables)?; - node = RelNode::CrossJoin { - left: Box::new(node), - right: Box::new(right), - }; - } - Ok(node) -} - -fn build_table_with_joins( - twj: &TableWithJoins, - seen_tables: &mut HashSet, -) -> Result { - let mut node = build_table_factor(&twj.relation, seen_tables)?; - for join in &twj.joins { - node = build_join(node, join, seen_tables)?; - } - Ok(node) -} - -fn build_join( - left: RelNode, - join: &Join, - seen_tables: &mut HashSet, -) -> Result { - let right = build_table_factor(&join.relation, seen_tables)?; - match &join.join_operator { - JoinOperator::Join(constraint) | JoinOperator::Inner(constraint) => { - let on_expr = require_on(constraint)?; - let on = extract_equality_keys(on_expr)?; - Ok(RelNode::Join { - left: Box::new(left), - right: Box::new(right), - on, - outer: false, - }) - } - JoinOperator::Left(constraint) | JoinOperator::LeftOuter(constraint) => { - let on_expr = require_on(constraint)?; - let on = extract_equality_keys(on_expr)?; - Ok(RelNode::Join { - left: Box::new(left), - right: Box::new(right), - on, - outer: true, - }) - } - JoinOperator::CrossJoin(_) => Ok(RelNode::CrossJoin { - left: Box::new(left), - right: Box::new(right), - }), - other => Err(InterpError::Build(format!( - "Unsupported JOIN type: {other:?} — only inner JOIN ... ON and CROSS JOIN are supported" - ))), - } -} - -fn build_table_factor( - factor: &TableFactor, - seen_tables: &mut HashSet, -) -> Result { - match factor { - TableFactor::Table { name, alias, .. } => { - let table = name.to_string(); - // Track the EFFECTIVE output name (alias if present, else the real - // table name) — this is the key each relation's Row is stored - // under, so a collision here (whether from a true self-join like - // `FROM a JOIN a ON ...` or an alias collision like - // `FROM a JOIN b AS a ON ...`) would cause one side's data to - // silently overwrite the other's during row merging. - let effective_name = match &alias { - Some(a) => a.name.value.clone(), - None => table.clone(), - }; - if !seen_tables.insert(effective_name.clone()) { - return Err(InterpError::Build(format!( - "table '{effective_name}' is referenced more than once in FROM/JOIN — \ - self-joins and alias collisions are not supported" - ))); - } - let scan = RelNode::TableScan { table }; - Ok(match alias { - Some(a) => RelNode::SubqueryAlias { - input: Box::new(scan), - alias: a.name.value.clone(), - }, - None => scan, - }) - } - _ => Err(InterpError::Build("Unsupported FROM clause".to_string())), - } -} - -fn require_on(constraint: &JoinConstraint) -> Result<&SqlExpr, InterpError> { - match constraint { - JoinConstraint::On(e) => Ok(e), - _ => Err(InterpError::Build( - "JOIN requires an ON condition".to_string(), - )), - } -} - -fn extract_equality_keys(expr: &SqlExpr) -> Result, InterpError> { - match expr { - SqlExpr::BinaryOp { - left, - op: BinaryOperator::And, - right, - } => { - let mut pairs = extract_equality_keys(left)?; - pairs.extend(extract_equality_keys(right)?); - Ok(pairs) - } - SqlExpr::BinaryOp { - left, - op: BinaryOperator::Eq, - right, - } => Ok(vec![( - crate::expr_build::convert_expr(left)?, - crate::expr_build::convert_expr(right)?, - )]), - _ => Err(InterpError::Build( - "JOIN ON condition must be an equality, or an AND of equalities, between columns" - .to_string(), - )), - } -} - -fn build_projection(items: &[SelectItem]) -> Result, InterpError> { - let mut out = Vec::new(); - for item in items { - match item { - SelectItem::UnnamedExpr(e) => { - let name = column_name(e)?; - out.push((name, crate::expr_build::convert_expr(e)?)); - } - SelectItem::ExprWithAlias { expr, alias } => { - // The output alias folds like any identifier (`AS Foo` -> `foo`). - out.push(( - crate::expr_build::fold_ident(alias), - crate::expr_build::convert_expr(expr)?, - )); - } - _ => return Err(InterpError::Build("Unsupported SELECT item".to_string())), - } - } - Ok(out) -} - -fn column_name(e: &SqlExpr) -> Result { - match e { - SqlExpr::Identifier(ident) => Ok(crate::expr_build::fold_ident(ident)), - SqlExpr::CompoundIdentifier(parts) => Ok(parts - .last() - .map(crate::expr_build::fold_ident) - .unwrap_or_default()), - // `unnest(struct_expr)` expands into per-field columns during - // validate_columns (once the arg's type is known), which replaces - // this placeholder name entirely. Only reachable unaliased here - // because DataFusion allows it; other bare function calls still - // require an alias below. - SqlExpr::Function(func) if func.name.to_string().eq_ignore_ascii_case("unnest") => { - Ok("unnest".to_string()) - } - _ => Err(InterpError::Build( - "Expression in SELECT list needs an alias (AS name)".to_string(), - )), - } -} - -/// Walks a built `Plan`, rewriting any `Join` node where exactly one side is -/// a scan of a table named in `static_tables` into a `RelNode::LookupJoin`. -/// Returns an error if both sides of a `Join` are static tables (a -/// static-to-static join isn't a lookup and isn't supported). -pub fn optimize( - plan: Plan, - static_tables: &HashSet, -) -> Result<(Plan, Vec), InterpError> { - let mut specs = Vec::new(); - let input = optimize_rel(plan.input, static_tables, &mut specs)?; - Ok(( - Plan { - projection: plan.projection, - input, - }, - specs, - )) -} - -fn optimize_rel( - node: RelNode, - static_tables: &HashSet, - specs: &mut Vec, -) -> Result { - match node { - RelNode::Join { - left, - right, - on, - outer, - } => { - let left = optimize_rel(*left, static_tables, specs)?; - let right = optimize_rel(*right, static_tables, specs)?; - let left_static = scan_table_name(&left).filter(|t| static_tables.contains(*t)); - let right_static = scan_table_name(&right).filter(|t| static_tables.contains(*t)); - match (left_static, right_static) { - (Some(_), Some(_)) => Err(InterpError::Build( - "Joining two static tables together is not supported".to_string(), - )), - (None, Some(table)) => { - let table = table.to_string(); - let (keys, key_columns) = split_keys(&on, &table)?; - specs.push(LookupSpec { - static_table: table.clone(), - key_columns, - }); - Ok(RelNode::LookupJoin { - input: Box::new(left), - table, - keys, - outer, - }) - } - (Some(table), None) => { - let table = table.to_string(); - let (keys, key_columns) = split_keys(&on, &table)?; - specs.push(LookupSpec { - static_table: table.clone(), - key_columns, - }); - Ok(RelNode::LookupJoin { - input: Box::new(right), - table, - keys, - outer, - }) - } - (None, None) => { - if outer { - return Err(InterpError::Build( - "LEFT JOIN is only supported against a static lookup table".to_string(), - )); - } - Ok(RelNode::Join { - left: Box::new(left), - right: Box::new(right), - on, - outer, - }) - } - } - } - RelNode::CrossJoin { left, right } => Ok(RelNode::CrossJoin { - left: Box::new(optimize_rel(*left, static_tables, specs)?), - right: Box::new(optimize_rel(*right, static_tables, specs)?), - }), - RelNode::Filter { input, predicate } => Ok(RelNode::Filter { - input: Box::new(optimize_rel(*input, static_tables, specs)?), - predicate, - }), - RelNode::SubqueryAlias { input, alias } => Ok(RelNode::SubqueryAlias { - input: Box::new(optimize_rel(*input, static_tables, specs)?), - alias, - }), - other => Ok(other), - } -} - -fn scan_table_name(node: &RelNode) -> Option<&str> { - match node { - RelNode::TableScan { table } => Some(table), - RelNode::SubqueryAlias { input, .. } => scan_table_name(input), - _ => None, - } -} - -/// The qualifier (`table` part) of a plain `Expr::Column`, or `None` for -/// anything else (unqualified column, literal, expression, ...). -fn column_qualifier(e: &Expr) -> Option<&str> { - match e { - Expr::Column { table: Some(t), .. } => Some(t.as_str()), - _ => None, - } -} - -/// Splits each ON-clause equality pair into (the static table's key column -/// name, the row-side expression to evaluate it against). -/// -/// The ON clause's tuple order reflects how the equality was *written* -/// (`a = b` vs `b = a`), which is independent of which side of the JOIN is -/// structurally left/right in the FROM clause — so this identifies the -/// static side per-pair by matching each operand's column qualifier against -/// `static_table`, rather than assuming a fixed position. -fn split_keys( - on: &[(Expr, Expr)], - static_table: &str, -) -> Result<(Vec, Vec), InterpError> { - let mut row_side_keys = Vec::new(); - let mut static_col_names = Vec::new(); - for (l, r) in on { - let static_expr = match (column_qualifier(l), column_qualifier(r)) { - (Some(t), _) if t == static_table => l, - (_, Some(t)) if t == static_table => r, - _ => { - return Err(InterpError::Build(format!( - "JOIN ON keys against static table '{static_table}' must reference \ - the static table's columns by name (e.g. {static_table}.col)" - ))) - } - }; - let row_expr = if std::ptr::eq(static_expr, l) { r } else { l }; - match static_expr { - Expr::Column { name, .. } => static_col_names.push(name.clone()), - _ => { - return Err(InterpError::Build(format!( - "JOIN ON keys against static table '{static_table}' must be plain columns" - ))) - } - } - row_side_keys.push(row_expr.clone()); - } - Ok((row_side_keys, static_col_names)) -} - -pub fn execute( - plan: &Plan, - tables: &HashMap>>, - lookups: &HashMap, -) -> Result>, InterpError> { - let rows = execute_rel(&plan.input, tables, lookups)?; - let mut out = Vec::with_capacity(rows.len()); - for row in &rows { - let mut result = HashMap::new(); - for (alias, e) in &plan.projection { - result.insert(alias.clone(), crate::expr::eval(e, row)?); - } - out.push(result); - } - Ok(out) -} - -fn execute_rel( - node: &RelNode, - tables: &HashMap>>, - lookups: &HashMap, -) -> Result, InterpError> { - match node { - RelNode::TableScan { table } => { - let flat_rows = tables.get(table).ok_or_else(|| { - InterpError::Build(format!("Unknown table in FROM clause: {table}")) - })?; - Ok(flat_rows - .iter() - .map(|r| { - let mut row = Row::new(); - row.insert(table.clone(), r.clone()); - row - }) - .collect()) - } - RelNode::Filter { input, predicate } => { - let rows = execute_rel(input, tables, lookups)?; - let mut out = Vec::new(); - for row in rows { - if let Value::Bool(true) = crate::expr::eval(predicate, &row)? { - out.push(row); - } - } - Ok(out) - } - RelNode::CrossJoin { left, right } => { - let left_rows = execute_rel(left, tables, lookups)?; - let right_rows = execute_rel(right, tables, lookups)?; - let mut out = Vec::with_capacity(left_rows.len() * right_rows.len()); - for l in &left_rows { - for r in &right_rows { - let mut merged = l.clone(); - merged.extend(r.clone()); - out.push(merged); - } - } - Ok(out) - } - RelNode::Join { - left, right, on, .. - } => { - let left_rows = execute_rel(left, tables, lookups)?; - let right_rows = execute_rel(right, tables, lookups)?; - let mut out = Vec::new(); - for l in &left_rows { - for r in &right_rows { - let mut merged = l.clone(); - merged.extend(r.clone()); - let mut all_match = true; - for (le, re) in on { - let lv = crate::expr::eval(le, &merged)?; - let rv = crate::expr::eval(re, &merged)?; - if matches!(lv, Value::Null) || matches!(rv, Value::Null) || lv != rv { - all_match = false; - break; - } - } - if all_match { - out.push(merged); - } - } - } - Ok(out) - } - RelNode::SubqueryAlias { input, alias } => { - let rows = execute_rel(input, tables, lookups)?; - Ok(rows - .into_iter() - .map(|row| { - let inner = row.into_values().next().unwrap_or_default(); - let mut renamed = Row::new(); - renamed.insert(alias.clone(), inner); - renamed - }) - .collect()) - } - RelNode::LookupJoin { - input, - table, - keys, - outer, - } => { - let rows = execute_rel(input, tables, lookups)?; - let index = lookups.get(table).ok_or_else(|| { - InterpError::Build(format!("No lookup index built for table: {table}")) - })?; - let mut out = Vec::with_capacity(rows.len()); - for mut row in rows { - let key: Vec = keys - .iter() - .map(|k| crate::expr::eval(k, &row)) - .collect::>()?; - match index.index.get(&key) { - Some(hit) => { - row.insert(table.clone(), hit.clone()); - } - None if *outer => { - let null_row: HashMap = index - .value_columns - .iter() - .map(|c| (c.clone(), Value::Null)) - .collect(); - row.insert(table.clone(), null_row); - } - None => { - let key_repr: Vec = - key.iter().map(crate::expr::display_value).collect(); - return Err(InterpError::MissingKey(format!( - "No row in static table '{table}' matches key ({})", - key_repr.join(", ") - ))); - } - } - out.push(row); - } - Ok(out) - } - RelNode::Unnest { - input, - list_expr, - output_col, - } => { - let rows = execute_rel(input, tables, lookups)?; - let mut out = Vec::new(); - for row in rows { - match crate::expr::eval(list_expr, &row)? { - Value::List(items) => { - // An empty list falls out here as zero iterations. - for item in items { - let mut new_row = row.clone(); - let mut bound = HashMap::new(); - bound.insert(output_col.clone(), item); - new_row.insert(UNNEST_KEY.to_string(), bound); - out.push(new_row); - } - } - // NULL list -> zero rows (matches DataFusion). - Value::Null => {} - other => { - return Err(InterpError::Eval(format!( - "unnest() expected a list, got a {} value", - crate::expr::type_name(&other) - ))) - } - } - } - Ok(out) - } - } -} - -pub struct ColumnValidation { - pub row_table_columns: HashMap>, - pub effective_schemas: HashMap, -} - -/// Maps each relation's EFFECTIVE name (its alias if aliased, else its real -/// table name — the qualifier `Expr::Column` references use after -/// `SubqueryAlias` renaming) to its real table name and whether it's a row -/// table (vs. static). Walks the already-optimized Plan, so any `Join` with -/// a static side has already become a `LookupJoin`. -fn resolve_tables( - node: &RelNode, - row_table_names: &HashSet, - nullable: bool, - out: &mut HashMap, - nullable_out: &mut HashSet, -) { - match node { - RelNode::TableScan { table } => { - let is_row = row_table_names.contains(table); - out.insert(table.clone(), (table.clone(), is_row)); - if nullable { - nullable_out.insert(table.clone()); - } - } - RelNode::SubqueryAlias { input, alias } => { - if let Some(real) = scan_table_name(input) { - let is_row = row_table_names.contains(real); - out.insert(alias.clone(), (real.to_string(), is_row)); - if nullable { - nullable_out.insert(alias.clone()); - } - } - } - RelNode::Filter { input, .. } => { - resolve_tables(input, row_table_names, nullable, out, nullable_out) - } - RelNode::CrossJoin { left, right } => { - resolve_tables(left, row_table_names, nullable, out, nullable_out); - resolve_tables(right, row_table_names, nullable, out, nullable_out); - } - // A LEFT join makes its right side nullable; nested joins stay nullable. - // NB: post-optimize `outer` is structurally always false here -- a LEFT - // Join with a static side becomes a LookupJoin, and a row-to-row LEFT - // JOIN is rejected in optimize_rel. The `|| *outer` is kept correct in - // case that restriction is ever relaxed. - RelNode::Join { - left, right, outer, .. - } => { - resolve_tables(left, row_table_names, nullable, out, nullable_out); - resolve_tables(right, row_table_names, nullable || *outer, out, nullable_out); - } - RelNode::LookupJoin { - input, table, outer, .. - } => { - resolve_tables(input, row_table_names, nullable, out, nullable_out); - out.insert(table.clone(), (table.clone(), false)); - if nullable || *outer { - nullable_out.insert(table.clone()); - } - } - // The emitted column lives under a synthetic key resolved via - // effective_schemas, not `resolved` — just recurse into the input. - RelNode::Unnest { input, .. } => { - resolve_tables(input, row_table_names, nullable, out, nullable_out) - } - } -} - -/// Validates every `Expr::Column` reference in the plan (projection, WHERE, -/// JOIN ON) against the resolved table schemas, and collects — per row -/// table's REAL name — the set of columns the query actually references. -/// Also returns the effective-name -> Schema map (aliases resolved), reused -/// by the output type-inference pass. -pub fn validate_columns( - plan: &mut Plan, - row_table_names: &HashSet, - row_schemas: &HashMap, - static_schemas: &HashMap, -) -> Result { - let mut resolved = HashMap::new(); - let mut nullable_tables = HashSet::new(); - resolve_tables( - &plan.input, - row_table_names, - false, - &mut resolved, - &mut nullable_tables, - ); - - let mut effective_schemas = HashMap::new(); - for (effective_name, (real_name, is_row)) in &resolved { - let schema = if *is_row { - row_schemas.get(real_name) - } else { - static_schemas.get(real_name) - }; - if let Some(s) = schema { - let mut s = s.clone(); - // Columns from the nullable side of an outer join can be NULL on an - // unmatched row, so the synthesized output type must be nullable even - // when the source table declares the column non-nullable. - if nullable_tables.contains(effective_name) { - for ft in s.values_mut() { - ft.nullable = true; - } - } - effective_schemas.insert(effective_name.clone(), s); - } - } - - let mut used_columns: HashMap> = HashMap::new(); - let mut expanded_projection = Vec::with_capacity(plan.projection.len()); - let mut unnest_seen = false; - for (name, mut e) in std::mem::take(&mut plan.projection) { - validate_expr( - &mut e, - &resolved, - row_schemas, - static_schemas, - &effective_schemas, - &mut used_columns, - )?; - // Task 6: `unnest(list)` multiplies rows. Wrap the input rel in an - // `Unnest` node and replace the projection item with a plain reference - // to the emitted column. (The `unnest(struct)` case types as a struct - // and is handled by `expand_unnest_struct` below.) - if let Some((list_expr, elem_ft)) = unnest_list_element(&e, &effective_schemas)? { - if unnest_seen { - return Err(InterpError::Build( - "Only one unnest(list) per query is supported".to_string(), - )); - } - unnest_seen = true; - let old_input = - std::mem::replace(&mut plan.input, RelNode::TableScan { table: String::new() }); - plan.input = RelNode::Unnest { - input: Box::new(old_input), - list_expr, - output_col: name.clone(), - }; - // Register the emitted column so output-model synthesis (`infer_type`) - // and downstream validation resolve the unqualified `output_col`. - effective_schemas - .entry(UNNEST_KEY.to_string()) - .or_default() - .insert(name.clone(), elem_ft); - expanded_projection.push((name.clone(), Expr::Column { table: None, name })); - continue; - } - match expand_unnest_struct(&e, &effective_schemas)? { - Some(fields) => expanded_projection.extend(fields), - None => expanded_projection.push((name, e)), - } - } - plan.projection = expanded_projection; - validate_rel( - &mut plan.input, - &resolved, - row_schemas, - static_schemas, - &effective_schemas, - &mut used_columns, - )?; - - Ok(ColumnValidation { - row_table_columns: used_columns - .into_iter() - .map(|(k, v)| (k, v.into_iter().collect())) - .collect(), - effective_schemas, - }) -} - -fn validate_rel( - node: &mut RelNode, - resolved: &HashMap, - row_schemas: &HashMap, - static_schemas: &HashMap, - effective_schemas: &HashMap, - used_columns: &mut HashMap>, -) -> Result<(), InterpError> { - match node { - RelNode::TableScan { .. } => Ok(()), - RelNode::Filter { input, predicate } => { - validate_expr( - predicate, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - validate_rel( - input, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - RelNode::CrossJoin { left, right } => { - validate_rel( - left, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - validate_rel( - right, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - RelNode::Join { - left, right, on, .. - } => { - for (l, r) in on.iter_mut() { - validate_expr( - l, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - validate_expr( - r, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - } - validate_rel( - left, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - validate_rel( - right, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - RelNode::SubqueryAlias { input, .. } => validate_rel( - input, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ), - RelNode::LookupJoin { input, keys, .. } => { - for k in keys.iter_mut() { - validate_expr( - k, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - } - validate_rel( - input, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - RelNode::Unnest { - input, list_expr, .. - } => { - validate_expr( - list_expr, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - validate_rel( - input, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - } -} - -/// If `e` is `unnest()` and `` types as a struct, returns the -/// per-field projection columns it expands into (DataFusion flattens a -/// struct-typed `unnest()` into one output column per field, named -/// `"."`, ignoring any alias on the SELECT item -- see -/// `unnest_display_name`). Returns `None` for anything else (including -/// `unnest()` on a list, left untouched for Task 6's list-unnest). -/// If `e` is `unnest()` and `` types as a list, returns the argument -/// (the list expression) and the list's element `FieldType`. Returns `None` for -/// anything else (a struct-typed `unnest`, a non-`unnest` expression, ...). -fn unnest_list_element( - e: &Expr, - effective_schemas: &HashMap, -) -> Result, InterpError> { - let Expr::Function { name, args } = e else { - return Ok(None); - }; - if name != "unnest" || args.len() != 1 { - return Ok(None); - } - let arg_ty = crate::types::infer_type(&args[0], effective_schemas)?; - let crate::types::Base::List(elem) = arg_ty.base else { - return Ok(None); - }; - Ok(Some((args[0].clone(), *elem))) -} - -fn expand_unnest_struct( - e: &Expr, - effective_schemas: &HashMap, -) -> Result>, InterpError> { - let Expr::Function { name, args } = e else { - return Ok(None); - }; - if name != "unnest" || args.len() != 1 { - return Ok(None); - } - let arg = &args[0]; - let arg_ty = crate::types::infer_type(arg, effective_schemas)?; - let crate::types::Base::Struct(fields) = &arg_ty.base else { - return Ok(None); - }; - let arg_display = unnest_display_name(arg, effective_schemas)?; - Ok(Some( - fields - .iter() - .map(|(field_name, _)| { - ( - format!("{arg_display}.{field_name}"), - Expr::FieldAccess { - base: Box::new(arg.clone()), - field: field_name.clone(), - }, - ) - }) - .collect(), - )) -} - -/// Renders an expression the way DataFusion's logical-plan `Expr::Display` -/// does, for the shapes `unnest()`'s argument can take -- a (possibly -/// qualified) column, a struct-field access, or a `named_struct(...)` -/// construction. This is what DataFusion derives its `unnest(...)` output -/// column names from, so matching it exactly is required for the -/// differential tests to agree column-for-column. -/// -/// ponytail: only covers the node shapes reachable as an `unnest()` arg -/// today (struct columns, struct field access, `named_struct`/`struct()` -/// literals over plain columns). `Expr::Struct` can't tell `named_struct(...)` -/// and `struct(...)` apart post-conversion (both collapse to the same node), -/// so this always renders as `named_struct(...)`; DataFusion names a -/// `struct()`-built unnest differently. Widen if that combination needs -/// differential coverage. -fn unnest_display_name( - e: &Expr, - effective_schemas: &HashMap, -) -> Result { - match e { - Expr::Column { - table: Some(t), - name, - } => Ok(format!("{t}.{name}")), - Expr::Column { table: None, name } => { - let qualifier = effective_schemas - .iter() - .find(|(_, schema)| schema.contains_key(name)) - .map(|(qualifier, _)| qualifier.clone()) - .ok_or_else(|| InterpError::Build(format!("Unknown column: {name}")))?; - Ok(format!("{qualifier}.{name}")) - } - Expr::FieldAccess { base, field } => Ok(format!( - "{}.{field}", - unnest_display_name(base, effective_schemas)? - )), - Expr::Struct(fields) => { - let inner = fields - .iter() - .map(|(key, value)| { - Ok(format!( - "Utf8(\"{key}\"),{}", - unnest_display_name(value, effective_schemas)? - )) - }) - .collect::, InterpError>>()? - .join(","); - Ok(format!("named_struct({inner})")) - } - _ => Err(InterpError::Build( - "unnest() argument is too complex to name".to_string(), - )), - } -} - -fn validate_expr( - e: &mut Expr, - resolved: &HashMap, - row_schemas: &HashMap, - static_schemas: &HashMap, - effective_schemas: &HashMap, - used_columns: &mut HashMap>, -) -> Result<(), InterpError> { - match e { - Expr::Column { - table: Some(t), - name, - } => { - if let Some((real, is_row)) = resolved.get(t.as_str()) { - check_column(real, *is_row, name, row_schemas, static_schemas)?; - if *is_row { - used_columns - .entry(real.clone()) - .or_default() - .insert(name.clone()); - } - return Ok(()); - } - // `t` isn't a relation alias -- the "table.column" parse was - // wrong; reinterpret it as struct field access: `t` an in-scope - // column, `name` one of its struct fields. Precedence rule: a - // relation alias always wins, so this fallback only runs once - // the alias lookup above has failed. - let base_name = t.clone(); - let field = name.clone(); - *e = Expr::FieldAccess { - base: Box::new(Expr::Column { - table: None, - name: base_name, - }), - field, - }; - validate_expr( - e, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - Expr::Column { table: None, name } => { - let mut matches: Vec<(&String, bool)> = Vec::new(); - for (real, is_row) in resolved.values() { - let schema = if *is_row { - row_schemas.get(real) - } else { - static_schemas.get(real) - }; - if schema.is_some_and(|s| s.contains_key(name)) { - matches.push((real, *is_row)); - } - } - match matches.as_slice() { - [] => Err(InterpError::Build(format!("Unknown column: {name}"))), - [(real, is_row)] => { - if *is_row { - used_columns - .entry((*real).clone()) - .or_default() - .insert(name.clone()); - } - Ok(()) - } - _ => Err(InterpError::Build(format!( - "Ambiguous column reference: {name}" - ))), - } - } - Expr::Literal(_) => Ok(()), - Expr::BinaryOp { left, right, .. } => { - validate_expr( - left, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - validate_expr( - right, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ) - } - Expr::Not(inner) | Expr::Cast { expr: inner, .. } => validate_expr( - inner, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ), - Expr::Function { args, .. } | Expr::List(args) => { - for a in args { - validate_expr( - a, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - } - Ok(()) - } - Expr::Struct(fields) => { - for (_, v) in fields { - validate_expr( - v, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - } - Ok(()) - } - // Compiler-mandated: `Expr` exhaustiveness is checked at compile time - // and this match has no catch-all. Transformer resolution runs AFTER - // validate_columns, so no `Transform` node reaches here today; recurse - // into `arg` anyway so column validation stays correct if that ordering - // ever changes. - Expr::Transform { arg, .. } => validate_expr( - arg, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - ), - Expr::FieldAccess { base, field } => { - validate_expr( - base, - resolved, - row_schemas, - static_schemas, - effective_schemas, - used_columns, - )?; - let base_ty = crate::types::infer_type(base, effective_schemas)?; - match &base_ty.base { - crate::types::Base::Struct(fields) => { - if fields.iter().any(|(name, _)| name == field) { - Ok(()) - } else { - Err(InterpError::Build(format!("Unknown struct field: {field}"))) - } - } - _ => Err(InterpError::Build(format!( - "Cannot access field '{field}' on a non-struct column" - ))), - } - } - Expr::Case { arms, default } => { - for (cond, result) in arms { - validate_expr( - cond, resolved, row_schemas, static_schemas, effective_schemas, - used_columns, - )?; - validate_expr( - result, resolved, row_schemas, static_schemas, effective_schemas, - used_columns, - )?; - } - if let Some(d) = default { - validate_expr( - d, resolved, row_schemas, static_schemas, effective_schemas, - used_columns, - )?; - } - Ok(()) - } - } -} - -fn check_column( - real_table: &str, - is_row: bool, - name: &str, - row_schemas: &HashMap, - static_schemas: &HashMap, -) -> Result<(), InterpError> { - let schema = if is_row { - row_schemas.get(real_table) - } else { - static_schemas.get(real_table) - }; - match schema { - Some(s) if s.contains_key(name) => Ok(()), - Some(_) => Err(InterpError::Build(format!( - "Unknown column: {real_table}.{name}" - ))), - None => Err(InterpError::Build(format!("Unknown table: {real_table}"))), - } -} diff --git a/src/schema.rs b/src/schema.rs index 2a41384..f2a2464 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,7 +1,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; -use crate::plan::InterpError; +use crate::error::InterpError; use crate::types::{Base, FieldType, Schema}; /// Extract a Schema from a Pydantic v2 model class's `model_fields`. diff --git a/src/types.rs b/src/types.rs index 48adad7..fe37c99 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,4 +1,10 @@ -use std::collections::{HashMap, HashSet}; +//! The type vocabulary shared by every interpreter engine. +//! +//! `Base`/`FieldType`/`Schema` describe the shape of a column; they carry no +//! SQL semantics. Which type a given *expression* produces is engine-specific +//! and lives in each engine's own `types::infer_type`. + +use std::collections::HashMap; #[derive(Clone, PartialEq, Eq, Debug)] pub enum Base { @@ -24,264 +30,6 @@ pub struct FieldType { pub type Schema = HashMap; -use crate::expr::{BinOp, CastType, Expr, Value}; -use crate::plan::InterpError; - -/// Statically infers the FieldType of a projection expression, mirroring -/// `crate::expr::eval()`'s structure but computing a type instead of a -/// value. Sound but not tight on nullability: `nullable: true` means -/// "cannot prove this can't be NULL," not "will be NULL." -pub fn infer_type( - expr: &Expr, - schemas: &HashMap, -) -> Result { - match expr { - Expr::Column { table, name } => resolve_column_type(table.as_deref(), name, schemas), - Expr::Literal(v) => Ok(literal_type(v)), - Expr::BinaryOp { op, left, right } => { - let l = infer_type(left, schemas)?; - let r = infer_type(right, schemas)?; - Ok(binary_op_type(*op, l, r)) - } - Expr::Not(inner) => { - let inner_ty = infer_type(inner, schemas)?; - Ok(FieldType { - base: Base::Bool, - nullable: inner_ty.nullable, - }) - } - Expr::Cast { expr, target } => { - let inner_ty = infer_type(expr, schemas)?; - Ok(FieldType { - base: cast_target_base(*target), - nullable: inner_ty.nullable, - }) - } - Expr::Function { name, args } => { - let arg_types: Vec = args - .iter() - .map(|a| infer_type(a, schemas)) - .collect::>()?; - Ok(function_type(name, &arg_types)) - } - Expr::Struct(fields) => { - let field_types = fields - .iter() - .map(|(name, e)| Ok((name.clone(), infer_type(e, schemas)?))) - .collect::>()?; - Ok(FieldType { - base: Base::Struct(field_types), - nullable: false, - }) - } - Expr::List(items) => { - let item_types = items - .iter() - .map(|e| infer_type(e, schemas)) - .collect::, _>>()?; - let elem = unify_list_element_types(&item_types); - Ok(FieldType { - base: Base::List(Box::new(elem)), - nullable: false, - }) - } - Expr::FieldAccess { base, field } => { - let base_ty = infer_type(base, schemas)?; - match &base_ty.base { - Base::Struct(fields) => fields - .iter() - .find(|(name, _)| name == field) - .map(|(_, ft)| FieldType { - base: ft.base.clone(), - nullable: ft.nullable || base_ty.nullable, - }) - .ok_or_else(|| InterpError::Build(format!("Unknown struct field: {field}"))), - _ => Err(InterpError::Build(format!( - "Cannot access field '{field}' on a non-struct column" - ))), - } - } - Expr::Transform { - input_features, - output_fields, - arg, - .. - } => { - let arg_ty = infer_type(arg, schemas)?; - match &arg_ty.base { - Base::Struct(fields) => { - let got: HashSet<&String> = fields.iter().map(|(n, _)| n).collect(); - let want: HashSet<&String> = input_features.iter().collect(); - if got != want { - return Err(InterpError::Build(format!( - "transformer input struct fields {:?} do not match \ - feature_names_in_ {:?}", - fields.iter().map(|(n, _)| n).collect::>(), - input_features - ))); - } - } - _ => { - return Err(InterpError::Build( - "transformer argument must be a struct (e.g. named_struct(...))" - .to_string(), - )) - } - } - Ok(FieldType { - base: Base::Struct(output_fields.clone()), - nullable: false, - }) - } - Expr::Case { arms, default } => { - let mut branch_types: Vec = arms - .iter() - .map(|(_, result)| infer_type(result, schemas)) - .collect::>()?; - let has_else = default.is_some(); - if let Some(d) = default { - branch_types.push(infer_type(d, schemas)?); - } - // No explicit ELSE => an unmatched row yields NULL, so nullable - // regardless of the branch types. - let nullable = !has_else || branch_types.iter().any(|t| t.nullable); - Ok(FieldType { - base: common_base(&branch_types), - nullable, - }) - } - } -} - -/// Unify element types for a list literal: identical types (by FieldType -/// equality, so nullability must also agree) collapse to that type; -/// anything else (including an empty list) is unresolvable. -fn unify_list_element_types(item_types: &[FieldType]) -> FieldType { - let mut iter = item_types.iter(); - let Some(first) = iter.next() else { - return FieldType { - base: Base::Other, - nullable: true, - }; - }; - if iter.all(|t| t == first) { - first.clone() - } else { - FieldType { - base: Base::Other, - nullable: true, - } - } -} - -fn resolve_column_type( - table: Option<&str>, - name: &str, - schemas: &HashMap, -) -> Result { - if let Some(t) = table { - return schemas - .get(t) - .and_then(|s| s.get(name)) - .cloned() - .ok_or_else(|| InterpError::Build(format!("Unknown column: {t}.{name}"))); - } - let mut found = None; - for s in schemas.values() { - if let Some(ft) = s.get(name) { - if found.is_some() { - return Err(InterpError::Build(format!( - "Ambiguous column reference: {name}" - ))); - } - found = Some(ft.clone()); - } - } - found.ok_or_else(|| InterpError::Build(format!("Unknown column: {name}"))) -} - -fn literal_type(v: &Value) -> FieldType { - match v { - Value::Int(_) => FieldType { - base: Base::Int, - nullable: false, - }, - Value::Float(_) => FieldType { - base: Base::Float, - nullable: false, - }, - Value::Str(_) => FieldType { - base: Base::Str, - nullable: false, - }, - Value::Bool(_) => FieldType { - base: Base::Bool, - nullable: false, - }, - Value::Null | Value::Object(_) => FieldType { - base: Base::Other, - nullable: true, - }, - Value::Struct(fields) => FieldType { - base: Base::Struct( - fields - .iter() - .map(|(name, v)| (name.clone(), literal_type(v))) - .collect(), - ), - nullable: false, - }, - Value::List(items) => { - let inner = items.first().map(literal_type).unwrap_or(FieldType { - base: Base::Other, - nullable: true, - }); - FieldType { - base: Base::List(Box::new(inner)), - nullable: false, - } - } - } -} - -fn binary_op_type(op: BinOp, l: FieldType, r: FieldType) -> FieldType { - let nullable = l.nullable || r.nullable; - match op { - BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div | BinOp::Mod => { - let base = if l.base == Base::Int && r.base == Base::Int { - Base::Int - } else { - Base::Float - }; - FieldType { base, nullable } - } - BinOp::Eq - | BinOp::NotEq - | BinOp::Lt - | BinOp::Gt - | BinOp::LtEq - | BinOp::GtEq - | BinOp::And - | BinOp::Or => FieldType { - base: Base::Bool, - nullable, - }, - BinOp::Concat => FieldType { - base: Base::Str, - nullable, - }, - } -} - -fn cast_target_base(target: CastType) -> Base { - match target { - CastType::Str => Base::Str, - CastType::Int => Base::Int, - CastType::Float => Base::Float, - CastType::Bool => Base::Bool, - } -} - /// Is `inferred` provably safe to store in a field declared as `declared`? /// Anything not provably wrong is allowed through — Pydantic's own /// `model_validate()` is the real authority at `.infer()` time for @@ -311,55 +59,6 @@ pub fn compatible(inferred: &Base, declared: &Base) -> bool { } } -/// Common result base for variadic same-shape functions (COALESCE/NULLIF): -/// all-equal keeps that base; a mix of Int/Float widens to Float (DataFusion's -/// numeric supertype); anything else is left unresolved for Pydantic to judge. -fn common_base(args: &[FieldType]) -> Base { - match args.first() { - None => Base::Other, - Some(first) if args.iter().all(|a| a.base == first.base) => first.base.clone(), - _ if args.iter().all(|a| matches!(a.base, Base::Int | Base::Float)) => Base::Float, - _ => Base::Other, - } -} - -fn function_type(name: &str, args: &[FieldType]) -> FieldType { - let any_nullable = args.iter().any(|a| a.nullable); - match name { - "upper" | "lower" | "trim" | "substr" | "substring" => FieldType { - base: Base::Str, - nullable: any_nullable, - }, - // ABS preserves its argument's type; ROUND always yields Float (DataFusion - // returns Float64 even for an integer argument). - "abs" => { - let base = args.first().map(|a| a.base.clone()).unwrap_or(Base::Other); - FieldType { - base, - nullable: any_nullable, - } - } - "round" => FieldType { - base: Base::Float, - nullable: any_nullable, - }, - "concat" => FieldType { - base: Base::Str, - nullable: false, - }, - // DataFusion types these as the common supertype of the arguments - // (COALESCE(int, float) -> float), not args[0]'s type. - "coalesce" | "nullif" => FieldType { - base: common_base(args), - nullable: true, - }, - _ => FieldType { - base: Base::Other, - nullable: true, - }, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/value.rs b/src/value.rs new file mode 100644 index 0000000..28b6c4f --- /dev/null +++ b/src/value.rs @@ -0,0 +1,224 @@ +//! Runtime scalar values shared by every interpreter engine. +//! +//! The representation and its Python marshalling are engine-agnostic; +//! anything with SQL semantics attached (comparison, arithmetic, display) +//! lives in the per-engine `expr` module instead. + +use pyo3::prelude::*; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString}; + +#[derive(Debug)] +pub enum Value { + Int(i64), + Float(f64), + Str(String), + Bool(bool), + Null, + /// Opaque passthrough for row values that aren't a SQL primitive + /// (e.g. a nested dict). Round-trips unchanged through column refs; + /// arithmetic/comparison on it is a runtime error. + Object(Py), + /// Ordered field list (name, value). Field order is significant for + /// equality/hash, mirroring `Base::Struct`. + Struct(Vec<(String, Value)>), + /// Ordered element list. + List(Vec), +} + +impl Clone for Value { + // Py isn't Clone (cloning it requires a GIL token to bump the + // refcount safely), so this can't be derived; Python::attach supplies + // the token for the Object case. + fn clone(&self) -> Self { + match self { + Value::Int(i) => Value::Int(*i), + Value::Float(f) => Value::Float(*f), + Value::Str(s) => Value::Str(s.clone()), + Value::Bool(b) => Value::Bool(*b), + Value::Null => Value::Null, + Value::Object(o) => Python::attach(|py| Value::Object(o.clone_ref(py))), + Value::Struct(fields) => { + Value::Struct(fields.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + } + Value::List(items) => Value::List(items.iter().map(|v| v.clone()).collect()), + } + } +} + +impl PartialEq for Value { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Value::Int(a), Value::Int(b)) => a == b, + (Value::Float(a), Value::Float(b)) => a == b, + (Value::Str(a), Value::Str(b)) => a == b, + (Value::Bool(a), Value::Bool(b)) => a == b, + (Value::Null, Value::Null) => true, + (Value::Object(a), Value::Object(b)) => a.as_ptr() == b.as_ptr(), + (Value::Struct(a), Value::Struct(b)) => a == b, + (Value::List(a), Value::List(b)) => a == b, + _ => false, + } + } +} + +impl Eq for Value {} + +impl std::hash::Hash for Value { + fn hash(&self, state: &mut H) { + match self { + Value::Int(i) => { + 0u8.hash(state); + i.hash(state); + } + Value::Float(f) => { + 1u8.hash(state); + f.to_bits().hash(state); + } + Value::Str(s) => { + 2u8.hash(state); + s.hash(state); + } + Value::Bool(b) => { + 3u8.hash(state); + b.hash(state); + } + Value::Null => 4u8.hash(state), + Value::Object(o) => { + 5u8.hash(state); + (o.as_ptr() as usize).hash(state); + } + Value::Struct(fields) => { + 6u8.hash(state); + fields.hash(state); + } + Value::List(items) => { + 7u8.hash(state); + items.hash(state); + } + } + } +} + +/// Human-readable type name for error messages. +pub fn type_name(v: &Value) -> &'static str { + match v { + Value::Int(_) => "int", + Value::Float(_) => "float", + Value::Str(_) => "string", + Value::Bool(_) => "bool", + Value::Null => "null", + Value::Object(_) => "object", + Value::Struct(_) => "struct", + Value::List(_) => "list", + } +} + +impl Value { + pub fn from_pyobject(obj: &Bound<'_, PyAny>) -> PyResult { + if obj.is_none() { + return Ok(Value::Null); + } + if let Ok(b) = obj.cast::() { + return Ok(Value::Bool(b.is_true())); + } + if let Ok(i) = obj.cast::() { + return Ok(Value::Int(i.extract::()?)); + } + if let Ok(f) = obj.cast::() { + return Ok(Value::Float(f.extract::()?)); + } + if let Ok(s) = obj.cast::() { + return Ok(Value::Str(s.extract::()?)); + } + Ok(Value::Object(obj.clone().unbind())) + } + + /// Schema-driven read: converts a Python value into a `Value` per the + /// field's declared `Base`, recursing into `Struct`/`List` so a nested + /// dict/list is marshalled by its declared shape rather than falling + /// through to an opaque `Value::Object`. Scalars behave exactly like + /// `from_pyobject`. `obj` may be a raw `dict`/`list` OR (when the row + /// model declares a nested pydantic submodel / `list[X]`) an already- + /// validated nested `BaseModel` instance / `list` — struct field access + /// falls back from dict indexing to attribute access to cover both. + pub fn from_pyobject_typed( + obj: &Bound<'_, PyAny>, + base: &crate::types::Base, + ) -> PyResult { + use crate::types::Base; + if obj.is_none() { + return Ok(Value::Null); + } + match base { + Base::Struct(fields) => { + // Accept a dict (read by key) or a pydantic-model-like object + // (read by attr) as struct-shaped input; anything else (e.g. + // a bare scalar) is a genuine type mismatch and must error, + // not silently marshal into an all-null struct. + let dict = obj.cast::().ok(); + // Probe `model_fields` on the CLASS, not the instance: instance + // access is deprecated in Pydantic 2.11 (PydanticDeprecatedSince211) + // and removed in 3.0, where the instance probe would return false + // and misclassify a struct value. Class access stays valid. + if dict.is_none() && !obj.get_type().hasattr("model_fields").unwrap_or(false) { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "Expected a struct/dict value for a struct-typed field: got {}", + obj.get_type().name()? + ))); + } + let mut out = Vec::with_capacity(fields.len()); + for (name, field_ft) in fields { + let field_val = if let Some(dict) = &dict { + dict.get_item(name)? + } else { + obj.getattr(name.as_str()).ok() + }; + let v = match field_val { + Some(item) => Value::from_pyobject_typed(&item, &field_ft.base)?, + None => Value::Null, + }; + out.push((name.clone(), v)); + } + Ok(Value::Struct(out)) + } + Base::List(inner) => { + let list = obj.cast::().map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "Expected a list value for a list-typed field: {e}" + )) + })?; + let mut out = Vec::with_capacity(list.len()); + for item in list.iter() { + out.push(Value::from_pyobject_typed(&item, &inner.base)?); + } + Ok(Value::List(out)) + } + _ => Value::from_pyobject(obj), + } + } + + pub fn to_pyobject(&self, py: Python<'_>) -> PyResult> { + Ok(match self { + Value::Int(i) => i.into_pyobject(py)?.into_any().unbind(), + Value::Float(f) => f.into_pyobject(py)?.into_any().unbind(), + Value::Str(s) => s.into_pyobject(py)?.into_any().unbind(), + Value::Bool(b) => b.into_pyobject(py)?.to_owned().into_any().unbind(), + Value::Null => py.None(), + Value::Object(o) => o.clone_ref(py), + Value::Struct(fields) => { + let dict = PyDict::new(py); + for (k, v) in fields { + dict.set_item(k, v.to_pyobject(py)?)?; + } + dict.into_any().unbind() + } + Value::List(items) => { + let elements = items + .iter() + .map(|v| v.to_pyobject(py)) + .collect::>>()?; + PyList::new(py, elements)?.into_any().unbind() + } + }) + } +} diff --git a/tests/test_duckdb_interpreter.py b/tests/test_duckdb_interpreter.py new file mode 100644 index 0000000..2465f65 --- /dev/null +++ b/tests/test_duckdb_interpreter.py @@ -0,0 +1,25 @@ +"""DuckDBInferFn is a stub: it parses in the DuckDB dialect and nothing else.""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel + +from sql_transform._interpreter import DuckDBInferFn + + +class Row(BaseModel): + age: int + + +def test_builds_and_infer_raises(): + fn = DuckDBInferFn( + "SELECT age FROM __THIS__", row_tables={"__THIS__": Row}, static_tables={} + ) + with pytest.raises(NotImplementedError): + fn.infer({"__THIS__": [Row(age=1)]}) + + +def test_bad_sql_is_a_build_error(): + with pytest.raises(ValueError, match="SQL parse error"): + DuckDBInferFn("SELECT FROM", row_tables={}, static_tables={}) From 37209930a016482450e5f6cf2ca3c341c548e6b1 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 04:23:43 +0200 Subject: [PATCH 02/11] docs(specs): SQL specializer design + loop-execution design Design doc: partial-evaluation architecture (BTA collapses static subtrees at prepare; row-major AoS ABI with a prepare-time-generated marshaller; DuckDB as frontend/static-evaluator/oracle; interpreter oracle then Cranelift). Loop doc: prep checklist and the /loop-spine + workflow-fan-out execution design. duckdb/ source clone gitignored as a reference corpus. Co-Authored-By: Claude Opus 5 --- .gitignore | 3 + .../2026-07-25-sql-specializer-design.md | 278 ++++++++++++++++++ ...5-sql-specializer-loop-execution-design.md | 181 ++++++++++++ 3 files changed, 462 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-25-sql-specializer-design.md create mode 100644 docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md diff --git a/.gitignore b/.gitignore index 1c09cc8..fa8a96a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ wheels/ # Rust build artifacts /target +# DuckDB source clone — reference corpus for the SQL specializer, not a dependency +/duckdb/ + # Compiled PyO3 extension module (built in-place by `maturin develop`) sql_transform/_interpreter*.so sql_transform/_interpreter*.pyd diff --git a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md new file mode 100644 index 0000000..a8df795 --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md @@ -0,0 +1,278 @@ +# SQL Specializer for ML Model Serving — Design + +**Status:** draft for review +**Replaces the framing of:** the "DuckDB native interpreter" stub (this is not an +interpreter; it is a partial evaluator that emits one) +**Relates to:** `2026-07-14-sqltransform-rust-backend-design.md` (the current +row-at-a-time engine), `2026-07-17-codegen-inferfn-design.md` (the Python codegen +engine), the boundary-cost benchmark (memory: inference is boundary-bound). + +## 1. What this is + +A prepare-once / run-millions engine: + +``` +prepare(sql, static_tables) -> f # slow path, seconds are fine +f(batch) -> batch # hot path, ns/call budget, allocation-free +``` + +The first Futamura projection applied to a query: the query and every static +relation are inputs to a *specializer*, and what falls out is a native function +whose only remaining variable is the request payload (`__THIS__`, n ≈ 1–100 +rows). This matches exactly what `SQLTransform.fit()` already produces +conceptually — frozen state + a callable — but replaces "interpret a plan +against the frozen state" with "the frozen state was compiled into the code". + +### How it maps onto the existing Python API + +Same surface as `SQLTransform` minus transformer refs (v0 explicitly excludes +the transformer callout): + +```python +t = SpecializedTransform("SELECT age / mean_age AS z FROM __THIS__ JOIN stats ON ...") +t.fit(train_table) # STAGE 0+1: extract state, prepare, compile +t.infer_batch(rows) # STAGE 2: run(in, out, scratch) +``` + +## 2. Restatement of the architecture (compressed) + +Binding-time separation is the invariant: + +- **STAGE 0 (load):** static tables → immutable, indexed. +- **STAGE 1 (prepare):** SQL → bind → relational IR → rewrite → **binding-time + analysis** → static subtrees evaluated NOW and replaced by constants (scalars, + perfect-hash tables, packed arrays) → produce/consume lowering of the dynamic + frontier → imperative IR → backend (interpreter or codegen). +- **STAGE 2 (execute):** `void run(const Batch* in, Batch* out, Arena* scratch)` + — columnar, caller-owned, `|out|` known up front, zero malloc/syscalls. + +Two IRs (relational: rewrites + BTA annotation; imperative: SSA, typed, explicit +null lane, `StaticRef(id)` handles, verifier + round-trippable text format). Two +backends behind one interface (closure-compiled interpreter as oracle/fallback; +codegen for production). Differential testing of codegen against the interpreter +on randomized inputs is what makes the codegen backend safe to write. + +The load-bearing insight: for this workload, BTA collapses almost the whole plan. +A hash join against a static build side is a probe of a prepare-time structure; +with no pipeline breakers left, the query is a straight-line function over a +`.rodata` blob. **Most of the win is in BTA, before any backend choice.** + +## 3. Disagreements and flags (requested — "disagreement over compliance") + +1. **The engine is not the bottleneck until the boundary is fixed — and the + boundary is row-major, so the prompt's "columnar in and out" ABI is wrong + for this workload.** We measured the first half (2026-07: FFI/pydantic + marshalling dominates; compute is noise at n≈1). The second half follows + from how requests arrive: as row-major structures (dict / pydantic model / + proto). At n≈1–100 a transpose into any columnar format is a per-call fixed + cost — exactly the species of overhead this design exists to eliminate — + and "zero-copy" was never on the table for row-born data. Layout is also + computationally irrelevant at this scale: the batch is L1-resident either + way, and produce/consume generates row-at-a-time code regardless. + + The fix is the Futamura move applied to the boundary itself: the row schema + is static input, so **the marshaller is generated at prepare time** — + fixed field order, interned attribute names, direct unbox into a packed row + struct (no generic `Value` dispatch), output filled `model_construct`-style + into fixed slots rather than run through `model_validate`. The pydantic + classes remain the API; their generic marshalling path does not. + + Columnar retains two roles only: static tables at prepare time (already + `pa.Table`, cold path), and a possible *alternate* large-n batch entry point + — deferred until a consumer at n ≥ ~1k actually exists. + +2. **Substrait is a bet, not a given.** `duckdb-substrait-extension` gets us + parse+bind+typecheck+optimize for free and a serializable plan — the right + default. But the extension has coverage gaps (window frames, some casts, DDL + of temp state) and version drift vs. DuckDB releases. Mitigation: keep the + frontend behind a `plan/` interface; the fallback frontend is our existing + sqlparser wiring (already speaks the DuckDB dialect from the stub work). We + validate substrait coverage against our v0 SQL subset in week one — it's a + prep item, not an article of faith. + +3. **Prepare-time static evaluation should be DuckDB itself.** The prompt says + "eval NOW" for static subtrees. We don't need to implement that evaluator: + at prepare time, run the static subtree *as SQL in DuckDB* and materialize + the result. DuckDB is simultaneously the frontend, the static-subtree + evaluator, and the differential oracle. We only ever implement the dynamic + frontier — which is thin by construction. + +4. **Two IRs + verifier + text format is the biggest cost center — and worth it + here specifically** because this project will be built by a loop/workflow + (see companion doc). Machine-checkable gates (verifier passes, text format + round-trips, differential suite green) are what let an unattended loop make + safe progress. The verifier is not engineering hygiene; it is the loop's + review substitute between human checkpoints. + +5. **Defer QuickScorer.** Branchless ensemble traversal is a codegen pattern for + a workload we don't serve yet (no tree-encoded static tables exist in this + repo today). It stays in the doc as the marquee example of "codegen pattern + recognized during lowering", but it is milestone-last, behind a real model. + +6. **Copy-and-patch is out for v0.** Research-grade in Rust, and our prepare + happens at deploy time (fit), so we have milliseconds. LLVM ORC vs Cranelift + is the real choice — resolved below. + +7. **Cache/epoch lifecycle is mostly out of scope here.** In this repo the + "static-table epoch" *is* the fit. Refit → re-prepare → new `f`; the old one + keeps serving until swapped. Double-buffered rebuild is serving-infra work, + not engine work; one paragraph in the ops doc, no design budget. + +## 4. Open decisions — resolved + +| decision | choice | why | +|---|---|---| +| Language | Rust, same crate/workspace as `_interpreter` | existing pyo3 wiring, team velocity, wasm door stays open | +| Codegen backend | **Interpreter first (oracle), Cranelift second, LLVM ORC only if measured gap matters** | Cranelift is a mature Rust-native dep (`cranelift-jit`); −10–30% vs LLVM is invisible next to the boundary win; no C++ toolchain in the build | +| Frontend | DuckDB + substrait extension; sqlparser fallback behind the same `plan/` interface | flag 2 | +| v0 SQL subset | projection + WHERE over `__THIS__`, LEFT/INNER equi-join to static tables, CASE, arithmetic/comparison/logic, the current builtin set (upper/lower/trim/substr/abs/round/coalesce/nullif/concat), CAST | this is precisely the surface the differential corpus already covers — the oracle tests exist | +| Batch layout | row-major packed AoS: per-schema `#[repr(C)]` row structs frozen at prepare; optional fields are `(u8 flag, T)` pairs — the IR's null lane laid flat | requests are born row-major (flag 1); at this n, columnar buys nothing and costs a transpose | +| Stage 2 ABI | `run(in: *const RowIn, n, out: *mut RowOut, scratch: *mut Arena)` | amends the prompt's columnar `Batch*`; `|out|` still known up front, still zero alloc | +| Null typing strength | strong enough that the verifier rejects 3VL mistakes statically | `T?` and `T` are distinct IR types; ops on `T?` must go through `null_check`/`unwrap_or`; no implicit coercion — see §6 | +| Multi-language future | imperative IR keeps a wasm-compatible profile (no host callbacks in the hot path) | the wasm spike showed one Rust→wasm artifact serves Go/Java near-native; Cranelift is wasmtime's backend — same lowering can target both eventually | + +## 5. Stage 1 pipeline, concretely + +``` +sql text + │ duckdb: PREPARE + get_substrait(sql) (or sqlparser fallback) + ▼ +substrait plan ──► plan/: decode to Relational IR + │ + ▼ rewrite rules (predicate pushdown, projection pruning — most of this + │ already happened inside DuckDB's optimizer; we keep only what BTA needs) + ▼ +binding-time analysis: + taint(__THIS__); propagate up. + for each maximal static subtree S: + result = duckdb.execute(sql_of(S)) # flag 3: DuckDB evals statics + replace S with Const(materialize(result)) # scalar | array | perfect hash + │ + ▼ +dynamic frontier (probe → filter → project ribbon) + │ produce/consume lowering (Neumann push model, fused, no materialization) + ▼ +Imperative IR ──► verifier ──► { interpreter (oracle) | cranelift (prod) } +``` + +Static-structure selection at `Const` materialization: + +```rust +enum StaticStruct { + Scalar(Value), // 1×1 result (e.g. MEAN OVER ()) + DenseArray { base: i64, values: Column }, // dense int keys → direct index + PerfectHash(PtHashMap), // general equi-join build side + Inline(SmallVec), // tiny tables → unrolled compare chain +} +``` + +## 6. Imperative IR sketch + +SSA, typed, no allocation vocabulary, explicit null lane. Text format is the +diagnostic surface and must round-trip (`parse(print(ir)) == ir`). + +``` +;; SELECT age / s.mean_age AS z FROM __THIS__ t LEFT JOIN stats s ON t.seg = s.seg +;; after BTA: stats collapsed into staticref @0 (perfect hash: seg -> mean_age) + +fn run(in: batch{age: i64?, seg: str}, out: batch{z: f64?}) { +entry(row: idx): + %age.f, %age.v = load.opt in.age, row ; (i1 flag, i64 payload) + %seg = load in.seg, row ; NOT NULL lane: no flag + %hit.f, %m = probe.opt @0, %seg ; miss -> flag=0 (LEFT JOIN) + %num = cast f64, %age.v + %q = fdiv %num, %m + %z.f = and %age.f, %hit.f ; null iff either input null + store.opt out.z, row, %z.f, %q + next row +} +``` + +Verifier rules (initial set): + +1. SSA: single def, defs dominate uses. +2. Type check every op; `load.opt`/`probe.opt`/`store.opt` are the only ops that + touch flags; a `T?` value cannot flow into an arithmetic op — the verifier + rejects it (this is the "3VL mistakes are statically impossible" property). +3. `StaticRef` ids must resolve against the plan's static-structure table, with + matching key/value types. +4. No op allocates; `scratch` is reachable only from varlen `store` ops. +5. `out` column set and row count semantics must match the declared plan shape + (`|out| = |in|` for pure projection; filter introduces the one allowed + divergence and must declare it). + +## 7. Backends + +**Interpreter (oracle):** closure-compile the IR once — pre-traverse, build a +tree of `Box`; ~50 LOC of dispatch. Never optimized, always +correct, always available; also the fallback for ops Cranelift doesn't cover yet. + +**Cranelift:** straight-line mapping from the IR above (it is deliberately shaped +like CLIF: SSA, explicit flags, no implicit control flow). `StaticRef` resolves +to absolute addresses of the prepare-time structures, which are owned by the +compiled artifact and dropped together with it. + +Every prepared query is validated interpreter-vs-codegen on randomized inputs at +prepare time (cheap; prepare is cold) and in CI on the full corpus. + +## 8. Testing + +Three rings, outside in: + +1. **DuckDB as end-to-end oracle** — the existing `tests/differential.py` + harness pattern, new backend id `"specialized"`, oracle = `duckdb` (python + package) instead of DataFusion. Same xfail-strict bug process as decision-1, + with DuckDB as the semantics authority for this engine. +2. **Corpus mining** — `duckdb/test/sql/` (cloned in-repo) filtered to the v0 + subset: extract `query`/`statement ok` blocks over projections, filters, and + joins-to-constant-tables; skip everything touching DDL/transactions/multi- + statement state. A script materializes these as parametrized differential + cases. This is fan-out work (see companion doc). +3. **IR-level** — verifier unit tests, text-format round-trip property tests, + interpreter-vs-cranelift differential on random IR programs. + +## 9. Module layout + +Extend the existing crate (no workspace split until it hurts): + +``` +src/ + value.rs types.rs error.rs schema.rs lookup.rs # shared (already exists) + datafusion/ # existing engine, untouched + specializer/ + catalog.rs # static tables, prepared static structures + frontend.rs # duckdb/substrait (or sqlparser fallback) -> Relational IR + plan.rs # relational IR + rewrites + BTA + lower.rs # produce/consume -> imperative IR + ir/ # imperative IR: defs, verifier, printer, parser + exec/ # interp.rs, cranelift.rs + runtime.rs # arena, batch layout, perfect hash, string ops +``` + +The earlier `DuckDBInferFn` stub becomes the pyclass shell of this module +(`prepare` in `__init__`, `run` on Arrow batches); its NotImplementedError body +is replaced by the real Stage 2 entry point. + +## 10. Measurement discipline + +Before optimizing anything: baseline the boundary (row extraction, FFI crossing, +output construction) with a no-op `f` — both through the generic pydantic path +and through the generated marshaller, so the marshaller's win is itself a +measured number rather than an assumption. Then report p50/p99 ns/call at n ∈ {1, 8, 64, +1024}, always with the interpreter backend as control, and always next to the +current native engine and codegen engine numbers so the comparison is against +what we ship today, not against nothing. + +## 11. Milestones (= the review boundaries in "How to proceed") + +1. **M-restate** — this document, argued and amended. ✅ you are here +2. **M-ir** — imperative IR: grammar, types, verifier, text format; round-trip + green. +3. **M-interp** — interpreter backend over the IR; hand-written IR programs pass. +4. **M-lower** — frontend + BTA + lowering for the v0 subset; differential suite + vs DuckDB green; corpus-mined tests running. +5. **M-cranelift** — codegen backend; interpreter-vs-cranelift green; first + ns/call numbers vs baseline. +6. **M-boundary** — generated row marshaller wired into the Python API (flag 1); + end-to-end p50/p99 vs current engines. +7. (later, behind a real model) M-quickscorer. diff --git a/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md b/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md new file mode 100644 index 0000000..15619d5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md @@ -0,0 +1,181 @@ +# Building the SQL Specializer with a Loop / Dynamic Workflow + +**Status:** draft for review — the "how we build it" companion to +`2026-07-25-sql-specializer-design.md`. Covers (a) what must be prepared before +any unattended execution starts, and (b) the design of the loop itself. + +## 0. The shape of the problem + +The specializer build is a long, mostly-sequential grind (IR → interpreter → +lowering → codegen) with two properties that decide the tooling: + +- **Between milestones it is sequential and judgment-heavy** → human review + gates, one milestone at a time. A loop's job here is *pacing and persistence*, + not parallelism. +- **Within a milestone it is wide and mechanical** (one lowering rule per + operator × one differential test per rule; hundreds of corpus cases; N + verifier rules) → fan-out via dynamic workflows *inside* a loop iteration. + +So: **/loop is the spine, Workflow is the muscle.** Not either/or. + +The design doc's verifier + text format + differential suite are what make this +safe: an unattended iteration cannot self-certify with prose — it either turns +a machine-checkable gate green or it doesn't. + +## 1. Preparation checklist (all before the first unattended iteration) + +Everything here is small, and every item removes a way for the loop to stall or +silently go wrong. + +### 1.1 Decisions locked +- [ ] Design doc reviewed and amended by AmirHossein (M-restate gate). +- [ ] Fate of the pending working-tree diff decided: the `src/duckdb/` stub + + shared-module refactor (`error.rs`/`value.rs`/`types.rs` split) is + currently uncommitted on this branch. Recommendation: land it as the PR + for this branch — the refactor is exactly the shared substrate + `specializer/` needs, and the stub is the future pyclass shell. + +### 1.2 Oracle and frontend availability (validate, don't assume) +- [ ] `uv add --dev duckdb` — the differential oracle. (Project dep, not a + system install.) +- [ ] **Spike:** substrait extension actually loads and covers the v0 subset: + `INSTALL substrait FROM community; LOAD substrait; CALL get_substrait(...)` + on ~10 representative v0 queries. Needs network once at setup; result is + cached in the extension dir. If coverage fails → flip the frontend flag to + the sqlparser fallback *now*, not mid-loop. +- [ ] **Spike:** `cranelift-jit` compiles and runs a hello-world fn on this + machine (Windows ABI quirks are real). Cheap now, expensive at M-cranelift. + +### 1.3 The gate command (the loop's definition of done) +One command, exit-code-honest, that every iteration must leave green: + +```toml +# mise.toml +[tasks.gate-specializer] +run = [ + "cargo test --features specializer-tests", # IR unit + round-trip + verifier + "uv run pytest tests/ -q", # existing suites + differential +] +``` + +Plus a per-milestone extension (e.g. M-lower adds the corpus suite). The loop +never reports progress that `mise gate-specializer` can't confirm — this is the +"validate, don't assume" rule made mechanical. + +### 1.4 Task ledger +The loop needs durable, machine-readable state that survives context loss. +Use `backlog.md` (already wired via MCP): one milestone note + one task per unit +of work, each with acceptance = "gate green + which new tests exist". The loop's +first act each iteration is `task_list`, its last is `task_edit`. No progress +lives only in conversation memory. + +- [ ] Create milestone `sql-specializer` with the M-ir … M-boundary tasks + seeded from the design doc (needs AmirHossein's go — PM dispatch rule). + +### 1.5 Corpus extraction (pre-mined, not mined mid-loop) +- [ ] `scripts/mine_duckdb_corpus.py`: walk `duckdb/test/sql/{projection, + filter,join,case,cast,function/string,function/numeric}`, parse sqllogictest + blocks, keep single-statement queries within the v0 subset grammar, emit + `tests/corpus/duckdb_mined.jsonl` (sql, input schema, expected via duckdb + python at extraction time). Checked in, so loop iterations replay it + offline and deterministically. +- [ ] The `duckdb/` clone stays untracked (it's a reference corpus, not a dep); + add to `.gitignore`. + +### 1.6 Hygiene rails (mechanical, from memory/feedback) +- [ ] Branch per milestone: `git checkout -b specializer-m2-ir origin/master` + as the *first* act of a milestone (branch-first rule). +- [ ] Land via PR to `origin`, never ref-push (land-via-PR rule). One PR per + milestone, opened at the review gate. +- [ ] Bug protocol: specializer disagrees with DuckDB → xfail-strict test + + ledger ticket, never an inline semantics patch (adapted decision-1 with + DuckDB as this engine's oracle). + +## 2. Loop design + +### 2.1 State machine + +``` + ┌────────────────────────────────────────────┐ + ▼ │ + read ledger ─► milestone done? ──yes──► open PR, notify user, STOP (review gate) + │no + ▼ + pick next task ─► TDD it (red → green → gate) ─► commit ─► ledger update + │ │ + │ blocked/ambiguous? ─► write blocker note, notify, STOP │ + └────────────── ScheduleWakeup ◄─────────────────────────┘ +``` + +Hard rules: +- **STOP at milestone boundaries.** The design doc's "stopping for review at + each boundary" is a contract; the loop opens the PR and does not start the + next milestone until told to. No self-granted approvals. +- **STOP on ambiguity** (surface-confusion-early rule). A blocker note in the + ledger + a message beats an unattended guess. +- **Every commit passes the gate.** An iteration that can't get green either + reverts to last green or stops with a red-state note — it never commits red. + +### 2.2 Iteration budget and pacing +- One task per iteration (a task is sized ≤ ~1h of work: one IR op family, one + lowering rule, one verifier rule + its tests). +- Dynamic pacing: `ScheduleWakeup` long (1200s+) after a STOP; short only when + a background build/test run is the wait. +- Kill switch: the loop halts if the same task fails the gate in two consecutive + iterations → blocker note + notify (prevents grinding a wall). + +### 2.3 Where dynamic workflows plug in + +Inside a single loop iteration, when the task is wide-and-mechanical: + +| task shape | workflow pattern | +|---|---| +| implement lowering for N operators (M-lower) | `pipeline(ops, implement-in-worktree, differential-verify)` — worktree isolation per op, verify stage runs the op's mined corpus slice | +| corpus triage (M-lower) | fan-out readers over `duckdb_mined.jsonl` failures → cluster by root cause → one ledger ticket per cluster, not per case | +| verifier adversarial pass (M-ir) | N agents each try to construct an IR program that passes the verifier but breaks the interpreter; loop-until-dry | +| codegen parity sweep (M-cranelift) | random-IR generator fan-out, interpreter-vs-cranelift adversarial verify | + +Sketch of the M-lower fan-out an iteration would launch: + +```js +const results = await pipeline( + OPS, // e.g. ["fdiv", "case", "probe", ...] + op => agent(`Implement lowering for ${op}; TDD; run mise gate-specializer`, + {isolation: "worktree", phase: "Implement"}), + (r, op) => agent(`Run the ${op} slice of tests/corpus/duckdb_mined.jsonl + against the interpreter backend; report mismatches as + structured findings`, {phase: "Verify", schema: FINDINGS}), +) +``` + +The loop (not the workflow) merges surviving worktrees and runs the full gate — +one integrator, many implementers. + +### 2.4 What the loop prompt contains (draft) + +The recurring prompt must be self-contained (survives compaction): + +> Work the `sql-specializer` milestone ledger in backlog.md. Read +> `docs/superpowers/specs/2026-07-25-sql-specializer-design.md` and this doc +> first. One task per iteration; TDD; `mise gate-specializer` must be green +> before any commit; update the ledger after. STOP and notify at milestone +> boundaries (open a PR) or on any ambiguity/blocked task. Never start the next +> milestone without explicit approval. + +### 2.5 Loop vs. plain sessions — honest tradeoff + +M-ir and M-interp are compact enough that an attended session each might beat a +loop (tight feedback, no pacing overhead). The loop earns its keep from M-lower +onward, where the work is a long tail of similar units against mechanical gates. +A defensible plan: attended sessions through M-interp, loop for M-lower and +M-cranelift. Decide at the M-interp review gate. + +## 3. Open questions for the loop conversation + +1. Land the pending refactor+stub PR first, or fold it into M-ir's branch? +2. Attended vs loop for M-ir/M-interp (§2.5)? +3. Token/notification budget per unattended iteration, and quiet hours? +4. Ledger seeding needs your go (PM-dispatch rule) — seed all milestones now or + only M-ir? +5. Workflow size guideline is "medium" (≤15 agents) in this session — fine for + the sweeps above, or raise it for the corpus triage? From 32026a44fa26ce1c2eece2d58c555eba7923c7a5 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 04:32:57 +0200 Subject: [PATCH 03/11] =?UTF-8?q?chore(specializer):=20pre-loop=20preparat?= =?UTF-8?q?ion=20=E2=80=94=20gate,=20oracle,=20corpus,=20ledger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - duckdb 1.5.5 as dev dep (differential oracle). Substrait spike: extension unavailable for 1.5.5/windows_amd64 (404 all repos) -> frontend flag flipped to the sqlparser fallback; json_serialize_sql noted as parser cross-check. - cranelift-jit 0.126 spike: works on x86_64-pc-windows-msvc. - cargo test made runnable: extension-module moved out of the crate's default pyo3 features (maturin enables it via pyproject); scripts/gate.py puts the uv CPython's python3.dll on PATH. mise task: gate-specializer. - pytest testpaths pinned so the duckdb/ clone's own scripts aren't collected. - scripts/mine_duckdb_corpus.py: 678 replayable cases mined from duckdb's sqllogictests into tests/corpus/duckdb_mined.jsonl (three-outcome contract). - backlog: milestone m-7 seeded, TASK-41..45 (M-ir..M-boundary) chained. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 5 +- backlog/milestones/m-7 - sql-specializer.md | 8 + ...224-grammar-types-verifier-text-format.md" | 28 + ...piled-IR-interpreter-the-oracle-backend.md | 29 + ...duce-consume-lowering-for-the-v0-subset.md | 33 + ...degen-backend-behind-the-same-interface.md | 29 + ...ary-generated-row-marshaller-Python-API.md | 30 + .../2026-07-25-sql-specializer-design.md | 2 +- ...5-sql-specializer-loop-execution-design.md | 70 +- mise.toml | 4 + pyproject.toml | 7 + scripts/gate.py | 21 + scripts/mine_duckdb_corpus.py | 179 +++++ tests/corpus/duckdb_mined.jsonl | 678 ++++++++++++++++++ uv.lock | 17 + 15 files changed, 1105 insertions(+), 35 deletions(-) create mode 100644 backlog/milestones/m-7 - sql-specializer.md create mode 100644 "backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" create mode 100644 backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md create mode 100644 backlog/tasks/task-43 - Specializer-M-lower-frontend-BTA-produce-consume-lowering-for-the-v0-subset.md create mode 100644 backlog/tasks/task-44 - Specializer-M-cranelift-codegen-backend-behind-the-same-interface.md create mode 100644 backlog/tasks/task-45 - Specializer-M-boundary-generated-row-marshaller-Python-API.md create mode 100644 scripts/gate.py create mode 100644 scripts/mine_duckdb_corpus.py create mode 100644 tests/corpus/duckdb_mined.jsonl diff --git a/Cargo.toml b/Cargo.toml index 725c6cc..c2aa715 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" name = "_interpreter" crate-type = ["cdylib"] +# extension-module is deliberately NOT a default feature: it suppresses the +# libpython link, which is right for the wheel but breaks `cargo test`. +# maturin enables it via [tool.maturin] features in pyproject.toml. [dependencies] -pyo3 = { version = "0.29", features = ["extension-module", "abi3-py314"] } +pyo3 = { version = "0.29", features = ["abi3-py314"] } sqlparser = "0.62" diff --git a/backlog/milestones/m-7 - sql-specializer.md b/backlog/milestones/m-7 - sql-specializer.md new file mode 100644 index 0000000..28e2144 --- /dev/null +++ b/backlog/milestones/m-7 - sql-specializer.md @@ -0,0 +1,8 @@ +--- +id: m-7 +title: "sql-specializer" +--- + +## Description + +SQL specializer for ML model serving: prepare-once/run-millions engine built on binding-time separation. Design: docs/superpowers/specs/2026-07-25-sql-specializer-design.md; execution process: docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md. diff --git "a/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" "b/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" new file mode 100644 index 0000000..4f6e840 --- /dev/null +++ "b/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" @@ -0,0 +1,28 @@ +--- +id: TASK-41 +title: 'Specializer M-ir: imperative IR — grammar, types, verifier, text format' +status: To Do +assignee: [] +created_date: '2026-07-25 02:31' +labels: [] +milestone: m-7 +dependencies: [] +documentation: + - docs/superpowers/specs/2026-07-25-sql-specializer-design.md +type: feature +ordinal: 35000 +--- + +## Description + + +Define the specializer's imperative IR per §6 of docs/superpowers/specs/2026-07-25-sql-specializer-design.md: SSA over typed scalars with a separate null lane (T? vs T are distinct types; ops on T? only via the .opt instructions), StaticRef handles, no allocation vocabulary. Deliverables are the IR definitions, the verifier, and a round-trippable text format under src/specializer/ir/. This is the diagnostic surface for the whole engine and the loop's machine-checkable gate substrate — the boundary must be airtight before anything targets it. + + +## Acceptance Criteria + +- [ ] #1 Verifier rejects: non-SSA defs, type mismatches, any arithmetic on a T? value not routed through the null-lane ops, unresolvable StaticRef ids, allocating constructs +- [ ] #2 Text format round-trips: parse(print(ir)) == ir on every test program, including a property/fuzz round-trip test +- [ ] #3 Hand-written IR programs covering every instruction exist as test fixtures +- [ ] #4 mise gate-specializer green + diff --git a/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md b/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md new file mode 100644 index 0000000..b92a926 --- /dev/null +++ b/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md @@ -0,0 +1,29 @@ +--- +id: TASK-42 +title: 'Specializer M-interp: closure-compiled IR interpreter (the oracle backend)' +status: To Do +assignee: [] +created_date: '2026-07-25 02:31' +labels: [] +milestone: m-7 +dependencies: + - TASK-41 +documentation: + - docs/superpowers/specs/2026-07-25-sql-specializer-design.md +type: feature +ordinal: 36000 +--- + +## Description + + +Implement the interpreter backend over the imperative IR (design doc §7): one pre-traversal builds a closure tree, then execution is plain dispatch. This backend is the differential-testing oracle for every future codegen backend and the fallback for uncovered ops — correctness and coverage over speed, never optimized. Depends on M-ir for the IR definitions and verifier. + + +## Acceptance Criteria + +- [ ] #1 Every IR instruction executes; the M-ir fixture programs all produce expected outputs +- [ ] #2 Runs only verifier-accepted IR; rejects unverified programs +- [ ] #3 No allocation during execution (arena-only for varlen), asserted by a test +- [ ] #4 mise gate-specializer green + diff --git a/backlog/tasks/task-43 - Specializer-M-lower-frontend-BTA-produce-consume-lowering-for-the-v0-subset.md b/backlog/tasks/task-43 - Specializer-M-lower-frontend-BTA-produce-consume-lowering-for-the-v0-subset.md new file mode 100644 index 0000000..822baed --- /dev/null +++ b/backlog/tasks/task-43 - Specializer-M-lower-frontend-BTA-produce-consume-lowering-for-the-v0-subset.md @@ -0,0 +1,33 @@ +--- +id: TASK-43 +title: >- + Specializer M-lower: frontend + BTA + produce/consume lowering for the v0 + subset +status: To Do +assignee: [] +created_date: '2026-07-25 02:31' +labels: [] +milestone: m-7 +dependencies: + - TASK-42 +documentation: + - docs/superpowers/specs/2026-07-25-sql-specializer-design.md + - docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md +type: feature +ordinal: 37000 +--- + +## Description + + +The load-bearing milestone (design doc §5): sqlparser(DuckDB dialect) frontend to relational IR; binding-time analysis that taints __THIS__ and evaluates every all-static subtree in DuckDB at prepare time, materializing Const structures (scalar / dense array / perfect hash / inline); produce/consume lowering of the dynamic frontier to imperative IR. v0 subset per design doc §4. Differential oracle is DuckDB (python pkg); the mined corpus at tests/corpus/duckdb_mined.jsonl replays under the three-outcome contract (match / clean-unsupported / FAIL) documented in scripts/mine_duckdb_corpus.py. Wide-mechanical: run per-operator fan-out via workflows per the loop-execution doc §2.3. + + +## Acceptance Criteria + +- [ ] #1 v0-subset queries prepare end-to-end: sql + static tables -> verified imperative IR running on the interpreter backend +- [ ] #2 All-static subtrees are evaluated at prepare time: a static-tables-only query lowers to a constant emitter with no probe/filter ops in its IR +- [ ] #3 Differential suite vs DuckDB green on hand-written v0 cases; engine-vs-oracle disagreement follows the xfail-strict + ticket protocol +- [ ] #4 Corpus replay reports match / clean-unsupported / FAIL counts; zero FAILs; every unsupported rejection is a clean build-time error naming the construct +- [ ] #5 mise gate-specializer green (corpus replay wired into it) + diff --git a/backlog/tasks/task-44 - Specializer-M-cranelift-codegen-backend-behind-the-same-interface.md b/backlog/tasks/task-44 - Specializer-M-cranelift-codegen-backend-behind-the-same-interface.md new file mode 100644 index 0000000..7b25a82 --- /dev/null +++ b/backlog/tasks/task-44 - Specializer-M-cranelift-codegen-backend-behind-the-same-interface.md @@ -0,0 +1,29 @@ +--- +id: TASK-44 +title: 'Specializer M-cranelift: codegen backend behind the same interface' +status: To Do +assignee: [] +created_date: '2026-07-25 02:31' +labels: [] +milestone: m-7 +dependencies: + - TASK-43 +documentation: + - docs/superpowers/specs/2026-07-25-sql-specializer-design.md +type: feature +ordinal: 38000 +--- + +## Description + + +Cranelift-jit backend for the imperative IR (design doc §7; cranelift-jit 0.126 spike-verified on x86_64-pc-windows-msvc 2026-07-25). StaticRef handles resolve to absolute addresses of prepare-time structures owned by the compiled artifact. Interpreter-vs-cranelift differential on random IR programs plus the full corpus; first ns/call numbers per the measurement discipline (design doc §10). + + +## Acceptance Criteria + +- [ ] #1 Every v0-subset prepared query compiles under cranelift and agrees with the interpreter backend on the corpus and on randomized inputs +- [ ] #2 Uncovered ops fall back to the interpreter backend rather than failing prepare +- [ ] #3 p50/p99 ns/call reported at n in {1, 8, 64, 1024} against the interpreter control and the existing native + codegen engines +- [ ] #4 mise gate-specializer green + diff --git a/backlog/tasks/task-45 - Specializer-M-boundary-generated-row-marshaller-Python-API.md b/backlog/tasks/task-45 - Specializer-M-boundary-generated-row-marshaller-Python-API.md new file mode 100644 index 0000000..20a8350 --- /dev/null +++ b/backlog/tasks/task-45 - Specializer-M-boundary-generated-row-marshaller-Python-API.md @@ -0,0 +1,30 @@ +--- +id: TASK-45 +title: 'Specializer M-boundary: generated row marshaller + Python API' +status: To Do +assignee: [] +created_date: '2026-07-25 02:32' +labels: [] +milestone: m-7 +dependencies: + - TASK-44 +documentation: + - docs/superpowers/specs/2026-07-25-sql-specializer-design.md +type: feature +ordinal: 39000 +--- + +## Description + + +Wire the specializer into the Python surface: SpecializedTransform (SQLTransform API minus transformer refs) whose fit() runs prepare and whose infer/infer_batch cross the boundary through a prepare-time-generated marshaller — fixed field order, interned names, packed row structs, model_construct-style output fill (design doc §3 flag 1: input is row-major; no columnar transpose in the hot path). Baseline the boundary with a no-op f through both the generic pydantic path and the generated marshaller so the win is measured, then end-to-end p50/p99 vs the current engines. + + +## Acceptance Criteria + +- [ ] #1 SpecializedTransform fit/infer/infer_batch works end-to-end on the v0 subset with dict and pydantic-model rows +- [ ] #2 No-op-f boundary baseline reported: generic pydantic path vs generated marshaller, p50/p99 at n in {1, 8, 64, 1024} +- [ ] #3 End-to-end p50/p99 vs the current native and codegen engines reported +- [ ] #4 Steady-state hot path allocates nothing per call beyond the output objects; arena reset only +- [ ] #5 mise gate-specializer green + diff --git a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md index a8df795..4630f41 100644 --- a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md +++ b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md @@ -124,7 +124,7 @@ with no pipeline breakers left, the query is a straight-line function over a |---|---|---| | Language | Rust, same crate/workspace as `_interpreter` | existing pyo3 wiring, team velocity, wasm door stays open | | Codegen backend | **Interpreter first (oracle), Cranelift second, LLVM ORC only if measured gap matters** | Cranelift is a mature Rust-native dep (`cranelift-jit`); −10–30% vs LLVM is invisible next to the boundary win; no C++ toolchain in the build | -| Frontend | DuckDB + substrait extension; sqlparser fallback behind the same `plan/` interface | flag 2 | +| Frontend | **sqlparser (DuckDB dialect)** — the substrait extension does not exist for duckdb 1.5.5 / windows_amd64 (spiked 2026-07-25: HTTP 404 from community, core, and nightly repos). `json_serialize_sql` (core, no extension) exposes DuckDB's own parse as JSON: use it as a differential check on our parser, and as the fallback frontend if sqlparser dialect drift ever bites | flag 2, resolved by spike | | v0 SQL subset | projection + WHERE over `__THIS__`, LEFT/INNER equi-join to static tables, CASE, arithmetic/comparison/logic, the current builtin set (upper/lower/trim/substr/abs/round/coalesce/nullif/concat), CAST | this is precisely the surface the differential corpus already covers — the oracle tests exist | | Batch layout | row-major packed AoS: per-schema `#[repr(C)]` row structs frozen at prepare; optional fields are `(u8 flag, T)` pairs — the IR's null lane laid flat | requests are born row-major (flag 1); at this n, columnar buys nothing and costs a transpose | | Stage 2 ABI | `run(in: *const RowIn, n, out: *mut RowOut, scratch: *mut Arena)` | amends the prompt's columnar `Batch*`; `|out|` still known up front, still zero alloc | diff --git a/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md b/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md index 15619d5..604c1a7 100644 --- a/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md +++ b/docs/superpowers/specs/2026-07-25-sql-specializer-loop-execution-design.md @@ -28,39 +28,40 @@ Everything here is small, and every item removes a way for the loop to stall or silently go wrong. ### 1.1 Decisions locked -- [ ] Design doc reviewed and amended by AmirHossein (M-restate gate). -- [ ] Fate of the pending working-tree diff decided: the `src/duckdb/` stub + - shared-module refactor (`error.rs`/`value.rs`/`types.rs` split) is - currently uncommitted on this branch. Recommendation: land it as the PR - for this branch — the refactor is exactly the shared substrate - `specializer/` needs, and the stub is the future pyclass shell. +- [ ] Design doc reviewed and amended by AmirHossein (M-restate gate). Partially + done in conversation (row-major ABI amendment, 2026-07-25); final go still + pending. +- [x] Pending diff landed on this branch (2026-07-25): the `src/duckdb/` stub + + shared-module refactor (`error.rs`/`value.rs`/`types.rs` split) — the + shared substrate `specializer/` sits on; the stub is the future pyclass + shell. ### 1.2 Oracle and frontend availability (validate, don't assume) -- [ ] `uv add --dev duckdb` — the differential oracle. (Project dep, not a - system install.) -- [ ] **Spike:** substrait extension actually loads and covers the v0 subset: - `INSTALL substrait FROM community; LOAD substrait; CALL get_substrait(...)` - on ~10 representative v0 queries. Needs network once at setup; result is - cached in the extension dir. If coverage fails → flip the frontend flag to - the sqlparser fallback *now*, not mid-loop. -- [ ] **Spike:** `cranelift-jit` compiles and runs a hello-world fn on this - machine (Windows ABI quirks are real). Cheap now, expensive at M-cranelift. +- [x] `uv add --dev duckdb` — the differential oracle. duckdb 1.5.5 installed. +- [x] **Spike (2026-07-25): substrait is UNAVAILABLE** — HTTP 404 for + duckdb 1.5.5 / windows_amd64 from community, core, and nightly repos. + Frontend flag flipped to the sqlparser fallback (design doc §4 updated). + Bonus finding: `json_serialize_sql` (core, extension-free) exposes + DuckDB's own AST as JSON — usable as a differential check on our parser. +- [x] **Spike (2026-07-25): `cranelift-jit` 0.126 works on + x86_64-pc-windows-msvc** — built and called `f(x) = x*2+42` at runtime, + correct results. Version pin recorded for M-cranelift. ### 1.3 The gate command (the loop's definition of done) One command, exit-code-honest, that every iteration must leave green: ```toml -# mise.toml +# mise.toml — wired 2026-07-25 and green (cargo test + 574 pytest) [tasks.gate-specializer] -run = [ - "cargo test --features specializer-tests", # IR unit + round-trip + verifier - "uv run pytest tests/ -q", # existing suites + differential -] +run = "uv run python scripts/gate.py" # cargo test + pytest, one exit code ``` -Plus a per-milestone extension (e.g. M-lower adds the corpus suite). The loop -never reports progress that `mise gate-specializer` can't confirm — this is the -"validate, don't assume" rule made mechanical. +`scripts/gate.py` also handles the Windows wrinkle: tests link libpython +(extension-module moved to a maturin-only feature), so the runner puts the +uv-managed CPython's `python3.dll` on PATH. Per-milestone suites get appended +there (e.g. M-lower adds the corpus replay). The loop never reports progress +that `mise gate-specializer` can't confirm — "validate, don't assume" made +mechanical. ### 1.4 Task ledger The loop needs durable, machine-readable state that survives context loss. @@ -69,18 +70,21 @@ of work, each with acceptance = "gate green + which new tests exist". The loop's first act each iteration is `task_list`, its last is `task_edit`. No progress lives only in conversation memory. -- [ ] Create milestone `sql-specializer` with the M-ir … M-boundary tasks - seeded from the design doc (needs AmirHossein's go — PM dispatch rule). +- [x] Milestone `sql-specializer` (m-7) seeded 2026-07-25 with TASK-41 (M-ir) → + TASK-42 (M-interp) → TASK-43 (M-lower) → TASK-44 (M-cranelift) → + TASK-45 (M-boundary), dependency-chained. Working the chain (dispatch) + still needs AmirHossein's go — PM dispatch rule. ### 1.5 Corpus extraction (pre-mined, not mined mid-loop) -- [ ] `scripts/mine_duckdb_corpus.py`: walk `duckdb/test/sql/{projection, - filter,join,case,cast,function/string,function/numeric}`, parse sqllogictest - blocks, keep single-statement queries within the v0 subset grammar, emit - `tests/corpus/duckdb_mined.jsonl` (sql, input schema, expected via duckdb - python at extraction time). Checked in, so loop iterations replay it - offline and deterministically. -- [ ] The `duckdb/` clone stays untracked (it's a reference corpus, not a dep); - add to `.gitignore`. +- [x] `scripts/mine_duckdb_corpus.py` (2026-07-25): 678 cases from 2758 queries + across 250 files → `tests/corpus/duckdb_mined.jsonl` (262 KB, checked in; + setup statements + sql + duckdb-computed expected rows). Replay contract + is three-outcome — match / clean-unsupported / FAIL — so the corpus + includes SQL beyond the v0 builtin list on purpose: each case the engine + learns flips from clean-unsupported to must-match. See the script + docstring. +- [x] The `duckdb/` clone stays untracked; `.gitignore`d. `testpaths` pinned in + pyproject so pytest never collects the clone's own test_*.py scripts. ### 1.6 Hygiene rails (mechanical, from memory/feedback) - [ ] Branch per milestone: `git checkout -b specializer-m2-ir origin/master` diff --git a/mise.toml b/mise.toml index 26c3c98..932eb60 100644 --- a/mise.toml +++ b/mise.toml @@ -9,6 +9,10 @@ run = ["uv run ruff check .", "uv run ruff format ."] description = "Run tests" run = "uv run pytest" +[tasks.gate-specializer] +description = "Specializer loop gate: cargo test + pytest, one exit code" +run = "uv run python scripts/gate.py" + [tasks.test-watch] description = "Run tests in watch mode" run = "uv run pytest --no-cov -x -s -vv --ff --lf" diff --git a/pyproject.toml b/pyproject.toml index 689ca20..696e521 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ [dependency-groups] dev = [ + "duckdb>=1.5.5", "ipdb>=0.13.13", "ipython>=9.2.0", "maturin>=1.14,<2.0", @@ -33,6 +34,9 @@ build-backend = "maturin" module-name = "sql_transform._interpreter" python-source = "." exclude = ["**/*_test.py", "tests/**"] +# See the note in Cargo.toml: extension-module only for the built wheel, +# so `cargo test` can link libpython. +features = ["pyo3/extension-module"] [tool.ruff] fix = true @@ -47,3 +51,6 @@ lint.ignore = ["S101"] [tool.pytest.ini_options] python_files = ["*_test.py", "test_*.py"] +# Scope collection: the duckdb/ source clone (reference corpus) carries its own +# test_*.py scripts that break under our pytest. +testpaths = ["tests", "sql_transform"] diff --git a/scripts/gate.py b/scripts/gate.py new file mode 100644 index 0000000..66f1261 --- /dev/null +++ b/scripts/gate.py @@ -0,0 +1,21 @@ +"""The specializer loop's definition of done: cargo test + pytest, one exit code. + +`cargo test` needs python3.dll on PATH at runtime (the crate links libpython for +tests; see the note in Cargo.toml). Run under `uv run` so sys.base_prefix is the +uv-managed CPython that owns that DLL. +""" + +import os +import subprocess +import sys + +env = os.environ.copy() +env["PATH"] = sys.base_prefix + os.pathsep + env["PATH"] + +for cmd in (["cargo", "test"], [sys.executable, "-m", "pytest", "-q"]): + print(f"gate: {' '.join(cmd)}", flush=True) + # noqa justification: fixed argv, no untrusted input. + result = subprocess.run(cmd, env=env) # noqa: S603 + if result.returncode != 0: + sys.exit(result.returncode) +print("gate: green") diff --git a/scripts/mine_duckdb_corpus.py b/scripts/mine_duckdb_corpus.py new file mode 100644 index 0000000..b769747 --- /dev/null +++ b/scripts/mine_duckdb_corpus.py @@ -0,0 +1,179 @@ +"""Mine the duckdb/ source clone's sqllogictest corpus into replayable cases. + +Walks a fixed set of test/sql subtrees, replays each file's setup statements in +an in-memory DuckDB, and keeps every `query` block whose SQL falls inside the +specializer's v0 subset. Expected outputs are recomputed by DuckDB at mining +time (the file's own expected blocks are ignored — sort modes and hashing make +them fiddly, and DuckDB itself is our oracle anyway). + +Output: tests/corpus/duckdb_mined.jsonl, one case per line: + {"source": ..., "setup": [...], "sql": ..., "cols": [...], "rows": [[...]]} + +Replay contract: run `setup` in a fresh DuckDB to reconstruct the input tables, +feed them to the engine under test, and classify into THREE outcomes: + match — engine output equals `rows` + clean-unsupported — engine rejects at build time with an "unsupported" error + FAIL — mismatch, wrong error, or crash +The shape filter below only drops what can never be v0 (aggregation, windows, +set ops, subqueries). It deliberately keeps SQL beyond the v0 builtin list +(exotic string functions, star-expansion sugar, `::` casts): those must fail +*cleanly* today, and each one the engine learns flips from clean-unsupported to +must-match — the corpus is the growth ladder, not a fixed pass bar. + +ponytail: line-oriented parse, no real sqllogictest grammar. Files using +directives we don't model (require, loop, mode, ...) are skipped whole; a +mis-parsed edge case at worst drops a case, never fabricates one, because every +kept SQL is re-executed by DuckDB before it is recorded. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import duckdb + +REPO = Path(__file__).resolve().parent.parent +CORPUS_DIRS = [ + "projection", + "filter", + "conjunction", + "cast", + "select", + "join/inner", + "join/left_outer", + "function/string", + "function/numeric", + "function/generic", + "function/operator", +] +OUT = REPO / "tests" / "corpus" / "duckdb_mined.jsonl" + +# A whole file is skipped when it uses machinery the replayer doesn't model. +FILE_SKIP = re.compile( + r"^(require|load|loop|foreach|endloop|mode|hash-threshold|restart|sleep|concurrentloop)\b", + re.MULTILINE, +) + +# v0 subset filter: single plain SELECT, no set ops / aggregation / windows / +# subqueries / CTEs. Conservative — a false reject only shrinks the corpus. +BANNED = re.compile( + r"\b(WITH|UNION|EXCEPT|INTERSECT|GROUP\s+BY|HAVING|ORDER\s+BY|LIMIT|OFFSET|" + r"OVER|DISTINCT|EXISTS|VALUES|SAMPLE|QUALIFY|WINDOW|LATERAL|PIVOT|UNNEST|" + r"RECURSIVE|IN\s*\()\b", + re.IGNORECASE, +) +SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE) +SETUP_KEEP = re.compile(r"^(CREATE|INSERT|DROP|UPDATE|DELETE)\b", re.IGNORECASE) +SCALARS = (int, float, str, bool, type(None)) + +MAX_ROWS = 64 # bigger results are batch tests, not unit cases + + +def v0_ok(sql: str) -> bool: + flat = " ".join(sql.split()) + return ( + flat.upper().startswith("SELECT") + and ";" not in flat.rstrip(";") + and not BANNED.search(flat) + and not SUBQUERY.search(flat) + and " FROM " in flat.upper() # pure-literal SELECTs have no dynamic input + ) + + +def blocks(text: str): + """Yield ('statement'|'query', sql) for the blocks we understand.""" + lines = text.splitlines() + i = 0 + while i < len(lines): + line = lines[i].strip() + if line.startswith("statement ok") or line.startswith("query "): + kind = "statement" if line.startswith("statement") else "query" + i += 1 + sql_lines = [] + while i < len(lines) and lines[i].strip() not in ("", "----"): + sql_lines.append(lines[i]) + i += 1 + # skip the file's expected block; we recompute via duckdb + while i < len(lines) and lines[i].strip() != "": + i += 1 + yield kind, "\n".join(sql_lines).strip() + else: + i += 1 + + +def mine_file(path: Path, rel: str, out) -> tuple[int, int]: + text = path.read_text(encoding="utf-8", errors="replace") + if FILE_SKIP.search(text): + return 0, 0 + con = duckdb.connect() + setup: list[str] = [] + kept = seen = 0 + for kind, sql in blocks(text): + if not sql: + continue + if kind == "statement": + try: + con.execute(sql) + except Exception: # noqa: BLE001 -- broken state; stop replaying + break + if SETUP_KEEP.match(sql): + setup.append(sql) + continue + seen += 1 + if not v0_ok(sql): + continue + try: + cur = con.execute(sql) + cols = [d[0] for d in cur.description] + rows = cur.fetchall() + except Exception: # noqa: BLE001, S112 -- needs state we skipped; drop case + continue + if len(rows) > MAX_ROWS: + continue + if any(not isinstance(v, SCALARS) for row in rows for v in row): + continue + out.write( + json.dumps( + { + "source": rel, + "setup": list(setup), + "sql": sql, + "cols": cols, + "rows": [list(r) for r in rows], + } + ) + + "\n" + ) + kept += 1 + con.close() + return kept, seen + + +def main() -> int: + root = REPO / "duckdb" / "test" / "sql" + if not root.is_dir(): + print(f"duckdb clone not found at {root}", file=sys.stderr) + return 1 + OUT.parent.mkdir(parents=True, exist_ok=True) + total_kept = total_seen = files = 0 + with OUT.open("w", encoding="utf-8") as out: + for d in CORPUS_DIRS: + dir_kept = 0 + for path in sorted((root / d).rglob("*.test")): + rel = path.relative_to(REPO / "duckdb").as_posix() + kept, seen = mine_file(path, rel, out) + dir_kept += kept + total_kept += kept + total_seen += seen + files += 1 + print(f"{d:24s} +{dir_kept}") + print(f"\n{total_kept} cases kept / {total_seen} queries seen / {files} files") + print(f"-> {OUT.relative_to(REPO)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/corpus/duckdb_mined.jsonl b/tests/corpus/duckdb_mined.jsonl new file mode 100644 index 0000000..31e72f5 --- /dev/null +++ b/tests/corpus/duckdb_mined.jsonl @@ -0,0 +1,678 @@ +{"source": "test/sql/projection/coalesce_error.test", "setup": ["CREATE TABLE vals AS SELECT * FROM (\n\tVALUES (1, 'hello'), (NULL, '2'), (3, NULL)\n) tbl(a, b)"], "sql": "SELECT COALESCE(a, b::INT) FROM vals", "cols": ["COALESCE(a, CAST(b AS INTEGER))"], "rows": [[1], [2], [3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE i FROM integers", "cols": ["j", "k"], "rows": [[2, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (i, j) FROM integers", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (j) FROM integers", "cols": ["i", "k"], "rows": [[1, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (j) FROM integers", "cols": ["i", "k"], "rows": [[1, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (\"J\") FROM integers", "cols": ["i", "k"], "rows": [[1, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* EXCLUDE (i) FROM integers", "cols": ["j", "k"], "rows": [[2, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* EXCLUDE ('i') FROM integers", "cols": ["j", "k"], "rows": [[2, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* EXCLUDE (i, j) FROM integers", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* EXCLUDE (j) FROM integers", "cols": ["i", "k"], "rows": [[1, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* EXCLUDE (i, j), * EXCLUDE (i, j), * EXCLUDE (i, k) FROM integers", "cols": ["k", "k", "j"], "rows": [[3, 3, 2]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (i, j) FROM integers i1, integers i2", "cols": ["k", "k"], "rows": [[3, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT i1.* EXCLUDE (i, j), i2.* EXCLUDE (i, j, k) FROM integers i1, integers i2", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT i1.* EXCLUDE (i, j), i2.* EXCLUDE (k) FROM integers i1, integers i2", "cols": ["k", "i", "j"], "rows": [[3, 1, 2]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (i) FROM integers i1 JOIN integers i2 USING (i)", "cols": ["j", "k", "j", "k"], "rows": [[2, 3, 2, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE integers.i FROM integers", "cols": ["j", "k"], "rows": [[2, 3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (integers.i, integers.j) FROM integers", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* EXCLUDE (integers.i, integers.j) FROM integers", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (INTEGERS.i, integers.J) FROM integers", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_exclude.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (i1.i, i2.i) FROM integers i1 JOIN integers i2 USING (i)", "cols": ["j", "k", "j", "k"], "rows": [[2, 3, 2, 3]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT COLUMNS(lambda x: x LIKE 'col%') FROM integers", "cols": ["col1", "col2"], "rows": [[1, 2]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * LIKE 'col%' FROM integers", "cols": ["col1", "col2"], "rows": [[1, 2]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * NOT LIKE 'col%' FROM integers", "cols": ["k"], "rows": [[3]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * ILIKE 'COL%' FROM integers", "cols": ["col1", "col2"], "rows": [[1, 2]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * SIMILAR TO '.*col.*' FROM integers", "cols": ["col1", "col2"], "rows": [[1, 2]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (col1) SIMILAR TO '.*col.*' FROM integers", "cols": ["col2"], "rows": [[2]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)", "CREATE TABLE t1(id INTEGER, col1 INTEGER, col2 INTEGER)", "INSERT INTO t1 VALUES (1, 10, 20)", "CREATE TABLE t2(name VARCHAR, category VARCHAR, col2 INTEGER)", "INSERT INTO t2 VALUES ('foo', 'bar', 30)"], "sql": "SELECT t1.* LIKE 'col%' FROM t1, t2", "cols": ["col1", "col2"], "rows": [[10, 20]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)", "CREATE TABLE t1(id INTEGER, col1 INTEGER, col2 INTEGER)", "INSERT INTO t1 VALUES (1, 10, 20)", "CREATE TABLE t2(name VARCHAR, category VARCHAR, col2 INTEGER)", "INSERT INTO t2 VALUES ('foo', 'bar', 30)"], "sql": "SELECT t2.* LIKE 'col%' FROM t1, t2", "cols": ["col2"], "rows": [[30]]} +{"source": "test/sql/projection/select_star_like.test", "setup": ["CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)", "CREATE TABLE t1(id INTEGER, col1 INTEGER, col2 INTEGER)", "INSERT INTO t1 VALUES (1, 10, 20)", "CREATE TABLE t2(name VARCHAR, category VARCHAR, col2 INTEGER)", "INSERT INTO t2 VALUES ('foo', 'bar', 30)"], "sql": "SELECT t1.* LIKE 'col%', t2.* LIKE 'col%' FROM t1, t2", "cols": ["col1", "col2", "col2"], "rows": [[10, 20, 30]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * REPLACE i+100 AS i FROM integers", "cols": ["i", "j", "k"], "rows": [[101, 2, 3]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (j, k) REPLACE (i+100 AS i), * EXCLUDE (j) REPLACE (i+100 AS i), * EXCLUDE (j, k) REPLACE (i+101 AS i) FROM integers", "cols": ["i", "i", "k", "i"], "rows": [[101, 101, 3, 102]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * REPLACE (i+100 AS i, j+200 AS \"J\") FROM integers", "cols": ["i", "J", "k"], "rows": [[101, 202, 3]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT integers.* REPLACE (i+100 AS i) FROM integers", "cols": ["i", "j", "k"], "rows": [[101, 2, 3]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)", "CREATE TABLE test_star(id INTEGER, col1 VARCHAR, col2 VARCHAR, col3 INTEGER);", "INSERT INTO test_star VALUES (1, 'val1', 'val2', 42);"], "sql": "SELECT test_star.* REPLACE ('computed: ' || (id * 2)::VARCHAR AS col1) FROM test_star;", "cols": ["id", "col1", "col2", "col3"], "rows": [[1, "computed: 2", "val2", 42]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)", "CREATE TABLE test_star(id INTEGER, col1 VARCHAR, col2 VARCHAR, col3 INTEGER);", "INSERT INTO test_star VALUES (1, 'val1', 'val2', 42);"], "sql": "SELECT test_star.* EXCLUDE (col2) REPLACE (id * 2 AS col1) RENAME (col3 AS final3) FROM test_star;", "cols": ["id", "col1", "final3"], "rows": [[1, 2, 42]]} +{"source": "test/sql/projection/select_star_replace.test", "setup": ["CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)", "CREATE TABLE test_star(id INTEGER, col1 VARCHAR, col2 VARCHAR, col3 INTEGER);", "INSERT INTO test_star VALUES (1, 'val1', 'val2', 42);"], "sql": "SELECT test_star.* EXCLUDE (col2) REPLACE ('replaced: ' || col3::VARCHAR AS col1) RENAME (id AS renamed_id, col3 AS final3) FROM test_star;", "cols": ["renamed_id", "col1", "final3"], "rows": [[1, "replaced: 42", 42]]} +{"source": "test/sql/projection/select_struct_star.test", "setup": ["CREATE TABLE test(a STRUCT(i INT, j INT));"], "sql": "SELECT a.* FROM test;", "cols": ["i", "j"], "rows": []} +{"source": "test/sql/projection/select_struct_star.test", "setup": ["CREATE TABLE test(a STRUCT(i INT, j INT));"], "sql": "SELECT a.* EXCLUDE(j) FROM test;", "cols": ["i"], "rows": []} +{"source": "test/sql/projection/select_struct_star.test", "setup": ["CREATE TABLE test(a STRUCT(i INT, j INT));"], "sql": "SELECT a.* EXCLUDE(i) FROM test;", "cols": ["j"], "rows": []} +{"source": "test/sql/projection/select_struct_star.test", "setup": ["CREATE TABLE test(a STRUCT(i INT, j INT));"], "sql": "SELECT a.* REPLACE(a.i + 3 AS i) FROM test;", "cols": ["i", "j"], "rows": []} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* REPLACE a AS a) FROM range(1) t1(a) NATURAL JOIN range(1) t2(a)", "cols": ["a"], "rows": [[0]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT * FROM range(1) t1(a) NATURAL JOIN range(1) t2(a)", "cols": ["a"], "rows": [[0]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* REPLACE a AS a) FROM range(1) t1(a) JOIN range(1) t2(a) USING (a)", "cols": ["a"], "rows": [[0]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* REPLACE (a + 1 AS a)) FROM range(1) t1(a) NATURAL JOIN range(1) t2(a)", "cols": ["a"], "rows": [[1]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* REPLACE (a * 2 AS a)) FROM range(3) t1(a) NATURAL JOIN range(3) t2(a)", "cols": ["a"], "rows": [[0], [2], [4]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* REPLACE (a AS a)) FROM range(1) t1(a) NATURAL JOIN range(1) t2(a) NATURAL JOIN range(1) t3(a)", "cols": ["a"], "rows": [[0]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* REPLACE (a + 1 AS a)) FROM range(2) t1(a) NATURAL JOIN range(2) t2(a) NATURAL JOIN range(2) t3(a)", "cols": ["a"], "rows": [[1], [2]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": [], "sql": "SELECT COLUMNS(* RENAME a AS b) FROM range(1) t1(a) NATURAL JOIN range(1) t2(a)", "cols": ["b"], "rows": [[0]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": ["CREATE TABLE test_t1 AS SELECT 1 as a, 2 as b, 3 as c", "CREATE TABLE test_t2 AS SELECT 1 as a, 4 as d, 5 as e"], "sql": "SELECT COLUMNS(* REPLACE (a + 10 AS a, b + 20 AS b)) FROM test_t1 NATURAL JOIN test_t2", "cols": ["a", "b", "c", "d", "e"], "rows": [[11, 22, 3, 4, 5]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": ["CREATE TABLE test_t1 AS SELECT 1 as a, 2 as b, 3 as c", "CREATE TABLE test_t2 AS SELECT 1 as a, 4 as d, 5 as e"], "sql": "SELECT COLUMNS(* REPLACE (concat(a::VARCHAR, '_joined') AS a)) FROM test_t1 NATURAL JOIN test_t2", "cols": ["a", "b", "c", "d", "e"], "rows": [["1_joined", 2, 3, 4, 5]]} +{"source": "test/sql/projection/test_columns_replace_using.test", "setup": ["CREATE TABLE test_t1 AS SELECT 1 as a, 2 as b, 3 as c", "CREATE TABLE test_t2 AS SELECT 1 as a, 4 as d, 5 as e", "CREATE TABLE integers(i INTEGER, j INTEGER, k INTEGER)", "INSERT INTO integers VALUES (1, 2, 3)"], "sql": "SELECT * EXCLUDE (i1.i, i2.i) FROM integers i1 JOIN integers i2 USING (i)", "cols": ["j", "k", "j", "k"], "rows": [[2, 3, 2, 3]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT * FROM exprtest", "cols": ["a", "b"], "rows": [[42, 10], [43, 100], [null, 1], [45, -1]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT a FROM exprtest WHERE a BETWEEN 43 AND 44", "cols": ["a"], "rows": [[43]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT a FROM exprtest WHERE a NOT BETWEEN 43 AND 44", "cols": ["a"], "rows": [[42], [45]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT a FROM exprtest WHERE a BETWEEN b AND 44", "cols": ["a"], "rows": [[42]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT CASE a WHEN 42 THEN 100 WHEN 43 THEN 200 ELSE 300 END FROM exprtest", "cols": ["CASE WHEN ((a = 42)) THEN (100) WHEN ((a = 43)) THEN (200) ELSE 300 END"], "rows": [[100], [200], [300], [300]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT CASE WHEN a = 42 THEN 100 WHEN a = 43 THEN 200 ELSE 300 END FROM exprtest", "cols": ["CASE WHEN ((a = 42)) THEN (100) WHEN ((a = 43)) THEN (200) ELSE 300 END"], "rows": [[100], [200], [300], [300]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT CASE WHEN a = 42 THEN 100 WHEN a = 43 THEN 200 END FROM exprtest", "cols": ["CASE WHEN ((a = 42)) THEN (100) WHEN ((a = 43)) THEN (200) ELSE NULL END"], "rows": [[100], [200], [null], [null]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)"], "sql": "SELECT ABS(b) FROM exprtest", "cols": ["abs(b)"], "rows": [[10], [100], [1], [1]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);"], "sql": "SELECT * FROM intest WHERE NULL IN ('a', 'b')", "cols": ["a", "b", "c"], "rows": []} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);"], "sql": "SELECT * FROM intest WHERE NULL NOT IN ('a', 'b')", "cols": ["a", "b", "c"], "rows": []} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);", "CREATE TABLE strtest (a INTEGER, b VARCHAR)", "INSERT INTO strtest VALUES (1, 'a'), (2, 'h'), (3, 'd')", "INSERT INTO strtest VALUES (4, NULL)"], "sql": "SELECT a FROM strtest WHERE b = 'a'", "cols": ["a"], "rows": [[1]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);", "CREATE TABLE strtest (a INTEGER, b VARCHAR)", "INSERT INTO strtest VALUES (1, 'a'), (2, 'h'), (3, 'd')", "INSERT INTO strtest VALUES (4, NULL)"], "sql": "SELECT a FROM strtest WHERE b <> 'a'", "cols": ["a"], "rows": [[2], [3]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);", "CREATE TABLE strtest (a INTEGER, b VARCHAR)", "INSERT INTO strtest VALUES (1, 'a'), (2, 'h'), (3, 'd')", "INSERT INTO strtest VALUES (4, NULL)"], "sql": "SELECT a FROM strtest WHERE b < 'h'", "cols": ["a"], "rows": [[1], [3]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);", "CREATE TABLE strtest (a INTEGER, b VARCHAR)", "INSERT INTO strtest VALUES (1, 'a'), (2, 'h'), (3, 'd')", "INSERT INTO strtest VALUES (4, NULL)"], "sql": "SELECT a FROM strtest WHERE b <= 'h'", "cols": ["a"], "rows": [[1], [2], [3]]} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);", "CREATE TABLE strtest (a INTEGER, b VARCHAR)", "INSERT INTO strtest VALUES (1, 'a'), (2, 'h'), (3, 'd')", "INSERT INTO strtest VALUES (4, NULL)"], "sql": "SELECT a FROM strtest WHERE b > 'h'", "cols": ["a"], "rows": []} +{"source": "test/sql/projection/test_complex_expressions.test", "setup": ["CREATE TABLE exprtest (a INTEGER, b INTEGER)", "INSERT INTO exprtest VALUES (42, 10), (43, 100), (NULL, 1), (45, -1)", "CREATE TABLE intest (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO intest VALUES (42, 42, 42), (43, 42, 42), (44, 41, 44);", "CREATE TABLE strtest (a INTEGER, b VARCHAR)", "INSERT INTO strtest VALUES (1, 'a'), (2, 'h'), (3, 'd')", "INSERT INTO strtest VALUES (4, NULL)"], "sql": "SELECT a FROM strtest WHERE b >= 'h'", "cols": ["a"], "rows": [[2]]} +{"source": "test/sql/projection/test_row_id.test", "setup": ["create table a(i integer);", "insert into a values (42), (44);"], "sql": "SELECT rowid, * FROM a", "cols": ["rowid", "i"], "rows": [[0, 42], [1, 44]]} +{"source": "test/sql/projection/test_row_id.test", "setup": ["create table a(i integer);", "insert into a values (42), (44);"], "sql": "SELECT rowid+1 FROM a WHERE CASE WHEN i=42 THEN rowid=0 ELSE rowid=1 END;", "cols": ["(rowid + 1)"], "rows": [[1], [2]]} +{"source": "test/sql/projection/test_row_id.test", "setup": ["create table a(i integer);", "insert into a values (42), (44);"], "sql": "SELECT * FROM a", "cols": ["i"], "rows": [[42], [44]]} +{"source": "test/sql/projection/test_row_id.test", "setup": ["create table a(i integer);", "insert into a values (42), (44);", "create table b(rowid integer);", "insert into b values (42), (22);", "UPDATE b SET rowid=5", "INSERT INTO b (rowid) VALUES (5)"], "sql": "SELECT * FROM b", "cols": ["rowid"], "rows": [[5], [5], [5]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT rowid + 1, rowid - 1, rowid + rowid, i + rowid FROM a", "cols": ["(rowid + 1)", "(rowid - 1)", "(rowid + rowid)", "(i + rowid)"], "rows": [[1, -1, 0, 42]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT -rowid, +rowid, abs(rowid) FROM a", "cols": ["-(rowid)", "+(rowid)", "abs(rowid)"], "rows": [[0, 0, 0]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT rowid BETWEEN -1 AND 1, 0 BETWEEN rowid AND 1, 1 BETWEEN -3 AND rowid FROM a", "cols": ["(rowid BETWEEN -1 AND 1)", "(0 BETWEEN rowid AND 1)", "(1 BETWEEN -3 AND rowid)"], "rows": [[true, true, false]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT rowid < i, rowid = NULL, rowid = i, rowid <> 0 FROM a", "cols": ["(rowid < i)", "(rowid = NULL)", "(rowid = i)", "(rowid != 0)"], "rows": [[true, null, false, false]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT SUM(rowid), MIN(rowid), MAX(rowid), COUNT(rowid), FIRST(rowid) FROM a", "cols": ["sum(rowid)", "min(rowid)", "max(rowid)", "count(rowid)", "\"first\"(rowid)"], "rows": [[0, 0, 0, 1, 0]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT SUM(rowid), MIN(rowid), MAX(rowid), COUNT(rowid), LAST(rowid) FROM a", "cols": ["sum(rowid)", "min(rowid)", "max(rowid)", "count(rowid)", "\"last\"(rowid)"], "rows": [[0, 0, 0, 1, 0]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT COUNT(*) FROM a", "cols": ["count_star()"], "rows": [[1]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT * FROM a, a a2 WHERE a.rowid=a2.rowid", "cols": ["i", "i"], "rows": [[42, 42]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT * FROM a, a a2 WHERE a.rowid<>a2.rowid", "cols": ["i", "i"], "rows": []} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);"], "sql": "SELECT * FROM a, a a2 WHERE a.rowid>=a2.rowid", "cols": ["i", "i"], "rows": [[42, 42]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);", "INSERT INTO a SELECT rowid FROM a", "UPDATE a SET i=rowid"], "sql": "SELECT * FROM a WHERE rowid=0", "cols": ["i"], "rows": [[0]]} +{"source": "test/sql/projection/test_row_id_expression.test", "setup": ["create table a(i integer);", "insert into a values (42);", "INSERT INTO a SELECT rowid FROM a", "UPDATE a SET i=rowid"], "sql": "SELECT * FROM a WHERE rowid=0 OR rowid=1", "cols": ["i"], "rows": [[0], [1]]} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);"], "sql": "SELECT i, j FROM a;", "cols": ["i", "j"], "rows": []} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);"], "sql": "SELECT * FROM a;", "cols": ["i", "j"], "rows": []} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);"], "sql": "SELECT * FROM a;", "cols": ["i", "j"], "rows": []} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);"], "sql": "SELECT x, y FROM a i1(x, y);", "cols": ["x", "y"], "rows": []} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);", "CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT a, b FROM test;", "cols": ["a", "b"], "rows": [[11, 22], [12, 21], [13, 22]]} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);", "CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT a + 2, b FROM test WHERE a = 11;", "cols": ["(a + 2)", "b"], "rows": [[13, 22]]} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);", "CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT a + 2, b FROM test WHERE a = 12;", "cols": ["(a + 2)", "b"], "rows": [[14, 21]]} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);", "CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT cast(a AS VARCHAR) FROM test;", "cols": ["CAST(a AS VARCHAR)"], "rows": [["11"], ["12"], ["13"]]} +{"source": "test/sql/projection/test_simple_projection.test", "setup": ["CREATE TABLE a (i integer, j integer);", "CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT cast(cast(a AS VARCHAR) as INTEGER) FROM test;", "cols": ["CAST(CAST(a AS VARCHAR) AS INTEGER)"], "rows": [[11], [12], [13]]} +{"source": "test/sql/projection/test_table_star.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER)", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT * FROM test", "cols": ["a", "b"], "rows": [[11, 22], [12, 21], [13, 22]]} +{"source": "test/sql/projection/test_table_star.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER)", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT test.* FROM test", "cols": ["a", "b"], "rows": [[11, 22], [12, 21], [13, 22]]} +{"source": "test/sql/projection/test_table_star.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER)", "INSERT INTO test VALUES (11, 22), (12, 21), (13, 22)"], "sql": "SELECT t.* FROM test t", "cols": ["a", "b"], "rows": [[11, 22], [12, 21], [13, 22]]} +{"source": "test/sql/filter/filter_cache.test", "setup": ["CREATE TABLE integers AS SELECT a FROM generate_series(0, 9999, 1) tbl(a), generate_series(0, 9, 1) tbl2(b);"], "sql": "SELECT COUNT(*) FROM integers WHERE a<5;", "cols": ["count_star()"], "rows": [[50]]} +{"source": "test/sql/filter/test_alias_filter.test", "setup": ["CREATE TABLE integers(i INTEGER)", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT i % 2 AS k FROM integers WHERE k<>0;", "cols": ["k"], "rows": [[1], [1]]} +{"source": "test/sql/filter/test_alias_filter.test", "setup": ["CREATE TABLE integers(i INTEGER)", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT i % 2 AS i FROM integers WHERE i<>0;", "cols": ["i"], "rows": [[1], [0], [1]]} +{"source": "test/sql/filter/test_alias_filter.test", "setup": ["CREATE TABLE integers(i INTEGER)", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT i % 2 AS k FROM integers WHERE integers.i<>0;", "cols": ["k"], "rows": [[1], [0], [1]]} +{"source": "test/sql/filter/test_alias_filter.test", "setup": ["CREATE TABLE integers(i INTEGER)", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT i % 2 AS k FROM integers WHERE k=k;", "cols": ["k"], "rows": [[1], [0], [1]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2<>3", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2>1", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2>=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2<3", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2<=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT a=NULL FROM integers", "cols": ["(a = NULL)"], "rows": [[null]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT NULL=a FROM integers", "cols": ["(NULL = a)"], "rows": [[null]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE 2 IN (((1*2)+(1*0))*1, 3, 4, 5)", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_constant_comparisons.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (2, 12)"], "sql": "SELECT * FROM integers WHERE CASE WHEN 2=2 THEN true ELSE false END;", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a>0", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>0 AND a=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a<4", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<4 AND a=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a<=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a>=2", "cols": ["a", "b"], "rows": [[2, 12]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>2 AND a>4", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>4 AND a>2", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>4 AND a>=4", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>=4 AND a>4", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<2 AND a<4", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<4 AND a<2", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<2 AND a<=2", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<=2 AND a<2", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<2 AND a<>3", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<=1 AND a<>3", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>4 AND a<>2", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>=5 AND a<>2", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>=4 AND a<>4 AND a<>4", "cols": ["a", "b"], "rows": [[5, null]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<3 AND a<4 AND a<5 AND a<10 AND a<2 AND a<20", "cols": ["a", "b"], "rows": [[1, 10]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a=4", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a>4", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>4 AND a=2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a>2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a>=4 AND a=2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=4 AND a<2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<2 AND a=4", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a<2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<=2 AND a=4", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<2 AND a>4", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a=2 AND a<>2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<>2 AND a=2", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE 0", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)"], "sql": "SELECT * FROM integers WHERE a<2 AND 0", "cols": ["a", "b"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s='hello'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s='world'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s<>'hello'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s<>'world'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s>'a'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s>='hello'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s<='hello'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s<'z'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s<='a'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s<'hello'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s>'hello'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s='hello' AND s>='z'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s<>'hello' AND s<='a'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s<>'hello' AND s<'hello'", "cols": ["s"], "rows": []} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s<>'hello' AND s>'hello'", "cols": ["s"], "rows": [["world"]]} +{"source": "test/sql/filter/test_obsolete_filters.test", "setup": ["CREATE TABLE integers(a INTEGER, b INTEGER)", "INSERT INTO integers VALUES (1, 10), (2, 12), (3, 14), (4, 16), (5, NULL), (NULL, NULL)", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('world'), (NULL)"], "sql": "SELECT * FROM strings WHERE s<>'world' AND s>='hello'", "cols": ["s"], "rows": [["hello"]]} +{"source": "test/sql/filter/test_or_pushdown.test", "setup": ["create table t1 as select range a, 'foo' || (range%100)::VARCHAR b, 100000-range c, timestamp '1992-01-01' + interval (range * 20) minute d from range(900000);"], "sql": "select count(*) from t1 where a < 5 or b = 'foo8';", "cols": ["count_star()"], "rows": [[9005]]} +{"source": "test/sql/filter/test_or_pushdown.test", "setup": ["create table t1 as select range a, 'foo' || (range%100)::VARCHAR b, 100000-range c, timestamp '1992-01-01' + interval (range * 20) minute d from range(900000);"], "sql": "select a from t1 where a=5 or a=899999;", "cols": ["a"], "rows": [[5], [899999]]} +{"source": "test/sql/filter/test_or_pushdown.test", "setup": ["create table t1 as select range a, 'foo' || (range%100)::VARCHAR b, 100000-range c, timestamp '1992-01-01' + interval (range * 20) minute d from range(900000);"], "sql": "select a from t1 where d=timestamp '1992-01-01 01:40:00' or d=timestamp '2026-03-22 23:40:00'", "cols": ["a"], "rows": [[5], [899999]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i=5 AND j>=i", "cols": ["i", "j"], "rows": [[5, 5], [5, 6]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i>9 AND j>=i", "cols": ["i", "j"], "rows": [[10, 10], [10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i>=10 AND j>=i", "cols": ["i", "j"], "rows": [[10, 10], [10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i<1 AND j>=i", "cols": ["i", "j"], "rows": [[0, 0], [0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i<=0 AND j>=i", "cols": ["i", "j"], "rows": [[0, 0], [0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i=5 AND j<=i", "cols": ["i", "j"], "rows": [[5, 5], [5, 4]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i>9 AND j<=i", "cols": ["i", "j"], "rows": [[10, 10], [10, 9]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i>=10 AND j<=i", "cols": ["i", "j"], "rows": [[10, 10], [10, 9]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i<1 AND j<=i", "cols": ["i", "j"], "rows": [[0, 0], [0, -1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i<=0 AND j<=i", "cols": ["i", "j"], "rows": [[0, 0], [0, -1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i=5 AND j>i", "cols": ["i", "j"], "rows": [[5, 6]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i>9 AND j>i", "cols": ["i", "j"], "rows": [[10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i>=10 AND j>i", "cols": ["i", "j"], "rows": [[10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i<1 AND j>i", "cols": ["i", "j"], "rows": [[0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i<=0 AND j>i", "cols": ["i", "j"], "rows": [[0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE i=5 AND j9 AND j=10 AND j=i AND i=5", "cols": ["i", "j"], "rows": [[5, 5], [5, 6]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>=i AND i>9", "cols": ["i", "j"], "rows": [[10, 10], [10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>=i AND i>=10", "cols": ["i", "j"], "rows": [[10, 10], [10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>=i AND i<1", "cols": ["i", "j"], "rows": [[0, 0], [0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>=i AND i<=0", "cols": ["i", "j"], "rows": [[0, 0], [0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j<=i AND i=5", "cols": ["i", "j"], "rows": [[5, 5], [5, 4]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j<=i AND i>9", "cols": ["i", "j"], "rows": [[10, 10], [10, 9]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j<=i AND i>=10", "cols": ["i", "j"], "rows": [[10, 10], [10, 9]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j<=i AND i<1", "cols": ["i", "j"], "rows": [[0, 0], [0, -1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j<=i AND i<=0", "cols": ["i", "j"], "rows": [[0, 0], [0, -1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>i AND i=5", "cols": ["i", "j"], "rows": [[5, 6]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>i AND i>9", "cols": ["i", "j"], "rows": [[10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>i AND i>=10", "cols": ["i", "j"], "rows": [[10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>i AND i<1", "cols": ["i", "j"], "rows": [[0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j>i AND i<=0", "cols": ["i", "j"], "rows": [[0, 1]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j9", "cols": ["i", "j"], "rows": [[10, 9]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j=10", "cols": ["i", "j"], "rows": [[10, 9]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i"], "sql": "SELECT * FROM vals1 WHERE j9 AND k>=j AND j>=i AND l>=k", "cols": ["i", "j", "k", "l"], "rows": [[10, 10, 10, 10], [10, 10, 10, 11]]} +{"source": "test/sql/filter/test_transitive_filters.test", "setup": ["CREATE TABLE vals1 AS SELECT i AS i, i AS j FROM range(0, 11, 1) t1(i)", "INSERT INTO vals1 SELECT i, i+1 FROM vals1", "INSERT INTO vals1 SELECT DISTINCT(i), i-1 FROM vals1 ORDER by i", "CREATE TABLE vals2(k BIGINT, l BIGINT)", "INSERT INTO vals2 SELECT * FROM vals1"], "sql": "SELECT * FROM vals1, vals2 WHERE i<1 AND k<=j AND j<=i AND l<=k", "cols": ["i", "j", "k", "l"], "rows": [[0, 0, 0, 0], [0, 0, 0, -1]]} +{"source": "test/sql/conjunction/or_between.test", "setup": ["CREATE TABLE tab0(pk INTEGER PRIMARY KEY, col0 INTEGER, col1 FLOAT, col2 VARCHAR, col3 INTEGER, col4 FLOAT, col5 VARCHAR);", "INSERT INTO tab0 VALUES(0,22,43.95999908447265625,'yoyca',0,80.1399993896484375,'eoenc');", "INSERT INTO tab0 VALUES(1,51,34.900001525878905361,'zeqhw',44,13.489999771118164062,'easox');", "INSERT INTO tab0 VALUES(2,42,59.759998321533203125,'ylshk',15,4.4499998092651367187,'xgrvy');", "INSERT INTO tab0 VALUES(3,67,90.660003662109378552,'rnadc',77,50.360000610351560723,'knooo');", "INSERT INTO tab0 VALUES(4,48,53.099998474121097302,'txhlv',75,9.770000457763671875,'gvudx');", "INSERT INTO tab0 VALUES(5,18,40.580001831054683947,'wgfxz',96,12.5,'mmxbj');", "INSERT INTO tab0 VALUES(6,84,24.239999771118165838,'ttodp',31,72.999999999999998223,'wujjl');", "INSERT INTO tab0 VALUES(7,86,67.449996948242185723,'mwgbl',38,10.479999542236329013,'ypcha');", "INSERT INTO tab0 VALUES(8,68,38.470001220703125,'kaoqh',8,41.500000000000003552,'fyhzl');", "INSERT INTO tab0 VALUES(9,29,19.600000381469726562,'kbenw',20,19.579999923706054687,'gsszq');"], "sql": "SELECT pk FROM tab0 WHERE (col0 BETWEEN 67 AND 0 OR col0 > 17 AND (col0 > 3));", "cols": ["pk"], "rows": [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]]} +{"source": "test/sql/conjunction/or_comparison.test", "setup": ["CREATE TABLE tab0(pk INTEGER PRIMARY KEY, col0 INTEGER, col1 FLOAT, col2 VARCHAR, col3 INTEGER, col4 FLOAT, col5 VARCHAR);;", "INSERT INTO tab0 VALUES(0,86,94.959999084472652697,'vuopk',91,47.779998779296875,'sikuk');", "INSERT INTO tab0 VALUES(1,37,20.850000381469726562,'zfuqj',30,10.689999580383300337,'rtcha');", "INSERT INTO tab0 VALUES(2,6,47.540000915527347302,'vubiv',38,46.630001068115234375,'nnuey');", "INSERT INTO tab0 VALUES(3,72,30.629999160766603338,'rezor',59,11.189999580383300781,'mdler');", "INSERT INTO tab0 VALUES(4,12,29.620000839233400213,'gvmir',68,69.669998168945310723,'yfnxa');", "INSERT INTO tab0 VALUES(5,64,21.489999771118162286,'ruygp',31,13.659999847412109819,'fhwkn');", "INSERT INTO tab0 VALUES(6,90,9.3699998855590820312,'tehgk',71,65.720001220703121447,'uibwi');", "INSERT INTO tab0 VALUES(7,38,43.139999389648435723,'axgch',27,98.949996948242180394,'oikcl');", "INSERT INTO tab0 VALUES(8,3,22.399999618530275213,'yigev',19,45.650001525878902697,'tbbkw');", "INSERT INTO tab0 VALUES(9,95,14.31999969482421875,'vssyg',8,12.930000305175781605,'epgyr');"], "sql": "SELECT pk FROM tab0 WHERE col0 < 84 OR col0 < 8 ;", "cols": ["pk"], "rows": [[1], [2], [3], [4], [5], [7], [8]]} +{"source": "test/sql/conjunction/or_comparison.test", "setup": ["CREATE TABLE tab0(pk INTEGER PRIMARY KEY, col0 INTEGER, col1 FLOAT, col2 VARCHAR, col3 INTEGER, col4 FLOAT, col5 VARCHAR);;", "INSERT INTO tab0 VALUES(0,86,94.959999084472652697,'vuopk',91,47.779998779296875,'sikuk');", "INSERT INTO tab0 VALUES(1,37,20.850000381469726562,'zfuqj',30,10.689999580383300337,'rtcha');", "INSERT INTO tab0 VALUES(2,6,47.540000915527347302,'vubiv',38,46.630001068115234375,'nnuey');", "INSERT INTO tab0 VALUES(3,72,30.629999160766603338,'rezor',59,11.189999580383300781,'mdler');", "INSERT INTO tab0 VALUES(4,12,29.620000839233400213,'gvmir',68,69.669998168945310723,'yfnxa');", "INSERT INTO tab0 VALUES(5,64,21.489999771118162286,'ruygp',31,13.659999847412109819,'fhwkn');", "INSERT INTO tab0 VALUES(6,90,9.3699998855590820312,'tehgk',71,65.720001220703121447,'uibwi');", "INSERT INTO tab0 VALUES(7,38,43.139999389648435723,'axgch',27,98.949996948242180394,'oikcl');", "INSERT INTO tab0 VALUES(8,3,22.399999618530275213,'yigev',19,45.650001525878902697,'tbbkw');", "INSERT INTO tab0 VALUES(9,95,14.31999969482421875,'vssyg',8,12.930000305175781605,'epgyr');"], "sql": "select pk from tab0 where col0 = 37 or col0 = 86", "cols": ["pk"], "rows": [[0], [1]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::BOOLEAN FROM bits;", "cols": ["CAST(b AS BOOLEAN)"], "rows": [[true]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::TINYINT FROM bits;", "cols": ["CAST(b AS \"TINYINT\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::SMALLINT FROM bits;", "cols": ["CAST(b AS SMALLINT)"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::INTEGER FROM bits;", "cols": ["CAST(b AS INTEGER)"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::BIGINT FROM bits;", "cols": ["CAST(b AS BIGINT)"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::UTINYINT FROM bits;", "cols": ["CAST(b AS \"UTINYINT\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::USMALLINT FROM bits;", "cols": ["CAST(b AS \"USMALLINT\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::UINTEGER FROM bits;", "cols": ["CAST(b AS \"UINTEGER\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::UBIGINT FROM bits;", "cols": ["CAST(b AS \"UBIGINT\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::HUGEINT FROM bits;", "cols": ["CAST(b AS \"HUGEINT\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::UHUGEINT FROM bits;", "cols": ["CAST(b AS \"UHUGEINT\")"], "rows": [[15]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::FLOAT FROM bits;", "cols": ["CAST(b AS FLOAT)"], "rows": [[2.1019476964872256e-44]]} +{"source": "test/sql/cast/test_bit_cast.test", "setup": ["CREATE TABLE bits (b bit);", "INSERT INTO bits VALUES('00001111');"], "sql": "SELECT b::DOUBLE FROM bits;", "cols": ["CAST(b AS \"DOUBLE\")"], "rows": [[7.4e-323]]} +{"source": "test/sql/cast/test_boolean_cast.test", "setup": ["CREATE TABLE tbl AS SELECT 0 AS yes"], "sql": "SELECT CAST(yes AS BOOLEAN) FROM tbl", "cols": ["CAST(\"yes\" AS BOOLEAN)"], "rows": [[false]]} +{"source": "test/sql/cast/test_try_cast.test", "setup": ["CREATE TABLE try_cast(try_cast INTEGER);", "INSERT INTO try_cast VALUES (3);"], "sql": "SELECT try_cast FROM try_cast;", "cols": ["try_cast"], "rows": [[3]]} +{"source": "test/sql/cast/test_try_cast.test", "setup": ["CREATE TABLE try_cast(try_cast INTEGER);", "INSERT INTO try_cast VALUES (3);"], "sql": "SELECT try_cast(try_cast as bigint) FROM try_cast;", "cols": ["TRY_CAST(\"try_cast\" AS BIGINT)"], "rows": [[3]]} +{"source": "test/sql/cast/test_try_cast.test", "setup": ["CREATE TABLE try_cast(try_cast INTEGER);", "INSERT INTO try_cast VALUES (3);"], "sql": "SELECT try_cast(try_cast(try_cast as integer) as integer) FROM try_cast;", "cols": ["TRY_CAST(TRY_CAST(\"try_cast\" AS INTEGER) AS INTEGER)"], "rows": [[3]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);"], "sql": "SELECT test.tbl.col FROM test.tbl;", "cols": ["col"], "rows": [[1], [2], [3]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);", "CREATE SCHEMA t", "CREATE TABLE t.t(t ROW(t INTEGER));", "INSERT INTO t.t VALUES ({'t': 42});"], "sql": "SELECT t.t.t.t FROM t.t;", "cols": ["t"], "rows": [[42]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);", "CREATE SCHEMA t", "CREATE TABLE t.t(t ROW(t INTEGER));", "INSERT INTO t.t VALUES ({'t': 42});", "DROP SCHEMA t CASCADE;", "CREATE SCHEMA t", "CREATE TABLE t.t AS SELECT {'t': {'t': {'t': {'t': {'t': 42}}}}} t"], "sql": "SELECT t.t.t.t.t.t.t.t FROM t.t;", "cols": ["t"], "rows": [[42]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);", "CREATE SCHEMA t", "CREATE TABLE t.t(t ROW(t INTEGER));", "INSERT INTO t.t VALUES ({'t': 42});", "DROP SCHEMA t CASCADE;", "CREATE SCHEMA t", "CREATE TABLE t.t AS SELECT {'t': {'t': {'t': {'t': {'t': 42}}}}} t", "DROP SCHEMA t CASCADE", "CREATE SCHEMA s1", "CREATE SCHEMA s2", "CREATE TABLE s1.t1 AS SELECT 42 t", "CREATE TABLE s2.t1 AS SELECT 84 t"], "sql": "SELECT s1.t1.t FROM s1.t1, s2.t1", "cols": ["t"], "rows": [[42]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);", "CREATE SCHEMA t", "CREATE TABLE t.t(t ROW(t INTEGER));", "INSERT INTO t.t VALUES ({'t': 42});", "DROP SCHEMA t CASCADE;", "CREATE SCHEMA t", "CREATE TABLE t.t AS SELECT {'t': {'t': {'t': {'t': {'t': 42}}}}} t", "DROP SCHEMA t CASCADE", "CREATE SCHEMA s1", "CREATE SCHEMA s2", "CREATE TABLE s1.t1 AS SELECT 42 t", "CREATE TABLE s2.t1 AS SELECT 84 t"], "sql": "SELECT * EXCLUDE (s1.t1.t) FROM s1.t1, s2.t1", "cols": ["t"], "rows": [[84]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);", "CREATE SCHEMA t", "CREATE TABLE t.t(t ROW(t INTEGER));", "INSERT INTO t.t VALUES ({'t': 42});", "DROP SCHEMA t CASCADE;", "CREATE SCHEMA t", "CREATE TABLE t.t AS SELECT {'t': {'t': {'t': {'t': {'t': 42}}}}} t", "DROP SCHEMA t CASCADE", "CREATE SCHEMA s1", "CREATE SCHEMA s2", "CREATE TABLE s1.t1 AS SELECT 42 t", "CREATE TABLE s2.t1 AS SELECT 84 t"], "sql": "SELECT * EXCLUDE (S1.T1.T) FROM s1.t1, s2.t1", "cols": ["t"], "rows": [[84]]} +{"source": "test/sql/select/test_multi_column_reference.test", "setup": ["CREATE SCHEMA test", "CREATE TABLE test.tbl(col INTEGER);", "INSERT INTO test.tbl VALUES (1), (2), (3);", "CREATE SCHEMA t", "CREATE TABLE t.t(t ROW(t INTEGER));", "INSERT INTO t.t VALUES ({'t': 42});", "DROP SCHEMA t CASCADE;", "CREATE SCHEMA t", "CREATE TABLE t.t AS SELECT {'t': {'t': {'t': {'t': {'t': 42}}}}} t", "DROP SCHEMA t CASCADE", "CREATE SCHEMA s1", "CREATE SCHEMA s2", "CREATE TABLE s1.t1 AS SELECT 42 t", "CREATE TABLE s2.t1 AS SELECT 84 t"], "sql": "SELECT s2.t1.t FROM s1.t1, s2.t1", "cols": ["t"], "rows": [[84]]} +{"source": "test/sql/select/test_positional_reference.test", "setup": [], "sql": "SELECT #1 FROM range(1)", "cols": ["range"], "rows": [[0]]} +{"source": "test/sql/select/test_positional_reference.test", "setup": [], "sql": "SELECT #1+#2 FROM range(1) tbl, range(1) tbl2", "cols": ["(#1 + #2)"], "rows": [[0]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": [], "sql": "select column_name from (describe SELECT j : 42)", "cols": ["column_name"], "rows": [["j"]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": [], "sql": "select column_name from (describe SELECT \"hel lo\" : 42)", "cols": ["column_name"], "rows": [["hel lo"]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": ["CREATE TABLE a (i INTEGER);", "INSERT INTO a VALUES (42);"], "sql": "SELECT j : i FROM a", "cols": ["j"], "rows": [[42]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": ["CREATE TABLE a (i INTEGER);", "INSERT INTO a VALUES (42);"], "sql": "SELECT \"j\" : \"i\" FROM a", "cols": ["j"], "rows": [[42]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": ["CREATE TABLE a (i INTEGER);", "INSERT INTO a VALUES (42);"], "sql": "SELECT * FROM b : a", "cols": ["i"], "rows": [[42]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": ["CREATE TABLE a (i INTEGER);", "INSERT INTO a VALUES (42);"], "sql": "SELECT * FROM \"b\" : a", "cols": ["i"], "rows": [[42]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": ["CREATE TABLE a (i INTEGER);", "INSERT INTO a VALUES (42);"], "sql": "SELECT i FROM b : a", "cols": ["i"], "rows": [[42]]} +{"source": "test/sql/select/test_select_alias_prefix_colon.test", "setup": ["CREATE TABLE a (i INTEGER);", "INSERT INTO a VALUES (42);"], "sql": "SELECT b.i FROM b : a", "cols": ["i"], "rows": [[42]]} +{"source": "test/sql/select/test_select_empty_table.test", "setup": ["CREATE TABLE integers(i INTEGER)"], "sql": "SELECT * FROM integers", "cols": ["i"], "rows": []} +{"source": "test/sql/join/inner/empty_tinyint_column.test", "setup": ["CREATE TABLE t1(c0 INT4, c1 VARCHAR);", "CREATE TABLE t2(c0 TINYINT, PRIMARY KEY(c0));", "INSERT INTO t1(c0) VALUES (14161972);", "INSERT INTO t1(c0, c1) VALUES (-1.438515327E9, 4.43806148E8);"], "sql": "SELECT * FROM t1 INNER JOIN t2 ON t1.c0 = t2.c0;", "cols": ["c0", "c1", "c0"], "rows": []} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[-128, -128], [127, 127]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[-32768, -32768], [32767, 32767]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[-2147483648, -2147483648], [2147483647, 2147483647]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 BIGINT);", "INSERT INTO t VALUES (-9223372036854775808), (9223372036854775807);", "CREATE TABLE u(u_k0 BIGINT);", "INSERT INTO u VALUES (-9223372036854775808), (9223372036854775807);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[-9223372036854775808, -9223372036854775808], [9223372036854775807, 9223372036854775807]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 BIGINT);", "INSERT INTO t VALUES (-9223372036854775808), (9223372036854775807);", "CREATE TABLE u(u_k0 BIGINT);", "INSERT INTO u VALUES (-9223372036854775808), (9223372036854775807);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 HUGEINT);", "INSERT INTO t VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "CREATE TABLE u(u_k0 HUGEINT);", "INSERT INTO u VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[-170141183460469231731687303715884105728, -170141183460469231731687303715884105728], [170141183460469231731687303715884105727, 170141183460469231731687303715884105727]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 BIGINT);", "INSERT INTO t VALUES (-9223372036854775808), (9223372036854775807);", "CREATE TABLE u(u_k0 BIGINT);", "INSERT INTO u VALUES (-9223372036854775808), (9223372036854775807);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 HUGEINT);", "INSERT INTO t VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "CREATE TABLE u(u_k0 HUGEINT);", "INSERT INTO u VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UTINYINT);", "INSERT INTO t VALUES (0), (255);", "CREATE TABLE u(u_k0 UTINYINT);", "INSERT INTO u VALUES (0), (255);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[0, 0], [255, 255]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 BIGINT);", "INSERT INTO t VALUES (-9223372036854775808), (9223372036854775807);", "CREATE TABLE u(u_k0 BIGINT);", "INSERT INTO u VALUES (-9223372036854775808), (9223372036854775807);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 HUGEINT);", "INSERT INTO t VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "CREATE TABLE u(u_k0 HUGEINT);", "INSERT INTO u VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UTINYINT);", "INSERT INTO t VALUES (0), (255);", "CREATE TABLE u(u_k0 UTINYINT);", "INSERT INTO u VALUES (0), (255);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 USMALLINT);", "INSERT INTO t VALUES (0), (65535);", "CREATE TABLE u(u_k0 USMALLINT);", "INSERT INTO u VALUES (0), (65535);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[0, 0], [65535, 65535]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 BIGINT);", "INSERT INTO t VALUES (-9223372036854775808), (9223372036854775807);", "CREATE TABLE u(u_k0 BIGINT);", "INSERT INTO u VALUES (-9223372036854775808), (9223372036854775807);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 HUGEINT);", "INSERT INTO t VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "CREATE TABLE u(u_k0 HUGEINT);", "INSERT INTO u VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UTINYINT);", "INSERT INTO t VALUES (0), (255);", "CREATE TABLE u(u_k0 UTINYINT);", "INSERT INTO u VALUES (0), (255);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 USMALLINT);", "INSERT INTO t VALUES (0), (65535);", "CREATE TABLE u(u_k0 USMALLINT);", "INSERT INTO u VALUES (0), (65535);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UINTEGER);", "INSERT INTO t VALUES (0), (4294967295);", "CREATE TABLE u(u_k0 UINTEGER);", "INSERT INTO u VALUES (0), (4294967295);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[0, 0], [4294967295, 4294967295]]} +{"source": "test/sql/join/inner/equality_join_limits.test", "setup": ["CREATE TABLE t(t_k0 TINYINT);", "INSERT INTO t VALUES (-128), (127);", "CREATE TABLE u(u_k0 TINYINT);", "INSERT INTO u VALUES (-128), (127);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 SMALLINT);", "INSERT INTO t VALUES (-32768), (32767);", "CREATE TABLE u(u_k0 SMALLINT);", "INSERT INTO u VALUES (-32768), (32767);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 INTEGER);", "INSERT INTO t VALUES (-2147483648), (2147483647);", "CREATE TABLE u(u_k0 INTEGER);", "INSERT INTO u VALUES (-2147483648), (2147483647);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 BIGINT);", "INSERT INTO t VALUES (-9223372036854775808), (9223372036854775807);", "CREATE TABLE u(u_k0 BIGINT);", "INSERT INTO u VALUES (-9223372036854775808), (9223372036854775807);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 HUGEINT);", "INSERT INTO t VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "CREATE TABLE u(u_k0 HUGEINT);", "INSERT INTO u VALUES (-170141183460469231731687303715884105728), (170141183460469231731687303715884105727);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UTINYINT);", "INSERT INTO t VALUES (0), (255);", "CREATE TABLE u(u_k0 UTINYINT);", "INSERT INTO u VALUES (0), (255);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 USMALLINT);", "INSERT INTO t VALUES (0), (65535);", "CREATE TABLE u(u_k0 USMALLINT);", "INSERT INTO u VALUES (0), (65535);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UINTEGER);", "INSERT INTO t VALUES (0), (4294967295);", "CREATE TABLE u(u_k0 UINTEGER);", "INSERT INTO u VALUES (0), (4294967295);", "DROP TABLE t;", "DROP TABLE u;", "CREATE TABLE t(t_k0 UBIGINT);", "INSERT INTO t VALUES (0), (18446744073709551615);", "CREATE TABLE u(u_k0 UBIGINT);", "INSERT INTO u VALUES (0), (18446744073709551615);"], "sql": "SELECT t_k0, u_k0 FROM t, u WHERE t_k0 = u_k0;", "cols": ["t_k0", "u_k0"], "rows": [[0, 0], [18446744073709551615, 18446744073709551615]]} +{"source": "test/sql/join/inner/join_cache.test", "setup": ["CREATE TABLE smalltable AS SELECT 1::INTEGER a;", "CREATE TABLE bigtable AS SELECT a::INTEGER a FROM generate_series(0, 10000, 1) tbl(a), generate_series(0, 9, 1) tbl2(b);"], "sql": "SELECT COUNT(*) FROM bigtable JOIN smalltable USING (a)", "cols": ["count_star()"], "rows": [[10]]} +{"source": "test/sql/join/inner/join_cache.test", "setup": ["CREATE TABLE smalltable AS SELECT 1::INTEGER a;", "CREATE TABLE bigtable AS SELECT a::INTEGER a FROM generate_series(0, 10000, 1) tbl(a), generate_series(0, 9, 1) tbl2(b);"], "sql": "SELECT COUNT(*) FROM bigtable JOIN smalltable USING (a) JOIN smalltable t3 USING (a)", "cols": ["count_star()"], "rows": [[10]]} +{"source": "test/sql/join/inner/join_cache.test", "setup": ["CREATE TABLE smalltable AS SELECT 1::INTEGER a;", "CREATE TABLE bigtable AS SELECT a::INTEGER a FROM generate_series(0, 10000, 1) tbl(a), generate_series(0, 9, 1) tbl2(b);"], "sql": "SELECT COUNT(*) FROM bigtable JOIN smalltable USING (a) JOIN smalltable t3 USING (a) JOIN smalltable t4 USING (a);", "cols": ["count_star()"], "rows": [[10]]} +{"source": "test/sql/join/inner/join_cache.test", "setup": ["CREATE TABLE smalltable AS SELECT 1::INTEGER a;", "CREATE TABLE bigtable AS SELECT a::INTEGER a FROM generate_series(0, 10000, 1) tbl(a), generate_series(0, 9, 1) tbl2(b);"], "sql": "SELECT * FROM bigtable JOIN smalltable USING (a)", "cols": ["a"], "rows": [[1], [1], [1], [1], [1], [1], [1], [1], [1], [1]]} +{"source": "test/sql/join/inner/list_join.test", "setup": ["CREATE TABLE test (id INTEGER, l VARCHAR[]);", "INSERT INTO test SELECT i, case when (i/1000)%2=0 then ARRAY[1::VARCHAR, 1::VARCHAR, 1::VARCHAR] else ARRAY[2::VARCHAR, 2::VARCHAR] end FROM generate_series(0, 1999, 1) tbl(i);"], "sql": "SELECT * FROM test AS t1 LEFT JOIN test AS t2 ON t1.id=t2.id WHERE t1.l!=t2.l or t1.id!=t2.id;", "cols": ["id", "l", "id", "l"], "rows": []} +{"source": "test/sql/join/inner/not_between_is_null.test", "setup": ["CREATE TABLE t1(c0 INT);\nCREATE TABLE t2(c0 INT);", "INSERT INTO t1(c0) VALUES (-18), (NULL);", "INSERT INTO t2(c0) VALUES (NULL);"], "sql": "SELECT * FROM t1 INNER JOIN t2 ON ((t1.c0 NOT BETWEEN t2.c0 AND t2.c0) IS NULL);", "cols": ["c0", "c0"], "rows": [[-18, null], [null, null]]} +{"source": "test/sql/join/inner/test_inner_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 3, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nINNER JOIN right_table r ON l.id = r.id AND l.val > 1 AND r.category = 1;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[3, 2, 60, 3, 1, 1500]]} +{"source": "test/sql/join/inner/test_inner_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 3, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nINNER JOIN right_table r ON l.id = r.id AND r.category = 1;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[1, 1, 50, 1, 1, 1000], [3, 2, 60, 3, 1, 1500]]} +{"source": "test/sql/join/inner/test_inner_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 3, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nINNER JOIN right_table r ON l.id = r.id AND l.val > 1;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[3, 2, 60, 3, 1, 1500]]} +{"source": "test/sql/join/inner/test_inner_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 3, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nINNER JOIN right_table r ON l.id = r.id AND true;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[1, 1, 50, 1, 1, 1000], [2, 1, 75, 2, 2, 2000], [3, 2, 60, 3, 1, 1500]]} +{"source": "test/sql/join/inner/test_inner_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 3, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nINNER JOIN right_table r ON l.id = r.id AND false;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": []} +{"source": "test/sql/join/inner/test_inner_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 3, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nINNER JOIN right_table r ON l.id = r.id AND l.amount + r.budget > 1100;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[2, 1, 75, 2, 2, 2000], [3, 2, 60, 3, 1, 1500]]} +{"source": "test/sql/join/inner/test_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)"], "sql": "SELECT a, test.b, c FROM test, test2 WHERE test.b=test2.b AND test.a-1=test2.c", "cols": ["a", "b", "c"], "rows": [[11, 1, 10]]} +{"source": "test/sql/join/inner/test_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)"], "sql": "SELECT a, test.b, c FROM test INNER JOIN test2 ON test2.b = test.b and test.b = 2;", "cols": ["a", "b", "c"], "rows": [[12, 2, 30]]} +{"source": "test/sql/join/inner/test_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)"], "sql": "SELECT a, test.b, c FROM test INNER JOIN test2 ON NULL = 2;", "cols": ["a", "b", "c"], "rows": []} +{"source": "test/sql/join/inner/test_join_duplicates.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 AS SELECT * FROM repeat(1, 10*1024) t1(b), (SELECT 10) t2(c);"], "sql": "SELECT COUNT(*) FROM test2;", "cols": ["count_star()"], "rows": [[10240]]} +{"source": "test/sql/join/inner/test_join_duplicates.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 AS SELECT * FROM repeat(1, 10*1024) t1(b), (SELECT 10) t2(c);"], "sql": "SELECT COUNT(*) FROM test INNER JOIN test2 ON test.b=test2.b", "cols": ["count_star()"], "rows": [[10240]]} +{"source": "test/sql/join/inner/test_range_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)"], "sql": "SELECT test.b, test2.b FROM test, test2 WHERE test.btest2.c AND test.b <= test2.b", "cols": ["a", "b", "b", "c"], "rows": [[11, 1, 1, 10]]} +{"source": "test/sql/join/inner/test_range_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)", "INSERT INTO test VALUES (11, NULL), (NULL, 1)"], "sql": "SELECT test.a, test.b, test2.b, test2.c FROM test, test2 WHERE test.a>test2.c AND test.b <= test2.b", "cols": ["a", "b", "b", "c"], "rows": [[11, 1, 1, 10]]} +{"source": "test/sql/join/inner/test_range_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)", "INSERT INTO test VALUES (11, NULL), (NULL, 1)", "INSERT INTO test2 VALUES (1, NULL), (NULL, 10)"], "sql": "SELECT test.a, test.b, test2.b, test2.c FROM test, test2 WHERE test.a>test2.c AND test.b <= test2.b", "cols": ["a", "b", "b", "c"], "rows": [[11, 1, 1, 10]]} +{"source": "test/sql/join/inner/test_range_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)", "INSERT INTO test VALUES (11, NULL), (NULL, 1)", "INSERT INTO test2 VALUES (1, NULL), (NULL, 10)", "CREATE TABLE issue4419 (x INT, y VARCHAR);", "INSERT INTO issue4419 VALUES (1, 'sssssssssssssssssueufuheuooefef');", "INSERT INTO issue4419 VALUES (2, 'sssssssssssssssssueufuheuooefesffff');", "INSERT INTO issue4419 VALUES (2, 'sssssssssssssssssueufuheuooefesffffsssssssieiffih');"], "sql": "SELECT * FROM issue4419 t1 INNER JOIN issue4419 t2 ON t1.x < t2.x;", "cols": ["x", "y", "x", "y"], "rows": [[1, "sssssssssssssssssueufuheuooefef", 2, "sssssssssssssssssueufuheuooefesffff"], [1, "sssssssssssssssssueufuheuooefef", 2, "sssssssssssssssssueufuheuooefesffffsssssssieiffih"]]} +{"source": "test/sql/join/inner/test_unequal_join.test", "setup": ["CREATE TABLE test (a INTEGER, b INTEGER);", "INSERT INTO test VALUES (11, 1), (12, 2), (13, 3)", "CREATE TABLE test2 (b INTEGER, c INTEGER);", "INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30)", "INSERT INTO test VALUES (NULL, NULL)", "INSERT INTO test2 VALUES (NULL, NULL)", "create table a (i integer)", "insert into a values ('28579'),('16098'),('25281'),('28877'),('18048'),('26820'),('26971'),('22812'),('11757'),('21851'),('27752'),('28354'),('29843'),('28828'),('16668'),('20534'),('28222'),('24244'),('28877'),('20150'),('23451'),('23683'),('20419'),('28048'),('24244'),('28605'),('25752'),('24466'),('26557'),('16098'),('29454'),('24854'),('13298'),('29584'),('13394'),('24843'),('22477'),('14593'),('24244'),('28722'),('25124'),('16668'),('26787'),('28877'),('27752'),('28482'),('24408'),('25752'),('24136'),('28222'),('17683'),('24244'),('19275'),('21087'),('26594'),('22293'),('25281'),('12898'),('23451'),('12898'),('21757'),('20965'),('25709'),('26614'),('10399'),('28773'),('11933'),('29584'),('29003'),('26871'),('17746'),('24092'),('26192'),('19310'),('10965'),('29275'),('20191'),('29101'),('28059'),('29584'),('20399'),('24338'),('26192'),('25124'),('28605'),('13003'),('16668'),('23511'),('26534'),('24107')", "create table b (j integer)", "insert into b values ('31904'),('31904'),('31904'),('31904'),('35709'),('31904'),('31904'),('35709'),('31904'),('31904'),('31904'),('31904')"], "sql": "select count(*) from a,b where i <> j", "cols": ["count_star()"], "rows": [[1080]]} +{"source": "test/sql/join/inner/test_unequal_join_duplicates.test", "setup": ["CREATE TABLE test (b INTEGER);", "INSERT INTO test VALUES (1), (2)", "CREATE TABLE test2 AS SELECT * FROM repeat(1, 10*1024) t1(b);"], "sql": "SELECT COUNT(*) FROM test2;", "cols": ["count_star()"], "rows": [[10240]]} +{"source": "test/sql/join/inner/test_unequal_join_duplicates.test", "setup": ["CREATE TABLE test (b INTEGER);", "INSERT INTO test VALUES (1), (2)", "CREATE TABLE test2 AS SELECT * FROM repeat(1, 10*1024) t1(b);"], "sql": "SELECT COUNT(*) FROM test INNER JOIN test2 ON test.b<>test2.b", "cols": ["count_star()"], "rows": [[10240]]} +{"source": "test/sql/join/inner/test_using_chain.test", "setup": ["CREATE TABLE t1 (a INTEGER, b INTEGER)", "INSERT INTO t1 VALUES (1, 2)", "CREATE TABLE t2 (b INTEGER, c INTEGER)", "INSERT INTO t2 VALUES (2, 3)", "CREATE TABLE t3 (c INTEGER, d INTEGER)", "INSERT INTO t3 VALUES (3, 4)", "DROP TABLE t1", "DROP TABLE t2", "DROP TABLE t3", "CREATE TABLE t1 (a INTEGER, b INTEGER, c INTEGER)", "INSERT INTO t1 VALUES (1, 2, 2)", "CREATE TABLE t2 (b INTEGER, c INTEGER, d INTEGER, e INTEGER)", "INSERT INTO t2 VALUES (2, 2, 3, 4)", "CREATE TABLE t3 (d INTEGER, e INTEGER)", "INSERT INTO t3 VALUES (3, 4)"], "sql": "SELECT * FROM t1 JOIN t2 USING (b, c) JOIN t3 USING (d, e);", "cols": ["a", "b", "c", "d", "e"], "rows": [[1, 2, 2, 3, 4]]} +{"source": "test/sql/join/inner/test_using_join.test", "setup": ["CREATE TABLE t1 (a INTEGER, b INTEGER, c INTEGER);", "INSERT INTO t1 VALUES (1,2,3);", "CREATE TABLE t2 (a INTEGER, b INTEGER, c INTEGER);", "INSERT INTO t2 VALUES (1,2,3), (2,2,4), (1,3,4);"], "sql": "SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING(a,b)", "cols": ["a", "b", "c"], "rows": [[1, 2, 3]]} +{"source": "test/sql/join/inner/test_using_join.test", "setup": ["CREATE TABLE t1 (a INTEGER, b INTEGER, c INTEGER);", "INSERT INTO t1 VALUES (1,2,3);", "CREATE TABLE t2 (a INTEGER, b INTEGER, c INTEGER);", "INSERT INTO t2 VALUES (1,2,3), (2,2,4), (1,3,4);"], "sql": "SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING(a,b,c)", "cols": ["a", "b", "c"], "rows": [[1, 2, 3]]} +{"source": "test/sql/join/inner/test_using_join.test", "setup": ["CREATE TABLE t1 (a INTEGER, b INTEGER, c INTEGER);", "INSERT INTO t1 VALUES (1,2,3);", "CREATE TABLE t2 (a INTEGER, b INTEGER, c INTEGER);", "INSERT INTO t2 VALUES (1,2,3), (2,2,4), (1,3,4);"], "sql": "SELECT * FROM t1 JOIN t2 USING(a,b)", "cols": ["a", "b", "c", "c"], "rows": [[1, 2, 3, 3]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);"], "sql": "select * from t1 left join t2 on t1.id = t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);"], "sql": "select * from t1 left join t2 on t1.id > t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);"], "sql": "select * from t1 left join t2 on t1.id <> t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);", "insert into t2 values (1);"], "sql": "select * from t1 left join t2 on t1.id = t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);", "insert into t2 values (1);"], "sql": "select * from t1 left join t2 on t1.id > t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);", "insert into t2 values (1);"], "sql": "select * from t1 left join t2 on t1.id <> t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);", "insert into t2 values (1);", "insert into t2 values (NULL), (NULL);"], "sql": "select * from t1 left join t2 on t1.id = t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);", "insert into t2 values (1);", "insert into t2 values (NULL), (NULL);"], "sql": "select * from t1 left join t2 on t1.id > t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/left_join_issue_1172.test", "setup": ["drop table if exists t1;", "drop table if exists t2;", "create table t1 (id string);", "create table t2 (id string);", "insert into t1 values\n(NULL);", "insert into t2 values (1), (1);", "insert into t2 values (1);", "insert into t2 values (NULL), (NULL);"], "sql": "select * from t1 left join t2 on t1.id <> t2.id;", "cols": ["id", "id"], "rows": [[null, null]]} +{"source": "test/sql/join/left_outer/test_left_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 99, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nLEFT JOIN right_table r ON l.id = r.id AND l.val > 1 AND r.category = 1;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[3, 2, 60, 3, 1, 1500], [1, 1, 50, null, null, null], [2, 1, 75, null, null, null], [4, 2, 90, null, null, null], [5, 99, 100, null, null, null]]} +{"source": "test/sql/join/left_outer/test_left_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 99, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nLEFT JOIN right_table r ON l.id = r.id AND r.category = 1;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[1, 1, 50, 1, 1, 1000], [3, 2, 60, 3, 1, 1500], [2, 1, 75, null, null, null], [4, 2, 90, null, null, null], [5, 99, 100, null, null, null]]} +{"source": "test/sql/join/left_outer/test_left_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 99, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nLEFT JOIN right_table r ON l.id = r.id AND l.val > 1;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[3, 2, 60, 3, 1, 1500], [1, 1, 50, null, null, null], [2, 1, 75, null, null, null], [4, 2, 90, null, null, null], [5, 99, 100, null, null, null]]} +{"source": "test/sql/join/left_outer/test_left_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 99, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nLEFT JOIN right_table r ON l.id = r.id AND true;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[1, 1, 50, 1, 1, 1000], [2, 1, 75, 2, 2, 2000], [3, 2, 60, 3, 1, 1500], [4, 2, 90, null, null, null], [5, 99, 100, null, null, null]]} +{"source": "test/sql/join/left_outer/test_left_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 99, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nLEFT JOIN right_table r ON l.id = r.id AND false;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[1, 1, 50, null, null, null], [2, 1, 75, null, null, null], [3, 2, 60, null, null, null], [4, 2, 90, null, null, null], [5, 99, 100, null, null, null]]} +{"source": "test/sql/join/left_outer/test_left_join_filter_pushdown.test", "setup": ["CREATE TABLE left_table (id INTEGER, val INTEGER, amount INTEGER);", "INSERT INTO left_table VALUES (1, 1, 50), (2, 1, 75), (3, 2, 60), (4, 2, 90), (5, 99, 100);", "CREATE TABLE right_table (id INTEGER, category INTEGER, budget INTEGER);", "INSERT INTO right_table VALUES (1, 1, 1000), (2, 2, 2000), (3, 1, 1500);"], "sql": "SELECT * FROM left_table l\nLEFT JOIN right_table r ON l.id = r.id AND l.amount + r.budget > 1100;", "cols": ["id", "val", "amount", "id", "category", "budget"], "rows": [[2, 1, 75, 2, 2, 2000], [3, 2, 60, 3, 1, 1500], [1, 1, 50, null, null, null], [4, 2, 90, null, null, null], [5, 99, 100, null, null, null]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)"], "sql": "SELECT f.fk, f.value, d.label\nFROM fact f LEFT JOIN dim d ON f.fk = d.id", "cols": ["fk", "value", "label"], "rows": [[1, 40, "a"], [2, 50, "b"], [3, 30, "c"], [1, 10, "a"], [2, 20, "b"]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)"], "sql": "SELECT f.fk, f.value, d.label\nFROM fact f LEFT JOIN empty_dim d ON f.fk = d.id", "cols": ["fk", "value", "label"], "rows": [[1, 10, null], [2, 20, null], [3, 30, null], [1, 40, null], [2, 50, null], [99, 999, null]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')"], "sql": "SELECT f.fk, f.value, d.label\nFROM fact f LEFT JOIN dim_unique d ON f.fk = d.id\nWHERE f.fk <= 2", "cols": ["fk", "value", "label"], "rows": [[1, 40, "x"], [2, 50, "y"], [1, 10, "x"], [2, 20, "y"]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)"], "sql": "SELECT f.fa, f.fb, f.value, d.label\nFROM fact_multi f LEFT JOIN dim_multi d ON f.fa = d.a AND f.fb = d.b", "cols": ["fa", "fb", "value", "label"], "rows": [[1, 1, 10, "aa"], [2, 2, 20, "bb"], [3, 3, 30, null]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)"], "sql": "SELECT COUNT(*) FROM big_fact f LEFT JOIN big_dim d ON f.fk = d.id", "cols": ["count_star()"], "rows": [[50000]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)"], "sql": "SELECT COUNT(d.label) FROM big_fact f LEFT JOIN big_dim d ON f.fk = d.id", "cols": ["count(d.\"label\")"], "rows": [[35000]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)"], "sql": "SELECT SUM(f.fk) FROM fact f LEFT JOIN dim d ON f.fk = d.id", "cols": ["sum(f.fk)"], "rows": [[108]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)", "CREATE TABLE dim_wide(id INTEGER PRIMARY KEY, a TEXT, b TEXT, c INTEGER)", "INSERT INTO dim_wide VALUES (1, 'x', 'y', 100), (2, 'p', 'q', 200)"], "sql": "SELECT f.fk, f.value, d.a, d.b, d.c\nFROM fact f LEFT JOIN dim_wide d ON f.fk = d.id\nWHERE f.fk <= 3", "cols": ["fk", "value", "a", "b", "c"], "rows": [[1, 40, "x", "y", 100], [2, 50, "p", "q", 200], [1, 10, "x", "y", 100], [2, 20, "p", "q", 200], [3, 30, null, null, null]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)", "CREATE TABLE dim_wide(id INTEGER PRIMARY KEY, a TEXT, b TEXT, c INTEGER)", "INSERT INTO dim_wide VALUES (1, 'x', 'y', 100), (2, 'p', 'q', 200)", "INSERT INTO fact VALUES (NULL, 777)"], "sql": "SELECT f.fk, f.value, d.label\nFROM fact f LEFT JOIN dim d ON f.fk = d.id\nWHERE f.value = 777", "cols": ["fk", "value", "label"], "rows": [[null, 777, null]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)", "CREATE TABLE dim_wide(id INTEGER PRIMARY KEY, a TEXT, b TEXT, c INTEGER)", "INSERT INTO dim_wide VALUES (1, 'x', 'y', 100), (2, 'p', 'q', 200)", "INSERT INTO fact VALUES (NULL, 777)"], "sql": "SELECT f.fk, f.value, d.label\nFROM fact f FULL OUTER JOIN dim d ON f.fk = d.id\nWHERE d.id > 3 OR f.fk IS NULL", "cols": ["fk", "value", "label"], "rows": [[null, null, "d"], [null, null, "e"], [null, 777, null]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)", "CREATE TABLE dim_wide(id INTEGER PRIMARY KEY, a TEXT, b TEXT, c INTEGER)", "INSERT INTO dim_wide VALUES (1, 'x', 'y', 100), (2, 'p', 'q', 200)", "INSERT INTO fact VALUES (NULL, 777)"], "sql": "SELECT COUNT(*) - COUNT(d.label) FROM big_fact f LEFT JOIN big_dim d ON f.fk = d.id", "cols": ["(count_star() - count(d.\"label\"))"], "rows": [[15000]]} +{"source": "test/sql/join/left_outer/unique_left_join.test", "setup": ["CREATE TABLE dim(id INTEGER PRIMARY KEY, label TEXT)", "INSERT INTO dim VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')", "CREATE TABLE fact(fk INTEGER, value INTEGER)", "INSERT INTO fact VALUES (1, 10), (2, 20), (3, 30), (1, 40), (2, 50)", "INSERT INTO fact VALUES (99, 999)", "CREATE TABLE empty_dim(id INTEGER PRIMARY KEY, label TEXT)", "CREATE TABLE dim_unique(id INTEGER UNIQUE, label TEXT)", "INSERT INTO dim_unique VALUES (1, 'x'), (2, 'y')", "CREATE TABLE dim_multi(a INTEGER, b INTEGER, label TEXT, PRIMARY KEY(a, b))", "INSERT INTO dim_multi VALUES (1, 1, 'aa'), (2, 2, 'bb')", "CREATE TABLE fact_multi(fa INTEGER, fb INTEGER, value INTEGER)", "INSERT INTO fact_multi VALUES (1, 1, 10), (2, 2, 20), (3, 3, 30)", "CREATE TABLE big_dim AS SELECT range::INTEGER AS id, 'val_' || range AS label FROM range(1, 10001)", "CREATE TABLE big_fact AS SELECT (range % 15000)::INTEGER AS fk FROM range(1, 50001)", "CREATE TABLE dim_wide(id INTEGER PRIMARY KEY, a TEXT, b TEXT, c INTEGER)", "INSERT INTO dim_wide VALUES (1, 'x', 'y', 100), (2, 'p', 'q', 200)", "INSERT INTO fact VALUES (NULL, 777)"], "sql": "SELECT SUM(f.fk) FROM big_fact f LEFT JOIN big_dim d ON f.fk = d.id", "cols": ["sum(f.fk)"], "rows": [[349980000]]} +{"source": "test/sql/function/string/hex.test", "setup": [], "sql": "SELECT to_hex(columns('^(.*int|varchar|bignum)$')) FROM test_all_types();", "cols": ["tinyint", "smallint", "int", "bigint", "hugeint", "uhugeint", "utinyint", "usmallint", "uint", "ubigint", "bignum", "varchar"], "rows": [["FFFFFFFFFFFFFF80", "FFFFFFFFFFFF8000", "FFFFFFFF80000000", "8000000000000000", "80000000000000000000000000000000", "0", "0", "0", "0", "0", "7FFF7F00000000000007FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", "F09FA686F09FA686F09FA686F09FA686F09FA686F09FA686"], ["7F", "7FFF", "7FFFFFFF", "7FFFFFFFFFFFFFFF", "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", "FF", "FFFF", "FFFFFFFF", "FFFFFFFFFFFFFFFF", "800080FFFFFFFFFFFFF800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "676F6F007365"], [null, null, null, null, null, null, null, null, null, null, null, null]]} +{"source": "test/sql/function/string/hex.test", "setup": [], "sql": "SELECT to_binary(columns('^(.*int|varchar|bignum)$')) FROM test_all_types();", "cols": ["tinyint", "smallint", "int", "bigint", "hugeint", "uhugeint", "utinyint", "usmallint", "uint", "ubigint", "bignum", "varchar"], "rows": [["1111111111111111111111111111111111111111111111111111111110000000", "1111111111111111111111111111111111111111111111111000000000000000", "1111111111111111111111111111111110000000000000000000000000000000", "1000000000000000000000000000000000000000000000000000000000000000", "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "0", "0", "0", "0", "0", "0111111111111111011111110000000000000000000000000000000000000000000000000000011111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "111100001001111110100110100001101111000010011111101001101000011011110000100111111010011010000110111100001001111110100110100001101111000010011111101001101000011011110000100111111010011010000110"], ["1111111", "111111111111111", "1111111111111111111111111111111", "111111111111111111111111111111111111111111111111111111111111111", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "11111111", "1111111111111111", "11111111111111111111111111111111", "1111111111111111111111111111111111111111111111111111111111111111", "1000000000000000100000001111111111111111111111111111111111111111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "011001110110111101101111000000000111001101100101"], [null, null, null, null, null, null, null, null, null, null, null, null]]} +{"source": "test/sql/function/string/like_unicode.test", "setup": ["CREATE TABLE t0 (c0 VARCHAR);", "INSERT INTO t0 VALUES ('\u7968'),('t'),('%'),('\u4e11'),('\u591a'), ('\ud83e\udd86');"], "sql": "SELECT count(*) FROM t0 WHERE t0.c0 LIKE '_';", "cols": ["count_star()"], "rows": [[6]]} +{"source": "test/sql/function/string/like_unicode.test", "setup": ["CREATE TABLE t0 (c0 VARCHAR);", "INSERT INTO t0 VALUES ('\u7968'),('t'),('%'),('\u4e11'),('\u591a'), ('\ud83e\udd86');"], "sql": "SELECT count(*) FROM t0 WHERE t0.c0 ILIKE '_';", "cols": ["count_star()"], "rows": [[6]]} +{"source": "test/sql/function/string/regex_extract.test", "setup": ["CREATE TABLE test (s VARCHAR, p VARCHAR, i INT)", "INSERT INTO test VALUES\n ('foobarbaz', 'b..', 0),\n ('foobarbaz', 'b..', 1),\n ('foobarbaz', '(b..)(b..)', 0),\n ('foobarbaz', '(b..)(b..)', 1),\n ('foobarbaz', '(b..)(b..)', 2)"], "sql": "SELECT regexp_extract(s, p, 0) FROM test", "cols": ["regexp_extract(s, p, 0)"], "rows": [["bar"], ["bar"], ["barbaz"], ["barbaz"], ["barbaz"]]} +{"source": "test/sql/function/string/regex_extract.test", "setup": ["CREATE TABLE test (s VARCHAR, p VARCHAR, i INT)", "INSERT INTO test VALUES\n ('foobarbaz', 'b..', 0),\n ('foobarbaz', 'b..', 1),\n ('foobarbaz', '(b..)(b..)', 0),\n ('foobarbaz', '(b..)(b..)', 1),\n ('foobarbaz', '(b..)(b..)', 2)"], "sql": "SELECT regexp_extract(s, 'b..', 0) FROM test", "cols": ["regexp_extract(s, 'b..', 0)"], "rows": [["bar"], ["bar"], ["bar"], ["bar"], ["bar"]]} +{"source": "test/sql/function/string/regex_filter_pushdown.test", "setup": ["CREATE TABLE regex(s STRING)", "INSERT INTO regex VALUES ('asdf'), ('xxxx'), ('aaaa')"], "sql": "SELECT s FROM regex WHERE REGEXP_MATCHES(s, 'as(c|d|e)f')", "cols": ["s"], "rows": [["asdf"]]} +{"source": "test/sql/function/string/regex_filter_pushdown.test", "setup": ["CREATE TABLE regex(s STRING)", "INSERT INTO regex VALUES ('asdf'), ('xxxx'), ('aaaa')"], "sql": "SELECT s FROM regex WHERE NOT REGEXP_MATCHES(s, 'as(c|d|e)f')", "cols": ["s"], "rows": [["xxxx"], ["aaaa"]]} +{"source": "test/sql/function/string/regex_filter_pushdown.test", "setup": ["CREATE TABLE regex(s STRING)", "INSERT INTO regex VALUES ('asdf'), ('xxxx'), ('aaaa')"], "sql": "SELECT s FROM regex WHERE REGEXP_MATCHES(s, 'as(c|d|e)f') AND s = 'asdf'", "cols": ["s"], "rows": [["asdf"]]} +{"source": "test/sql/function/string/regex_filter_pushdown.test", "setup": ["CREATE TABLE regex(s STRING)", "INSERT INTO regex VALUES ('asdf'), ('xxxx'), ('aaaa')"], "sql": "SELECT s FROM regex WHERE REGEXP_MATCHES(s, 'as(c|d|e)f') AND REGEXP_MATCHES(s, 'as[a-z]f')", "cols": ["s"], "rows": [["asdf"]]} +{"source": "test/sql/function/string/regex_operators.test", "setup": ["CREATE TABLE regex_columns_partial(abc INTEGER, bcd INTEGER, def INTEGER);", "INSERT INTO regex_columns_partial VALUES (1, 2, 3);"], "sql": "SELECT * ~ 'b' FROM regex_columns_partial", "cols": ["abc", "bcd"], "rows": [[1, 2]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, NULL) from t0", "cols": ["regexp_matches(c0, NULL)"], "rows": [[null]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, '.*sd.*') from t0;", "cols": ["regexp_matches(c0, '.*sd.*')"], "rows": [[true]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, '.*yu.*') from t0;", "cols": ["regexp_matches(c0, '.*yu.*')"], "rows": [[false]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, '') from t0;", "cols": ["regexp_matches(c0, '')"], "rows": [[true]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, 'sd') from t0;", "cols": ["regexp_matches(c0, 'sd')"], "rows": [[true]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_full_match(c0, 'sd') from t0;", "cols": ["regexp_full_match(c0, 'sd')"], "rows": [[false]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_full_match(c0, '.sd.') from t0;", "cols": ["regexp_full_match(c0, '.sd.')"], "rows": [[true]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, '^sdf$') from t0;", "cols": ["regexp_matches(c0, '^sdf$')"], "rows": [[false]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);"], "sql": "SELECT regexp_matches(c0, CAST(NULL AS STRING)) from t0;", "cols": ["regexp_matches(c0, CAST(NULL AS VARCHAR))"], "rows": [[null]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);", "CREATE TABLE regex(s STRING, p STRING)", "INSERT INTO regex VALUES ('asdf', 'sd'), ('asdf', '^sd'), (NULL, '^sd'), ('asdf', NULL)"], "sql": "SELECT regexp_matches(s, '.*') FROM regex", "cols": ["regexp_matches(s, '.*')"], "rows": [[true], [true], [null], [true]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);", "CREATE TABLE regex(s STRING, p STRING)", "INSERT INTO regex VALUES ('asdf', 'sd'), ('asdf', '^sd'), (NULL, '^sd'), ('asdf', NULL)"], "sql": "SELECT regexp_matches(s, p) FROM regex", "cols": ["regexp_matches(s, p)"], "rows": [[true], [false], [null], [null]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);", "CREATE TABLE regex(s STRING, p STRING)", "INSERT INTO regex VALUES ('asdf', 'sd'), ('asdf', '^sd'), (NULL, '^sd'), ('asdf', NULL)"], "sql": "SELECT regexp_matches(c0, '.*SD.*', 'i') from t0;", "cols": ["regexp_matches(c0, '.*SD.*', 'i')"], "rows": [[true]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);", "CREATE TABLE regex(s STRING, p STRING)", "INSERT INTO regex VALUES ('asdf', 'sd'), ('asdf', '^sd'), (NULL, '^sd'), ('asdf', NULL)"], "sql": "SELECT regexp_matches(c0, '.*SD.*', 'c') from t0;", "cols": ["regexp_matches(c0, '.*SD.*', 'c')"], "rows": [[false]]} +{"source": "test/sql/function/string/regex_search.test", "setup": ["CREATE TABLE t0 as FROM VALUES('asdf') t(c0);", "CREATE TABLE regex(s STRING, p STRING)", "INSERT INTO regex VALUES ('asdf', 'sd'), ('asdf', '^sd'), (NULL, '^sd'), ('asdf', NULL)"], "sql": "SELECT regexp_matches(c0, '.*SD.*', ' i \t') from t0;", "cols": ["regexp_matches(c0, '.*SD.*', ' i \t')"], "rows": [[true]]} +{"source": "test/sql/function/string/regexp_unicode_literal.test", "setup": ["CREATE TABLE data(wsc INT, zipcode VARCHAR)", "INSERT INTO data VALUES (32, '00' || chr(32) || '001'), (160, '00' || chr(160) || '001'), (0, '00\ud83e\udd86001');"], "sql": "select *\nfrom data\nwhere regexp_matches(zipcode, '^00\\x{00A0}001$')\nand regexp_matches(zipcode, '^00\\x{0020}001$')", "cols": ["wsc", "zipcode"], "rows": []} +{"source": "test/sql/function/string/strip_accents.test", "setup": ["CREATE TABLE collate_test(s VARCHAR, str VARCHAR)", "INSERT INTO collate_test VALUES ('\u00e4\u00e4\u00e4', 'aaa')", "INSERT INTO collate_test VALUES ('h\u00e4nn\u00ebs m\u00fchl\u00eb\u00efs\u00ebn', 'hannes muhleisen')", "INSERT INTO collate_test VALUES ('ol\u00e1', 'ola')", "INSERT INTO collate_test VALUES ('\u00f4\u00e2\u00ea\u00f3\u00e1\u00eb\u00f2\u00f5\u00e7', 'oaeoaeooc')"], "sql": "SELECT strip_accents(s)=strip_accents(str) FROM collate_test", "cols": ["(strip_accents(s) = strip_accents(str))"], "rows": [[true], [true], [true], [true]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, 2) FROM strings", "cols": ["array_extract(s, 2)"], "rows": [["e"], ["o"], [""], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, 3) FROM strings", "cols": ["array_extract(s, 3)"], "rows": [["l"], ["r"], [""], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, off) FROM strings", "cols": ["array_extract(s, \"off\")"], "rows": [["h"], ["o"], ["b"], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, 2) FROM strings", "cols": ["array_extract(s, 2)"], "rows": [["e"], ["o"], [""], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract('hello', off) FROM strings", "cols": ["array_extract('hello', \"off\")"], "rows": [["h"], ["e"], ["h"], ["e"]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(NULL::VARCHAR, off) FROM strings", "cols": ["array_extract(CAST(NULL AS VARCHAR), \"off\")"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract('hello', NULL) FROM strings", "cols": ["array_extract('hello', NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(NULL::VARCHAR, NULL) FROM strings", "cols": ["array_extract(CAST(NULL AS VARCHAR), NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(NULL::VARCHAR, off) FROM strings", "cols": ["array_extract(CAST(NULL AS VARCHAR), \"off\")"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(NULL::VARCHAR, NULL) FROM strings", "cols": ["array_extract(CAST(NULL AS VARCHAR), NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, -1) FROM strings", "cols": ["array_extract(s, -1)"], "rows": [["o"], ["d"], ["b"], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, 1) FROM strings", "cols": ["array_extract(s, 1)"], "rows": [["h"], ["w"], ["b"], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, 6) FROM strings", "cols": ["array_extract(s, 6)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, 2147483646) FROM strings", "cols": ["array_extract(s, 2147483646)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_array_extract.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT array_extract(s, -2147483647) FROM strings", "cols": ["array_extract(s, -2147483647)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_bar.test", "setup": [], "sql": "select bar(x * x, 0, 100) from range(0, 11) t(x)", "cols": ["bar((x * x), 0, 100)"], "rows": [[" "], ["\u258a "], ["\u2588\u2588\u2588\u258f "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258f "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258a "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258a "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258f "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258f "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258a "], ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588"]]} +{"source": "test/sql/function/string/test_bit_length.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Zero'), ('$', NULL), ('\u00a2','Two'), ('\u20ac', NULL), ('\ud800\udf48','Four')"], "sql": "select BIT_LENGTH(a) FROM strings", "cols": ["bit_length(a)"], "rows": [[0], [8], [16], [24], [32]]} +{"source": "test/sql/function/string/test_bit_length.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Zero'), ('$', NULL), ('\u00a2','Two'), ('\u20ac', NULL), ('\ud800\udf48','Four')"], "sql": "select BIT_LENGTH(b) FROM strings", "cols": ["bit_length(b)"], "rows": [[32], [null], [24], [null], [32]]} +{"source": "test/sql/function/string/test_bit_length.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Zero'), ('$', NULL), ('\u00a2','Two'), ('\u20ac', NULL), ('\ud800\udf48','Four')"], "sql": "select BIT_LENGTH(a) FROM strings WHERE b IS NOT NULL", "cols": ["bit_length(a)"], "rows": [[0], [16], [32]]} +{"source": "test/sql/function/string/test_caseconvert.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select UPPER(a), UCASE(a) FROM strings", "cols": ["upper(a)", "ucase(a)"], "rows": [["HELLO", "HELLO"], ["HULLD", "HULLD"], ["MOT\u00d6RHEAD", "MOT\u00d6RHEAD"]]} +{"source": "test/sql/function/string/test_caseconvert.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select LOWER(a), LCASE(a) FROM strings", "cols": ["lower(a)", "lcase(a)"], "rows": [["hello", "hello"], ["hulld", "hulld"], ["mot\u00f6rhead", "mot\u00f6rhead"]]} +{"source": "test/sql/function/string/test_caseconvert.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select LOWER(b), LCASE(b) FROM strings", "cols": ["lower(b)", "lcase(b)"], "rows": [["world", "world"], [null, null], ["r\u00e4cks", "r\u00e4cks"]]} +{"source": "test/sql/function/string/test_caseconvert.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select UPPER(a), LOWER(a), UCASE(a), LCASE(a) FROM strings WHERE b IS NOT NULL", "cols": ["upper(a)", "lower(a)", "ucase(a)", "lcase(a)"], "rows": [["HELLO", "hello", "HELLO", "hello"], ["MOT\u00d6RHEAD", "mot\u00f6rhead", "MOT\u00d6RHEAD", "mot\u00f6rhead"]]} +{"source": "test/sql/function/string/test_concat_function.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT(a, 'SUFFIX') FROM strings", "cols": ["concat(a, 'SUFFIX')"], "rows": [["HelloSUFFIX"], ["HuLlDSUFFIX"], ["Mot\u00f6rHeadSUFFIX"]]} +{"source": "test/sql/function/string/test_concat_function.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT('PREFIX', b) FROM strings", "cols": ["concat('PREFIX', b)"], "rows": [["PREFIXWorld"], ["PREFIX"], ["PREFIXR\u00c4cks"]]} +{"source": "test/sql/function/string/test_concat_function.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT(a, b) FROM strings", "cols": ["concat(a, b)"], "rows": [["HelloWorld"], ["HuLlD"], ["Mot\u00f6rHeadR\u00c4cks"]]} +{"source": "test/sql/function/string/test_concat_function.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT(a, b, 'SUFFIX') FROM strings", "cols": ["concat(a, b, 'SUFFIX')"], "rows": [["HelloWorldSUFFIX"], ["HuLlDSUFFIX"], ["Mot\u00f6rHeadR\u00c4cksSUFFIX"]]} +{"source": "test/sql/function/string/test_concat_function.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT(a, b, a) FROM strings", "cols": ["concat(a, b, a)"], "rows": [["HelloWorldHello"], ["HuLlDHuLlD"], ["Mot\u00f6rHeadR\u00c4cksMot\u00f6rHead"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS(',',a, 'SUFFIX') FROM strings", "cols": ["concat_ws(',', a, 'SUFFIX')"], "rows": [["Hello,SUFFIX"], ["HuLlD,SUFFIX"], ["Mot\u00f6rHead,SUFFIX"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS('@','PREFIX', b) FROM strings", "cols": ["concat_ws('@', 'PREFIX', b)"], "rows": [["PREFIX@World"], ["PREFIX"], ["PREFIX@R\u00c4cks"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS('$',a, b) FROM strings", "cols": ["concat_ws('$', a, b)"], "rows": [["Hello$World"], ["HuLlD"], ["Mot\u00f6rHead$R\u00c4cks"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS(a, b, 'SUFFIX') FROM strings", "cols": ["concat_ws(a, b, 'SUFFIX')"], "rows": [["WorldHelloSUFFIX"], ["SUFFIX"], ["R\u00c4cksMot\u00f6rHeadSUFFIX"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS(a, b, b) FROM strings", "cols": ["concat_ws(a, b, b)"], "rows": [["WorldHelloWorld"], [""], ["R\u00c4cksMot\u00f6rHeadR\u00c4cks"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS(',', a, 'SUFFIX') FROM strings WHERE a != 'Hello'", "cols": ["concat_ws(',', a, 'SUFFIX')"], "rows": [["HuLlD,SUFFIX"], ["Mot\u00f6rHead,SUFFIX"]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS(NULL, b, 'SUFFIX') FROM strings", "cols": ["concat_ws(NULL, b, 'SUFFIX')"], "rows": [[null], [null], [null]]} +{"source": "test/sql/function/string/test_concat_ws.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select CONCAT_WS(',', NULL, 'SUFFIX') FROM strings", "cols": ["concat_ws(',', NULL, 'SUFFIX')"], "rows": [["SUFFIX"], ["SUFFIX"], ["SUFFIX"]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'h') FROM strings", "cols": ["contains(s, 'h')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'e') FROM strings", "cols": ["contains(s, 'e')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'d') FROM strings", "cols": ["contains(s, 'd')"], "rows": [[false], [true], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'he') FROM strings", "cols": ["contains(s, 'he')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'ello') FROM strings", "cols": ["contains(s, 'ello')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'lo') FROM strings", "cols": ["contains(s, 'lo')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'he-man') FROM strings", "cols": ["contains(s, 'he-man')"], "rows": [[false], [false], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'o') FROM strings", "cols": ["contains(s, 'o')"], "rows": [[true], [true], [false], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(NULL,'o') FROM strings", "cols": ["contains(NULL, 'o')"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,NULL) FROM strings", "cols": ["contains(s, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_contains.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT contains(s,'') FROM strings", "cols": ["contains(s, '')"], "rows": [[true], [true], [true], [null]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'\u00e1') FROM strings", "cols": ["contains(s, '\u00e1')"], "rows": [[true], [true], [false], [false]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'ol\u00e1 mundo') FROM strings", "cols": ["contains(s, 'ol\u00e1 mundo')"], "rows": [[false], [true], [false], [false]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'\u4f60\u597d\u4e16\u754c') FROM strings", "cols": ["contains(s, '\u4f60\u597d\u4e16\u754c')"], "rows": [[false], [false], [true], [false]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'two \u00f1 thr') FROM strings", "cols": ["contains(s, 'two \u00f1 thr')"], "rows": [[false], [false], [false], [true]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'\u00f1') FROM strings", "cols": ["contains(s, '\u00f1')"], "rows": [[false], [false], [false], [true]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'\u20a1 four \ud83e\udd86 e') FROM strings", "cols": ["contains(s, '\u20a1 four \ud83e\udd86 e')"], "rows": [[false], [false], [false], [true]]} +{"source": "test/sql/function/string/test_contains_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT contains(s,'\ud83e\udd86 end') FROM strings", "cols": ["contains(s, '\ud83e\udd86 end')"], "rows": [[false], [false], [false], [true]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')"], "sql": "SELECT damerau_levenshtein(NULL, s) FROM strings", "cols": ["damerau_levenshtein(NULL, s)"], "rows": [[null], [null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')"], "sql": "SELECT damerau_levenshtein(NULL, s) FROM strings", "cols": ["damerau_levenshtein(NULL, s)"], "rows": [[null], [null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein(s, NULL) from strings", "cols": ["damerau_levenshtein(s, NULL)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein(NULL, s) from strings", "cols": ["damerau_levenshtein(NULL, s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein('test', s) from strings", "cols": ["damerau_levenshtein('test', s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein(s, 'test') from strings", "cols": ["damerau_levenshtein(s, 'test')"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein('null', s) from strings", "cols": ["damerau_levenshtein('null', s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein('', s) FROM strings", "cols": ["damerau_levenshtein('', s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT damerau_levenshtein(s, '') FROM strings", "cols": ["damerau_levenshtein(s, '')"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein(NULL, s) FROM strings", "cols": ["damerau_levenshtein(NULL, s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein(s, NULL) FROM strings", "cols": ["damerau_levenshtein(s, NULL)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein(s, '') FROM strings", "cols": ["damerau_levenshtein(s, '')"], "rows": [[0]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein('', s) FROM strings", "cols": ["damerau_levenshtein('', s)"], "rows": [[0]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein(s, 'test') FROM strings", "cols": ["damerau_levenshtein(s, 'test')"], "rows": [[4]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein('test', s) FROM strings", "cols": ["damerau_levenshtein('test', s)"], "rows": [[4]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT damerau_levenshtein('null', s) FROM strings", "cols": ["damerau_levenshtein('null', s)"], "rows": [[4]]} +{"source": "test/sql/function/string/test_damerau_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('here'), ('heres'), ('there'), ('three'), ('threes')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')", "DROP TABLE strings", "CREATE TABLE strings(s_left VARCHAR, s_right VARCHAR)", "INSERT INTO strings VALUES \t('identical', 'identical'), ('dientical', 'identical'),\n\t\t\t\t\t\t\t('dinetcila', 'identical'), ('abcdefghijk', 'bacdfzzeghki'),\n\t\t\t\t\t\t\t('abcd', 'bcda'), ('great', 'greta'),\n\t\t\t\t\t\t\t('abcdefghijklmnopqrstuvwxyz', 'abdcpoxwz'),\n\t\t\t\t\t\t\t('a_considerably_longer_string', 'a_ocnsiderably_longre_tsrig'),\n\t\t\t\t\t\t\t('another-quite-long-string', 'naothre-quit-elongstrnig'),\n\t\t\t\t\t\t\t('littlehampton', 'littlerhamptoner'),\n\t\t\t\t\t\t\t('an_incredibly_long_string_to_compare', 'na_incerdibl_ylong_sr56ting_ot_ocmrpe'),\n\t\t\t\t\t\t\t('smaller', 'notsmaller,longer'),\n\t\t\t\t\t\t\t('againalongerstring', 'string'),\n\t\t\t\t\t\t\t(NULL, NULL), ('', ''),\n\t\t\t\t\t\t\t(NULL, 'test'), ('test', NULL),\n\t\t\t\t\t\t\t('four', ''), ('', 'four'),\n\t\t\t\t\t\t\t(NULL, ''), ('', NULL)"], "sql": "SELECT damerau_levenshtein(s_left, s_right) FROM strings", "cols": ["damerau_levenshtein(s_left, s_right)"], "rows": [[0], [1], [4], [6], [2], [1], [20], [4], [5], [3], [10], [10], [12], [null], [0], [null], [null], [4], [4], [null], [null]]} +{"source": "test/sql/function/string/test_glob.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab*'), ('aaa', 'a?a'), ('aaa', '*b*')"], "sql": "SELECT s FROM strings WHERE s GLOB 'ab*'", "cols": ["s"], "rows": [["abab"]]} +{"source": "test/sql/function/string/test_glob.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab*'), ('aaa', 'a?a'), ('aaa', '*b*')"], "sql": "SELECT s FROM strings WHERE 'aba' GLOB pat", "cols": ["s"], "rows": [["abab"], ["aaa"], ["aaa"]]} +{"source": "test/sql/function/string/test_glob.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab*'), ('aaa', 'a?a'), ('aaa', '*b*')"], "sql": "SELECT s FROM strings WHERE s GLOB pat", "cols": ["s"], "rows": [["abab"], ["aaa"]]} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE s LIKE 'ab%'", "cols": ["s"], "rows": [["abab"]]} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE 'aba' ILIKE pat", "cols": ["s"], "rows": [["abab"], ["aaa"], ["aaa"]]} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE 'aba' NOT ILIKE pat", "cols": ["s"], "rows": []} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE s ILIKE pat", "cols": ["s"], "rows": [["abab"], ["aaa"]]} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE s NOT ILIKE pat", "cols": ["s"], "rows": [["aaa"]]} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')", "CREATE TABLE unicode_strings(s STRING, pat STRING);", "INSERT INTO unicode_strings VALUES ('\u00f6\u00e4b', '\u00d6%B'), ('aa\u00c4', 'A_\u00e4'), ('aaa', '%b%')"], "sql": "SELECT s FROM unicode_strings WHERE s ILIKE pat", "cols": ["s"], "rows": [["\u00f6\u00e4b"], ["aa\u00c4"]]} +{"source": "test/sql/function/string/test_ilike.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'Ab%'), ('aaa', 'A_a'), ('aaa', '%b%')", "CREATE TABLE unicode_strings(s STRING, pat STRING);", "INSERT INTO unicode_strings VALUES ('\u00f6\u00e4b', '\u00d6%B'), ('aa\u00c4', 'A_\u00e4'), ('aaa', '%b%')"], "sql": "SELECT s FROM unicode_strings WHERE s NOT ILIKE pat", "cols": ["s"], "rows": [["aaa"]]} +{"source": "test/sql/function/string/test_ilike_embedded_null.test", "setup": ["CREATE TABLE t(a VARCHAR, b VARCHAR);", "INSERT INTO t VALUES ('goo'||chr(0)||'se', 'goo'||chr(0)||'se'), ('\ud83e\udd86','\ud83e\udd86');"], "sql": "SELECT a ILIKE b FROM t", "cols": ["(a ~~* b)"], "rows": [[false], [true]]} +{"source": "test/sql/function/string/test_ilike_embedded_null.test", "setup": ["CREATE TABLE t(a VARCHAR, b VARCHAR);", "INSERT INTO t VALUES ('goo'||chr(0)||'se', 'goo'||chr(0)||'se'), ('\ud83e\udd86','\ud83e\udd86');", "CREATE TABLE ascii_only(a VARCHAR, b VARCHAR);", "INSERT INTO ascii_only VALUES ('goo' || chr(0) || 'se', 'goo' || chr(0) || 'se');"], "sql": "SELECT a ILIKE b FROM ascii_only", "cols": ["(a ~~* b)"], "rows": [[true]]} +{"source": "test/sql/function/string/test_ilike_escape.test", "setup": ["CREATE TABLE tbl(str VARCHAR, pat VARCHAR);", "INSERT INTO tbl VALUES ('a%c', 'a$%C');"], "sql": "SELECT str ILIKE pat ESCAPE '$' FROM tbl", "cols": ["main.ilike_escape(str, pat, '$')"], "rows": [[true]]} +{"source": "test/sql/function/string/test_ilike_escape.test", "setup": ["CREATE TABLE tbl(str VARCHAR, pat VARCHAR);", "INSERT INTO tbl VALUES ('a%c', 'a$%C');"], "sql": "SELECT str NOT ILIKE pat ESCAPE '$' FROM tbl", "cols": ["main.not_ilike_escape(str, pat, '$')"], "rows": [[false]]} +{"source": "test/sql/function/string/test_ilike_escape.test", "setup": ["CREATE TABLE tbl(str VARCHAR, pat VARCHAR);", "INSERT INTO tbl VALUES ('a%c', 'a$%C');"], "sql": "SELECT NULL ILIKE pat ESCAPE '$' FROM tbl", "cols": ["main.ilike_escape(NULL, pat, '$')"], "rows": [[null]]} +{"source": "test/sql/function/string/test_ilike_escape.test", "setup": ["CREATE TABLE tbl(str VARCHAR, pat VARCHAR);", "INSERT INTO tbl VALUES ('a%c', 'a$%C');"], "sql": "SELECT str ILIKE NULL ESCAPE '$' FROM tbl", "cols": ["main.ilike_escape(str, NULL, '$')"], "rows": [[null]]} +{"source": "test/sql/function/string/test_ilike_escape.test", "setup": ["CREATE TABLE tbl(str VARCHAR, pat VARCHAR);", "INSERT INTO tbl VALUES ('a%c', 'a$%C');"], "sql": "SELECT str ILIKE pat ESCAPE NULL FROM tbl", "cols": ["main.ilike_escape(str, pat, NULL)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'h') FROM strings", "cols": ["instr(s, 'h')"], "rows": [[1], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT position('h' in s) FROM strings", "cols": ["main.\"position\"(s, 'h')"], "rows": [[1], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'e') FROM strings", "cols": ["instr(s, 'e')"], "rows": [[2], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'d') FROM strings", "cols": ["instr(s, 'd')"], "rows": [[0], [5], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'he') FROM strings", "cols": ["instr(s, 'he')"], "rows": [[1], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT position('he' in s) FROM strings", "cols": ["main.\"position\"(s, 'he')"], "rows": [[1], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'ello') FROM strings", "cols": ["instr(s, 'ello')"], "rows": [[2], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'lo') FROM strings", "cols": ["instr(s, 'lo')"], "rows": [[4], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'he-man') FROM strings", "cols": ["instr(s, 'he-man')"], "rows": [[0], [0], [0], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,'o'),s FROM strings", "cols": ["instr(s, 'o')", "s"], "rows": [[5, "hello"], [2, "world"], [0, "b"], [null, null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(NULL,'o') FROM strings", "cols": ["instr(NULL, 'o')"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(s,NULL) FROM strings", "cols": ["instr(s, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_instr.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('b', 1, 1), (NULL, 2, 2)"], "sql": "SELECT instr(NULL,NULL) FROM strings", "cols": ["instr(NULL, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT INSTR(s,'\u00e1') FROM strings", "cols": ["instr(s, '\u00e1')"], "rows": [[1], [3], [0], [0]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT POSITION('\u00e1' in s) FROM strings", "cols": ["main.\"position\"(s, '\u00e1')"], "rows": [[1], [3], [0], [0]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT INSTR(s,'ol\u00e1 mundo') FROM strings", "cols": ["instr(s, 'ol\u00e1 mundo')"], "rows": [[0], [1], [0], [0]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT INSTR(s,'\u4f60\u597d\u4e16\u754c') FROM strings", "cols": ["instr(s, '\u4f60\u597d\u4e16\u754c')"], "rows": [[0], [0], [1], [0]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT instr(s,'two \u00f1 thr') FROM strings", "cols": ["instr(s, 'two \u00f1 thr')"], "rows": [[0], [0], [0], [1]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT instr(s,'\u00f1') FROM strings", "cols": ["instr(s, '\u00f1')"], "rows": [[0], [0], [0], [5]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT instr(s,'\u20a1 four \ud83e\udd86 e') FROM strings", "cols": ["instr(s, '\u20a1 four \ud83e\udd86 e')"], "rows": [[0], [0], [0], [13]]} +{"source": "test/sql/function/string/test_instr_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT instr(s,'\ud83e\udd86 end') FROM strings", "cols": ["instr(s, '\ud83e\udd86 end')"], "rows": [[0], [0], [0], [20]]} +{"source": "test/sql/function/string/test_issue_1812.test", "setup": ["CREATE TABLE t (str VARCHAR);", "INSERT INTO t VALUES ('hello1'), ('hello2'), ('hello3'), ('world1'), ('world2'), ('world3');"], "sql": "SELECT COUNT(*) FROM t WHERE str LIKE '%o%' OR (str LIKE '%o%' AND str LIKE '%rld%');", "cols": ["count_star()"], "rows": [[6]]} +{"source": "test/sql/function/string/test_issue_1812.test", "setup": ["CREATE TABLE t (str VARCHAR);", "INSERT INTO t VALUES ('hello1'), ('hello2'), ('hello3'), ('world1'), ('world2'), ('world3');"], "sql": "SELECT COUNT(*) FROM t\nWHERE (str LIKE '%o%' AND str LIKE '%rld%')\n OR str LIKE '%o%';", "cols": ["count_star()"], "rows": [[6]]} +{"source": "test/sql/function/string/test_issue_1812.test", "setup": ["CREATE TABLE t (str VARCHAR);", "INSERT INTO t VALUES ('hello1'), ('hello2'), ('hello3'), ('world1'), ('world2'), ('world3');"], "sql": "SELECT COUNT(*) FROM t\nWHERE (str LIKE '%o%' AND str LIKE '%rld%')\n OR (str LIKE '%o%')\n OR (str LIKE '%o%');", "cols": ["count_star()"], "rows": [[6]]} +{"source": "test/sql/function/string/test_issue_1812.test", "setup": ["CREATE TABLE t (str VARCHAR);", "INSERT INTO t VALUES ('hello1'), ('hello2'), ('hello3'), ('world1'), ('world2'), ('world3');"], "sql": "SELECT COUNT(*) FROM t\nWHERE (str LIKE '%o%' AND str LIKE '%rld%')\n OR (str LIKE '%o%')\n OR (str LIKE '%o%' AND str LIKE 'blabla%');", "cols": ["count_star()"], "rows": [[6]]} +{"source": "test/sql/function/string/test_issue_1812.test", "setup": ["CREATE TABLE t (str VARCHAR);", "INSERT INTO t VALUES ('hello1'), ('hello2'), ('hello3'), ('world1'), ('world2'), ('world3');"], "sql": "SELECT COUNT(*) FROM t\nWHERE (str LIKE '%o%' AND str LIKE '%1%')\n OR (str LIKE '%o%' AND str LIKE '%1%' AND str LIKE 'blabla%')\n OR (str LIKE '%o%' AND str LIKE '%1%' AND str LIKE 'blabla2%')", "cols": ["count_star()"], "rows": [[2]]} +{"source": "test/sql/function/string/test_jaccard.test", "setup": ["CREATE TABLE strings(s VARCHAR, t VARCHAR)", "INSERT INTO strings VALUES ('hello', 'hallo'), ('aloha', 'fello'), ('fellow', 'ducks'), (NULL, NULL)"], "sql": "select round(jaccard(s, t), 1) from strings", "cols": ["round(jaccard(s, t), 1)"], "rows": [[0.6], [0.3], [0.0], [null]]} +{"source": "test/sql/function/string/test_jaccard.test", "setup": ["CREATE TABLE strings(s VARCHAR, t VARCHAR)", "INSERT INTO strings VALUES ('hello', 'hallo'), ('aloha', 'fello'), ('fellow', 'ducks'), (NULL, NULL)"], "sql": "select round(jaccard(s, 'hallo'), 1) from strings", "cols": ["round(jaccard(s, 'hallo'), 1)"], "rows": [[0.6], [1.0], [0.3], [null]]} +{"source": "test/sql/function/string/test_jaccard.test", "setup": ["CREATE TABLE strings(s VARCHAR, t VARCHAR)", "INSERT INTO strings VALUES ('hello', 'hallo'), ('aloha', 'fello'), ('fellow', 'ducks'), (NULL, NULL)"], "sql": "select round(jaccard('hallo', t), 1) from strings", "cols": ["round(jaccard('hallo', t), 1)"], "rows": [[1.0], [0.3], [0.0], [null]]} +{"source": "test/sql/function/string/test_jaccard.test", "setup": ["CREATE TABLE strings(s VARCHAR, t VARCHAR)", "INSERT INTO strings VALUES ('hello', 'hallo'), ('aloha', 'fello'), ('fellow', 'ducks'), (NULL, NULL)"], "sql": "select round(jaccard(NULL, t), 1) from strings", "cols": ["round(jaccard(NULL, t), 1)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_jaccard.test", "setup": ["CREATE TABLE strings(s VARCHAR, t VARCHAR)", "INSERT INTO strings VALUES ('hello', 'hallo'), ('aloha', 'fello'), ('fellow', 'ducks'), (NULL, NULL)"], "sql": "select round(jaccard(s, NULL), 1) from strings", "cols": ["round(jaccard(s, NULL), 1)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')"], "sql": "SELECT levenshtein(NULL, s) FROM strings", "cols": ["levenshtein(NULL, s)"], "rows": [[null], [null], [null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')"], "sql": "SELECT levenshtein(NULL, s) FROM strings", "cols": ["levenshtein(NULL, s)"], "rows": [[null], [null], [null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT levenshtein(s, NULL) from strings", "cols": ["levenshtein(s, NULL)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT levenshtein(NULL, s) from strings", "cols": ["levenshtein(NULL, s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT levenshtein('hi', s) from strings", "cols": ["levenshtein('hi', s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT levenshtein(s, 'hi') from strings", "cols": ["levenshtein(s, 'hi')"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT levenshtein('', s) FROM strings", "cols": ["levenshtein('', s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)"], "sql": "SELECT levenshtein(s, '') FROM strings", "cols": ["levenshtein(s, '')"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT levenshtein(NULL, s) FROM strings", "cols": ["levenshtein(NULL, s)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT levenshtein(s, NULL) FROM strings", "cols": ["levenshtein(s, NULL)"], "rows": [[null]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT levenshtein(s, '') FROM strings", "cols": ["levenshtein(s, '')"], "rows": [[0]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT levenshtein('', s) FROM strings", "cols": ["levenshtein('', s)"], "rows": [[0]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT levenshtein(s, 'hi') FROM strings", "cols": ["levenshtein(s, 'hi')"], "rows": [[2]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT levenshtein('hi', s) FROM strings", "cols": ["levenshtein('hi', s)"], "rows": [[2]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')"], "sql": "SELECT editdist3(s, 'hello') FROM strings", "cols": ["editdist3(s, 'hello')"], "rows": [[5]]} +{"source": "test/sql/function/string/test_levenshtein.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('fello'), ('fellow'), ('ducks')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('')", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR, t VARCHAR)", "INSERT INTO strings VALUES \t('hello', 'hello'), ('hello', 'hallo'), ('flaw', 'lawn'),\n\t\t\t\t\t\t\t('sitting', 'kitten'), ('hallo', 'aloha'), ('hello', 'aloha'),\n\t\t\t\t\t\t\t(NULL, NULL), ('', ''),\n\t\t\t\t\t\t\t(NULL, 'bora'), ('bora', NULL),\n\t\t\t\t\t\t\t('hi', ''), ('', 'hi'),\n\t\t\t\t\t\t\t(NULL, ''), ('', NULL)"], "sql": "SELECT levenshtein(s, t) ld FROM strings", "cols": ["ld"], "rows": [[0], [1], [2], [3], [4], [5], [null], [0], [null], [null], [2], [2], [null], [null]]} +{"source": "test/sql/function/string/test_like.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE s LIKE 'ab%'", "cols": ["s"], "rows": [["abab"]]} +{"source": "test/sql/function/string/test_like.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE 'aba' LIKE pat", "cols": ["s"], "rows": [["abab"], ["aaa"], ["aaa"]]} +{"source": "test/sql/function/string/test_like.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a_a'), ('aaa', '%b%')"], "sql": "SELECT s FROM strings WHERE s LIKE pat", "cols": ["s"], "rows": [["abab"], ["aaa"]]} +{"source": "test/sql/function/string/test_like_escape.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a*_a'), ('aaa', '*%b'), ('bbb', 'a%');"], "sql": "SELECT s FROM strings;", "cols": ["s"], "rows": [["abab"], ["aaa"], ["aaa"], ["bbb"]]} +{"source": "test/sql/function/string/test_like_escape.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a*_a'), ('aaa', '*%b'), ('bbb', 'a%');"], "sql": "SELECT pat FROM strings;", "cols": ["pat"], "rows": [["ab%"], ["a*_a"], ["*%b"], ["a%"]]} +{"source": "test/sql/function/string/test_like_escape.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a*_a'), ('aaa', '*%b'), ('bbb', 'a%');"], "sql": "SELECT s FROM strings WHERE pat LIKE 'a*%' ESCAPE '*';", "cols": ["s"], "rows": [["bbb"]]} +{"source": "test/sql/function/string/test_like_escape.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a*_a'), ('aaa', '*%b'), ('bbb', 'a%');"], "sql": "SELECT s FROM strings WHERE 'aba' LIKE pat ESCAPE '*';", "cols": ["s"], "rows": [["abab"], ["bbb"]]} +{"source": "test/sql/function/string/test_like_escape.test", "setup": ["CREATE TABLE strings(s STRING, pat STRING);", "INSERT INTO strings VALUES ('abab', 'ab%'), ('aaa', 'a*_a'), ('aaa', '*%b'), ('bbb', 'a%');"], "sql": "SELECT s FROM strings WHERE s LIKE pat ESCAPE '*';", "cols": ["s"], "rows": [["abab"]]} +{"source": "test/sql/function/string/test_mismatches.test", "setup": ["CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('hallo'), ('aloha'), ('world'), (NULL)", "DROP TABLE strings", "CREATE TABLE strings(s VARCHAR)", "INSERT INTO strings VALUES ('hello'), ('halo'), (NULL)"], "sql": "SELECT hamming(s, 'hallo') FROM strings WHERE s = 'hello'", "cols": ["hamming(s, 'hallo')"], "rows": [[1]]} +{"source": "test/sql/function/string/test_pad.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select LPAD(a, 16, b), RPAD(a, 16, b) FROM strings", "cols": ["lpad(a, 16, b)", "rpad(a, 16, b)"], "rows": [["WorldWorldWHello", "HelloWorldWorldW"], [null, null], ["R\u00c4cksR\u00c4Mot\u00f6rHead", "Mot\u00f6rHeadR\u00c4cksR\u00c4"]]} +{"source": "test/sql/function/string/test_pad.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks')"], "sql": "select LPAD(a, 12, b), RPAD(a, 12, b), UCASE(a), LCASE(a) FROM strings WHERE b IS NOT NULL", "cols": ["lpad(a, 12, b)", "rpad(a, 12, b)", "ucase(a)", "lcase(a)"], "rows": [["WorldWoHello", "HelloWorldWo", "HELLO", "hello"], ["R\u00c4cMot\u00f6rHead", "Mot\u00f6rHeadR\u00c4c", "MOT\u00d6RHEAD", "mot\u00f6rhead"]]} +{"source": "test/sql/function/string/test_prefix.test", "setup": ["CREATE TABLE t0(c0 VARCHAR)"], "sql": "SELECT * FROM t0 WHERE PREFIX(t0.c0, '')", "cols": ["c0"], "rows": []} +{"source": "test/sql/function/string/test_prefix.test", "setup": ["CREATE TABLE t0(c0 VARCHAR)", "INSERT INTO t0 VALUES ('a'), ('b'), ('d')"], "sql": "SELECT COUNT(*) FROM t0 WHERE prefix(t0.c0, '');", "cols": ["count_star()"], "rows": [[3]]} +{"source": "test/sql/function/string/test_prefix.test", "setup": ["CREATE TABLE t0(c0 VARCHAR)", "INSERT INTO t0 VALUES ('a'), ('b'), ('d')", "INSERT INTO t0 VALUES (NULL)"], "sql": "SELECT COUNT(*) FROM t0 WHERE prefix(t0.c0, '');", "cols": ["count_star()"], "rows": [[3]]} +{"source": "test/sql/function/string/test_repeat.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REPEAT(a, 3) FROM strings", "cols": ["repeat(a, 3)"], "rows": [["HelloHelloHello"], ["HuLlDHuLlDHuLlD"], ["Mot\u00f6rHeadMot\u00f6rHeadMot\u00f6rHead"], [""]]} +{"source": "test/sql/function/string/test_repeat.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REPEAT(b, 2) FROM strings", "cols": ["repeat(b, 2)"], "rows": [["WorldWorld"], [null], ["R\u00c4cksR\u00c4cks"], [null]]} +{"source": "test/sql/function/string/test_repeat.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REPEAT(a, 4) FROM strings WHERE b IS NOT NULL", "cols": ["repeat(a, 4)"], "rows": [["HelloHelloHelloHello"], ["Mot\u00f6rHeadMot\u00f6rHeadMot\u00f6rHeadMot\u00f6rHead"]]} +{"source": "test/sql/function/string/test_replace.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REPLACE(a, 'l', '-') FROM strings", "cols": ["\"replace\"(a, 'l', '-')"], "rows": [["He--o"], ["HuL-D"], ["Mot\u00f6rHead"], [""]]} +{"source": "test/sql/function/string/test_replace.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REPLACE(b, '\u00c4', '--') FROM strings", "cols": ["\"replace\"(b, '\u00c4', '--')"], "rows": [["World"], [null], ["R--cks"], [null]]} +{"source": "test/sql/function/string/test_replace.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REPLACE(a, 'H', '') FROM strings WHERE b IS NOT NULL", "cols": ["\"replace\"(a, 'H', '')"], "rows": [["ello"], ["Mot\u00f6read"]]} +{"source": "test/sql/function/string/test_reverse.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REVERSE(a) FROM strings", "cols": ["reverse(a)"], "rows": [["olleH"], ["DlLuH"], ["daeHr\u00f6toM"], [""]]} +{"source": "test/sql/function/string/test_reverse.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REVERSE(b) FROM strings", "cols": ["reverse(b)"], "rows": [["dlroW"], [null], ["skc\u00c4R"], [null]]} +{"source": "test/sql/function/string/test_reverse.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL)"], "sql": "select REVERSE(a) FROM strings WHERE b IS NOT NULL", "cols": ["reverse(a)"], "rows": [["olleH"], ["daeHr\u00f6toM"]]} +{"source": "test/sql/function/string/test_similar_to.test", "setup": ["CREATE TABLE strings (s STRING, p STRING);", "INSERT INTO strings VALUES('aaa', 'a[a-z]a'), ('abab', 'ab.*'), ('aaa', 'a[a-z]a'), ('aaa', '.*b.*');"], "sql": "SELECT s FROM strings WHERE s SIMILAR TO 'ab.*'", "cols": ["s"], "rows": [["abab"]]} +{"source": "test/sql/function/string/test_similar_to.test", "setup": ["CREATE TABLE strings (s STRING, p STRING);", "INSERT INTO strings VALUES('aaa', 'a[a-z]a'), ('abab', 'ab.*'), ('aaa', 'a[a-z]a'), ('aaa', '.*b.*');"], "sql": "SELECT s FROM strings WHERE 'aba' SIMILAR TO p", "cols": ["s"], "rows": [["aaa"], ["abab"], ["aaa"], ["aaa"]]} +{"source": "test/sql/function/string/test_similar_to.test", "setup": ["CREATE TABLE strings (s STRING, p STRING);", "INSERT INTO strings VALUES('aaa', 'a[a-z]a'), ('abab', 'ab.*'), ('aaa', 'a[a-z]a'), ('aaa', '.*b.*');"], "sql": "SELECT s FROM strings WHERE s SIMILAR TO p", "cols": ["s"], "rows": [["aaa"], ["abab"], ["aaa"]]} +{"source": "test/sql/function/string/test_similar_to.test", "setup": ["CREATE TABLE strings (s STRING, p STRING);", "INSERT INTO strings VALUES('aaa', 'a[a-z]a'), ('abab', 'ab.*'), ('aaa', 'a[a-z]a'), ('aaa', '.*b.*');"], "sql": "SELECT s FROM strings WHERE s NOT SIMILAR TO p", "cols": ["s"], "rows": [["aaa"]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(s,'h') FROM strings", "cols": ["starts_with(s, 'h')"], "rows": [[true], [false], [true], [null]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(s,'he') FROM strings", "cols": ["starts_with(s, 'he')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(s,'he-man') FROM strings", "cols": ["starts_with(s, 'he-man')"], "rows": [[false], [false], [false], [null]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(NULL,'h') FROM strings", "cols": ["starts_with(NULL, 'h')"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(s,NULL) FROM strings", "cols": ["starts_with(s, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(NULL,NULL) FROM strings", "cols": ["starts_with(NULL, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_starts_with_function.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT starts_with(s,'') FROM strings", "cols": ["starts_with(s, '')"], "rows": [[true], [true], [true], [null]]} +{"source": "test/sql/function/string/test_starts_with_function_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT starts_with(s,'\u00e1') FROM strings", "cols": ["starts_with(s, '\u00e1')"], "rows": [[true], [false], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_function_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT starts_with(s,'ol\u00e1 mundo') FROM strings", "cols": ["starts_with(s, 'ol\u00e1 mundo')"], "rows": [[false], [true], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_function_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT starts_with(s,'\u4f60\u597d\u4e16\u754c') FROM strings", "cols": ["starts_with(s, '\u4f60\u597d\u4e16\u754c')"], "rows": [[false], [false], [true], [false]]} +{"source": "test/sql/function/string/test_starts_with_function_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT starts_with(s,'two \u00f1 thr') FROM strings", "cols": ["starts_with(s, 'two \u00f1 thr')"], "rows": [[false], [false], [false], [true]]} +{"source": "test/sql/function/string/test_starts_with_function_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT starts_with(s,'\u00f1') FROM strings", "cols": ["starts_with(s, '\u00f1')"], "rows": [[false], [false], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_function_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT starts_with(s,'\u20a1 four \ud83e\udd86 e') FROM strings", "cols": ["starts_with(s, '\u20a1 four \ud83e\udd86 e')"], "rows": [[false], [false], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT s ^@ 'h' FROM strings", "cols": ["(s ^@ 'h')"], "rows": [[true], [false], [true], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT s ^@ 'he' FROM strings", "cols": ["(s ^@ 'he')"], "rows": [[true], [false], [false], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT s ^@ 'he-man' FROM strings", "cols": ["(s ^@ 'he-man')"], "rows": [[false], [false], [false], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT NULL ^@ 'h' FROM strings", "cols": ["(NULL ^@ 'h')"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT s ^@ NULL FROM strings", "cols": ["(s ^@ NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT NULL ^@ NULL FROM strings", "cols": ["(NULL ^@ NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 1, 2), ('world', 2, 3), ('h', 1, 1), (NULL, 2, 2)"], "sql": "SELECT s ^@ '' FROM strings", "cols": ["(s ^@ '')"], "rows": [[true], [true], [true], [null]]} +{"source": "test/sql/function/string/test_starts_with_operator_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT s ^@ '\u00e1' FROM strings", "cols": ["(s ^@ '\u00e1')"], "rows": [[true], [false], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_operator_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT s ^@ 'ol\u00e1 mundo' FROM strings", "cols": ["(s ^@ 'ol\u00e1 mundo')"], "rows": [[false], [true], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_operator_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT s ^@ '\u4f60\u597d\u4e16\u754c' FROM strings", "cols": ["(s ^@ '\u4f60\u597d\u4e16\u754c')"], "rows": [[false], [false], [true], [false]]} +{"source": "test/sql/function/string/test_starts_with_operator_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT s ^@ 'two \u00f1 thr' FROM strings", "cols": ["(s ^@ 'two \u00f1 thr')"], "rows": [[false], [false], [false], [true]]} +{"source": "test/sql/function/string/test_starts_with_operator_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT s ^@ '\u00f1' FROM strings", "cols": ["(s ^@ '\u00f1')"], "rows": [[false], [false], [false], [false]]} +{"source": "test/sql/function/string/test_starts_with_operator_utf8.test", "setup": ["CREATE TABLE strings(s VARCHAR);", "INSERT INTO strings VALUES ('\u00e1tomo')", "INSERT INTO strings VALUES ('ol\u00e1 mundo')", "INSERT INTO strings VALUES ('\u4f60\u597d\u4e16\u754c')", "INSERT INTO strings VALUES ('two \u00f1 three \u20a1 four \ud83e\udd86 end')"], "sql": "SELECT s ^@ '\u20a1 four \ud83e\udd86 e' FROM strings", "cols": ["(s ^@ '\u20a1 four \ud83e\udd86 e')"], "rows": [[false], [false], [false], [false]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 0, 2) FROM strings", "cols": ["array_slice(s, 0, 2)"], "rows": [["he"], ["wo"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT list_slice(s, 0, 2) FROM strings", "cols": ["list_slice(s, 0, 2)"], "rows": [["he"], ["wo"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 1, 3) FROM strings", "cols": ["array_slice(s, 1, 3)"], "rows": [["hel"], ["wor"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 2, 3) FROM strings", "cols": ["array_slice(s, 2, 3)"], "rows": [["el"], ["or"], [""], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, off, length+off) FROM strings", "cols": ["array_slice(s, \"off\", (length + \"off\"))"], "rows": [["he"], ["worl"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, off, 2+off) FROM strings", "cols": ["array_slice(s, \"off\", (2 + \"off\"))"], "rows": [["he"], ["wor"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 0, length) FROM strings", "cols": ["array_slice(s, 0, length)"], "rows": [["he"], ["wor"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice('hello', off, length+off) FROM strings", "cols": ["array_slice('hello', \"off\", (length + \"off\"))"], "rows": [["he"], ["hell"], ["h"], ["hel"]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(NULL::varchar, off, length+off) FROM strings", "cols": ["array_slice(CAST(NULL AS VARCHAR), \"off\", (length + \"off\"))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice('hello', NULL, length+NULL) FROM strings", "cols": ["array_slice('hello', NULL, (length + NULL))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice('hello', off+1, NULL+off) FROM strings", "cols": ["array_slice('hello', (\"off\" + 1), (NULL + \"off\"))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(NULL::varchar, NULL, length+NULL) FROM strings", "cols": ["array_slice(CAST(NULL AS VARCHAR), NULL, (length + NULL))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice('hello', NULL, NULL+NULL) FROM strings", "cols": ["array_slice('hello', NULL, (NULL + NULL))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(NULL::varchar, off, NULL+off) FROM strings", "cols": ["array_slice(CAST(NULL AS VARCHAR), \"off\", (NULL + \"off\"))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(NULL::varchar, NULL, NULL+NULL) FROM strings", "cols": ["array_slice(CAST(NULL AS VARCHAR), NULL, (NULL + NULL))"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, -2, NULL) FROM strings", "cols": ["array_slice(s, -2, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 0, 1) FROM strings", "cols": ["array_slice(s, 0, 1)"], "rows": [["h"], ["w"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, -4, -2) FROM strings", "cols": ["array_slice(s, -4, -2)"], "rows": [["ell"], ["orl"], [""], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 1, 0) FROM strings", "cols": ["array_slice(s, 1, 0)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, 2, NULL) FROM strings", "cols": ["array_slice(s, 2, NULL)"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, (2147483647-1), 1) FROM strings", "cols": ["array_slice(s, (2147483647 - 1), 1)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, (2147483647-1), -1) FROM strings", "cols": ["array_slice(s, (2147483647 - 1), -1)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, (-2147483646-1), -1) FROM strings", "cols": ["array_slice(s, (-2147483646 - 1), -1)"], "rows": [["hello"], ["world"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_array_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)"], "sql": "SELECT array_slice(s, (-2147483646-1), -2147483647) FROM strings", "cols": ["array_slice(s, (-2147483646 - 1), -2147483647)"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[0:2] FROM strings", "cols": ["s[0:2]"], "rows": [["he"], ["wo"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[1:3] FROM strings", "cols": ["s[1:3]"], "rows": [["hel"], ["wor"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[2:3] FROM strings", "cols": ["s[2:3]"], "rows": [["el"], ["or"], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[off:length+off] FROM strings", "cols": ["s[\"off\":(length + \"off\")]"], "rows": [["he"], ["worl"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[off:2+off] FROM strings", "cols": ["s[\"off\":(2 + \"off\")]"], "rows": [["he"], ["wor"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[0:length] FROM strings", "cols": ["s[0:length]"], "rows": [["he"], ["wor"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT 'hello'[off:length+off] FROM strings", "cols": ["'hello'[\"off\":(length + \"off\")]"], "rows": [["he"], ["hell"], ["h"], ["hel"]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT n[off:length+off] FROM strings, nulltable", "cols": ["n[\"off\":(length + \"off\")]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT 'hello'[NULL:length+NULL] FROM strings", "cols": ["'hello'[NULL:(length + NULL)]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT 'hello'[off:NULL+off] FROM strings", "cols": ["'hello'[\"off\":(NULL + \"off\")]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT 'hello'[off+1:NULL+off] FROM strings", "cols": ["'hello'[(\"off\" + 1):(NULL + \"off\")]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT n[NULL:length+NULL] FROM strings, nulltable", "cols": ["n[NULL:(length + NULL)]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT 'hello'[NULL:NULL+NULL] FROM strings", "cols": ["'hello'[NULL:(NULL + NULL)]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT n[off:NULL+off] FROM strings, nulltable", "cols": ["n[\"off\":(NULL + \"off\")]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT n[NULL:NULL+NULL] FROM strings, nulltable", "cols": ["n[NULL:(NULL + NULL)]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[-2:] FROM strings", "cols": ["s[-2:]"], "rows": [["lo"], ["ld"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[0:1] FROM strings", "cols": ["s[0:1]"], "rows": [["h"], ["w"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[-4:-2] FROM strings", "cols": ["s[-4:-2]"], "rows": [["ell"], ["orl"], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[1:0] FROM strings", "cols": ["s[1:0]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[2:] FROM strings", "cols": ["s[2:]"], "rows": [["ello"], ["orld"], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[(2147483647-1):1] FROM strings", "cols": ["s[(2147483647 - 1):1]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[(2147483647-1):-1] FROM strings", "cols": ["s[(2147483647 - 1):-1]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[(-2147483646-1):-1] FROM strings", "cols": ["s[(-2147483646 - 1):-1]"], "rows": [["hello"], ["world"], ["b"], [null]]} +{"source": "test/sql/function/string/test_string_slice.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER, length INTEGER);", "INSERT INTO strings VALUES ('hello', 0, 2), ('world', 1, 3), ('b', 0, 1), (NULL, 1, 2)", "CREATE TABLE nulltable(n VARCHAR);", "INSERT INTO nulltable VALUES (NULL)"], "sql": "SELECT s[(-2147483646-1):-2147483647] FROM strings", "cols": ["s[(-2147483646 - 1):-2147483647]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[2] FROM strings", "cols": ["s[2]"], "rows": [["e"], ["o"], [""], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[3] FROM strings", "cols": ["s[3]"], "rows": [["l"], ["r"], [""], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[off] FROM strings", "cols": ["s[\"off\"]"], "rows": [["h"], ["o"], ["b"], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[2] FROM strings", "cols": ["s[2]"], "rows": [["e"], ["o"], [""], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT 'hello'[off] FROM strings", "cols": ["'hello'[\"off\"]"], "rows": [["h"], ["e"], ["h"], ["e"]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT 'hello'[NULL] FROM strings", "cols": ["'hello'[NULL]"], "rows": [[null], [null], [null], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[-1] FROM strings", "cols": ["s[-1]"], "rows": [["o"], ["d"], ["b"], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[1] FROM strings", "cols": ["s[1]"], "rows": [["h"], ["w"], ["b"], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[6] FROM strings", "cols": ["s[6]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[2147483646] FROM strings", "cols": ["s[2147483646]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_subscript.test", "setup": ["CREATE TABLE strings(s VARCHAR, off INTEGER);", "INSERT INTO strings VALUES ('hello', 1), ('world', 2), ('b', 1), (NULL, 2)"], "sql": "SELECT s[-2147483647] FROM strings", "cols": ["s[-2147483647]"], "rows": [[""], [""], [""], [null]]} +{"source": "test/sql/function/string/test_translate.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL), ('Hi', '\ud83e\udd86')"], "sql": "select TRANSLATE(a, '\u00f6Hl', 'oA-') FROM strings", "cols": ["translate(a, '\u00f6Hl', 'oA-')"], "rows": [["Ae--o"], ["AuL-D"], ["MotorAead"], [""], ["Ai"]]} +{"source": "test/sql/function/string/test_translate.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL), ('Hi', '\ud83e\udd86')"], "sql": "select TRANSLATE(a, 'loD', '\ud83e\udd86') FROM strings", "cols": ["translate(a, 'loD', '\ud83e\udd86')"], "rows": [["He\ud83e\udd86\ud83e\udd86"], ["HuL\ud83e\udd86"], ["Mt\u00f6rHead"], [""], ["Hi"]]} +{"source": "test/sql/function/string/test_translate.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL), ('Hi', '\ud83e\udd86')"], "sql": "select TRANSLATE(b, '\u00c4W\ud83e\udd86l', 'ow\ud83d\udc31') FROM strings", "cols": ["translate(b, '\u00c4W\ud83e\udd86l', 'ow\ud83d\udc31')"], "rows": [["word"], [null], ["Rocks"], [null], ["\ud83d\udc31"]]} +{"source": "test/sql/function/string/test_translate.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('Hello', 'World'), ('HuLlD', NULL), ('Mot\u00f6rHead','R\u00c4cks'), ('', NULL), ('Hi', '\ud83e\udd86')"], "sql": "select TRANSLATE(a, 'oel', 'OEL') FROM strings WHERE b IS NOT NULL", "cols": ["translate(a, 'oel', 'OEL')"], "rows": [["HELLO"], ["MOt\u00f6rHEad"], ["Hi"]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)"], "sql": "select LTRIM(a) FROM strings", "cols": ["ltrim(a)"], "rows": [[""], ["Leading"], ["Both "], [""]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)"], "sql": "select LTRIM(b) FROM strings", "cols": ["ltrim(b)"], "rows": [["Neither"], [null], ["Trailing "], [null]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)"], "sql": "select LTRIM(a) FROM strings WHERE b IS NOT NULL", "cols": ["ltrim(a)"], "rows": [[""], ["Both "]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)"], "sql": "select RTRIM(a) FROM strings", "cols": ["rtrim(a)"], "rows": [[""], [" Leading"], [" Both"], [""]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)"], "sql": "select RTRIM(b) FROM strings", "cols": ["rtrim(b)"], "rows": [["Neither"], [null], ["Trailing"], [null]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)"], "sql": "select RTRIM(a) FROM strings WHERE b IS NOT NULL", "cols": ["rtrim(a)"], "rows": [[""], [" Both"]]} +{"source": "test/sql/function/string/test_trim.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Neither'), (' Leading', NULL), (' Both ','Trailing '), ('', NULL)", "CREATE TABLE trim_test(a VARCHAR, b VARCHAR)", "INSERT INTO trim_test VALUES ('hello', 'ho'), ('test', 't'), ('m\u00fchleisen','m\u00fcn'), (NULL, ' '), ('', NULL), ('', ''), (NULL, NULL)"], "sql": "SELECT LTRIM(a, b), RTRIM(a, b), TRIM(a, b) FROM trim_test", "cols": ["ltrim(a, b)", "rtrim(a, b)", "main.\"trim\"(a, b)"], "rows": [["ello", "hell", "ell"], ["est", "tes", "es"], ["hleisen", "m\u00fchleise", "hleise"], [null, null, null], [null, null, null], ["", "", ""], [null, null, null]]} +{"source": "test/sql/function/string/test_unicode.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Zero'), ('$', NULL), ('\u00a2','Two'), ('\u20ac', NULL), ('\ud800\udf48','Four')"], "sql": "select UNICODE(a) FROM strings", "cols": ["unicode(a)"], "rows": [[-1], [36], [162], [8364], [66376]]} +{"source": "test/sql/function/string/test_unicode.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Zero'), ('$', NULL), ('\u00a2','Two'), ('\u20ac', NULL), ('\ud800\udf48','Four')"], "sql": "select UNICODE(b) FROM strings", "cols": ["unicode(b)"], "rows": [[90], [null], [84], [null], [70]]} +{"source": "test/sql/function/string/test_unicode.test", "setup": ["CREATE TABLE strings(a STRING, b STRING)", "INSERT INTO strings VALUES ('', 'Zero'), ('$', NULL), ('\u00a2','Two'), ('\u20ac', NULL), ('\ud800\udf48','Four')"], "sql": "select UNICODE(a) FROM strings WHERE b IS NOT NULL", "cols": ["unicode(a)"], "rows": [[-1], [162], [66376]]} +{"source": "test/sql/function/string/test_url_encode.test", "setup": [], "sql": "SELECT COUNT(*) from range(1000) t(n) WHERE url_decode(url_encode(chr(n::INT))) = chr(n::INT)", "cols": ["count_star()"], "rows": [[1000]]} +{"source": "test/sql/function/numeric/test_arithmetic_aliases.test", "setup": ["CREATE TABLE test(a integer)", "insert into test values (1), (2), (3), (NULL)"], "sql": "select add(a,a) from test", "cols": ["\"add\"(a, a)"], "rows": [[2], [4], [6], [null]]} +{"source": "test/sql/function/numeric/test_arithmetic_aliases.test", "setup": ["CREATE TABLE test(a integer)", "insert into test values (1), (2), (3), (NULL)"], "sql": "select subtract(a,a) from test", "cols": ["subtract(a, a)"], "rows": [[0], [0], [0], [null]]} +{"source": "test/sql/function/numeric/test_arithmetic_aliases.test", "setup": ["CREATE TABLE test(a integer)", "insert into test values (1), (2), (3), (NULL)"], "sql": "select multiply(a,a) from test", "cols": ["multiply(a, a)"], "rows": [[1], [4], [9], [null]]} +{"source": "test/sql/function/numeric/test_arithmetic_aliases.test", "setup": ["CREATE TABLE test(a integer)", "insert into test values (1), (2), (3), (NULL)"], "sql": "select divide(a,a) from test", "cols": ["divide(a, a)"], "rows": [[1], [1], [1], [null]]} +{"source": "test/sql/function/numeric/test_bit_count.test", "setup": ["CREATE TABLE bits(t tinyint, s smallint, i integer, b bigint, h hugeint)", "INSERT INTO bits VALUES (NULL, NULL, NULL, NULL, NULL), (31, 1023, 11834119, 50827156903621017, 3141592653589793238462643383279528841), (-59, -517, -575693, -9876543210, -148873535527910577765226390751398592512)"], "sql": "select bit_count(t), bit_count(s), bit_count(i), bit_count(b), bit_count(h) from bits", "cols": ["bit_count(t)", "bit_count(s)", "bit_count(i)", "bit_count(b)", "bit_count(h)"], "rows": [[null, null, null, null, null], [5, 10, 11, 27, 67], [4, 14, 24, 49, 2]]} +{"source": "test/sql/function/numeric/test_even.test", "setup": [], "sql": "select i, even(i + 0.4) from generate_series(-4,4) tbl(i);", "cols": ["i", "even((i + 0.4))"], "rows": [[-4, -4.0], [-3, -4.0], [-2, -2.0], [-1, -2.0], [0, 2.0], [1, 2.0], [2, 4.0], [3, 4.0], [4, 6.0]]} +{"source": "test/sql/function/numeric/test_even.test", "setup": [], "sql": "select i, even(i + 0.9) from generate_series(-4,4) tbl(i);", "cols": ["i", "even((i + 0.9))"], "rows": [[-4, -4.0], [-3, -4.0], [-2, -2.0], [-1, -2.0], [0, 2.0], [1, 2.0], [2, 4.0], [3, 4.0], [4, 6.0]]} +{"source": "test/sql/function/numeric/test_fdiv_fmod.test", "setup": ["CREATE TABLE rs(x DOUBLE, y INTEGER)", "INSERT INTO rs VALUES (10, 3),(10,-3),(-10,3),(-10,-3),(0,1),(1,1),(NULL,10),(10,NULL),(NULL,NULL)"], "sql": "SELECT fmod(x, y) FROM rs", "cols": ["fmod(x, y)"], "rows": [[1.0], [-2.0], [2.0], [-1.0], [0.0], [0.0], [null], [null], [null]]} +{"source": "test/sql/function/numeric/test_fdiv_fmod.test", "setup": ["CREATE TABLE rs(x DOUBLE, y INTEGER)", "INSERT INTO rs VALUES (10, 3),(10,-3),(-10,3),(-10,-3),(0,1),(1,1),(NULL,10),(10,NULL),(NULL,NULL)"], "sql": "SELECT fdiv(x, y) FROM rs", "cols": ["fdiv(x, y)"], "rows": [[3.0], [-4.0], [-4.0], [3.0], [0.0], [1.0], [null], [null], [null]]} +{"source": "test/sql/function/numeric/test_geomean.test", "setup": ["CREATE TABLE numbers(x DOUBLE)", "INSERT INTO numbers VALUES (NULL), (1), (2)"], "sql": "SELECT geomean(x) FROM numbers", "cols": ["geomean(x)"], "rows": [[1.414213562373095]]} +{"source": "test/sql/function/numeric/test_geomean.test", "setup": ["CREATE TABLE numbers(x DOUBLE)", "INSERT INTO numbers VALUES (NULL), (1), (2)"], "sql": "SELECT geomean(x::integer) FROM numbers", "cols": ["geomean(CAST(x AS INTEGER))"], "rows": [[1.414213562373095]]} +{"source": "test/sql/function/numeric/test_geomean.test", "setup": ["CREATE TABLE numbers(x DOUBLE)", "INSERT INTO numbers VALUES (NULL), (1), (2)"], "sql": "SELECT geomean(i) FROM generate_series(1000, 2000) tbl(i);", "cols": ["geomean(i)"], "rows": [[1471.459313166229]]} +{"source": "test/sql/function/numeric/test_mod.test", "setup": ["CREATE TABLE modme(a DOUBLE, b INTEGER)", "INSERT INTO modme VALUES (42.123456, 3)"], "sql": "select mod(a, 40) from modme", "cols": ["mod(a, 40)"], "rows": [[2.1234559999999973]]} +{"source": "test/sql/function/numeric/test_mod.test", "setup": ["CREATE TABLE modme(a DOUBLE, b INTEGER)", "INSERT INTO modme VALUES (42.123456, 3)"], "sql": "select mod(a, 2) from modme", "cols": ["mod(a, 2)"], "rows": [[0.12345599999999735]]} +{"source": "test/sql/function/numeric/test_nextafter.test", "setup": ["create table test (a FLOAT)", "INSERT INTO test VALUES (10),(20),(30),(40)"], "sql": "select nextafter(a, 0::FLOAT) from test", "cols": ["nextafter(a, CAST(0 AS FLOAT))"], "rows": [[9.999999046325684], [19.999998092651367], [29.999998092651367], [39.999996185302734]]} +{"source": "test/sql/function/numeric/test_nextafter.test", "setup": ["create table test (a FLOAT)", "INSERT INTO test VALUES (10),(20),(30),(40)", "create table test_twoc (a FLOAT, b FLOAT)", "INSERT INTO test_twoc VALUES (10,1),(20,21),(30,1),(40,41)"], "sql": "select nextafter(a, b) from test_twoc", "cols": ["nextafter(a, b)"], "rows": [[9.999999046325684], [20.000001907348633], [29.999998092651367], [40.000003814697266]]} +{"source": "test/sql/function/numeric/test_pow.test", "setup": ["CREATE TABLE powerme(a DOUBLE, b INTEGER)", "INSERT INTO powerme VALUES (2.1, 3)"], "sql": "select pow(a, 0) from powerme", "cols": ["pow(a, 0)"], "rows": [[1.0]]} +{"source": "test/sql/function/numeric/test_pow.test", "setup": ["CREATE TABLE powerme(a DOUBLE, b INTEGER)", "INSERT INTO powerme VALUES (2.1, 3)"], "sql": "select pow(b, -2) from powerme", "cols": ["pow(b, -2)"], "rows": [[0.1111111111111111]]} +{"source": "test/sql/function/numeric/test_pow.test", "setup": ["CREATE TABLE powerme(a DOUBLE, b INTEGER)", "INSERT INTO powerme VALUES (2.1, 3)"], "sql": "select pow(a, b) from powerme", "cols": ["pow(a, b)"], "rows": [[9.261000000000001]]} +{"source": "test/sql/function/numeric/test_pow.test", "setup": ["CREATE TABLE powerme(a DOUBLE, b INTEGER)", "INSERT INTO powerme VALUES (2.1, 3)"], "sql": "select pow(b, a) from powerme", "cols": ["pow(b, a)"], "rows": [[10.04510856630514]]} +{"source": "test/sql/function/numeric/test_pow.test", "setup": ["CREATE TABLE powerme(a DOUBLE, b INTEGER)", "INSERT INTO powerme VALUES (2.1, 3)"], "sql": "select power(b, a) from powerme", "cols": ["power(b, a)"], "rows": [[10.04510856630514]]} +{"source": "test/sql/function/numeric/test_round.test", "setup": ["CREATE TABLE roundme(a DOUBLE, b INTEGER)", "INSERT INTO roundme VALUES (42.123456, 3)"], "sql": "select round(a, 1) from roundme", "cols": ["round(a, 1)"], "rows": [[42.1]]} +{"source": "test/sql/function/numeric/test_round.test", "setup": ["CREATE TABLE roundme(a DOUBLE, b INTEGER)", "INSERT INTO roundme VALUES (42.123456, 3)"], "sql": "select round(b, 1) from roundme", "cols": ["round(b, 1)"], "rows": [[3]]} +{"source": "test/sql/function/numeric/test_round.test", "setup": ["CREATE TABLE roundme(a DOUBLE, b INTEGER)", "INSERT INTO roundme VALUES (42.123456, 3)"], "sql": "select round(a, b) from roundme", "cols": ["round(a, b)"], "rows": [[42.123]]} +{"source": "test/sql/function/numeric/test_round_even.test", "setup": [], "sql": "select i, round_even(i + 0.5, 0) from generate_series(-2,4) tbl(i);", "cols": ["i", "round_even((i + 0.5), 0)"], "rows": [[-2, -2.0], [-1, -0.0], [0, 0.0], [1, 2.0], [2, 2.0], [3, 4.0], [4, 4.0]]} +{"source": "test/sql/function/numeric/test_round_even.test", "setup": [], "sql": "select i, round_even(i + 0.55, 0) from generate_series(-2,4) tbl(i);", "cols": ["i", "round_even((i + 0.55), 0)"], "rows": [[-2, -1.0], [-1, 0.0], [0, 1.0], [1, 2.0], [2, 3.0], [3, 4.0], [4, 5.0]]} +{"source": "test/sql/function/numeric/test_round_even.test", "setup": [], "sql": "select i, roundBankers(i + 0.55, 0) from generate_series(-2,4) tbl(i);", "cols": ["i", "roundbankers((i + 0.55), 0)"], "rows": [[-2, -1.0], [-1, 0.0], [0, 1.0], [1, 2.0], [2, 3.0], [3, 4.0], [4, 5.0]]} +{"source": "test/sql/function/numeric/test_round_integers.test", "setup": ["CREATE TABLE zz AS\n SELECT\n CAST(i AS SMALLINT) AS id,\n CAST(i AS SMALLINT) AS si\n FROM generate_series(1, 1000) t(i);"], "sql": "select round(100::INTEGER, int)\nfrom test_all_types();", "cols": ["round(CAST(100 AS INTEGER), \"int\")"], "rows": [[0], [100], [null]]} +{"source": "test/sql/function/numeric/test_unary.test", "setup": ["CREATE TABLE test(i INTEGER)", "INSERT INTO test VALUES (2)"], "sql": "SELECT ++-++-+i FROM test", "cols": ["+(+(-(+(+(-(+(i)))))))"], "rows": [[2]]} +{"source": "test/sql/function/numeric/test_unary.test", "setup": ["CREATE TABLE test(i INTEGER)", "INSERT INTO test VALUES (2)"], "sql": "SELECT +i FROM test", "cols": ["+(i)"], "rows": [[2]]} +{"source": "test/sql/function/numeric/test_unary.test", "setup": ["CREATE TABLE test(i INTEGER)", "INSERT INTO test VALUES (2)"], "sql": "SELECT -i FROM test", "cols": ["-(i)"], "rows": [[-2]]} +{"source": "test/sql/function/numeric/test_unary.test", "setup": ["CREATE TABLE test(i INTEGER)", "INSERT INTO test VALUES (2)"], "sql": "SELECT +++++++i FROM test", "cols": ["+(+(+(+(+(+(+(i)))))))"], "rows": [[2]]} +{"source": "test/sql/function/numeric/test_unary.test", "setup": ["CREATE TABLE test(i INTEGER)", "INSERT INTO test VALUES (2)"], "sql": "SELECT ++-++-+i FROM test", "cols": ["+(+(-(+(+(-(+(i)))))))"], "rows": [[2]]} +{"source": "test/sql/function/numeric/test_unary.test", "setup": ["CREATE TABLE test(i INTEGER)", "INSERT INTO test VALUES (2)"], "sql": "SELECT -+-+-+-+-i FROM test", "cols": ["-(+(-(+(-(+(-(+(-(i)))))))))"], "rows": [[-2]]} +{"source": "test/sql/function/generic/case_condition.test", "setup": ["CREATE TABLE tbl AS SELECT * FROM range(10) tbl(i)"], "sql": "SELECT * FROM tbl\nWHERE\nCASE WHEN i%2=0 THEN 1 ELSE 0 END\nAND\nCASE WHEN i<5 THEN 1 ELSE 0 END", "cols": ["i"], "rows": [[0], [2], [4]]} +{"source": "test/sql/function/generic/case_varchar.test", "setup": ["CREATE TABLE tbl AS SELECT i, 'thisisalongstring' || i::VARCHAR s FROM range(10) tbl(i)"], "sql": "SELECT i, s, CASE WHEN i%2=0 THEN s ELSE s END FROM tbl", "cols": ["i", "s", "CASE WHEN (((i % 2) = 0)) THEN (s) ELSE s END"], "rows": [[0, "thisisalongstring0", "thisisalongstring0"], [1, "thisisalongstring1", "thisisalongstring1"], [2, "thisisalongstring2", "thisisalongstring2"], [3, "thisisalongstring3", "thisisalongstring3"], [4, "thisisalongstring4", "thisisalongstring4"], [5, "thisisalongstring5", "thisisalongstring5"], [6, "thisisalongstring6", "thisisalongstring6"], [7, "thisisalongstring7", "thisisalongstring7"], [8, "thisisalongstring8", "thisisalongstring8"], [9, "thisisalongstring9", "thisisalongstring9"]]} +{"source": "test/sql/function/generic/cast_to_type.test", "setup": ["CREATE OR REPLACE MACRO try_trim_null(s) AS CASE WHEN typeof(s)=='VARCHAR' THEN cast_to_type(nullif(trim(s::VARCHAR), ''), s) ELSE s END;", "create table tbl(i int, v varchar);", "insert into tbl values (42, ' hello '), (100, ' ');"], "sql": "SELECT try_trim_null(COLUMNS(*)) FROM tbl", "cols": ["i", "v"], "rows": [[42, "hello"], [100, null]]} +{"source": "test/sql/function/generic/constant_or_null.test", "setup": [], "sql": "SELECT constant_or_null(1, case when i%2=0 then null else i end) from range(5) tbl(i)", "cols": ["constant_or_null(1, CASE WHEN (((i % 2) = 0)) THEN (NULL) ELSE i END)"], "rows": [[null], [1], [null], [1], [null]]} +{"source": "test/sql/function/generic/constant_or_null.test", "setup": [], "sql": "SELECT constant_or_null(1, case when i%2=0 then null else i end, case when i%2=1 then null else i end) from range(5) tbl(i)", "cols": ["constant_or_null(1, CASE WHEN (((i % 2) = 0)) THEN (NULL) ELSE i END, CASE WHEN (((i % 2) = 1)) THEN (NULL) ELSE i END)"], "rows": [[null], [null], [null], [null], [null]]} +{"source": "test/sql/function/generic/replace_type.test", "setup": ["CREATE OR REPLACE MACRO try_trim_null(s) AS CASE WHEN typeof(s)=='VARCHAR' THEN replace_type(nullif(trim(s::VARCHAR), ''), NULL::VARCHAR, s) ELSE s END;", "create table tbl(i int, v varchar);", "insert into tbl values (42, ' hello '), (100, ' ');"], "sql": "SELECT try_trim_null(COLUMNS(*)) FROM tbl", "cols": ["i", "v"], "rows": [[42, "hello"], [100, null]]} +{"source": "test/sql/function/generic/table_func_varargs.test", "setup": [], "sql": "SELECT * FROM repeat_row(1, 2, 'foo', num_rows=3)", "cols": ["column0", "column1", "column2"], "rows": [[1, 2, "foo"], [1, 2, "foo"], [1, 2, "foo"]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 10 BETWEEN 10 AND 20", "cols": ["count_star()"], "rows": [[4]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 9 BETWEEN 10 AND 20", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 10 BETWEEN NULL AND 20", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 30 BETWEEN NULL AND 20", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 10 BETWEEN 10 AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 9 BETWEEN 10 AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN 10 AND 20", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN NULL AND 20", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN 10 AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN NULL AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN 1 AND 2", "cols": ["count_star()"], "rows": [[2]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN NULL AND 2", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN 2 AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN -1 AND +1", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 0 BETWEEN -1 AND +1", "cols": ["count_star()"], "rows": [[4]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN i-1 AND i+1", "cols": ["count_star()"], "rows": [[3]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN i-1 AND 10", "cols": ["count_star()"], "rows": [[3]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN NULL AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN i-1 AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN 0 AND i+1", "cols": ["count_star()"], "rows": [[3]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE i BETWEEN NULL AND i+1", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 1 BETWEEN i-1 AND i+1", "cols": ["count_star()"], "rows": [[2]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN i-1 AND i+1", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE NULL BETWEEN i-1 AND NULL", "cols": ["count_star()"], "rows": [[0]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 1 BETWEEN i-1 AND 100", "cols": ["count_star()"], "rows": [[2]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT COUNT(*) FROM integers WHERE 1 BETWEEN 0 AND i-1", "cols": ["count_star()"], "rows": [[2]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT * FROM integers WHERE i >= 1 AND i < 2", "cols": ["i"], "rows": [[1]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT * FROM integers WHERE i > 1 AND i <= 2", "cols": ["i"], "rows": [[2]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT * FROM range(4) tbl(i) WHERE i >= 1 AND i < 2", "cols": ["i"], "rows": [[1]]} +{"source": "test/sql/function/generic/test_between.test", "setup": ["CREATE TABLE integers(i INTEGER);", "INSERT INTO integers VALUES (1), (2), (3), (NULL)"], "sql": "SELECT * FROM range(4) tbl(i) WHERE i > 1 AND i <= 2", "cols": ["i"], "rows": [[2]]} +{"source": "test/sql/function/generic/test_least_greatest.test", "setup": ["CREATE TABLE t1(i INTEGER, j INTEGER);", "INSERT INTO t1 VALUES (1, NULL), (2, 1), (3, 7);"], "sql": "SELECT LEAST(REPEAT(i::VARCHAR, 20), j::VARCHAR) FROM t1;", "cols": ["least(repeat(CAST(i AS VARCHAR), 20), CAST(j AS VARCHAR))"], "rows": [["11111111111111111111"], ["1"], ["33333333333333333333"]]} +{"source": "test/sql/function/generic/test_null_if.test", "setup": ["CREATE TABLE test (a STRING);", "INSERT INTO test VALUES ('hello'), ('world'), ('test')", "CREATE TABLE test2 (a STRING, b STRING);", "INSERT INTO test2 VALUES ('blabla', 'b'), ('blabla2', 'c'), ('blabla3', 'd')", "DROP TABLE test;", "CREATE TABLE test3 (a INTEGER, b INTEGER);", "INSERT INTO test3 VALUES (11, 22), (13, 22), (12, 21)"], "sql": "SELECT NULLIF(CAST(a AS VARCHAR), '11') FROM test3;", "cols": ["\"nullif\"(CAST(a AS VARCHAR), '11')"], "rows": [[null], ["13"], ["12"]]} +{"source": "test/sql/function/operator/test_bitwise_ops_types.test", "setup": ["CREATE TABLE bitwise_test(i TINYINT, j TINYINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)"], "sql": "SELECT i << j, i >> j, i & j, i | j, xor(i, j) FROM bitwise_test", "cols": ["(i << j)", "(i >> j)", "(i & j)", "(i | j)", "xor(i, j)"], "rows": [[2, 0, 1, 1, 0], [1, 1, 0, 1, 1], [0, 0, 0, 1, 1], [0, 0, 0, 0, 0], [null, null, null, null, null], [null, null, null, null, null], [null, null, null, null, null]]} +{"source": "test/sql/function/operator/test_bitwise_ops_types.test", "setup": ["CREATE TABLE bitwise_test(i TINYINT, j TINYINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)", "CREATE TABLE bitwise_test(i SMALLINT, j SMALLINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)"], "sql": "SELECT i << j, i >> j, i & j, i | j, xor(i, j) FROM bitwise_test", "cols": ["(i << j)", "(i >> j)", "(i & j)", "(i | j)", "xor(i, j)"], "rows": [[2, 0, 1, 1, 0], [1, 1, 0, 1, 1], [0, 0, 0, 1, 1], [0, 0, 0, 0, 0], [null, null, null, null, null], [null, null, null, null, null], [null, null, null, null, null]]} +{"source": "test/sql/function/operator/test_bitwise_ops_types.test", "setup": ["CREATE TABLE bitwise_test(i TINYINT, j TINYINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)", "CREATE TABLE bitwise_test(i SMALLINT, j SMALLINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)", "CREATE TABLE bitwise_test(i INTEGER, j INTEGER)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)"], "sql": "SELECT i << j, i >> j, i & j, i | j, xor(i, j) FROM bitwise_test", "cols": ["(i << j)", "(i >> j)", "(i & j)", "(i | j)", "xor(i, j)"], "rows": [[2, 0, 1, 1, 0], [1, 1, 0, 1, 1], [0, 0, 0, 1, 1], [0, 0, 0, 0, 0], [null, null, null, null, null], [null, null, null, null, null], [null, null, null, null, null]]} +{"source": "test/sql/function/operator/test_bitwise_ops_types.test", "setup": ["CREATE TABLE bitwise_test(i TINYINT, j TINYINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)", "CREATE TABLE bitwise_test(i SMALLINT, j SMALLINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)", "CREATE TABLE bitwise_test(i INTEGER, j INTEGER)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)", "CREATE TABLE bitwise_test(i BIGINT, j BIGINT)", "INSERT INTO bitwise_test VALUES (1, 1), (1, 0), (0, 1), (0, 0), (1, NULL), (NULL, 1), (NULL, NULL)"], "sql": "SELECT i << j, i >> j, i & j, i | j, xor(i, j) FROM bitwise_test", "cols": ["(i << j)", "(i >> j)", "(i & j)", "(i | j)", "xor(i, j)"], "rows": [[2, 0, 1, 1, 0], [1, 1, 0, 1, 1], [0, 0, 0, 1, 1], [0, 0, 0, 0, 0], [null, null, null, null, null], [null, null, null, null, null], [null, null, null, null, null]]} +{"source": "test/sql/function/operator/test_conjunction.test", "setup": ["CREATE TABLE a (i integer, j integer);", "INSERT INTO a VALUES (3, 4), (4, 5), (5, 6);"], "sql": "SELECT * FROM a WHERE (i > 3 AND j < 5) OR (i > 3 AND j > 5);", "cols": ["i", "j"], "rows": [[5, 6]]} +{"source": "test/sql/function/operator/test_in_empty_table.test", "setup": ["CREATE OR REPLACE TABLE test (a INTEGER)"], "sql": "SELECT * FROM test WHERE a IN ('a', 'b', 'c', 'd', 'e')", "cols": ["a"], "rows": []} diff --git a/uv.lock b/uv.lock index 8c2c488..f70b162 100644 --- a/uv.lock +++ b/uv.lock @@ -360,6 +360,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, ] +[[package]] +name = "duckdb" +version = "1.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, + { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, + { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1650,6 +1665,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "duckdb" }, { name = "ipdb" }, { name = "ipython" }, { name = "maturin" }, @@ -1672,6 +1688,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "duckdb", specifier = ">=1.5.5" }, { name = "ipdb", specifier = ">=0.13.13" }, { name = "ipython", specifier = ">=9.2.0" }, { name = "maturin", specifier = ">=1.14,<2.0" }, From 9e45d37d911e4cc164695d6eaeffef2ee26e1822 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 04:41:44 +0200 Subject: [PATCH 04/11] =?UTF-8?q?fix(interpreter):=20commit=20src/datafusi?= =?UTF-8?q?on/=20=E2=80=94=20an=20unanchored=20local=20exclude=20rule=20hi?= =?UTF-8?q?d=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo-root DataFusion clone is ignored via .git/info/exclude with the bare pattern `datafusion/`, which matches at any depth and silently swallowed the refactor's src/datafusion/ move (local builds passed off the working tree; CI got a commit without the module). Exclude rule anchored locally to /datafusion/. Co-Authored-By: Claude Opus 5 --- src/datafusion/expr.rs | 659 ++++++++++++++++++ src/datafusion/expr_build.rs | 278 ++++++++ src/datafusion/mod.rs | 355 ++++++++++ src/datafusion/plan.rs | 1239 ++++++++++++++++++++++++++++++++++ src/datafusion/types.rs | 314 +++++++++ 5 files changed, 2845 insertions(+) create mode 100644 src/datafusion/expr.rs create mode 100644 src/datafusion/expr_build.rs create mode 100644 src/datafusion/mod.rs create mode 100644 src/datafusion/plan.rs create mode 100644 src/datafusion/types.rs diff --git a/src/datafusion/expr.rs b/src/datafusion/expr.rs new file mode 100644 index 0000000..b5efef6 --- /dev/null +++ b/src/datafusion/expr.rs @@ -0,0 +1,659 @@ +use pyo3::prelude::*; +use pyo3::types::PyList; + +use std::sync::Arc; + +use crate::datafusion::types::FieldType; + +// Re-exported so the engine-internal paths (`crate::datafusion::expr::Value`) +// keep resolving now that the value representation is shared. +pub use crate::value::{type_name, Value}; + + +/// String form used by CONCAT and CAST(.. AS VARCHAR). +pub fn display_value(v: &Value) -> String { + match v { + Value::Int(i) => i.to_string(), + // `{:?}` matches Arrow/DataFusion's Float64->Utf8 rendering (1.0 -> "1.0", + // 1e300 -> "1e300"), unlike `to_string` (1.0 -> "1", 1e300 -> 300 digits) + // -- EXCEPT |x| in [1e-5, 1e-4): there `{:?}` uses exponential ('1e-5') + // but DataFusion stays fixed-decimal ('0.00001'). `{}` is fixed-point + // shortest-round-trip and matches the oracle for exactly that band (the + // only magnitude where the two disagree -- see TASK-7 probe). + Value::Float(f) => { + if (1e-5..1e-4).contains(&f.abs()) { + format!("{f}") + } else { + format!("{f:?}") + } + } + Value::Str(s) => s.clone(), + Value::Bool(b) => b.to_string(), + Value::Null => String::new(), + Value::Object(_) => "".to_string(), + Value::Struct(fields) => { + let inner = fields + .iter() + .map(|(k, v)| format!("{k}: {}", display_value(v))) + .collect::>() + .join(", "); + format!("{{{inner}}}") + } + Value::List(items) => { + let inner = items + .iter() + .map(display_value) + .collect::>() + .join(", "); + format!("[{inner}]") + } + } +} + + +#[derive(Clone)] +pub enum Expr { + Column { + table: Option, + name: String, + }, + Literal(Value), + BinaryOp { + op: BinOp, + left: Box, + right: Box, + }, + Not(Box), + Function { + name: String, + args: Vec, + }, + Cast { + expr: Box, + target: CastType, + }, + Struct(Vec<(String, Expr)>), + List(Vec), + FieldAccess { + base: Box, + field: String, + }, + /// Callout to an opaque, already-fitted Python transformer. `obj` is the + /// fitted sklearn object; `input_features` is its `feature_names_in_` + /// (the struct arg is reordered to this before `.transform`); + /// `output_fields` is the caller-declared output schema (marshalling + /// target, order significant); `arg` evaluates to the input `Value::Struct`. + Transform { + obj: Arc>, + input_features: Vec, + output_fields: Vec<(String, FieldType)>, + arg: Box, + }, + /// SQL CASE. `arms` are (condition, result) pairs evaluated left to right; + /// the first arm whose condition is `Bool(true)` wins. `default` is the ELSE + /// result, or `None` when there is no ELSE (an unmatched row yields NULL). + Case { + arms: Vec<(Expr, Expr)>, + default: Option>, + }, +} + +#[derive(Clone, Copy)] +pub enum CastType { + Str, + Int, + Float, + Bool, +} + +#[derive(Clone, Copy, PartialEq)] +pub enum BinOp { + Add, + Sub, + Mul, + Div, + Mod, + Eq, + NotEq, + Lt, + Gt, + LtEq, + GtEq, + And, + Or, + Concat, +} + +pub fn eval(expr: &Expr, row: &crate::datafusion::plan::Row) -> Result { + match expr { + Expr::Column { table, name } => resolve_column(row, table.as_deref(), name), + Expr::Literal(v) => Ok(v.clone()), + Expr::BinaryOp { op, left, right } => { + let l = eval(left, row)?; + let r = eval(right, row)?; + eval_binary_op(*op, l, r) + } + Expr::Not(inner) => { + let v = eval(inner, row)?; + match as_tribool(&v)? { + Some(b) => Ok(Value::Bool(!b)), + None => Ok(Value::Null), + } + } + Expr::Function { name, args } => { + let values: Vec = args + .iter() + .map(|a| eval(a, row)) + .collect::>()?; + eval_builtin(name, values) + } + Expr::Cast { expr, target } => { + let v = eval(expr, row)?; + eval_cast(v, *target) + } + Expr::Struct(fields) => { + let values = fields + .iter() + .map(|(k, e)| Ok((k.clone(), eval(e, row)?))) + .collect::>()?; + Ok(Value::Struct(values)) + } + Expr::List(items) => { + let values = items + .iter() + .map(|e| eval(e, row)) + .collect::>()?; + Ok(Value::List(values)) + } + Expr::FieldAccess { base, field } => { + let v = eval(base, row)?; + match v { + Value::Null => Ok(Value::Null), + Value::Struct(fields) => fields + .into_iter() + .find(|(name, _)| name == field) + .map(|(_, v)| v) + .ok_or_else(|| { + crate::datafusion::plan::InterpError::Eval(format!("Unknown struct field: {field}")) + }), + other => Err(crate::datafusion::plan::InterpError::Eval(format!( + "Cannot access field '{field}' on a {} value", + type_name(&other) + ))), + } + } + Expr::Transform { + obj, + input_features, + output_fields, + arg, + } => { + let fields = match eval(arg, row)? { + Value::Struct(f) => f, + Value::Null => return Ok(Value::Null), + other => { + return Err(crate::datafusion::plan::InterpError::Eval(format!( + "transformer argument must be a struct, got a {} value", + type_name(&other) + ))) + } + }; + // infer() already holds the GIL; attach is cheap and re-entrant, so + // eval() stays a pure-Rust signature (no py token threaded through). + Python::attach(|py| -> Result { + // Reorder the struct's fields to feature_names_in_ order, then + // build a 1-row Python list-of-lists. sklearn's check_array + // coerces it -- no numpy on the Rust side. + let mut ordered: Vec> = Vec::with_capacity(input_features.len()); + for feat in input_features { + let value = fields + .iter() + .find(|(n, _)| n == feat) + .map(|(_, v)| v) + .ok_or_else(|| { + crate::datafusion::plan::InterpError::Eval(format!( + "transformer input struct is missing field '{feat}'" + )) + })?; + let py_val = value.to_pyobject(py).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "marshalling transformer input failed: {e}" + )) + })?; + ordered.push(py_val); + } + let row_list = PyList::new(py, ordered).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "building transformer input row failed: {e}" + )) + })?; + let x = PyList::new(py, [row_list]).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "building transformer input matrix failed: {e}" + )) + })?; + let y = obj.bind(py).call_method1("transform", (x,)).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!("transformer.transform failed: {e}")) + })?; + // .tolist() turns numpy scalars into Python builtins so + // from_pyobject sees float/int/str, not opaque numpy objects. + let y_list = y.call_method0("tolist").map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "transformer output .tolist() failed: {e}" + )) + })?; + let y0 = y_list.get_item(0).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "transformer produced no output row: {e}" + )) + })?; + // Marshal each output position through the declared field + // type. Parity invariant: the declared output dtype must equal + // the transform's natural dtype -- here it drives pydantic + // coercion at model-validate time, and the DataFusion oracle + // reaches the same type via a pyarrow cast; the two agree only + // when no real coercion is needed (see _transformer_udf.py). + let mut out = Vec::with_capacity(output_fields.len()); + for (i, (fname, ft)) in output_fields.iter().enumerate() { + let elem = y0.get_item(i).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "transformer output missing position {i} for field '{fname}': {e}" + )) + })?; + let val = Value::from_pyobject_typed(&elem, &ft.base).map_err(|e| { + crate::datafusion::plan::InterpError::Eval(format!( + "marshalling transformer output field '{fname}' failed: {e}" + )) + })?; + out.push((fname.clone(), val)); + } + Ok(Value::Struct(out)) + }) + } + Expr::Case { arms, default } => { + // Short-circuit: evaluate conditions left to right, stop at the first + // Bool(true), and evaluate ONLY that arm's result. A non-matching + // arm's result is never touched, so CASE WHEN x>0 THEN 1/x ELSE 0 END + // does not divide by zero at x=0 -- matching the oracle. + for (cond, result) in arms { + if let Some(true) = as_tribool(&eval(cond, row)?)? { + return eval(result, row); + } + } + match default { + Some(d) => eval(d, row), + None => Ok(Value::Null), + } + } + } +} + +fn resolve_column( + row: &crate::datafusion::plan::Row, + table: Option<&str>, + name: &str, +) -> Result { + use crate::datafusion::plan::InterpError; + + if let Some(t) = table { + return row + .get(t) + .and_then(|cols| cols.get(name)) + .cloned() + .ok_or_else(|| InterpError::Build(format!("Unknown column: {t}.{name}"))); + } + let mut found: Option<&Value> = None; + for cols in row.values() { + if let Some(v) = cols.get(name) { + if found.is_some() { + return Err(InterpError::Build(format!( + "Ambiguous column reference: {name}" + ))); + } + found = Some(v); + } + } + found + .cloned() + .ok_or_else(|| InterpError::Build(format!("Unknown column: {name}"))) +} + +fn eval_binary_op(op: BinOp, l: Value, r: Value) -> Result { + match op { + BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div | BinOp::Mod => arithmetic(op, l, r), + BinOp::Eq | BinOp::NotEq | BinOp::Lt | BinOp::Gt | BinOp::LtEq | BinOp::GtEq => { + comparison(op, l, r) + } + BinOp::And | BinOp::Or => logic(op, l, r), + BinOp::Concat => concat_op(l, r), + } +} + +/// `||` string concatenation. NULL-propagating (SQL/DataFusion semantics: any +/// NULL operand yields NULL, unlike CONCAT which skips NULLs). +fn concat_op(l: Value, r: Value) -> Result { + if matches!(l, Value::Null) || matches!(r, Value::Null) { + return Ok(Value::Null); + } + Ok(Value::Str(format!( + "{}{}", + display_value(&l), + display_value(&r) + ))) +} + +fn arithmetic(op: BinOp, l: Value, r: Value) -> Result { + if matches!(l, Value::Null) || matches!(r, Value::Null) { + return Ok(Value::Null); + } + match (l, r) { + (Value::Int(a), Value::Int(b)) => { + if matches!(op, BinOp::Div | BinOp::Mod) && b == 0 { + return Err(crate::datafusion::plan::InterpError::Eval( + "division by zero".to_string(), + )); + } + Ok(match op { + BinOp::Add => Value::Int(a + b), + BinOp::Sub => Value::Int(a - b), + BinOp::Mul => Value::Int(a * b), + BinOp::Div => Value::Int(a / b), + BinOp::Mod => Value::Int(a % b), + _ => unreachable!(), + }) + } + (a, b) => { + let af = as_f64(&a)?; + let bf = as_f64(&b)?; + Ok(match op { + BinOp::Add => Value::Float(af + bf), + BinOp::Sub => Value::Float(af - bf), + BinOp::Mul => Value::Float(af * bf), + BinOp::Div => Value::Float(af / bf), + BinOp::Mod => Value::Float(af % bf), + _ => unreachable!(), + }) + } + } +} + +fn as_f64(v: &Value) -> Result { + match v { + Value::Int(i) => Ok(*i as f64), + Value::Float(f) => Ok(*f), + other => Err(crate::datafusion::plan::InterpError::Eval(format!( + "Cannot use a {} value in an arithmetic expression", + type_name(other) + ))), + } +} + +fn comparison(op: BinOp, l: Value, r: Value) -> Result { + if matches!(l, Value::Null) || matches!(r, Value::Null) { + return Ok(Value::Null); + } + // Structs/lists support deep structural equality (DataFusion parity) but + // not ordering; route Eq/NotEq through the existing structural `PartialEq` + // on `Value` before falling into `compare_values`, which only handles + // scalars and would error on a container. + if matches!(op, BinOp::Eq | BinOp::NotEq) + && matches!( + (&l, &r), + (Value::Struct(_), _) + | (Value::List(_), _) + | (_, Value::Struct(_)) + | (_, Value::List(_)) + ) + { + let eq = l == r; + return Ok(Value::Bool(if op == BinOp::Eq { eq } else { !eq })); + } + let ordering = compare_values(&l, &r)?; + Ok(Value::Bool(match op { + BinOp::Eq => ordering == std::cmp::Ordering::Equal, + BinOp::NotEq => ordering != std::cmp::Ordering::Equal, + BinOp::Lt => ordering == std::cmp::Ordering::Less, + BinOp::Gt => ordering == std::cmp::Ordering::Greater, + BinOp::LtEq => ordering != std::cmp::Ordering::Greater, + BinOp::GtEq => ordering != std::cmp::Ordering::Less, + _ => unreachable!(), + })) +} + +fn compare_values(l: &Value, r: &Value) -> Result { + match (l, r) { + (Value::Int(a), Value::Int(b)) => Ok(a.cmp(b)), + (Value::Str(a), Value::Str(b)) => Ok(a.cmp(b)), + (Value::Bool(a), Value::Bool(b)) => Ok(a.cmp(b)), + (a, b) => { + let af = as_f64(a)?; + let bf = as_f64(b)?; + // DataFusion float ordering: NaN == NaN and NaN sorts greatest. + // Keep partial_cmp for the normal path (so 0.0 == -0.0) and only + // special-case NaN, which partial_cmp reports as incomparable. + Ok(af + .partial_cmp(&bf) + .unwrap_or_else(|| match (af.is_nan(), bf.is_nan()) { + (true, true) => std::cmp::Ordering::Equal, + (true, false) => std::cmp::Ordering::Greater, + (false, true) => std::cmp::Ordering::Less, + (false, false) => unreachable!("partial_cmp only fails on NaN"), + })) + } + } +} + +fn logic(op: BinOp, l: Value, r: Value) -> Result { + let lb = as_tribool(&l)?; + let rb = as_tribool(&r)?; + Ok(match op { + BinOp::And => match (lb, rb) { + (Some(false), _) | (_, Some(false)) => Value::Bool(false), + (Some(true), Some(true)) => Value::Bool(true), + _ => Value::Null, + }, + BinOp::Or => match (lb, rb) { + (Some(true), _) | (_, Some(true)) => Value::Bool(true), + (Some(false), Some(false)) => Value::Bool(false), + _ => Value::Null, + }, + _ => unreachable!(), + }) +} + +fn as_tribool(v: &Value) -> Result, crate::datafusion::plan::InterpError> { + match v { + Value::Bool(b) => Ok(Some(*b)), + Value::Null => Ok(None), + other => Err(crate::datafusion::plan::InterpError::Eval(format!( + "Expected a boolean expression, got a {} value", + type_name(other) + ))), + } +} + +fn eval_builtin(name: &str, args: Vec) -> Result { + use crate::datafusion::plan::InterpError; + + if matches!(name, "upper" | "lower" | "trim" | "substr" | "substring") + && args.iter().any(|a| matches!(a, Value::Null)) + { + return Ok(Value::Null); + } + + match name { + "upper" => Ok(Value::Str(as_str(&args, 0)?.to_uppercase())), + "lower" => Ok(Value::Str(as_str(&args, 0)?.to_lowercase())), + "trim" => Ok(Value::Str(as_str(&args, 0)?.trim().to_string())), + "concat" => { + let mut s = String::new(); + for a in &args { + if !matches!(a, Value::Null) { + s.push_str(&display_value(a)); + } + } + Ok(Value::Str(s)) + } + "abs" => match &args[0] { + Value::Int(i) => Ok(Value::Int(i.abs())), + Value::Float(f) => Ok(Value::Float(f.abs())), + Value::Null => Ok(Value::Null), + other => Err(InterpError::Eval(format!( + "ABS expects a number, got a {} value", + type_name(other) + ))), + }, + "round" => match &args[0] { + Value::Float(f) => Ok(Value::Float(f.round())), + // DataFusion's ROUND returns Float64 even for an integer arg. + Value::Int(i) => Ok(Value::Float(*i as f64)), + Value::Null => Ok(Value::Null), + other => Err(InterpError::Eval(format!( + "ROUND expects a number, got a {} value", + type_name(other) + ))), + }, + "substr" | "substring" => { + let s = as_str(&args, 0)?; + let start = as_i64(&args, 1)?; + let length = if args.len() > 2 { + Some(as_i64(&args, 2)?) + } else { + None + }; + Ok(Value::Str(substr(s, start, length))) + } + "coalesce" => Ok(args + .into_iter() + .find(|v| !matches!(v, Value::Null)) + .unwrap_or(Value::Null)), + "nullif" => { + if args.len() != 2 { + return Err(InterpError::Eval("NULLIF expects 2 arguments".to_string())); + } + // DataFusion coerces numerically (NULLIF(1, 1.0) -> NULL), so + // compare through the numeric-aware comparison, not Value's + // variant-tagged PartialEq. + let equal = matches!( + comparison(BinOp::Eq, args[0].clone(), args[1].clone())?, + Value::Bool(true) + ); + if equal { + Ok(Value::Null) + } else { + Ok(args[0].clone()) + } + } + other => Err(InterpError::Eval(format!("Unknown function: {other}"))), + } +} + +fn as_str(args: &[Value], idx: usize) -> Result<&str, crate::datafusion::plan::InterpError> { + match args.get(idx) { + Some(Value::Str(s)) => Ok(s.as_str()), + other => Err(crate::datafusion::plan::InterpError::Eval(format!( + "Expected a string argument at position {idx}, got {:?}", + other.map(type_name) + ))), + } +} + +fn as_i64(args: &[Value], idx: usize) -> Result { + match args.get(idx) { + Some(Value::Int(i)) => Ok(*i), + other => Err(crate::datafusion::plan::InterpError::Eval(format!( + "Expected an integer argument at position {idx}, got {:?}", + other.map(type_name) + ))), + } +} + +fn substr(s: &str, start: i64, length: Option) -> String { + // Postgres/DataFusion windowing: the 1-based window is [start, start+length); + // positions < 1 are consumed by the length. SUBSTR('hello',0,3) -> 'he', + // SUBSTR('hello',-2,5) -> 'he'. `start` clamps up to 1; the end is + // start+length-1 (0-based exclusive), both clamped into [0, len]. + let chars: Vec = s.chars().collect(); + let len = chars.len() as i64; + let begin = (start.max(1) - 1).clamp(0, len) as usize; + let end = match length { + Some(l) => (start + l - 1).clamp(0, len) as usize, + None => len as usize, + }; + let end = end.max(begin); + chars[begin..end].iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn struct_and_list_value_equality() { + let a = Value::Struct(vec![("x".into(), Value::Int(1))]); + let b = Value::Struct(vec![("x".into(), Value::Int(1))]); + let c = Value::List(vec![Value::Int(1), Value::Int(2)]); + assert_eq!(a, b); + assert_ne!(a, c); + assert_eq!(c.clone(), c); + } +} + +fn eval_cast(v: Value, target: CastType) -> Result { + use crate::datafusion::plan::InterpError; + + if matches!(v, Value::Null) { + return Ok(Value::Null); + } + Ok(match target { + CastType::Str => Value::Str(display_value(&v)), + CastType::Int => match v { + Value::Int(i) => Value::Int(i), + Value::Float(f) => Value::Int(f.trunc() as i64), + // No trim: DataFusion's Utf8->Int cast rejects surrounding + // whitespace (CAST(' 42 ' AS BIGINT) errors), so we must too. + Value::Str(s) => Value::Int( + s.parse::() + .map_err(|_| InterpError::Eval(format!("Cannot cast '{s}' to INT")))?, + ), + Value::Bool(b) => Value::Int(b as i64), + Value::Null | Value::Object(_) | Value::Struct(_) | Value::List(_) => { + return Err(InterpError::Eval( + "Cannot cast this value to INT".to_string(), + )) + } + }, + CastType::Float => match v { + Value::Int(i) => Value::Float(i as f64), + Value::Float(f) => Value::Float(f), + // No trim: match DataFusion, which rejects surrounding whitespace. + Value::Str(s) => Value::Float( + s.parse::() + .map_err(|_| InterpError::Eval(format!("Cannot cast '{s}' to FLOAT")))?, + ), + Value::Bool(b) => Value::Float(if b { 1.0 } else { 0.0 }), + Value::Null | Value::Object(_) | Value::Struct(_) | Value::List(_) => { + return Err(InterpError::Eval( + "Cannot cast this value to FLOAT".to_string(), + )) + } + }, + CastType::Bool => match v { + Value::Bool(b) => Value::Bool(b), + Value::Int(i) => Value::Bool(i != 0), + Value::Float(f) => Value::Bool(f != 0.0), + // DataFusion/Arrow's Utf8->Boolean accept set (case-insensitive); + // anything else errors, unlike the old "only 'true'" behavior. + Value::Str(s) => match s.to_ascii_lowercase().as_str() { + "true" | "t" | "yes" | "y" | "1" | "on" => Value::Bool(true), + "false" | "f" | "no" | "n" | "0" | "off" => Value::Bool(false), + _ => return Err(InterpError::Eval(format!("Cannot cast '{s}' to BOOLEAN"))), + }, + Value::Null | Value::Object(_) | Value::Struct(_) | Value::List(_) => { + return Err(InterpError::Eval( + "Cannot cast this value to BOOLEAN".to_string(), + )) + } + }, + }) +} diff --git a/src/datafusion/expr_build.rs b/src/datafusion/expr_build.rs new file mode 100644 index 0000000..8262101 --- /dev/null +++ b/src/datafusion/expr_build.rs @@ -0,0 +1,278 @@ +use sqlparser::ast::{ + Array, BinaryOperator, DataType, Expr as SqlExpr, Function, FunctionArg, FunctionArgExpr, + FunctionArguments, Ident, UnaryOperator, Value as SqlValue, +}; + +use crate::datafusion::expr::{BinOp, CastType, Expr, Value}; +use crate::datafusion::plan::InterpError; + +/// DataFusion identifier folding: an unquoted identifier folds to lowercase, a +/// double-quoted one stays case-exact. sqlparser keeps the unquoted text in +/// `.value` regardless, and the quoting in `.quote_style` -- so the fold key is +/// purely whether a quote style was present. Matches the oracle so `SELECT Age` +/// on a column named `Age` folds to `age` and misses. +pub fn fold_ident(ident: &Ident) -> String { + match ident.quote_style { + Some(_) => ident.value.clone(), + None => ident.value.to_lowercase(), + } +} + +pub fn convert_expr(e: &SqlExpr) -> Result { + match e { + SqlExpr::Identifier(ident) => Ok(Expr::Column { + table: None, + name: fold_ident(ident), + }), + // `a.b` is ambiguous at parse time -- it's either `table.column` or + // `struct_col.field`, and we don't know the relation-alias set here + // (that only exists once the row/static tables are supplied, in + // plan::validate_columns). Parse the first two parts as + // Column{table,name} (today's `table.column` shape) and layer any + // further dotted parts on top as FieldAccess (so `s.a.b` parses as + // field `b` of field `a` of column-or-table `s`). validate_expr + // rewrites the Column node into a FieldAccess when its `table` part + // turns out not to be a relation alias -- see plan.rs. + SqlExpr::CompoundIdentifier(parts) if parts.len() >= 2 => { + // Fold the column/field parts (`parts[1..]`); leave the leading + // qualifier (`parts[0]`) raw -- it names a relation (`__THIS__`/ + // generated), never a data column. ponytail: a real CamelCase table + // or struct-column qualifier won't fold like DataFusion here; not + // reachable today (qualifiers are always library-internal). Widen to + // fold parts[0] too if user-named CamelCase relations ever appear. + let mut expr = Expr::Column { + table: Some(parts[0].value.clone()), + name: fold_ident(&parts[1]), + }; + for part in &parts[2..] { + expr = Expr::FieldAccess { + base: Box::new(expr), + field: fold_ident(part), + }; + } + Ok(expr) + } + SqlExpr::Value(vws) => Ok(Expr::Literal(convert_literal(&vws.value)?)), + SqlExpr::Nested(inner) => convert_expr(inner), + SqlExpr::UnaryOp { + op: UnaryOperator::Not, + expr, + } => Ok(Expr::Not(Box::new(convert_expr(expr)?))), + // Unary minus: lower to `0 - x`, reusing Sub's numeric promotion + // (int stays int, float stays float) to match DataFusion. + SqlExpr::UnaryOp { + op: UnaryOperator::Minus, + expr, + } => Ok(Expr::BinaryOp { + op: BinOp::Sub, + left: Box::new(Expr::Literal(Value::Int(0))), + right: Box::new(convert_expr(expr)?), + }), + SqlExpr::BinaryOp { left, op, right } => { + let bin_op = convert_binary_operator(op)?; + Ok(Expr::BinaryOp { + op: bin_op, + left: Box::new(convert_expr(left)?), + right: Box::new(convert_expr(right)?), + }) + } + SqlExpr::Function(func) => convert_function(func), + SqlExpr::Array(Array { elem, .. }) => Ok(Expr::List( + elem.iter().map(convert_expr).collect::>()?, + )), + SqlExpr::Cast { + expr, data_type, .. + } => Ok(Expr::Cast { + expr: Box::new(convert_expr(expr)?), + target: convert_cast_type(data_type)?, + }), + // sqlparser 0.62 parses SUBSTR/SUBSTRING into a dedicated AST node + // rather than a generic Function call; normalize it to + // Expr::Function("substr", ...) so eval_builtin's dispatch handles + // both call syntaxes uniformly. + SqlExpr::Substring { + expr, + substring_from, + substring_for, + .. + } => { + if substring_from.is_none() && substring_for.is_none() { + return Err(InterpError::Build( + "SUBSTRING requires FROM and/or FOR".to_string(), + )); + } + let mut args = vec![convert_expr(expr)?]; + match substring_from { + Some(from) => args.push(convert_expr(from)?), + // SQL-92: SUBSTRING(expr FOR n) with no FROM means "the + // first n characters", equivalent to + // SUBSTRING(expr FROM 1 FOR n). + None => args.push(Expr::Literal(Value::Int(1))), + } + if let Some(for_) = substring_for { + args.push(convert_expr(for_)?); + } + Ok(Expr::Function { + name: "substr".to_string(), + args, + }) + } + // Likewise TRIM(expr) is a dedicated AST node. Only the plain form + // (no BOTH/LEADING/TRAILING side, no explicit trim characters) maps + // onto eval_builtin's "trim" (Rust's str::trim, whitespace only). + SqlExpr::Trim { + expr, + trim_where: None, + trim_what: None, + trim_characters: None, + .. + } => Ok(Expr::Function { + name: "trim".to_string(), + args: vec![convert_expr(expr)?], + }), + SqlExpr::Case { + operand, + conditions, + else_result, + .. + } => { + let mut arms = Vec::with_capacity(conditions.len()); + for when in conditions { + // Simple form: `CASE WHEN ...` normalizes each arm's + // condition to `operand = v`, matching DataFusion/codegen. A NULL + // operand/value makes the `=` NULL, so the arm doesn't match. + let cond = match operand { + Some(op) => Expr::BinaryOp { + op: BinOp::Eq, + left: Box::new(convert_expr(op)?), + right: Box::new(convert_expr(&when.condition)?), + }, + None => convert_expr(&when.condition)?, + }; + arms.push((cond, convert_expr(&when.result)?)); + } + let default = match else_result { + Some(e) => Some(Box::new(convert_expr(e)?)), + None => None, + }; + Ok(Expr::Case { arms, default }) + } + _ => Err(InterpError::Build(format!("Unsupported expression: {e}"))), + } +} + +fn convert_function(func: &Function) -> Result { + let name = func.name.to_string().to_lowercase(); + let args = match &func.args { + FunctionArguments::List(list) => list + .args + .iter() + .map(convert_function_arg) + .collect::, _>>()?, + FunctionArguments::None => Vec::new(), + FunctionArguments::Subquery(_) => { + return Err(InterpError::Build(format!( + "Subquery arguments are not supported in function: {name}" + ))) + } + }; + if name == "named_struct" { + if args.len() % 2 != 0 { + return Err(InterpError::Build( + "named_struct expects an even number of arguments (key, value, ...)".to_string(), + )); + } + let mut fields = Vec::with_capacity(args.len() / 2); + let mut it = args.into_iter(); + while let (Some(key), Some(value)) = (it.next(), it.next()) { + let Expr::Literal(Value::Str(key)) = key else { + return Err(InterpError::Build( + "named_struct field names must be string literals".to_string(), + )); + }; + fields.push((key, value)); + } + return Ok(Expr::Struct(fields)); + } + if name == "struct" { + let fields = args + .into_iter() + .enumerate() + .map(|(i, e)| (format!("c{i}"), e)) + .collect(); + return Ok(Expr::Struct(fields)); + } + Ok(Expr::Function { name, args }) +} + +fn convert_function_arg(arg: &FunctionArg) -> Result { + match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => convert_expr(e), + _ => Err(InterpError::Build( + "Only plain positional function arguments are supported".to_string(), + )), + } +} + +fn convert_cast_type(dt: &DataType) -> Result { + let name = dt.to_string().to_uppercase(); + if name.starts_with("VARCHAR") + || name.starts_with("TEXT") + || name.starts_with("STRING") + || name.starts_with("CHAR") + { + Ok(CastType::Str) + } else if name.starts_with("BIGINT") || name.starts_with("INT") { + Ok(CastType::Int) + } else if name.starts_with("DOUBLE") || name.starts_with("FLOAT") || name.starts_with("REAL") { + Ok(CastType::Float) + } else if name.starts_with("BOOL") { + Ok(CastType::Bool) + } else { + Err(InterpError::Build(format!( + "Unsupported CAST target type: {name}" + ))) + } +} + +fn convert_literal(v: &SqlValue) -> Result { + match v { + SqlValue::Null => Ok(Value::Null), + SqlValue::Boolean(b) => Ok(Value::Bool(*b)), + SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => { + Ok(Value::Str(s.clone())) + } + SqlValue::Number(text, _) => { + if text.contains('.') || text.to_lowercase().contains('e') { + text.parse::() + .map(Value::Float) + .map_err(|_| InterpError::Build(format!("Invalid numeric literal: {text}"))) + } else { + text.parse::() + .map(Value::Int) + .map_err(|_| InterpError::Build(format!("Invalid numeric literal: {text}"))) + } + } + other => Err(InterpError::Build(format!("Unsupported literal: {other}"))), + } +} + +fn convert_binary_operator(op: &BinaryOperator) -> Result { + match op { + BinaryOperator::Plus => Ok(BinOp::Add), + BinaryOperator::Minus => Ok(BinOp::Sub), + BinaryOperator::Multiply => Ok(BinOp::Mul), + BinaryOperator::Divide => Ok(BinOp::Div), + BinaryOperator::Modulo => Ok(BinOp::Mod), + BinaryOperator::Eq => Ok(BinOp::Eq), + BinaryOperator::NotEq => Ok(BinOp::NotEq), + BinaryOperator::Lt => Ok(BinOp::Lt), + BinaryOperator::Gt => Ok(BinOp::Gt), + BinaryOperator::LtEq => Ok(BinOp::LtEq), + BinaryOperator::GtEq => Ok(BinOp::GtEq), + BinaryOperator::And => Ok(BinOp::And), + BinaryOperator::Or => Ok(BinOp::Or), + BinaryOperator::StringConcat => Ok(BinOp::Concat), + other => Err(InterpError::Build(format!("Unsupported operator: {other}"))), + } +} diff --git a/src/datafusion/mod.rs b/src/datafusion/mod.rs new file mode 100644 index 0000000..e753a1a --- /dev/null +++ b/src/datafusion/mod.rs @@ -0,0 +1,355 @@ +//! The DataFusion-semantics interpreter: `InferFn`. +//! +//! DataFusion is the oracle this engine is differentially tested against, so +//! every semantic choice here (identifier folding, NULL propagation, numeric +//! coercion, float ordering) mirrors DataFusion rather than standard SQL. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::*; +use pyo3::types::PyDict; + +pub mod expr; +pub mod expr_build; +pub mod plan; +pub mod types; + +use crate::lookup::{self, LookupIndex}; +use crate::schema; +use crate::value::Value; +use expr::Expr; +use plan::Plan; + +#[pyclass] +pub struct InferFn { + plan: Plan, + lookups: HashMap, + row_table_columns: HashMap>, + row_schemas: HashMap, + #[pyo3(get)] + output_model: Py, +} + +fn synthesize_output_model( + py: Python<'_>, + projection: &[(String, Expr)], + schemas: &HashMap, +) -> PyResult> { + let pydantic = PyModule::import(py, "pydantic")?; + let create_model = pydantic.getattr("create_model")?; + let builtins = PyModule::import(py, "builtins")?; + let ellipsis = builtins.getattr("Ellipsis")?; + + let kwargs = PyDict::new(py); + for (alias, expr) in projection { + let ft = types::infer_type(expr, schemas)?; + let py_type = schema::field_type_to_python(py, ft)?; + kwargs.set_item(alias, (py_type, &ellipsis))?; + } + let model = create_model.call(("OutputRow",), Some(&kwargs))?; + Ok(model.unbind()) +} + +/// Validates a caller-supplied `output_model` against the query's inferred +/// output shape. Only rejects what can be *proven* wrong at build time: a +/// missing/extra field vs. the projection's aliases, or a base-type mismatch +/// `types::compatible` can't excuse. Nullability mismatches are never a +/// build-time error — see `types::compatible`'s docs and Task 2's +/// deliberately conservative `infer_type`. +fn validate_output_model( + py: Python<'_>, + model: &Py, + projection: &[(String, Expr)], + schemas: &HashMap, +) -> PyResult<()> { + let declared_schema = schema::from_pydantic_model(py, model)?; + + let mut projected_aliases: HashSet = HashSet::new(); + for (alias, expr) in projection { + projected_aliases.insert(alias.clone()); + let declared = declared_schema.get(alias).ok_or_else(|| { + plan::InterpError::Build(format!( + "output_model is missing field '{alias}' produced by the query" + )) + })?; + let inferred = types::infer_type(expr, schemas)?; + if !crate::types::compatible(&inferred.base, &declared.base) { + return Err(plan::InterpError::Build(format!( + "output_model field '{alias}' is declared as a type incompatible with the \ + query's inferred output ({:?} vs declared {:?})", + inferred.base, declared.base + )) + .into()); + } + } + + let declared_fields: HashSet = declared_schema.keys().cloned().collect(); + let extra: Vec<&String> = declared_fields.difference(&projected_aliases).collect(); + if !extra.is_empty() { + return Err(plan::InterpError::Build(format!( + "output_model declares fields not produced by the query: {extra:?}" + )) + .into()); + } + Ok(()) +} + +/// A `transformers` registry entry resolved at build time: the fitted object, +/// its `feature_names_in_` (input alignment order), and the caller-declared +/// output field list (marshalling target, order significant). +struct ResolvedTransformer { + obj: std::sync::Arc>, + input_features: Vec, + output_fields: Vec<(String, types::FieldType)>, +} + +/// Reads `obj.feature_names_in_.tolist()`. Absence is a clear build error: +/// the object was fit on bare arrays, so we cannot align inputs by name. +fn read_feature_names_in(py: Python<'_>, obj: &Py) -> Result, plan::InterpError> { + let bound = obj.bind(py); + let attr = bound.getattr("feature_names_in_").map_err(|_| { + plan::InterpError::Build( + "transformer has no `feature_names_in_`; fit it on named data \ + (e.g. a pandas DataFrame) so input columns can be aligned by name" + .to_string(), + ) + })?; + attr.call_method0("tolist") + .and_then(|l| l.extract::>()) + .map_err(|e| plan::InterpError::Build(format!("could not read feature_names_in_: {e}"))) +} + +/// Rewrites every `Expr::Function` whose name is a registered transformer into +/// an `Expr::Transform`, recursing through the whole expression tree so a +/// transformer call nested inside arithmetic is still resolved. A transformer +/// call must have exactly one argument. +fn resolve_transformers( + expr: Expr, + resolved: &HashMap, +) -> Result { + match expr { + Expr::Function { name, args } => { + let mut new_args = Vec::with_capacity(args.len()); + for a in args { + new_args.push(resolve_transformers(a, resolved)?); + } + if let Some(rt) = resolved.get(&name) { + if new_args.len() != 1 { + return Err(plan::InterpError::Build(format!( + "transformer '{name}' takes exactly one argument, got {}", + new_args.len() + ))); + } + let arg = new_args.into_iter().next().unwrap(); + return Ok(Expr::Transform { + obj: rt.obj.clone(), + input_features: rt.input_features.clone(), + output_fields: rt.output_fields.clone(), + arg: Box::new(arg), + }); + } + Ok(Expr::Function { name, args: new_args }) + } + Expr::BinaryOp { op, left, right } => Ok(Expr::BinaryOp { + op, + left: Box::new(resolve_transformers(*left, resolved)?), + right: Box::new(resolve_transformers(*right, resolved)?), + }), + Expr::Not(inner) => Ok(Expr::Not(Box::new(resolve_transformers(*inner, resolved)?))), + Expr::Cast { expr, target } => Ok(Expr::Cast { + expr: Box::new(resolve_transformers(*expr, resolved)?), + target, + }), + Expr::Struct(fields) => { + let mut out = Vec::with_capacity(fields.len()); + for (k, v) in fields { + out.push((k, resolve_transformers(v, resolved)?)); + } + Ok(Expr::Struct(out)) + } + Expr::List(items) => { + let mut out = Vec::with_capacity(items.len()); + for e in items { + out.push(resolve_transformers(e, resolved)?); + } + Ok(Expr::List(out)) + } + Expr::FieldAccess { base, field } => Ok(Expr::FieldAccess { + base: Box::new(resolve_transformers(*base, resolved)?), + field, + }), + Expr::Case { arms, default } => { + let mut new_arms = Vec::with_capacity(arms.len()); + for (cond, result) in arms { + new_arms.push(( + resolve_transformers(cond, resolved)?, + resolve_transformers(result, resolved)?, + )); + } + let default = match default { + Some(d) => Some(Box::new(resolve_transformers(*d, resolved)?)), + None => None, + }; + Ok(Expr::Case { arms: new_arms, default }) + } + // Column, Literal, and an already-built Transform pass through. + other => Ok(other), + } +} + +#[pymethods] +impl InferFn { + #[new] + #[pyo3(signature = (sql, row_tables, static_tables, output_model=None, transformers=None))] + fn new( + py: Python<'_>, + sql: String, + row_tables: HashMap>, + static_tables: HashMap>, + output_model: Option>, + transformers: Option, Py)>>, + ) -> PyResult { + let raw_plan = plan::build_plan(&sql)?; + let row_table_names: HashSet = row_tables.keys().cloned().collect(); + let static_table_names: HashSet = static_tables.keys().cloned().collect(); + let (mut optimized_plan, specs) = plan::optimize(raw_plan, &static_table_names)?; + + let mut row_schemas = HashMap::new(); + for (name, model_class) in &row_tables { + row_schemas.insert(name.clone(), schema::from_pydantic_model(py, model_class)?); + } + let mut static_schemas = HashMap::new(); + for (name, table_obj) in &static_tables { + static_schemas.insert(name.clone(), schema::from_arrow_table(py, table_obj)?); + } + + let column_validation = plan::validate_columns( + &mut optimized_plan, + &row_table_names, + &row_schemas, + &static_schemas, + )?; + + // Resolve registered transformers AFTER column validation (so + // validate_columns sees the plain Expr::Function and its named_struct + // arg -- no Transform arm needed there) and BEFORE output-model + // synthesis (so infer_type sees Expr::Transform and returns the + // declared output struct type). Reads feature_names_in_ here (with py). + let transformers = transformers.unwrap_or_default(); + if !transformers.is_empty() { + let mut resolved: HashMap = HashMap::new(); + for (name, (obj, out_schema_obj)) in &transformers { + let input_features = read_feature_names_in(py, obj)?; + let output_fields = schema::arrow_schema_to_ordered_fields(py, out_schema_obj)?; + resolved.insert( + name.clone(), + ResolvedTransformer { + obj: std::sync::Arc::new(obj.clone_ref(py)), + input_features, + output_fields, + }, + ); + } + let projection = std::mem::take(&mut optimized_plan.projection); + let mut new_projection = Vec::with_capacity(projection.len()); + for (alias, expr) in projection { + new_projection.push((alias, resolve_transformers(expr, &resolved)?)); + } + optimized_plan.projection = new_projection; + } + + let output_model = match output_model { + Some(supplied) => { + validate_output_model( + py, + &supplied, + &optimized_plan.projection, + &column_validation.effective_schemas, + )?; + supplied + } + None => synthesize_output_model( + py, + &optimized_plan.projection, + &column_validation.effective_schemas, + )?, + }; + + let mut lookups = HashMap::new(); + for spec in specs { + let table_obj = static_tables.get(&spec.static_table).ok_or_else(|| { + plan::InterpError::Build(format!( + "SQL references static table '{}' that was not provided", + spec.static_table + )) + })?; + let index = lookup::build_index(py, table_obj, &spec.key_columns)?; + lookups.insert(spec.static_table, index); + } + + Ok(InferFn { + plan: optimized_plan, + lookups, + row_table_columns: column_validation.row_table_columns, + row_schemas, + output_model, + }) + } + + #[pyo3(signature = (tables=None, **kwargs))] + fn infer( + &self, + py: Python<'_>, + tables: Option>>>, + kwargs: Option>, + ) -> PyResult>> { + let mut merged: HashMap>> = tables.unwrap_or_default(); + if let Some(kwargs) = kwargs { + for (k, v) in kwargs.iter() { + let key: String = k.extract()?; + let rows: Vec> = v.extract()?; + merged.insert(key, rows); + } + } + + let empty: Vec = Vec::new(); + let mut value_tables: HashMap>> = HashMap::new(); + for (table, rows) in &merged { + let columns = self.row_table_columns.get(table).unwrap_or(&empty); + let schema = self.row_schemas.get(table); + let mut out_rows = Vec::with_capacity(rows.len()); + for row_obj in rows { + let bound = row_obj.bind(py); + let mut row: HashMap = HashMap::new(); + for col in columns { + let attr = bound.getattr(col.as_str()).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "Row for table '{table}' is missing attribute '{col}': {e}" + )) + })?; + let value = match schema.and_then(|s| s.get(col)) { + Some(ft) => Value::from_pyobject_typed(&attr, &ft.base)?, + None => Value::from_pyobject(&attr)?, + }; + row.insert(col.clone(), value); + } + out_rows.push(row); + } + value_tables.insert(table.clone(), out_rows); + } + + let result_rows = plan::execute(&self.plan, &value_tables, &self.lookups)?; + + let output_model = self.output_model.bind(py); + let mut out = Vec::with_capacity(result_rows.len()); + for row in &result_rows { + let dict = PyDict::new(py); + for (k, v) in row { + dict.set_item(k, v.to_pyobject(py)?)?; + } + let instance = output_model.call_method1("model_validate", (dict,))?; + out.push(instance.unbind()); + } + Ok(out) + } +} diff --git a/src/datafusion/plan.rs b/src/datafusion/plan.rs new file mode 100644 index 0000000..b22b9c6 --- /dev/null +++ b/src/datafusion/plan.rs @@ -0,0 +1,1239 @@ +use std::collections::{HashMap, HashSet}; + +use sqlparser::ast::{ + BinaryOperator, Expr as SqlExpr, Join, JoinConstraint, JoinOperator, SelectItem, SetExpr, + Statement, TableFactor, TableWithJoins, +}; +use sqlparser::dialect::GenericDialect; +use sqlparser::parser::Parser; + +use crate::datafusion::expr::{Expr, Value}; +use crate::datafusion::types::Schema; + +// Re-exported so the engine-internal paths (`crate::datafusion::plan::InterpError`) +// keep resolving now that the error type is shared. +pub use crate::error::InterpError; + +pub type Row = HashMap>; + +pub enum RelNode { + TableScan { + table: String, + }, + Filter { + input: Box, + predicate: Expr, + }, + CrossJoin { + left: Box, + right: Box, + }, + Join { + left: Box, + right: Box, + on: Vec<(Expr, Expr)>, + outer: bool, + }, + SubqueryAlias { + input: Box, + alias: String, + }, + LookupJoin { + input: Box, + table: String, + keys: Vec, + outer: bool, + }, + /// Row-multiplying `unnest(list)`: evaluates `list_expr` per input row and + /// emits one output row per list element, binding it to `output_col`. A + /// NULL or empty list emits zero rows (matches DataFusion). + Unnest { + input: Box, + list_expr: Expr, + output_col: String, + }, +} + +/// Synthetic outer key under which `RelNode::Unnest` binds its emitted element +/// column — in the runtime `Row` and in the validation-time effective-schema +/// map. The NUL byte can never collide with a real SQL table/alias identifier. +const UNNEST_KEY: &str = "\0unnest"; + +pub struct Plan { + pub projection: Vec<(String, Expr)>, + pub input: RelNode, +} + +pub struct LookupSpec { + pub static_table: String, + pub key_columns: Vec, +} + +pub fn build_plan(sql: &str) -> Result { + let dialect = GenericDialect {}; + let statements = Parser::parse_sql(&dialect, sql) + .map_err(|e| InterpError::Build(format!("SQL parse error: {e}")))?; + + if statements.len() != 1 { + return Err(InterpError::Build( + "Expected exactly one SQL statement".to_string(), + )); + } + + let select = match &statements[0] { + Statement::Query(query) => match query.body.as_ref() { + SetExpr::Select(select) => select.as_ref(), + _ => { + return Err(InterpError::Build( + "Only SELECT queries are supported".to_string(), + )) + } + }, + _ => { + return Err(InterpError::Build( + "Only SELECT queries are supported".to_string(), + )) + } + }; + + let mut input = build_from(&select.from)?; + if let Some(predicate) = &select.selection { + input = RelNode::Filter { + input: Box::new(input), + predicate: crate::datafusion::expr_build::convert_expr(predicate)?, + }; + } + let projection = build_projection(&select.projection)?; + + Ok(Plan { projection, input }) +} + +fn build_from(from: &[TableWithJoins]) -> Result { + if from.is_empty() { + return Err(InterpError::Build("FROM clause is required".to_string())); + } + let mut seen_tables: HashSet = HashSet::new(); + let mut node = build_table_with_joins(&from[0], &mut seen_tables)?; + for twj in &from[1..] { + let right = build_table_with_joins(twj, &mut seen_tables)?; + node = RelNode::CrossJoin { + left: Box::new(node), + right: Box::new(right), + }; + } + Ok(node) +} + +fn build_table_with_joins( + twj: &TableWithJoins, + seen_tables: &mut HashSet, +) -> Result { + let mut node = build_table_factor(&twj.relation, seen_tables)?; + for join in &twj.joins { + node = build_join(node, join, seen_tables)?; + } + Ok(node) +} + +fn build_join( + left: RelNode, + join: &Join, + seen_tables: &mut HashSet, +) -> Result { + let right = build_table_factor(&join.relation, seen_tables)?; + match &join.join_operator { + JoinOperator::Join(constraint) | JoinOperator::Inner(constraint) => { + let on_expr = require_on(constraint)?; + let on = extract_equality_keys(on_expr)?; + Ok(RelNode::Join { + left: Box::new(left), + right: Box::new(right), + on, + outer: false, + }) + } + JoinOperator::Left(constraint) | JoinOperator::LeftOuter(constraint) => { + let on_expr = require_on(constraint)?; + let on = extract_equality_keys(on_expr)?; + Ok(RelNode::Join { + left: Box::new(left), + right: Box::new(right), + on, + outer: true, + }) + } + JoinOperator::CrossJoin(_) => Ok(RelNode::CrossJoin { + left: Box::new(left), + right: Box::new(right), + }), + other => Err(InterpError::Build(format!( + "Unsupported JOIN type: {other:?} — only inner JOIN ... ON and CROSS JOIN are supported" + ))), + } +} + +fn build_table_factor( + factor: &TableFactor, + seen_tables: &mut HashSet, +) -> Result { + match factor { + TableFactor::Table { name, alias, .. } => { + let table = name.to_string(); + // Track the EFFECTIVE output name (alias if present, else the real + // table name) — this is the key each relation's Row is stored + // under, so a collision here (whether from a true self-join like + // `FROM a JOIN a ON ...` or an alias collision like + // `FROM a JOIN b AS a ON ...`) would cause one side's data to + // silently overwrite the other's during row merging. + let effective_name = match &alias { + Some(a) => a.name.value.clone(), + None => table.clone(), + }; + if !seen_tables.insert(effective_name.clone()) { + return Err(InterpError::Build(format!( + "table '{effective_name}' is referenced more than once in FROM/JOIN — \ + self-joins and alias collisions are not supported" + ))); + } + let scan = RelNode::TableScan { table }; + Ok(match alias { + Some(a) => RelNode::SubqueryAlias { + input: Box::new(scan), + alias: a.name.value.clone(), + }, + None => scan, + }) + } + _ => Err(InterpError::Build("Unsupported FROM clause".to_string())), + } +} + +fn require_on(constraint: &JoinConstraint) -> Result<&SqlExpr, InterpError> { + match constraint { + JoinConstraint::On(e) => Ok(e), + _ => Err(InterpError::Build( + "JOIN requires an ON condition".to_string(), + )), + } +} + +fn extract_equality_keys(expr: &SqlExpr) -> Result, InterpError> { + match expr { + SqlExpr::BinaryOp { + left, + op: BinaryOperator::And, + right, + } => { + let mut pairs = extract_equality_keys(left)?; + pairs.extend(extract_equality_keys(right)?); + Ok(pairs) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::Eq, + right, + } => Ok(vec![( + crate::datafusion::expr_build::convert_expr(left)?, + crate::datafusion::expr_build::convert_expr(right)?, + )]), + _ => Err(InterpError::Build( + "JOIN ON condition must be an equality, or an AND of equalities, between columns" + .to_string(), + )), + } +} + +fn build_projection(items: &[SelectItem]) -> Result, InterpError> { + let mut out = Vec::new(); + for item in items { + match item { + SelectItem::UnnamedExpr(e) => { + let name = column_name(e)?; + out.push((name, crate::datafusion::expr_build::convert_expr(e)?)); + } + SelectItem::ExprWithAlias { expr, alias } => { + // The output alias folds like any identifier (`AS Foo` -> `foo`). + out.push(( + crate::datafusion::expr_build::fold_ident(alias), + crate::datafusion::expr_build::convert_expr(expr)?, + )); + } + _ => return Err(InterpError::Build("Unsupported SELECT item".to_string())), + } + } + Ok(out) +} + +fn column_name(e: &SqlExpr) -> Result { + match e { + SqlExpr::Identifier(ident) => Ok(crate::datafusion::expr_build::fold_ident(ident)), + SqlExpr::CompoundIdentifier(parts) => Ok(parts + .last() + .map(crate::datafusion::expr_build::fold_ident) + .unwrap_or_default()), + // `unnest(struct_expr)` expands into per-field columns during + // validate_columns (once the arg's type is known), which replaces + // this placeholder name entirely. Only reachable unaliased here + // because DataFusion allows it; other bare function calls still + // require an alias below. + SqlExpr::Function(func) if func.name.to_string().eq_ignore_ascii_case("unnest") => { + Ok("unnest".to_string()) + } + _ => Err(InterpError::Build( + "Expression in SELECT list needs an alias (AS name)".to_string(), + )), + } +} + +/// Walks a built `Plan`, rewriting any `Join` node where exactly one side is +/// a scan of a table named in `static_tables` into a `RelNode::LookupJoin`. +/// Returns an error if both sides of a `Join` are static tables (a +/// static-to-static join isn't a lookup and isn't supported). +pub fn optimize( + plan: Plan, + static_tables: &HashSet, +) -> Result<(Plan, Vec), InterpError> { + let mut specs = Vec::new(); + let input = optimize_rel(plan.input, static_tables, &mut specs)?; + Ok(( + Plan { + projection: plan.projection, + input, + }, + specs, + )) +} + +fn optimize_rel( + node: RelNode, + static_tables: &HashSet, + specs: &mut Vec, +) -> Result { + match node { + RelNode::Join { + left, + right, + on, + outer, + } => { + let left = optimize_rel(*left, static_tables, specs)?; + let right = optimize_rel(*right, static_tables, specs)?; + let left_static = scan_table_name(&left).filter(|t| static_tables.contains(*t)); + let right_static = scan_table_name(&right).filter(|t| static_tables.contains(*t)); + match (left_static, right_static) { + (Some(_), Some(_)) => Err(InterpError::Build( + "Joining two static tables together is not supported".to_string(), + )), + (None, Some(table)) => { + let table = table.to_string(); + let (keys, key_columns) = split_keys(&on, &table)?; + specs.push(LookupSpec { + static_table: table.clone(), + key_columns, + }); + Ok(RelNode::LookupJoin { + input: Box::new(left), + table, + keys, + outer, + }) + } + (Some(table), None) => { + let table = table.to_string(); + let (keys, key_columns) = split_keys(&on, &table)?; + specs.push(LookupSpec { + static_table: table.clone(), + key_columns, + }); + Ok(RelNode::LookupJoin { + input: Box::new(right), + table, + keys, + outer, + }) + } + (None, None) => { + if outer { + return Err(InterpError::Build( + "LEFT JOIN is only supported against a static lookup table".to_string(), + )); + } + Ok(RelNode::Join { + left: Box::new(left), + right: Box::new(right), + on, + outer, + }) + } + } + } + RelNode::CrossJoin { left, right } => Ok(RelNode::CrossJoin { + left: Box::new(optimize_rel(*left, static_tables, specs)?), + right: Box::new(optimize_rel(*right, static_tables, specs)?), + }), + RelNode::Filter { input, predicate } => Ok(RelNode::Filter { + input: Box::new(optimize_rel(*input, static_tables, specs)?), + predicate, + }), + RelNode::SubqueryAlias { input, alias } => Ok(RelNode::SubqueryAlias { + input: Box::new(optimize_rel(*input, static_tables, specs)?), + alias, + }), + other => Ok(other), + } +} + +fn scan_table_name(node: &RelNode) -> Option<&str> { + match node { + RelNode::TableScan { table } => Some(table), + RelNode::SubqueryAlias { input, .. } => scan_table_name(input), + _ => None, + } +} + +/// The qualifier (`table` part) of a plain `Expr::Column`, or `None` for +/// anything else (unqualified column, literal, expression, ...). +fn column_qualifier(e: &Expr) -> Option<&str> { + match e { + Expr::Column { table: Some(t), .. } => Some(t.as_str()), + _ => None, + } +} + +/// Splits each ON-clause equality pair into (the static table's key column +/// name, the row-side expression to evaluate it against). +/// +/// The ON clause's tuple order reflects how the equality was *written* +/// (`a = b` vs `b = a`), which is independent of which side of the JOIN is +/// structurally left/right in the FROM clause — so this identifies the +/// static side per-pair by matching each operand's column qualifier against +/// `static_table`, rather than assuming a fixed position. +fn split_keys( + on: &[(Expr, Expr)], + static_table: &str, +) -> Result<(Vec, Vec), InterpError> { + let mut row_side_keys = Vec::new(); + let mut static_col_names = Vec::new(); + for (l, r) in on { + let static_expr = match (column_qualifier(l), column_qualifier(r)) { + (Some(t), _) if t == static_table => l, + (_, Some(t)) if t == static_table => r, + _ => { + return Err(InterpError::Build(format!( + "JOIN ON keys against static table '{static_table}' must reference \ + the static table's columns by name (e.g. {static_table}.col)" + ))) + } + }; + let row_expr = if std::ptr::eq(static_expr, l) { r } else { l }; + match static_expr { + Expr::Column { name, .. } => static_col_names.push(name.clone()), + _ => { + return Err(InterpError::Build(format!( + "JOIN ON keys against static table '{static_table}' must be plain columns" + ))) + } + } + row_side_keys.push(row_expr.clone()); + } + Ok((row_side_keys, static_col_names)) +} + +pub fn execute( + plan: &Plan, + tables: &HashMap>>, + lookups: &HashMap, +) -> Result>, InterpError> { + let rows = execute_rel(&plan.input, tables, lookups)?; + let mut out = Vec::with_capacity(rows.len()); + for row in &rows { + let mut result = HashMap::new(); + for (alias, e) in &plan.projection { + result.insert(alias.clone(), crate::datafusion::expr::eval(e, row)?); + } + out.push(result); + } + Ok(out) +} + +fn execute_rel( + node: &RelNode, + tables: &HashMap>>, + lookups: &HashMap, +) -> Result, InterpError> { + match node { + RelNode::TableScan { table } => { + let flat_rows = tables.get(table).ok_or_else(|| { + InterpError::Build(format!("Unknown table in FROM clause: {table}")) + })?; + Ok(flat_rows + .iter() + .map(|r| { + let mut row = Row::new(); + row.insert(table.clone(), r.clone()); + row + }) + .collect()) + } + RelNode::Filter { input, predicate } => { + let rows = execute_rel(input, tables, lookups)?; + let mut out = Vec::new(); + for row in rows { + if let Value::Bool(true) = crate::datafusion::expr::eval(predicate, &row)? { + out.push(row); + } + } + Ok(out) + } + RelNode::CrossJoin { left, right } => { + let left_rows = execute_rel(left, tables, lookups)?; + let right_rows = execute_rel(right, tables, lookups)?; + let mut out = Vec::with_capacity(left_rows.len() * right_rows.len()); + for l in &left_rows { + for r in &right_rows { + let mut merged = l.clone(); + merged.extend(r.clone()); + out.push(merged); + } + } + Ok(out) + } + RelNode::Join { + left, right, on, .. + } => { + let left_rows = execute_rel(left, tables, lookups)?; + let right_rows = execute_rel(right, tables, lookups)?; + let mut out = Vec::new(); + for l in &left_rows { + for r in &right_rows { + let mut merged = l.clone(); + merged.extend(r.clone()); + let mut all_match = true; + for (le, re) in on { + let lv = crate::datafusion::expr::eval(le, &merged)?; + let rv = crate::datafusion::expr::eval(re, &merged)?; + if matches!(lv, Value::Null) || matches!(rv, Value::Null) || lv != rv { + all_match = false; + break; + } + } + if all_match { + out.push(merged); + } + } + } + Ok(out) + } + RelNode::SubqueryAlias { input, alias } => { + let rows = execute_rel(input, tables, lookups)?; + Ok(rows + .into_iter() + .map(|row| { + let inner = row.into_values().next().unwrap_or_default(); + let mut renamed = Row::new(); + renamed.insert(alias.clone(), inner); + renamed + }) + .collect()) + } + RelNode::LookupJoin { + input, + table, + keys, + outer, + } => { + let rows = execute_rel(input, tables, lookups)?; + let index = lookups.get(table).ok_or_else(|| { + InterpError::Build(format!("No lookup index built for table: {table}")) + })?; + let mut out = Vec::with_capacity(rows.len()); + for mut row in rows { + let key: Vec = keys + .iter() + .map(|k| crate::datafusion::expr::eval(k, &row)) + .collect::>()?; + match index.index.get(&key) { + Some(hit) => { + row.insert(table.clone(), hit.clone()); + } + None if *outer => { + let null_row: HashMap = index + .value_columns + .iter() + .map(|c| (c.clone(), Value::Null)) + .collect(); + row.insert(table.clone(), null_row); + } + None => { + let key_repr: Vec = + key.iter().map(crate::datafusion::expr::display_value).collect(); + return Err(InterpError::MissingKey(format!( + "No row in static table '{table}' matches key ({})", + key_repr.join(", ") + ))); + } + } + out.push(row); + } + Ok(out) + } + RelNode::Unnest { + input, + list_expr, + output_col, + } => { + let rows = execute_rel(input, tables, lookups)?; + let mut out = Vec::new(); + for row in rows { + match crate::datafusion::expr::eval(list_expr, &row)? { + Value::List(items) => { + // An empty list falls out here as zero iterations. + for item in items { + let mut new_row = row.clone(); + let mut bound = HashMap::new(); + bound.insert(output_col.clone(), item); + new_row.insert(UNNEST_KEY.to_string(), bound); + out.push(new_row); + } + } + // NULL list -> zero rows (matches DataFusion). + Value::Null => {} + other => { + return Err(InterpError::Eval(format!( + "unnest() expected a list, got a {} value", + crate::datafusion::expr::type_name(&other) + ))) + } + } + } + Ok(out) + } + } +} + +pub struct ColumnValidation { + pub row_table_columns: HashMap>, + pub effective_schemas: HashMap, +} + +/// Maps each relation's EFFECTIVE name (its alias if aliased, else its real +/// table name — the qualifier `Expr::Column` references use after +/// `SubqueryAlias` renaming) to its real table name and whether it's a row +/// table (vs. static). Walks the already-optimized Plan, so any `Join` with +/// a static side has already become a `LookupJoin`. +fn resolve_tables( + node: &RelNode, + row_table_names: &HashSet, + nullable: bool, + out: &mut HashMap, + nullable_out: &mut HashSet, +) { + match node { + RelNode::TableScan { table } => { + let is_row = row_table_names.contains(table); + out.insert(table.clone(), (table.clone(), is_row)); + if nullable { + nullable_out.insert(table.clone()); + } + } + RelNode::SubqueryAlias { input, alias } => { + if let Some(real) = scan_table_name(input) { + let is_row = row_table_names.contains(real); + out.insert(alias.clone(), (real.to_string(), is_row)); + if nullable { + nullable_out.insert(alias.clone()); + } + } + } + RelNode::Filter { input, .. } => { + resolve_tables(input, row_table_names, nullable, out, nullable_out) + } + RelNode::CrossJoin { left, right } => { + resolve_tables(left, row_table_names, nullable, out, nullable_out); + resolve_tables(right, row_table_names, nullable, out, nullable_out); + } + // A LEFT join makes its right side nullable; nested joins stay nullable. + // NB: post-optimize `outer` is structurally always false here -- a LEFT + // Join with a static side becomes a LookupJoin, and a row-to-row LEFT + // JOIN is rejected in optimize_rel. The `|| *outer` is kept correct in + // case that restriction is ever relaxed. + RelNode::Join { + left, right, outer, .. + } => { + resolve_tables(left, row_table_names, nullable, out, nullable_out); + resolve_tables(right, row_table_names, nullable || *outer, out, nullable_out); + } + RelNode::LookupJoin { + input, table, outer, .. + } => { + resolve_tables(input, row_table_names, nullable, out, nullable_out); + out.insert(table.clone(), (table.clone(), false)); + if nullable || *outer { + nullable_out.insert(table.clone()); + } + } + // The emitted column lives under a synthetic key resolved via + // effective_schemas, not `resolved` — just recurse into the input. + RelNode::Unnest { input, .. } => { + resolve_tables(input, row_table_names, nullable, out, nullable_out) + } + } +} + +/// Validates every `Expr::Column` reference in the plan (projection, WHERE, +/// JOIN ON) against the resolved table schemas, and collects — per row +/// table's REAL name — the set of columns the query actually references. +/// Also returns the effective-name -> Schema map (aliases resolved), reused +/// by the output type-inference pass. +pub fn validate_columns( + plan: &mut Plan, + row_table_names: &HashSet, + row_schemas: &HashMap, + static_schemas: &HashMap, +) -> Result { + let mut resolved = HashMap::new(); + let mut nullable_tables = HashSet::new(); + resolve_tables( + &plan.input, + row_table_names, + false, + &mut resolved, + &mut nullable_tables, + ); + + let mut effective_schemas = HashMap::new(); + for (effective_name, (real_name, is_row)) in &resolved { + let schema = if *is_row { + row_schemas.get(real_name) + } else { + static_schemas.get(real_name) + }; + if let Some(s) = schema { + let mut s = s.clone(); + // Columns from the nullable side of an outer join can be NULL on an + // unmatched row, so the synthesized output type must be nullable even + // when the source table declares the column non-nullable. + if nullable_tables.contains(effective_name) { + for ft in s.values_mut() { + ft.nullable = true; + } + } + effective_schemas.insert(effective_name.clone(), s); + } + } + + let mut used_columns: HashMap> = HashMap::new(); + let mut expanded_projection = Vec::with_capacity(plan.projection.len()); + let mut unnest_seen = false; + for (name, mut e) in std::mem::take(&mut plan.projection) { + validate_expr( + &mut e, + &resolved, + row_schemas, + static_schemas, + &effective_schemas, + &mut used_columns, + )?; + // Task 6: `unnest(list)` multiplies rows. Wrap the input rel in an + // `Unnest` node and replace the projection item with a plain reference + // to the emitted column. (The `unnest(struct)` case types as a struct + // and is handled by `expand_unnest_struct` below.) + if let Some((list_expr, elem_ft)) = unnest_list_element(&e, &effective_schemas)? { + if unnest_seen { + return Err(InterpError::Build( + "Only one unnest(list) per query is supported".to_string(), + )); + } + unnest_seen = true; + let old_input = + std::mem::replace(&mut plan.input, RelNode::TableScan { table: String::new() }); + plan.input = RelNode::Unnest { + input: Box::new(old_input), + list_expr, + output_col: name.clone(), + }; + // Register the emitted column so output-model synthesis (`infer_type`) + // and downstream validation resolve the unqualified `output_col`. + effective_schemas + .entry(UNNEST_KEY.to_string()) + .or_default() + .insert(name.clone(), elem_ft); + expanded_projection.push((name.clone(), Expr::Column { table: None, name })); + continue; + } + match expand_unnest_struct(&e, &effective_schemas)? { + Some(fields) => expanded_projection.extend(fields), + None => expanded_projection.push((name, e)), + } + } + plan.projection = expanded_projection; + validate_rel( + &mut plan.input, + &resolved, + row_schemas, + static_schemas, + &effective_schemas, + &mut used_columns, + )?; + + Ok(ColumnValidation { + row_table_columns: used_columns + .into_iter() + .map(|(k, v)| (k, v.into_iter().collect())) + .collect(), + effective_schemas, + }) +} + +fn validate_rel( + node: &mut RelNode, + resolved: &HashMap, + row_schemas: &HashMap, + static_schemas: &HashMap, + effective_schemas: &HashMap, + used_columns: &mut HashMap>, +) -> Result<(), InterpError> { + match node { + RelNode::TableScan { .. } => Ok(()), + RelNode::Filter { input, predicate } => { + validate_expr( + predicate, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + validate_rel( + input, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + RelNode::CrossJoin { left, right } => { + validate_rel( + left, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + validate_rel( + right, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + RelNode::Join { + left, right, on, .. + } => { + for (l, r) in on.iter_mut() { + validate_expr( + l, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + validate_expr( + r, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + } + validate_rel( + left, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + validate_rel( + right, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + RelNode::SubqueryAlias { input, .. } => validate_rel( + input, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ), + RelNode::LookupJoin { input, keys, .. } => { + for k in keys.iter_mut() { + validate_expr( + k, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + } + validate_rel( + input, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + RelNode::Unnest { + input, list_expr, .. + } => { + validate_expr( + list_expr, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + validate_rel( + input, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + } +} + +/// If `e` is `unnest()` and `` types as a struct, returns the +/// per-field projection columns it expands into (DataFusion flattens a +/// struct-typed `unnest()` into one output column per field, named +/// `"."`, ignoring any alias on the SELECT item -- see +/// `unnest_display_name`). Returns `None` for anything else (including +/// `unnest()` on a list, left untouched for Task 6's list-unnest). +/// If `e` is `unnest()` and `` types as a list, returns the argument +/// (the list expression) and the list's element `FieldType`. Returns `None` for +/// anything else (a struct-typed `unnest`, a non-`unnest` expression, ...). +fn unnest_list_element( + e: &Expr, + effective_schemas: &HashMap, +) -> Result, InterpError> { + let Expr::Function { name, args } = e else { + return Ok(None); + }; + if name != "unnest" || args.len() != 1 { + return Ok(None); + } + let arg_ty = crate::datafusion::types::infer_type(&args[0], effective_schemas)?; + let crate::datafusion::types::Base::List(elem) = arg_ty.base else { + return Ok(None); + }; + Ok(Some((args[0].clone(), *elem))) +} + +fn expand_unnest_struct( + e: &Expr, + effective_schemas: &HashMap, +) -> Result>, InterpError> { + let Expr::Function { name, args } = e else { + return Ok(None); + }; + if name != "unnest" || args.len() != 1 { + return Ok(None); + } + let arg = &args[0]; + let arg_ty = crate::datafusion::types::infer_type(arg, effective_schemas)?; + let crate::datafusion::types::Base::Struct(fields) = &arg_ty.base else { + return Ok(None); + }; + let arg_display = unnest_display_name(arg, effective_schemas)?; + Ok(Some( + fields + .iter() + .map(|(field_name, _)| { + ( + format!("{arg_display}.{field_name}"), + Expr::FieldAccess { + base: Box::new(arg.clone()), + field: field_name.clone(), + }, + ) + }) + .collect(), + )) +} + +/// Renders an expression the way DataFusion's logical-plan `Expr::Display` +/// does, for the shapes `unnest()`'s argument can take -- a (possibly +/// qualified) column, a struct-field access, or a `named_struct(...)` +/// construction. This is what DataFusion derives its `unnest(...)` output +/// column names from, so matching it exactly is required for the +/// differential tests to agree column-for-column. +/// +/// ponytail: only covers the node shapes reachable as an `unnest()` arg +/// today (struct columns, struct field access, `named_struct`/`struct()` +/// literals over plain columns). `Expr::Struct` can't tell `named_struct(...)` +/// and `struct(...)` apart post-conversion (both collapse to the same node), +/// so this always renders as `named_struct(...)`; DataFusion names a +/// `struct()`-built unnest differently. Widen if that combination needs +/// differential coverage. +fn unnest_display_name( + e: &Expr, + effective_schemas: &HashMap, +) -> Result { + match e { + Expr::Column { + table: Some(t), + name, + } => Ok(format!("{t}.{name}")), + Expr::Column { table: None, name } => { + let qualifier = effective_schemas + .iter() + .find(|(_, schema)| schema.contains_key(name)) + .map(|(qualifier, _)| qualifier.clone()) + .ok_or_else(|| InterpError::Build(format!("Unknown column: {name}")))?; + Ok(format!("{qualifier}.{name}")) + } + Expr::FieldAccess { base, field } => Ok(format!( + "{}.{field}", + unnest_display_name(base, effective_schemas)? + )), + Expr::Struct(fields) => { + let inner = fields + .iter() + .map(|(key, value)| { + Ok(format!( + "Utf8(\"{key}\"),{}", + unnest_display_name(value, effective_schemas)? + )) + }) + .collect::, InterpError>>()? + .join(","); + Ok(format!("named_struct({inner})")) + } + _ => Err(InterpError::Build( + "unnest() argument is too complex to name".to_string(), + )), + } +} + +fn validate_expr( + e: &mut Expr, + resolved: &HashMap, + row_schemas: &HashMap, + static_schemas: &HashMap, + effective_schemas: &HashMap, + used_columns: &mut HashMap>, +) -> Result<(), InterpError> { + match e { + Expr::Column { + table: Some(t), + name, + } => { + if let Some((real, is_row)) = resolved.get(t.as_str()) { + check_column(real, *is_row, name, row_schemas, static_schemas)?; + if *is_row { + used_columns + .entry(real.clone()) + .or_default() + .insert(name.clone()); + } + return Ok(()); + } + // `t` isn't a relation alias -- the "table.column" parse was + // wrong; reinterpret it as struct field access: `t` an in-scope + // column, `name` one of its struct fields. Precedence rule: a + // relation alias always wins, so this fallback only runs once + // the alias lookup above has failed. + let base_name = t.clone(); + let field = name.clone(); + *e = Expr::FieldAccess { + base: Box::new(Expr::Column { + table: None, + name: base_name, + }), + field, + }; + validate_expr( + e, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + Expr::Column { table: None, name } => { + let mut matches: Vec<(&String, bool)> = Vec::new(); + for (real, is_row) in resolved.values() { + let schema = if *is_row { + row_schemas.get(real) + } else { + static_schemas.get(real) + }; + if schema.is_some_and(|s| s.contains_key(name)) { + matches.push((real, *is_row)); + } + } + match matches.as_slice() { + [] => Err(InterpError::Build(format!("Unknown column: {name}"))), + [(real, is_row)] => { + if *is_row { + used_columns + .entry((*real).clone()) + .or_default() + .insert(name.clone()); + } + Ok(()) + } + _ => Err(InterpError::Build(format!( + "Ambiguous column reference: {name}" + ))), + } + } + Expr::Literal(_) => Ok(()), + Expr::BinaryOp { left, right, .. } => { + validate_expr( + left, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + validate_expr( + right, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ) + } + Expr::Not(inner) | Expr::Cast { expr: inner, .. } => validate_expr( + inner, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ), + Expr::Function { args, .. } | Expr::List(args) => { + for a in args { + validate_expr( + a, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + } + Ok(()) + } + Expr::Struct(fields) => { + for (_, v) in fields { + validate_expr( + v, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + } + Ok(()) + } + // Compiler-mandated: `Expr` exhaustiveness is checked at compile time + // and this match has no catch-all. Transformer resolution runs AFTER + // validate_columns, so no `Transform` node reaches here today; recurse + // into `arg` anyway so column validation stays correct if that ordering + // ever changes. + Expr::Transform { arg, .. } => validate_expr( + arg, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + ), + Expr::FieldAccess { base, field } => { + validate_expr( + base, + resolved, + row_schemas, + static_schemas, + effective_schemas, + used_columns, + )?; + let base_ty = crate::datafusion::types::infer_type(base, effective_schemas)?; + match &base_ty.base { + crate::datafusion::types::Base::Struct(fields) => { + if fields.iter().any(|(name, _)| name == field) { + Ok(()) + } else { + Err(InterpError::Build(format!("Unknown struct field: {field}"))) + } + } + _ => Err(InterpError::Build(format!( + "Cannot access field '{field}' on a non-struct column" + ))), + } + } + Expr::Case { arms, default } => { + for (cond, result) in arms { + validate_expr( + cond, resolved, row_schemas, static_schemas, effective_schemas, + used_columns, + )?; + validate_expr( + result, resolved, row_schemas, static_schemas, effective_schemas, + used_columns, + )?; + } + if let Some(d) = default { + validate_expr( + d, resolved, row_schemas, static_schemas, effective_schemas, + used_columns, + )?; + } + Ok(()) + } + } +} + +fn check_column( + real_table: &str, + is_row: bool, + name: &str, + row_schemas: &HashMap, + static_schemas: &HashMap, +) -> Result<(), InterpError> { + let schema = if is_row { + row_schemas.get(real_table) + } else { + static_schemas.get(real_table) + }; + match schema { + Some(s) if s.contains_key(name) => Ok(()), + Some(_) => Err(InterpError::Build(format!( + "Unknown column: {real_table}.{name}" + ))), + None => Err(InterpError::Build(format!("Unknown table: {real_table}"))), + } +} diff --git a/src/datafusion/types.rs b/src/datafusion/types.rs new file mode 100644 index 0000000..9bbf4af --- /dev/null +++ b/src/datafusion/types.rs @@ -0,0 +1,314 @@ +//! DataFusion-flavoured static type inference over a projection expression. + +use std::collections::{HashMap, HashSet}; + +// Re-exported so the engine-internal paths (`crate::datafusion::types::Base`) +// keep resolving now that the type vocabulary is shared. +pub use crate::types::{Base, FieldType, Schema}; + +use crate::datafusion::expr::{BinOp, CastType, Expr, Value}; +use crate::datafusion::plan::InterpError; + +/// Statically infers the FieldType of a projection expression, mirroring +/// `crate::datafusion::expr::eval()`'s structure but computing a type instead of a +/// value. Sound but not tight on nullability: `nullable: true` means +/// "cannot prove this can't be NULL," not "will be NULL." +pub fn infer_type( + expr: &Expr, + schemas: &HashMap, +) -> Result { + match expr { + Expr::Column { table, name } => resolve_column_type(table.as_deref(), name, schemas), + Expr::Literal(v) => Ok(literal_type(v)), + Expr::BinaryOp { op, left, right } => { + let l = infer_type(left, schemas)?; + let r = infer_type(right, schemas)?; + Ok(binary_op_type(*op, l, r)) + } + Expr::Not(inner) => { + let inner_ty = infer_type(inner, schemas)?; + Ok(FieldType { + base: Base::Bool, + nullable: inner_ty.nullable, + }) + } + Expr::Cast { expr, target } => { + let inner_ty = infer_type(expr, schemas)?; + Ok(FieldType { + base: cast_target_base(*target), + nullable: inner_ty.nullable, + }) + } + Expr::Function { name, args } => { + let arg_types: Vec = args + .iter() + .map(|a| infer_type(a, schemas)) + .collect::>()?; + Ok(function_type(name, &arg_types)) + } + Expr::Struct(fields) => { + let field_types = fields + .iter() + .map(|(name, e)| Ok((name.clone(), infer_type(e, schemas)?))) + .collect::>()?; + Ok(FieldType { + base: Base::Struct(field_types), + nullable: false, + }) + } + Expr::List(items) => { + let item_types = items + .iter() + .map(|e| infer_type(e, schemas)) + .collect::, _>>()?; + let elem = unify_list_element_types(&item_types); + Ok(FieldType { + base: Base::List(Box::new(elem)), + nullable: false, + }) + } + Expr::FieldAccess { base, field } => { + let base_ty = infer_type(base, schemas)?; + match &base_ty.base { + Base::Struct(fields) => fields + .iter() + .find(|(name, _)| name == field) + .map(|(_, ft)| FieldType { + base: ft.base.clone(), + nullable: ft.nullable || base_ty.nullable, + }) + .ok_or_else(|| InterpError::Build(format!("Unknown struct field: {field}"))), + _ => Err(InterpError::Build(format!( + "Cannot access field '{field}' on a non-struct column" + ))), + } + } + Expr::Transform { + input_features, + output_fields, + arg, + .. + } => { + let arg_ty = infer_type(arg, schemas)?; + match &arg_ty.base { + Base::Struct(fields) => { + let got: HashSet<&String> = fields.iter().map(|(n, _)| n).collect(); + let want: HashSet<&String> = input_features.iter().collect(); + if got != want { + return Err(InterpError::Build(format!( + "transformer input struct fields {:?} do not match \ + feature_names_in_ {:?}", + fields.iter().map(|(n, _)| n).collect::>(), + input_features + ))); + } + } + _ => { + return Err(InterpError::Build( + "transformer argument must be a struct (e.g. named_struct(...))" + .to_string(), + )) + } + } + Ok(FieldType { + base: Base::Struct(output_fields.clone()), + nullable: false, + }) + } + Expr::Case { arms, default } => { + let mut branch_types: Vec = arms + .iter() + .map(|(_, result)| infer_type(result, schemas)) + .collect::>()?; + let has_else = default.is_some(); + if let Some(d) = default { + branch_types.push(infer_type(d, schemas)?); + } + // No explicit ELSE => an unmatched row yields NULL, so nullable + // regardless of the branch types. + let nullable = !has_else || branch_types.iter().any(|t| t.nullable); + Ok(FieldType { + base: common_base(&branch_types), + nullable, + }) + } + } +} + +/// Unify element types for a list literal: identical types (by FieldType +/// equality, so nullability must also agree) collapse to that type; +/// anything else (including an empty list) is unresolvable. +fn unify_list_element_types(item_types: &[FieldType]) -> FieldType { + let mut iter = item_types.iter(); + let Some(first) = iter.next() else { + return FieldType { + base: Base::Other, + nullable: true, + }; + }; + if iter.all(|t| t == first) { + first.clone() + } else { + FieldType { + base: Base::Other, + nullable: true, + } + } +} + +fn resolve_column_type( + table: Option<&str>, + name: &str, + schemas: &HashMap, +) -> Result { + if let Some(t) = table { + return schemas + .get(t) + .and_then(|s| s.get(name)) + .cloned() + .ok_or_else(|| InterpError::Build(format!("Unknown column: {t}.{name}"))); + } + let mut found = None; + for s in schemas.values() { + if let Some(ft) = s.get(name) { + if found.is_some() { + return Err(InterpError::Build(format!( + "Ambiguous column reference: {name}" + ))); + } + found = Some(ft.clone()); + } + } + found.ok_or_else(|| InterpError::Build(format!("Unknown column: {name}"))) +} + +fn literal_type(v: &Value) -> FieldType { + match v { + Value::Int(_) => FieldType { + base: Base::Int, + nullable: false, + }, + Value::Float(_) => FieldType { + base: Base::Float, + nullable: false, + }, + Value::Str(_) => FieldType { + base: Base::Str, + nullable: false, + }, + Value::Bool(_) => FieldType { + base: Base::Bool, + nullable: false, + }, + Value::Null | Value::Object(_) => FieldType { + base: Base::Other, + nullable: true, + }, + Value::Struct(fields) => FieldType { + base: Base::Struct( + fields + .iter() + .map(|(name, v)| (name.clone(), literal_type(v))) + .collect(), + ), + nullable: false, + }, + Value::List(items) => { + let inner = items.first().map(literal_type).unwrap_or(FieldType { + base: Base::Other, + nullable: true, + }); + FieldType { + base: Base::List(Box::new(inner)), + nullable: false, + } + } + } +} + +fn binary_op_type(op: BinOp, l: FieldType, r: FieldType) -> FieldType { + let nullable = l.nullable || r.nullable; + match op { + BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div | BinOp::Mod => { + let base = if l.base == Base::Int && r.base == Base::Int { + Base::Int + } else { + Base::Float + }; + FieldType { base, nullable } + } + BinOp::Eq + | BinOp::NotEq + | BinOp::Lt + | BinOp::Gt + | BinOp::LtEq + | BinOp::GtEq + | BinOp::And + | BinOp::Or => FieldType { + base: Base::Bool, + nullable, + }, + BinOp::Concat => FieldType { + base: Base::Str, + nullable, + }, + } +} + +fn cast_target_base(target: CastType) -> Base { + match target { + CastType::Str => Base::Str, + CastType::Int => Base::Int, + CastType::Float => Base::Float, + CastType::Bool => Base::Bool, + } +} + +/// Common result base for variadic same-shape functions (COALESCE/NULLIF): +/// all-equal keeps that base; a mix of Int/Float widens to Float (DataFusion's +/// numeric supertype); anything else is left unresolved for Pydantic to judge. +fn common_base(args: &[FieldType]) -> Base { + match args.first() { + None => Base::Other, + Some(first) if args.iter().all(|a| a.base == first.base) => first.base.clone(), + _ if args.iter().all(|a| matches!(a.base, Base::Int | Base::Float)) => Base::Float, + _ => Base::Other, + } +} + +fn function_type(name: &str, args: &[FieldType]) -> FieldType { + let any_nullable = args.iter().any(|a| a.nullable); + match name { + "upper" | "lower" | "trim" | "substr" | "substring" => FieldType { + base: Base::Str, + nullable: any_nullable, + }, + // ABS preserves its argument's type; ROUND always yields Float (DataFusion + // returns Float64 even for an integer argument). + "abs" => { + let base = args.first().map(|a| a.base.clone()).unwrap_or(Base::Other); + FieldType { + base, + nullable: any_nullable, + } + } + "round" => FieldType { + base: Base::Float, + nullable: any_nullable, + }, + "concat" => FieldType { + base: Base::Str, + nullable: false, + }, + // DataFusion types these as the common supertype of the arguments + // (COALESCE(int, float) -> float), not args[0]'s type. + "coalesce" | "nullif" => FieldType { + base: common_base(args), + nullable: true, + }, + _ => FieldType { + base: Base::Other, + nullable: true, + }, + } +} From 81a6c2a4e634b89a039682fb47ff857edc0cafd3 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 04:43:45 +0200 Subject: [PATCH 05/11] fix(lint): pin datafusion/duckdb as third-party for isort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/datafusion/ (Rust module) sits under a ruff src root, so isort's scan started classifying the datafusion pip package as first-party on CI's --all-files run (local pre-commit only checks changed files, so it never fired). duckdb pinned too — the source clone collides the same way. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 696e521..26894ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,12 @@ fix = true lint.select = ["E", "F", "B", "I", "UP", "C4", "S"] lint.ignore = ["S101"] +[tool.ruff.lint.isort] +# datafusion and duckdb are pip packages, but same-named directories exist +# in-repo (src/datafusion Rust module; duckdb/ source clone), which makes +# isort's src-root scan misclassify the imports as first-party. +known-third-party = ["datafusion", "duckdb"] + [tool.ruff.lint.per-file-ignores] "*test*.py" = ["S608"] "_state.py" = ["S608"] From 8b50641364eda4d03a1aac416cd158b33891c6fd Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 15:17:57 +0200 Subject: [PATCH 06/11] =?UTF-8?q?feat(specializer):=20imperative=20IR=20?= =?UTF-8?q?=E2=80=94=20types,=20verifier,=20text=20format,=20fuzz=20(M-ir)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/specializer/ir/: SSA over bare typed scalars with the null lane split into explicit i1 flags (.opt instruction forms) so a nullable value cannot reach arithmetic by construction; strict block-param SSA (no dominance analysis needed); acyclic CFG; emit/skip/trap terminators carrying the store-completeness contract (|out|==|in| iff skip unreachable). verify.rs enforces six rule groups, each with a rejecting test; print/parse round-trip exactly (parse(print(p)) == p, bitwise literal equality so nan/-0.0 survive); gen.rs is a seeded xorshift program generator — 300-seed round-trip fuzz in tests, reused later for interpreter-vs-codegen fuzzing. Design doc §6 updated to the implemented form. TASK-41. Co-Authored-By: Claude Opus 5 --- ...224-grammar-types-verifier-text-format.md" | 13 +- .../2026-07-25-sql-specializer-design.md | 70 +- src/lib.rs | 1 + src/specializer/ir/fixtures.rs | 147 +++ src/specializer/ir/gen.rs | 430 +++++++ src/specializer/ir/mod.rs | 541 +++++++++ src/specializer/ir/parse.rs | 1046 +++++++++++++++++ src/specializer/ir/print.rs | 280 +++++ src/specializer/ir/tests.rs | 505 ++++++++ src/specializer/ir/verify.rs | 560 +++++++++ src/specializer/mod.rs | 11 + 11 files changed, 3578 insertions(+), 26 deletions(-) create mode 100644 src/specializer/ir/fixtures.rs create mode 100644 src/specializer/ir/gen.rs create mode 100644 src/specializer/ir/mod.rs create mode 100644 src/specializer/ir/parse.rs create mode 100644 src/specializer/ir/print.rs create mode 100644 src/specializer/ir/tests.rs create mode 100644 src/specializer/ir/verify.rs create mode 100644 src/specializer/mod.rs diff --git "a/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" "b/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" index 4f6e840..4a88e46 100644 --- "a/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" +++ "b/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" @@ -1,9 +1,10 @@ --- id: TASK-41 title: 'Specializer M-ir: imperative IR — grammar, types, verifier, text format' -status: To Do +status: In Progress assignee: [] created_date: '2026-07-25 02:31' +updated_date: '2026-07-25 02:58' labels: [] milestone: m-7 dependencies: [] @@ -26,3 +27,13 @@ Define the specializer's imperative IR per §6 of docs/superpowers/specs/2026-07 - [ ] #3 Hand-written IR programs covering every instruction exist as test fixtures - [ ] #4 mise gate-specializer green + +## Implementation Plan + + +1. Freeze IR design decisions (recorded in design doc §6 update): implicit row cursors (no idx params), strict block-param SSA (cross-block value uses forbidden — no dominance analysis needed), acyclic CFG for v0, terminators emit/skip/trap/jump/brif, store-completeness dataflow at emit. +2. Implement src/specializer/ir/: core types + instructions (mod.rs), verifier (verify.rs), printer (print.rs), parser (parse.rs), shared fixtures (fixtures.rs). +3. Tests: per-instruction fixture programs (parse+verify+round-trip), negative verifier tests per rule, negative parser tests, deterministic seeded fuzz round-trip (hand-rolled xorshift generator, no new deps). +4. Adversarial workflow: fan-out attackers on verifier soundness + round-trip edge cases (float/string literals), plus design-conformance and simplification reviewers; fix confirmed findings. +5. Gate green; stacked PR onto claude/duckdb-native-interpreter-36d016. + diff --git a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md index 4630f41..35f6921 100644 --- a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md +++ b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md @@ -166,40 +166,60 @@ enum StaticStruct { } ``` -## 6. Imperative IR sketch +## 6. Imperative IR (as implemented — M-ir, `src/specializer/ir/`) SSA, typed, no allocation vocabulary, explicit null lane. Text format is the -diagnostic surface and must round-trip (`parse(print(ir)) == ir`). +diagnostic surface and round-trips: `parse(print(p)) == p`. The module docs of +`src/specializer/ir/mod.rs` are the normative spec; highlights: ``` -;; SELECT age / s.mean_age AS z FROM __THIS__ t LEFT JOIN stats s ON t.seg = s.seg -;; after BTA: stats collapsed into staticref @0 (perfect hash: seg -> mean_age) +static @0: map(str) -> (f64) fn run(in: batch{age: i64?, seg: str}, out: batch{z: f64?}) { -entry(row: idx): - %age.f, %age.v = load.opt in.age, row ; (i1 flag, i64 payload) - %seg = load in.seg, row ; NOT NULL lane: no flag - %hit.f, %m = probe.opt @0, %seg ; miss -> flag=0 (LEFT JOIN) - %num = cast f64, %age.v - %q = fdiv %num, %m - %z.f = and %age.f, %hit.f ; null iff either input null - store.opt out.z, row, %z.f, %q - next row +entry: + %age_f, %age_v = load.opt in.age # (i1 flag, i64 payload); row cursor implicit + %seg = load in.seg # NOT NULL lane: no flag + %hit, %mean = probe @0, %seg # miss -> hit=false (LEFT JOIN) + %num = itof %age_v + %q = fdiv %num, %mean + %zf = and %age_f, %hit # null iff either input null + store.opt out.z, %zf, %q + emit } ``` -Verifier rules (initial set): - -1. SSA: single def, defs dominate uses. -2. Type check every op; `load.opt`/`probe.opt`/`store.opt` are the only ops that - touch flags; a `T?` value cannot flow into an arithmetic op — the verifier - rejects it (this is the "3VL mistakes are statically impossible" property). -3. `StaticRef` ids must resolve against the plan's static-structure table, with - matching key/value types. -4. No op allocates; `scratch` is reachable only from varlen `store` ops. -5. `out` column set and row count semantics must match the declared plan shape - (`|out| = |in|` for pure projection; filter introduces the one allowed - divergence and must declare it). +Decisions made at implementation (deviations from the original sketch, all in +the direction of a smaller, more verifiable core): + +- **Implicit row cursor** — no `idx` values, no `row` operand; nothing in v0 + needs random row access, and it removes a whole type from the verifier. +- **Terminators carry the row protocol**: `emit` (row complete; every out + column stored exactly once on the path), `skip` (filter drop; path must + store nothing), `trap "msg"` (runtime error — CAST failures, div guards), + plus `jump`/`brif`. `|out| == |in|` iff `skip` is unreachable — statically + known, per the "filter must declare divergence" requirement. +- **Strict block-param SSA**: values cross blocks only as branch args to + block params, so the verifier needs no dominance analysis. CFG acyclic in + v0 (lift when a lowering pattern needs loops, e.g. QuickScorer). +- **Nullability is structural, not checked**: SSA values are always bare + scalars; only `load.opt`/`store.opt`/`probe`/`sload.opt` touch flags. A + `T?` cannot reach arithmetic because the type system cannot express it. +- Names in text are presentation-only; printing canonicalizes to `%vN`/`bN`. + NaN payloads canonicalize to one `nan` token (literal equality is bitwise, + so `-0.0` and NaN round-trip). + +Verifier rules (each has a rejecting test in `ir/tests.rs`): structure (entry +has no params, unique column names, no branch to entry), SSA (single def, +same-block visibility), per-op operand types incl. mandatory/forbidden `.opt` +pairing against column/static nullability, static resolution + kind/arity/key +types, CFG reachability + acyclicity + branch-arg typing, and the +store-completeness dataflow (exactly-once per column at `emit`, zero at +`skip`, agreement at joins). + +Deferred to M-interp, pinned there against the DuckDB oracle: `ftoi.round` +tie behavior and `fcmp` NaN ordering. `idiv`/`irem` trap on zero/overflow; +SQL-level NULL-on-zero (or error) semantics are a lowering decision built +from `brif` + `trap`. ## 7. Backends diff --git a/src/lib.rs b/src/lib.rs index 3aac431..e7fab14 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,7 @@ mod duckdb; mod error; mod lookup; mod schema; +pub mod specializer; mod types; mod value; diff --git a/src/specializer/ir/fixtures.rs b/src/specializer/ir/fixtures.rs new file mode 100644 index 0000000..69ad1dc --- /dev/null +++ b/src/specializer/ir/fixtures.rs @@ -0,0 +1,147 @@ +//! Hand-written IR programs, one habitat per instruction. Written in the +//! human-friendly form (named values, named labels) that the parser accepts +//! and the printer canonicalizes — the round-trip tests exercise exactly that +//! path. M-interp reuses these as its first executable programs. + +/// The design-doc example: nullable arithmetic against a probed static. +/// Covers: load, load.opt, probe, itof, fdiv, and, store.opt, emit. +pub const PROJECTION: &str = r#" +static @0: map(str) -> (f64) + +fn run(in: batch{age: i64?, seg: str}, out: batch{z: f64?}) { +entry: + %age_f, %age_v = load.opt in.age + %seg = load in.seg + %hit, %mean = probe @0, %seg + %num = itof %age_v + %q = fdiv %num, %mean + %zf = and %age_f, %hit + store.opt out.z, %zf, %q + emit +} +"#; + +/// A WHERE clause: the one construct allowed to change |out|. +/// Covers: const.f64, fcmp, brif (no args), skip, store. +pub const FILTER: &str = r#" +fn keep_positive(in: batch{score: f64}, out: batch{score: f64}) { +entry: + %s = load in.score + %zero = const.f64 0.0 + %pos = fcmp.gt %s, %zero + brif %pos, keep, drop +keep: + %s2 = load in.score + store out.score, %s2 + emit +drop: + skip +} +"#; + +/// A CASE diamond joining through a block param, plus the i1 algebra. +/// Covers: const.i64, const.i1, const.str, icmp, jump with args, block +/// params, xor, not, select, store of i1. +pub const CASE_DIAMOND: &str = r#" +fn bucket(in: batch{age: i64}, out: batch{label: str, flag: i1}) { +entry: + %age = load in.age + %lim = const.i64 30 + %old = icmp.ge %age, %lim + brif %old, old_b, young_b +old_b: + %a = const.str "old" + jump join(%a) +young_b: + %b = const.str "young" + jump join(%b) +join(%label: str): + store out.label, %label + %t = const.i1 true + %f = const.i1 false + %x = xor %t, %f + %n = not %x + %pick = select %n, %t, %f + store out.flag, %pick + emit +} +"#; + +/// The cast family and scalar statics; a failed parse traps (SQL CAST +/// semantics — TRY_CAST lowers to a select instead). +/// Covers: stoi.opt, sload, sload.opt, ftoi.round, ftoi.trunc, isub, ftos, +/// sconcat, scmp, trap. +pub const CASTS: &str = r#" +static @0: scalar +static @1: scalar + +fn casts(in: batch{s: str}, out: batch{n: i64, msg: str}) { +entry: + %raw = load in.s + %ok, %parsed = stoi.opt %raw + brif %ok, good, bad +good: + %base = sload @0 + %r1 = ftoi.round %base + %t1 = ftoi.trunc %base + %d = isub %r1, %t1 + %pf, %pv = sload.opt @1 + %n2 = select %pf, %pv, %d + store out.n, %n2 + %ftxt = ftos %base + %sep = const.str ":" + %msg1 = sconcat %ftxt, %sep + %same = scmp.eq %msg1, %sep + %sel = select %same, %msg1, %sep + store out.msg, %sel + emit +bad: + trap "cast to i64 failed" +} +"#; + +/// Everything else: the full integer/float binary set, a compound-key probe +/// with multiple value columns, and the remaining casts. +/// Covers: iadd, imul, idiv, irem, fadd, fsub, fmul, or, icmp.ne, fcmp.lt, +/// itos, stof.opt, multi-key/multi-value probe, quoted column names. +pub const KITCHEN: &str = r#" +static @0: map(i64, str) -> (f64, i64) + +fn kitchen(in: batch{a: i64, b: i64?, k: str, x: f64}, out: batch{"r?": f64?, s: str}) { +entry: + %a = load in.a + %bf, %bv = load.opt in.b + %sum = iadd %a, %bv + %prod = imul %sum, %a + %quot = idiv %prod, %a + %rem = irem %quot, %a + %k = load in.k + %h, %vf, %vi = probe @0, %rem, %k + %x = load in.x + %fa = fadd %x, %vf + %fs = fsub %fa, %vf + %fm = fmul %fs, %x + %lt = fcmp.lt %fm, %x + %ne = icmp.ne %vi, %a + %both = or %lt, %ne + %rf = and %both, %bf + %rv = select %lt, %fm, %x + store.opt out."r?", %rf, %rv + %istr = itos %vi + %ff, %fv = stof.opt %istr + %num2 = select %ff, %fv, %fm + %ns = ftos %num2 + store out.s, %ns + emit +} +"#; + +pub fn all() -> Vec<(&'static str, &'static str)> { + vec![ + ("projection", PROJECTION), + ("filter", FILTER), + ("case_diamond", CASE_DIAMOND), + ("casts", CASTS), + ("kitchen", KITCHEN), + ] +} diff --git a/src/specializer/ir/gen.rs b/src/specializer/ir/gen.rs new file mode 100644 index 0000000..8f409f1 --- /dev/null +++ b/src/specializer/ir/gen.rs @@ -0,0 +1,430 @@ +//! Deterministic random-program generator for round-trip and (later) +//! differential fuzzing. Every generated program is verifier-valid by +//! construction and built with dense definition-ordered value ids, so +//! `parse(print(p)) == p` must hold exactly — any divergence is a bug in the +//! printer, the parser, or this generator, and all three are worth knowing. +//! +//! Hand-rolled xorshift instead of a proptest dependency: round-trip failures +//! shrink trivially (the seed pins the program), and the generator doubles as +//! the input source for M-interp's interpreter-vs-codegen fuzzing. + +use super::{ + BinOp, Block, BlockId, Builder, Col, ColTy, CmpPred, Inst, Lit, Program, RoundMode, StaticTy, + Term, Ty, Value, +}; + +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Rng { + // xorshift64* must not be seeded with 0. + Rng(seed.wrapping_mul(2685821657736338717).max(1)) + } + + pub fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(2685821657736338717) + } + + pub fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + + pub fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } +} + +const TYS: [Ty; 4] = [Ty::I1, Ty::I64, Ty::F64, Ty::Str]; + +/// Column-name pool: mixes plain identifiers with names that force the +/// quoted form (SQL-derived output names contain arbitrary characters). +const NASTY_NAMES: [&str; 4] = ["COALESCE(a, b)", "weird name", "a\"quote", "tab\there"]; + +fn rand_ty(rng: &mut Rng) -> Ty { + TYS[rng.below(4) as usize] +} + +fn rand_col_ty(rng: &mut Rng) -> ColTy { + ColTy { + ty: rand_ty(rng), + nullable: rng.chance(40), + } +} + +fn rand_lit(rng: &mut Rng, ty: Ty) -> Lit { + match ty { + Ty::I1 => Lit::I1(rng.chance(50)), + Ty::I64 => Lit::I64(match rng.below(6) { + 0 => 0, + 1 => -1, + 2 => i64::MAX, + 3 => i64::MIN, + _ => rng.next() as i64 % 1_000_000, + }), + Ty::F64 => Lit::F64(match rng.below(9) { + 0 => 0.0, + 1 => -0.0, + 2 => f64::NAN, + 3 => f64::INFINITY, + 4 => f64::NEG_INFINITY, + 5 => 1e300, + 6 => 1e-5, + 7 => 0.1, + _ => (rng.next() as i64 % 1000) as f64 / 8.0, + }), + Ty::Str => Lit::Str( + match rng.below(6) { + 0 => "", + 1 => "plain", + 2 => "quote\"backslash\\", + 3 => "line\nbreak\ttab", + 4 => "unicode é ✓", + _ => "x", + } + .to_string(), + ), + } +} + +/// Per-block generation state: the values in scope, by type. +struct Scope { + avail: Vec<(Value, Ty)>, +} + +impl Scope { + fn new() -> Scope { + Scope { avail: Vec::new() } + } + + fn add(&mut self, v: Value, ty: Ty) { + self.avail.push((v, ty)); + } + + fn pick(&self, rng: &mut Rng, ty: Ty) -> Option { + let of_ty: Vec = self + .avail + .iter() + .filter(|(_, t)| *t == ty) + .map(|(v, _)| *v) + .collect(); + if of_ty.is_empty() { + None + } else { + Some(of_ty[rng.below(of_ty.len() as u64) as usize]) + } + } +} + +/// Get a value of `ty`, minting a const when none is in scope (or sometimes +/// just because — consts are where the literal edge cases enter). +fn ensure(rng: &mut Rng, b: &mut Builder, scope: &mut Scope, insts: &mut Vec, ty: Ty) -> Value { + if rng.chance(70) { + if let Some(v) = scope.pick(rng, ty) { + return v; + } + } + let dst = b.fresh(); + insts.push(Inst::Const { dst, lit: rand_lit(rng, ty) }); + scope.add(dst, ty); + dst +} + +/// Loads for every input column plus reads of every static — the block's +/// raw material. +fn load_all( + rng: &mut Rng, + b: &mut Builder, + scope: &mut Scope, + insts: &mut Vec, + p_in: &[Col], + statics: &[StaticTy], +) { + for (ci, c) in p_in.iter().enumerate() { + if c.ty.nullable { + let flag = b.fresh(); + let dst = b.fresh(); + insts.push(Inst::LoadOpt { flag, dst, col: ci as u32 }); + scope.add(flag, Ty::I1); + scope.add(dst, c.ty.ty); + } else { + let dst = b.fresh(); + insts.push(Inst::Load { dst, col: ci as u32 }); + scope.add(dst, c.ty.ty); + } + } + for (si, st) in statics.iter().enumerate() { + match st { + StaticTy::Scalar(ct) if ct.nullable => { + let flag = b.fresh(); + let dst = b.fresh(); + insts.push(Inst::SloadOpt { static_id: si as u32, flag, dst }); + scope.add(flag, Ty::I1); + scope.add(dst, ct.ty); + } + StaticTy::Scalar(ct) => { + let dst = b.fresh(); + insts.push(Inst::Sload { static_id: si as u32, dst }); + scope.add(dst, ct.ty); + } + StaticTy::Map { keys, values } => { + let key_vals: Vec = keys + .iter() + .map(|kt| ensure(rng, b, scope, insts, *kt)) + .collect(); + let hit = b.fresh(); + scope.add(hit, Ty::I1); + let mut dsts = Vec::with_capacity(values.len()); + for vt in values { + let d = b.fresh(); + scope.add(d, *vt); + dsts.push(d); + } + insts.push(Inst::Probe { static_id: si as u32, hit, dsts, keys: key_vals }); + } + } + } +} + +/// A few random compute instructions over whatever is in scope. +fn compute(rng: &mut Rng, b: &mut Builder, scope: &mut Scope, insts: &mut Vec) { + let n = rng.below(7); + for _ in 0..n { + match rng.below(8) { + 0 => { + let ops = [ + BinOp::Iadd, + BinOp::Isub, + BinOp::Imul, + BinOp::Fadd, + BinOp::Fsub, + BinOp::Fmul, + BinOp::Fdiv, + BinOp::And, + BinOp::Or, + BinOp::Xor, + ]; + // idiv/irem excluded: they trap on zero, and generated + // programs must stay executable for M-interp fuzzing. + let op = ops[rng.below(ops.len() as u64) as usize]; + let (ot, rt) = op.sig(); + let a = ensure(rng, b, scope, insts, ot); + let rhs = ensure(rng, b, scope, insts, ot); + let dst = b.fresh(); + insts.push(Inst::Bin { op, dst, a, b: rhs }); + scope.add(dst, rt); + } + 1 => { + let ty = [Ty::I64, Ty::F64, Ty::Str][rng.below(3) as usize]; + let preds = [ + CmpPred::Eq, + CmpPred::Ne, + CmpPred::Lt, + CmpPred::Le, + CmpPred::Gt, + CmpPred::Ge, + ]; + let pred = preds[rng.below(6) as usize]; + let a = ensure(rng, b, scope, insts, ty); + let rhs = ensure(rng, b, scope, insts, ty); + let dst = b.fresh(); + insts.push(Inst::Cmp { pred, ty, dst, a, b: rhs }); + scope.add(dst, Ty::I1); + } + 2 => { + let a = ensure(rng, b, scope, insts, Ty::I1); + let dst = b.fresh(); + insts.push(Inst::Not { dst, a }); + scope.add(dst, Ty::I1); + } + 3 => { + let ty = rand_ty(rng); + let cond = ensure(rng, b, scope, insts, Ty::I1); + let a = ensure(rng, b, scope, insts, ty); + let rhs = ensure(rng, b, scope, insts, ty); + let dst = b.fresh(); + insts.push(Inst::Select { dst, cond, a, b: rhs }); + scope.add(dst, ty); + } + 4 => { + let a = ensure(rng, b, scope, insts, Ty::I64); + let dst = b.fresh(); + insts.push(Inst::Itof { dst, a }); + scope.add(dst, Ty::F64); + } + 5 => { + let mode = if rng.chance(50) { RoundMode::Trunc } else { RoundMode::Round }; + let a = ensure(rng, b, scope, insts, Ty::F64); + let dst = b.fresh(); + insts.push(Inst::Ftoi { mode, dst, a }); + scope.add(dst, Ty::I64); + } + 6 => { + let a = ensure(rng, b, scope, insts, Ty::Str); + let rhs = ensure(rng, b, scope, insts, Ty::Str); + let dst = b.fresh(); + insts.push(Inst::Sconcat { dst, a, b: rhs }); + scope.add(dst, Ty::Str); + } + _ => { + let (from, mk): (Ty, fn(Value, Value) -> Inst) = if rng.chance(50) { + (Ty::I64, |dst, a| Inst::Itos { dst, a }) + } else { + (Ty::F64, |dst, a| Inst::Ftos { dst, a }) + }; + let a = ensure(rng, b, scope, insts, from); + let dst = b.fresh(); + insts.push(mk(dst, a)); + scope.add(dst, Ty::Str); + } + } + } +} + +/// One store per out column, then the given terminator's stores contract. +fn stores( + rng: &mut Rng, + b: &mut Builder, + scope: &mut Scope, + insts: &mut Vec, + out_cols: &[Col], +) { + for (ci, c) in out_cols.iter().enumerate() { + let val = ensure(rng, b, scope, insts, c.ty.ty); + if c.ty.nullable { + let flag = ensure(rng, b, scope, insts, Ty::I1); + insts.push(Inst::StoreOpt { col: ci as u32, flag, val }); + } else { + insts.push(Inst::Store { col: ci as u32, val }); + } + } +} + +pub fn gen_program(seed: u64) -> Program { + let mut rng = Rng::new(seed); + let mut b = Builder::new(); + + let statics: Vec = (0..rng.below(3)) + .map(|_| { + if rng.chance(50) { + StaticTy::Scalar(rand_col_ty(&mut rng)) + } else { + StaticTy::Map { + keys: (0..1 + rng.below(2)).map(|_| rand_ty(&mut rng)).collect(), + values: (0..1 + rng.below(2)).map(|_| rand_ty(&mut rng)).collect(), + } + } + }) + .collect(); + + let name_of = |i: usize, prefix: &str, rng: &mut Rng| -> String { + if rng.chance(20) { + format!("{} {}", NASTY_NAMES[rng.below(4) as usize], i) + } else { + format!("{prefix}{i}") + } + }; + let in_cols: Vec = (0..1 + rng.below(3) as usize) + .map(|i| Col { name: name_of(i, "c", &mut rng), ty: rand_col_ty(&mut rng) }) + .collect(); + let out_cols: Vec = (0..1 + rng.below(2) as usize) + .map(|i| Col { name: name_of(i, "o", &mut rng), ty: rand_col_ty(&mut rng) }) + .collect(); + + let shape = rng.below(3); + let mut blocks = Vec::new(); + match shape { + // Straight line: load, compute, store, emit. + 0 => { + let mut scope = Scope::new(); + let mut insts = Vec::new(); + load_all(&mut rng, &mut b, &mut scope, &mut insts, &in_cols, &statics); + compute(&mut rng, &mut b, &mut scope, &mut insts); + stores(&mut rng, &mut b, &mut scope, &mut insts, &out_cols); + blocks.push(Block { params: vec![], insts, term: Term::Emit }); + } + // Filter: entry branches to a storing block or a skip/trap block. + 1 => { + let mut scope = Scope::new(); + let mut insts = Vec::new(); + load_all(&mut rng, &mut b, &mut scope, &mut insts, &in_cols, &statics); + compute(&mut rng, &mut b, &mut scope, &mut insts); + let cond = ensure(&mut rng, &mut b, &mut scope, &mut insts, Ty::I1); + blocks.push(Block { + params: vec![], + insts, + term: Term::Brif { + cond, + then_to: BlockId(1), + then_args: vec![], + else_to: BlockId(2), + else_args: vec![], + }, + }); + let mut keep_scope = Scope::new(); + let mut keep_insts = Vec::new(); + load_all(&mut rng, &mut b, &mut keep_scope, &mut keep_insts, &in_cols, &statics); + stores(&mut rng, &mut b, &mut keep_scope, &mut keep_insts, &out_cols); + blocks.push(Block { params: vec![], insts: keep_insts, term: Term::Emit }); + let drop_term = if rng.chance(80) { + Term::Skip + } else { + Term::Trap { msg: "generated trap".to_string() } + }; + blocks.push(Block { params: vec![], insts: vec![], term: drop_term }); + } + // Diamond: both arms feed a join param that lands in a store. + _ => { + let join_ty = rand_ty(&mut rng); + let mut scope = Scope::new(); + let mut insts = Vec::new(); + load_all(&mut rng, &mut b, &mut scope, &mut insts, &in_cols, &statics); + let cond = ensure(&mut rng, &mut b, &mut scope, &mut insts, Ty::I1); + blocks.push(Block { + params: vec![], + insts, + term: Term::Brif { + cond, + then_to: BlockId(1), + then_args: vec![], + else_to: BlockId(2), + else_args: vec![], + }, + }); + for _ in 0..2 { + let mut arm_scope = Scope::new(); + let mut arm_insts = Vec::new(); + let v = ensure(&mut rng, &mut b, &mut arm_scope, &mut arm_insts, join_ty); + blocks.push(Block { + params: vec![], + insts: arm_insts, + term: Term::Jump { to: BlockId(3), args: vec![v] }, + }); + } + let param = b.fresh(); + let mut join_scope = Scope::new(); + join_scope.add(param, join_ty); + let mut join_insts = Vec::new(); + load_all(&mut rng, &mut b, &mut join_scope, &mut join_insts, &in_cols, &statics); + compute(&mut rng, &mut b, &mut join_scope, &mut join_insts); + stores(&mut rng, &mut b, &mut join_scope, &mut join_insts, &out_cols); + blocks.push(Block { + params: vec![(param, join_ty)], + insts: join_insts, + term: Term::Emit, + }); + } + } + + Program { + statics, + name: "fuzzed".to_string(), + in_cols, + out_cols, + blocks, + } +} diff --git a/src/specializer/ir/mod.rs b/src/specializer/ir/mod.rs new file mode 100644 index 0000000..44f8142 --- /dev/null +++ b/src/specializer/ir/mod.rs @@ -0,0 +1,541 @@ +//! The specializer's imperative IR: SSA over typed scalars, explicit null +//! lane, no allocation vocabulary. This module is the spec; `verify`, `print` +//! and `parse` implement its three mandatory properties (airtight boundary, +//! canonical text, round-trip). +//! +//! # Execution model +//! +//! A [`Program`] describes one specialized function `run(in, out, scratch)` +//! executed once **per input row** (the produce/consume row loop is implicit — +//! there is no row-index value in the IR). The function reads the current +//! input row via `load`/`load.opt`, computes, writes the current output row +//! via `store`/`store.opt`, and finishes with a terminator: +//! +//! * `emit` — the output row is complete (every out column stored exactly +//! once on this path); advance both cursors. +//! * `skip` — no output row for this input row (filters); nothing may have +//! been stored on this path. +//! * `trap "msg"` — abort the whole call with a runtime error (CAST failures, +//! division guards — whatever the lowered dialect defines as an error). +//! +//! `|out| == |in|` therefore holds exactly when `skip` is unreachable, which +//! is statically known — the design doc's "filter is the one allowed +//! divergence and must declare it". +//! +//! # The null lane +//! +//! SSA values are always bare scalars — there is no nullable SSA type, so a +//! nullable value cannot reach an arithmetic instruction *by construction* +//! (the type system has no way to express it; this is how three-valued-logic +//! bugs are made unrepresentable rather than merely checked). Nullability +//! exists only at the edges: a nullable column or static is accessed through +//! the `.opt` instruction forms, which split it into an `i1` validity flag +//! plus a payload of the bare type; NULL logic is then ordinary `i1` algebra +//! (`and`, `or`, `select`), and `store.opt` reassembles flag + payload into a +//! nullable output column. On a false flag the payload is the type's default +//! value (0 / 0.0 / "" / false) — defined, deterministic, never poison. +//! +//! # SSA discipline +//! +//! Strict block-param form: a value may be used only in the block that +//! defines it (after its definition) or received as a block parameter. +//! Cross-block direct uses are illegal — everything flowing between blocks +//! rides on branch arguments. This removes the need for dominance analysis in +//! the verifier; the CFG must also be acyclic in v0 (no operator we lower +//! needs a loop yet; lift when one does, e.g. QuickScorer). +//! +//! # Statics +//! +//! Prepare-time structures are referenced through opaque `@N` handles typed +//! by the program header — `scalar`/`scalar` read with +//! `sload`/`sload.opt`, and `map(K..) -> (V..)` probed with `probe` (returns +//! a hit flag plus one value per declared column; a LEFT-join miss is just +//! `hit = false`). How a map is materialized (perfect hash, dense array, +//! inline chain) is a backend decision invisible here. Nullable static map +//! columns are flattened to an `i1` column + payload column at prepare time, +//! so map key/value types are bare [`Ty`]. +//! +//! # Allocation +//! +//! No instruction allocates on a heap. The only instructions that produce new +//! varlen data — `itos`, `ftos`, `sconcat` — write into the caller's bump +//! arena (`scratch` in the ABI), reset per call. Everything else moves +//! scalars between registers. +//! +//! # Text format +//! +//! Canonical, round-trippable; `parse(print(p)) == p` for every program whose +//! value ids are dense and in definition order (the parser and [`gen`] both +//! produce that form, so the property is closed under round-trip). Names in +//! the text are presentation only — they are not stored in the IR; printing +//! uses canonical `%vN` / `bN` names. +//! +//! ```text +//! program := static* func +//! static := "static" "@" INT ":" static_ty +//! static_ty := "scalar" "<" col_ty ">" +//! | "map" "(" ty ("," ty)* ")" "->" "(" ty ("," ty)* ")" +//! func := "fn" IDENT "(" "in" ":" batch "," "out" ":" batch ")" "{" block+ "}" +//! batch := "batch" "{" [col ("," col)*] "}" +//! col := (IDENT | STRING) ":" col_ty // STRING for non-ident names +//! col_ty := ty ["?"] +//! ty := "i1" | "i64" | "f64" | "str" +//! block := IDENT ["(" VALUE ":" ty ("," VALUE ":" ty)* ")"] ":" inst* term +//! inst := VALUE ("," VALUE)* "=" OPCODE operands +//! term := "jump" target +//! | "brif" VALUE "," target "," target +//! | "emit" | "skip" | "trap" STRING +//! target := IDENT ["(" VALUE ("," VALUE)* ")"] +//! VALUE := "%" IDENT +//! ``` +//! +//! Instruction surface (`%d` result, `%f` an `i1` flag result): +//! +//! ```text +//! %d = const.i1 true|false %d = const.i64 INT +//! %d = const.f64 FLOAT %d = const.str STRING +//! %d = iadd|isub|imul|idiv|irem %a, %b // i64; idiv/irem trap on 0 or overflow +//! %d = fadd|fsub|fmul|fdiv %a, %b // f64, IEEE (inf/nan flow, no trap) +//! %d = and|or|xor %a, %b // i1 +//! %d = not %a // i1 +//! %d = icmp.P|fcmp.P|scmp.P %a, %b // P in eq ne lt le gt ge; -> i1 +//! %d = select %c, %a, %b // %c: i1; %a, %b same type +//! %d = itof %a // i64 -> f64 +//! %d = ftoi.trunc|ftoi.round %a // f64 -> i64; traps out of range +//! %d = itos %a | ftos %a // -> str (arena) +//! %f, %d = stoi.opt %a | stof.opt %a // parse; %f=false on failure +//! %d = sconcat %a, %b // str (arena) +//! %d = load in.COL // COL not nullable +//! %f, %d = load.opt in.COL // COL nullable +//! store out.COL, %v // COL not nullable +//! store.opt out.COL, %f, %v // COL nullable +//! %hit, %v1, .. = probe @N, %k1, .. // map static +//! %d = sload @N // scalar static +//! %f, %d = sload.opt @N // scalar static +//! ``` +//! +//! Numeric-semantics pins deferred to M-interp (settled against the DuckDB +//! oracle there): `ftoi.round` tie behavior, `fcmp` NaN ordering. The IR +//! names the operations; the interpreter pins their edge cases with tests. + +pub mod fixtures; +pub mod gen; +pub mod parse; +pub mod print; +pub mod verify; + +#[cfg(test)] +mod tests; + +/// Bare scalar type of an SSA value. There is deliberately no nullable +/// variant — see the module docs. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum Ty { + I1, + I64, + F64, + Str, +} + +impl Ty { + pub fn name(self) -> &'static str { + match self { + Ty::I1 => "i1", + Ty::I64 => "i64", + Ty::F64 => "f64", + Ty::Str => "str", + } + } +} + +/// Column type: a bare type plus nullability. Only batch columns and scalar +/// statics carry nullability; SSA values never do. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ColTy { + pub ty: Ty, + pub nullable: bool, +} + +/// A named batch column. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Col { + pub name: String, + pub ty: ColTy, +} + +/// Type of a prepare-time static structure, addressed as `@N`. +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum StaticTy { + Scalar(ColTy), + Map { keys: Vec, values: Vec }, +} + +/// SSA value id. Presentation names are not stored; the printer derives +/// `%vN` from the id. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, PartialOrd, Ord)] +pub struct Value(pub u32); + +/// Block id = index into `Program::blocks`. Block 0 is the entry. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub struct BlockId(pub u32); + +/// Literal for `const.*`. `F64` equality is bitwise so that programs +/// containing NaN still satisfy `parse(print(p)) == p` (the text form +/// canonicalizes NaN payloads to the one `nan` token). +#[derive(Clone, Debug)] +pub enum Lit { + I1(bool), + I64(i64), + F64(f64), + Str(String), +} + +impl PartialEq for Lit { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Lit::I1(a), Lit::I1(b)) => a == b, + (Lit::I64(a), Lit::I64(b)) => a == b, + (Lit::F64(a), Lit::F64(b)) => a.to_bits() == b.to_bits(), + (Lit::Str(a), Lit::Str(b)) => a == b, + _ => false, + } + } +} +impl Eq for Lit {} + +impl Lit { + pub fn ty(&self) -> Ty { + match self { + Lit::I1(_) => Ty::I1, + Lit::I64(_) => Ty::I64, + Lit::F64(_) => Ty::F64, + Lit::Str(_) => Ty::Str, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum BinOp { + Iadd, + Isub, + Imul, + Idiv, + Irem, + Fadd, + Fsub, + Fmul, + Fdiv, + And, + Or, + Xor, +} + +impl BinOp { + /// (operand type, result type). Uniform for all v0 binary ops. + pub fn sig(self) -> (Ty, Ty) { + match self { + BinOp::Iadd | BinOp::Isub | BinOp::Imul | BinOp::Idiv | BinOp::Irem => { + (Ty::I64, Ty::I64) + } + BinOp::Fadd | BinOp::Fsub | BinOp::Fmul | BinOp::Fdiv => (Ty::F64, Ty::F64), + BinOp::And | BinOp::Or | BinOp::Xor => (Ty::I1, Ty::I1), + } + } + + pub fn name(self) -> &'static str { + match self { + BinOp::Iadd => "iadd", + BinOp::Isub => "isub", + BinOp::Imul => "imul", + BinOp::Idiv => "idiv", + BinOp::Irem => "irem", + BinOp::Fadd => "fadd", + BinOp::Fsub => "fsub", + BinOp::Fmul => "fmul", + BinOp::Fdiv => "fdiv", + BinOp::And => "and", + BinOp::Or => "or", + BinOp::Xor => "xor", + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum CmpPred { + Eq, + Ne, + Lt, + Le, + Gt, + Ge, +} + +impl CmpPred { + pub fn name(self) -> &'static str { + match self { + CmpPred::Eq => "eq", + CmpPred::Ne => "ne", + CmpPred::Lt => "lt", + CmpPred::Le => "le", + CmpPred::Gt => "gt", + CmpPred::Ge => "ge", + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum RoundMode { + Trunc, + Round, +} + +#[derive(Clone, PartialEq, Debug)] +pub enum Inst { + Const { + dst: Value, + lit: Lit, + }, + Bin { + op: BinOp, + dst: Value, + a: Value, + b: Value, + }, + /// `icmp.P` / `fcmp.P` / `scmp.P` — `ty` is the operand type + /// (I64/F64/Str), result is I1. + Cmp { + pred: CmpPred, + ty: Ty, + dst: Value, + a: Value, + b: Value, + }, + Not { + dst: Value, + a: Value, + }, + /// Branchless pick; `a`/`b` any one type, `cond` I1. + Select { + dst: Value, + cond: Value, + a: Value, + b: Value, + }, + Itof { + dst: Value, + a: Value, + }, + Ftoi { + mode: RoundMode, + dst: Value, + a: Value, + }, + Itos { + dst: Value, + a: Value, + }, + Ftos { + dst: Value, + a: Value, + }, + StoiOpt { + flag: Value, + dst: Value, + a: Value, + }, + StofOpt { + flag: Value, + dst: Value, + a: Value, + }, + Sconcat { + dst: Value, + a: Value, + b: Value, + }, + /// Read a NOT NULL input column at the current row. + Load { + dst: Value, + col: u32, + }, + /// Read a nullable input column: validity flag + payload. + LoadOpt { + flag: Value, + dst: Value, + col: u32, + }, + /// Write a NOT NULL output column at the current row. + Store { + col: u32, + val: Value, + }, + /// Write a nullable output column: flag=false stores NULL. + StoreOpt { + col: u32, + flag: Value, + val: Value, + }, + /// Probe a map static: hit flag + one value per declared value column. + Probe { + static_id: u32, + hit: Value, + dsts: Vec, + keys: Vec, + }, + /// Read a `scalar` static. + Sload { + static_id: u32, + dst: Value, + }, + /// Read a `scalar` static: validity flag + payload. + SloadOpt { + static_id: u32, + flag: Value, + dst: Value, + }, +} + +#[derive(Clone, PartialEq, Debug)] +pub enum Term { + Jump { + to: BlockId, + args: Vec, + }, + Brif { + cond: Value, + then_to: BlockId, + then_args: Vec, + else_to: BlockId, + else_args: Vec, + }, + /// Output row complete; advance both cursors. + Emit, + /// Drop this input row; nothing may have been stored on this path. + Skip, + /// Abort the whole call with a runtime error. + Trap { + msg: String, + }, +} + +#[derive(Clone, PartialEq, Debug)] +pub struct Block { + pub params: Vec<(Value, Ty)>, + pub insts: Vec, + pub term: Term, +} + +#[derive(Clone, PartialEq, Debug)] +pub struct Program { + pub statics: Vec, + pub name: String, + pub in_cols: Vec, + pub out_cols: Vec, + pub blocks: Vec, +} + +impl Inst { + /// Values this instruction defines, in definition order. + pub fn dsts(&self) -> Vec { + match self { + Inst::Const { dst, .. } + | Inst::Bin { dst, .. } + | Inst::Cmp { dst, .. } + | Inst::Not { dst, .. } + | Inst::Select { dst, .. } + | Inst::Itof { dst, .. } + | Inst::Ftoi { dst, .. } + | Inst::Itos { dst, .. } + | Inst::Ftos { dst, .. } + | Inst::Sconcat { dst, .. } + | Inst::Load { dst, .. } + | Inst::Sload { dst, .. } => vec![*dst], + Inst::StoiOpt { flag, dst, .. } + | Inst::StofOpt { flag, dst, .. } + | Inst::LoadOpt { flag, dst, .. } + | Inst::SloadOpt { flag, dst, .. } => vec![*flag, *dst], + Inst::Probe { hit, dsts, .. } => { + let mut all = vec![*hit]; + all.extend(dsts.iter().copied()); + all + } + Inst::Store { .. } | Inst::StoreOpt { .. } => vec![], + } + } + + /// Values this instruction uses. + pub fn uses(&self) -> Vec { + match self { + Inst::Const { .. } + | Inst::Load { .. } + | Inst::LoadOpt { .. } + | Inst::Sload { .. } + | Inst::SloadOpt { .. } => vec![], + Inst::Not { a, .. } + | Inst::Itof { a, .. } + | Inst::Ftoi { a, .. } + | Inst::Itos { a, .. } + | Inst::Ftos { a, .. } + | Inst::StoiOpt { a, .. } + | Inst::StofOpt { a, .. } => vec![*a], + Inst::Bin { a, b, .. } | Inst::Cmp { a, b, .. } | Inst::Sconcat { a, b, .. } => { + vec![*a, *b] + } + Inst::Select { cond, a, b, .. } => vec![*cond, *a, *b], + Inst::Store { val, .. } => vec![*val], + Inst::StoreOpt { flag, val, .. } => vec![*flag, *val], + Inst::Probe { keys, .. } => keys.clone(), + } + } +} + +impl Term { + /// Successor blocks with their branch arguments. + pub fn successors(&self) -> Vec<(BlockId, &[Value])> { + match self { + Term::Jump { to, args } => vec![(*to, args.as_slice())], + Term::Brif { + then_to, + then_args, + else_to, + else_args, + .. + } => vec![ + (*then_to, then_args.as_slice()), + (*else_to, else_args.as_slice()), + ], + Term::Emit | Term::Skip | Term::Trap { .. } => vec![], + } + } + + pub fn uses(&self) -> Vec { + let mut out = Vec::new(); + if let Term::Brif { cond, .. } = self { + out.push(*cond); + } + for (_, args) in self.successors() { + out.extend_from_slice(args); + } + out + } +} + +/// Builds programs with dense, definition-ordered value ids — the canonical +/// form for which `parse(print(p)) == p` holds exactly. Lowering (M-lower) +/// and the fuzz generator both construct through this. +pub struct Builder { + next: u32, +} + +impl Builder { + #[allow(clippy::new_without_default)] + pub fn new() -> Builder { + Builder { next: 0 } + } + + pub fn fresh(&mut self) -> Value { + let v = Value(self.next); + self.next += 1; + v + } +} diff --git a/src/specializer/ir/parse.rs b/src/specializer/ir/parse.rs new file mode 100644 index 0000000..607b3ce --- /dev/null +++ b/src/specializer/ir/parse.rs @@ -0,0 +1,1046 @@ +//! Parser for the IR text format — the inverse of [`print`]. Hand-rolled +//! lexer + recursive descent; no dependencies. The parser owns *syntax* only: +//! it interns value names to dense, definition-ordered ids (the canonical +//! form) and rejects malformed text, redefinition of a value name, use of an +//! unknown value/label/column, and unknown opcodes. Everything semantic +//! (types, SSA scoping, CFG shape, store completeness) is the verifier's job. +//! +//! [`print`]: super::print + +use std::collections::HashMap; + +use super::{ + Block, BlockId, BinOp, Col, ColTy, CmpPred, Inst, Lit, Program, RoundMode, StaticTy, Term, + Ty, Value, +}; + +#[derive(Debug)] +pub struct ParseError { + pub line: u32, + pub msg: String, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "line {}: {}", self.line, self.msg) + } +} + +pub fn parse(text: &str) -> Result { + let tokens = lex(text)?; + Parser { + toks: tokens, + pos: 0, + values: HashMap::new(), + next_value: 0, + } + .program() +} + +// ---------------------------------------------------------------- lexer -- + +#[derive(Clone, PartialEq, Debug)] +enum Tok { + Ident(String), + Val(String), // %name + Num(String), // integer or float text, optional leading '-' + Str(String), // unescaped content + At, // @ + Comma, + Colon, + Dot, + Eq, + Question, + Arrow, // -> + LParen, + RParen, + LBrace, + RBrace, + Lt, + Gt, + Eof, +} + +impl Tok { + fn show(&self) -> String { + match self { + Tok::Ident(s) => format!("'{s}'"), + Tok::Val(s) => format!("'%{s}'"), + Tok::Num(s) => format!("number '{s}'"), + Tok::Str(_) => "string literal".to_string(), + Tok::At => "'@'".to_string(), + Tok::Comma => "','".to_string(), + Tok::Colon => "':'".to_string(), + Tok::Dot => "'.'".to_string(), + Tok::Eq => "'='".to_string(), + Tok::Question => "'?'".to_string(), + Tok::Arrow => "'->'".to_string(), + Tok::LParen => "'('".to_string(), + Tok::RParen => "')'".to_string(), + Tok::LBrace => "'{'".to_string(), + Tok::RBrace => "'}'".to_string(), + Tok::Lt => "'<'".to_string(), + Tok::Gt => "'>'".to_string(), + Tok::Eof => "end of input".to_string(), + } + } +} + +fn lex(text: &str) -> Result, ParseError> { + let mut out = Vec::new(); + let mut chars = text.chars().peekable(); + let mut line: u32 = 1; + while let Some(&c) = chars.peek() { + match c { + '\n' => { + line += 1; + chars.next(); + } + c if c.is_whitespace() => { + chars.next(); + } + '#' => { + // comment to end of line + for c in chars.by_ref() { + if c == '\n' { + line += 1; + break; + } + } + } + '@' => { + chars.next(); + out.push((Tok::At, line)); + } + ',' => { + chars.next(); + out.push((Tok::Comma, line)); + } + ':' => { + chars.next(); + out.push((Tok::Colon, line)); + } + '.' => { + chars.next(); + out.push((Tok::Dot, line)); + } + '=' => { + chars.next(); + out.push((Tok::Eq, line)); + } + '?' => { + chars.next(); + out.push((Tok::Question, line)); + } + '(' => { + chars.next(); + out.push((Tok::LParen, line)); + } + ')' => { + chars.next(); + out.push((Tok::RParen, line)); + } + '{' => { + chars.next(); + out.push((Tok::LBrace, line)); + } + '}' => { + chars.next(); + out.push((Tok::RBrace, line)); + } + '<' => { + chars.next(); + out.push((Tok::Lt, line)); + } + '>' => { + chars.next(); + out.push((Tok::Gt, line)); + } + '%' => { + chars.next(); + let name = take_ident(&mut chars); + if name.is_empty() { + return Err(ParseError { + line, + msg: "expected a value name after '%'".to_string(), + }); + } + out.push((Tok::Val(name), line)); + } + '"' => { + chars.next(); + let mut s = String::new(); + loop { + match chars.next() { + None => { + return Err(ParseError { + line, + msg: "unterminated string literal".to_string(), + }) + } + Some('"') => break, + Some('\\') => match chars.next() { + Some('"') => s.push('"'), + Some('\\') => s.push('\\'), + Some('n') => s.push('\n'), + Some('r') => s.push('\r'), + Some('t') => s.push('\t'), + Some('u') => { + if chars.next() != Some('{') { + return Err(ParseError { + line, + msg: "expected '{' after \\u".to_string(), + }); + } + let mut hex = String::new(); + loop { + match chars.next() { + Some('}') => break, + Some(h) if h.is_ascii_hexdigit() => hex.push(h), + _ => { + return Err(ParseError { + line, + msg: "bad \\u{...} escape".to_string(), + }) + } + } + } + let cp = u32::from_str_radix(&hex, 16).ok(); + match cp.and_then(char::from_u32) { + Some(c) => s.push(c), + None => { + return Err(ParseError { + line, + msg: format!("bad codepoint in \\u{{{hex}}}"), + }) + } + } + } + other => { + return Err(ParseError { + line, + msg: format!("unknown escape: \\{}", show_esc(other)), + }) + } + }, + Some('\n') => { + return Err(ParseError { + line, + msg: "newline in string literal (use \\n)".to_string(), + }) + } + Some(c) => s.push(c), + } + } + out.push((Tok::Str(s), line)); + } + '-' => { + chars.next(); + match chars.peek() { + Some('>') => { + chars.next(); + out.push((Tok::Arrow, line)); + } + Some(c) if c.is_ascii_digit() => { + let num = take_number(&mut chars); + out.push((Tok::Num(format!("-{num}")), line)); + } + Some('i') => { + let word = take_ident(&mut chars); + if word == "inf" { + out.push((Tok::Num("-inf".to_string()), line)); + } else { + return Err(ParseError { + line, + msg: format!("unexpected '-{word}'"), + }); + } + } + _ => { + return Err(ParseError { + line, + msg: "unexpected '-'".to_string(), + }) + } + } + } + c if c.is_ascii_digit() => { + let num = take_number(&mut chars); + out.push((Tok::Num(num), line)); + } + c if c.is_ascii_alphabetic() || c == '_' => { + let word = take_ident(&mut chars); + out.push((Tok::Ident(word), line)); + } + other => { + return Err(ParseError { + line, + msg: format!("unexpected character '{other}'"), + }) + } + } + } + out.push((Tok::Eof, line)); + Ok(out) +} + +fn show_esc(c: Option) -> String { + match c { + Some(c) => c.to_string(), + None => "".to_string(), + } +} + +fn take_ident(chars: &mut std::iter::Peekable>) -> String { + let mut s = String::new(); + while let Some(&c) = chars.peek() { + if c.is_ascii_alphanumeric() || c == '_' { + s.push(c); + chars.next(); + } else { + break; + } + } + s +} + +/// Number text: digits, optional fraction, optional exponent. The '.' is +/// consumed only when followed by a digit, so `1.` never eats a field access +/// dot (which cannot follow a number anyway in this grammar). +fn take_number(chars: &mut std::iter::Peekable>) -> String { + let mut s = String::new(); + while let Some(&c) = chars.peek() { + if c.is_ascii_digit() { + s.push(c); + chars.next(); + } else { + break; + } + } + if let Some('.') = chars.peek() { + let mut ahead = chars.clone(); + ahead.next(); + if matches!(ahead.peek(), Some(d) if d.is_ascii_digit()) { + s.push('.'); + chars.next(); + while let Some(&c) = chars.peek() { + if c.is_ascii_digit() { + s.push(c); + chars.next(); + } else { + break; + } + } + } + } + if matches!(chars.peek(), Some('e') | Some('E')) { + let mut ahead = chars.clone(); + ahead.next(); + let sign = matches!(ahead.peek(), Some('+') | Some('-')); + if sign { + ahead.next(); + } + if matches!(ahead.peek(), Some(d) if d.is_ascii_digit()) { + s.push(chars.next().unwrap()); + if sign { + s.push(chars.next().unwrap()); + } + while let Some(&c) = chars.peek() { + if c.is_ascii_digit() { + s.push(c); + chars.next(); + } else { + break; + } + } + } + } + s +} + +// --------------------------------------------------------------- parser -- + +struct Parser { + toks: Vec<(Tok, u32)>, + pos: usize, + values: HashMap, + next_value: u32, +} + +/// A block body before label resolution: terminator targets are still names. +struct RawBlock { + label: String, + params: Vec<(Value, Ty)>, + insts: Vec, + term: RawTerm, + line: u32, +} + +enum RawTerm { + Jump(String, Vec), + Brif(Value, (String, Vec), (String, Vec)), + Emit, + Skip, + Trap(String), +} + +impl Parser { + fn peek(&self) -> &Tok { + &self.toks[self.pos].0 + } + + fn line(&self) -> u32 { + self.toks[self.pos].1 + } + + fn bump(&mut self) -> Tok { + let t = self.toks[self.pos].0.clone(); + if self.pos + 1 < self.toks.len() { + self.pos += 1; + } + t + } + + fn err(&self, msg: impl Into) -> ParseError { + ParseError { + line: self.line(), + msg: msg.into(), + } + } + + fn expect(&mut self, tok: Tok) -> Result<(), ParseError> { + if *self.peek() == tok { + self.bump(); + Ok(()) + } else { + Err(self.err(format!("expected {}, found {}", tok.show(), self.peek().show()))) + } + } + + fn ident(&mut self, what: &str) -> Result { + match self.bump() { + Tok::Ident(s) => Ok(s), + other => Err(self.err(format!("expected {what}, found {}", other.show()))), + } + } + + /// Keyword = identifier with a required spelling. + fn keyword(&mut self, kw: &str) -> Result<(), ParseError> { + match self.peek() { + Tok::Ident(s) if s == kw => { + self.bump(); + Ok(()) + } + other => Err(self.err(format!("expected '{kw}', found {}", other.show()))), + } + } + + fn define_value(&mut self, name: String) -> Result { + if self.values.contains_key(&name) { + return Err(self.err(format!("value '%{name}' defined twice"))); + } + let v = Value(self.next_value); + self.next_value += 1; + self.values.insert(name, v); + Ok(v) + } + + fn use_value(&mut self) -> Result { + match self.bump() { + Tok::Val(name) => self + .values + .get(&name) + .copied() + .ok_or_else(|| self.err(format!("use of undefined value '%{name}'"))), + other => Err(self.err(format!("expected a %value, found {}", other.show()))), + } + } + + fn program(mut self) -> Result { + let mut statics = Vec::new(); + while matches!(self.peek(), Tok::Ident(s) if s == "static") { + self.bump(); + self.expect(Tok::At)?; + let idx = self.int_literal("static index")?; + if idx != statics.len() as i64 { + return Err(self.err(format!( + "static ids must be dense and in order: expected @{}, found @{idx}", + statics.len() + ))); + } + self.expect(Tok::Colon)?; + statics.push(self.static_ty()?); + } + + self.keyword("fn")?; + let name = self.ident("function name")?; + self.expect(Tok::LParen)?; + self.keyword("in")?; + self.expect(Tok::Colon)?; + let in_cols = self.batch()?; + self.expect(Tok::Comma)?; + self.keyword("out")?; + self.expect(Tok::Colon)?; + let out_cols = self.batch()?; + self.expect(Tok::RParen)?; + self.expect(Tok::LBrace)?; + + let mut raw_blocks: Vec = Vec::new(); + while *self.peek() != Tok::RBrace { + let block = self.block(&statics, &in_cols, &out_cols)?; + raw_blocks.push(block); + } + self.expect(Tok::RBrace)?; + if *self.peek() != Tok::Eof { + return Err(self.err(format!("trailing input: {}", self.peek().show()))); + } + if raw_blocks.is_empty() { + return Err(self.err("a function needs at least one block")); + } + + // Resolve labels. + let mut label_ids: HashMap = HashMap::new(); + for (i, rb) in raw_blocks.iter().enumerate() { + if label_ids.insert(rb.label.clone(), BlockId(i as u32)).is_some() { + return Err(ParseError { + line: rb.line, + msg: format!("block label '{}' defined twice", rb.label), + }); + } + } + let resolve = |label: &str, line: u32| -> Result { + label_ids.get(label).copied().ok_or(ParseError { + line, + msg: format!("jump to unknown block '{label}'"), + }) + }; + let mut blocks = Vec::with_capacity(raw_blocks.len()); + for rb in raw_blocks { + let term = match rb.term { + RawTerm::Jump(label, args) => Term::Jump { + to: resolve(&label, rb.line)?, + args, + }, + RawTerm::Brif(cond, (tl, ta), (el, ea)) => Term::Brif { + cond, + then_to: resolve(&tl, rb.line)?, + then_args: ta, + else_to: resolve(&el, rb.line)?, + else_args: ea, + }, + RawTerm::Emit => Term::Emit, + RawTerm::Skip => Term::Skip, + RawTerm::Trap(msg) => Term::Trap { msg }, + }; + blocks.push(Block { + params: rb.params, + insts: rb.insts, + term, + }); + } + + Ok(Program { + statics, + name, + in_cols, + out_cols, + blocks, + }) + } + + fn int_literal(&mut self, what: &str) -> Result { + match self.bump() { + Tok::Num(s) => s + .parse::() + .map_err(|_| self.err(format!("bad {what}: '{s}'"))), + other => Err(self.err(format!("expected {what}, found {}", other.show()))), + } + } + + fn static_ty(&mut self) -> Result { + let kind = self.ident("'scalar' or 'map'")?; + match kind.as_str() { + "scalar" => { + self.expect(Tok::Lt)?; + let ct = self.col_ty()?; + self.expect(Tok::Gt)?; + Ok(StaticTy::Scalar(ct)) + } + "map" => { + self.expect(Tok::LParen)?; + let keys = self.ty_list()?; + self.expect(Tok::RParen)?; + self.expect(Tok::Arrow)?; + self.expect(Tok::LParen)?; + let values = self.ty_list()?; + self.expect(Tok::RParen)?; + Ok(StaticTy::Map { keys, values }) + } + other => Err(self.err(format!("expected 'scalar' or 'map', found '{other}'"))), + } + } + + fn ty_list(&mut self) -> Result, ParseError> { + let mut list = vec![self.ty()?]; + while *self.peek() == Tok::Comma { + self.bump(); + list.push(self.ty()?); + } + Ok(list) + } + + fn ty(&mut self) -> Result { + let name = self.ident("a type")?; + match name.as_str() { + "i1" => Ok(Ty::I1), + "i64" => Ok(Ty::I64), + "f64" => Ok(Ty::F64), + "str" => Ok(Ty::Str), + other => Err(self.err(format!("unknown type '{other}'"))), + } + } + + fn col_ty(&mut self) -> Result { + let ty = self.ty()?; + let nullable = if *self.peek() == Tok::Question { + self.bump(); + true + } else { + false + }; + Ok(ColTy { ty, nullable }) + } + + fn batch(&mut self) -> Result, ParseError> { + self.keyword("batch")?; + self.expect(Tok::LBrace)?; + let mut cols = Vec::new(); + if *self.peek() != Tok::RBrace { + loop { + let name = match self.bump() { + Tok::Ident(s) => s, + Tok::Str(s) => s, + other => { + return Err( + self.err(format!("expected a column name, found {}", other.show())) + ) + } + }; + if cols.iter().any(|c: &Col| c.name == name) { + return Err(self.err(format!("column '{name}' declared twice"))); + } + self.expect(Tok::Colon)?; + let ty = self.col_ty()?; + cols.push(Col { name, ty }); + if *self.peek() == Tok::Comma { + self.bump(); + } else { + break; + } + } + } + self.expect(Tok::RBrace)?; + Ok(cols) + } + + fn block( + &mut self, + statics: &[StaticTy], + in_cols: &[Col], + out_cols: &[Col], + ) -> Result { + let line = self.line(); + let label = self.ident("a block label")?; + let mut params = Vec::new(); + if *self.peek() == Tok::LParen { + self.bump(); + loop { + let name = match self.bump() { + Tok::Val(n) => n, + other => { + return Err( + self.err(format!("expected a %param, found {}", other.show())) + ) + } + }; + self.expect(Tok::Colon)?; + let ty = self.ty()?; + let v = self.define_value(name)?; + params.push((v, ty)); + if *self.peek() == Tok::Comma { + self.bump(); + } else { + break; + } + } + self.expect(Tok::RParen)?; + } + self.expect(Tok::Colon)?; + + let mut insts = Vec::new(); + let term = loop { + if let Some(term) = self.try_terminator()? { + break term; + } + insts.push(self.inst(statics, in_cols, out_cols)?); + }; + Ok(RawBlock { + label, + params, + insts, + term, + line, + }) + } + + fn try_terminator(&mut self) -> Result, ParseError> { + let kw = match self.peek() { + Tok::Ident(s) => s.clone(), + _ => return Ok(None), + }; + match kw.as_str() { + "emit" => { + self.bump(); + Ok(Some(RawTerm::Emit)) + } + "skip" => { + self.bump(); + Ok(Some(RawTerm::Skip)) + } + "trap" => { + self.bump(); + match self.bump() { + Tok::Str(msg) => Ok(Some(RawTerm::Trap(msg))), + other => Err(self.err(format!( + "expected a string message after 'trap', found {}", + other.show() + ))), + } + } + "jump" => { + self.bump(); + let (label, args) = self.target()?; + Ok(Some(RawTerm::Jump(label, args))) + } + "brif" => { + self.bump(); + let cond = self.use_value()?; + self.expect(Tok::Comma)?; + let then_t = self.target()?; + self.expect(Tok::Comma)?; + let else_t = self.target()?; + Ok(Some(RawTerm::Brif(cond, then_t, else_t))) + } + _ => Ok(None), + } + } + + fn target(&mut self) -> Result<(String, Vec), ParseError> { + let label = self.ident("a block label")?; + let mut args = Vec::new(); + if *self.peek() == Tok::LParen { + self.bump(); + loop { + args.push(self.use_value()?); + if *self.peek() == Tok::Comma { + self.bump(); + } else { + break; + } + } + self.expect(Tok::RParen)?; + } + Ok((label, args)) + } + + /// One instruction line: `dsts = opcode operands`. + fn inst( + &mut self, + statics: &[StaticTy], + in_cols: &[Col], + out_cols: &[Col], + ) -> Result { + // store/store.opt have no dsts and start with an Ident. + if matches!(self.peek(), Tok::Ident(s) if s == "store") { + self.bump(); + let opt = self.dot_suffix()?; + let col = self.col_ref("out", out_cols)?; + self.expect(Tok::Comma)?; + return match opt.as_deref() { + None => { + let val = self.use_value()?; + Ok(Inst::Store { col, val }) + } + Some("opt") => { + let flag = self.use_value()?; + self.expect(Tok::Comma)?; + let val = self.use_value()?; + Ok(Inst::StoreOpt { col, flag, val }) + } + Some(other) => Err(self.err(format!("unknown opcode 'store.{other}'"))), + }; + } + + // Everything else: one or more dsts, '=', opcode. + let mut dst_names = Vec::new(); + loop { + match self.bump() { + Tok::Val(n) => dst_names.push(n), + other => { + return Err(self.err(format!( + "expected an instruction or terminator, found {}", + other.show() + ))) + } + } + if *self.peek() == Tok::Comma { + self.bump(); + } else { + break; + } + } + self.expect(Tok::Eq)?; + let head = self.ident("an opcode")?; + let suffix = self.dot_suffix()?; + let opcode = match &suffix { + Some(sfx) => format!("{head}.{sfx}"), + None => head.clone(), + }; + + let want_dsts = |n: usize, this: &Parser| -> Result<(), ParseError> { + if dst_names.len() != n { + Err(this.err(format!( + "'{opcode}' defines {n} value(s), found {}", + dst_names.len() + ))) + } else { + Ok(()) + } + }; + + // Helper closures cannot borrow self mutably twice; do defs inline. + macro_rules! def { + ($i:expr) => { + self.define_value(dst_names[$i].clone())? + }; + } + + let inst = match opcode.as_str() { + "const.i1" | "const.i64" | "const.f64" | "const.str" => { + want_dsts(1, self)?; + let lit = match opcode.as_str() { + "const.i1" => match self.ident("'true' or 'false'")?.as_str() { + "true" => Lit::I1(true), + "false" => Lit::I1(false), + other => { + return Err( + self.err(format!("expected 'true' or 'false', found '{other}'")) + ) + } + }, + "const.i64" => Lit::I64(self.int_literal("integer literal")?), + "const.f64" => Lit::F64(self.f64_literal()?), + _ => match self.bump() { + Tok::Str(s) => Lit::Str(s), + other => { + return Err( + self.err(format!("expected a string, found {}", other.show())) + ) + } + }, + }; + Inst::Const { dst: def!(0), lit } + } + "iadd" | "isub" | "imul" | "idiv" | "irem" | "fadd" | "fsub" | "fmul" | "fdiv" + | "and" | "or" | "xor" => { + want_dsts(1, self)?; + let op = match opcode.as_str() { + "iadd" => BinOp::Iadd, + "isub" => BinOp::Isub, + "imul" => BinOp::Imul, + "idiv" => BinOp::Idiv, + "irem" => BinOp::Irem, + "fadd" => BinOp::Fadd, + "fsub" => BinOp::Fsub, + "fmul" => BinOp::Fmul, + "fdiv" => BinOp::Fdiv, + "and" => BinOp::And, + "or" => BinOp::Or, + _ => BinOp::Xor, + }; + let a = self.use_value()?; + self.expect(Tok::Comma)?; + let b = self.use_value()?; + Inst::Bin { op, dst: def!(0), a, b } + } + _ if head == "icmp" || head == "fcmp" || head == "scmp" => { + want_dsts(1, self)?; + let ty = match head.as_str() { + "icmp" => Ty::I64, + "fcmp" => Ty::F64, + _ => Ty::Str, + }; + let pred = match suffix.as_deref() { + Some("eq") => CmpPred::Eq, + Some("ne") => CmpPred::Ne, + Some("lt") => CmpPred::Lt, + Some("le") => CmpPred::Le, + Some("gt") => CmpPred::Gt, + Some("ge") => CmpPred::Ge, + _ => return Err(self.err(format!("unknown opcode '{opcode}'"))), + }; + let a = self.use_value()?; + self.expect(Tok::Comma)?; + let b = self.use_value()?; + Inst::Cmp { pred, ty, dst: def!(0), a, b } + } + "not" => { + want_dsts(1, self)?; + let a = self.use_value()?; + Inst::Not { dst: def!(0), a } + } + "select" => { + want_dsts(1, self)?; + let cond = self.use_value()?; + self.expect(Tok::Comma)?; + let a = self.use_value()?; + self.expect(Tok::Comma)?; + let b = self.use_value()?; + Inst::Select { dst: def!(0), cond, a, b } + } + "itof" => { + want_dsts(1, self)?; + let a = self.use_value()?; + Inst::Itof { dst: def!(0), a } + } + "ftoi.trunc" | "ftoi.round" => { + want_dsts(1, self)?; + let mode = if opcode.ends_with("trunc") { + RoundMode::Trunc + } else { + RoundMode::Round + }; + let a = self.use_value()?; + Inst::Ftoi { mode, dst: def!(0), a } + } + "itos" => { + want_dsts(1, self)?; + let a = self.use_value()?; + Inst::Itos { dst: def!(0), a } + } + "ftos" => { + want_dsts(1, self)?; + let a = self.use_value()?; + Inst::Ftos { dst: def!(0), a } + } + "stoi.opt" | "stof.opt" => { + want_dsts(2, self)?; + let a = self.use_value()?; + let flag = def!(0); + let dst = def!(1); + if opcode.starts_with("stoi") { + Inst::StoiOpt { flag, dst, a } + } else { + Inst::StofOpt { flag, dst, a } + } + } + "sconcat" => { + want_dsts(1, self)?; + let a = self.use_value()?; + self.expect(Tok::Comma)?; + let b = self.use_value()?; + Inst::Sconcat { dst: def!(0), a, b } + } + "load" => { + want_dsts(1, self)?; + let col = self.col_ref("in", in_cols)?; + Inst::Load { dst: def!(0), col } + } + "load.opt" => { + want_dsts(2, self)?; + let col = self.col_ref("in", in_cols)?; + Inst::LoadOpt { flag: def!(0), dst: def!(1), col } + } + "probe" => { + if dst_names.is_empty() { + return Err(self.err("'probe' defines at least a hit flag")); + } + let static_id = self.static_ref(statics)?; + let mut keys = Vec::new(); + while *self.peek() == Tok::Comma { + self.bump(); + keys.push(self.use_value()?); + } + let hit = def!(0); + let mut dsts = Vec::with_capacity(dst_names.len() - 1); + for i in 1..dst_names.len() { + dsts.push(def!(i)); + } + Inst::Probe { static_id, hit, dsts, keys } + } + "sload" => { + want_dsts(1, self)?; + let static_id = self.static_ref(statics)?; + Inst::Sload { static_id, dst: def!(0) } + } + "sload.opt" => { + want_dsts(2, self)?; + let static_id = self.static_ref(statics)?; + Inst::SloadOpt { static_id, flag: def!(0), dst: def!(1) } + } + other => return Err(self.err(format!("unknown opcode '{other}'"))), + }; + Ok(inst) + } + + fn dot_suffix(&mut self) -> Result, ParseError> { + if *self.peek() == Tok::Dot { + self.bump(); + Ok(Some(self.ident("an opcode suffix")?)) + } else { + Ok(None) + } + } + + /// `in.NAME` / `out.NAME` (NAME bare or quoted) -> column index. + fn col_ref(&mut self, side: &str, cols: &[Col]) -> Result { + self.keyword(side)?; + self.expect(Tok::Dot)?; + let name = match self.bump() { + Tok::Ident(s) => s, + Tok::Str(s) => s, + other => { + return Err(self.err(format!("expected a column name, found {}", other.show()))) + } + }; + cols.iter() + .position(|c| c.name == name) + .map(|i| i as u32) + .ok_or_else(|| self.err(format!("unknown column '{side}.{name}'"))) + } + + fn static_ref(&mut self, statics: &[StaticTy]) -> Result { + self.expect(Tok::At)?; + let idx = self.int_literal("static index")?; + if idx < 0 || idx as usize >= statics.len() { + return Err(self.err(format!("unknown static '@{idx}'"))); + } + Ok(idx as u32) + } + + fn f64_literal(&mut self) -> Result { + match self.bump() { + Tok::Num(s) => { + if s == "-inf" { + Ok(f64::NEG_INFINITY) + } else { + s.parse::() + .map_err(|_| self.err(format!("bad float literal '{s}'"))) + } + } + Tok::Ident(s) if s == "inf" => Ok(f64::INFINITY), + Tok::Ident(s) if s == "nan" => Ok(f64::NAN), + other => Err(self.err(format!("expected a float literal, found {}", other.show()))), + } + } +} diff --git a/src/specializer/ir/print.rs b/src/specializer/ir/print.rs new file mode 100644 index 0000000..9297fb9 --- /dev/null +++ b/src/specializer/ir/print.rs @@ -0,0 +1,280 @@ +//! Canonical text form of a [`Program`]. The inverse of [`parse`]; every +//! choice here (names, spacing, literal forms) is pinned by the round-trip +//! property, so change both sides together or not at all. +//! +//! [`parse`]: super::parse + +use std::fmt::Write; + +use super::{Block, ColTy, Inst, Lit, Program, StaticTy, Term, Ty, Value}; + +pub fn print(p: &Program) -> String { + let mut s = String::new(); + for (i, st) in p.statics.iter().enumerate() { + let _ = write!(s, "static @{i}: "); + match st { + StaticTy::Scalar(ct) => { + let _ = writeln!(s, "scalar<{}>", col_ty(*ct)); + } + StaticTy::Map { keys, values } => { + let _ = writeln!(s, "map({}) -> ({})", tys(keys), tys(values)); + } + } + } + if !p.statics.is_empty() { + s.push('\n'); + } + let _ = writeln!( + s, + "fn {}(in: {}, out: {}) {{", + p.name, + batch(&p.in_cols), + batch(&p.out_cols) + ); + for (bi, b) in p.blocks.iter().enumerate() { + let _ = write!(s, "b{bi}"); + if !b.params.is_empty() { + let params: Vec = b + .params + .iter() + .map(|(v, t)| format!("{}: {}", val(*v), t.name())) + .collect(); + let _ = write!(s, "({})", params.join(", ")); + } + s.push_str(":\n"); + for inst in &b.insts { + s.push_str(" "); + print_inst(&mut s, p, inst); + s.push('\n'); + } + s.push_str(" "); + print_term(&mut s, b); + s.push('\n'); + } + s.push_str("}\n"); + s +} + +fn print_inst(s: &mut String, p: &Program, inst: &Inst) { + let dsts = inst.dsts(); + if !dsts.is_empty() { + let names: Vec = dsts.iter().map(|v| val(*v)).collect(); + let _ = write!(s, "{} = ", names.join(", ")); + } + match inst { + Inst::Const { lit, .. } => match lit { + Lit::I1(b) => { + let _ = write!(s, "const.i1 {b}"); + } + Lit::I64(i) => { + let _ = write!(s, "const.i64 {i}"); + } + Lit::F64(f) => { + let _ = write!(s, "const.f64 {}", f64_text(*f)); + } + Lit::Str(t) => { + let _ = write!(s, "const.str {}", quote(t)); + } + }, + Inst::Bin { op, a, b, .. } => { + let _ = write!(s, "{} {}, {}", op.name(), val(*a), val(*b)); + } + Inst::Cmp { pred, ty, a, b, .. } => { + let prefix = match ty { + Ty::I64 => "icmp", + Ty::F64 => "fcmp", + Ty::Str => "scmp", + // Unreachable in verified programs; printed anyway so a bad + // program still prints for diagnostics. + Ty::I1 => "icmp", + }; + let _ = write!(s, "{prefix}.{} {}, {}", pred.name(), val(*a), val(*b)); + } + Inst::Not { a, .. } => { + let _ = write!(s, "not {}", val(*a)); + } + Inst::Select { cond, a, b, .. } => { + let _ = write!(s, "select {}, {}, {}", val(*cond), val(*a), val(*b)); + } + Inst::Itof { a, .. } => { + let _ = write!(s, "itof {}", val(*a)); + } + Inst::Ftoi { mode, a, .. } => { + let m = match mode { + super::RoundMode::Trunc => "trunc", + super::RoundMode::Round => "round", + }; + let _ = write!(s, "ftoi.{m} {}", val(*a)); + } + Inst::Itos { a, .. } => { + let _ = write!(s, "itos {}", val(*a)); + } + Inst::Ftos { a, .. } => { + let _ = write!(s, "ftos {}", val(*a)); + } + Inst::StoiOpt { a, .. } => { + let _ = write!(s, "stoi.opt {}", val(*a)); + } + Inst::StofOpt { a, .. } => { + let _ = write!(s, "stof.opt {}", val(*a)); + } + Inst::Sconcat { a, b, .. } => { + let _ = write!(s, "sconcat {}, {}", val(*a), val(*b)); + } + Inst::Load { col, .. } => { + let _ = write!(s, "load in.{}", col_name(p, false, *col)); + } + Inst::LoadOpt { col, .. } => { + let _ = write!(s, "load.opt in.{}", col_name(p, false, *col)); + } + Inst::Store { col, val: v } => { + let _ = write!(s, "store out.{}, {}", col_name(p, true, *col), val(*v)); + } + Inst::StoreOpt { col, flag, val: v } => { + let _ = write!( + s, + "store.opt out.{}, {}, {}", + col_name(p, true, *col), + val(*flag), + val(*v) + ); + } + Inst::Probe { + static_id, keys, .. + } => { + let ks: Vec = keys.iter().map(|k| val(*k)).collect(); + let _ = write!(s, "probe @{static_id}, {}", ks.join(", ")); + } + Inst::Sload { static_id, .. } => { + let _ = write!(s, "sload @{static_id}"); + } + Inst::SloadOpt { static_id, .. } => { + let _ = write!(s, "sload.opt @{static_id}"); + } + } +} + +fn print_term(s: &mut String, b: &Block) { + match &b.term { + Term::Jump { to, args } => { + let _ = write!(s, "jump {}", target(to.0, args)); + } + Term::Brif { + cond, + then_to, + then_args, + else_to, + else_args, + } => { + let _ = write!( + s, + "brif {}, {}, {}", + val(*cond), + target(then_to.0, then_args), + target(else_to.0, else_args) + ); + } + Term::Emit => s.push_str("emit"), + Term::Skip => s.push_str("skip"), + Term::Trap { msg } => { + let _ = write!(s, "trap {}", quote(msg)); + } + } +} + +fn target(block: u32, args: &[Value]) -> String { + if args.is_empty() { + format!("b{block}") + } else { + let a: Vec = args.iter().map(|v| val(*v)).collect(); + format!("b{block}({})", a.join(", ")) + } +} + +fn val(v: Value) -> String { + format!("%v{}", v.0) +} + +fn tys(list: &[Ty]) -> String { + list.iter() + .map(|t| t.name()) + .collect::>() + .join(", ") +} + +fn col_ty(ct: ColTy) -> String { + if ct.nullable { + format!("{}?", ct.ty.name()) + } else { + ct.ty.name().to_string() + } +} + +fn batch(cols: &[super::Col]) -> String { + let inner: Vec = cols + .iter() + .map(|c| format!("{}: {}", ident_or_quoted(&c.name), col_ty(c.ty))) + .collect(); + format!("batch{{{}}}", inner.join(", ")) +} + +fn col_name(p: &Program, out: bool, idx: u32) -> String { + let cols = if out { &p.out_cols } else { &p.in_cols }; + match cols.get(idx as usize) { + Some(c) => ident_or_quoted(&c.name), + // Out-of-range in an unverified program; keep printing diagnosable. + None => format!("\"\""), + } +} + +/// Column and function names print bare when they are identifiers, quoted +/// otherwise (SQL-derived output names like `COALESCE(a, b)` survive). +fn ident_or_quoted(name: &str) -> String { + if is_ident(name) { + name.to_string() + } else { + quote(name) + } +} + +pub(super) fn is_ident(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// `f64` text form: `{:?}` for finite values (guaranteed shortest +/// round-trip), dedicated tokens for the specials. NaN payloads collapse to +/// the single canonical `nan`. +pub(super) fn f64_text(f: f64) -> String { + if f.is_nan() { + "nan".to_string() + } else if f == f64::INFINITY { + "inf".to_string() + } else if f == f64::NEG_INFINITY { + "-inf".to_string() + } else { + format!("{f:?}") + } +} + +pub(super) fn quote(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\u{{{:x}}}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} diff --git a/src/specializer/ir/tests.rs b/src/specializer/ir/tests.rs new file mode 100644 index 0000000..9fe6d2e --- /dev/null +++ b/src/specializer/ir/tests.rs @@ -0,0 +1,505 @@ +//! IR boundary tests: fixtures round-trip and verify; every verifier rule +//! has a program that trips it; every parser guard has a text that trips it; +//! seeded fuzz pins `parse(print(p)) == p` across the generator's whole +//! surface. + +use super::{fixtures, gen, parse::parse, print::print, verify::verify, Program}; + +fn parsed(text: &str) -> Program { + match parse(text) { + Ok(p) => p, + Err(e) => panic!("parse failed: {e}\n---\n{text}"), + } +} + +fn verified(text: &str) -> Program { + let p = parsed(text); + if let Err(errs) = verify(&p) { + let msgs: Vec = errs.iter().map(|e| e.to_string()).collect(); + panic!("verify failed: {}\n---\n{text}", msgs.join("; ")); + } + p +} + +// ------------------------------------------------------------- fixtures -- + +#[test] +fn fixtures_verify_and_round_trip() { + for (name, text) in fixtures::all() { + let p = verified(text); + let printed = print(&p); + let p2 = parsed(&printed); + assert_eq!(p2, p, "round-trip changed '{name}':\n{printed}"); + assert_eq!(print(&p2), printed, "printing is not a fixpoint for '{name}'"); + verify(&p2).unwrap_or_else(|_| panic!("canonical form of '{name}' fails verify")); + } +} + +/// Every opcode and terminator must appear in at least one fixture — this is +/// the acceptance criterion "hand-written programs covering every +/// instruction", kept honest mechanically. +#[test] +fn fixtures_cover_every_opcode() { + let all: String = fixtures::all().iter().map(|(_, t)| *t).collect(); + let opcodes = [ + "const.i1", "const.i64", "const.f64", "const.str", "iadd", "isub", "imul", "idiv", + "irem", "fadd", "fsub", "fmul", "fdiv", "and", "or", "xor", "not", "icmp.", "fcmp.", + "scmp.", "select", "itof", "ftoi.trunc", "ftoi.round", "itos", "ftos", "stoi.opt", + "stof.opt", "sconcat", "load in.", "load.opt in.", "store out.", "store.opt out.", + "probe @", "sload @", "sload.opt @", "jump", "brif", "emit", "skip", "trap", + ]; + let missing: Vec<&str> = opcodes.iter().filter(|op| !all.contains(**op)).copied().collect(); + assert!(missing.is_empty(), "no fixture covers: {missing:?}"); +} + +// ------------------------------------------------------ verifier rejects -- + +/// Parse must succeed and verify must fail with a message containing `needle`. +fn assert_verify_rejects(text: &str, needle: &str) { + let p = parsed(text); + match verify(&p) { + Ok(()) => panic!("verifier accepted a bad program (wanted '{needle}'):\n{text}"), + Err(errs) => { + let all: Vec = errs.iter().map(|e| e.to_string()).collect(); + assert!( + all.iter().any(|m| m.contains(needle)), + "expected an error containing '{needle}', got: {all:?}\n---\n{text}" + ); + } + } +} + +#[test] +fn rejects_cross_block_use() { + // %x is defined in entry and used in b1 without riding a branch arg. + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + %t = const.i1 true + brif %t, one, two +one: + store out.o, %x + emit +two: + skip +}"#, + "cross blocks only as branch args", + ); +} + +#[test] +fn rejects_use_before_def() { + // Text form: the parser's name table already refuses a forward %use... + assert_parse_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %y = iadd %x, %x + %x = load in.a + store out.o, %y + emit +}"#, + "undefined value '%x'", + ); + // ...and the verifier independently rejects the same shape when the + // program is constructed through the API (the path lowering will use). + use super::{Block, Col, ColTy, Inst, Program, Term, Ty, Value, BinOp}; + let p = Program { + statics: vec![], + name: "f".into(), + in_cols: vec![Col { name: "a".into(), ty: ColTy { ty: Ty::I64, nullable: false } }], + out_cols: vec![Col { name: "o".into(), ty: ColTy { ty: Ty::I64, nullable: false } }], + blocks: vec![Block { + params: vec![], + insts: vec![ + Inst::Bin { op: BinOp::Iadd, dst: Value(0), a: Value(1), b: Value(1) }, + Inst::Load { dst: Value(1), col: 0 }, + Inst::Store { col: 0, val: Value(0) }, + ], + term: Term::Emit, + }], + }; + let errs = verify(&p).expect_err("use-before-def must not verify"); + assert!( + errs.iter().any(|e| e.to_string().contains("used before any definition")), + "wrong errors: {:?}", + errs.iter().map(|e| e.to_string()).collect::>() + ); +} + +#[test] +fn rejects_arith_type_mismatch() { + assert_verify_rejects( + r#"fn f(in: batch{a: f64}, out: batch{o: f64}) { +entry: + %x = load in.a + %y = iadd %x, %x + %z = itof %y + store out.o, %z + emit +}"#, + "must be i64, got f64", + ); +} + +#[test] +fn rejects_skipping_the_null_lane() { + // A nullable column read with the non-opt load: the arithmetic below it + // would silently operate on a maybe-NULL — exactly the 3VL bug class the + // IR is designed to make unrepresentable. + assert_verify_rejects( + r#"fn f(in: batch{a: i64?}, out: batch{o: i64}) { +entry: + %x = load in.a + store out.o, %x + emit +}"#, + "is nullable: use load.opt", + ); +} + +#[test] +fn rejects_inventing_a_null_lane() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %f, %x = load.opt in.a + store out.o, %x + emit +}"#, + "is not nullable: use load", + ); +} + +#[test] +fn rejects_store_without_flag_on_nullable_out() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64?}) { +entry: + %x = load in.a + store out.o, %x + emit +}"#, + "is nullable: use store.opt", + ); +} + +#[test] +fn rejects_probe_on_scalar_static() { + assert_verify_rejects( + r#"static @0: scalar +fn f(in: batch{k: str}, out: batch{o: f64}) { +entry: + %k = load in.k + %h, %v = probe @0, %k + store out.o, %v + emit +}"#, + "is a scalar: use sload", + ); +} + +#[test] +fn rejects_probe_key_arity_and_type() { + assert_verify_rejects( + r#"static @0: map(i64, str) -> (f64) +fn f(in: batch{k: str}, out: batch{o: f64}) { +entry: + %k = load in.k + %h, %v = probe @0, %k + store out.o, %v + emit +}"#, + "has 2 key(s), probe passes 1", + ); + assert_verify_rejects( + r#"static @0: map(i64) -> (f64) +fn f(in: batch{k: str}, out: batch{o: f64}) { +entry: + %k = load in.k + %h, %v = probe @0, %k + store out.o, %v + emit +}"#, + "probe key %v0 must be i64, got str", + ); +} + +#[test] +fn rejects_probe_value_arity() { + assert_verify_rejects( + r#"static @0: map(str) -> (f64, i64) +fn f(in: batch{k: str}, out: batch{o: f64}) { +entry: + %k = load in.k + %h, %v = probe @0, %k + store out.o, %v + emit +}"#, + "has 2 value column(s), probe defines 1", + ); +} + +#[test] +fn rejects_missing_store_at_emit() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64, p: i64}) { +entry: + %x = load in.a + store out.o, %x + emit +}"#, + "emit without storing out.p", + ); +} + +#[test] +fn rejects_double_store() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + store out.o, %x + store out.o, %x + emit +}"#, + "stored more than once", + ); +} + +#[test] +fn rejects_store_before_skip() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + store out.o, %x + skip +}"#, + "skip after storing", + ); +} + +#[test] +fn rejects_join_with_disagreeing_stores() { + // One arm stores out.o before the join, the other doesn't. + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %t = const.i1 true + brif %t, one, two +one: + %x = const.i64 1 + store out.o, %x + jump join +two: + jump join +join: + emit +}"#, + "disagree on which out columns are stored", + ); +} + +#[test] +fn rejects_cycle() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %t = const.i1 true + brif %t, one, two +one: + jump two +two: + jump one +}"#, + "control-flow cycle", + ); +} + +#[test] +fn rejects_unreachable_block() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + store out.o, %x + emit +island: + skip +}"#, + "unreachable block", + ); +} + +#[test] +fn rejects_branch_arg_mismatch() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + jump join(%x) +join(%p: f64): + %t = ftoi.trunc %p + store out.o, %t + emit +}"#, + "branch arg %v0 must be f64, got i64", + ); +} + +#[test] +fn rejects_select_arm_mismatch() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64, b: f64}, out: batch{o: i64}) { +entry: + %x = load in.a + %y = load in.b + %t = const.i1 true + %s = select %t, %x, %y + store out.o, %s + emit +}"#, + "select arms differ", + ); +} + +// -------------------------------------------------------- parser rejects -- + +fn assert_parse_rejects(text: &str, needle: &str) { + match parse(text) { + Ok(_) => panic!("parser accepted bad text (wanted '{needle}'):\n{text}"), + Err(e) => assert!( + e.to_string().contains(needle), + "expected a parse error containing '{needle}', got '{e}'" + ), + } +} + +#[test] +fn parser_rejects_unknown_opcode() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n %x = alloc 8\n emit\n}", + "unknown opcode 'alloc'", + ); +} + +#[test] +fn parser_rejects_value_redefinition() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n %x = load in.a\n %x = load in.a\n emit\n}", + "defined twice", + ); +} + +#[test] +fn parser_rejects_unknown_value() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n store out.o, %ghost\n emit\n}", + "undefined value '%ghost'", + ); +} + +#[test] +fn parser_rejects_unknown_column() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n %x = load in.b\n emit\n}", + "unknown column 'in.b'", + ); +} + +#[test] +fn parser_rejects_unknown_label() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n jump nowhere\n}", + "unknown block 'nowhere'", + ); +} + +#[test] +fn parser_rejects_unknown_static() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n %x = sload @0\n emit\n}", + "unknown static '@0'", + ); +} + +#[test] +fn parser_rejects_sparse_static_ids() { + assert_parse_rejects( + "static @1: scalar\nfn f(in: batch{a: i64}, out: batch{o: i64}) {\nentry:\n emit\n}", + "dense and in order", + ); +} + +#[test] +fn parser_rejects_bad_escape_and_unterminated_string() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{o: str}) {\nentry:\n %x = const.str \"\\q\"\n emit\n}", + "unknown escape", + ); + assert_parse_rejects("fn f(in: batch{a: i64}, out: batch{o: str}) { \"", "unterminated"); +} + +#[test] +fn parser_rejects_trailing_garbage() { + assert_parse_rejects( + "fn f(in: batch{a: i64}, out: batch{}) {\nentry:\n emit\n}\nextra", + "trailing input", + ); +} + +#[test] +fn parser_rejects_wrong_dst_count() { + assert_parse_rejects( + "fn f(in: batch{a: i64?}, out: batch{o: i64}) {\nentry:\n %x = load.opt in.a\n emit\n}", + "'load.opt' defines 2 value(s), found 1", + ); +} + +// ------------------------------------------------------------- literals -- + +#[test] +fn literal_edge_cases_round_trip() { + let text = r#"fn lits(in: batch{a: i64}, out: batch{o: str}) { +entry: + %nan = const.f64 nan + %pinf = const.f64 inf + %ninf = const.f64 -inf + %nzero = const.f64 -0.0 + %tiny = const.f64 1e-5 + %big = const.f64 1e300 + %min = const.i64 -9223372036854775808 + %max = const.i64 9223372036854775807 + %esc = const.str "q\"b\\n\nl\tt\u{7}" + store out.o, %esc + emit +}"#; + let p = verified(text); + let printed = print(&p); + let p2 = parsed(&printed); + assert_eq!(p2, p, "literal round-trip changed the program:\n{printed}"); + // -0.0 must survive as -0.0 (bitwise equality, not IEEE ==). + let has_neg_zero = printed.contains("-0.0"); + assert!(has_neg_zero, "printer lost the sign of -0.0:\n{printed}"); +} + +// ----------------------------------------------------------------- fuzz -- + +#[test] +fn fuzz_round_trip() { + for seed in 0..300u64 { + let p = gen::gen_program(seed); + if let Err(errs) = verify(&p) { + let msgs: Vec = errs.iter().map(|e| e.to_string()).collect(); + panic!("generator produced an invalid program (seed {seed}): {msgs:?}\n{}", print(&p)); + } + let text = print(&p); + let p2 = match parse(&text) { + Ok(p2) => p2, + Err(e) => panic!("canonical text failed to parse (seed {seed}): {e}\n{text}"), + }; + assert_eq!(p2, p, "round-trip changed the program (seed {seed}):\n{text}"); + assert_eq!(print(&p2), text, "printing is not a fixpoint (seed {seed})"); + } +} diff --git a/src/specializer/ir/verify.rs b/src/specializer/ir/verify.rs new file mode 100644 index 0000000..54386a2 --- /dev/null +++ b/src/specializer/ir/verify.rs @@ -0,0 +1,560 @@ +//! The verifier — the airtight boundary. Everything upstream (lowering) and +//! downstream (backends) is allowed to assume a verified program; nothing may +//! execute or compile an unverified one. +//! +//! Rules enforced (numbers referenced from tests): +//! 1. Structure: at least one block; entry (b0) has no params and is never a +//! branch target; batch column names unique per side. +//! 2. SSA: every value defined exactly once function-wide; every use sees a +//! definition earlier in the same block or a param of the same block +//! (strict block-param form — cross-block uses are illegal, which is what +//! lets the verifier skip dominance analysis entirely). +//! 3. Types: every operand matches its instruction's signature; `.opt` forms +//! are mandatory for nullable columns/statics and illegal on non-nullable +//! ones (the null lane can be neither skipped nor invented). +//! 4. Statics: every `@N` resolves; probe/sload match the static's kind, +//! arity, and types. +//! 5. CFG: all blocks reachable from entry; no cycles (v0); branch args +//! match target params in count and type. +//! 6. Stores: on every path to `emit`, each out column is stored exactly +//! once; paths to `skip` store nothing; store states must agree at joins. + +use std::collections::HashMap; + +use super::{Block, Col, Inst, Program, StaticTy, Term, Ty, Value}; + +#[derive(Debug, PartialEq, Eq)] +pub struct VerifyError { + /// Block index, when the error is inside a block. + pub block: Option, + /// Instruction index within the block; `None` for param/terminator errors. + pub inst: Option, + pub msg: String, +} + +impl std::fmt::Display for VerifyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match (self.block, self.inst) { + (Some(b), Some(i)) => write!(f, "b{b}[{i}]: {}", self.msg), + (Some(b), None) => write!(f, "b{b}: {}", self.msg), + _ => write!(f, "{}", self.msg), + } + } +} + +pub fn verify(p: &Program) -> Result<(), Vec> { + let mut errs = Vec::new(); + + check_structure(p, &mut errs); + // Function-wide definition table: value -> (type, defining block), + // discovered at definition sites. Collected fully before use-checking so + // an early error doesn't cascade into spurious "undefined value" noise. + let mut def_types: HashMap = HashMap::new(); + collect_defs(p, &mut def_types, &mut errs); + for (bi, b) in p.blocks.iter().enumerate() { + check_block(p, bi, b, &def_types, &mut errs); + } + check_cfg_and_stores(p, &mut errs); + + if errs.is_empty() { + Ok(()) + } else { + Err(errs) + } +} + +fn err(errs: &mut Vec, block: Option, inst: Option, msg: String) { + errs.push(VerifyError { block, inst, msg }); +} + +fn check_structure(p: &Program, errs: &mut Vec) { + if p.blocks.is_empty() { + err(errs, None, None, "function has no blocks".to_string()); + return; + } + if !p.blocks[0].params.is_empty() { + err(errs, Some(0), None, "entry block cannot have params".to_string()); + } + for (side, cols) in [("in", &p.in_cols), ("out", &p.out_cols)] { + let mut seen: HashMap<&str, ()> = HashMap::new(); + for c in cols.iter() { + if seen.insert(c.name.as_str(), ()).is_some() { + err(errs, None, None, format!("duplicate {side} column '{}'", c.name)); + } + } + } +} + +/// First pass: register every definition (params + inst dsts) with its type, +/// flagging double definitions. A definition's type is derivable from the +/// instruction plus the column/static tables alone — except `select`, whose +/// result type equals its operands'; it is registered per its `a` operand +/// during the per-block pass (see `check_block`), and as a placeholder here +/// only to occupy the SSA slot. +fn collect_defs( + p: &Program, + def_types: &mut HashMap, + errs: &mut Vec, +) { + for (bi, b) in p.blocks.iter().enumerate() { + for (v, ty) in &b.params { + if def_types.insert(v.0, (*ty, bi)).is_some() { + err(errs, Some(bi), None, format!("%v{} defined more than once", v.0)); + } + } + for (ii, inst) in b.insts.iter().enumerate() { + for (dst, ty) in dst_types(p, inst) { + if def_types.insert(dst.0, (ty, bi)).is_some() { + err(errs, Some(bi), Some(ii), format!("%v{} defined more than once", dst.0)); + } + } + } + } +} + +/// Result types of an instruction's definitions, best-effort when the +/// program is malformed (bad indices fall back so verification continues and +/// the real error is reported at the site check). +fn dst_types(p: &Program, inst: &Inst) -> Vec<(Value, Ty)> { + let in_col = |c: u32| p.in_cols.get(c as usize).map(|c| c.ty.ty); + let scalar_ty = |id: u32| match p.statics.get(id as usize) { + Some(StaticTy::Scalar(ct)) => ct.ty, + _ => Ty::I1, + }; + match inst { + Inst::Const { dst, lit } => vec![(*dst, lit.ty())], + Inst::Bin { op, dst, .. } => vec![(*dst, op.sig().1)], + Inst::Cmp { dst, .. } | Inst::Not { dst, .. } => vec![(*dst, Ty::I1)], + // Placeholder; corrected per-block (see collect_defs docs). + Inst::Select { dst, .. } => vec![(*dst, Ty::I1)], + Inst::Itof { dst, .. } => vec![(*dst, Ty::F64)], + Inst::Ftoi { dst, .. } => vec![(*dst, Ty::I64)], + Inst::Itos { dst, .. } | Inst::Ftos { dst, .. } | Inst::Sconcat { dst, .. } => { + vec![(*dst, Ty::Str)] + } + Inst::StoiOpt { flag, dst, .. } => vec![(*flag, Ty::I1), (*dst, Ty::I64)], + Inst::StofOpt { flag, dst, .. } => vec![(*flag, Ty::I1), (*dst, Ty::F64)], + Inst::Load { dst, col } => vec![(*dst, in_col(*col).unwrap_or(Ty::I1))], + Inst::LoadOpt { flag, dst, col } => { + vec![(*flag, Ty::I1), (*dst, in_col(*col).unwrap_or(Ty::I1))] + } + Inst::Store { .. } | Inst::StoreOpt { .. } => vec![], + Inst::Probe { static_id, hit, dsts, .. } => { + let mut v = vec![(*hit, Ty::I1)]; + if let Some(StaticTy::Map { values, .. }) = p.statics.get(*static_id as usize) { + for (d, ty) in dsts.iter().zip(values.iter()) { + v.push((*d, *ty)); + } + } + // Dsts beyond the declared value columns keep no type; the site + // check reports the arity mismatch. + v + } + Inst::Sload { static_id, dst } => vec![(*dst, scalar_ty(*static_id))], + Inst::SloadOpt { static_id, flag, dst } => { + vec![(*flag, Ty::I1), (*dst, scalar_ty(*static_id))] + } + } +} + +/// Look up a use in the block's scope, reporting scope violations with a +/// message that distinguishes "defined later in this block" (use before def) +/// from "defined in another block" (illegal crossing) from "never defined". +fn scope_ty( + in_scope: &HashMap, + def_types: &HashMap, + v: Value, + what: &str, + bi: usize, + ii: Option, + errs: &mut Vec, +) -> Option { + match in_scope.get(&v.0) { + Some(ty) => Some(*ty), + None => { + let msg = match def_types.get(&v.0) { + Some((_, def_bi)) if *def_bi != bi => format!( + "{what} %v{} is not visible here: values cross blocks only as branch \ + args to block params", + v.0 + ), + _ => format!("{what} %v{} is used before any definition", v.0), + }; + err(errs, Some(bi), ii, msg); + None + } + } +} + +#[allow(clippy::too_many_arguments)] +fn want( + in_scope: &HashMap, + def_types: &HashMap, + v: Value, + ty: Ty, + what: &str, + bi: usize, + ii: Option, + errs: &mut Vec, +) { + if let Some(actual) = scope_ty(in_scope, def_types, v, what, bi, ii, errs) { + if actual != ty { + err( + errs, + Some(bi), + ii, + format!("{what} %v{} must be {}, got {}", v.0, ty.name(), actual.name()), + ); + } + } +} + +/// Second pass, per block: scoping (uses see only same-block earlier defs or +/// own params) and per-instruction operand typing. +fn check_block( + p: &Program, + bi: usize, + b: &Block, + def_types: &HashMap, + errs: &mut Vec, +) { + // Values visible at the current point of this block. + let mut in_scope: HashMap = HashMap::new(); + for (v, ty) in &b.params { + in_scope.insert(v.0, *ty); + } + + for (ii, inst) in b.insts.iter().enumerate() { + let i = Some(ii); + match inst { + Inst::Const { .. } => {} + Inst::Bin { op, a, b: rhs, .. } => { + let (operand, _) = op.sig(); + want(&in_scope, def_types, *a, operand, "operand", bi, i, errs); + want(&in_scope, def_types, *rhs, operand, "operand", bi, i, errs); + } + Inst::Cmp { ty, a, b: rhs, .. } => { + if *ty == Ty::I1 { + err(errs, Some(bi), i, "cmp on i1 is not defined; use xor/not".into()); + } + want(&in_scope, def_types, *a, *ty, "operand", bi, i, errs); + want(&in_scope, def_types, *rhs, *ty, "operand", bi, i, errs); + } + Inst::Not { a, .. } => { + want(&in_scope, def_types, *a, Ty::I1, "operand", bi, i, errs) + } + Inst::Select { dst, cond, a, b: rhs } => { + want(&in_scope, def_types, *cond, Ty::I1, "condition", bi, i, errs); + let ta = scope_ty(&in_scope, def_types, *a, "operand", bi, i, errs); + let tb = scope_ty(&in_scope, def_types, *rhs, "operand", bi, i, errs); + if let (Some(ta), Some(tb)) = (ta, tb) { + if ta != tb { + err( + errs, + Some(bi), + i, + format!("select arms differ: {} vs {}", ta.name(), tb.name()), + ); + } + } + // Correct the placeholder from collect_defs with the real type. + if let Some(ta) = ta { + in_scope.insert(dst.0, ta); + } + } + Inst::Itof { a, .. } => { + want(&in_scope, def_types, *a, Ty::I64, "operand", bi, i, errs) + } + Inst::Ftoi { a, .. } => { + want(&in_scope, def_types, *a, Ty::F64, "operand", bi, i, errs) + } + Inst::Itos { a, .. } => { + want(&in_scope, def_types, *a, Ty::I64, "operand", bi, i, errs) + } + Inst::Ftos { a, .. } => { + want(&in_scope, def_types, *a, Ty::F64, "operand", bi, i, errs) + } + Inst::StoiOpt { a, .. } | Inst::StofOpt { a, .. } => { + want(&in_scope, def_types, *a, Ty::Str, "operand", bi, i, errs) + } + Inst::Sconcat { a, b: rhs, .. } => { + want(&in_scope, def_types, *a, Ty::Str, "operand", bi, i, errs); + want(&in_scope, def_types, *rhs, Ty::Str, "operand", bi, i, errs); + } + Inst::Load { col, .. } => match p.in_cols.get(*col as usize) { + None => err(errs, Some(bi), i, format!("unknown in column {col}")), + Some(Col { ty, name }) if ty.nullable => { + err(errs, Some(bi), i, format!("in.{name} is nullable: use load.opt")) + } + Some(_) => {} + }, + Inst::LoadOpt { col, .. } => match p.in_cols.get(*col as usize) { + None => err(errs, Some(bi), i, format!("unknown in column {col}")), + Some(Col { ty, name }) if !ty.nullable => { + err(errs, Some(bi), i, format!("in.{name} is not nullable: use load")) + } + Some(_) => {} + }, + Inst::Store { col, val } => match p.out_cols.get(*col as usize) { + None => err(errs, Some(bi), i, format!("unknown out column {col}")), + Some(c) if c.ty.nullable => { + err(errs, Some(bi), i, format!("out.{} is nullable: use store.opt", c.name)) + } + Some(c) => { + want(&in_scope, def_types, *val, c.ty.ty, "stored value", bi, i, errs) + } + }, + Inst::StoreOpt { col, flag, val } => match p.out_cols.get(*col as usize) { + None => err(errs, Some(bi), i, format!("unknown out column {col}")), + Some(c) if !c.ty.nullable => { + err(errs, Some(bi), i, format!("out.{} is not nullable: use store", c.name)) + } + Some(c) => { + want(&in_scope, def_types, *flag, Ty::I1, "validity flag", bi, i, errs); + want(&in_scope, def_types, *val, c.ty.ty, "stored value", bi, i, errs); + } + }, + Inst::Probe { static_id, dsts, keys, .. } => { + match p.statics.get(*static_id as usize) { + None => err(errs, Some(bi), i, format!("unknown static @{static_id}")), + Some(StaticTy::Scalar(_)) => { + err(errs, Some(bi), i, format!("@{static_id} is a scalar: use sload")) + } + Some(StaticTy::Map { keys: kts, values: vts }) => { + if keys.len() != kts.len() { + err( + errs, + Some(bi), + i, + format!( + "@{static_id} has {} key(s), probe passes {}", + kts.len(), + keys.len() + ), + ); + } else { + for (k, kt) in keys.iter().zip(kts.iter()) { + want(&in_scope, def_types, *k, *kt, "probe key", bi, i, errs); + } + } + if dsts.len() != vts.len() { + err( + errs, + Some(bi), + i, + format!( + "@{static_id} has {} value column(s), probe defines {}", + vts.len(), + dsts.len() + ), + ); + } + } + } + } + Inst::Sload { static_id, .. } => match p.statics.get(*static_id as usize) { + None => err(errs, Some(bi), i, format!("unknown static @{static_id}")), + Some(StaticTy::Map { .. }) => { + err(errs, Some(bi), i, format!("@{static_id} is a map: use probe")) + } + Some(StaticTy::Scalar(ct)) if ct.nullable => { + err(errs, Some(bi), i, format!("@{static_id} is nullable: use sload.opt")) + } + Some(_) => {} + }, + Inst::SloadOpt { static_id, .. } => match p.statics.get(*static_id as usize) { + None => err(errs, Some(bi), i, format!("unknown static @{static_id}")), + Some(StaticTy::Map { .. }) => { + err(errs, Some(bi), i, format!("@{static_id} is a map: use probe")) + } + Some(StaticTy::Scalar(ct)) if !ct.nullable => { + err(errs, Some(bi), i, format!("@{static_id} is not nullable: use sload")) + } + Some(_) => {} + }, + } + + // Definitions become visible AFTER the instruction (no self-use). + for d in inst.dsts() { + let ty = def_types.get(&d.0).map(|(t, _)| *t).unwrap_or(Ty::I1); + // Select already inserted its corrected type above; keep it. + in_scope.entry(d.0).or_insert(ty); + } + } + + // Terminator: cond and branch args are uses in this block's final scope. + if let Term::Brif { cond, .. } = &b.term { + want(&in_scope, def_types, *cond, Ty::I1, "branch condition", bi, None, errs); + } + for (target, args) in b.term.successors() { + match p.blocks.get(target.0 as usize) { + None => err(errs, Some(bi), None, format!("branch to unknown block b{}", target.0)), + Some(tb) => { + if args.len() != tb.params.len() { + err( + errs, + Some(bi), + None, + format!( + "b{} expects {} arg(s), got {}", + target.0, + tb.params.len(), + args.len() + ), + ); + } else { + for (arg, (_, pty)) in args.iter().zip(tb.params.iter()) { + want(&in_scope, def_types, *arg, *pty, "branch arg", bi, None, errs); + } + } + if target.0 == 0 { + err(errs, Some(bi), None, "branch to entry block".to_string()); + } + } + } + } +} + +/// CFG shape (reachability, acyclicity) and the store-completeness dataflow. +fn check_cfg_and_stores(p: &Program, errs: &mut Vec) { + let n = p.blocks.len(); + if n == 0 { + return; + } + + // DFS for reachability + cycle detection (0 white, 1 gray, 2 black). + fn dfs(p: &Program, b: usize, color: &mut [u8], cyclic: &mut bool) { + color[b] = 1; + for (succ, _) in p.blocks[b].term.successors() { + let s = succ.0 as usize; + if s >= color.len() { + continue; // reported by check_block + } + match color[s] { + 0 => dfs(p, s, color, cyclic), + 1 => *cyclic = true, + _ => {} + } + } + color[b] = 2; + } + let mut color = vec![0u8; n]; + let mut cyclic = false; + dfs(p, 0, &mut color, &mut cyclic); + if cyclic { + err(errs, None, None, "control-flow cycle (v0 CFGs must be acyclic)".to_string()); + return; // the store dataflow below needs a DAG + } + for (bi, c) in color.iter().enumerate() { + if *c == 0 { + err(errs, Some(bi), None, "unreachable block".to_string()); + } + } + + // Store dataflow over the DAG in topological order. State: per out + // column, how many times it has been stored on every path reaching this + // point; all paths into a join must agree. + let ncols = p.out_cols.len(); + let mut entry_state: Vec>> = vec![None; n]; + entry_state[0] = Some(vec![0; ncols]); + + for bi in topo_order(p, n) { + let Some(state) = entry_state[bi].clone() else { + continue; // unreachable; already reported + }; + let mut state_out = state; + for (ii, inst) in p.blocks[bi].insts.iter().enumerate() { + let col = match inst { + Inst::Store { col, .. } | Inst::StoreOpt { col, .. } => *col as usize, + _ => continue, + }; + if col < ncols { + state_out[col] = state_out[col].saturating_add(1); + if state_out[col] > 1 { + err( + errs, + Some(bi), + Some(ii), + format!("out.{} stored more than once on this path", p.out_cols[col].name), + ); + } + } + } + match &p.blocks[bi].term { + Term::Emit => { + for (ci, count) in state_out.iter().enumerate() { + if *count == 0 { + err( + errs, + Some(bi), + None, + format!("emit without storing out.{}", p.out_cols[ci].name), + ); + } + } + } + Term::Skip => { + if state_out.iter().any(|c| *c > 0) { + err( + errs, + Some(bi), + None, + "skip after storing (a skipped row must store nothing)".to_string(), + ); + } + } + Term::Trap { .. } => {} + _ => { + for (succ, _) in p.blocks[bi].term.successors() { + let s = succ.0 as usize; + if s >= n { + continue; + } + match &entry_state[s] { + None => entry_state[s] = Some(state_out.clone()), + Some(existing) if *existing != state_out => { + err( + errs, + Some(s), + None, + "paths joining here disagree on which out columns are stored" + .to_string(), + ); + } + Some(_) => {} + } + } + } + } + } +} + +/// Topological order of the (already acyclicity-checked) CFG via Kahn's +/// algorithm; unreachable blocks may appear anywhere, which is fine — they +/// have no entry state and are skipped. +fn topo_order(p: &Program, n: usize) -> Vec { + let mut indegree = vec![0usize; n]; + for b in &p.blocks { + for (succ, _) in b.term.successors() { + let s = succ.0 as usize; + if s < n { + indegree[s] += 1; + } + } + } + let mut stack: Vec = (0..n).filter(|&i| indegree[i] == 0).collect(); + let mut order = Vec::with_capacity(n); + while let Some(b) = stack.pop() { + order.push(b); + for (succ, _) in p.blocks[b].term.successors() { + let s = succ.0 as usize; + if s < n { + indegree[s] -= 1; + if indegree[s] == 0 { + stack.push(s); + } + } + } + } + order +} diff --git a/src/specializer/mod.rs b/src/specializer/mod.rs new file mode 100644 index 0000000..c4c38c9 --- /dev/null +++ b/src/specializer/mod.rs @@ -0,0 +1,11 @@ +//! The SQL specializer: a partial evaluator that turns (fixed SQL + static +//! tables) into a specialized native function `f : Rows -> Rows`, prepared +//! once and invoked millions of times with a small dynamic input relation. +//! +//! Design: docs/superpowers/specs/2026-07-25-sql-specializer-design.md. +//! Build order (backlog milestone m-7): the imperative IR below (M-ir), then +//! the closure-compiled interpreter oracle (M-interp), the frontend + BTA + +//! lowering (M-lower), the Cranelift backend (M-cranelift), and the generated +//! Python-boundary marshaller (M-boundary). + +pub mod ir; From c7b0479b076c49e5466d93471afcb2953188fb53 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 15:46:30 +0200 Subject: [PATCH 07/11] =?UTF-8?q?fix(specializer):=20harden=20the=20IR=20b?= =?UTF-8?q?oundary=20against=20adversarial=20findings=20=E2=80=94=20TASK-4?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six agents (4 attack lenses + 2 reviews) probed the initial IR; every confirmed finding is fixed and pinned by a regression test: - reachability DFS made iterative: a ~8k-block legal CFG stack-overflowed the process; 50k blocks now verify (deep_cfg_verifies_without_crashing) - verified programs must print to parseable text: reject empty map static signatures (grammar can't express `map() -> ()`) and non-identifier function names - Lit::F64 equality treats all NaNs as one class, matching the text form's canonical `nan` token — non-canonical payloads round-trip again - store dataflow runs over the reachable subgraph: an unreachable cyclic island no longer starves a reachable join and masks its store errors - docs aligned: double stores rejected on ALL paths incl. trap (deliberate), grammar EBNF covers store/comments/lenient %names, canonical-form note - dead Inst::uses/Term::uses deleted; 7 missing rule-coverage tests added (entry params, duplicate cols, branch-to-entry, sload-on-map, scalar-static .opt pairing both ways, store.opt on non-nullable) cargo 45 passed; gate green. Co-Authored-By: Claude Opus 5 --- .../2026-07-25-sql-specializer-design.md | 19 +- src/specializer/ir/mod.rs | 68 ++--- src/specializer/ir/print.rs | 6 +- src/specializer/ir/tests.rs | 244 ++++++++++++++++++ src/specializer/ir/verify.rs | 93 +++++-- 5 files changed, 354 insertions(+), 76 deletions(-) diff --git a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md index 35f6921..4e639af 100644 --- a/docs/superpowers/specs/2026-07-25-sql-specializer-design.md +++ b/docs/superpowers/specs/2026-07-25-sql-specializer-design.md @@ -209,12 +209,19 @@ the direction of a smaller, more verifiable core): so `-0.0` and NaN round-trip). Verifier rules (each has a rejecting test in `ir/tests.rs`): structure (entry -has no params, unique column names, no branch to entry), SSA (single def, -same-block visibility), per-op operand types incl. mandatory/forbidden `.opt` -pairing against column/static nullability, static resolution + kind/arity/key -types, CFG reachability + acyclicity + branch-arg typing, and the -store-completeness dataflow (exactly-once per column at `emit`, zero at -`skip`, agreement at joins). +has no params, unique column names, no branch to entry, identifier function +name, non-empty map signatures — a verified program must print to parseable +text), SSA (single def, same-block visibility), per-op operand types incl. +mandatory/forbidden `.opt` pairing against column/static nullability, static +resolution + kind/arity/key types, CFG reachability + acyclicity + branch-arg +typing (iterative DFS — deep legal CFGs must not overflow the native stack), +and the store-completeness dataflow (never twice on any path, exactly-once +per column at `emit`, zero at `skip`, agreement at joins, computed over the +reachable subgraph so an unreachable island can't mask a join's errors). + +An adversarial pass (4 attack lenses + design-conformance + simplification +reviews, 2026-07-25) ran against the initial implementation; every confirmed +finding is pinned by a regression test in `ir/tests.rs`. Deferred to M-interp, pinned there against the DuckDB oracle: `ftoi.round` tie behavior and `fcmp` NaN ordering. `idiv`/`irem` trap on zero/overflow; diff --git a/src/specializer/ir/mod.rs b/src/specializer/ir/mod.rs index 44f8142..2dd8dd4 100644 --- a/src/specializer/ir/mod.rs +++ b/src/specializer/ir/mod.rs @@ -18,6 +18,10 @@ //! * `trap "msg"` — abort the whole call with a runtime error (CAST failures, //! division guards — whatever the lowered dialect defines as an error). //! +//! No path may store the same column twice, whatever its terminator — a +//! double store is always a lowering bug, so the verifier rejects it even on +//! paths that end in `trap`. +//! //! `|out| == |in|` therefore holds exactly when `skip` is unreachable, which //! is statically known — the design doc's "filter is the one allowed //! divergence and must declare it". @@ -66,12 +70,16 @@ //! //! Canonical, round-trippable; `parse(print(p)) == p` for every program whose //! value ids are dense and in definition order (the parser and [`gen`] both -//! produce that form, so the property is closed under round-trip). Names in -//! the text are presentation only — they are not stored in the IR; printing -//! uses canonical `%vN` / `bN` names. +//! produce that form, so the property is closed under round-trip). A program +//! with sparse or out-of-order ids still verifies and still prints; the +//! round-trip then performs a bijective renumbering into canonical form, so +//! structural equality is only meaningful between canonical programs. Names +//! in the text are presentation only — they are not stored in the IR; +//! printing uses canonical `%vN` / `bN` names. //! //! ```text //! program := static* func +//! comment := "#" ... end-of-line // allowed anywhere whitespace is //! static := "static" "@" INT ":" static_ty //! static_ty := "scalar" "<" col_ty ">" //! | "map" "(" ty ("," ty)* ")" "->" "(" ty ("," ty)* ")" @@ -81,12 +89,15 @@ //! col_ty := ty ["?"] //! ty := "i1" | "i64" | "f64" | "str" //! block := IDENT ["(" VALUE ":" ty ("," VALUE ":" ty)* ")"] ":" inst* term -//! inst := VALUE ("," VALUE)* "=" OPCODE operands +//! inst := [VALUE ("," VALUE)* "="] OPCODE operands +//! // only store/store.opt omit the dests; everything else +//! // defines at least one value //! term := "jump" target //! | "brif" VALUE "," target "," target //! | "emit" | "skip" | "trap" STRING //! target := IDENT ["(" VALUE ("," VALUE)* ")"] -//! VALUE := "%" IDENT +//! VALUE := "%" NAME // NAME: any run of [A-Za-z0-9_]; +//! // the printer only emits %vN //! ``` //! //! Instruction surface (`%d` result, `%f` an `i1` flag result): @@ -179,9 +190,11 @@ pub struct Value(pub u32); #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub struct BlockId(pub u32); -/// Literal for `const.*`. `F64` equality is bitwise so that programs -/// containing NaN still satisfy `parse(print(p)) == p` (the text form -/// canonicalizes NaN payloads to the one `nan` token). +/// Literal for `const.*`. `F64` equality is bitwise — so `-0.0 != 0.0` +/// survives a round-trip — EXCEPT that all NaNs compare equal: the text form +/// canonicalizes every NaN payload to the one `nan` token, and equality must +/// match what the text format can distinguish or `parse(print(p)) == p` +/// would fail for non-canonical payloads (found by adversarial fuzzing). #[derive(Clone, Debug)] pub enum Lit { I1(bool), @@ -195,7 +208,9 @@ impl PartialEq for Lit { match (self, other) { (Lit::I1(a), Lit::I1(b)) => a == b, (Lit::I64(a), Lit::I64(b)) => a == b, - (Lit::F64(a), Lit::F64(b)) => a.to_bits() == b.to_bits(), + (Lit::F64(a), Lit::F64(b)) => { + a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()) + } (Lit::Str(a), Lit::Str(b)) => a == b, _ => false, } @@ -463,30 +478,6 @@ impl Inst { } } - /// Values this instruction uses. - pub fn uses(&self) -> Vec { - match self { - Inst::Const { .. } - | Inst::Load { .. } - | Inst::LoadOpt { .. } - | Inst::Sload { .. } - | Inst::SloadOpt { .. } => vec![], - Inst::Not { a, .. } - | Inst::Itof { a, .. } - | Inst::Ftoi { a, .. } - | Inst::Itos { a, .. } - | Inst::Ftos { a, .. } - | Inst::StoiOpt { a, .. } - | Inst::StofOpt { a, .. } => vec![*a], - Inst::Bin { a, b, .. } | Inst::Cmp { a, b, .. } | Inst::Sconcat { a, b, .. } => { - vec![*a, *b] - } - Inst::Select { cond, a, b, .. } => vec![*cond, *a, *b], - Inst::Store { val, .. } => vec![*val], - Inst::StoreOpt { flag, val, .. } => vec![*flag, *val], - Inst::Probe { keys, .. } => keys.clone(), - } - } } impl Term { @@ -507,17 +498,6 @@ impl Term { Term::Emit | Term::Skip | Term::Trap { .. } => vec![], } } - - pub fn uses(&self) -> Vec { - let mut out = Vec::new(); - if let Term::Brif { cond, .. } = self { - out.push(*cond); - } - for (_, args) in self.successors() { - out.extend_from_slice(args); - } - out - } } /// Builds programs with dense, definition-ordered value ids — the canonical diff --git a/src/specializer/ir/print.rs b/src/specializer/ir/print.rs index 9297fb9..9026858 100644 --- a/src/specializer/ir/print.rs +++ b/src/specializer/ir/print.rs @@ -227,8 +227,10 @@ fn col_name(p: &Program, out: bool, idx: u32) -> String { } } -/// Column and function names print bare when they are identifiers, quoted -/// otherwise (SQL-derived output names like `COALESCE(a, b)` survive). +/// Column names print bare when they are identifiers, quoted otherwise +/// (SQL-derived output names like `COALESCE(a, b)` survive). Function names +/// are NOT routed through this: the verifier requires them to be +/// identifiers, so the printer writes them raw. fn ident_or_quoted(name: &str) -> String { if is_ident(name) { name.to_string() diff --git a/src/specializer/ir/tests.rs b/src/specializer/ir/tests.rs index 9fe6d2e..42ba1dd 100644 --- a/src/specializer/ir/tests.rs +++ b/src/specializer/ir/tests.rs @@ -457,6 +457,250 @@ fn parser_rejects_wrong_dst_count() { ); } +// ----------------------------------------- adversarial-pass regressions -- +// Each of these pins a fix for a confirmed finding from the 2026-07-25 +// adversarial workflow (4 attack lenses + 2 reviews). + +/// Build the smallest valid API program shell around custom parts. +fn api_program( + statics: Vec, + name: &str, + blocks: Vec, +) -> Program { + use super::{Col, ColTy, Ty}; + Program { + statics, + name: name.into(), + in_cols: vec![], + out_cols: vec![Col { name: "o".into(), ty: ColTy { ty: Ty::I64, nullable: false } }], + blocks, + } +} + +fn store_emit_block() -> super::Block { + use super::{Block, Inst, Lit, Term, Value}; + Block { + params: vec![], + insts: vec![ + Inst::Const { dst: Value(0), lit: Lit::I64(1) }, + Inst::Store { col: 0, val: Value(0) }, + ], + term: Term::Emit, + } +} + +/// Deep-but-legal CFGs must verify, not abort the process — the reachability +/// DFS was recursive and stack-overflowed at ~8k blocks. +#[test] +fn deep_cfg_verifies_without_crashing() { + use super::{Block, BlockId, Term}; + let n: u32 = 50_000; + let mut blocks: Vec = (0..n - 1) + .map(|i| Block { + params: vec![], + insts: vec![], + term: Term::Jump { to: BlockId(i + 1), args: vec![] }, + }) + .collect(); + blocks.push(store_emit_block()); + let p = api_program(vec![], "deep", blocks); + verify(&p).expect("a deep linear CFG is legal"); +} + +/// `map() -> (..)` / `map(..) -> ()` cannot be expressed by the grammar, so +/// a verified program containing one could not be reloaded. +#[test] +fn rejects_empty_map_static_signatures() { + use super::{StaticTy, Ty}; + for st in [ + StaticTy::Map { keys: vec![], values: vec![Ty::I64] }, + StaticTy::Map { keys: vec![Ty::I64], values: vec![] }, + ] { + let p = api_program(vec![st], "f", vec![store_emit_block()]); + let errs = verify(&p).expect_err("empty map signature must not verify"); + assert!( + errs.iter().any(|e| e.to_string().contains("at least one key and one value")), + "wrong errors: {:?}", + errs.iter().map(|e| e.to_string()).collect::>() + ); + } +} + +/// A non-identifier (or empty) function name prints as unparseable text. +#[test] +fn rejects_non_identifier_function_name() { + for bad in ["weird name", "", "1fn", "a-b"] { + let p = api_program(vec![], bad, vec![store_emit_block()]); + let errs = verify(&p).expect_err("non-identifier fn name must not verify"); + assert!( + errs.iter().any(|e| e.to_string().contains("must be an identifier")), + "'{bad}': wrong errors: {:?}", + errs.iter().map(|e| e.to_string()).collect::>() + ); + } +} + +/// Every NaN payload canonicalizes to the one `nan` token; equality must +/// treat NaNs as one class or non-canonical payloads break the round-trip. +#[test] +fn non_canonical_nan_payload_round_trips() { + use super::{Block, Inst, Lit, Term, Value, Col, ColTy, Ty}; + let neg_quiet_nan = f64::from_bits(0xFFF8_0000_0000_0000); + let p = Program { + statics: vec![], + name: "f".into(), + in_cols: vec![], + out_cols: vec![Col { name: "o".into(), ty: ColTy { ty: Ty::F64, nullable: false } }], + blocks: vec![Block { + params: vec![], + insts: vec![ + Inst::Const { dst: Value(0), lit: Lit::F64(neg_quiet_nan) }, + Inst::Store { col: 0, val: Value(0) }, + ], + term: Term::Emit, + }], + }; + verify(&p).expect("NaN const is legal"); + assert_eq!(parsed(&print(&p)), p, "non-canonical NaN payload broke the round-trip"); +} + +/// Double stores are a lowering bug on ANY path, including trap-terminated +/// ones (deliberate: stricter than "only emit/skip paths"). +#[test] +fn rejects_double_store_on_trap_path() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + store out.o, %x + store out.o, %x + trap "boom" +}"#, + "stored more than once", + ); +} + +/// An unreachable island with an edge into a reachable join must not starve +/// the join's store dataflow — both the island error AND the join's store +/// error must surface in one verify pass. +#[test] +fn unreachable_island_does_not_mask_store_errors() { + let p = parsed( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + jump join +u1: + jump u2 +u2: + %c = const.i1 true + brif %c, u1, join +join: + emit +}"#, + ); + let errs = verify(&p).expect_err("island + missing store must not verify"); + let all: Vec = errs.iter().map(|e| e.to_string()).collect(); + assert!(all.iter().any(|m| m.contains("unreachable block")), "missing island error: {all:?}"); + assert!( + all.iter().any(|m| m.contains("emit without storing out.o")), + "store error masked by the island: {all:?}" + ); +} + +// Rule-coverage tests the design-conformance review found missing. + +#[test] +fn rejects_entry_with_params() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry(%p: i64): + store out.o, %p + emit +}"#, + "entry block cannot have params", + ); +} + +#[test] +fn rejects_duplicate_columns() { + use super::{Col, ColTy, Ty}; + let mut p = api_program(vec![], "f", vec![store_emit_block()]); + p.out_cols = vec![ + Col { name: "o".into(), ty: ColTy { ty: Ty::I64, nullable: false } }, + Col { name: "o".into(), ty: ColTy { ty: Ty::I64, nullable: false } }, + ]; + // Second column now unstored; the duplicate-name error is what matters. + let errs = verify(&p).expect_err("duplicate columns must not verify"); + assert!( + errs.iter().any(|e| e.to_string().contains("duplicate out column 'o'")), + "wrong errors: {:?}", + errs.iter().map(|e| e.to_string()).collect::>() + ); +} + +#[test] +fn rejects_branch_to_entry() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + jump entry +}"#, + "branch to entry block", + ); +} + +#[test] +fn rejects_sload_on_map_static() { + assert_verify_rejects( + r#"static @0: map(str) -> (i64) +fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %v = sload @0 + store out.o, %v + emit +}"#, + "is a map: use probe", + ); +} + +#[test] +fn rejects_wrong_opt_pairing_on_scalar_statics() { + assert_verify_rejects( + r#"static @0: scalar +fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %v = sload @0 + store out.o, %v + emit +}"#, + "is nullable: use sload.opt", + ); + assert_verify_rejects( + r#"static @0: scalar +fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %f, %v = sload.opt @0 + store out.o, %v + emit +}"#, + "is not nullable: use sload", + ); +} + +#[test] +fn rejects_store_opt_on_non_nullable_out() { + assert_verify_rejects( + r#"fn f(in: batch{a: i64}, out: batch{o: i64}) { +entry: + %x = load in.a + %t = const.i1 true + store.opt out.o, %t, %x + emit +}"#, + "is not nullable: use store", + ); +} + // ------------------------------------------------------------- literals -- #[test] diff --git a/src/specializer/ir/verify.rs b/src/specializer/ir/verify.rs index 54386a2..968dbed 100644 --- a/src/specializer/ir/verify.rs +++ b/src/specializer/ir/verify.rs @@ -4,7 +4,9 @@ //! //! Rules enforced (numbers referenced from tests): //! 1. Structure: at least one block; entry (b0) has no params and is never a -//! branch target; batch column names unique per side. +//! branch target; batch column names unique per side; the function name +//! is an identifier and map statics have >= 1 key and >= 1 value column +//! (a verified program must print to parseable canonical text). //! 2. SSA: every value defined exactly once function-wide; every use sees a //! definition earlier in the same block or a param of the same block //! (strict block-param form — cross-block uses are illegal, which is what @@ -16,8 +18,10 @@ //! arity, and types. //! 5. CFG: all blocks reachable from entry; no cycles (v0); branch args //! match target params in count and type. -//! 6. Stores: on every path to `emit`, each out column is stored exactly -//! once; paths to `skip` store nothing; store states must agree at joins. +//! 6. Stores: no path stores a column twice, whatever its terminator +//! (including `trap` — a double store is always a lowering bug); paths +//! to `emit` store every column exactly once; paths to `skip` store +//! nothing; store states must agree at joins. use std::collections::HashMap; @@ -68,6 +72,29 @@ fn err(errs: &mut Vec, block: Option, inst: Option, m } fn check_structure(p: &Program, errs: &mut Vec) { + // A verified program must print to parseable canonical text, so text-only + // constraints (identifier function name, non-empty map signatures — the + // grammar cannot express `map() -> ()`) are verifier rules too. + if !super::print::is_ident(&p.name) { + err( + errs, + None, + None, + format!("function name '{}' must be an identifier", p.name), + ); + } + for (i, st) in p.statics.iter().enumerate() { + if let StaticTy::Map { keys, values } = st { + if keys.is_empty() || values.is_empty() { + err( + errs, + None, + None, + format!("@{i}: map statics need at least one key and one value column"), + ); + } + } + } if p.blocks.is_empty() { err(errs, None, None, "function has no blocks".to_string()); return; @@ -422,31 +449,43 @@ fn check_cfg_and_stores(p: &Program, errs: &mut Vec) { return; } - // DFS for reachability + cycle detection (0 white, 1 gray, 2 black). - fn dfs(p: &Program, b: usize, color: &mut [u8], cyclic: &mut bool) { - color[b] = 1; - for (succ, _) in p.blocks[b].term.successors() { - let s = succ.0 as usize; - if s >= color.len() { + // Iterative DFS for reachability + cycle detection (0 white, 1 gray, + // 2 black). Explicit stack, NOT recursion: a deep-but-legal CFG (large + // CASE/decision-tree lowerings) must not abort the process — recursion + // stack-overflowed at ~8k blocks under adversarial fuzzing. + let mut color = vec![0u8; n]; + let mut cyclic = false; + let mut stack: Vec<(usize, usize)> = vec![(0, 0)]; // (block, next successor index) + color[0] = 1; + while let Some(frame) = stack.last_mut() { + let (b, si) = *frame; + let succs = p.blocks[b].term.successors(); + if si < succs.len() { + frame.1 += 1; + let s = succs[si].0 .0 as usize; + if s >= n { continue; // reported by check_block } match color[s] { - 0 => dfs(p, s, color, cyclic), - 1 => *cyclic = true, + 0 => { + color[s] = 1; + stack.push((s, 0)); + } + 1 => cyclic = true, _ => {} } + } else { + color[b] = 2; + stack.pop(); } - color[b] = 2; } - let mut color = vec![0u8; n]; - let mut cyclic = false; - dfs(p, 0, &mut color, &mut cyclic); if cyclic { err(errs, None, None, "control-flow cycle (v0 CFGs must be acyclic)".to_string()); return; // the store dataflow below needs a DAG } - for (bi, c) in color.iter().enumerate() { - if *c == 0 { + let reachable: Vec = color.iter().map(|c| *c != 0).collect(); + for (bi, r) in reachable.iter().enumerate() { + if !r { err(errs, Some(bi), None, "unreachable block".to_string()); } } @@ -458,7 +497,7 @@ fn check_cfg_and_stores(p: &Program, errs: &mut Vec) { let mut entry_state: Vec>> = vec![None; n]; entry_state[0] = Some(vec![0; ncols]); - for bi in topo_order(p, n) { + for bi in topo_order(p, n, &reachable) { let Some(state) = entry_state[bi].clone() else { continue; // unreachable; already reported }; @@ -529,12 +568,18 @@ fn check_cfg_and_stores(p: &Program, errs: &mut Vec) { } } -/// Topological order of the (already acyclicity-checked) CFG via Kahn's -/// algorithm; unreachable blocks may appear anywhere, which is fine — they -/// have no entry state and are skipped. -fn topo_order(p: &Program, n: usize) -> Vec { +/// Topological order of the reachable, acyclicity-checked subgraph via +/// Kahn's algorithm. Indegrees count only edges whose SOURCE is reachable: +/// an edge out of an unreachable island (which may itself be cyclic and +/// never drain) must not starve a reachable join of its dataflow visit — +/// that would mask the join's store errors behind the island's +/// unreachable-block errors and make them reappear one fix later. +fn topo_order(p: &Program, n: usize, reachable: &[bool]) -> Vec { let mut indegree = vec![0usize; n]; - for b in &p.blocks { + for (bi, b) in p.blocks.iter().enumerate() { + if !reachable[bi] { + continue; + } for (succ, _) in b.term.successors() { let s = succ.0 as usize; if s < n { @@ -542,7 +587,7 @@ fn topo_order(p: &Program, n: usize) -> Vec { } } } - let mut stack: Vec = (0..n).filter(|&i| indegree[i] == 0).collect(); + let mut stack: Vec = (0..n).filter(|&i| reachable[i] && indegree[i] == 0).collect(); let mut order = Vec::with_capacity(n); while let Some(b) = stack.pop() { order.push(b); From 362595fd6132ff2640fedf70de7bc28e3d5bc8d2 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 15:50:04 +0200 Subject: [PATCH 08/11] chore(backlog): TASK-41 acceptance criteria checked + implementation notes Co-Authored-By: Claude Opus 5 --- ...00\224-grammar-types-verifier-text-format.md" | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git "a/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" "b/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" index 4a88e46..66a27a6 100644 --- "a/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" +++ "b/backlog/tasks/task-41 - Specializer-M-ir-imperative-IR-\342\200\224-grammar-types-verifier-text-format.md" @@ -4,7 +4,7 @@ title: 'Specializer M-ir: imperative IR — grammar, types, verifier, text forma status: In Progress assignee: [] created_date: '2026-07-25 02:31' -updated_date: '2026-07-25 02:58' +updated_date: '2026-07-25 13:46' labels: [] milestone: m-7 dependencies: [] @@ -22,10 +22,10 @@ Define the specializer's imperative IR per §6 of docs/superpowers/specs/2026-07 ## Acceptance Criteria -- [ ] #1 Verifier rejects: non-SSA defs, type mismatches, any arithmetic on a T? value not routed through the null-lane ops, unresolvable StaticRef ids, allocating constructs -- [ ] #2 Text format round-trips: parse(print(ir)) == ir on every test program, including a property/fuzz round-trip test -- [ ] #3 Hand-written IR programs covering every instruction exist as test fixtures -- [ ] #4 mise gate-specializer green +- [x] #1 Verifier rejects: non-SSA defs, type mismatches, any arithmetic on a T? value not routed through the null-lane ops, unresolvable StaticRef ids, allocating constructs +- [x] #2 Text format round-trips: parse(print(ir)) == ir on every test program, including a property/fuzz round-trip test +- [x] #3 Hand-written IR programs covering every instruction exist as test fixtures +- [x] #4 mise gate-specializer green ## Implementation Plan @@ -37,3 +37,9 @@ Define the specializer's imperative IR per §6 of docs/superpowers/specs/2026-07 4. Adversarial workflow: fan-out attackers on verifier soundness + round-trip edge cases (float/string literals), plus design-conformance and simplification reviewers; fix confirmed findings. 5. Gate green; stacked PR onto claude/duckdb-native-interpreter-36d016. + +## Implementation Notes + + +Implemented in src/specializer/ir/ (mod, verify, print, parse, fixtures, gen, tests). Key decisions vs the original sketch (design doc §6 updated): implicit row cursor, emit/skip/trap terminators carrying the store contract, strict block-param SSA (no dominance analysis), acyclic v0 CFG, presentation-only names (canonical %vN/bN). Adversarial pass (6 agents) found and I fixed: recursive-DFS process abort on deep CFGs (now iterative, 50k blocks OK), empty map static signatures and non-ident fn names verifying but printing unparseably, non-canonical NaN payloads breaking bitwise round-trip equality (NaN-class equality now), unreachable-island starvation masking join store errors (topo indegrees over reachable sources only), doc/grammar drift, dead uses() code. 45 cargo tests; every verifier rule and parser guard has a rejecting test; 300-seed fuzz round-trip. + From 35328002529ddffa98b59ce57174cfbe6b164fc5 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 20:52:07 +0200 Subject: [PATCH 09/11] =?UTF-8?q?feat(specializer):=20closure-compiled=20I?= =?UTF-8?q?R=20interpreter=20=E2=80=94=20the=20oracle=20(M-interp)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/specializer/exec/: runtime substrate (Batch, byte-backed bump Arena with StrRef spans, sorted-vec map statics probed by allocation-free binary search) + interp.rs (compile verifies first — nothing executes unverified IR — then one pre-traversal builds per-block closures; terminators are a small enum in the row loop; branch-arg copies are safe by SSA disjointness). Steady state allocates nothing: registers/arena/out builders live in a reusable RunState; a thread-local counting global allocator asserts zero allocs on a warmed run. Semantics pins documented in interp.rs (checked integer arithmetic, IEEE floats, half-away-from-zero ftoi.round, exact-parse stoi/stof) — provisional until the DuckDB differential at M-lower. All five M-ir fixtures execute against hand-computed expectations incl. the trap path; 150-seed generated programs execute deterministically. TASK-42. Co-Authored-By: Claude Opus 5 --- ...piled-IR-interpreter-the-oracle-backend.md | 13 +- src/specializer/exec/interp.rs | 817 ++++++++++++++++++ src/specializer/exec/mod.rs | 215 +++++ src/specializer/exec/tests.rs | 468 ++++++++++ src/specializer/mod.rs | 1 + 5 files changed, 1513 insertions(+), 1 deletion(-) create mode 100644 src/specializer/exec/interp.rs create mode 100644 src/specializer/exec/mod.rs create mode 100644 src/specializer/exec/tests.rs diff --git a/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md b/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md index b92a926..d57e41a 100644 --- a/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md +++ b/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md @@ -1,9 +1,10 @@ --- id: TASK-42 title: 'Specializer M-interp: closure-compiled IR interpreter (the oracle backend)' -status: To Do +status: In Progress assignee: [] created_date: '2026-07-25 02:31' +updated_date: '2026-07-25 18:42' labels: [] milestone: m-7 dependencies: @@ -27,3 +28,13 @@ Implement the interpreter backend over the imperative IR (design doc §7): one p - [ ] #3 No allocation during execution (arena-only for varlen), asserted by a test - [ ] #4 mise gate-specializer green + +## Implementation Plan + + +1. Runtime substrate in src/specializer/exec/: Batch (typed columns + validity), Arena (bump String, spans as (u32,u32) StrRef), StaticData (Scalar / Map keyed by bit-hashed key tuples) type-checked against Program.statics at compile. +2. interp.rs: compile(program, statics) runs verify() first (unverified IR is rejected — AC#2), then one pre-traversal builds per-block closure lists + terminator thunks; Frame of Copy registers indexed by Value id, reused across rows; stores push straight into pre-capacitied out builders (verifier's exactly-once contract keeps columns row-aligned). +3. Semantics pins (documented in interp.rs, oracle-differential at M-lower): checked integer arithmetic traps on overflow; idiv/irem trap on 0 and MIN/-1; fdiv IEEE; fcmp IEEE-ordered (NaN compares false); ftoi.trunc toward zero, ftoi.round half-away-from-zero (DuckDB CAST), both trap out of i64 range; stoi/stof exact parse. +4. Tests: all 5 M-ir fixtures executed against hand-computed inputs/statics/outputs; unverified-program rejection; counting global allocator asserts zero heap allocs on a warmed second run (arena/regs/builders reused); executor fuzz over gen::gen_program seeds with random inputs (no panics, |out| <= |in|, deterministic). +5. Adversarial workflow (semantics vs spec, alloc discipline, trap/edge paths, fuzz), fix confirmed findings; gate green; stacked PR onto claude/duckdb-native-interpreter-36d016. + diff --git a/src/specializer/exec/interp.rs b/src/specializer/exec/interp.rs new file mode 100644 index 0000000..b1a3adf --- /dev/null +++ b/src/specializer/exec/interp.rs @@ -0,0 +1,817 @@ +//! The closure-compiled interpreter backend — the oracle. One pre-traversal +//! of a VERIFIED program builds a vector of instruction closures per block; +//! execution is plain dispatch. Never optimized: correctness and coverage +//! over speed, and it stays that way (design doc §7). +//! +//! # Semantics pins (provisional until the DuckDB differential at M-lower) +//! +//! * `iadd`/`isub`/`imul` trap on i64 overflow (SQL overflow is an error, +//! not a wrap); `idiv`/`irem` trap on zero and on `i64::MIN / -1`. +//! * `f*` arithmetic is raw IEEE — inf/nan flow through, no traps. +//! * `fcmp` is IEEE-ordered: every predicate involving NaN is false, except +//! `ne`, which is `!(a == b)` and therefore true. SQL NULL/NaN policy is +//! the lowering's job, expressed with flags around these primitives. +//! * `icmp` is i64 order; `scmp` is byte order (Rust `str` cmp). +//! * `itof` is `as f64` (may round — same as DuckDB's BIGINT->DOUBLE). +//! * `ftoi.trunc` rounds toward zero; `ftoi.round` half-away-from-zero +//! (matches DuckDB CAST); both trap on non-finite or out-of-i64-range. +//! * `stoi.opt`/`stof.opt` are exact `str::parse` — no whitespace trimming; +//! whether SQL CAST trims is a lowering decision to pin at M-lower. +//! * `itos`/`ftos` format into the arena; `ftos` uses Rust's shortest +//! round-trip form (provisional; oracle-pinned at M-lower). +//! * On a false validity flag the payload is the type default; `load.opt` +//! normalizes even if the input batch carries garbage in invalid slots. + +use std::fmt::Write as _; + +use super::super::ir::{ + self, BinOp, CmpPred, Inst, Program, RoundMode, StaticTy, Term, Ty, Value, +}; +use super::super::ir::verify::{verify, VerifyError}; +use super::{ + Arena, Batch, ColData, KeyBits, OutCol, RegVal, RunState, ScalarVal, StaticData, StrRef, Trap, +}; + +#[derive(Debug)] +pub enum CompileError { + /// The program failed verification — nothing executes unverified IR. + Verify(Vec), + /// The static data does not match the program's static declarations. + Static(String), +} + +impl std::fmt::Display for CompileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CompileError::Verify(errs) => { + write!(f, "program failed verification: ")?; + for e in errs { + write!(f, "[{e}] ")?; + } + Ok(()) + } + CompileError::Static(msg) => write!(f, "static data mismatch: {msg}"), + } + } +} + +/// One compiled instruction: reads registers/input/statics, writes registers +/// or output builders. `Ok(())` or a call-aborting trap. +type InstFn = Box Fn(&mut Ctx<'a>) -> Result<(), Trap>>; + +/// Everything a closure can touch during one row. +pub struct Ctx<'a> { + regs: &'a mut [RegVal], + arena: &'a mut Arena, + out: &'a mut [OutCol], + input: &'a Batch, + statics: &'a [PreparedStatic], + row: usize, +} + +enum PreparedStatic { + Scalar { valid: bool, val: ScalarVal }, + /// Sorted by key; probed by allocation-free binary search. + Map { + entries: Vec<(Vec, Vec)>, + }, +} + +struct CBlock { + insts: Vec, + term: CTerm, +} + +/// Terminators are a small enum interpreted by the row loop — no closure +/// indirection needed. Branch-arg copies are (src, dst) register pairs; +/// sources and destinations are disjoint by SSA single-definition, so +/// sequential copies are safe. +enum CTerm { + Jump { to: usize, moves: Vec<(u32, u32)> }, + Brif { cond: u32, then_to: usize, then_moves: Vec<(u32, u32)>, else_to: usize, else_moves: Vec<(u32, u32)> }, + Emit, + Skip, + Trap(String), +} + +pub struct InterpFn { + blocks: Vec, + nregs: usize, + statics: Vec, + in_decl: Vec<(Ty, bool)>, + out_decl: Vec<(Ty, bool)>, +} + +pub fn compile(p: &Program, statics: Vec) -> Result { + verify(p).map_err(CompileError::Verify)?; + let prepared = prepare_statics(p, statics)?; + + let nregs = max_value_id(p) + 1; + let mut blocks = Vec::with_capacity(p.blocks.len()); + for b in &p.blocks { + let mut insts: Vec = Vec::with_capacity(b.insts.len()); + for inst in &b.insts { + insts.push(compile_inst(p, inst)); + } + blocks.push(CBlock { insts, term: compile_term(p, &b.term) }); + } + + Ok(InterpFn { + blocks, + nregs, + statics: prepared, + in_decl: p.in_cols.iter().map(|c| (c.ty.ty, c.ty.nullable)).collect(), + out_decl: p.out_cols.iter().map(|c| (c.ty.ty, c.ty.nullable)).collect(), + }) +} + +impl InterpFn { + /// Fresh reusable buffers for `run`. Allocate once, reuse per call. + pub fn new_state(&self) -> RunState { + RunState { + regs: vec![RegVal::I64(0); self.nregs], + arena: Arena::default(), + out: self + .out_decl + .iter() + .map(|(ty, _)| match ty { + Ty::I1 => OutCol::I1(Vec::new()), + Ty::I64 => OutCol::I64(Vec::new()), + Ty::F64 => OutCol::F64(Vec::new()), + Ty::Str => OutCol::Str(Vec::new()), + }) + .collect(), + } + } + + /// Execute over `input`, filling `st.out` (cleared first, capacity kept). + /// On `Err` the output is meaningless and the whole call is void. + pub fn run(&self, input: &Batch, st: &mut RunState) -> Result<(), Trap> { + self.check_input(input)?; + self.check_state(st)?; + st.arena.clear(); + for col in st.out.iter_mut() { + col.clear(); + } + reserve_out(&mut st.out, input.rows); + + for row in 0..input.rows { + let mut ctx = Ctx { + regs: &mut st.regs, + arena: &mut st.arena, + out: &mut st.out, + input, + statics: &self.statics, + row, + }; + let mut bi = 0usize; + loop { + for f in &self.blocks[bi].insts { + f(&mut ctx)?; + } + match &self.blocks[bi].term { + CTerm::Jump { to, moves } => { + do_moves(ctx.regs, moves); + bi = *to; + } + CTerm::Brif { cond, then_to, then_moves, else_to, else_moves } => { + if as_i1(ctx.regs[*cond as usize]) { + do_moves(ctx.regs, then_moves); + bi = *then_to; + } else { + do_moves(ctx.regs, else_moves); + bi = *else_to; + } + } + CTerm::Emit | CTerm::Skip => break, + CTerm::Trap(msg) => return Err(Trap(msg.clone())), + } + } + } + Ok(()) + } + + /// A `RunState` is only valid for the `InterpFn` that created it — + /// reject a foreign one with a trap instead of an index panic. + fn check_state(&self, st: &RunState) -> Result<(), Trap> { + if st.regs.len() < self.nregs { + return Err(Trap(format!( + "RunState has {} register(s), this function needs {} — states are not \ + shareable across compiled functions", + st.regs.len(), + self.nregs + ))); + } + if st.out.len() != self.out_decl.len() { + return Err(Trap(format!( + "RunState has {} out column(s), this function declares {}", + st.out.len(), + self.out_decl.len() + ))); + } + for (ci, (col, (ty, _))) in st.out.iter().zip(self.out_decl.iter()).enumerate() { + let col_ty = match col { + OutCol::I1(_) => Ty::I1, + OutCol::I64(_) => Ty::I64, + OutCol::F64(_) => Ty::F64, + OutCol::Str(_) => Ty::Str, + }; + if col_ty != *ty { + return Err(Trap(format!( + "RunState out column {ci} is {}, this function declares {}", + col_ty.name(), + ty.name() + ))); + } + } + Ok(()) + } + + fn check_input(&self, input: &Batch) -> Result<(), Trap> { + if input.cols.len() != self.in_decl.len() { + return Err(Trap(format!( + "input has {} column(s), the program declares {}", + input.cols.len(), + self.in_decl.len() + ))); + } + for (ci, (col, (ty, _))) in input.cols.iter().zip(self.in_decl.iter()).enumerate() { + if col.ty() != *ty { + return Err(Trap(format!( + "input column {ci} is {}, the program declares {}", + col.ty().name(), + ty.name() + ))); + } + if col.len() != input.rows { + return Err(Trap(format!( + "input column {ci} has {} row(s), the batch declares {}", + col.len(), + input.rows + ))); + } + } + Ok(()) + } +} + +fn do_moves(regs: &mut [RegVal], moves: &[(u32, u32)]) { + for (src, dst) in moves { + regs[*dst as usize] = regs[*src as usize]; + } +} + +fn reserve_out(out: &mut [OutCol], rows: usize) { + for col in out { + match col { + OutCol::I1(v) => v.reserve(rows), + OutCol::I64(v) => v.reserve(rows), + OutCol::F64(v) => v.reserve(rows), + OutCol::Str(v) => v.reserve(rows), + } + } +} + +fn max_value_id(p: &Program) -> usize { + let mut max = 0usize; + for b in &p.blocks { + for (v, _) in &b.params { + max = max.max(v.0 as usize); + } + for inst in &b.insts { + for d in inst.dsts() { + max = max.max(d.0 as usize); + } + } + } + max +} + +fn prepare_statics( + p: &Program, + statics: Vec, +) -> Result, CompileError> { + if statics.len() != p.statics.len() { + return Err(CompileError::Static(format!( + "program declares {} static(s), {} provided", + p.statics.len(), + statics.len() + ))); + } + let mut prepared = Vec::with_capacity(statics.len()); + for (i, (decl, data)) in p.statics.iter().zip(statics.into_iter()).enumerate() { + match (decl, data) { + (StaticTy::Scalar(ct), StaticData::Scalar { valid, val }) => { + if val.ty() != ct.ty { + return Err(CompileError::Static(format!( + "@{i}: scalar is {}, declared {}", + val.ty().name(), + ct.ty.name() + ))); + } + if !valid && !ct.nullable { + return Err(CompileError::Static(format!( + "@{i}: NULL scalar for a non-nullable declaration" + ))); + } + prepared.push(PreparedStatic::Scalar { valid, val }); + } + (StaticTy::Map { keys, values }, StaticData::Map(mut entries)) => { + for (ei, (k, v)) in entries.iter().enumerate() { + let kt: Vec = k.iter().map(|kb| kb.ty()).collect(); + let vt: Vec = v.iter().map(|sv| sv.ty()).collect(); + if kt != *keys || vt != *values { + return Err(CompileError::Static(format!( + "@{i}: entry {ei} has shape ({kt:?}) -> ({vt:?}), declared \ + ({keys:?}) -> ({values:?})" + ))); + } + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + if entries.windows(2).any(|w| w[0].0 == w[1].0) { + return Err(CompileError::Static(format!("@{i}: duplicate map key"))); + } + prepared.push(PreparedStatic::Map { entries }); + } + (StaticTy::Scalar(_), StaticData::Map(_)) => { + return Err(CompileError::Static(format!( + "@{i}: declared scalar, got map data" + ))) + } + (StaticTy::Map { .. }, StaticData::Scalar { .. }) => { + return Err(CompileError::Static(format!( + "@{i}: declared map, got scalar data" + ))) + } + } + } + Ok(prepared) +} + +// ------------------------------------------------------------ registers -- +// Verified programs make these matches infallible; a miss is a bug in the +// verifier or the compiler, not in the program, hence unreachable!. + +fn as_i1(r: RegVal) -> bool { + match r { + RegVal::I1(b) => b, + _ => unreachable!("type hole past the verifier: expected i1"), + } +} + +fn as_i64(r: RegVal) -> i64 { + match r { + RegVal::I64(v) => v, + _ => unreachable!("type hole past the verifier: expected i64"), + } +} + +fn as_f64(r: RegVal) -> f64 { + match r { + RegVal::F64(v) => v, + _ => unreachable!("type hole past the verifier: expected f64"), + } +} + +fn as_str(r: RegVal) -> StrRef { + match r { + RegVal::Str(s) => s, + _ => unreachable!("type hole past the verifier: expected str"), + } +} + +fn scalar_to_reg(v: &ScalarVal, arena: &mut Arena) -> RegVal { + match v { + ScalarVal::I1(b) => RegVal::I1(*b), + ScalarVal::I64(i) => RegVal::I64(*i), + ScalarVal::F64(f) => RegVal::F64(*f), + ScalarVal::Str(s) => RegVal::Str(arena.push_str(s)), + } +} + +fn default_reg(ty: Ty) -> RegVal { + match ty { + Ty::I1 => RegVal::I1(false), + Ty::I64 => RegVal::I64(0), + Ty::F64 => RegVal::F64(0.0), + Ty::Str => RegVal::Str(StrRef { off: 0, len: 0 }), + } +} + +// ---------------------------------------------------------- compilation -- + +fn compile_term(p: &Program, t: &Term) -> CTerm { + let mk_moves = |to: ir::BlockId, args: &Vec| -> (usize, Vec<(u32, u32)>) { + let params = &p.blocks[to.0 as usize].params; + let moves = args + .iter() + .zip(params.iter()) + .map(|(src, (dst, _))| (src.0, dst.0)) + .collect(); + (to.0 as usize, moves) + }; + match t { + Term::Jump { to, args } => { + let (to, moves) = mk_moves(*to, args); + CTerm::Jump { to, moves } + } + Term::Brif { cond, then_to, then_args, else_to, else_args } => { + let (then_to, then_moves) = mk_moves(*then_to, then_args); + let (else_to, else_moves) = mk_moves(*else_to, else_args); + CTerm::Brif { cond: cond.0, then_to, then_moves, else_to, else_moves } + } + Term::Emit => CTerm::Emit, + Term::Skip => CTerm::Skip, + Term::Trap { msg } => CTerm::Trap(msg.clone()), + } +} + +/// Byte sink over the arena so `write!` formats without intermediate +/// allocation. +struct ArenaWriter<'a>(&'a mut Vec); + +impl std::fmt::Write for ArenaWriter<'_> { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + self.0.extend_from_slice(s.as_bytes()); + Ok(()) + } +} + +fn fmt_into_arena(arena: &mut Arena, args: std::fmt::Arguments<'_>) -> StrRef { + let off = arena.0.len() as u32; + let _ = ArenaWriter(&mut arena.0).write_fmt(args); + StrRef { off, len: arena.0.len() as u32 - off } +} + +fn compile_inst(p: &Program, inst: &Inst) -> InstFn { + match inst.clone() { + Inst::Const { dst, lit } => { + let dst = dst.0 as usize; + match lit { + ir::Lit::I1(b) => Box::new(move |ctx| { + ctx.regs[dst] = RegVal::I1(b); + Ok(()) + }), + ir::Lit::I64(i) => Box::new(move |ctx| { + ctx.regs[dst] = RegVal::I64(i); + Ok(()) + }), + ir::Lit::F64(f) => Box::new(move |ctx| { + ctx.regs[dst] = RegVal::F64(f); + Ok(()) + }), + ir::Lit::Str(s) => Box::new(move |ctx| { + ctx.regs[dst] = RegVal::Str(ctx.arena.push_str(&s)); + Ok(()) + }), + } + } + Inst::Bin { op, dst, a, b } => { + let (dst, a, b) = (dst.0 as usize, a.0 as usize, b.0 as usize); + match op { + BinOp::Iadd | BinOp::Isub | BinOp::Imul => Box::new(move |ctx| { + let (x, y) = (as_i64(ctx.regs[a]), as_i64(ctx.regs[b])); + let r = match op { + BinOp::Iadd => x.checked_add(y), + BinOp::Isub => x.checked_sub(y), + _ => x.checked_mul(y), + }; + match r { + Some(v) => { + ctx.regs[dst] = RegVal::I64(v); + Ok(()) + } + None => Err(Trap(format!("i64 overflow in {}", op.name()))), + } + }), + BinOp::Idiv | BinOp::Irem => Box::new(move |ctx| { + let (x, y) = (as_i64(ctx.regs[a]), as_i64(ctx.regs[b])); + if y == 0 { + return Err(Trap(format!("division by zero in {}", op.name()))); + } + let r = match op { + BinOp::Idiv => x.checked_div(y), + _ => x.checked_rem(y), + }; + match r { + Some(v) => { + ctx.regs[dst] = RegVal::I64(v); + Ok(()) + } + None => Err(Trap(format!("i64 overflow in {}", op.name()))), + } + }), + BinOp::Fadd | BinOp::Fsub | BinOp::Fmul | BinOp::Fdiv => Box::new(move |ctx| { + let (x, y) = (as_f64(ctx.regs[a]), as_f64(ctx.regs[b])); + let v = match op { + BinOp::Fadd => x + y, + BinOp::Fsub => x - y, + BinOp::Fmul => x * y, + _ => x / y, + }; + ctx.regs[dst] = RegVal::F64(v); + Ok(()) + }), + BinOp::And | BinOp::Or | BinOp::Xor => Box::new(move |ctx| { + let (x, y) = (as_i1(ctx.regs[a]), as_i1(ctx.regs[b])); + let v = match op { + BinOp::And => x && y, + BinOp::Or => x || y, + _ => x ^ y, + }; + ctx.regs[dst] = RegVal::I1(v); + Ok(()) + }), + } + } + Inst::Cmp { pred, ty, dst, a, b } => { + let (dst, a, b) = (dst.0 as usize, a.0 as usize, b.0 as usize); + Box::new(move |ctx| { + let v = match ty { + Ty::I64 => apply_ord(pred, as_i64(ctx.regs[a]).cmp(&as_i64(ctx.regs[b]))), + Ty::F64 => { + // IEEE partial order: NaN makes eq/lt/le/gt/ge false + // and ne true. + let (x, y) = (as_f64(ctx.regs[a]), as_f64(ctx.regs[b])); + match pred { + CmpPred::Eq => x == y, + CmpPred::Ne => x != y, + CmpPred::Lt => x < y, + CmpPred::Le => x <= y, + CmpPred::Gt => x > y, + CmpPred::Ge => x >= y, + } + } + Ty::Str => { + let (x, y) = (as_str(ctx.regs[a]), as_str(ctx.regs[b])); + apply_ord(pred, ctx.arena.get(x).cmp(ctx.arena.get(y))) + } + Ty::I1 => unreachable!("cmp on i1 is rejected by the verifier"), + }; + ctx.regs[dst] = RegVal::I1(v); + Ok(()) + }) + } + Inst::Not { dst, a } => { + let (dst, a) = (dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + ctx.regs[dst] = RegVal::I1(!as_i1(ctx.regs[a])); + Ok(()) + }) + } + Inst::Select { dst, cond, a, b } => { + let (dst, cond, a, b) = (dst.0 as usize, cond.0 as usize, a.0 as usize, b.0 as usize); + Box::new(move |ctx| { + ctx.regs[dst] = if as_i1(ctx.regs[cond]) { ctx.regs[a] } else { ctx.regs[b] }; + Ok(()) + }) + } + Inst::Itof { dst, a } => { + let (dst, a) = (dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + ctx.regs[dst] = RegVal::F64(as_i64(ctx.regs[a]) as f64); + Ok(()) + }) + } + Inst::Ftoi { mode, dst, a } => { + let (dst, a) = (dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + let x = as_f64(ctx.regs[a]); + let r = match mode { + RoundMode::Trunc => x.trunc(), + RoundMode::Round => x.round(), // half away from zero + }; + // 2^63 is exactly representable; anything in [-2^63, 2^63) + // fits i64 after rounding. + if r.is_finite() && r >= -(2f64.powi(63)) && r < 2f64.powi(63) { + ctx.regs[dst] = RegVal::I64(r as i64); + Ok(()) + } else { + Err(Trap(format!("f64 value {x:?} out of i64 range in ftoi"))) + } + }) + } + Inst::Itos { dst, a } => { + let (dst, a) = (dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + let v = as_i64(ctx.regs[a]); + ctx.regs[dst] = RegVal::Str(fmt_into_arena(ctx.arena, format_args!("{v}"))); + Ok(()) + }) + } + Inst::Ftos { dst, a } => { + let (dst, a) = (dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + let v = as_f64(ctx.regs[a]); + ctx.regs[dst] = RegVal::Str(fmt_into_arena(ctx.arena, format_args!("{v:?}"))); + Ok(()) + }) + } + Inst::StoiOpt { flag, dst, a } => { + let (flag, dst, a) = (flag.0 as usize, dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + let s = ctx.arena.get(as_str(ctx.regs[a])); + match s.parse::() { + Ok(v) => { + ctx.regs[flag] = RegVal::I1(true); + ctx.regs[dst] = RegVal::I64(v); + } + Err(_) => { + ctx.regs[flag] = RegVal::I1(false); + ctx.regs[dst] = RegVal::I64(0); + } + } + Ok(()) + }) + } + Inst::StofOpt { flag, dst, a } => { + let (flag, dst, a) = (flag.0 as usize, dst.0 as usize, a.0 as usize); + Box::new(move |ctx| { + let s = ctx.arena.get(as_str(ctx.regs[a])); + match s.parse::() { + Ok(v) => { + ctx.regs[flag] = RegVal::I1(true); + ctx.regs[dst] = RegVal::F64(v); + } + Err(_) => { + ctx.regs[flag] = RegVal::I1(false); + ctx.regs[dst] = RegVal::F64(0.0); + } + } + Ok(()) + }) + } + Inst::Sconcat { dst, a, b } => { + let (dst, a, b) = (dst.0 as usize, a.0 as usize, b.0 as usize); + Box::new(move |ctx| { + let (x, y) = (as_str(ctx.regs[a]), as_str(ctx.regs[b])); + ctx.regs[dst] = RegVal::Str(ctx.arena.concat(x, y)); + Ok(()) + }) + } + Inst::Load { dst, col } => { + let (dst, col) = (dst.0 as usize, col as usize); + Box::new(move |ctx| { + ctx.regs[dst] = load_payload(&ctx.input.cols[col], ctx.row, ctx.arena); + Ok(()) + }) + } + Inst::LoadOpt { flag, dst, col } => { + let (flag, dst, col) = (flag.0 as usize, dst.0 as usize, col as usize); + let ty = p.in_cols[col].ty.ty; + Box::new(move |ctx| { + let c = &ctx.input.cols[col]; + let valid = col_valid(c, ctx.row); + ctx.regs[flag] = RegVal::I1(valid); + ctx.regs[dst] = if valid { + load_payload(c, ctx.row, ctx.arena) + } else { + default_reg(ty) + }; + Ok(()) + }) + } + Inst::Store { col, val } => { + let (col, val) = (col as usize, val.0 as usize); + Box::new(move |ctx| { + push_out(&mut ctx.out[col], true, ctx.regs[val]); + Ok(()) + }) + } + Inst::StoreOpt { col, flag, val } => { + let (col, flag, val) = (col as usize, flag.0 as usize, val.0 as usize); + Box::new(move |ctx| { + let valid = as_i1(ctx.regs[flag]); + push_out(&mut ctx.out[col], valid, ctx.regs[val]); + Ok(()) + }) + } + Inst::Probe { static_id, hit, dsts, keys } => { + let static_id = static_id as usize; + let hit = hit.0 as usize; + let dsts: Vec = dsts.iter().map(|d| d.0 as usize).collect(); + let keys: Vec = keys.iter().map(|k| k.0 as usize).collect(); + let value_tys: Vec = match &p.statics[static_id] { + StaticTy::Map { values, .. } => values.clone(), + _ => unreachable!("probe on non-map is rejected by the verifier"), + }; + Box::new(move |ctx| { + let PreparedStatic::Map { entries } = &ctx.statics[static_id] else { + unreachable!("static kind checked at compile"); + }; + let found = entries + .binary_search_by(|(k, _)| cmp_key(k, &keys, ctx)) + .ok(); + match found { + Some(idx) => { + ctx.regs[hit] = RegVal::I1(true); + // Copy values out BEFORE writing regs that might + // alias arena growth: scalar_to_reg appends to the + // arena for str values, which is fine — spans stay + // valid, the arena only grows. + for (di, vi) in dsts.iter().zip(0..value_tys.len()) { + let v = &entries[idx].1[vi]; + ctx.regs[*di] = scalar_to_reg(v, ctx.arena); + } + } + None => { + ctx.regs[hit] = RegVal::I1(false); + for (di, ty) in dsts.iter().zip(value_tys.iter()) { + ctx.regs[*di] = default_reg(*ty); + } + } + } + Ok(()) + }) + } + Inst::Sload { static_id, dst } => { + let (static_id, dst) = (static_id as usize, dst.0 as usize); + Box::new(move |ctx| { + let PreparedStatic::Scalar { val, .. } = &ctx.statics[static_id] else { + unreachable!("static kind checked at compile"); + }; + ctx.regs[dst] = scalar_to_reg(val, ctx.arena); + Ok(()) + }) + } + Inst::SloadOpt { static_id, flag, dst } => { + let (static_id, flag, dst) = (static_id as usize, flag.0 as usize, dst.0 as usize); + let ty = match &p.statics[static_id] { + StaticTy::Scalar(ct) => ct.ty, + _ => unreachable!("sload on non-scalar is rejected by the verifier"), + }; + Box::new(move |ctx| { + let PreparedStatic::Scalar { valid, val } = &ctx.statics[static_id] else { + unreachable!("static kind checked at compile"); + }; + ctx.regs[flag] = RegVal::I1(*valid); + ctx.regs[dst] = if *valid { + scalar_to_reg(val, ctx.arena) + } else { + default_reg(ty) + }; + Ok(()) + }) + } + } +} + +/// Compare a stored key tuple against the probe registers, position-wise — +/// the search-time mirror of `KeyBits: Ord` (which the entries were sorted +/// by), so the binary search stays allocation-free. +fn cmp_key(stored: &[KeyBits], key_regs: &[usize], ctx: &Ctx<'_>) -> std::cmp::Ordering { + use std::cmp::Ordering; + for (kb, reg) in stored.iter().zip(key_regs.iter()) { + let ord = match (kb, ctx.regs[*reg]) { + (KeyBits::I1(s), RegVal::I1(v)) => s.cmp(&v), + (KeyBits::I64(s), RegVal::I64(v)) => s.cmp(&v), + (KeyBits::F64(s), RegVal::F64(v)) => s.cmp(&v.to_bits()), + (KeyBits::Str(s), RegVal::Str(v)) => s.as_str().cmp(ctx.arena.get(v)), + _ => unreachable!("probe key types checked at compile"), + }; + if ord != Ordering::Equal { + return ord; + } + } + Ordering::Equal +} + +fn apply_ord(pred: CmpPred, ord: std::cmp::Ordering) -> bool { + use std::cmp::Ordering::*; + match pred { + CmpPred::Eq => ord == Equal, + CmpPred::Ne => ord != Equal, + CmpPred::Lt => ord == Less, + CmpPred::Le => ord != Greater, + CmpPred::Gt => ord == Greater, + CmpPred::Ge => ord != Less, + } +} + +fn col_valid(c: &ColData, row: usize) -> bool { + match c { + ColData::I1 { valid, .. } + | ColData::I64 { valid, .. } + | ColData::F64 { valid, .. } + | ColData::Str { valid, .. } => valid.get(row).copied().unwrap_or(true), + } +} + +fn load_payload(c: &ColData, row: usize, arena: &mut Arena) -> RegVal { + match c { + ColData::I1 { data, .. } => RegVal::I1(data[row]), + ColData::I64 { data, .. } => RegVal::I64(data[row]), + ColData::F64 { data, .. } => RegVal::F64(data[row]), + ColData::Str { data, .. } => RegVal::Str(arena.push_str(&data[row])), + } +} + +fn push_out(col: &mut OutCol, valid: bool, v: RegVal) { + match (col, v) { + (OutCol::I1(vec), RegVal::I1(b)) => vec.push((valid, b)), + (OutCol::I64(vec), RegVal::I64(i)) => vec.push((valid, i)), + (OutCol::F64(vec), RegVal::F64(f)) => vec.push((valid, f)), + (OutCol::Str(vec), RegVal::Str(s)) => vec.push((valid, s)), + _ => unreachable!("store type checked by the verifier"), + } +} diff --git a/src/specializer/exec/mod.rs b/src/specializer/exec/mod.rs new file mode 100644 index 0000000..9b05291 --- /dev/null +++ b/src/specializer/exec/mod.rs @@ -0,0 +1,215 @@ +//! Execution substrate shared by the interpreter backend (and, at +//! M-cranelift, the codegen backend): batches, the bump arena, prepared +//! static structures, and the run-state buffers. Everything here obeys the +//! Stage-2 discipline — steady-state execution allocates nothing on the +//! heap; the only growable memory is the arena and the pre-reserved output +//! builders, both owned by [`RunState`] and reused across calls. +//! +//! Lives under `exec/` rather than a separate `runtime/` until the Cranelift +//! backend actually shares it — one home per concept until two consumers +//! exist. + +pub mod interp; + +#[cfg(test)] +mod tests; + +use super::ir::Ty; + +/// A runtime error that aborts the whole call (division by zero, integer +/// overflow, CAST failure routed to `trap`, input-shape mismatch). +/// Constructing one allocates — that is fine, it is the error path. +#[derive(Debug, PartialEq, Eq)] +pub struct Trap(pub String); + +impl std::fmt::Display for Trap { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "trap: {}", self.0) + } +} + +/// Columnar input batch. `valid` is meaningful only for nullable columns — +/// for non-nullable columns the interpreter ignores it entirely. The payload +/// slot of an invalid row may hold anything; readers normalize it to the +/// type's default so downstream behavior never depends on garbage. +pub struct Batch { + pub rows: usize, + pub cols: Vec, +} + +pub enum ColData { + I1 { valid: Vec, data: Vec }, + I64 { valid: Vec, data: Vec }, + F64 { valid: Vec, data: Vec }, + Str { valid: Vec, data: Vec }, +} + +impl ColData { + pub fn ty(&self) -> Ty { + match self { + ColData::I1 { .. } => Ty::I1, + ColData::I64 { .. } => Ty::I64, + ColData::F64 { .. } => Ty::F64, + ColData::Str { .. } => Ty::Str, + } + } + + pub fn len(&self) -> usize { + match self { + ColData::I1 { data, .. } => data.len(), + ColData::I64 { data, .. } => data.len(), + ColData::F64 { data, .. } => data.len(), + ColData::Str { data, .. } => data.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// A span into the run arena. All `str`-typed register values are arena +/// spans — input/const/static strings are copied in on read, so registers +/// stay `Copy` and nothing borrows across rows. +#[derive(Clone, Copy, Debug)] +pub struct StrRef { + pub off: u32, + pub len: u32, +} + +/// Bump arena for varlen values, reset per call. Byte-backed so +/// `extend_from_within` (sconcat) works without intermediate allocation; +/// only whole UTF-8 strings are ever appended, so spans are always valid +/// UTF-8. +#[derive(Default)] +pub struct Arena(pub Vec); + +impl Arena { + pub fn clear(&mut self) { + self.0.clear(); + } + + pub fn push_str(&mut self, s: &str) -> StrRef { + let off = self.0.len() as u32; + self.0.extend_from_slice(s.as_bytes()); + StrRef { off, len: s.len() as u32 } + } + + pub fn concat(&mut self, a: StrRef, b: StrRef) -> StrRef { + let off = self.0.len() as u32; + self.0.extend_from_within(a.off as usize..(a.off + a.len) as usize); + self.0.extend_from_within(b.off as usize..(b.off + b.len) as usize); + StrRef { off, len: a.len + b.len } + } + + pub fn get(&self, r: StrRef) -> &str { + // Spans only ever come from push_str/concat of whole &strs, so this + // never fails; checked conversion keeps the module unsafe-free. + std::str::from_utf8(&self.0[r.off as usize..(r.off + r.len) as usize]) + .expect("arena spans are always whole UTF-8 strings") + } +} + +/// An owned scalar, used by static structures and test expectations. +#[derive(Clone, Debug, PartialEq)] +pub enum ScalarVal { + I1(bool), + I64(i64), + F64(f64), + Str(String), +} + +impl ScalarVal { + pub fn ty(&self) -> Ty { + match self { + ScalarVal::I1(_) => Ty::I1, + ScalarVal::I64(_) => Ty::I64, + ScalarVal::F64(_) => Ty::F64, + ScalarVal::Str(_) => Ty::Str, + } + } +} + +/// One component of a map-static key. F64 keys compare and match by bit +/// pattern (total, deterministic; NaN == NaN by bits) — an internal +/// convention of the prepared structure, not a SQL semantics statement. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum KeyBits { + I1(bool), + I64(i64), + F64(u64), + Str(String), +} + +impl KeyBits { + pub fn ty(&self) -> Ty { + match self { + KeyBits::I1(_) => Ty::I1, + KeyBits::I64(_) => Ty::I64, + KeyBits::F64(_) => Ty::F64, + KeyBits::Str(_) => Ty::Str, + } + } +} + +/// Runtime payload for a static structure, supplied to `compile` alongside +/// the program and type-checked against its `StaticTy` declaration. +pub enum StaticData { + Scalar { valid: bool, val: ScalarVal }, + /// Entries are sorted + deduped at compile into a probe table; the + /// binary-search probe is allocation-free (design doc: how a map is + /// materialized is a backend decision — the oracle picks the simplest + /// correct structure, perfect hashing is the codegen backend's game). + Map(Vec<(Vec, Vec)>), +} + +/// Output column builder: `(valid, value)` pairs; strings are arena spans. +pub enum OutCol { + I1(Vec<(bool, bool)>), + I64(Vec<(bool, i64)>), + F64(Vec<(bool, f64)>), + Str(Vec<(bool, StrRef)>), +} + +impl OutCol { + fn clear(&mut self) { + match self { + OutCol::I1(v) => v.clear(), + OutCol::I64(v) => v.clear(), + OutCol::F64(v) => v.clear(), + OutCol::Str(v) => v.clear(), + } + } + + pub fn len(&self) -> usize { + match self { + OutCol::I1(v) => v.len(), + OutCol::I64(v) => v.len(), + OutCol::F64(v) => v.len(), + OutCol::Str(v) => v.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// A register: always a bare scalar, mirroring the IR's type system. +#[derive(Clone, Copy)] +pub enum RegVal { + I1(bool), + I64(i64), + F64(f64), + Str(StrRef), +} + +/// Reusable per-call buffers: registers, arena, output builders. Create once +/// with [`interp::InterpFn::new_state`], reuse across calls — `run` clears +/// (capacity-preserving) and refills them; after the first warm call the +/// steady state performs zero heap allocations. +pub struct RunState { + pub regs: Vec, + pub arena: Arena, + pub out: Vec, +} diff --git a/src/specializer/exec/tests.rs b/src/specializer/exec/tests.rs new file mode 100644 index 0000000..2d92348 --- /dev/null +++ b/src/specializer/exec/tests.rs @@ -0,0 +1,468 @@ +//! Interpreter-backend tests: every M-ir fixture executes against +//! hand-computed expectations; unverified IR and mismatched statics are +//! rejected; the steady state performs zero heap allocations (counting +//! global allocator); generated programs execute deterministically. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +use super::super::ir::{fixtures, gen, parse::parse, verify::verify, Program, StaticTy, Ty}; +use super::interp::{compile, CompileError, InterpFn}; +use super::{Batch, ColData, KeyBits, OutCol, RunState, ScalarVal, StaticData, Trap}; + +// ------------------------------------------------- counting allocator -- +// Thread-local counter so parallel tests don't disturb each other; const- +// initialized Cell so the TLS access itself never allocates. try_with +// swallows accesses during thread teardown. + +struct CountingAlloc; + +std::thread_local! { + static ALLOCS: Cell = const { Cell::new(0) }; +} + +fn bump() { + let _ = ALLOCS.try_with(|c| c.set(c.get() + 1)); +} + +fn alloc_count() -> u64 { + ALLOCS.with(|c| c.get()) +} + +unsafe impl GlobalAlloc for CountingAlloc { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + bump(); + unsafe { System.alloc(l) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + unsafe { System.dealloc(p, l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, n: usize) -> *mut u8 { + bump(); + unsafe { System.realloc(p, l, n) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + bump(); + unsafe { System.alloc_zeroed(l) } + } +} + +#[global_allocator] +static GA: CountingAlloc = CountingAlloc; + +// ------------------------------------------------------------- helpers -- + +fn c_i1(vals: &[Option]) -> ColData { + ColData::I1 { + valid: vals.iter().map(|v| v.is_some()).collect(), + data: vals.iter().map(|v| v.unwrap_or(false)).collect(), + } +} + +fn c_i64(vals: &[Option]) -> ColData { + ColData::I64 { + valid: vals.iter().map(|v| v.is_some()).collect(), + data: vals.iter().map(|v| v.unwrap_or(0)).collect(), + } +} + +fn c_f64(vals: &[Option]) -> ColData { + ColData::F64 { + valid: vals.iter().map(|v| v.is_some()).collect(), + data: vals.iter().map(|v| v.unwrap_or(0.0)).collect(), + } +} + +fn c_str(vals: &[Option<&str>]) -> ColData { + ColData::Str { + valid: vals.iter().map(|v| v.is_some()).collect(), + data: vals.iter().map(|v| v.unwrap_or("").to_string()).collect(), + } +} + +fn batch(rows: usize, cols: Vec) -> Batch { + Batch { rows, cols } +} + +fn built(text: &str) -> Program { + let p = parse(text).expect("fixture parses"); + verify(&p).expect("fixture verifies"); + p +} + +/// Snapshot the output as strings, masking NULL payloads (a NULL's payload +/// is meaningless downstream by contract). Allocates — test side only. +fn snapshot(st: &RunState) -> Vec> { + let ncols = st.out.len(); + let nrows = st.out.first().map(|c| c.len()).unwrap_or(0); + (0..nrows) + .map(|r| { + (0..ncols) + .map(|c| match &st.out[c] { + OutCol::I1(v) => render(v[r].0, format!("{}", v[r].1)), + OutCol::I64(v) => render(v[r].0, format!("{}", v[r].1)), + OutCol::F64(v) => render(v[r].0, format!("{:?}", v[r].1)), + OutCol::Str(v) => render(v[r].0, st.arena.get(v[r].1).to_string()), + }) + .collect() + }) + .collect() +} + +fn render(valid: bool, s: String) -> String { + if valid { + s + } else { + "NULL".to_string() + } +} + +fn run_snapshot(f: &InterpFn, input: &Batch) -> Result>, Trap> { + let mut st = f.new_state(); + f.run(input, &mut st)?; + Ok(snapshot(&st)) +} + +fn rows(v: &[&[&str]]) -> Vec> { + v.iter() + .map(|r| r.iter().map(|s| s.to_string()).collect()) + .collect() +} + +// ------------------------------------------------------------ fixtures -- + +#[test] +fn projection_fixture_executes() { + let p = built(fixtures::PROJECTION); + let statics = vec![StaticData::Map(vec![( + vec![KeyBits::Str("a".into())], + vec![ScalarVal::F64(10.0)], + )])]; + let f = compile(&p, statics).unwrap(); + let input = batch( + 3, + vec![ + c_i64(&[Some(40), None, Some(40)]), + c_str(&[Some("a"), Some("a"), Some("x")]), + ], + ); + // Row 1: 40/10 = 4. Row 2: age NULL -> NULL. Row 3: probe miss -> NULL. + assert_eq!( + run_snapshot(&f, &input).unwrap(), + rows(&[&["4.0"], &["NULL"], &["NULL"]]) + ); +} + +#[test] +fn filter_fixture_drops_rows() { + let p = built(fixtures::FILTER); + let f = compile(&p, vec![]).unwrap(); + let input = batch(3, vec![c_f64(&[Some(1.5), Some(-2.0), Some(0.0)])]); + assert_eq!(run_snapshot(&f, &input).unwrap(), rows(&[&["1.5"]])); +} + +#[test] +fn case_diamond_fixture_executes() { + let p = built(fixtures::CASE_DIAMOND); + let f = compile(&p, vec![]).unwrap(); + let input = batch(3, vec![c_i64(&[Some(31), Some(30), Some(5)])]); + assert_eq!( + run_snapshot(&f, &input).unwrap(), + rows(&[ + &["old", "false"], + &["old", "false"], + &["young", "false"], + ]) + ); +} + +#[test] +fn casts_fixture_executes_and_traps() { + let p = built(fixtures::CASTS); + let statics = |snd: StaticData| { + vec![ + StaticData::Scalar { valid: true, val: ScalarVal::F64(2.5) }, + snd, + ] + }; + // @1 NULL: n = round(2.5) - trunc(2.5) = 3 - 2 = 1; msg = select over + // scmp.eq("2.5:", ":") = false -> ":". + let f = compile( + &p, + statics(StaticData::Scalar { valid: false, val: ScalarVal::I64(0) }), + ) + .unwrap(); + let input = batch(1, vec![c_str(&[Some("12")])]); + assert_eq!(run_snapshot(&f, &input).unwrap(), rows(&[&["1", ":"]])); + + // @1 = 7: the select picks the static instead. + let f7 = compile( + &p, + statics(StaticData::Scalar { valid: true, val: ScalarVal::I64(7) }), + ) + .unwrap(); + assert_eq!(run_snapshot(&f7, &input).unwrap(), rows(&[&["7", ":"]])); + + // Unparseable input routes to the trap block and aborts the call. + let bad = batch(1, vec![c_str(&[Some("xx")])]); + assert_eq!( + run_snapshot(&f, &bad).unwrap_err(), + Trap("cast to i64 failed".into()) + ); +} + +#[test] +fn kitchen_fixture_executes() { + let p = built(fixtures::KITCHEN); + let statics = vec![StaticData::Map(vec![( + vec![KeyBits::I64(2), KeyBits::Str("k1".into())], + vec![ScalarVal::F64(0.5), ScalarVal::I64(9)], + )])]; + let f = compile(&p, statics).unwrap(); + let input = batch( + 2, + vec![ + c_i64(&[Some(4), Some(10)]), + c_i64(&[Some(2), None]), + c_str(&[Some("k1"), Some("nope")]), + c_f64(&[Some(1.5), Some(-1.0)]), + ], + ); + // Row 1: probe hit (2,"k1"); r = select(2.25<1.5, ..., x)=1.5 valid; + // s = ftos(stof(itos(9))) = "9.0". + // Row 2: b NULL -> r NULL; probe miss -> defaults -> s = "0.0". + assert_eq!( + run_snapshot(&f, &input).unwrap(), + rows(&[&["1.5", "9.0"], &["NULL", "0.0"]]) + ); +} + +// ------------------------------------------------------------ rejects -- + +#[test] +fn rejects_unverified_program() { + // Parses fine, fails verification (iadd on f64). + let p = parse( + r#"fn f(in: batch{a: f64}, out: batch{o: f64}) { +entry: + %x = load in.a + %y = iadd %x, %x + %z = itof %y + store out.o, %z + emit +}"#, + ) + .unwrap(); + assert!(verify(&p).is_err(), "precondition: program must be unverifiable"); + match compile(&p, vec![]) { + Err(CompileError::Verify(errs)) => assert!(!errs.is_empty()), + Err(CompileError::Static(m)) => panic!("wrong error kind: {m}"), + Ok(_) => panic!("compile accepted an unverified program"), + } +} + +#[test] +fn rejects_mismatched_statics() { + let p = built(fixtures::PROJECTION); // declares one map(str) -> (f64) + for (data, needle) in [ + (vec![], "declares 1 static(s), 0 provided"), + ( + vec![StaticData::Scalar { valid: true, val: ScalarVal::F64(1.0) }], + "declared map, got scalar", + ), + ( + vec![StaticData::Map(vec![( + vec![KeyBits::I64(1)], + vec![ScalarVal::F64(1.0)], + )])], + "entry 0 has shape", + ), + ( + vec![StaticData::Map(vec![ + (vec![KeyBits::Str("a".into())], vec![ScalarVal::F64(1.0)]), + (vec![KeyBits::Str("a".into())], vec![ScalarVal::F64(2.0)]), + ])], + "duplicate map key", + ), + ] { + match compile(&p, data) { + Err(CompileError::Static(msg)) => { + assert!(msg.contains(needle), "expected '{needle}' in '{msg}'") + } + Err(CompileError::Verify(_)) => panic!("fixture failed verify?"), + Ok(_) => panic!("compile accepted bad statics (wanted '{needle}')"), + } + } +} + +#[test] +fn rejects_input_shape_mismatch() { + let p = built(fixtures::FILTER); // one f64 column + let f = compile(&p, vec![]).unwrap(); + let mut st = f.new_state(); + let wrong_ty = batch(1, vec![c_i64(&[Some(1)])]); + assert!(f.run(&wrong_ty, &mut st).unwrap_err().0.contains("is i64")); + let wrong_count = batch(1, vec![]); + assert!(f.run(&wrong_count, &mut st).unwrap_err().0.contains("0 column(s)")); + let wrong_len = Batch { rows: 2, cols: vec![c_f64(&[Some(1.0)])] }; + assert!(f.run(&wrong_len, &mut st).unwrap_err().0.contains("1 row(s)")); +} + +// ---------------------------------------------------------- allocation -- + +/// AC#3: after warmup, run() performs ZERO heap allocations — registers, +/// arena, and output builders are all reused; varlen goes arena-only. +/// PROJECTION covers str loads, a probe, arithmetic, and store.opt. +#[test] +fn steady_state_run_allocates_nothing() { + let p = built(fixtures::PROJECTION); + let statics = vec![StaticData::Map(vec![( + vec![KeyBits::Str("a".into())], + vec![ScalarVal::F64(10.0)], + )])]; + let f = compile(&p, statics).unwrap(); + let input = batch( + 3, + vec![ + c_i64(&[Some(40), None, Some(40)]), + c_str(&[Some("a"), Some("a"), Some("x")]), + ], + ); + let mut st = f.new_state(); + // Warm: buffers reach steady capacity. + f.run(&input, &mut st).unwrap(); + f.run(&input, &mut st).unwrap(); + + let before = alloc_count(); + for _ in 0..5 { + f.run(&input, &mut st).unwrap(); + } + let delta = alloc_count() - before; + assert_eq!(delta, 0, "steady-state run heap-allocated {delta} time(s)"); +} + +// ---------------------------------------------------------------- fuzz -- + +fn gen_scalar(rng: &mut gen::Rng, ty: Ty) -> ScalarVal { + match ty { + Ty::I1 => ScalarVal::I1(rng.chance(50)), + Ty::I64 => ScalarVal::I64(rng.next() as i64 % 1000), + Ty::F64 => ScalarVal::F64((rng.next() as i64 % 1000) as f64 / 4.0), + Ty::Str => ScalarVal::Str(format!("s{}", rng.below(5))), + } +} + +/// Static data matching a program's declarations; entry keys made distinct +/// through the first component's index. +fn gen_statics(rng: &mut gen::Rng, p: &Program) -> Vec { + p.statics + .iter() + .map(|st| match st { + StaticTy::Scalar(ct) => StaticData::Scalar { + valid: !ct.nullable || rng.chance(70), + val: gen_scalar(rng, ct.ty), + }, + StaticTy::Map { keys, values } => { + let n = if keys[0] == Ty::I1 { 2 } else { 1 + rng.below(3) as usize }; + let entries = (0..n) + .map(|j| { + let key: Vec = keys + .iter() + .enumerate() + .map(|(pos, kt)| match (pos, kt) { + (0, Ty::I1) => KeyBits::I1(j % 2 == 1), + (0, Ty::I64) => KeyBits::I64(j as i64), + (0, Ty::F64) => KeyBits::F64((j as f64).to_bits()), + (0, Ty::Str) => KeyBits::Str(format!("k{j}")), + (_, Ty::I1) => KeyBits::I1(true), + (_, Ty::I64) => KeyBits::I64(7), + (_, Ty::F64) => KeyBits::F64(1.5f64.to_bits()), + (_, Ty::Str) => KeyBits::Str("fix".into()), + }) + .collect(); + let vals = values.iter().map(|vt| gen_scalar(rng, *vt)).collect(); + (key, vals) + }) + .collect(); + StaticData::Map(entries) + } + }) + .collect() +} + +fn gen_input(rng: &mut gen::Rng, p: &Program) -> Batch { + let rows = rng.below(5) as usize; + let cols = p + .in_cols + .iter() + .map(|c| { + let mk_valid = |rng: &mut gen::Rng| !c.ty.nullable || rng.chance(70); + match c.ty.ty { + Ty::I1 => c_i1( + &(0..rows) + .map(|_| mk_valid(rng).then(|| rng.chance(50))) + .collect::>(), + ), + Ty::I64 => c_i64( + &(0..rows) + .map(|_| mk_valid(rng).then(|| rng.next() as i64 % 100_000)) + .collect::>(), + ), + Ty::F64 => c_f64( + &(0..rows) + .map(|_| { + mk_valid(rng).then(|| match rng.below(5) { + 0 => f64::NAN, + 1 => f64::INFINITY, + _ => (rng.next() as i64 % 1000) as f64 / 8.0, + }) + }) + .collect::>(), + ), + Ty::Str => { + let opts: Vec> = (0..rows) + .map(|_| { + mk_valid(rng).then(|| match rng.below(4) { + 0 => String::new(), + 1 => "k1".to_string(), + 2 => "unicode é".to_string(), + _ => format!("v{}", rng.below(9)), + }) + }) + .collect(); + let refs: Vec> = opts.iter().map(|o| o.as_deref()).collect(); + c_str(&refs) + } + } + }) + .collect(); + Batch { rows, cols } +} + +/// Generated programs compile and execute without panics; when they run to +/// completion the output is rectangular, |out| <= |in|, and a second run is +/// byte-identical (determinism). Traps are a legal outcome (i64::MAX consts +/// meeting iadd, ftoi on inf, ...). +#[test] +fn fuzz_generated_programs_execute_deterministically() { + for seed in 0..150u64 { + let p = gen::gen_program(seed); + let mut rng = gen::Rng::new(seed ^ 0x9E37_79B9_7F4A_7C15); + let statics = gen_statics(&mut rng, &p); + let f = match compile(&p, statics) { + Ok(f) => f, + Err(e) => panic!("seed {seed}: generated program failed to compile: {e}"), + }; + let input = gen_input(&mut rng, &p); + let first = run_snapshot(&f, &input); + let second = run_snapshot(&f, &input); + match (&first, &second) { + (Ok(a), Ok(b)) => { + assert_eq!(a, b, "seed {seed}: nondeterministic output"); + assert!(a.len() <= input.rows, "seed {seed}: |out| > |in|"); + } + (Err(a), Err(b)) => assert_eq!(a, b, "seed {seed}: nondeterministic trap"), + _ => panic!("seed {seed}: run flipped between Ok and Err"), + } + } +} diff --git a/src/specializer/mod.rs b/src/specializer/mod.rs index c4c38c9..8c3c8d7 100644 --- a/src/specializer/mod.rs +++ b/src/specializer/mod.rs @@ -8,4 +8,5 @@ //! lowering (M-lower), the Cranelift backend (M-cranelift), and the generated //! Python-boundary marshaller (M-boundary). +pub mod exec; pub mod ir; From b713c486923914ab31207027ccde7c4f21df9ba5 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 21:18:20 +0200 Subject: [PATCH 10/11] =?UTF-8?q?fix(specializer):=20harden=20the=20interp?= =?UTF-8?q?reter=20against=20adversarial=20findings=20=E2=80=94=20TASK-42?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six agents (4 attack lenses + 2 reviews) probed the oracle; confirmed findings fixed and pinned: - StrRef offsets widened u32 -> usize: a single call legitimately exceeding 4 GiB of varlen data silently aliased old spans (Ok with corrupt output — the one thing an oracle must never do) - store.opt with a false flag now stores the type-default payload the spec pins, not the live register - check_input validates the validity lane of nullable columns (a short valid vec silently meant "all valid") - register slots assigned densely at compile: sparse-but-legal value ids can no longer inflate the frame (Value(u32::MAX) demanded a 64 GiB vec) - zero-allocation claim restated honestly: warmth is per content profile (branch paths, probe hits, non-NULL varlen), growth one-time + monotone — refuted-as-written by branch/probe/validity flips at identical shape - RunState.emitted reports the row count (observable with 0 out columns) - Ctx made private; out_decl drops its unread nullable flag; probe hit arm zips entry values directly - semantics pins now all tested: overflow/div traps incl. irem MIN%-1, fcmp NaN table, ftoi ties + range traps, IEEE inf/nan flow, scmp order, exact stoi parse, load.opt garbage normalization cargo 65 passed; gate green. Co-Authored-By: Claude Opus 5 --- src/specializer/exec/interp.rs | 156 +++++++++++++++---------- src/specializer/exec/mod.rs | 41 ++++--- src/specializer/exec/tests.rs | 201 +++++++++++++++++++++++++++++++++ 3 files changed, 326 insertions(+), 72 deletions(-) diff --git a/src/specializer/exec/interp.rs b/src/specializer/exec/interp.rs index b1a3adf..aabae64 100644 --- a/src/specializer/exec/interp.rs +++ b/src/specializer/exec/interp.rs @@ -22,6 +22,7 @@ //! * On a false validity flag the payload is the type default; `load.opt` //! normalizes even if the input batch carries garbage in invalid slots. +use std::collections::HashMap; use std::fmt::Write as _; use super::super::ir::{ @@ -60,7 +61,7 @@ impl std::fmt::Display for CompileError { type InstFn = Box Fn(&mut Ctx<'a>) -> Result<(), Trap>>; /// Everything a closure can touch during one row. -pub struct Ctx<'a> { +struct Ctx<'a> { regs: &'a mut [RegVal], arena: &'a mut Arena, out: &'a mut [OutCol], @@ -99,21 +100,38 @@ pub struct InterpFn { nregs: usize, statics: Vec, in_decl: Vec<(Ty, bool)>, - out_decl: Vec<(Ty, bool)>, + out_decl: Vec, } pub fn compile(p: &Program, statics: Vec) -> Result { verify(p).map_err(CompileError::Verify)?; let prepared = prepare_statics(p, statics)?; - let nregs = max_value_id(p) + 1; + // Register slots are assigned densely in definition order, decoupling + // the frame size from raw value ids — a verified program with sparse ids + // (they are legal) must not force a huge register vector (adversarial + // finding: Value(u32::MAX) would have demanded a 64 GiB frame). + let mut slots: HashMap = HashMap::new(); + for b in &p.blocks { + for (v, _) in &b.params { + let n = slots.len() as u32; + slots.entry(v.0).or_insert(n); + } + for inst in &b.insts { + for d in inst.dsts() { + let n = slots.len() as u32; + slots.entry(d.0).or_insert(n); + } + } + } + let nregs = slots.len(); let mut blocks = Vec::with_capacity(p.blocks.len()); for b in &p.blocks { let mut insts: Vec = Vec::with_capacity(b.insts.len()); for inst in &b.insts { - insts.push(compile_inst(p, inst)); + insts.push(compile_inst(p, inst, &slots)); } - blocks.push(CBlock { insts, term: compile_term(p, &b.term) }); + blocks.push(CBlock { insts, term: compile_term(p, &b.term, &slots) }); } Ok(InterpFn { @@ -121,7 +139,7 @@ pub fn compile(p: &Program, statics: Vec) -> Result RunState { RunState { regs: vec![RegVal::I64(0); self.nregs], + emitted: 0, arena: Arena::default(), out: self .out_decl .iter() - .map(|(ty, _)| match ty { + .map(|ty| match ty { Ty::I1 => OutCol::I1(Vec::new()), Ty::I64 => OutCol::I64(Vec::new()), Ty::F64 => OutCol::F64(Vec::new()), @@ -150,11 +169,13 @@ impl InterpFn { self.check_input(input)?; self.check_state(st)?; st.arena.clear(); + st.emitted = 0; for col in st.out.iter_mut() { col.clear(); } reserve_out(&mut st.out, input.rows); + let mut emitted = 0usize; for row in 0..input.rows { let mut ctx = Ctx { regs: &mut st.regs, @@ -183,11 +204,16 @@ impl InterpFn { bi = *else_to; } } - CTerm::Emit | CTerm::Skip => break, + CTerm::Emit => { + emitted += 1; + break; + } + CTerm::Skip => break, CTerm::Trap(msg) => return Err(Trap(msg.clone())), } } } + st.emitted = emitted; Ok(()) } @@ -209,7 +235,7 @@ impl InterpFn { self.out_decl.len() ))); } - for (ci, (col, (ty, _))) in st.out.iter().zip(self.out_decl.iter()).enumerate() { + for (ci, (col, ty)) in st.out.iter().zip(self.out_decl.iter()).enumerate() { let col_ty = match col { OutCol::I1(_) => Ty::I1, OutCol::I64(_) => Ty::I64, @@ -250,6 +276,17 @@ impl InterpFn { input.rows ))); } + // For a nullable column the validity lane is part of the input + // shape — a short vector must not silently mean "valid". + let (_, nullable) = self.in_decl[ci]; + if nullable && valid_len(col) != input.rows { + return Err(Trap(format!( + "input column {ci} is nullable but its validity vector has {} \ + entries for {} row(s)", + valid_len(col), + input.rows + ))); + } } Ok(()) } @@ -272,21 +309,6 @@ fn reserve_out(out: &mut [OutCol], rows: usize) { } } -fn max_value_id(p: &Program) -> usize { - let mut max = 0usize; - for b in &p.blocks { - for (v, _) in &b.params { - max = max.max(v.0 as usize); - } - for inst in &b.insts { - for d in inst.dsts() { - max = max.max(d.0 as usize); - } - } - } - max -} - fn prepare_statics( p: &Program, statics: Vec, @@ -400,13 +422,13 @@ fn default_reg(ty: Ty) -> RegVal { // ---------------------------------------------------------- compilation -- -fn compile_term(p: &Program, t: &Term) -> CTerm { +fn compile_term(p: &Program, t: &Term, slots: &HashMap) -> CTerm { let mk_moves = |to: ir::BlockId, args: &Vec| -> (usize, Vec<(u32, u32)>) { let params = &p.blocks[to.0 as usize].params; let moves = args .iter() .zip(params.iter()) - .map(|(src, (dst, _))| (src.0, dst.0)) + .map(|(src, (dst, _))| (sl(slots, *src) as u32, sl(slots, *dst) as u32)) .collect(); (to.0 as usize, moves) }; @@ -418,7 +440,7 @@ fn compile_term(p: &Program, t: &Term) -> CTerm { Term::Brif { cond, then_to, then_args, else_to, else_args } => { let (then_to, then_moves) = mk_moves(*then_to, then_args); let (else_to, else_moves) = mk_moves(*else_to, else_args); - CTerm::Brif { cond: cond.0, then_to, then_moves, else_to, else_moves } + CTerm::Brif { cond: sl(slots, *cond) as u32, then_to, then_moves, else_to, else_moves } } Term::Emit => CTerm::Emit, Term::Skip => CTerm::Skip, @@ -438,15 +460,20 @@ impl std::fmt::Write for ArenaWriter<'_> { } fn fmt_into_arena(arena: &mut Arena, args: std::fmt::Arguments<'_>) -> StrRef { - let off = arena.0.len() as u32; + let off = arena.0.len(); let _ = ArenaWriter(&mut arena.0).write_fmt(args); - StrRef { off, len: arena.0.len() as u32 - off } + StrRef { off, len: arena.0.len() - off } +} + +/// Value id -> dense register slot. +fn sl(slots: &HashMap, v: Value) -> usize { + slots[&v.0] as usize } -fn compile_inst(p: &Program, inst: &Inst) -> InstFn { +fn compile_inst(p: &Program, inst: &Inst, slots: &HashMap) -> InstFn { match inst.clone() { Inst::Const { dst, lit } => { - let dst = dst.0 as usize; + let dst = sl(slots, dst); match lit { ir::Lit::I1(b) => Box::new(move |ctx| { ctx.regs[dst] = RegVal::I1(b); @@ -467,7 +494,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { } } Inst::Bin { op, dst, a, b } => { - let (dst, a, b) = (dst.0 as usize, a.0 as usize, b.0 as usize); + let (dst, a, b) = (sl(slots, dst), sl(slots, a), sl(slots, b)); match op { BinOp::Iadd | BinOp::Isub | BinOp::Imul => Box::new(move |ctx| { let (x, y) = (as_i64(ctx.regs[a]), as_i64(ctx.regs[b])); @@ -525,7 +552,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { } } Inst::Cmp { pred, ty, dst, a, b } => { - let (dst, a, b) = (dst.0 as usize, a.0 as usize, b.0 as usize); + let (dst, a, b) = (sl(slots, dst), sl(slots, a), sl(slots, b)); Box::new(move |ctx| { let v = match ty { Ty::I64 => apply_ord(pred, as_i64(ctx.regs[a]).cmp(&as_i64(ctx.regs[b]))), @@ -553,28 +580,28 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Not { dst, a } => { - let (dst, a) = (dst.0 as usize, a.0 as usize); + let (dst, a) = (sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { ctx.regs[dst] = RegVal::I1(!as_i1(ctx.regs[a])); Ok(()) }) } Inst::Select { dst, cond, a, b } => { - let (dst, cond, a, b) = (dst.0 as usize, cond.0 as usize, a.0 as usize, b.0 as usize); + let (dst, cond, a, b) = (sl(slots, dst), sl(slots, cond), sl(slots, a), sl(slots, b)); Box::new(move |ctx| { ctx.regs[dst] = if as_i1(ctx.regs[cond]) { ctx.regs[a] } else { ctx.regs[b] }; Ok(()) }) } Inst::Itof { dst, a } => { - let (dst, a) = (dst.0 as usize, a.0 as usize); + let (dst, a) = (sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { ctx.regs[dst] = RegVal::F64(as_i64(ctx.regs[a]) as f64); Ok(()) }) } Inst::Ftoi { mode, dst, a } => { - let (dst, a) = (dst.0 as usize, a.0 as usize); + let (dst, a) = (sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { let x = as_f64(ctx.regs[a]); let r = match mode { @@ -592,7 +619,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Itos { dst, a } => { - let (dst, a) = (dst.0 as usize, a.0 as usize); + let (dst, a) = (sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { let v = as_i64(ctx.regs[a]); ctx.regs[dst] = RegVal::Str(fmt_into_arena(ctx.arena, format_args!("{v}"))); @@ -600,7 +627,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Ftos { dst, a } => { - let (dst, a) = (dst.0 as usize, a.0 as usize); + let (dst, a) = (sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { let v = as_f64(ctx.regs[a]); ctx.regs[dst] = RegVal::Str(fmt_into_arena(ctx.arena, format_args!("{v:?}"))); @@ -608,7 +635,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::StoiOpt { flag, dst, a } => { - let (flag, dst, a) = (flag.0 as usize, dst.0 as usize, a.0 as usize); + let (flag, dst, a) = (sl(slots, flag), sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { let s = ctx.arena.get(as_str(ctx.regs[a])); match s.parse::() { @@ -625,7 +652,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::StofOpt { flag, dst, a } => { - let (flag, dst, a) = (flag.0 as usize, dst.0 as usize, a.0 as usize); + let (flag, dst, a) = (sl(slots, flag), sl(slots, dst), sl(slots, a)); Box::new(move |ctx| { let s = ctx.arena.get(as_str(ctx.regs[a])); match s.parse::() { @@ -642,7 +669,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Sconcat { dst, a, b } => { - let (dst, a, b) = (dst.0 as usize, a.0 as usize, b.0 as usize); + let (dst, a, b) = (sl(slots, dst), sl(slots, a), sl(slots, b)); Box::new(move |ctx| { let (x, y) = (as_str(ctx.regs[a]), as_str(ctx.regs[b])); ctx.regs[dst] = RegVal::Str(ctx.arena.concat(x, y)); @@ -650,14 +677,14 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Load { dst, col } => { - let (dst, col) = (dst.0 as usize, col as usize); + let (dst, col) = (sl(slots, dst), col as usize); Box::new(move |ctx| { ctx.regs[dst] = load_payload(&ctx.input.cols[col], ctx.row, ctx.arena); Ok(()) }) } Inst::LoadOpt { flag, dst, col } => { - let (flag, dst, col) = (flag.0 as usize, dst.0 as usize, col as usize); + let (flag, dst, col) = (sl(slots, flag), sl(slots, dst), col as usize); let ty = p.in_cols[col].ty.ty; Box::new(move |ctx| { let c = &ctx.input.cols[col]; @@ -672,25 +699,29 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Store { col, val } => { - let (col, val) = (col as usize, val.0 as usize); + let (col, val) = (col as usize, sl(slots, val)); Box::new(move |ctx| { push_out(&mut ctx.out[col], true, ctx.regs[val]); Ok(()) }) } Inst::StoreOpt { col, flag, val } => { - let (col, flag, val) = (col as usize, flag.0 as usize, val.0 as usize); + let (col, flag, val) = (col as usize, sl(slots, flag), sl(slots, val)); + // Spec: on a false flag the stored payload is the type default — + // never the live register (adversarial finding). + let default = default_reg(p.out_cols[col].ty.ty); Box::new(move |ctx| { let valid = as_i1(ctx.regs[flag]); - push_out(&mut ctx.out[col], valid, ctx.regs[val]); + let v = if valid { ctx.regs[val] } else { default }; + push_out(&mut ctx.out[col], valid, v); Ok(()) }) } Inst::Probe { static_id, hit, dsts, keys } => { let static_id = static_id as usize; - let hit = hit.0 as usize; - let dsts: Vec = dsts.iter().map(|d| d.0 as usize).collect(); - let keys: Vec = keys.iter().map(|k| k.0 as usize).collect(); + let hit = sl(slots, hit); + let dsts: Vec = dsts.iter().map(|d| sl(slots, *d)).collect(); + let keys: Vec = keys.iter().map(|k| sl(slots, *k)).collect(); let value_tys: Vec = match &p.statics[static_id] { StaticTy::Map { values, .. } => values.clone(), _ => unreachable!("probe on non-map is rejected by the verifier"), @@ -705,12 +736,9 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { match found { Some(idx) => { ctx.regs[hit] = RegVal::I1(true); - // Copy values out BEFORE writing regs that might - // alias arena growth: scalar_to_reg appends to the - // arena for str values, which is fine — spans stay - // valid, the arena only grows. - for (di, vi) in dsts.iter().zip(0..value_tys.len()) { - let v = &entries[idx].1[vi]; + // scalar_to_reg may append to the arena for str + // values — fine, spans stay valid, it only grows. + for (di, v) in dsts.iter().zip(entries[idx].1.iter()) { ctx.regs[*di] = scalar_to_reg(v, ctx.arena); } } @@ -725,7 +753,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::Sload { static_id, dst } => { - let (static_id, dst) = (static_id as usize, dst.0 as usize); + let (static_id, dst) = (static_id as usize, sl(slots, dst)); Box::new(move |ctx| { let PreparedStatic::Scalar { val, .. } = &ctx.statics[static_id] else { unreachable!("static kind checked at compile"); @@ -735,7 +763,7 @@ fn compile_inst(p: &Program, inst: &Inst) -> InstFn { }) } Inst::SloadOpt { static_id, flag, dst } => { - let (static_id, flag, dst) = (static_id as usize, flag.0 as usize, dst.0 as usize); + let (static_id, flag, dst) = (static_id as usize, sl(slots, flag), sl(slots, dst)); let ty = match &p.statics[static_id] { StaticTy::Scalar(ct) => ct.ty, _ => unreachable!("sload on non-scalar is rejected by the verifier"), @@ -788,6 +816,18 @@ fn apply_ord(pred: CmpPred, ord: std::cmp::Ordering) -> bool { } } +fn valid_len(c: &ColData) -> usize { + match c { + ColData::I1 { valid, .. } + | ColData::I64 { valid, .. } + | ColData::F64 { valid, .. } + | ColData::Str { valid, .. } => valid.len(), + } +} + +/// Only called for nullable columns, whose validity length check_input has +/// already enforced — the unwrap_or is unreachable there and don't-care for +/// non-nullable columns (which never reach this). fn col_valid(c: &ColData, row: usize) -> bool { match c { ColData::I1 { valid, .. } diff --git a/src/specializer/exec/mod.rs b/src/specializer/exec/mod.rs index 9b05291..0c4bb66 100644 --- a/src/specializer/exec/mod.rs +++ b/src/specializer/exec/mod.rs @@ -1,9 +1,10 @@ //! Execution substrate shared by the interpreter backend (and, at //! M-cranelift, the codegen backend): batches, the bump arena, prepared //! static structures, and the run-state buffers. Everything here obeys the -//! Stage-2 discipline — steady-state execution allocates nothing on the -//! heap; the only growable memory is the arena and the pre-reserved output -//! builders, both owned by [`RunState`] and reused across calls. +//! Stage-2 discipline: the only growable memory is the arena and the +//! pre-reserved output builders, both owned by [`RunState`] and reused +//! across calls — see [`RunState`]'s docs for the precise zero-allocation +//! contract. //! //! Lives under `exec/` rather than a separate `runtime/` until the Cranelift //! backend actually shares it — one home per concept until two consumers @@ -70,11 +71,13 @@ impl ColData { /// A span into the run arena. All `str`-typed register values are arena /// spans — input/const/static strings are copied in on read, so registers -/// stay `Copy` and nothing borrows across rows. +/// stay `Copy` and nothing borrows across rows. Offsets are usize: a single +/// call can legitimately accumulate more than 4 GiB of varlen data, and a +/// u32 span silently aliased old data past the wrap (adversarial finding). #[derive(Clone, Copy, Debug)] pub struct StrRef { - pub off: u32, - pub len: u32, + pub off: usize, + pub len: usize, } /// Bump arena for varlen values, reset per call. Byte-backed so @@ -90,22 +93,22 @@ impl Arena { } pub fn push_str(&mut self, s: &str) -> StrRef { - let off = self.0.len() as u32; + let off = self.0.len(); self.0.extend_from_slice(s.as_bytes()); - StrRef { off, len: s.len() as u32 } + StrRef { off, len: s.len() } } pub fn concat(&mut self, a: StrRef, b: StrRef) -> StrRef { - let off = self.0.len() as u32; - self.0.extend_from_within(a.off as usize..(a.off + a.len) as usize); - self.0.extend_from_within(b.off as usize..(b.off + b.len) as usize); + let off = self.0.len(); + self.0.extend_from_within(a.off..a.off + a.len); + self.0.extend_from_within(b.off..b.off + b.len); StrRef { off, len: a.len + b.len } } pub fn get(&self, r: StrRef) -> &str { // Spans only ever come from push_str/concat of whole &strs, so this // never fails; checked conversion keeps the module unsafe-free. - std::str::from_utf8(&self.0[r.off as usize..(r.off + r.len) as usize]) + std::str::from_utf8(&self.0[r.off..r.off + r.len]) .expect("arena spans are always whole UTF-8 strings") } } @@ -206,10 +209,20 @@ pub enum RegVal { /// Reusable per-call buffers: registers, arena, output builders. Create once /// with [`interp::InterpFn::new_state`], reuse across calls — `run` clears -/// (capacity-preserving) and refills them; after the first warm call the -/// steady state performs zero heap allocations. +/// (capacity-preserving) and refills them. +/// +/// Zero-allocation contract, stated precisely (the naive "after one warm +/// call" claim was refuted by adversarial testing): a run allocates only +/// when the input's *arena footprint* — branch paths taken, probe hits, +/// non-NULL varlen values, output row count — exceeds every previous run's +/// high-water mark. Such growth is one-time and monotone: repeating any +/// content profile already seen allocates nothing. Traps may allocate +/// (error path). pub struct RunState { pub regs: Vec, pub arena: Arena, pub out: Vec, + /// Rows emitted by the last `run` — the row count even when the + /// program has zero output columns. + pub emitted: usize, } diff --git a/src/specializer/exec/tests.rs b/src/specializer/exec/tests.rs index 2d92348..378f3fc 100644 --- a/src/specializer/exec/tests.rs +++ b/src/specializer/exec/tests.rs @@ -341,6 +341,207 @@ fn steady_state_run_allocates_nothing() { assert_eq!(delta, 0, "steady-state run heap-allocated {delta} time(s)"); } +// ----------------------------------------- adversarial-pass regressions -- +// Each pins a fix for a confirmed finding from the 2026-07-26 adversarial +// workflow against the interpreter. + +/// store.opt with a false flag stores the TYPE DEFAULT payload, never the +/// live register (spec pin in ir/mod.rs). +#[test] +fn store_opt_false_flag_stores_type_default() { + let p = built( + r#"fn f(in: batch{a: i64}, out: batch{o: i64?, s: str?}) { +entry: + %x = load in.a + %f = const.i1 false + store.opt out.o, %f, %x + %t = itos %x + store.opt out.s, %f, %t + emit +}"#, + ); + let f = compile(&p, vec![]).unwrap(); + let mut st = f.new_state(); + f.run(&batch(1, vec![c_i64(&[Some(42)])]), &mut st).unwrap(); + match &st.out[0] { + OutCol::I64(v) => assert_eq!(v[0], (false, 0), "payload must be the default, not 42"), + _ => unreachable!(), + } + match &st.out[1] { + OutCol::Str(v) => { + assert!(!v[0].0); + assert_eq!(st.arena.get(v[0].1), "", "payload must be the empty default"); + } + _ => unreachable!(), + } +} + +/// A nullable column's validity vector is part of the input shape. +#[test] +fn short_validity_vector_traps() { + let p = built( + r#"fn f(in: batch{a: i64?}, out: batch{o: i64}) { +entry: + %f, %v = load.opt in.a + %z = select %f, %v, %v + store out.o, %z + emit +}"#, + ); + let f = compile(&p, vec![]).unwrap(); + let mut st = f.new_state(); + let bad = Batch { + rows: 1, + cols: vec![ColData::I64 { valid: vec![], data: vec![7] }], + }; + let err = f.run(&bad, &mut st).unwrap_err(); + assert!(err.0.contains("validity vector"), "wrong trap: {}", err.0); +} + +/// Register slots are dense: sparse value ids must not inflate the frame. +#[test] +fn sparse_value_ids_use_dense_register_slots() { + use super::super::ir::{Block, Col, ColTy, Inst, Lit, Program, Term, Ty, Value}; + let p = Program { + statics: vec![], + name: "sparse".into(), + in_cols: vec![], + out_cols: vec![Col { name: "o".into(), ty: ColTy { ty: Ty::I64, nullable: false } }], + blocks: vec![Block { + params: vec![], + insts: vec![ + Inst::Const { dst: Value(9_999_999), lit: Lit::I64(5) }, + Inst::Store { col: 0, val: Value(9_999_999) }, + ], + term: Term::Emit, + }], + }; + let f = compile(&p, vec![]).unwrap(); + let st = f.new_state(); + assert!(st.regs.len() <= 4, "frame sized by ids, not defs: {}", st.regs.len()); +} + +/// The emitted counter reports the row count even without reading columns. +#[test] +fn emitted_counts_rows() { + let p = built(fixtures::FILTER); + let f = compile(&p, vec![]).unwrap(); + let mut st = f.new_state(); + f.run(&batch(3, vec![c_f64(&[Some(1.5), Some(-2.0), Some(0.0)])]), &mut st).unwrap(); + assert_eq!(st.emitted, 1); +} + +// ------------------------------------------------------- semantics pins -- +// One test per documented pin in interp.rs that the design review found +// untested. Each tiny program computes one edge through the text form. + +/// Run a one-output program over no input rows... rather, over one dummy row. +fn eval1(body: &str, out_col: &str) -> Result>, Trap> { + let text = format!( + "fn f(in: batch{{d: i64}}, out: batch{{{out_col}}}) {{\nentry:\n{body}\n emit\n}}" + ); + let p = built(&text); + let f = compile(&p, vec![]).unwrap(); + run_snapshot(&f, &batch(1, vec![c_i64(&[Some(0)])])) +} + +#[test] +fn pin_integer_overflow_and_division_traps() { + for (expr, needle) in [ + (" %a = const.i64 9223372036854775807\n %b = const.i64 1\n %r = iadd %a, %b", "overflow in iadd"), + (" %a = const.i64 -9223372036854775808\n %b = const.i64 1\n %r = isub %a, %b", "overflow in isub"), + (" %a = const.i64 4611686018427387904\n %b = const.i64 4\n %r = imul %a, %b", "overflow in imul"), + (" %a = const.i64 1\n %b = const.i64 0\n %r = idiv %a, %b", "division by zero in idiv"), + (" %a = const.i64 1\n %b = const.i64 0\n %r = irem %a, %b", "division by zero in irem"), + (" %a = const.i64 -9223372036854775808\n %b = const.i64 -1\n %r = idiv %a, %b", "overflow in idiv"), + (" %a = const.i64 -9223372036854775808\n %b = const.i64 -1\n %r = irem %a, %b", "overflow in irem"), + ] { + let body = format!("{expr}\n store out.o, %r"); + let err = eval1(&body, "o: i64").unwrap_err(); + assert!(err.0.contains(needle), "expected '{needle}', got '{}'", err.0); + } +} + +#[test] +fn pin_fcmp_nan_ordering() { + // Every predicate involving NaN is false except ne. + let body = " %n = const.f64 nan\n %x = const.f64 1.0\n\ + \x20 %eq = fcmp.eq %n, %x\n %ne = fcmp.ne %n, %x\n\ + \x20 %lt = fcmp.lt %n, %x\n %le = fcmp.le %n, %x\n\ + \x20 %gt = fcmp.gt %n, %x\n %ge = fcmp.ge %n, %x\n\ + \x20 store out.eq, %eq\n store out.ne, %ne\n store out.lt, %lt\n\ + \x20 store out.le, %le\n store out.gt, %gt\n store out.ge, %ge"; + let got = eval1(body, "eq: i1, ne: i1, lt: i1, le: i1, gt: i1, ge: i1").unwrap(); + assert_eq!(got, rows(&[&["false", "true", "false", "false", "false", "false"]])); +} + +#[test] +fn pin_ftoi_rounding_and_traps() { + for (lit, mode, expect) in [ + ("2.5", "round", "3"), + ("-2.5", "round", "-3"), + ("0.5", "round", "1"), + ("-0.5", "round", "-1"), + ("2.5", "trunc", "2"), + ("-2.5", "trunc", "-2"), + ] { + let body = format!(" %a = const.f64 {lit}\n %r = ftoi.{mode} %a\n store out.o, %r"); + assert_eq!( + eval1(&body, "o: i64").unwrap(), + rows(&[&[expect]]), + "ftoi.{mode}({lit})" + ); + } + for lit in ["nan", "inf", "-inf", "1e19"] { + let body = format!(" %a = const.f64 {lit}\n %r = ftoi.trunc %a\n store out.o, %r"); + let err = eval1(&body, "o: i64").unwrap_err(); + assert!(err.0.contains("out of i64 range"), "ftoi({lit}): {}", err.0); + } +} + +#[test] +fn pin_ieee_flow_and_scmp_and_concat() { + let body = " %z = const.f64 0.0\n %o = const.f64 1.0\n %m = const.f64 -1.0\n\ + \x20 %nan = fdiv %z, %z\n %pinf = fdiv %o, %z\n %ninf = fdiv %m, %z\n\ + \x20 store out.a, %nan\n store out.b, %pinf\n store out.c, %ninf\n\ + \x20 %s1 = const.str \"Z\"\n %s2 = const.str \"a\"\n %lt = scmp.lt %s1, %s2\n\ + \x20 store out.d, %lt\n\ + \x20 %e1 = const.str \"\"\n %cc = sconcat %e1, %e1\n store out.e, %cc"; + let got = eval1(body, "a: f64, b: f64, c: f64, d: i1, e: str").unwrap(); + assert_eq!(got, rows(&[&["NaN", "inf", "-inf", "true", ""]])); +} + +#[test] +fn pin_stoi_exact_parse_no_trim() { + for (s, ok) in [(" 5", false), ("5 ", false), ("+5", true), ("0x10", false), ("", false)] { + let body = format!( + " %s = const.str \"{s}\"\n %f, %v = stoi.opt %s\n store out.f, %f\n store out.v, %v" + ); + let got = eval1(&body, "f: i1, v: i64").unwrap(); + assert_eq!(got[0][0], ok.to_string(), "stoi({s:?})"); + } +} + +#[test] +fn pin_load_opt_normalizes_garbage_payloads() { + // Invalid slot carries 999; the flag is false so downstream must see the + // type default, not the garbage. + let p = built( + r#"fn f(in: batch{a: i64?}, out: batch{o: i64}) { +entry: + %f, %v = load.opt in.a + store out.o, %v + emit +}"#, + ); + let f = compile(&p, vec![]).unwrap(); + let input = Batch { + rows: 1, + cols: vec![ColData::I64 { valid: vec![false], data: vec![999] }], + }; + assert_eq!(run_snapshot(&f, &input).unwrap(), rows(&[&["0"]])); +} + // ---------------------------------------------------------------- fuzz -- fn gen_scalar(rng: &mut gen::Rng, ty: Ty) -> ScalarVal { From 590b820a72be8ab30645a199d74418537aa294ff Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Sat, 25 Jul 2026 21:18:41 +0200 Subject: [PATCH 11/11] chore(backlog): TASK-42 acceptance criteria checked + implementation notes Co-Authored-By: Claude Opus 5 --- ...compiled-IR-interpreter-the-oracle-backend.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md b/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md index d57e41a..94e699c 100644 --- a/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md +++ b/backlog/tasks/task-42 - Specializer-M-interp-closure-compiled-IR-interpreter-the-oracle-backend.md @@ -4,7 +4,7 @@ title: 'Specializer M-interp: closure-compiled IR interpreter (the oracle backen status: In Progress assignee: [] created_date: '2026-07-25 02:31' -updated_date: '2026-07-25 18:42' +updated_date: '2026-07-25 19:18' labels: [] milestone: m-7 dependencies: @@ -23,10 +23,10 @@ Implement the interpreter backend over the imperative IR (design doc §7): one p ## Acceptance Criteria -- [ ] #1 Every IR instruction executes; the M-ir fixture programs all produce expected outputs -- [ ] #2 Runs only verifier-accepted IR; rejects unverified programs -- [ ] #3 No allocation during execution (arena-only for varlen), asserted by a test -- [ ] #4 mise gate-specializer green +- [x] #1 Every IR instruction executes; the M-ir fixture programs all produce expected outputs +- [x] #2 Runs only verifier-accepted IR; rejects unverified programs +- [x] #3 No allocation during execution (arena-only for varlen), asserted by a test +- [x] #4 mise gate-specializer green ## Implementation Plan @@ -38,3 +38,9 @@ Implement the interpreter backend over the imperative IR (design doc §7): one p 4. Tests: all 5 M-ir fixtures executed against hand-computed inputs/statics/outputs; unverified-program rejection; counting global allocator asserts zero heap allocs on a warmed second run (arena/regs/builders reused); executor fuzz over gen::gen_program seeds with random inputs (no panics, |out| <= |in|, deterministic). 5. Adversarial workflow (semantics vs spec, alloc discipline, trap/edge paths, fuzz), fix confirmed findings; gate green; stacked PR onto claude/duckdb-native-interpreter-36d016. + +## Implementation Notes + + +Implemented in src/specializer/exec/ (mod.rs substrate + interp.rs + tests.rs). compile() verifies first; closures built in one pre-traversal; terminators interpreted as a small enum in the row loop; register slots densely remapped at compile. Semantics pins documented and every one tested. Adversarial pass (6 agents) found and I fixed: u32 arena-span wrap silently corrupting output past 4 GiB (offsets now usize), store.opt false-flag storing the live register instead of the pinned type default, unvalidated validity-lane length, sparse value ids inflating the register frame, and an overbroad zero-allocation claim (restated: warmth is per content profile, growth one-time+monotone). RunState.emitted added. cargo 65 tests; alloc-free steady state asserted via thread-local counting global allocator; 150-seed executor fuzz deterministic. +