Skip to content

Commit a02a467

Browse files
carli2carli2
authored
Top-down scalar compiler refactor (#247)
* Refactor scalar subquery compile phases * Fix scalar marker leak in CASE null checks * Raise advertised max_allowed_packet * Add scalar wrapper regression coverage
1 parent c0d26ca commit a02a467

3 files changed

Lines changed: 362 additions & 50 deletions

File tree

‎lib/queryplan.scm‎

Lines changed: 156 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ builds, because their truth value depends on current session state. */
195195
(define latest_def (_latest_schema_for_alias tblvar ignorecase))
196196
(if (nil? latest_def) '() (_expand_alias_cols tblvar latest_def)))
197197
(list col expr)
198-
)))))))
198+
)))))))
199199
/* materialized_source_schema: resolve schema for a materialized temp source
200200
(keytable, prejoin) using planner-internal metadata only. No storage access --
201201
keytables/prejoins may not exist at compile time (runtime-only creation). */
@@ -1560,7 +1560,7 @@ helper filters can close over them at runtime. */
15601560
'((symbol get_column) alias_ ti col _) (if ((if ti equal?? equal?) alias_ tblvar) (list (if pick_col col dir)) '())
15611561
'((quote get_column) alias_ ti col _) (if ((if ti equal?? equal?) alias_ tblvar) (list (if pick_col col dir)) '())
15621562
_ '()
1563-
)))))))
1563+
)))))))
15641564

15651565
(define extract_scan_order_cols_for_tblvar (lambda (order_items tblvar)
15661566
(_extract_scan_order_terms_for_tblvar order_items tblvar true)
@@ -2342,7 +2342,9 @@ carrier until session domains are modeled explicitly. */
23422342
(define union_parts (query_union_all_parts rewritten_query))
23432343
(if (nil? union_parts)
23442344
(if (query_is_select_core rewritten_query)
2345-
(make_select_core_term (apply untangle_query (merge rewritten_query (list outer_schemas))))
2345+
(begin
2346+
(define uq_result (apply untangle_query (merge rewritten_query (list outer_schemas))))
2347+
(make_select_core_term uq_result))
23462348
(error "invalid SELECT query term"))
23472349
(match union_parts '(branches order limit offset) (begin
23482350
(if (or (nil? branches) (equal? branches '()))
@@ -2951,6 +2953,7 @@ seeing the correctly prefixed outer alias. */
29512953
(define outer_schemas_chain (coalesceNil outer_schemas_param '()))
29522954
(define sq_cache (newsession))
29532955
(sq_cache "init" '())
2956+
(define dep_scalar_cache (newsession))
29542957

29552958
/* COUNT(DISTINCT) rewrite helpers - do not descend into inner_select nodes (subqueries are processed separately) */
29562959
(define _cd_is_subquery (lambda (sym) (match sym
@@ -3196,55 +3199,96 @@ seeing the correctly prefixed outer alias. */
31963199
(expr_uses_session_state subquery)
31973200
(_raw_query_contains_skip_level_nested_outer_ref subquery (_raw_query_local_aliases subquery))))
31983201
nil)))
3202+
(define scalar_subselect_inline_raw_flags (lambda (subquery) (match subquery
3203+
'(_ _ _ _ g h o l off)
3204+
(list
3205+
g h o l off
3206+
(expr_uses_session_state subquery)
3207+
(_raw_query_contains_skip_level_nested_outer_ref subquery (_raw_query_local_aliases subquery)))
3208+
nil)))
3209+
(define scalar_subselect_lowering_facts (lambda (subquery outer_schemas) (match subquery
3210+
'(_ _ flds _ g h o _ _) (begin
3211+
(define value_expr (match flds
3212+
(cons _ (cons v _)) v
3213+
nil))
3214+
(define has_outer (_subquery_has_outer_refs subquery outer_schemas))
3215+
(list
3216+
g h o
3217+
value_expr
3218+
has_outer
3219+
(if has_outer
3220+
(_subquery_outer_refs_are_direct_columns subquery outer_schemas)
3221+
true)
3222+
(_contains_inner_select_marker subquery)
3223+
(not (equal? (if (nil? value_expr) '() (extract_aggregates value_expr)) '()))))
3224+
nil)))
31993225
(define scalar_subselect_inline_reason planner_scalar_subselect_inline_reason)
32003226
(define scalar_subselect_inline_strategy planner_scalar_subselect_inline_strategy)
32013227
(define scalar_subselect_lowering_reason_from_facts planner_scalar_subselect_lowering_reason_from_facts)
3228+
(define untangle_scalar_subquery_scope (lambda (subquery outer_schemas raw_group raw_having raw_order raw_limit raw_offset) (begin
3229+
/* Shared logical scope preparation for scalar subqueries.
3230+
This keeps recursive untangle + default stage synthesis in one place so the
3231+
future top-down dependent-join pass can replace exactly this boundary. */
3232+
(match (apply untangle_query (merge subquery (list outer_schemas)))
3233+
'(schema2 tables2 fields2 condition2 groups2 schemas2 replace_find_column2 _init2)
3234+
(begin
3235+
(define groups2 (coalesceNil groups2 '()))
3236+
(define groups2 (if (or (nil? groups2) (equal? groups2 '()))
3237+
(if (or raw_group raw_having raw_order raw_limit raw_offset)
3238+
(list (make_group_stage raw_group raw_having raw_order raw_limit raw_offset nil nil))
3239+
groups2)
3240+
groups2))
3241+
(list schema2 tables2 fields2 condition2 groups2 schemas2 replace_find_column2 _init2))
3242+
nil))))
3243+
(define prepare_scalar_subselect_inline_scope (lambda (subquery outer_schemas raw_group raw_having raw_order raw_limit raw_offset) (begin
3244+
/* This is the logical scope-normalization boundary for scalar inline lowering.
3245+
It must stay free of runtime scan/promise construction so a future top-down
3246+
dependent-join pass can hook in here without re-walking the old fallback code. */
3247+
(match (untangle_scalar_subquery_scope subquery outer_schemas raw_group raw_having raw_order raw_limit raw_offset)
3248+
'(schema2 tables2 fields2 condition2 groups2 schemas2 replace_find_column2 _init2)
3249+
(begin
3250+
(define replace_find_column_subselect (make_replace_find_column_subselect schemas2 outer_schemas false))
3251+
(define field_exprs (extract_assoc fields2 (lambda (k v) v)))
3252+
(define value_expr (match field_exprs
3253+
(cons only '()) only
3254+
_ (error "scalar subselect must return single column")
3255+
))
3256+
(set fields2 (map_assoc fields2 (lambda (k v) (replace_find_column_subselect v))))
3257+
(set condition2 (replace_find_column_subselect (coalesceNil condition2 true)))
3258+
/* wrap remaining unresolved qualified get_column refs as (outer tbl.col).
3259+
These are outer-outer refs that weren't in _s or _o — wrapping them
3260+
preserves them through replace_columns_from_expr and allows
3261+
replace_column_alias to prefix them during derived-table flattening. */
3262+
(define wrap_unresolved_outer (lambda (e) (match e
3263+
'((symbol get_column) alias_ ti col ci) (if (and (not (nil? alias_)) (or ti ci)
3264+
/* only wrap as (outer) if the alias is actually in outer_schemas;
3265+
if not in outer_schemas either, leave as-is for scan-context resolution
3266+
(e.g. joinexpr refs to sibling tables like v.ID) */
3267+
(not (nil? (reduce_assoc outer_schemas (lambda (a k v) (or a (equal?? k alias_))) false))))
3268+
(list (quote outer) (symbol (concat alias_ "." col)))
3269+
e)
3270+
(cons sym args) (cons (wrap_unresolved_outer sym) (map args wrap_unresolved_outer))
3271+
e
3272+
)))
3273+
(set fields2 (map_assoc fields2 (lambda (k v) (wrap_unresolved_outer v))))
3274+
(set condition2 (wrap_unresolved_outer condition2))
3275+
(list schema2 tables2 fields2 condition2 groups2 schemas2 replace_find_column_subselect _init2 value_expr))
3276+
nil))))
32023277
(define build_scalar_subselect_inline_with_strategy (lambda (subquery outer_schemas) (begin
32033278
(define union_parts (query_union_all_parts subquery))
32043279
(if (not (nil? union_parts))
32053280
(begin
32063281
(planner_debug_record_scalar_event (quote inline-strategy) (quote inline-union-all-not-supported))
32073282
(error "scalar subselect UNION ALL is not supported yet"))
32083283
(begin
3209-
(match (scalar_subselect_shape_facts subquery outer_schemas)
3210-
'(raw_group raw_having raw_order raw_limit raw_offset _raw_value_expr _raw_has_outer _raw_outer_refs_are_direct_columns _raw_contains_inner_select_marker _raw_has_aggregate scalar_uses_session_state raw_contains_skip_level_nested_outer_ref)
3284+
(match (scalar_subselect_inline_raw_flags subquery)
3285+
'(raw_group raw_having raw_order raw_limit raw_offset scalar_uses_session_state raw_contains_skip_level_nested_outer_ref)
32113286
(begin
3212-
/* pass full outer schema chain so nested subqueries inside this scalar
3213-
subselect can still resolve grandparent references (skip-level correlation) */
3214-
(match (apply untangle_query (merge subquery (list outer_schemas)))
3215-
'(schema2 tables2 fields2 condition2 groups2 schemas2 replace_find_column2 _init2)
3287+
(match (prepare_scalar_subselect_inline_scope
3288+
subquery outer_schemas
3289+
raw_group raw_having raw_order raw_limit raw_offset)
3290+
'(schema2 tables2 fields2 condition2 groups2 schemas2 replace_find_column_subselect _init2 value_expr)
32163291
(begin
3217-
(define groups2 (coalesceNil groups2 '()))
3218-
(define groups2 (if (or (nil? groups2) (equal? groups2 '()))
3219-
(if (or raw_group raw_having raw_order raw_limit raw_offset)
3220-
(list (make_group_stage raw_group raw_having raw_order raw_limit raw_offset nil nil))
3221-
groups2)
3222-
groups2))
3223-
(define replace_find_column_subselect (make_replace_find_column_subselect schemas2 outer_schemas false))
3224-
(define field_exprs (extract_assoc fields2 (lambda (k v) v)))
3225-
(define value_expr (match field_exprs
3226-
(cons only '()) only
3227-
_ (error "scalar subselect must return single column")
3228-
))
3229-
(set fields2 (map_assoc fields2 (lambda (k v) (replace_find_column_subselect v))))
3230-
(set condition2 (replace_find_column_subselect (coalesceNil condition2 true)))
3231-
/* wrap remaining unresolved qualified get_column refs as (outer tbl.col).
3232-
These are outer-outer refs that weren't in _s or _o — wrapping them
3233-
preserves them through replace_columns_from_expr and allows
3234-
replace_column_alias to prefix them during derived-table flattening. */
3235-
(define wrap_unresolved_outer (lambda (e) (match e
3236-
'((symbol get_column) alias_ ti col ci) (if (and (not (nil? alias_)) (or ti ci)
3237-
/* only wrap as (outer) if the alias is actually in outer_schemas;
3238-
if not in outer_schemas either, leave as-is for scan-context resolution
3239-
(e.g. joinexpr refs to sibling tables like v.ID) */
3240-
(not (nil? (reduce_assoc outer_schemas (lambda (a k v) (or a (equal?? k alias_))) false))))
3241-
(list (quote outer) (symbol (concat alias_ "." col)))
3242-
e)
3243-
(cons sym args) (cons (wrap_unresolved_outer sym) (map args wrap_unresolved_outer))
3244-
e
3245-
)))
3246-
(set fields2 (map_assoc fields2 (lambda (k v) (wrap_unresolved_outer v))))
3247-
(set condition2 (wrap_unresolved_outer condition2))
32483292
/* Software contract: scalar aggregates are split by canonical
32493293
correlation, not by raw parser shape.
32503294
- uncorrelated aggregates go through the helper-table/keytable path
@@ -3493,8 +3537,12 @@ seeing the correctly prefixed outer alias. */
34933537
(define raw_order_us (nth raw_vals_us 2))
34943538
(define raw_limit_us (nth raw_vals_us 3))
34953539
(define raw_offset_us (nth raw_vals_us 4))
3496-
/* pass outer_schemas chain to recursive untangle so grandparent refs resolve */
3497-
(match (apply untangle_query (merge subquery (list outer_schemas)))
3540+
/* pass outer_schemas chain to recursive untangle so grandparent refs resolve.
3541+
Use the shared logical scope preparation so inline and unnest paths stay on
3542+
the same recursive normalization boundary. */
3543+
(match (untangle_scalar_subquery_scope
3544+
subquery outer_schemas
3545+
raw_group_us raw_having_us raw_order_us raw_limit_us raw_offset_us)
34983546
'(schema2_us tables2_us fields2_us condition2_us groups2_us schemas2_us rfcol2_us _init2_us) (begin
34993547
(if (and (not (nil? _init2_us)) (not (equal? _init2_us '())))
35003548
(sq_cache "init" (merge (coalesceNil (sq_cache "init") '()) _init2_us)))
@@ -3834,6 +3882,13 @@ seeing the correctly prefixed outer alias. */
38343882
)
38353883
)
38363884
)))
3885+
(define dependent_scalar_compile_marker (lambda (idx)
3886+
(list (quote dependent_scalar_compile) idx)))
3887+
(define dependent_scalar_compile_marker_id (lambda (expr) (match expr
3888+
'((quote dependent_scalar_compile) idx) idx
3889+
'((symbol dependent_scalar_compile) idx) idx
3890+
'(dependent_scalar_compile idx) idx
3891+
_ nil)))
38373892
(define not_symbol (lambda (sym) (match sym
38383893
(symbol not) true
38393894
'not true
@@ -4002,8 +4057,8 @@ seeing the correctly prefixed outer alias. */
40024057
false))
40034058
false)))
40044059
(define scalar_subselect_lowering_reason (lambda (subquery outer_schemas)
4005-
(match (scalar_subselect_shape_facts subquery outer_schemas)
4006-
'(_g h _o _l _off _value_expr _has_outer _outer_refs_are_direct_columns _contains_inner_select_marker _has_aggregate _uses_session_state _contains_skip_level_nested_outer_ref) (begin
4060+
(match (scalar_subselect_lowering_facts subquery outer_schemas)
4061+
'(_g h _o _value_expr _has_outer _outer_refs_are_direct_columns _contains_inner_select_marker _has_aggregate) (begin
40074062
/* ORDER/LIMIT-only correlated scalars already lower through the
40084063
normal non-aggregate partition-topk path, but only for the direct
40094064
single-level shape. Nested inner-select markers still stay on the
@@ -4311,7 +4366,56 @@ seeing the correctly prefixed outer alias. */
43114366
not_expr))
43124367
expr
43134368
)))
4314-
4369+
/* Compile-only scalar markers keep large expr trees free of eager scalar
4370+
lowering while the current scope is still being normalized. The marker never
4371+
escapes the compiler; it is resolved back into the normal logical plan before
4372+
_sq_* helper integration. */
4373+
(define nil_test_of_inner_select (lambda (expr) (match expr
4374+
(cons nil_sym (cons inner_expr '()))
4375+
(and
4376+
(or
4377+
(equal?? nil_sym (symbol nil?))
4378+
(equal?? nil_sym (quote nil?))
4379+
(equal?? nil_sym (quote (quote nil?))))
4380+
(match inner_expr
4381+
(cons inner_sym (cons _ '()))
4382+
(equal?? (inner_select_kind inner_sym) (quote inner_select))
4383+
_ false))
4384+
_ false)))
4385+
(define collect_dependent_scalar_compile_markers (lambda (expr outer_schemas)
4386+
(if (nil_test_of_inner_select expr)
4387+
(replace_inner_selects expr outer_schemas)
4388+
(match expr
4389+
(cons sym args) (begin
4390+
(define kind (inner_select_kind sym))
4391+
(if (equal?? kind (quote inner_select))
4392+
(match args
4393+
(cons subquery '()) (begin
4394+
(define dep_id (coalesceNil (dep_scalar_cache "idx") 1))
4395+
(dep_scalar_cache "idx" (+ dep_id 1))
4396+
(dep_scalar_cache dep_id subquery)
4397+
(dependent_scalar_compile_marker dep_id))
4398+
_ (replace_inner_selects expr outer_schemas))
4399+
(if (nil? kind)
4400+
(cons sym (map args (lambda (arg) (collect_dependent_scalar_compile_markers arg outer_schemas))))
4401+
(replace_inner_selects expr outer_schemas))))
4402+
_ expr))))
4403+
(define resolve_dependent_scalar_compile_markers (lambda (expr outer_schemas)
4404+
(match expr
4405+
(cons sym args) (begin
4406+
(define dep_id (dependent_scalar_compile_marker_id expr))
4407+
(if (nil? dep_id)
4408+
(if (_is_opaque_scope_sym sym)
4409+
expr
4410+
(cons sym (map args (lambda (arg)
4411+
(resolve_dependent_scalar_compile_markers arg outer_schemas)))))
4412+
(begin
4413+
(define subquery (dep_scalar_cache dep_id))
4414+
(coalesce
4415+
(build_scalar_subselect subquery outer_schemas)
4416+
(replace_inner_selects (list (quote inner_select) subquery) outer_schemas)
4417+
expr))))
4418+
_ expr)))
43154419
/* no-FROM rewrite: inject virtual one-row table ".(1)" (like Oracle DUAL).
43164420
Dot prefix hides from SHOW TABLES. Eliminates the no-table special case.
43174421
set tables= must wrap the if (set is scope-local in this Scheme dialect). */
@@ -4838,10 +4942,10 @@ seeing the correctly prefixed outer alias. */
48384942
(define _ris_schemas (merge schemas outer_schemas_chain))
48394943
(set tables (map tables (lambda (td) (match td
48404944
'(tv tschema ttbl toisOuter tje)
4841-
(list tv tschema ttbl toisOuter
4945+
(list tv tschema ttbl toisOuter
48424946
(if (nil? tje) nil (replace_inner_selects tje _ris_schemas)))
48434947
td))))
4844-
(set fields (map_assoc fields (lambda (k v) (replace_inner_selects v _ris_schemas))))
4948+
(set fields (map_assoc fields (lambda (k v) (collect_dependent_scalar_compile_markers v _ris_schemas))))
48454949
(set condition (replace_inner_selects condition _ris_schemas))
48464950
(set group (map group (lambda (g) (replace_inner_selects g _ris_schemas))))
48474951
(set having (begin
@@ -4869,6 +4973,8 @@ seeing the correctly prefixed outer alias. */
48694973
(cons sym (map args freeze_visible_field_refs)))
48704974
expr)))
48714975
(set fields (map_assoc fields (lambda (k v) (freeze_visible_field_refs v))))
4976+
(set fields (map_assoc fields (lambda (k v)
4977+
(resolve_dependent_scalar_compile_markers v _ris_schemas))))
48724978
/* integrate unnested scalar subselects from Neumann unnesting.
48734979
Tables from non-aggregate path (direct LEFT JOIN) do NOT need schema updates.
48744980
Tables from aggregate path (materialized derived) DO need schemas for build_queryplan. */
@@ -5088,7 +5194,9 @@ seeing the correctly prefixed outer alias. */
50885194

50895195
(define planner_visible_schemas (merge schemas outer_schemas_chain))
50905196
(define finalize_visible_expr (lambda (expr)
5091-
(finalize_logical_expr_scoped expr schemas planner_visible_schemas replace_rename enforce_planner_contract)))
5197+
(finalize_logical_expr_scoped
5198+
(resolve_dependent_scalar_compile_markers expr planner_visible_schemas)
5199+
schemas planner_visible_schemas replace_rename enforce_planner_contract)))
50925200
(define finalize_visible_table_ref (lambda (tbl)
50935201
(if (scan_tagged_table_needs_scan_order tbl)
50945202
(scan_tagged_table_with_outer_sources

‎scm/mysql.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -354,14 +354,15 @@ func (m *MySQLWrapper) ComQuery(session *driver.Session, query string, bindVaria
354354
}
355355
}()
356356
// max_allowed_packet: PHP PDO queries this to size buffers.
357-
// Return 32MB (33554432) so large result sets work.
357+
// Return 40MB so large result sets work without tripping client-side
358+
// packet buffer limits on wide login/dashboard views.
358359
if query == "select @@max_allowed_packet" || query == "SELECT @@max_allowed_packet" {
359360
callback(&sqltypes.Result{
360361
Fields: []*querypb.Field{
361362
{Name: "@@max_allowed_packet", Type: querypb.Type_INT64},
362363
},
363364
Rows: [][]sqltypes.Value{
364-
{sqltypes.MakeTrusted(querypb.Type_INT64, []byte("4194304"))},
365+
{sqltypes.MakeTrusted(querypb.Type_INT64, []byte("41943040"))},
365366
},
366367
})
367368
return nil

0 commit comments

Comments
 (0)