Merge pull request #146510 from DrewKimball/backport25.2-146005

DrewKimball · web-flow · commit c0150765fc2e · 2025-05-12T12:57:21.000-06:00
release-25.2: opt: add rule to pull filters out of EXISTS condition
diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go
@@ -4140,6 +4140,10 @@ func (m *sessionDataMutator) SetPropagateAdmissionHeaderToLeafTransactions(val b
 	m.data.PropagateAdmissionHeaderToLeafTransactions = val
 }
 
+func (m *sessionDataMutator) SetOptimizerUseExistsFilterHoistRule(val bool) {
+	m.data.OptimizerUseExistsFilterHoistRule = val
+}
+
 // Utility functions related to scrubbing sensitive information on SQL Stats.
 
 // quantizeCounts ensures that the Count field in the
diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema
@@ -4026,6 +4026,7 @@ optimizer_push_limit_into_project_filtered_scan            on
 optimizer_push_offset_into_index_join                      on
 optimizer_use_conditional_hoist_fix                        on
 optimizer_use_delete_range_fast_path                       on
+optimizer_use_exists_filter_hoist_rule                     off
 optimizer_use_forecasts                                    on
 optimizer_use_histograms                                   on
 optimizer_use_improved_computed_column_filters_derivation  on
diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog
@@ -3035,6 +3035,7 @@ optimizer_push_limit_into_project_filtered_scan            on                  N
 optimizer_push_offset_into_index_join                      on                  NULL      NULL        NULL        string
 optimizer_use_conditional_hoist_fix                        on                  NULL      NULL        NULL        string
 optimizer_use_delete_range_fast_path                       on                  NULL      NULL        NULL        string
+optimizer_use_exists_filter_hoist_rule                     off                 NULL      NULL        NULL        string
 optimizer_use_forecasts                                    on                  NULL      NULL        NULL        string
 optimizer_use_histograms                                   on                  NULL      NULL        NULL        string
 optimizer_use_improved_computed_column_filters_derivation  on                  NULL      NULL        NULL        string
@@ -3255,6 +3256,7 @@ optimizer_push_limit_into_project_filtered_scan            on                  N
 optimizer_push_offset_into_index_join                      on                  NULL  user     NULL      on                  on
 optimizer_use_conditional_hoist_fix                        on                  NULL  user     NULL      on                  on
 optimizer_use_delete_range_fast_path                       on                  NULL  user     NULL      on                  on
+optimizer_use_exists_filter_hoist_rule                     off                 NULL  user     NULL      off                 off
 optimizer_use_forecasts                                    on                  NULL  user     NULL      on                  on
 optimizer_use_histograms                                   on                  NULL  user     NULL      on                  on
 optimizer_use_improved_computed_column_filters_derivation  on                  NULL  user     NULL      on                  on
@@ -3474,6 +3476,7 @@ optimizer_push_limit_into_project_filtered_scan            NULL    NULL     NULL
 optimizer_push_offset_into_index_join                      NULL    NULL     NULL     NULL        NULL
 optimizer_use_conditional_hoist_fix                        NULL    NULL     NULL     NULL        NULL
 optimizer_use_delete_range_fast_path                       NULL    NULL     NULL     NULL        NULL
+optimizer_use_exists_filter_hoist_rule                     NULL    NULL     NULL     NULL        NULL
 optimizer_use_forecasts                                    NULL    NULL     NULL     NULL        NULL
 optimizer_use_histograms                                   NULL    NULL     NULL     NULL        NULL
 optimizer_use_improved_computed_column_filters_derivation  NULL    NULL     NULL     NULL        NULL
diff --git a/pkg/sql/logictest/testdata/logic_test/show_source b/pkg/sql/logictest/testdata/logic_test/show_source
@@ -151,6 +151,7 @@ optimizer_push_limit_into_project_filtered_scan            on
 optimizer_push_offset_into_index_join                      on
 optimizer_use_conditional_hoist_fix                        on
 optimizer_use_delete_range_fast_path                       on
+optimizer_use_exists_filter_hoist_rule                     off
 optimizer_use_forecasts                                    on
 optimizer_use_histograms                                   on
 optimizer_use_improved_computed_column_filters_derivation  on
diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go
@@ -206,6 +206,7 @@ type Memo struct {
 	useInsertFastPath                          bool
 	internal                                   bool
 	usePre_25_2VariadicBuiltins                bool
+	useExistsFilterHoistRule                   bool
 
 	// txnIsoLevel is the isolation level under which the plan was created. This
 	// affects the planning of some locking operations, so it must be included in
@@ -309,6 +310,7 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
 		useInsertFastPath:                          evalCtx.SessionData().InsertFastPath,
 		internal:                                   evalCtx.SessionData().Internal,
 		usePre_25_2VariadicBuiltins:                evalCtx.SessionData().UsePre_25_2VariadicBuiltins,
+		useExistsFilterHoistRule:                   evalCtx.SessionData().OptimizerUseExistsFilterHoistRule,
 		txnIsoLevel:                                evalCtx.TxnIsoLevel,
 	}
 	m.metadata.Init()
@@ -485,6 +487,7 @@ func (m *Memo) IsStale(
 		m.useInsertFastPath != evalCtx.SessionData().InsertFastPath ||
 		m.internal != evalCtx.SessionData().Internal ||
 		m.usePre_25_2VariadicBuiltins != evalCtx.SessionData().UsePre_25_2VariadicBuiltins ||
+		m.useExistsFilterHoistRule != evalCtx.SessionData().OptimizerUseExistsFilterHoistRule ||
 		m.txnIsoLevel != evalCtx.TxnIsoLevel {
 		return true, nil
 	}
diff --git a/pkg/sql/opt/memo/memo_test.go b/pkg/sql/opt/memo/memo_test.go
@@ -574,6 +574,11 @@ func TestMemoIsStale(t *testing.T) {
 	evalCtx.SessionData().UsePre_25_2VariadicBuiltins = false
 	notStale()
 
+	evalCtx.SessionData().OptimizerUseExistsFilterHoistRule = true
+	stale()
+	evalCtx.SessionData().OptimizerUseExistsFilterHoistRule = false
+	notStale()
+
 	// User no longer has access to view.
 	catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
 	_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)
diff --git a/pkg/sql/opt/memo/testdata/stats/project b/pkg/sql/opt/memo/testdata/stats/project
@@ -302,7 +302,7 @@ exec-ddl
 CREATE TABLE t124831 (a INT, b INT);
 ----
 
-norm disable=(SimplifyZeroCardinalityGroup,EliminateExistsZeroRows,SimplifyZeroCardinalitySemiJoin,PushFilterIntoJoinLeft)
+norm disable=(SimplifyZeroCardinalityGroup,EliminateExistsZeroRows,SimplifyZeroCardinalitySemiJoin,PushFilterIntoJoinLeft,HoistUnboundFilterFromExistsSubquery)
 SELECT a FROM t124831 WHERE NULL::INT IN (SELECT 1 LIMIT b);
 ----
 project
diff --git a/pkg/sql/opt/norm/decorrelate_funcs.go b/pkg/sql/opt/norm/decorrelate_funcs.go
@@ -1566,3 +1566,9 @@ func (c *CustomFuncs) MakeAnyNotNullScalarGroupBy(input memo.RelExpr) memo.RelEx
 		memo.EmptyGroupingPrivate,
 	)
 }
+
+// CanHoistUnboundFilterFromExistsSubquery returns true if the
+// HoistUnboundFilterFromExistsSubquery rule is enabled by session-setting.
+func (c *CustomFuncs) CanHoistUnboundFilterFromExistsSubquery() bool {
+	return c.f.evalCtx.SessionData().OptimizerUseExistsFilterHoistRule
+}
diff --git a/pkg/sql/opt/norm/general_funcs.go b/pkg/sql/opt/norm/general_funcs.go
@@ -769,6 +769,17 @@ func (c *CustomFuncs) RemoveFiltersItem(
 	return filters.RemoveFiltersItem(search)
 }
 
+// AppendFiltersItem returns a new list that is a copy of the given list, except
+// that the given item has been appended to the end of the list.
+func (c *CustomFuncs) AppendFiltersItem(
+	filters memo.FiltersExpr, toAppend opt.ScalarExpr,
+) memo.FiltersExpr {
+	newFilters := make(memo.FiltersExpr, len(filters)+1)
+	copy(newFilters, filters)
+	newFilters[len(filters)] = c.f.ConstructFiltersItem(toAppend)
+	return newFilters
+}
+
 // ReplaceFiltersItem returns a new list that is a copy of the given list,
 // except that the given search item has been replaced by the given replace
 // item. If the list contains the search item multiple times, then only the
diff --git a/pkg/sql/opt/norm/rules/decorrelate.opt b/pkg/sql/opt/norm/rules/decorrelate.opt
@@ -911,6 +911,105 @@
     (OutputCols2 $left $right)
 )
 
+# HoistUnboundFilterFromExistsSubquery pulls a filter condition out of an
+# Exists subquery if the filter condition only depends on columns from the
+# outer query. This is useful because it allows other optimization rules to
+# apply to the filter which was previously hidden inside the subquery.
+[HoistUnboundFilterFromExistsSubquery, Normalize]
+(Select
+    $input:* & (CanHoistUnboundFilterFromExistsSubquery)
+    $filters:[
+        ...
+        $item:(FiltersItem
+            (Exists
+                (Select
+                    $innerInput:*
+                    $innerFilters:[
+                        ...
+                        $innerItem:(FiltersItem $unboundCond:*) &
+                            (IsBoundBy
+                                $innerItem
+                                $inputCols:(OutputCols $input)
+                            )
+                        ...
+                    ]
+                )
+                $existsPrivate:*
+            )
+        )
+        ...
+    ]
+)
+=>
+(Select
+    $input
+    (AppendFiltersItem
+        (ReplaceFiltersItem
+            $filters
+            $item
+            (Exists
+                (Select
+                    $innerInput
+                    (RemoveFiltersItem $innerFilters $innerItem)
+                )
+                $existsPrivate
+            )
+        )
+        $unboundCond
+    )
+)
+
+# HoistUnboundJoinFilterFromExistsSubquery is similar to
+# HoistUnboundFilterFromExistsSubquery, but it applies to a join filter.
+[HoistUnboundJoinFilterFromExistsSubquery, Normalize]
+(Select
+    $input:* & (CanHoistUnboundFilterFromExistsSubquery)
+    $filters:[
+        ...
+        $item:(FiltersItem
+            (Exists
+                $join:(InnerJoin | InnerJoinApply | SemiJoin
+                        | SemiJoinApply
+                    $left:*
+                    $right:*
+                    $joinFilters:[
+                        ...
+                        $innerItem:(FiltersItem $unboundCond:*) &
+                            (IsBoundBy
+                                $innerItem
+                                $inputCols:(OutputCols $input)
+                            )
+                        ...
+                    ]
+                    $joinPrivate:*
+                )
+                $existsPrivate:*
+            )
+        )
+        ...
+    ]
+)
+=>
+(Select
+    $input
+    (AppendFiltersItem
+        (ReplaceFiltersItem
+            $filters
+            $item
+            (Exists
+                ((OpName $join)
+                    $left
+                    $right
+                    (RemoveFiltersItem $joinFilters $innerItem)
+                    $joinPrivate
+                )
+                $existsPrivate
+            )
+        )
+        $unboundCond
+    )
+)
+
 # HoistSelectExists extracts existential subqueries from Select filters,
 # turning them into semi-joins. This eliminates the subquery, which is often
 # expensive to execute and restricts the optimizer's plan choices.
diff --git a/pkg/sql/opt/norm/testdata/rules/decorrelate b/pkg/sql/opt/norm/testdata/rules/decorrelate
diff --git a/pkg/sql/opt/norm/testdata/rules/join b/pkg/sql/opt/norm/testdata/rules/join
diff --git a/pkg/sql/sessiondatapb/local_only_session_data.proto b/pkg/sql/sessiondatapb/local_only_session_data.proto
diff --git a/pkg/sql/vars.go b/pkg/sql/vars.go

Original file line number	Diff line number	Diff line change
`@@ -4140,6 +4140,10 @@ func (m *sessionDataMutator) SetPropagateAdmissionHeaderToLeafTransactions(val b`
`4140`	`4140`	`m.data.PropagateAdmissionHeaderToLeafTransactions = val`
`4141`	`4141`	`}`
`4142`	`4142`
	`4143`	`+func (m *sessionDataMutator) SetOptimizerUseExistsFilterHoistRule(val bool) {`
	`4144`	`+ m.data.OptimizerUseExistsFilterHoistRule = val`
	`4145`	`+}`
	`4146`	`+`
`4143`	`4147`	`// Utility functions related to scrubbing sensitive information on SQL Stats.`
`4144`	`4148`
`4145`	`4149`	`// quantizeCounts ensures that the Count field in the`
Original file line number	Diff line number	Diff line change
`@@ -1566,3 +1566,9 @@ func (c *CustomFuncs) MakeAnyNotNullScalarGroupBy(input memo.RelExpr) memo.RelEx`
`1566`	`1566`	`memo.EmptyGroupingPrivate,`
`1567`	`1567`	`)`
`1568`	`1568`	`}`
	`1569`	`+`
	`1570`	`+// CanHoistUnboundFilterFromExistsSubquery returns true if the`
	`1571`	`+// HoistUnboundFilterFromExistsSubquery rule is enabled by session-setting.`
	`1572`	`+func (c *CustomFuncs) CanHoistUnboundFilterFromExistsSubquery() bool {`
	`1573`	`+ return c.f.evalCtx.SessionData().OptimizerUseExistsFilterHoistRule`
	`1574`	`+}`