Skip to content

Commit 4c38b14

Browse files
DrewKimballDrewKimball
committed
opt: add rule to pull filters out of EXISTS join condition
This commit adds a new norm rule `HoistUnboundJoinFilterFromExistsSubquery` that pulls a filter from a join in an `EXISTS` subquery if the join filter only references columns from the outer query. This gives other optimizations a chance to apply before filter push-down rules move the EXISTS subquery and everything in it below joins and other operators in the outer query. Fixes #146000 Release note: None
1 parent 2a5413c commit 4c38b14

File tree

3 files changed

+343
-1
lines changed

3 files changed

+343
-1
lines changed

‎pkg/sql/opt/norm/rules/decorrelate.opt

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -959,6 +959,57 @@
959959
)
960960
)
961961

962+
# HoistUnboundJoinFilterFromExistsSubquery is similar to
963+
# HoistUnboundFilterFromExistsSubquery, but it applies to a join filter.
964+
[HoistUnboundJoinFilterFromExistsSubquery, Normalize]
965+
(Select
966+
$input:*
967+
$filters:[
968+
...
969+
$item:(FiltersItem
970+
(Exists
971+
$join:(InnerJoin | InnerJoinApply | SemiJoin
972+
| SemiJoinApply
973+
$left:*
974+
$right:*
975+
$joinFilters:[
976+
...
977+
$innerItem:(FiltersItem $unboundCond:*) &
978+
(IsBoundBy
979+
$innerItem
980+
$inputCols:(OutputCols $input)
981+
)
982+
...
983+
]
984+
$joinPrivate:*
985+
)
986+
$existsPrivate:*
987+
)
988+
)
989+
...
990+
]
991+
)
992+
=>
993+
(Select
994+
$input
995+
(AppendFiltersItem
996+
(ReplaceFiltersItem
997+
$filters
998+
$item
999+
(Exists
1000+
((OpName $join)
1001+
$left
1002+
$right
1003+
(RemoveFiltersItem $joinFilters $innerItem)
1004+
$joinPrivate
1005+
)
1006+
$existsPrivate
1007+
)
1008+
)
1009+
$unboundCond
1010+
)
1011+
)
1012+
9621013
# HoistSelectExists extracts existential subqueries from Select filters,
9631014
# turning them into semi-joins. This eliminates the subquery, which is often
9641015
# expensive to execute and restricts the optimizer's plan choices.

‎pkg/sql/opt/norm/testdata/rules/decorrelate

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8458,3 +8458,294 @@ project
84588458
│ └── key: (8)
84598459
└── filters
84608460
└── x:8 = k:1 [outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(1)==(8), (8)==(1)]
8461+
8462+
# --------------------------------------------------
8463+
# HoistUnboundJoinFilterFromExistsSubquery
8464+
# --------------------------------------------------
8465+
8466+
norm expect=HoistUnboundJoinFilterFromExistsSubquery
8467+
SELECT * FROM a WHERE EXISTS (
8468+
SELECT * FROM xy INNER JOIN uv ON x = u AND a.i = 100
8469+
);
8470+
----
8471+
select
8472+
├── columns: k:1!null i:2!null f:3 s:4 j:5
8473+
├── key: (1)
8474+
├── fd: ()-->(2), (1)-->(3-5)
8475+
├── scan a
8476+
│ ├── columns: k:1!null i:2 f:3 s:4 j:5
8477+
│ ├── key: (1)
8478+
│ └── fd: (1)-->(2-5)
8479+
└── filters
8480+
├── coalesce [subquery]
8481+
│ ├── subquery
8482+
│ │ └── project
8483+
│ │ ├── columns: column17:17!null
8484+
│ │ ├── cardinality: [0 - 1]
8485+
│ │ ├── key: ()
8486+
│ │ ├── fd: ()-->(17)
8487+
│ │ ├── limit
8488+
│ │ │ ├── columns: x:8!null u:12!null
8489+
│ │ │ ├── cardinality: [0 - 1]
8490+
│ │ │ ├── key: ()
8491+
│ │ │ ├── fd: ()-->(8,12), (8)==(12), (12)==(8)
8492+
│ │ │ ├── inner-join (hash)
8493+
│ │ │ │ ├── columns: x:8!null u:12!null
8494+
│ │ │ │ ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-one)
8495+
│ │ │ │ ├── key: (12)
8496+
│ │ │ │ ├── fd: (8)==(12), (12)==(8)
8497+
│ │ │ │ ├── limit hint: 1.00
8498+
│ │ │ │ ├── scan xy
8499+
│ │ │ │ │ ├── columns: x:8!null
8500+
│ │ │ │ │ └── key: (8)
8501+
│ │ │ │ ├── scan uv
8502+
│ │ │ │ │ ├── columns: u:12!null
8503+
│ │ │ │ │ └── key: (12)
8504+
│ │ │ │ └── filters
8505+
│ │ │ │ └── x:8 = u:12 [outer=(8,12), constraints=(/8: (/NULL - ]; /12: (/NULL - ]), fd=(8)==(12), (12)==(8)]
8506+
│ │ │ └── 1
8507+
│ │ └── projections
8508+
│ │ └── true [as=column17:17]
8509+
│ └── false
8510+
└── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
8511+
8512+
norm expect=HoistUnboundJoinFilterFromExistsSubquery
8513+
SELECT * FROM a WHERE EXISTS (
8514+
SELECT * FROM xy INNER JOIN LATERAL (
8515+
WITH foo(u, v) AS MATERIALIZED (SELECT * FROM uv WHERE y > 5) SELECT * FROM foo
8516+
) ON x = u AND a.i = 100
8517+
);
8518+
----
8519+
select
8520+
├── columns: k:1!null i:2!null f:3 s:4 j:5
8521+
├── key: (1)
8522+
├── fd: ()-->(2), (1)-->(3-5)
8523+
├── scan a
8524+
│ ├── columns: k:1!null i:2 f:3 s:4 j:5
8525+
│ ├── key: (1)
8526+
│ └── fd: (1)-->(2-5)
8527+
└── filters
8528+
├── coalesce [subquery]
8529+
│ ├── subquery
8530+
│ │ └── project
8531+
│ │ ├── columns: column19:19!null
8532+
│ │ ├── cardinality: [0 - 1]
8533+
│ │ ├── key: ()
8534+
│ │ ├── fd: ()-->(19)
8535+
│ │ ├── limit
8536+
│ │ │ ├── columns: x:8!null y:9 u:16!null
8537+
│ │ │ ├── cardinality: [0 - 1]
8538+
│ │ │ ├── key: ()
8539+
│ │ │ ├── fd: ()-->(8,9,16), (8)==(16), (16)==(8)
8540+
│ │ │ ├── inner-join-apply
8541+
│ │ │ │ ├── columns: x:8!null y:9 u:16!null
8542+
│ │ │ │ ├── key: (16)
8543+
│ │ │ │ ├── fd: (8)-->(9), (8)==(16), (16)==(8)
8544+
│ │ │ │ ├── limit hint: 1.00
8545+
│ │ │ │ ├── scan xy
8546+
│ │ │ │ │ ├── columns: x:8!null y:9
8547+
│ │ │ │ │ ├── key: (8)
8548+
│ │ │ │ │ └── fd: (8)-->(9)
8549+
│ │ │ │ ├── with &1 (foo)
8550+
│ │ │ │ │ ├── columns: u:16!null
8551+
│ │ │ │ │ ├── materialized
8552+
│ │ │ │ │ ├── outer: (9)
8553+
│ │ │ │ │ ├── key: (16)
8554+
│ │ │ │ │ ├── select
8555+
│ │ │ │ │ │ ├── columns: uv.u:12!null uv.v:13
8556+
│ │ │ │ │ │ ├── outer: (9)
8557+
│ │ │ │ │ │ ├── key: (12)
8558+
│ │ │ │ │ │ ├── fd: (12)-->(13)
8559+
│ │ │ │ │ │ ├── scan uv
8560+
│ │ │ │ │ │ │ ├── columns: uv.u:12!null uv.v:13
8561+
│ │ │ │ │ │ │ ├── key: (12)
8562+
│ │ │ │ │ │ │ └── fd: (12)-->(13)
8563+
│ │ │ │ │ │ └── filters
8564+
│ │ │ │ │ │ └── y:9 > 5 [outer=(9), constraints=(/9: [/6 - ]; tight)]
8565+
│ │ │ │ │ └── with-scan &1 (foo)
8566+
│ │ │ │ │ ├── columns: u:16!null
8567+
│ │ │ │ │ ├── mapping:
8568+
│ │ │ │ │ │ └── uv.u:12 => u:16
8569+
│ │ │ │ │ └── key: (16)
8570+
│ │ │ │ └── filters
8571+
│ │ │ │ └── x:8 = u:16 [outer=(8,16), constraints=(/8: (/NULL - ]; /16: (/NULL - ]), fd=(8)==(16), (16)==(8)]
8572+
│ │ │ └── 1
8573+
│ │ └── projections
8574+
│ │ └── true [as=column19:19]
8575+
│ └── false
8576+
└── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
8577+
8578+
norm expect=HoistUnboundJoinFilterFromExistsSubquery
8579+
SELECT * FROM a WHERE EXISTS (
8580+
SELECT * FROM xy WHERE EXISTS (
8581+
SELECT * FROM uv WHERE x = u AND a.i = 100
8582+
)
8583+
);
8584+
----
8585+
select
8586+
├── columns: k:1!null i:2!null f:3 s:4 j:5
8587+
├── key: (1)
8588+
├── fd: ()-->(2), (1)-->(3-5)
8589+
├── scan a
8590+
│ ├── columns: k:1!null i:2 f:3 s:4 j:5
8591+
│ ├── key: (1)
8592+
│ └── fd: (1)-->(2-5)
8593+
└── filters
8594+
├── coalesce [subquery]
8595+
│ ├── subquery
8596+
│ │ └── project
8597+
│ │ ├── columns: column18:18!null
8598+
│ │ ├── cardinality: [0 - 1]
8599+
│ │ ├── key: ()
8600+
│ │ ├── fd: ()-->(18)
8601+
│ │ ├── limit
8602+
│ │ │ ├── columns: x:8!null
8603+
│ │ │ ├── cardinality: [0 - 1]
8604+
│ │ │ ├── key: ()
8605+
│ │ │ ├── fd: ()-->(8)
8606+
│ │ │ ├── semi-join (hash)
8607+
│ │ │ │ ├── columns: x:8!null
8608+
│ │ │ │ ├── key: (8)
8609+
│ │ │ │ ├── limit hint: 1.00
8610+
│ │ │ │ ├── scan xy
8611+
│ │ │ │ │ ├── columns: x:8!null
8612+
│ │ │ │ │ └── key: (8)
8613+
│ │ │ │ ├── scan uv
8614+
│ │ │ │ │ ├── columns: u:12!null
8615+
│ │ │ │ │ └── key: (12)
8616+
│ │ │ │ └── filters
8617+
│ │ │ │ └── x:8 = u:12 [outer=(8,12), constraints=(/8: (/NULL - ]; /12: (/NULL - ]), fd=(8)==(12), (12)==(8)]
8618+
│ │ │ └── 1
8619+
│ │ └── projections
8620+
│ │ └── true [as=column18:18]
8621+
│ └── false
8622+
└── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
8623+
8624+
norm expect=HoistUnboundJoinFilterFromExistsSubquery
8625+
SELECT * FROM a WHERE EXISTS (
8626+
SELECT * FROM xy WHERE EXISTS (
8627+
SELECT * FROM (
8628+
WITH foo(u, v) AS MATERIALIZED (SELECT * FROM uv WHERE y > 5) SELECT * FROM foo
8629+
) WHERE x = u AND a.i = 100
8630+
)
8631+
);
8632+
----
8633+
select
8634+
├── columns: k:1!null i:2!null f:3 s:4 j:5
8635+
├── key: (1)
8636+
├── fd: ()-->(2), (1)-->(3-5)
8637+
├── scan a
8638+
│ ├── columns: k:1!null i:2 f:3 s:4 j:5
8639+
│ ├── key: (1)
8640+
│ └── fd: (1)-->(2-5)
8641+
└── filters
8642+
├── coalesce [subquery]
8643+
│ ├── subquery
8644+
│ │ └── project
8645+
│ │ ├── columns: column20:20!null
8646+
│ │ ├── cardinality: [0 - 1]
8647+
│ │ ├── key: ()
8648+
│ │ ├── fd: ()-->(20)
8649+
│ │ ├── limit
8650+
│ │ │ ├── columns: x:8!null y:9
8651+
│ │ │ ├── cardinality: [0 - 1]
8652+
│ │ │ ├── key: ()
8653+
│ │ │ ├── fd: ()-->(8,9)
8654+
│ │ │ ├── semi-join-apply
8655+
│ │ │ │ ├── columns: x:8!null y:9
8656+
│ │ │ │ ├── key: (8)
8657+
│ │ │ │ ├── fd: (8)-->(9)
8658+
│ │ │ │ ├── limit hint: 1.00
8659+
│ │ │ │ ├── scan xy
8660+
│ │ │ │ │ ├── columns: x:8!null y:9
8661+
│ │ │ │ │ ├── key: (8)
8662+
│ │ │ │ │ └── fd: (8)-->(9)
8663+
│ │ │ │ ├── with &1 (foo)
8664+
│ │ │ │ │ ├── columns: u:16!null
8665+
│ │ │ │ │ ├── materialized
8666+
│ │ │ │ │ ├── outer: (9)
8667+
│ │ │ │ │ ├── key: (16)
8668+
│ │ │ │ │ ├── select
8669+
│ │ │ │ │ │ ├── columns: uv.u:12!null uv.v:13
8670+
│ │ │ │ │ │ ├── outer: (9)
8671+
│ │ │ │ │ │ ├── key: (12)
8672+
│ │ │ │ │ │ ├── fd: (12)-->(13)
8673+
│ │ │ │ │ │ ├── scan uv
8674+
│ │ │ │ │ │ │ ├── columns: uv.u:12!null uv.v:13
8675+
│ │ │ │ │ │ │ ├── key: (12)
8676+
│ │ │ │ │ │ │ └── fd: (12)-->(13)
8677+
│ │ │ │ │ │ └── filters
8678+
│ │ │ │ │ │ └── y:9 > 5 [outer=(9), constraints=(/9: [/6 - ]; tight)]
8679+
│ │ │ │ │ └── with-scan &1 (foo)
8680+
│ │ │ │ │ ├── columns: u:16!null
8681+
│ │ │ │ │ ├── mapping:
8682+
│ │ │ │ │ │ └── uv.u:12 => u:16
8683+
│ │ │ │ │ └── key: (16)
8684+
│ │ │ │ └── filters
8685+
│ │ │ │ └── x:8 = u:16 [outer=(8,16), constraints=(/8: (/NULL - ]; /16: (/NULL - ]), fd=(8)==(16), (16)==(8)]
8686+
│ │ │ └── 1
8687+
│ │ └── projections
8688+
│ │ └── true [as=column20:20]
8689+
│ └── false
8690+
└── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
8691+
8692+
# No-op because of the left-join.
8693+
norm expect-not=HoistUnboundJoinFilterFromExistsSubquery
8694+
SELECT * FROM a WHERE EXISTS (
8695+
SELECT * FROM xy LEFT JOIN uv ON x = u AND a.i = 100
8696+
);
8697+
----
8698+
semi-join-apply
8699+
├── columns: k:1!null i:2 f:3 s:4 j:5
8700+
├── key: (1)
8701+
├── fd: (1)-->(2-5)
8702+
├── scan a
8703+
│ ├── columns: k:1!null i:2 f:3 s:4 j:5
8704+
│ ├── key: (1)
8705+
│ └── fd: (1)-->(2-5)
8706+
├── left-join (hash)
8707+
│ ├── columns: x:8!null u:12
8708+
│ ├── outer: (2)
8709+
│ ├── multiplicity: left-rows(exactly-one), right-rows(zero-or-one)
8710+
│ ├── key: (8)
8711+
│ ├── fd: (8)-->(12)
8712+
│ ├── scan xy
8713+
│ │ ├── columns: x:8!null
8714+
│ │ └── key: (8)
8715+
│ ├── scan uv
8716+
│ │ ├── columns: u:12!null
8717+
│ │ └── key: (12)
8718+
│ └── filters
8719+
│ ├── x:8 = u:12 [outer=(8,12), constraints=(/8: (/NULL - ]; /12: (/NULL - ]), fd=(8)==(12), (12)==(8)]
8720+
│ └── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
8721+
└── filters (true)
8722+
8723+
# No-op because of the full-join.
8724+
norm expect-not=HoistUnboundJoinFilterFromExistsSubquery
8725+
SELECT * FROM a WHERE EXISTS (
8726+
SELECT * FROM xy FULL JOIN uv ON x = u AND a.i = 100
8727+
);
8728+
----
8729+
semi-join-apply
8730+
├── columns: k:1!null i:2 f:3 s:4 j:5
8731+
├── key: (1)
8732+
├── fd: (1)-->(2-5)
8733+
├── scan a
8734+
│ ├── columns: k:1!null i:2 f:3 s:4 j:5
8735+
│ ├── key: (1)
8736+
│ └── fd: (1)-->(2-5)
8737+
├── full-join (hash)
8738+
│ ├── columns: x:8 u:12
8739+
│ ├── outer: (2)
8740+
│ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one)
8741+
│ ├── key: (8,12)
8742+
│ ├── scan xy
8743+
│ │ ├── columns: x:8!null
8744+
│ │ └── key: (8)
8745+
│ ├── scan uv
8746+
│ │ ├── columns: u:12!null
8747+
│ │ └── key: (12)
8748+
│ └── filters
8749+
│ ├── x:8 = u:12 [outer=(8,12), constraints=(/8: (/NULL - ]; /12: (/NULL - ]), fd=(8)==(12), (12)==(8)]
8750+
│ └── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
8751+
└── filters (true)

‎pkg/sql/opt/norm/testdata/rules/join

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1201,7 +1201,7 @@ inner-join (cross)
12011201

12021202
# Regression for issue 28818. Try to trigger undetectable cycle between the
12031203
# PushFilterIntoJoinLeftAndRight and TryDecorrelateSelect rules.
1204-
norm
1204+
norm disable=HoistUnboundJoinFilterFromExistsSubquery
12051205
SELECT 1
12061206
FROM a
12071207
WHERE EXISTS (

0 commit comments

Comments
 (0)