Skip to content

Commit da0be27

Browse files
SeavenSeaven
committed
[Enhancement] optimize constant value cte&subquery (#60814)
Signed-off-by: Seaven <[email protected]> (cherry picked from commit 5b96255)
1 parent 10ac170 commit da0be27

File tree

35 files changed

+642
-803
lines changed

35 files changed

+642
-803
lines changed

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,6 @@ private OptExpression logicalRuleRewrite(
486486

487487
SessionVariable sessionVariable = rootTaskContext.getOptimizerContext().getSessionVariable();
488488
CTEContext cteContext = context.getCteContext();
489-
CTEUtils.collectCteOperators(tree, context);
490489

491490
// see JoinPredicatePushdown
492491
if (sessionVariable.isEnableRboTablePrune()) {
@@ -497,13 +496,14 @@ private OptExpression logicalRuleRewrite(
497496
context.setEnableJoinPredicatePushDown(false);
498497
}
499498

499+
scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
500+
CTEUtils.collectCteOperators(tree, context);
500501
// inline CTE if consume use once
501502
while (cteContext.hasInlineCTE()) {
502503
scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES);
503504
CTEUtils.collectCteOperators(tree, context);
504505
}
505506

506-
scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
507507
CTEUtils.collectCteOperators(tree, context);
508508

509509
scheduler.rewriteOnce(tree, rootTaskContext, new IcebergPartitionsTableRewriteRule());
@@ -588,7 +588,6 @@ private OptExpression logicalRuleRewrite(
588588

589589
tree = pruneSubfield(tree, rootTaskContext, requiredColumns);
590590

591-
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES);
592591
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);
593592

594593
CTEUtils.collectCteOperators(tree, context);

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/SPMOptimizer.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ private OptExpression optimizeByRule(OptExpression tree,
9393
tree = OptExpression.create(new LogicalTreeAnchorOperator(), tree);
9494
deriveLogicalProperty(tree);
9595

96+
scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
9697
CTEContext cteContext = context.getCteContext();
9798
CTEUtils.collectCteOperators(tree, context);
9899

@@ -102,7 +103,6 @@ private OptExpression optimizeByRule(OptExpression tree,
102103
CTEUtils.collectCteOperators(tree, context);
103104
}
104105

105-
scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
106106
CTEUtils.collectCteOperators(tree, context);
107107

108108
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.AGGREGATE_REWRITE_RULES);
@@ -146,7 +146,6 @@ private OptExpression optimizeByRule(OptExpression tree,
146146
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
147147
scheduler.rewriteIterative(tree, rootTaskContext, new PushDownProjectLimitRule());
148148

149-
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES);
150149
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);
151150

152151
CTEUtils.collectCteOperators(tree, context);

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/NonDeterministicVisitor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator;
2323
import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
2424
import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
25+
import com.starrocks.sql.optimizer.operator.logical.LogicalValuesOperator;
2526
import com.starrocks.sql.optimizer.operator.logical.LogicalWindowOperator;
2627
import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
2728
import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
2829
import com.starrocks.sql.optimizer.operator.scalar.LambdaFunctionOperator;
2930
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
3031

32+
import java.util.List;
3133
import java.util.Map;
3234

3335
public class NonDeterministicVisitor extends OptExpressionVisitor<Boolean, Void> {
@@ -200,4 +202,4 @@ public Boolean visitLogicalRawValues(OptExpression optExpression, Void context)
200202
}
201203
return false;
202204
}
203-
}
205+
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@
8787
import com.starrocks.sql.optimizer.rule.transformation.MinMaxOptOnScanRule;
8888
import com.starrocks.sql.optimizer.rule.transformation.PartitionPruneRule;
8989
import com.starrocks.sql.optimizer.rule.transformation.PruneAggregateColumnsRule;
90-
import com.starrocks.sql.optimizer.rule.transformation.PruneAssertOneRowRule;
9190
import com.starrocks.sql.optimizer.rule.transformation.PruneCTEConsumeColumnsRule;
9291
import com.starrocks.sql.optimizer.rule.transformation.PruneCTEProduceRule;
9392
import com.starrocks.sql.optimizer.rule.transformation.PruneEmptyDirectRule;
@@ -310,11 +309,6 @@ public class RuleSet {
310309
new QuantifiedApply2OuterJoinRule()
311310
));
312311

313-
public static final Rule PRUNE_ASSERT_ROW_RULES =
314-
new CombinationRule(RuleType.GP_PRUNE_ASSERT_ROW, ImmutableList.of(
315-
new PruneAssertOneRowRule()
316-
));
317-
318312
public static final Rule PRUNE_UKFK_JOIN_RULES = new CombinationRule(RuleType.GP_PRUNE_UKFK_JOIN, ImmutableList.of(
319313
new PruneUKFKJoinRule()
320314
));

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,6 @@ public enum RuleType {
126126
TF_PUSH_DOWN_APPLY_AGG,
127127
TF_PUSH_DOWN_PROJECT_TO_CTE_ANCHOR,
128128

129-
TF_PRUNE_ASSERT_ONE_ROW,
130129
TF_PUSH_DOWN_ASSERT_ONE_ROW_PROJECT,
131130

132131
TF_MATERIALIZED_VIEW,

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/EliminateJoinWithConstantRule.java‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.starrocks.sql.optimizer.rule.transformation;
1616

17+
import com.google.common.base.Preconditions;
1718
import com.google.common.collect.Lists;
1819
import com.google.common.collect.Maps;
1920
import com.starrocks.analysis.JoinOperator;
@@ -61,16 +62,15 @@ public boolean check(OptExpression input, OptimizerContext context) {
6162
if (!context.getSessionVariable().isEnableConstantExecuteInFE()) {
6263
return false;
6364
}
64-
if (OperatorType.LOGICAL_PROJECT.equals(input.inputAt(constantIndex).getOp().getOpType())) {
65-
LogicalProjectOperator project = input.inputAt(constantIndex).getOp().cast();
66-
if (!project.getColumnRefMap().values().stream().allMatch(ScalarOperator::isConstant)) {
67-
return false;
68-
}
69-
OptExpression optExpression = input.inputAt(constantIndex);
70-
OptExpression valuesOpt = optExpression.inputAt(0);
71-
return checkValuesOptExpression(valuesOpt);
65+
if (!isTransformable((LogicalJoinOperator) input.getOp(), constantIndex)) {
66+
return false;
7267
}
73-
return false;
68+
OptExpression child = input.inputAt(constantIndex);
69+
if (child.getOp().getOpType() == OperatorType.LOGICAL_PROJECT) {
70+
child = child.inputAt(0);
71+
}
72+
Preconditions.checkState(child.getOp().getOpType() != OperatorType.LOGICAL_PROJECT);
73+
return checkValuesOptExpression(child);
7474
}
7575

7676
public boolean checkValuesOptExpression(OptExpression valuesOpt) {
@@ -84,9 +84,6 @@ public boolean checkValuesOptExpression(OptExpression valuesOpt) {
8484

8585
@Override
8686
public List<OptExpression> transform(OptExpression input, OptimizerContext context) {
87-
if (!isTransformable((LogicalJoinOperator) input.getOp(), constantIndex)) {
88-
return Lists.newArrayList(input);
89-
}
9087
OptExpression otherOpt = input.inputAt(1 - constantIndex);
9188
OptExpression valueOpt = input.inputAt(constantIndex);
9289
return onMatch(input, otherOpt, valueOpt, context);

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneAssertOneRowRule.java‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyJoinRule.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,14 @@ private PruneEmptyJoinRule(int index) {
4949

5050
@Override
5151
public boolean check(OptExpression input, OptimizerContext context) {
52-
if (!OperatorType.LOGICAL_VALUES.equals(input.inputAt(emptyIndex).getOp().getOpType())) {
52+
OptExpression child = input.inputAt(emptyIndex);
53+
while (child.getOp().getOpType() == OperatorType.LOGICAL_PROJECT) {
54+
child = child.inputAt(0);
55+
}
56+
if (!OperatorType.LOGICAL_VALUES.equals(child.getOp().getOpType())) {
5357
return false;
5458
}
55-
LogicalValuesOperator v = input.inputAt(emptyIndex).getOp().cast();
59+
LogicalValuesOperator v = child.getOp().cast();
5660
return v.getRows().isEmpty();
5761
}
5862

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ScalarApply2JoinRule.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator;
3737
import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
3838
import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
39+
import com.starrocks.sql.optimizer.operator.logical.LogicalValuesOperator;
3940
import com.starrocks.sql.optimizer.operator.pattern.Pattern;
4041
import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
4142
import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
@@ -249,15 +250,18 @@ private List<OptExpression> transformCorrelateWithoutCheckOneRows(OptExpression
249250
private List<OptExpression> transformUnCorrelateCheckOneRows(OptExpression input, LogicalApplyOperator apply,
250251
OptimizerContext context) {
251252
// assert one rows will check rows, and fill null row if result is empty
252-
OptExpression assertOptExpression = new OptExpression(LogicalAssertOneRowOperator.createLessEqOne(""));
253-
assertOptExpression.getInputs().add(input.getInputs().get(1));
253+
OptExpression rightInput = input.getInputs().get(1);
254+
if (!checkIsLessOneRows(rightInput)) {
255+
rightInput = new OptExpression(LogicalAssertOneRowOperator.createLessEqOne(""));
256+
rightInput.getInputs().add(input.getInputs().get(1));
257+
}
254258

255259
// use hint, forbidden reorder un-correlate subquery
256260
OptExpression joinOptExpression = new OptExpression(LogicalJoinOperator.builder()
257261
.setJoinType(JoinOperator.CROSS_JOIN)
258262
.setJoinHint(JoinOperator.HINT_BROADCAST).build());
259263
joinOptExpression.getInputs().add(input.getInputs().get(0));
260-
joinOptExpression.getInputs().add(assertOptExpression);
264+
joinOptExpression.getInputs().add(rightInput);
261265

262266
ColumnRefFactory factory = context.getColumnRefFactory();
263267
Map<ColumnRefOperator, ScalarOperator> allOutput = Maps.newHashMap();
@@ -272,4 +276,18 @@ private List<OptExpression> transformUnCorrelateCheckOneRows(OptExpression input
272276

273277
return Lists.newArrayList(projectExpression);
274278
}
279+
280+
private boolean checkIsLessOneRows(OptExpression input) {
281+
if (input.getOp().getOpType() == OperatorType.LOGICAL_AGGR) {
282+
// if child is aggregation and none grouping key, remove AssertOneRow node
283+
LogicalAggregationOperator lao = (LogicalAggregationOperator) input.getOp();
284+
return lao.getGroupingKeys().isEmpty() && (lao.getPredicate() == null);
285+
} else if (input.getOp().getOpType() == OperatorType.LOGICAL_PROJECT) {
286+
return checkIsLessOneRows(input.getInputs().get(0));
287+
} else if (input.getOp().getOpType() == OperatorType.LOGICAL_VALUES) {
288+
LogicalValuesOperator lvo = (LogicalValuesOperator) input.getOp();
289+
return lvo.getRows().size() <= 1;
290+
}
291+
return false;
292+
}
275293
}

‎fe/fe-core/src/test/java/com/starrocks/sql/plan/AggregateTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1347,7 +1347,7 @@ public void testMultiCountDistinctWithNoneGroup1() throws Exception {
13471347
"from test_all_type join tmp1 t1 join tmp2 t2 join tmp1 t3 join tmp2 t4";
13481348
Pair<String, ExecPlan> pair = UtFrameUtils.getPlanAndFragment(connectContext, sql);
13491349
System.out.println(pair.first);
1350-
assertContains(pair.first, "CTEAnchor(cteid=3)");
1350+
assertContains(pair.first, "CTEAnchor(cteid=1)");
13511351
FeConstants.runningUnitTest = false;
13521352
}
13531353

0 commit comments

Comments
 (0)