Skip to content

[BugFix] optimize constant value cte&subquery (backport #60814) #67237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mergify wants to merge 2 commits into
base: branch-3.5
Choose a base branch
from
Open

[BugFix] optimize constant value cte&subquery (backport #60814) #67237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
da0be27
[Enhancement] optimize constant value cte&subquery (#60814)
Seaven Jul 16, 2025
363748c
update
Seaven Dec 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,6 @@ private OptExpression logicalRuleRewrite(

SessionVariable sessionVariable = rootTaskContext.getOptimizerContext().getSessionVariable();
CTEContext cteContext = context.getCteContext();
CTEUtils.collectCteOperators(tree, context);

// see JoinPredicatePushdown
if (sessionVariable.isEnableRboTablePrune()) {
Expand All @@ -497,13 +496,14 @@ private OptExpression logicalRuleRewrite(
context.setEnableJoinPredicatePushDown(false);
}

scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
CTEUtils.collectCteOperators(tree, context);
// inline CTE if consume use once
while (cteContext.hasInlineCTE()) {
scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES);
CTEUtils.collectCteOperators(tree, context);
}

scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
CTEUtils.collectCteOperators(tree, context);

scheduler.rewriteOnce(tree, rootTaskContext, new IcebergPartitionsTableRewriteRule());
Expand Down Expand Up @@ -588,7 +588,6 @@ private OptExpression logicalRuleRewrite(

tree = pruneSubfield(tree, rootTaskContext, requiredColumns);

scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES);
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);

CTEUtils.collectCteOperators(tree, context);
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/SPMOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ private OptExpression optimizeByRule(OptExpression tree,
tree = OptExpression.create(new LogicalTreeAnchorOperator(), tree);
deriveLogicalProperty(tree);

scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
CTEContext cteContext = context.getCteContext();
CTEUtils.collectCteOperators(tree, context);

Expand All @@ -102,7 +103,6 @@ private OptExpression optimizeByRule(OptExpression tree,
CTEUtils.collectCteOperators(tree, context);
}

scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
CTEUtils.collectCteOperators(tree, context);

scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.AGGREGATE_REWRITE_RULES);
Expand Down Expand Up @@ -146,7 +146,6 @@ private OptExpression optimizeByRule(OptExpression tree,
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
scheduler.rewriteIterative(tree, rootTaskContext, new PushDownProjectLimitRule());

scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES);
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);

CTEUtils.collectCteOperators(tree, context);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/NonDeterministicVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,4 +200,4 @@ public Boolean visitLogicalRawValues(OptExpression optExpression, Void context)
}
return false;
}
}
}
6 changes: 0 additions & 6 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@
import com.starrocks.sql.optimizer.rule.transformation.MinMaxOptOnScanRule;
import com.starrocks.sql.optimizer.rule.transformation.PartitionPruneRule;
import com.starrocks.sql.optimizer.rule.transformation.PruneAggregateColumnsRule;
import com.starrocks.sql.optimizer.rule.transformation.PruneAssertOneRowRule;
import com.starrocks.sql.optimizer.rule.transformation.PruneCTEConsumeColumnsRule;
import com.starrocks.sql.optimizer.rule.transformation.PruneCTEProduceRule;
import com.starrocks.sql.optimizer.rule.transformation.PruneEmptyDirectRule;
Expand Down Expand Up @@ -310,11 +309,6 @@ public class RuleSet {
new QuantifiedApply2OuterJoinRule()
));

public static final Rule PRUNE_ASSERT_ROW_RULES =
new CombinationRule(RuleType.GP_PRUNE_ASSERT_ROW, ImmutableList.of(
new PruneAssertOneRowRule()
));

public static final Rule PRUNE_UKFK_JOIN_RULES = new CombinationRule(RuleType.GP_PRUNE_UKFK_JOIN, ImmutableList.of(
new PruneUKFKJoinRule()
));
Expand Down
1 change: 0 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,6 @@ public enum RuleType {
TF_PUSH_DOWN_APPLY_AGG,
TF_PUSH_DOWN_PROJECT_TO_CTE_ANCHOR,

TF_PRUNE_ASSERT_ONE_ROW,
TF_PUSH_DOWN_ASSERT_ONE_ROW_PROJECT,

TF_MATERIALIZED_VIEW,
Expand Down
21 changes: 9 additions & 12 deletions ...n/java/com/starrocks/sql/optimizer/rule/transformation/EliminateJoinWithConstantRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package com.starrocks.sql.optimizer.rule.transformation;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.starrocks.analysis.JoinOperator;
Expand Down Expand Up @@ -61,16 +62,15 @@ public boolean check(OptExpression input, OptimizerContext context) {
if (!context.getSessionVariable().isEnableConstantExecuteInFE()) {
return false;
}
if (OperatorType.LOGICAL_PROJECT.equals(input.inputAt(constantIndex).getOp().getOpType())) {
LogicalProjectOperator project = input.inputAt(constantIndex).getOp().cast();
if (!project.getColumnRefMap().values().stream().allMatch(ScalarOperator::isConstant)) {
return false;
}
OptExpression optExpression = input.inputAt(constantIndex);
OptExpression valuesOpt = optExpression.inputAt(0);
return checkValuesOptExpression(valuesOpt);
if (!isTransformable((LogicalJoinOperator) input.getOp(), constantIndex)) {
return false;
}
return false;
OptExpression child = input.inputAt(constantIndex);
if (child.getOp().getOpType() == OperatorType.LOGICAL_PROJECT) {
child = child.inputAt(0);
}
Preconditions.checkState(child.getOp().getOpType() != OperatorType.LOGICAL_PROJECT);
return checkValuesOptExpression(child);
}

public boolean checkValuesOptExpression(OptExpression valuesOpt) {
Expand All @@ -84,9 +84,6 @@ public boolean checkValuesOptExpression(OptExpression valuesOpt) {

@Override
public List<OptExpression> transform(OptExpression input, OptimizerContext context) {
if (!isTransformable((LogicalJoinOperator) input.getOp(), constantIndex)) {
return Lists.newArrayList(input);
}
OptExpression otherOpt = input.inputAt(1 - constantIndex);
OptExpression valueOpt = input.inputAt(constantIndex);
return onMatch(input, otherOpt, valueOpt, context);
Expand Down
50 changes: 0 additions & 50 deletions .../src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneAssertOneRowRule.java
View file
Open in desktop

This file was deleted.

8 changes: 6 additions & 2 deletions ...ore/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyJoinRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,14 @@ private PruneEmptyJoinRule(int index) {

@Override
public boolean check(OptExpression input, OptimizerContext context) {
if (!OperatorType.LOGICAL_VALUES.equals(input.inputAt(emptyIndex).getOp().getOpType())) {
OptExpression child = input.inputAt(emptyIndex);
while (child.getOp().getOpType() == OperatorType.LOGICAL_PROJECT) {
child = child.inputAt(0);
}
if (!OperatorType.LOGICAL_VALUES.equals(child.getOp().getOpType())) {
return false;
}
LogicalValuesOperator v = input.inputAt(emptyIndex).getOp().cast();
LogicalValuesOperator v = child.getOp().cast();
return v.getRows().isEmpty();
}

Expand Down
24 changes: 21 additions & 3 deletions ...e/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ScalarApply2JoinRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator;
import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
import com.starrocks.sql.optimizer.operator.logical.LogicalValuesOperator;
import com.starrocks.sql.optimizer.operator.pattern.Pattern;
import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
Expand Down Expand Up @@ -249,15 +250,18 @@ private List<OptExpression> transformCorrelateWithoutCheckOneRows(OptExpression
private List<OptExpression> transformUnCorrelateCheckOneRows(OptExpression input, LogicalApplyOperator apply,
OptimizerContext context) {
// assert one rows will check rows, and fill null row if result is empty
OptExpression assertOptExpression = new OptExpression(LogicalAssertOneRowOperator.createLessEqOne(""));
assertOptExpression.getInputs().add(input.getInputs().get(1));
OptExpression rightInput = input.getInputs().get(1);
if (!checkIsLessOneRows(rightInput)) {
rightInput = new OptExpression(LogicalAssertOneRowOperator.createLessEqOne(""));
rightInput.getInputs().add(input.getInputs().get(1));
}

// use hint, forbidden reorder un-correlate subquery
OptExpression joinOptExpression = new OptExpression(LogicalJoinOperator.builder()
.setJoinType(JoinOperator.CROSS_JOIN)
.setJoinHint(JoinOperator.HINT_BROADCAST).build());
joinOptExpression.getInputs().add(input.getInputs().get(0));
joinOptExpression.getInputs().add(assertOptExpression);
joinOptExpression.getInputs().add(rightInput);

ColumnRefFactory factory = context.getColumnRefFactory();
Map<ColumnRefOperator, ScalarOperator> allOutput = Maps.newHashMap();
Expand All @@ -272,4 +276,18 @@ private List<OptExpression> transformUnCorrelateCheckOneRows(OptExpression input

return Lists.newArrayList(projectExpression);
}

private boolean checkIsLessOneRows(OptExpression input) {
if (input.getOp().getOpType() == OperatorType.LOGICAL_AGGR) {
// if child is aggregation and none grouping key, remove AssertOneRow node
LogicalAggregationOperator lao = (LogicalAggregationOperator) input.getOp();
return lao.getGroupingKeys().isEmpty() && (lao.getPredicate() == null);
} else if (input.getOp().getOpType() == OperatorType.LOGICAL_PROJECT) {
return checkIsLessOneRows(input.getInputs().get(0));
} else if (input.getOp().getOpType() == OperatorType.LOGICAL_VALUES) {
LogicalValuesOperator lvo = (LogicalValuesOperator) input.getOp();
return lvo.getRows().size() <= 1;
}
return false;
}
}
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/com/starrocks/sql/plan/AggregateTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1347,7 +1347,7 @@ public void testMultiCountDistinctWithNoneGroup1() throws Exception {
"from test_all_type join tmp1 t1 join tmp2 t2 join tmp1 t3 join tmp2 t4";
Pair<String, ExecPlan> pair = UtFrameUtils.getPlanAndFragment(connectContext, sql);
System.out.println(pair.first);
assertContains(pair.first, "CTEAnchor(cteid=3)");
assertContains(pair.first, "CTEAnchor(cteid=1)");
FeConstants.runningUnitTest = false;
}

Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/com/starrocks/sql/plan/HiveTPCHPlanTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public static void afterClass() {
@ParameterizedTest(name = "Tpch.{0}")
@MethodSource("tpchSource")
public void testTPCH(String name, String sql, String resultFile) {
runFileUnitTest(sql, resultFile);
runFileUnitTest(sql, resultFile, true);
}

private static Stream<Arguments> tpchSource() {
Expand Down
15 changes: 7 additions & 8 deletions fe/fe-core/src/test/java/com/starrocks/sql/plan/JoinTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1960,13 +1960,12 @@ public void testJoinPredicateTransitivityWithSubqueryInWhereClause() throws Exce
" SELECT MAX(k9)\n" +
" FROM test.pushdown_test);";
String plan = starRocksAssert.query(sql).explainQuery();
assertContains(plan, " UNPARTITIONED\n" +
"\n" +
" 2:AGGREGATE (update finalize)\n" +
" | output: max(22: k9)\n" +
" | group by: \n" +
" | \n" +
" 1:OlapScanNode");
assertContains(plan, "2:AGGREGATE (update finalize)\n"
+ " | output: max(22: k9)\n"
+ " | group by: \n"
+ " | having: 23: max > 0.0\n"
+ " | \n"
+ " 1:OlapScanNode");
}

@Test
Expand Down Expand Up @@ -2495,7 +2494,7 @@ public void testSemiJoinReorderProjections() throws Exception {
" ) t;";
String plan = getFragmentPlan(sql);
// check no error
assertContains(plan, "11:ASSERT NUMBER OF ROWS");
assertContains(plan, "ASSERT NUMBER OF ROWS");
}

@Test
Expand Down
46 changes: 23 additions & 23 deletions fe/fe-core/src/test/java/com/starrocks/sql/plan/OrderByTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,18 +135,18 @@ void testOrderByWithSubquery() throws Exception {
"(select sum(v5) from t1) as x1, " +
"(select sum(v7) from t2) as x2 from t0 order by t0.v3";
String plan = getFragmentPlan(sql);
assertContains(plan, " 15:SORT\n" +
" | order by: <slot 3> 3: v3 ASC\n" +
" | offset: 0\n" +
" | \n" +
" 14:Project\n" +
" | <slot 1> : 1: v1\n" +
" | <slot 2> : 2: v2\n" +
" | <slot 3> : 3: v3\n" +
" | <slot 8> : 8: sum\n" +
" | <slot 13> : 12: sum\n" +
" | \n" +
" 13:NESTLOOP JOIN");
assertContains(plan, "SORT\n"
+ " | order by: <slot 3> 3: v3 ASC\n"
+ " | offset: 0\n"
+ " | \n"
+ " 10:Project\n"
+ " | <slot 1> : 1: v1\n"
+ " | <slot 2> : 2: v2\n"
+ " | <slot 3> : 3: v3\n"
+ " | <slot 8> : 8: sum\n"
+ " | <slot 13> : 12: sum\n"
+ " | \n"
+ " 9:NESTLOOP JOIN");
}

@Test
Expand Down Expand Up @@ -193,17 +193,17 @@ void testOrderByGroupByWithSubquery() throws Exception {
"(select sum(v5) from t1) as x1, " +
"(select sum(v7) from t2) as x2 from t0 group by v2 order by x3";
String plan = getFragmentPlan(sql);
assertContains(plan, " 16:SORT\n" +
" | order by: <slot 4> 4: sum ASC\n" +
" | offset: 0\n" +
" | \n" +
" 15:Project\n" +
" | <slot 2> : 2: v2\n" +
" | <slot 4> : 4: sum\n" +
" | <slot 9> : 9: sum\n" +
" | <slot 14> : 13: sum\n" +
" | \n" +
" 14:NESTLOOP JOIN\n");
assertContains(plan, "SORT\n"
+ " | order by: <slot 4> 4: sum ASC\n"
+ " | offset: 0\n"
+ " | \n"
+ " 11:Project\n"
+ " | <slot 2> : 2: v2\n"
+ " | <slot 4> : 4: sum\n"
+ " | <slot 9> : 9: sum\n"
+ " | <slot 14> : 13: sum\n"
+ " | \n"
+ " 10:NESTLOOP JOIN\n");
}

@Test
Expand Down
Loading
Loading