[BugFix] Fix view based mv rewrite bug (backport #62198) (#62230)

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
Co-authored-by: shuming.li <ming.moriarty@gmail.com>

Author: mergify[bot]

docs/en/sql-reference/System_variable.md

@@ -299,6 +299,12 @@ Used for MySQL client compatibility. No practical usage.
 * **Default**: true
 * **Introduced in**: v2.5.13, v3.0.7, v3.1.4, v3.2.0, v3.3.0
 
+### enable_cbo_based_mv_rewrite
+
+* **Description**: Whether to enable materialized view rewrite in CBO phase which can maximize the likelihood of successful query rewriting (e.g., when the join order differs between materialized views and queries), but it will increase the execution time of the optimizer phase.
+* **Default**: true
+* **Introduced in**: v3.5.5, v4.0.1
+
 ### enable_plan_advisor
 
 * **Description**: Whether to enable Query Feedback feature for slow queries and manually marked queries.

docs/ja/sql-reference/System_variable.md

@@ -298,6 +298,11 @@ MySQL クライアント互換性のために使用されます。実際の用
 * **デフォルト**: true
 * **導入バージョン**: v2.5.13, v3.0.7, v3.1.4, v3.2.0, v3.3.0
 
+### enable_cbo_based_mv_rewrite
+* **説明**: CBO フェーズでマテリアライズドビューの書き換えを有効にするかどうか。これにより、クエリ書き換えの成功率を最大化できます（例：マテリアライズドビューとクエリの結合順序が異なる場合）が、オプティマイザフェーズの実行時間が増加します。
+* **デフォルト**: true
+* **導入バージョン**: v3.5.5, v4.0.1
+
 ### enable_plan_advisor
 
 * **説明**: 遅いクエリや手動でマークされたクエリに対するクエリフィードバック機能を有効にするかどうか。

docs/zh/sql-reference/System_variable.md

@@ -303,6 +303,12 @@ ALTER USER 'jack' SET PROPERTIES ('session.query_timeout' = '600');
 * 默认值：true
 * 引入版本：v2.5.13，v3.0.7，v3.1.4，v3.2.0，v3.3.0
 
+### enable_cbo_based_mv_rewrite
+
+* 描述：是否在 CBO 阶段启用物化视图改写，这可以最大化查询改写成功的可能性（例如，当物化视图和查询之间的连接顺序不同时），但这会增加优化器阶段的执行时间。
+* 默认值：true
+* 引入版本：v3.5.5，v4.0.1
+
 ### enable_plan_advisor
 
 * 描述：是否为慢查询或手动标记查询开启 Query Feedback 功能。

fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java

@@ -655,6 +655,7 @@ public static MaterializedViewRewriteMode parse(String str) {
     public static final String ENABLE_VIEW_BASED_MV_REWRITE = "enable_view_based_mv_rewrite";
 
     public static final String ENABLE_CBO_VIEW_BASED_MV_REWRITE = "enable_cbo_view_based_mv_rewrite";
+    public static final String ENABLE_CBO_BASED_MV_REWRITE = "enable_cbo_based_mv_rewrite";
 
     public static final String ENABLE_BIG_QUERY_LOG = "enable_big_query_log";
     public static final String BIG_QUERY_LOG_CPU_SECOND_THRESHOLD = "big_query_log_cpu_second_threshold";
@@ -2083,6 +2084,11 @@ public long getConnectorSinkTargetMaxFileSize() {
     @VarAttr(name = ENABLE_CBO_VIEW_BASED_MV_REWRITE)
     private boolean enableCBOViewBasedMvRewrite = false;
 
+    // Whether enable mv rewrite in CBO phase, true by default which means will try best to use mv
+    // to rewrite in RBO and CBO phase.
+    @VarAttr(name = ENABLE_CBO_BASED_MV_REWRITE)
+    private boolean enableCBOBasedMVRewrite = true;
+
     /**
      * Materialized view rewrite rule output limit: how many MVs would be chosen in a Rule for an OptExpr ?
      */
@@ -4075,6 +4081,14 @@ public boolean isEnableCBOViewBasedMvRewrite() {
         return this.enableCBOViewBasedMvRewrite;
     }
 
+    public boolean isEnableCBOBasedMVRewrite() {
+        return enableCBOBasedMVRewrite;
+    }
+
+    public void setEnableCboBasedMvRewrite(boolean enableCBOBasedMVRewrite) {
+        this.enableCBOBasedMVRewrite = enableCBOBasedMVRewrite;
+    }
+
     public int getCboMaterializedViewRewriteRuleOutputLimit() {
         return cboMaterializedViewRewriteRuleOutputLimit;
     }

fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java

@@ -271,10 +271,6 @@ private OptExpression optimizeByCost(ConnectContext connectContext,
         }
 
         memo.init(logicOperatorTree);
-        if (context.getQueryMaterializationContext() != null) {
-            // LogicalTreeWithView is logically equivalent to logicOperatorTree
-            addViewBasedPlanIntoMemo(context.getQueryMaterializationContext().getQueryOptPlanWithView());
-        }
         OptimizerTraceUtil.log("after logical rewrite, root group:\n%s", memo.getRootGroup());
 
         // Currently, we cache output columns in logic property.
@@ -336,14 +332,6 @@ private OptExpression optimizeByCost(ConnectContext connectContext,
         }
     }
 
-    private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) {
-        if (logicalTreeWithView == null) {
-            return;
-        }
-        Memo memo = context.getMemo();
-        memo.copyIn(memo.getRootGroup(), logicalTreeWithView);
-    }
-
     private void prepare(ConnectContext connectContext,
                          ColumnRefFactory columnRefFactory,
                          OptExpression logicOperatorTree) {
@@ -481,7 +469,7 @@ private void doRuleBasedMaterializedViewRewrite(OptExpression tree,
         }
         if (mvRewriteStrategy.enableForceRBORewrite) {
             // use rule based mv rewrite strategy to do mv rewrite for multi tables query
-            if (mvRewriteStrategy.enableMultiTableRewrite) {
+            if (mvRewriteStrategy.enableCBOBasedMvRewrite && mvRewriteStrategy.enableMultiTableRewrite) {
                 ruleRewriteIterative(tree, rootTaskContext, RuleSetType.MULTI_TABLE_MV_REWRITE);
             }
             if (mvRewriteStrategy.enableSingleTableRewrite) {
@@ -767,6 +755,7 @@ private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskCont
         try (Timer ignored = Tracers.watchScope("MVViewRewrite")) {
             OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "try VIEW_BASED_MV_REWRITE");
             OptExpression treeWithView = queryMaterializationContext.getQueryOptPlanWithView();
+            OptimizerTraceUtil.logOptExpression("before ViewBasedMvRuleRewrite:\n%s", tree);
             // should add a LogicalTreeAnchorOperator for rewrite
             treeWithView = OptExpression.create(new LogicalTreeAnchorOperator(), treeWithView);
             if (mvRewriteStrategy.enableMultiTableRewrite) {
@@ -791,6 +780,7 @@ private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskCont
             }
             OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "original view scans size: {}, " +
                             "left view scans size: {}", origQueryViewScanOperators.size(), leftViewScanOperators.size());
+            OptimizerTraceUtil.logOptExpression("after ViewBasedMvRuleRewrite:\n%s", tree);
         } catch (Exception e) {
             OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE",
                     "single table view based mv rule rewrite failed.", e);

fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/logical/LogicalViewScanOperator.java

@@ -15,6 +15,7 @@
 package com.starrocks.sql.optimizer.operator.logical;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
 import com.starrocks.analysis.Expr;
 import com.starrocks.catalog.Column;
 import com.starrocks.catalog.Table;
@@ -26,6 +27,7 @@
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.OperatorVisitor;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
+import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils;
 
 import java.util.Map;
@@ -42,19 +44,25 @@ public class LogicalViewScanOperator  extends LogicalScanOperator {
     // Original plan evaluator(inlined view) for the input logical plan tree which has not done rule based rewrite yet,
     // this is only used when the view scan operator cannot be rewritten by view-based rewrite rules.
     private OptPlanEvaluator optPlanEvaluator;
+    // this maps original inlined view's column ref to non-inlined column ref which can be used for
+    // replacing from non-inlined column refs to inlined view's column ref.
+    private Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap;
 
     public LogicalViewScanOperator(
             int relationId,
             Table table,
             Map<ColumnRefOperator, Column> colRefToColumnMetaMap,
             Map<Column, ColumnRefOperator> columnMetaToColRefMap,
             ColumnRefSet outputColumnSet,
-            Map<Expr, ColumnRefOperator> expressionToColumns) {
+            Map<Expr, ColumnRefOperator> expressionToColumns,
+            Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap) {
         super(OperatorType.LOGICAL_VIEW_SCAN, table, colRefToColumnMetaMap,
                 columnMetaToColRefMap, Operator.DEFAULT_LIMIT, null, null);
         this.relationId = relationId;
         this.outputColumnSet = outputColumnSet;
         this.expressionToColumns = expressionToColumns;
+        // copy the originalColumnRefToInlinedColumnRefMap to avoid modification by projection
+        this.originalColumnRefToInlinedColumnRefMap = Maps.newHashMap(originalColumnRefToInlinedColumnRefMap);
     }
 
     private LogicalViewScanOperator() {
@@ -84,6 +92,10 @@ public OptExpression getOriginalPlanEvaluator() {
         return this.optPlanEvaluator.evaluate();
     }
 
+    public Map<ColumnRefOperator, ScalarOperator> getOriginalColumnRefToInlinedColumnRefMap() {
+        return originalColumnRefToInlinedColumnRefMap;
+    }
+
     /**
      * Evaluate the original plan and return the optimized plan for the input logical plan tree which is with inlined view and
      * has not done rule based rewrite yet.
@@ -157,6 +169,7 @@ public LogicalViewScanOperator.Builder withOperator(LogicalViewScanOperator scan
             builder.expressionToColumns = scanOperator.expressionToColumns;
             builder.outputColumnSet = scanOperator.outputColumnSet;
             builder.optPlanEvaluator = scanOperator.optPlanEvaluator;
+            builder.originalColumnRefToInlinedColumnRefMap = scanOperator.originalColumnRefToInlinedColumnRefMap;
             return this;
         }
     }

fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java

@@ -70,6 +70,7 @@ public int getOrdinal() {
     public boolean enableViewBasedRewrite = false;
     public boolean enableSingleTableRewrite = false;
     public boolean enableMultiTableRewrite = false;
+    public boolean enableCBOBasedMvRewrite = false;
 
     static class MvStrategyArbitrator {
         private final OptimizerConfig optimizerConfig;
@@ -130,7 +131,7 @@ private boolean isEnableRBOSingleTableRewrite(OptExpression queryPlan) {
             return true;
         }
 
-        private boolean isEnableCBOMultiTableRewrite(OptExpression queryPlan) {
+        private boolean isEnableMultiTableRewrite(OptExpression queryPlan) {
             if (!sessionVariable.isEnableMaterializedViewSingleTableViewDeltaRewrite() &&
                     MvUtils.getAllTables(queryPlan).size() <= 1) {
                 return false;
@@ -170,7 +171,8 @@ public static MvRewriteStrategy prepareRewriteStrategy(OptimizerContext optimize
         strategy.enableSingleTableRewrite = arbitrator.isEnableRBOSingleTableRewrite(queryPlan);
 
         // cbo strategies
-        strategy.enableMultiTableRewrite = arbitrator.isEnableCBOMultiTableRewrite(queryPlan);
+        strategy.enableCBOBasedMvRewrite = sessionVariable.isEnableCBOBasedMVRewrite();
+        strategy.enableMultiTableRewrite = arbitrator.isEnableMultiTableRewrite(queryPlan);
         return strategy;
     }
 

fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java

@@ -72,6 +72,7 @@
 import com.starrocks.sql.optimizer.Optimizer;
 import com.starrocks.sql.optimizer.OptimizerConfig;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerTraceUtil;
 import com.starrocks.sql.optimizer.QueryMaterializationContext;
 import com.starrocks.sql.optimizer.Utils;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
@@ -1180,11 +1181,17 @@ public static OptExpression replaceLogicalViewScanOperator(OptExpression queryEx
         // add a LogicalTreeAnchorOperator to replace the tree easier
         OptExpression anchorExpr = OptExpression.create(new LogicalTreeAnchorOperator(), queryExpression);
         doReplaceLogicalViewScanOperator(anchorExpr, 0, queryExpression);
+
+        // check if there is any view scan operator in the tree
         List<Operator> viewScanOperators = Lists.newArrayList();
         MvUtils.collectViewScanOperator(anchorExpr, viewScanOperators);
         if (!viewScanOperators.isEmpty()) {
+            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE",
+                    "After replacing logical view scan operator but found view scan operator in the tree, "
+                            + "so cannot rewrite the query expression: " + queryExpression);
             return null;
         }
+
         OptExpression newQuery = anchorExpr.inputAt(0);
         deriveLogicalProperty(newQuery);
         return newQuery;
@@ -1196,29 +1203,48 @@ private static void doReplaceLogicalViewScanOperator(OptExpression parent,
         LogicalOperator op = queryExpression.getOp().cast();
         if (op instanceof LogicalViewScanOperator) {
             LogicalViewScanOperator viewScanOperator = op.cast();
-            OptExpression viewPlan = viewScanOperator.getOriginalPlanEvaluator();
+            OptExpression inlineViewPlan = viewScanOperator.getOriginalPlanEvaluator();
             if (viewScanOperator.getPredicate() != null) {
-                // If viewScanOperator contains predicate, we need to rewrite them,
-                // otherwise predicate will be lost.
-                Map<ColumnRefOperator, ScalarOperator> reverseColumnRefMap =
-                        viewScanOperator.getProjection().getColumnRefMap().entrySet().stream()
-                                .collect(Collectors.toMap(e -> (ColumnRefOperator) e.getValue(), e -> e.getKey()));
-                ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(reverseColumnRefMap);
-                Operator.Builder builder = OperatorBuilderFactory.build(viewPlan.getOp());
-                builder.withOperator(viewPlan.getOp());
-                // rewrite predicate
-                builder.setPredicate(rewriter.rewrite(viewScanOperator.getPredicate()));
-                // rewrite projection
-                Map<ColumnRefOperator, ScalarOperator> newColumnRefMap = viewScanOperator.getProjection().getColumnRefMap()
+                // original map records inlined view's column ref to non-inlined view's column ref mapping,
+                // now we need to rewrite non-inlined view's column ref to inlined view's column ref,
+                // so we need to reverse the mapping.
+                Map<ColumnRefOperator, ScalarOperator> originalColumnRefToInlinedColumnRefMap =
+                        Maps.newHashMap(viewScanOperator.getOriginalColumnRefToInlinedColumnRefMap());
+                // add new added column ref mapping
+                if (viewScanOperator.getProjection() != null) {
+                    viewScanOperator.getProjection().getColumnRefMap()
+                            .entrySet()
+                            .stream()
+                            .forEach(e -> originalColumnRefToInlinedColumnRefMap.put(e.getKey(), e.getValue()));
+                }
+                Map<ColumnRefOperator, ScalarOperator> reverseColumnRefMap = originalColumnRefToInlinedColumnRefMap
                         .entrySet()
                         .stream()
-                        .map(e -> Maps.immutableEntry(e.getKey(), rewriter.rewrite(e.getValue())))
-                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-                builder.setProjection(new Projection(newColumnRefMap));
-                Operator newViewPlanOp = builder.build();
-                parent.setChild(index, OptExpression.create(newViewPlanOp, viewPlan.getInputs()));
+                        .collect(Collectors.toMap(e -> (ColumnRefOperator) e.getValue(), e -> e.getKey()));
+
+                // If viewScanOperator contains predicate, we need to rewrite them, otherwise predicate will be lost.
+                ReplaceColumnRefRewriter rewriter = new ReplaceColumnRefRewriter(reverseColumnRefMap);
+                Operator.Builder builder = OperatorBuilderFactory.build(inlineViewPlan.getOp());
+                builder.withOperator(inlineViewPlan.getOp());
+                if (viewScanOperator.getPredicate() != null) {
+                    // rewrite predicate
+                    ScalarOperator rewrittenPredicate = rewriter.rewrite(viewScanOperator.getPredicate());
+                    builder.setPredicate(rewrittenPredicate);
+                }
+                // rewrite projection
+                if (viewScanOperator.getPredicate() != null) {
+                    Map<ColumnRefOperator, ScalarOperator> newColumnRefMap = viewScanOperator
+                            .getProjection().getColumnRefMap()
+                            .entrySet()
+                            .stream()
+                            .map(e -> Maps.immutableEntry(e.getKey(), rewriter.rewrite(e.getValue())))
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    builder.setProjection(new Projection(newColumnRefMap));
+                }
+                Operator newInlineViewPlanOp = builder.build();
+                parent.setChild(index, OptExpression.create(newInlineViewPlanOp, inlineViewPlan.getInputs()));
             } else {
-                parent.setChild(index, viewPlan);
+                parent.setChild(index, inlineViewPlan);
             }
             return;
         }

fe/fe-core/src/main/java/com/starrocks/sql/optimizer/transformer/RelationTransformer.java

@@ -907,7 +907,7 @@ private LogicalViewScanOperator buildViewScan(
 
         LogicalViewScanOperator scanOperator = new LogicalViewScanOperator(relationId,
                 node.getView(), columnRefOperatorToColumn, columnMetaToColRefMap,
-                new ColumnRefSet(logicalPlan.getOutputColumn()), newExprMapping);
+                new ColumnRefSet(logicalPlan.getOutputColumn()), newExprMapping, projectionMap);
         if (inlineView) {
             // add a projection to make sure output columns keep the same,
             // because LogicalViewScanOperator should be logically equivalent to logicalPlan