diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index 8d3aeaa6a267a0..e93638a8853d05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -101,7 +101,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { } if (!runtimeFilters.isEmpty()) { output.append(prefix).append("runtime filters: "); - output.append(getRuntimeFilterExplainString(false)); + output.append(getRuntimeFilterExplainString()); } output.append(prefix); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 4f81dde82d97f2..7d020c7fa3b3f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids; +import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.hint.Hint; @@ -36,6 +37,7 @@ import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; import org.apache.doris.nereids.processor.post.TopnFilterContext; +import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterContextV2; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; @@ -47,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.ColumnStatistic; @@ -94,6 +97,7 @@ public class CascadesContext implements ScheduleContext { // subqueryExprIsAnalyzed: whether the subquery has been analyzed. private final Map subqueryExprIsAnalyzed; private final RuntimeFilterContext runtimeFilterContext; + private final RuntimeFilterContextV2 runtimeFilterV2Context; private final TopnFilterContext topnFilterContext = new TopnFilterContext(); private Optional outerScope = Optional.empty(); @@ -142,7 +146,10 @@ private CascadesContext(Optional parent, Optional curren this.jobScheduler = new SimpleJobScheduler(); this.currentJobContext = new JobContext(this, requireProperties, Double.MAX_VALUE); this.subqueryExprIsAnalyzed = new HashMap<>(); - this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable()); + IdGenerator runtimeFilterIdGen = RuntimeFilterId.createGenerator(); + this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable(), + runtimeFilterIdGen); + this.runtimeFilterV2Context = new RuntimeFilterContextV2(runtimeFilterIdGen); this.materializationContexts = new HashSet<>(); if (statementContext.getConnectContext() != null) { ConnectContext connectContext = statementContext.getConnectContext(); @@ -534,4 +541,8 @@ public int getDistinctAggLevel() { public boolean isEnableExprTrace() { return isEnableExprTrace; } + + public RuntimeFilterContextV2 getRuntimeFilterV2Context() { + return runtimeFilterV2Context; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 74c36e5970f1db..7a0b1f68b1d1ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -73,6 +73,7 @@ import org.apache.doris.fs.FileSystemDirectoryLister; import org.apache.doris.fs.TransactionScopeCachingDirectoryListerFactory; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterV2; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.DistributionSpecAny; import org.apache.doris.nereids.properties.DistributionSpecExecutionAny; @@ -822,6 +823,16 @@ public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTransla expr, olapScanNode, context) ) ); + + // translate rf v2 target + List rfV2s = context.getRuntimeFilterV2Context() + .getRuntimeFilterV2ByTargetPlan(olapScan); + for (RuntimeFilterV2 rfV2 : rfV2s) { + Expr targetExpr = rfV2.getTargetExpression().accept(ExpressionTranslator.INSTANCE, context); + rfV2.setLegacyTargetNode(olapScanNode); + rfV2.setLegacyTargetExpr(targetExpr); + } + context.getTopnFilterContext().translateTarget(olapScan, olapScanNode, context); olapScanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(olapScan.getRelationId())); // Create PlanFragment @@ -2159,6 +2170,28 @@ && findOlapScanNodesByPassExchangeAndJoinNode(setOperationFragment.getPlanRoot() return setOperationFragment; } + @Override + public PlanFragment visitPhysicalIntersect(PhysicalIntersect intersect, PlanTranslatorContext context) { + PlanFragment fragment = visitPhysicalSetOperation(intersect, context); + RunTimeFilterTranslatorV2.INSTANCE.createLegacyRuntimeFilters( + fragment.getPlanRoot(), + intersect.getRuntimeFiltersV2(), + context); + + return fragment; + } + + @Override + public PlanFragment visitPhysicalExcept(PhysicalExcept except, PlanTranslatorContext context) { + PlanFragment fragment = visitPhysicalSetOperation(except, context); + RunTimeFilterTranslatorV2.INSTANCE.createLegacyRuntimeFilters( + fragment.getPlanRoot(), + except.getRuntimeFiltersV2(), + context); + + return fragment; + } + /*- * Physical sort: * 1. Build sortInfo diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index f40a4e2e8f47c6..2992bbe67b5c4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -30,6 +30,7 @@ import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.processor.post.TopnFilterContext; +import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterContextV2; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -72,6 +73,7 @@ public class PlanTranslatorContext { private final DescriptorTable descTable = new DescriptorTable(); private final RuntimeFilterTranslator translator; + private final TopnFilterContext topnFilterContext; /** * index from Nereids' slot to legacy slot. @@ -113,11 +115,13 @@ public class PlanTranslatorContext { private final Map tablePushAggOp = Maps.newHashMap(); private final Map> statsUnknownColumnsMap = Maps.newHashMap(); + private final RuntimeFilterContextV2 runtimeFilterV2Context; public PlanTranslatorContext(CascadesContext ctx) { this.connectContext = ctx.getConnectContext(); this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext()); this.topnFilterContext = ctx.getTopnFilterContext(); + this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context(); } @VisibleForTesting @@ -125,6 +129,7 @@ public PlanTranslatorContext() { this.connectContext = null; this.translator = null; this.topnFilterContext = new TopnFilterContext(); + this.runtimeFilterV2Context = null; } /** @@ -337,4 +342,8 @@ public void setRelationPushAggOp(RelationId relationId, TPushAggOp aggOp) { public TPushAggOp getRelationPushAggOp(RelationId relationId) { return tablePushAggOp.getOrDefault(relationId, TPushAggOp.NONE); } + + public RuntimeFilterContextV2 getRuntimeFilterV2Context() { + return runtimeFilterV2Context; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RunTimeFilterTranslatorV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RunTimeFilterTranslatorV2.java new file mode 100644 index 00000000000000..6cfc1236af3dc8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RunTimeFilterTranslatorV2.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.Expr; +import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterV2; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.planner.RuntimeFilter; +import org.apache.doris.planner.RuntimeFilter.RuntimeFilterTarget; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * RunTimeFilterTranslatorV2 + */ +public class RunTimeFilterTranslatorV2 { + public static RunTimeFilterTranslatorV2 INSTANCE = new RunTimeFilterTranslatorV2(); + + /** + * createLegacyRuntimeFilters + */ + public void createLegacyRuntimeFilters(PlanNode sourceNode, + List filters, PlanTranslatorContext ctx) { + List filtersToTranslate = Lists.newArrayList(filters); + Set ignoreRuntimeFilterIds = ConnectContext.get() != null + ? ConnectContext.get().getSessionVariable().getIgnoredRuntimeFilterIds() + : new HashSet<>(); + while (!filtersToTranslate.isEmpty()) { + List translateRound = Lists.newArrayListWithCapacity(filtersToTranslate.size()); + List otherRound = Lists.newArrayListWithCapacity(filtersToTranslate.size()); + RuntimeFilterV2 head = filtersToTranslate.get(0); + if (!ignoreRuntimeFilterIds.contains(head.getRuntimeFilterId().asInt())) { + translateRound.add(head); + } + for (int i = 1; i < filtersToTranslate.size(); i++) { + if (!ignoreRuntimeFilterIds.contains(filtersToTranslate.get(i).getRuntimeFilterId().asInt())) { + if (head.getSourceExpression().equals(filtersToTranslate.get(i).getSourceExpression()) + && head.getType() == filtersToTranslate.get(i).getType()) { + translateRound.add(filtersToTranslate.get(i)); + } else { + otherRound.add(filtersToTranslate.get(i)); + } + } + } + if (!translateRound.isEmpty()) { + translateRuntimeFilterGroup(sourceNode, translateRound, ctx); + } + filtersToTranslate = otherRound; + } + + } + + /** + * a group of RFs if their source and type are the same, but their targets are + * different. + * example: + * rf1[bloom](a->T1.b) rf2[bloom](a->T2.c) rf3[min_max](a->t3.d) + * rf1 and rf2 are in one group, but rf3 is not + * + */ + private void translateRuntimeFilterGroup(PlanNode sourceNode, + List filters, PlanTranslatorContext ctx) { + if (filters.isEmpty()) { + return; + } + + RuntimeFilterV2 head = filters.get(0); + + Expr srcExpr = ExpressionTranslator.translate(head.getSourceExpression(), ctx); + + List targets = new ArrayList<>(); + for (RuntimeFilterV2 filter : filters) { + Expr targetExpr = filter.getLegacyTargetExpr(); + if (!srcExpr.getType().equals(targetExpr.getType())) { + targetExpr = new CastExpr(srcExpr.getType(), targetExpr); + } + targets.add(new RuntimeFilterTarget(filter.getLegacyTargetNode(), targetExpr)); + } + + RuntimeFilter legacyFilter = new RuntimeFilter( + head.getId(), + sourceNode, + srcExpr, + head.getExprOrder(), + targets, + head.getType(), + head.getBuildNdvOrRowCount(), + head.getTMinMaxRuntimeFilterType()); + + // finalize + legacyFilter.assignToPlanNodes(); + legacyFilter.extractTargetsPosition(); + legacyFilter.markFinalized(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java index a8654e27291c06..b09c94f0846665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.processor.post; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterV2Generator; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TRuntimeFilterMode; @@ -75,13 +76,14 @@ public List getProcessors() { if (!cascadesContext.getConnectContext().getSessionVariable().getRuntimeFilterMode() .toUpperCase().equals(TRuntimeFilterMode.OFF.name())) { builder.add(new RegisterParent()); - builder.add(new RuntimeFilterGenerator()); + builder.add(new org.apache.doris.nereids.processor.post.RuntimeFilterGenerator()); if (ConnectContext.get().getSessionVariable().enableRuntimeFilterPrune) { builder.add(new RuntimeFilterPruner()); if (ConnectContext.get().getSessionVariable().runtimeFilterPruneForExternal) { builder.add(new RuntimeFilterPrunerForExternalTable()); } } + builder.add(new RuntimeFilterV2Generator()); } builder.add(new Validator()); return builder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java index 04e254f549aa64..f3e101c748e5e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.processor.post; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -32,6 +33,7 @@ import org.apache.doris.planner.DataStreamSink; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.RuntimeFilterGenerator.FilterSizeLimits; +import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.planner.ScanNode; import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TRuntimeFilterType; @@ -125,6 +127,7 @@ public boolean equals(Object other) { private final List expandedRF = Lists.newArrayList(); private final Map> relationsUsedByPlan = Maps.newHashMap(); + private final IdGenerator runtimeFilterIdGen; /** * info about expand rf by inner join @@ -149,9 +152,10 @@ public ExpandRF(AbstractPhysicalJoin buildNode, PhysicalRelation srcNode, } } - public RuntimeFilterContext(SessionVariable sessionVariable) { + public RuntimeFilterContext(SessionVariable sessionVariable, IdGenerator runtimeFilterIdGen) { this.sessionVariable = sessionVariable; this.limits = new FilterSizeLimits(sessionVariable); + this.runtimeFilterIdGen = runtimeFilterIdGen; } /** @@ -369,4 +373,8 @@ public ExpandRF getExpandRfByJoin(AbstractPhysicalJoin join) { public List getExpandedRF() { return expandedRF; } + + public IdGenerator getRuntimeFilterIdGen() { + return runtimeFilterIdGen; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index f1ed71373eebfa..6254cecfa17d84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.processor.post; -import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.stats.ExpressionEstimation; @@ -56,7 +55,6 @@ import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.thrift.TMinMaxRuntimeFilterType; import org.apache.doris.thrift.TRuntimeFilterType; @@ -97,7 +95,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { PhysicalHashJoin.class ); - private final IdGenerator generator = RuntimeFilterId.createGenerator(); + public RuntimeFilterGenerator() { + } @Override public Plan processRoot(Plan plan, CascadesContext ctx) { @@ -282,7 +281,7 @@ public PhysicalPlan visitPhysicalHashJoin(PhysicalHashJoin { + public static PushDownVisitor INSTANCE = new PushDownVisitor(); + + @Override + public Boolean visit(Plan plan, PushDownContext ctx) { + boolean pushed = false; + for (Plan child : plan.children()) { + if (child.getOutputSet().containsAll(ctx.getTargetExpression().getInputSlots())) { + pushed |= child.accept(this, ctx); + } + } + return pushed; + } + + @Override + public Boolean visitPhysicalCatalogRelation(PhysicalCatalogRelation relation, PushDownContext ctx) { + if (ctx.getTargetExpression().getInputSlots().size() != 1) { + return false; + } + Slot targetSlot = ctx.getTargetExpression().getInputSlots().iterator().next(); + if (relation.getOutputSet().contains(targetSlot)) { + for (TRuntimeFilterType type : ctx.getRFContext().getTypes()) { + RuntimeFilterV2 rfV2 = new RuntimeFilterV2( + ctx.getRFContext().nextId(), + ctx.getSourceNode(), + ctx.getSourceExpression(), + ctx.getBuildNdvOrRowCount(), + ctx.getExprOrder(), + relation, + ctx.getTargetExpression(), + type); + ctx.getRFContext().addRuntimeFilterV2(rfV2); + relation.addRuntimeFilterV2(rfV2); + ctx.getSourceNode().addRuntimeFilterV2(rfV2); + } + return true; + } + return false; + } + + @Override + public Boolean visitPhysicalProject(PhysicalProject project, PushDownContext ctx) { + if (!project.getOutputSet().containsAll(ctx.getTargetExpression().getInputSlots())) { + return false; + } + + Map replaceMap = ExpressionUtils.generateReplaceMap(project.getProjects()); + Expression newTarget = ctx.getTargetExpression().rewriteDownShortCircuit( + e -> replaceMap.getOrDefault(e, e)); + if (newTarget.getInputSlots().size() == 1) { + return project.child().accept(this, ctx.withTarget(newTarget)); + } else { + return false; + } + } + + @Override + public Boolean visitPhysicalSetOperation(PhysicalSetOperation setOp, PushDownContext ctx) { + if (!setOp.getOutputSet().containsAll(ctx.getTargetExpression().getInputSlots())) { + return false; + } + + List output = setOp.getOutput(); + List> childrenOutput = setOp.getRegularChildrenOutputs(); + boolean pushed = false; + for (int i = 1; i < childrenOutput.size(); i++) { + Map replaceMap = new HashMap<>(); + for (int j = 0; j < output.size(); j++) { + replaceMap.put(output.get(j), childrenOutput.get(i).get(j)); + } + Expression newTarget = ctx.getTargetExpression().rewriteDownShortCircuit( + e -> replaceMap.getOrDefault(e, e)); + if (newTarget.getInputSlots().size() == 1) { + pushed |= setOp.child(i).accept(this, ctx.withTarget(newTarget)); + } + } + return pushed; + } + + @Override + public Boolean visitPhysicalHashJoin(PhysicalHashJoin join, PushDownContext ctx) { + if (!join.getOutputSet().containsAll(ctx.getTargetExpression().getInputSlots())) { + return false; + } + + if (ctx.isNullSafe() || join.getJoinType().isOuterJoin()) { + return false; + } + + boolean pushed = join.left().accept(this, ctx); + pushed |= join.right().accept(this, ctx); + + List hashJoinConditions = join.getHashJoinConjuncts(); + for (Expression hashJoinCondition : hashJoinConditions) { + if (hashJoinCondition instanceof EqualTo) { + EqualTo equal = (EqualTo) hashJoinCondition; + equal = (EqualTo) JoinUtils.swapEqualToForChildrenOrder(equal, join.left().getOutputSet()); + if (ctx.getTargetExpression().equals(equal.left())) { + pushed |= join.right().accept(this, ctx.withTarget(equal.right())); + } else if (ctx.getTargetExpression().equals(equal.right())) { + pushed |= join.left().accept(this, ctx.withTarget(equal.left())); + } + } + } + return pushed; + } + + @Override + public Boolean visitPhysicalNestedLoopJoin(PhysicalNestedLoopJoin join, + PushDownContext ctx) { + if (!join.getOutputSet().containsAll(ctx.getTargetExpression().getInputSlots())) { + return false; + } + + if (ctx.isNullSafe() || join.getJoinType().isOuterJoin()) { + return false; + } + + boolean pushed = join.left().accept(this, ctx); + pushed |= join.right().accept(this, ctx); + + return pushed; + } + + @Override + public Boolean visitPhysicalTopN(PhysicalTopN topN, PushDownContext ctx) { + return false; + } + + @Override + public Boolean visitPhysicalWindow( + PhysicalWindow window, PushDownContext ctx) { + Set commonPartitionKeys = window.getCommonPartitionKeyFromWindowExpressions(); + if (commonPartitionKeys.containsAll(ctx.getTargetExpression().getInputSlots())) { + return window.child().accept(this, ctx); + } + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterContextV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterContextV2.java new file mode 100644 index 00000000000000..dfdf7e173b1c47 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterContextV2.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post.runtimefilterv2; + +import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TRuntimeFilterType; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * RuntimeFilterContextV2 + */ +public class RuntimeFilterContextV2 { + + private final List rfsV2 = new ArrayList<>(); + + private final List types = new ArrayList<>(); + + private final IdGenerator idGenerator; + + /** + * constr + */ + public RuntimeFilterContextV2(IdGenerator runtimeFilterIdGen) { + int typesInt = 2; + if (ConnectContext.get() != null) { + typesInt = ConnectContext.get().getSessionVariable().getRuntimeFilterType(); + } + for (TRuntimeFilterType type : TRuntimeFilterType.values()) { + if ((type.getValue() & typesInt) > 0) { + types.add(type); + } + } + this.idGenerator = runtimeFilterIdGen; + } + + public RuntimeFilterId nextId() { + return idGenerator.getNextId(); + } + + public List getTypes() { + return types; + } + + public List getRuntimeFilterV2ByTargetPlan(AbstractPhysicalPlan targetPlan) { + return rfsV2.stream() + .filter(rf -> rf.getTargetNode().equals(targetPlan)) + .collect(Collectors.toList()); + } + + public void addRuntimeFilterV2(RuntimeFilterV2 rfv2) { + rfsV2.add(rfv2); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterV2.java new file mode 100644 index 00000000000000..844218e2966e3c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterV2.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post.runtimefilterv2; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.thrift.TMinMaxRuntimeFilterType; +import org.apache.doris.thrift.TRuntimeFilterType; + +/** + * RuntimeFilterV2 + */ +public class RuntimeFilterV2 { + private RuntimeFilterId id; + private AbstractPhysicalPlan sourceNode; + private Expression sourceExpression; + private long buildNdvOrRowCount; + private int exprOrder; + private AbstractPhysicalPlan targetNode; + private Expression targetExpression; + private TRuntimeFilterType type; + private TMinMaxRuntimeFilterType tMinMaxRuntimeFilterType = TMinMaxRuntimeFilterType.MIN_MAX; + + // translate + private PlanNode legacyTargetNode; + private Expr legacyTargetExpr; + + /** + * constr + */ + public RuntimeFilterV2(RuntimeFilterId id, + AbstractPhysicalPlan sourceNode, Expression source, long buildNdvOrRowCount, int exprOrder, + AbstractPhysicalPlan targetNode, Expression target, TRuntimeFilterType type) { + this.sourceNode = sourceNode; + this.sourceExpression = source; + this.buildNdvOrRowCount = buildNdvOrRowCount; + this.exprOrder = exprOrder; + this.targetNode = targetNode; + this.targetExpression = target; + this.id = id; + this.type = type; + } + + public RuntimeFilterId getId() { + return id; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("RF").append(id.asInt()).append("[").append(type).append("]") + .append("(").append(sourceExpression) + .append("->").append(targetExpression).append(")"); + return sb.toString(); + } + + public RuntimeFilterId getRuntimeFilterId() { + return id; + } + + public AbstractPhysicalPlan getSourceNode() { + return sourceNode; + } + + public Expression getSourceExpression() { + return sourceExpression; + } + + public AbstractPhysicalPlan getTargetNode() { + return targetNode; + } + + public Expression getTargetExpression() { + return targetExpression; + } + + public TRuntimeFilterType getType() { + return type; + } + + public int getExprOrder() { + return exprOrder; + } + + public long getBuildNdvOrRowCount() { + return buildNdvOrRowCount; + } + + public void setMinMaxSubType(TMinMaxRuntimeFilterType tMinMaxRuntimeFilterType) { + this.tMinMaxRuntimeFilterType = tMinMaxRuntimeFilterType; + } + + public TMinMaxRuntimeFilterType getTMinMaxRuntimeFilterType() { + return tMinMaxRuntimeFilterType; + } + + public PlanNode getLegacyTargetNode() { + return legacyTargetNode; + } + + public Expr getLegacyTargetExpr() { + return legacyTargetExpr; + } + + public void setLegacyTargetNode(PlanNode legacyTargetNode) { + this.legacyTargetNode = legacyTargetNode; + } + + public void setLegacyTargetExpr(Expr legacyTargetExpr) { + this.legacyTargetExpr = legacyTargetExpr; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterV2Generator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterV2Generator.java new file mode 100644 index 00000000000000..291eaa789400c4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/RuntimeFilterV2Generator.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post.runtimefilterv2; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.processor.post.PlanPostProcessor; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.Statistics; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * RuntimeFilterV2Generator + */ +public class RuntimeFilterV2Generator extends PlanPostProcessor { + + public RuntimeFilterV2Generator() { + } + + @Override + public Plan visitPhysicalIntersect(PhysicalIntersect intersect, CascadesContext context) { + for (int slotIdx : chooseSourceSlots(intersect)) { + for (int childId = 1; childId < intersect.children().size(); childId++) { + Plan child = intersect.children().get(childId); + Statistics stats = ((AbstractPlan) intersect.child(0)).getStats(); + long buildNdvOrRowCount = -1; + if (stats != null) { + buildNdvOrRowCount = (long) stats.getRowCount(); + ColumnStatistic colStats = stats.findColumnStatistics( + intersect.child(0).getOutput().get(slotIdx)); + if (colStats != null && !colStats.isUnKnown) { + buildNdvOrRowCount = Math.max(1, (long) colStats.ndv); + } + } + PushDownContext pushDownContext = new PushDownContext( + context.getRuntimeFilterV2Context(), + intersect, + intersect.child(0).getOutput().get(slotIdx), + buildNdvOrRowCount, + slotIdx, + intersect.child(childId).getOutput().get(slotIdx)); + child.accept(PushDownVisitor.INSTANCE, pushDownContext); + } + } + return visitPhysicalSetOperation(intersect, context); + } + + private List chooseSourceSlots(PhysicalIntersect intersect) { + // TODO: choose best slots by ndv + return ImmutableList.of(0); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectBetweenTopNAndAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectBetweenTopNAndAgg.java index f80c40cff7a98f..5a3055c42ef466 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectBetweenTopNAndAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectBetweenTopNAndAgg.java @@ -92,12 +92,8 @@ private Plan adjust(LogicalTopN topN) { } if (match) { if (project.getProjects().size() >= project.getInputSlots().size()) { - LOG.info("$$$$ before: project.getProjects() = " + project.getProjects()); - LOG.info("$$$$ before: project.getInputSlots() = " + project.getInputSlots()); - LOG.info("$$$$ before: " + topN.treeString()); topN = topN.withChildren(project.children()).withOrderKeys(newOrderKeys); project = (LogicalProject) project.withChildren(topN); - LOG.info("$$$$ after:" + project.treeString()); return project; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java index 194f6356045cc0..faa1447ff75c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java @@ -286,9 +286,13 @@ public String toString() { args.add(hint.getExplainString()); } if (!runtimeFilters.isEmpty()) { - args.add("runtimeFilters"); + args.add("RFs"); args.add(runtimeFilters.stream().map(rf -> rf.toString() + " ").collect(Collectors.toList())); } + if (!runtimeFiltersV2.isEmpty()) { + args.add("RFV2"); + args.add(runtimeFiltersV2); + } return Utils.toSqlString(this.getClass().getSimpleName() + "[" + id.asInt() + "]" + getGroupIdWithPrefix(), args.toArray()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 2146b9fe5de04a..ebff6dcbef918b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.physical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterV2; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.plans.AbstractPlan; @@ -40,6 +41,7 @@ */ public abstract class AbstractPhysicalPlan extends AbstractPlan implements PhysicalPlan, Explainable { protected final PhysicalProperties physicalProperties; + protected final List runtimeFiltersV2 = Lists.newArrayList(); private final List appliedRuntimeFilters = Lists.newArrayList(); public AbstractPhysicalPlan(PlanType type, LogicalProperties logicalProperties, Plan... children) { @@ -77,15 +79,23 @@ public AbstractPhysicalPlan copyStatsAndGroupId return newPlan; } - public List getAppliedRuntimeFilters() { + public List getAppliedRuntimeFilters() { return appliedRuntimeFilters; } - public void addAppliedRuntimeFilter(RuntimeFilter filter) { + public void addAppliedRuntimeFilter(org.apache.doris.nereids.trees.plans.physical.RuntimeFilter filter) { appliedRuntimeFilters.add(filter); } - public void removeAppliedRuntimeFilter(RuntimeFilter filter) { + public void addRuntimeFilterV2(RuntimeFilterV2 filter) { + runtimeFiltersV2.add(filter); + } + + public List getRuntimeFiltersV2() { + return runtimeFiltersV2; + } + + public void removeAppliedRuntimeFilter(org.apache.doris.nereids.trees.plans.physical.RuntimeFilter filter) { appliedRuntimeFilters.remove(filter); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java index 4a4b6b61ffd961..c834cea4b76d5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java @@ -152,6 +152,9 @@ public String shapeInfo() { getAppliedRuntimeFilters() .stream().forEach(rf -> shapeBuilder.append(" RF").append(rf.getId().asInt())); } + if (!runtimeFiltersV2.isEmpty()) { + shapeBuilder.append("RFV2: ").append(runtimeFiltersV2); + } return shapeBuilder.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index e4caab5bce3677..ca84e1afc2fd1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -220,6 +220,9 @@ public String shapeInfo() { builder.append(" build RFs:").append(runtimeFilters.stream() .map(rf -> rf.shapeInfo()).collect(Collectors.joining(";"))); } + if (!runtimeFiltersV2.isEmpty()) { + builder.append(" RFV2: ").append(runtimeFiltersV2); + } return builder.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java index 3194939a9e6481..6aec62b443743b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java @@ -80,7 +80,9 @@ public String toString() { "qualifier", qualifier, "outputs", outputs, "regularChildrenOutputs", regularChildrenOutputs, - "stats", statistics); + "stats", statistics, + "RFV2", runtimeFiltersV2 + ); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 698a189aa265ba..44d1ea0ce174ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -118,9 +118,10 @@ public List getBaseOutputs() { @Override public String toString() { - StringBuilder builder = new StringBuilder(); + StringBuilder jrfBuilder = new StringBuilder(); if (!getAppliedRuntimeFilters().isEmpty()) { - getAppliedRuntimeFilters().forEach(rf -> builder.append(" RF").append(rf.getId().asInt())); + getAppliedRuntimeFilters().forEach( + jrf -> jrfBuilder.append(" RF").append(jrf.getId().asInt())); } String index = ""; if (selectedIndexId != getTable().getBaseIndexId()) { @@ -131,9 +132,14 @@ public String toString() { if (selectedPartitionIds.size() != partitionCount) { partitions = " partitions(" + selectedPartitionIds.size() + "/" + partitionCount + ")"; } + String rfV2 = ""; + if (!runtimeFiltersV2.isEmpty()) { + rfV2 = runtimeFiltersV2.toString(); + } return Utils.toSqlString("PhysicalOlapScan[" + table.getName() + index + partitions + "]" + getGroupIdWithPrefix(), - "stats", statistics, "RFs", builder + "stats", statistics, "JRFs", jrfBuilder, + "RFV2", rfV2 ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java index 07326a8afb4d1c..20c18ab1bfb1b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java @@ -147,4 +147,14 @@ public List computeOutput() { public boolean isDistinct() { return qualifier == Qualifier.DISTINCT; } + + @Override + public String shapeInfo() { + StringBuilder sb = new StringBuilder(); + sb.append(this.getClass().getSimpleName()); + if (!runtimeFiltersV2.isEmpty()) { + sb.append("RFV2: ").append(runtimeFiltersV2); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java index ceda692aa4b01f..a23529a84063a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java @@ -189,7 +189,7 @@ protected String getRuntimeFilterExplainString(boolean isBuildNode, boolean isBr } List filtersStr = new ArrayList<>(); for (RuntimeFilter filter : runtimeFilters) { - filtersStr.add(filter.getExplainString(isBuildNode, isBrief, getExchNodeId())); + filtersStr.add(filter.getExplainString(getExchNodeId())); } return Joiner.on(", ").join(filtersStr) + "\n"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 2f6b75e0e0e0ca..b8614cfc60f043 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -823,10 +823,6 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve if (detailLevel == TExplainLevel.BRIEF) { output.append(detailPrefix).append( String.format("cardinality=%,d", cardinality)).append("\n"); - if (!runtimeFilters.isEmpty()) { - output.append(detailPrefix).append("runtime filters: "); - output.append(getRuntimeFilterExplainString(true, true)); - } return output.toString(); } @@ -844,10 +840,7 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve if (!conjuncts.isEmpty()) { output.append(detailPrefix).append("other predicates: ").append(getExplainString(conjuncts)).append("\n"); } - if (!runtimeFilters.isEmpty()) { - output.append(detailPrefix).append("runtime filters: "); - output.append(getRuntimeFilterExplainString(true)); - } + output.append(detailPrefix).append(String.format("cardinality=%,d", cardinality)).append("\n"); if (outputTupleDesc != null) { output.append(detailPrefix).append("vec output tuple id: ").append(outputTupleDesc.getId()).append("\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java index 9a58b7c27314bd..efbd5ec87e573c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java @@ -244,10 +244,7 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve if (!conjuncts.isEmpty()) { output.append(detailPrefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n"); } - if (!runtimeFilters.isEmpty()) { - output.append(detailPrefix).append("runtime filters: "); - output.append(getRuntimeFilterExplainString(true)); - } + output.append(detailPrefix).append("is output left side only: ").append(isOutputLeftSideOnly).append("\n"); output.append(detailPrefix).append(String.format("cardinality=%,d", cardinality)).append("\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 33965e30a73773..a12e06222f4f40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1335,10 +1335,6 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { if (cardinalityAfterFilter != -1) { output.append("\n").append(prefix).append(String.format("afterFilter=%,d", cardinalityAfterFilter)); } - if (!runtimeFilters.isEmpty()) { - output.append("\n").append(prefix).append("Apply RFs: "); - output.append(getRuntimeFilterExplainString(false, true)); - } if (!conjuncts.isEmpty()) { output.append("\n").append(prefix).append("PREDICATES: ").append(conjuncts.size()).append("\n"); } @@ -1374,10 +1370,6 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { Expr expr = convertConjunctsToAndCompoundPredicate(conjuncts); output.append(prefix).append("PREDICATES: ").append(expr.toSql()).append("\n"); } - if (!runtimeFilters.isEmpty()) { - output.append(prefix).append("runtime filters: "); - output.append(getRuntimeFilterExplainString(false)); - } String selectedPartitions = getSelectedPartitionIds().stream().sorted() .map(id -> olapTable.getPartition(id).getName()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 300dbe0a18a073..a61c2018880a55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -545,6 +545,10 @@ protected final String getExplainString(String rootPrefix, String prefix, TExpla } expBuilder.append("\n"); expBuilder.append(getNodeExplainString(detailPrefix, detailLevel)); + if (!runtimeFilters.isEmpty()) { + expBuilder.append(detailPrefix).append("runtime filters: "); + expBuilder.append(getRuntimeFilterExplainString()); + } if (limit != -1) { expBuilder.append(detailPrefix + "limit: " + limit + "\n"); } @@ -1215,21 +1219,17 @@ public void clearRuntimeFilters() { runtimeFilters.clear(); } - protected String getRuntimeFilterExplainString(boolean isBuildNode, boolean isBrief) { + protected String getRuntimeFilterExplainString() { if (runtimeFilters.isEmpty()) { return ""; } List filtersStr = new ArrayList<>(); for (RuntimeFilter filter : runtimeFilters) { - filtersStr.add(filter.getExplainString(isBuildNode, isBrief, getId())); + filtersStr.add(filter.getExplainString(getId())); } return Joiner.on(", ").join(filtersStr) + "\n"; } - protected String getRuntimeFilterExplainString(boolean isBuildNode) { - return getRuntimeFilterExplainString(isBuildNode, false); - } - /** * If an plan node implements this method, the plan node itself supports project optimization. * @param requiredSlotIdSet: The upper plan node's requirement slot set for the current plan node. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 80497798083dc7..b3f8c8162b3969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -52,6 +52,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Representation of a runtime filter. A runtime filter is generated from @@ -76,7 +77,7 @@ public final class RuntimeFilter { // Expr (lhs of join predicate) from which the targetExprs_ are generated. private final List origTargetExprs; // Runtime filter targets - private final List targets = new ArrayList<>(); + private List targets = new ArrayList<>(); // Slots from base table tuples that have value transfer from the slots // of 'origTargetExpr'. The slots are grouped by tuple id. private final List>> targetSlotsByTid; @@ -119,7 +120,7 @@ public final class RuntimeFilter { */ public static class RuntimeFilterTarget { // Scan node that applies the filter - public ScanNode node; + public PlanNode node; // Expr on which the filter is applied public Expr expr; // Indicates if 'expr' is bound only by partition columns @@ -127,7 +128,7 @@ public static class RuntimeFilterTarget { // Indicates if 'node' is in the same fragment as the join that produces the filter public final boolean isLocalTarget; - public RuntimeFilterTarget(ScanNode targetNode, Expr targetExpr, + public RuntimeFilterTarget(PlanNode targetNode, Expr targetExpr, boolean isBoundByKeyColumns, boolean isLocalTarget) { Preconditions.checkState(targetExpr.isBoundByTupleIds(targetNode.getTupleIds()) || targetNode instanceof CTEScanNode); @@ -137,6 +138,10 @@ public RuntimeFilterTarget(ScanNode targetNode, Expr targetExpr, this.isLocalTarget = isLocalTarget; } + public RuntimeFilterTarget(PlanNode targetNode, Expr targetExpr) { + this(targetNode, targetExpr, false, false); + } + @Override public String toString() { return "Target Id: " + node.getId() + " " @@ -164,6 +169,25 @@ private RuntimeFilter(RuntimeFilterId filterId, PlanNode filterSrcNode, Expr src calculateFilterSize(filterSizeLimits); } + public RuntimeFilter(RuntimeFilterId filterId, + PlanNode filterSrcNode, Expr srcExpr, int exprOrder, + List targets, + TRuntimeFilterType type, + long buildNdvOrRowCount, + TMinMaxRuntimeFilterType tMinMaxRuntimeFilterType) { + this.id = filterId; + this.builderNode = filterSrcNode; + this.srcExpr = srcExpr; + this.exprOrder = exprOrder; + this.targets = ImmutableList.copyOf(targets); + this.runtimeFilterType = type; + this.tMinMaxRuntimeFilterType = tMinMaxRuntimeFilterType; + calculateBloomFilterSize(buildNdvOrRowCount); + // TODO: remove after refactor v1 + origTargetExprs = targets.stream().map(target -> target.expr).collect(Collectors.toList()); + targetSlotsByTid = ImmutableList.of(); + } + private RuntimeFilter(RuntimeFilterId filterId, PlanNode filterSrcNode, Expr srcExpr, int exprOrder, List origTargetExprs, List>> targetSlots, TRuntimeFilterType type, RuntimeFilterGenerator.FilterSizeLimits filterSizeLimits, long buildSizeNdv) { @@ -676,6 +700,18 @@ public void calculateFilterSize(RuntimeFilterGenerator.FilterSizeLimits filterSi filterSizeBytes = Math.min(filterSizeBytes, filterSizeLimits.maxVal); } + public void calculateBloomFilterSize(long buildNdvOrRowCount) { + SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); + if (sessionVariable.useRuntimeFilterDefaultSize) { + filterSizeBytes = sessionVariable.getRuntimeBloomFilterSize(); + } else { + filterSizeBytes = expectRuntimeFilterSize(buildNdvOrRowCount); + expectFilterSizeBytes = filterSizeBytes; + filterSizeBytes = Math.max(filterSizeBytes, sessionVariable.getRuntimeBloomFilterMinSize()); + filterSizeBytes = Math.min(filterSizeBytes, sessionVariable.getRuntimeBloomFilterMaxSize()); + } + } + public static long expectRuntimeFilterSize(long ndv) { double fpp = FeConstants.default_bloom_filter_fpp; int logFilterSize = getMinLogSpaceForBloomFilter(ndv, fpp); @@ -749,22 +785,25 @@ public long getExpectFilterSizeBytes() { return expectFilterSizeBytes; } - public String getExplainString(boolean isBuildNode, boolean isBrief, PlanNodeId targetNodeId) { + public String getExplainString(PlanNodeId nodeId) { StringBuilder filterStr = new StringBuilder(); filterStr.append(getFilterId()); - if (!isBrief) { - filterStr.append("["); - filterStr.append(getTypeDesc()); - filterStr.append("]"); - if (isBuildNode) { - filterStr.append(" <- "); - filterStr.append(getSrcExpr().toSql()); - filterStr.append("(").append(getEstimateNdv()).append("/") - .append(getExpectFilterSizeBytes()).append("/") - .append(getFilterSizeBytes()).append(")"); - } else { + + filterStr.append("["); + filterStr.append(getTypeDesc()); + filterStr.append("]"); + if (getBuilderNode().getId().equals(nodeId)) { + // source side + filterStr.append(" <- "); + filterStr.append(getSrcExpr().toSql()); + filterStr.append("(").append(getEstimateNdv()).append("/") + .append(getExpectFilterSizeBytes()).append("/") + .append(getFilterSizeBytes()).append(")"); + } else { + // target side + if (getTargetExpr(nodeId) != null) { filterStr.append(" -> "); - filterStr.append(getTargetExpr(targetNodeId).toSql()); + filterStr.append(getTargetExpr(nodeId).toSql()); } } return filterStr.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java index 08a98fddbecc10..0afb114db6da89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java @@ -184,7 +184,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { } if (!runtimeFilters.isEmpty()) { output.append(prefix).append("runtime filters: "); - output.append(getRuntimeFilterExplainString(false)); + output.append(getRuntimeFilterExplainString()); } output.append(prefix).append(String.format("cardinality=%s", cardinality)) .append(String.format(", avgRowSize=%s", avgRowSize)).append(String.format(", numNodes=%s", numNodes)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java deleted file mode 100644 index 2b4d868249a365..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java +++ /dev/null @@ -1,711 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.planner; - -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.BaseTableRef; -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.JoinOperator; -import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.TableName; -import org.apache.doris.analysis.TableRef; -import org.apache.doris.analysis.TupleDescriptor; -import org.apache.doris.analysis.TupleId; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.TableTest; -import org.apache.doris.common.UserException; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.VariableMgr; -import org.apache.doris.thrift.TPartitionType; - -import com.google.common.collect.ImmutableList; -import mockit.Expectations; -import mockit.Mocked; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; - -public class RuntimeFilterGeneratorTest { - private Analyzer analyzer; - private PlanFragment testPlanFragment; - private HashJoinNode hashJoinNode; - private OlapScanNode lhsScanNode; - private OlapScanNode rhsScanNode; - @Mocked - private ConnectContext connectContext; - - @Before - public void setUp() throws UserException { - new Expectations() { - { - ConnectContext.get().getSessionVariable(); - minTimes = 0; - result = VariableMgr.newSessionVariable(); - } - }; - Env env = Deencapsulation.newInstance(Env.class); - analyzer = new Analyzer(env, connectContext); - TableRef tableRef = new TableRef(); - Deencapsulation.setField(tableRef, "isAnalyzed", true); - Deencapsulation.setField(tableRef, "joinOp", JoinOperator.INNER_JOIN); - - TupleDescriptor lhsTupleDescriptor = new TupleDescriptor(new TupleId(0)); - lhsScanNode = new OlapScanNode(new PlanNodeId(0), lhsTupleDescriptor, "LEFT SCAN"); - TableName lhsTableName = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, "test_db", - "test_lhs_tbl"); - SlotRef lhsExpr = new SlotRef(lhsTableName, "test_lhs_col"); - SlotDescriptor lhsSlotDescriptor = new SlotDescriptor(new SlotId(0), lhsTupleDescriptor); - Column k1 = new Column("test_lhs_col", PrimitiveType.BIGINT, false); - k1.setIsKey(true); - lhsSlotDescriptor.setColumn(k1); - lhsExpr.setDesc(lhsSlotDescriptor); - OlapTable lhsTable = TableTest.newOlapTable(0, "test_lhs_tbl", 0, ImmutableList.of(k1)); - BaseTableRef lhsTableRef = new BaseTableRef(tableRef, lhsTable, lhsTableName); - lhsTableRef.analyze(analyzer); - - TupleDescriptor rhsTupleDescriptor = new TupleDescriptor(new TupleId(1)); - rhsScanNode = new OlapScanNode(new PlanNodeId(1), rhsTupleDescriptor, "RIGHT SCAN"); - TableName rhsTableName = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, "test_db", - "test_rhs_tbl"); - SlotRef rhsExpr = new SlotRef(rhsTableName, "test_rhs_col"); - SlotDescriptor rhsSlotDescriptor = new SlotDescriptor(new SlotId(1), rhsTupleDescriptor); - Column k2 = new Column("test_rhs_col", PrimitiveType.INT, false); - k2.setIsKey(true); - rhsSlotDescriptor.setColumn(k2); - rhsExpr.setDesc(rhsSlotDescriptor); - OlapTable rhsTable = TableTest.newOlapTable(0, "test_rhs_tbl", 0, ImmutableList.of(k2)); - BaseTableRef rhsTableRef = new BaseTableRef(tableRef, rhsTable, rhsTableName); - rhsTableRef.analyze(analyzer); - - ArrayList testJoinExprs = new ArrayList<>(); - BinaryPredicate eqJoinConjunct = new BinaryPredicate(BinaryPredicate.Operator.EQ, lhsExpr, rhsExpr); - testJoinExprs.add(eqJoinConjunct); - - hashJoinNode = new HashJoinNode(new PlanNodeId(2), lhsScanNode, rhsScanNode, tableRef, testJoinExprs, - new ArrayList<>()); - testPlanFragment = new PlanFragment(new PlanFragmentId(0), hashJoinNode, - new DataPartition(TPartitionType.UNPARTITIONED)); - hashJoinNode.setFragment(testPlanFragment); - lhsScanNode.setFragment(testPlanFragment); - rhsScanNode.setFragment(testPlanFragment); - - new Expectations() { - { - analyzer.getSlotDesc(new SlotId(0)); - result = lhsSlotDescriptor; - analyzer.getSlotDesc(new SlotId(1)); - result = rhsSlotDescriptor; - - ConnectContext.get().getSessionVariable().getRuntimeFiltersMaxNum(); - result = 8; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMaxSize(); - result = 16777216; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMinSize(); - result = 1048576; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterSize(); - result = 2097152; - } - }; - } - - private void clearRuntimeFilterState() { - testPlanFragment.clearRuntimeFilters(); - analyzer.clearAssignedRuntimeFilters(); - hashJoinNode.clearRuntimeFilters(); - lhsScanNode.clearRuntimeFilters(); - } - - @Test - public void testGenerateRuntimeFiltersMode() { - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterMode(); - result = "GLOBAL"; - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 15; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 4); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 4); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 4); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 4); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 4); - String rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString, rfString.contains("RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains("RF001[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF003[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint")); - - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString, rfString.contains("RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains("RF001[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF003[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterMode(); - result = "LOCAL"; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 4); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 4); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 4); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 4); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 4); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString, rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains("RF001[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF003[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString, rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF003[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterMode(); - result = "REMOTE"; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 0); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 0); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 0); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 0); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 0); - Assert.assertEquals(hashJoinNode.getRuntimeFilterExplainString(true), ""); - Assert.assertEquals(lhsScanNode.getRuntimeFilterExplainString(false), ""); - } - - @Test(expected = IllegalStateException.class) - public void testGenerateRuntimeFiltersModeException() { - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 32; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - } - - @Test - public void testGenerateRuntimeFiltersType() { - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 0; - ConnectContext.get().getSessionVariable().getRuntimeFilterMode(); - result = "GLOBAL"; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(hashJoinNode.getRuntimeFilterExplainString(true), ""); - Assert.assertEquals(lhsScanNode.getRuntimeFilterExplainString(false), ""); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 0); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 0); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 0); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 0); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 0); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 1; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(hashJoinNode.getRuntimeFilterExplainString(true), - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)(-1/0/2097152)\n", - hashJoinNode.getRuntimeFilterExplainString(true)); - Assert.assertEquals(lhsScanNode.getRuntimeFilterExplainString(false), - lhsScanNode.getRuntimeFilterExplainString(false), - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`\n"); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 1); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 1); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 1); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 1); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 1); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 2; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(hashJoinNode.getRuntimeFilterExplainString(true), - "RF000[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)(-1/0/2097152)\n"); - Assert.assertEquals(lhsScanNode.getRuntimeFilterExplainString(false), - "RF000[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`\n"); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 1); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 1); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 1); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 1); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 1); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 3; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - String rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString, rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString, rfString.contains( - "RF001[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString, rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString, rfString.contains( - "RF001[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 2); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 2); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 2); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 2); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 2); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 4; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(hashJoinNode.getRuntimeFilterExplainString(true), - "RF000[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)(-1/0/2097152)\n"); - Assert.assertEquals(lhsScanNode.getRuntimeFilterExplainString(false), - "RF000[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`\n"); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 1); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 1); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 1); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 1); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 1); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 5; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 2); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 2); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 2); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 2); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 2); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 6; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 2); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 2); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 2); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 2); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 2); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 7; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 3); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 3); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 3); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 3); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 3); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 8; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(hashJoinNode.getRuntimeFilterExplainString(true), - "RF000[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)(-1/0/2097152)\n"); - Assert.assertEquals(lhsScanNode.getRuntimeFilterExplainString(false), - "RF000[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`\n"); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 1); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 1); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 1); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 1); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 1); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 9; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 2); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 2); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 2); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 2); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 2); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 10; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 2); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 2); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 2); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 2); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 2); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 11; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 3); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 3); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 3); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 3); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 3); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 12; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 2); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 2); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 2); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 2); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 2); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 13; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 3); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 3); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 3); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 3); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 3); - - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 14; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 3); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 3); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 3); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 3); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 3); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 15; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - rfString = hashJoinNode.getRuntimeFilterExplainString(true); - Assert.assertTrue(rfString.contains( - "RF000[in] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - Assert.assertTrue(rfString.contains( - "RF003[in_or_bloom] <- CAST(`test_db`.`test_rhs_tbl`.`test_rhs_col` AS bigint)")); - rfString = lhsScanNode.getRuntimeFilterExplainString(false); - Assert.assertTrue(rfString.contains( - "RF000[in] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF001[bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF002[min_max] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertTrue(rfString.contains( - "RF003[in_or_bloom] -> `test_db`.`test_lhs_tbl`.`test_lhs_col`")); - Assert.assertEquals(testPlanFragment.getTargetRuntimeFilterIds().size(), 4); - Assert.assertEquals(testPlanFragment.getBuilderRuntimeFilterIds().size(), 4); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().size(), 4); - Assert.assertEquals(hashJoinNode.getRuntimeFilters().size(), 4); - Assert.assertEquals(lhsScanNode.getRuntimeFilters().size(), 4); - } - - @Test(expected = IllegalStateException.class) - public void testGenerateRuntimeFiltersTypeExceptionLess() { - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = -1; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - } - - @Test(expected = IllegalStateException.class) - public void testGenerateRuntimeFiltersTypeExceptionMore() { - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 32; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - } - - @Test - public void testGenerateRuntimeFiltersSize() { - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeFilterMode(); - result = "GLOBAL"; - ConnectContext.get().getSessionVariable().getRuntimeFilterType(); - result = 2; - } - }; - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMaxSize(); - result = 16777216; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMinSize(); - result = 1048576; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterSize(); - result = 2097152; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().get(0).toThrift().getBloomFilterSizeBytes(), 2097152); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMaxSize(); - result = 16777216; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMinSize(); - result = 1048576; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterSize(); - result = 1; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().get(0).toThrift().getBloomFilterSizeBytes(), 1048576); - - clearRuntimeFilterState(); - new Expectations() { - { - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMaxSize(); - result = 16777216; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterMinSize(); - result = 1048576; - ConnectContext.get().getSessionVariable().getRuntimeBloomFilterSize(); - result = 999999999; - } - }; - RuntimeFilterGenerator.generateRuntimeFilters(analyzer, hashJoinNode); - Assert.assertEquals(analyzer.getAssignedRuntimeFilter().get(0).toThrift().getBloomFilterSizeBytes(), 16777216); - - // Use ndv and fpp to calculate the minimum space required for bloom filter - Assert.assertEquals(1048576, 1L - << RuntimeFilter.getMinLogSpaceForBloomFilter(1000000, 0.05)); - Assert.assertEquals(1048576, 1L - << RuntimeFilter.getMinLogSpaceForBloomFilter(1000000, 0.1)); - Assert.assertEquals(524288, 1L - << RuntimeFilter.getMinLogSpaceForBloomFilter(1000000, 0.3)); - Assert.assertEquals(8388608, 1L - << RuntimeFilter.getMinLogSpaceForBloomFilter(10000000, 0.1)); - Assert.assertEquals(1024, 1L - << RuntimeFilter.getMinLogSpaceForBloomFilter(1000, 0.1)); - } -} diff --git a/regression-test/data/query_p0/runtimefilterV2/rfv2.out b/regression-test/data/query_p0/runtimefilterV2/rfv2.out new file mode 100644 index 00000000000000..e71fba9f7d7a48 --- /dev/null +++ b/regression-test/data/query_p0/runtimefilterV2/rfv2.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +PhysicalResultSink +--PhysicalDistribute[DistributionSpecGather] +----PhysicalIntersectRFV2: [RF0[IN](a1#0->b1#3), RF1[BLOOM](a1#0->b1#3), RF2[IN](a1#0->c1#6), RF3[BLOOM](a1#0->c1#6)] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalOlapScan[a] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalOlapScan[b]RFV2: [RF0[IN](a1#0->b1#3), RF1[BLOOM](a1#0->b1#3)] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalOlapScan[c]RFV2: [RF2[IN](a1#0->c1#6), RF3[BLOOM](a1#0->c1#6)] + +-- !2 -- +PhysicalResultSink +--PhysicalDistribute[DistributionSpecGather] +----PhysicalIntersectRFV2: [RF0[IN](x#6->cast(b1#3 as BIGINT)), RF1[BLOOM](x#6->cast(b1#3 as BIGINT))] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------PhysicalOlapScan[a] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------PhysicalOlapScan[b]RFV2: [RF0[IN](x#6->cast(b1#3 as BIGINT)), RF1[BLOOM](x#6->cast(b1#3 as BIGINT))] + +-- !3 -- +PhysicalResultSink +--PhysicalDistribute[DistributionSpecGather] +----PhysicalIntersectRFV2: [RF0[IN](x#9->cast(b1#3 as BIGINT)), RF1[BLOOM](x#9->cast(b1#3 as BIGINT)), RF2[IN](x#9->cast(c1#6 as BIGINT)), RF3[BLOOM](x#9->cast(c1#6 as BIGINT))] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------PhysicalOlapScan[a] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------PhysicalOlapScan[b]RFV2: [RF0[IN](x#9->cast(b1#3 as BIGINT)), RF1[BLOOM](x#9->cast(b1#3 as BIGINT))] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------PhysicalOlapScan[c]RFV2: [RF2[IN](x#9->cast(c1#6 as BIGINT)), RF3[BLOOM](x#9->cast(c1#6 as BIGINT))] + diff --git a/regression-test/suites/query_p0/runtimefilterV2/rfv2.groovy b/regression-test/suites/query_p0/runtimefilterV2/rfv2.groovy new file mode 100644 index 00000000000000..fbc08313e1ee78 --- /dev/null +++ b/regression-test/suites/query_p0/runtimefilterV2/rfv2.groovy @@ -0,0 +1,63 @@ +suite("rfv2") { + sql """ + drop table if exists a; + create table a + ( + a1 int, + a2 int, + a3 varchar + ) engine=olap + duplicate key (a1) + distributed by hash(a1) buckets 3 + properties('replication_num'='1'); + + insert into a values(1, 2, 3), (1, 2, 3), (4, 5, 6), (7,8,9),(10, 11, 12); + + drop table if exists b; + create table b + ( + b1 int, + b2 int, + b3 varchar + ) engine=olap + duplicate key (b1) + distributed by hash(b1) buckets 3 + properties('replication_num'='1'); + + insert into b values (1, 2, 3); + + drop table if exists c; + create table c + ( + c1 int, + c2 int, + c3 varchar + ) engine=olap + duplicate key (c1) + distributed by hash(c1) buckets 3 + properties('replication_num'='1'); + + insert into c values (7,8,9); + + set runtime_filter_type=3; + """ + + qt_1 """ + explain shape plan + select * from ((select a1, a2, a3 from a) intersect (select b1, b2, b3 from b) intersect (select c1, c2, c3 from c)) t; + """ + + qt_2 """ + explain shape plan + select * from ((select a1+1 as x, a2, a3 from a) intersect (select b1, b2, b3 from b)) t; + """ + + qt_3 """ + explain shape plan + select * from ( + (select a1+1 as x, a2, a3 from a) + intersect + (select b1, b2, b3 from b intersect select c1, c2, c3 from c) + ) t; + """ +} \ No newline at end of file