Skip to content

Commit 6aa88b2

Browse files
silundongsilundong
committed
[CALCITE-6846] Support basic dphyp join reorder algorithm
1 parent 5a1b15c commit 6aa88b2

File tree

9 files changed

+1291
-0
lines changed

9 files changed

+1291
-0
lines changed

‎core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -816,4 +816,15 @@ private CoreRules() {}
816816
WINDOW_REDUCE_EXPRESSIONS =
817817
ReduceExpressionsRule.WindowReduceExpressionsRule.WindowReduceExpressionsRuleConfig
818818
.DEFAULT.toRule();
819+
820+
/** Rule that flattens a tree of {@link LogicalJoin}s
821+
* into a single {@link HyperGraph} with N inputs. */
822+
public static final JoinToHyperGraphRule JOIN_TO_HYPER_GRAPH =
823+
JoinToHyperGraphRule.Config.DEFAULT.toRule();
824+
825+
/** Rule that re-orders a {@link Join} tree using dphyp algorithm.
826+
*
827+
* @see #JOIN_TO_HYPER_GRAPH */
828+
public static final DphypJoinReorderRule HYPER_GRAPH_OPTIMIZE =
829+
DphypJoinReorderRule.Config.DEFAULT.toRule();
819830
}

‎core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to you under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.calcite.rel.rules;
18+
19+
import org.apache.calcite.plan.RelOptCost;
20+
import org.apache.calcite.rel.RelNode;
21+
import org.apache.calcite.rel.core.JoinRelType;
22+
import org.apache.calcite.rel.metadata.RelMetadataQuery;
23+
import org.apache.calcite.rex.RexNode;
24+
import org.apache.calcite.tools.RelBuilder;
25+
26+
import org.checkerframework.checker.nullness.qual.Nullable;
27+
28+
import java.util.HashMap;
29+
import java.util.List;
30+
31+
/**
32+
* The core process of dphyp enumeration algorithm.
33+
*/
34+
public class DpHyp {
35+
36+
private HyperGraph hyperGraph;
37+
38+
private HashMap<Long, RelNode> dpTable;
39+
40+
private RelBuilder builder;
41+
42+
private RelMetadataQuery mq;
43+
44+
public DpHyp(HyperGraph hyperGraph, RelBuilder builder, RelMetadataQuery relMetadataQuery) {
45+
this.hyperGraph = hyperGraph;
46+
this.dpTable = new HashMap<>();
47+
this.builder = builder;
48+
this.mq = relMetadataQuery;
49+
}
50+
51+
public void startEnumerateJoin() {
52+
int size = hyperGraph.getInputs().size();
53+
for (int i = 0; i < size; i++) {
54+
long singleNode = LongBitmap.newBitmap(i);
55+
dpTable.put(singleNode, hyperGraph.getInput(i));
56+
hyperGraph.initEdgeBitMap(singleNode);
57+
}
58+
59+
// start enumerating from the second to last
60+
for (int i = size - 2; i >= 0; i--) {
61+
long csg = LongBitmap.newBitmap(i);
62+
long forbidden = csg - 1;
63+
emitCsg(csg);
64+
enumerateCsgRec(csg, forbidden);
65+
}
66+
}
67+
68+
private void emitCsg(long csg) {
69+
long forbidden = csg | LongBitmap.getBvBitmap(csg);
70+
long neighbors = hyperGraph.getNeighborBitmap(csg, forbidden);
71+
72+
LongBitmap.ReverseIterator reverseIterator = new LongBitmap.ReverseIterator(neighbors);
73+
for (long cmp : reverseIterator) {
74+
List<HyperEdge> edges = hyperGraph.connectCsgCmp(csg, cmp);
75+
if (!edges.isEmpty()) {
76+
emitCsgCmp(csg, cmp, edges);
77+
}
78+
// forbidden the nodes that smaller than current cmp when extend cmp, e.g.
79+
// neighbors = {t1, t2}, t1 and t2 are connected.
80+
// when extented t2, we will get (t1, t2)
81+
// when extented t1, we will get (t1, t2) repeated
82+
long newForbidden =
83+
(cmp | LongBitmap.getBvBitmap(cmp)) & neighbors;
84+
newForbidden = newForbidden | forbidden;
85+
enumerateCmpRec(csg, cmp, newForbidden);
86+
}
87+
}
88+
89+
private void enumerateCsgRec(long csg, long forbidden) {
90+
long neighbors = hyperGraph.getNeighborBitmap(csg, forbidden);
91+
LongBitmap.SubsetIterator subsetIterator = new LongBitmap.SubsetIterator(neighbors);
92+
for (long subNeighbor : subsetIterator) {
93+
hyperGraph.updateEdgesForUnion(csg, subNeighbor);
94+
long newCsg = csg | subNeighbor;
95+
if (dpTable.containsKey(newCsg)) {
96+
emitCsg(newCsg);
97+
}
98+
}
99+
long newForbidden = forbidden | neighbors;
100+
subsetIterator.reset();
101+
for (long subNeighbor : subsetIterator) {
102+
long newCsg = csg | subNeighbor;
103+
enumerateCsgRec(newCsg, newForbidden);
104+
}
105+
}
106+
107+
private void enumerateCmpRec(long csg, long cmp, long forbidden) {
108+
long neighbors = hyperGraph.getNeighborBitmap(cmp, forbidden);
109+
LongBitmap.SubsetIterator subsetIterator = new LongBitmap.SubsetIterator(neighbors);
110+
for (long subNeighbor : subsetIterator) {
111+
long newCmp = cmp | subNeighbor;
112+
hyperGraph.updateEdgesForUnion(cmp, subNeighbor);
113+
if (dpTable.containsKey(newCmp)) {
114+
List<HyperEdge> edges = hyperGraph.connectCsgCmp(csg, newCmp);
115+
if (!edges.isEmpty()) {
116+
emitCsgCmp(csg, newCmp, edges);
117+
}
118+
}
119+
}
120+
long newForbidden = forbidden | neighbors;
121+
subsetIterator.reset();
122+
for (long subNeighbor : subsetIterator) {
123+
long newCmp = cmp | subNeighbor;
124+
enumerateCmpRec(csg, newCmp, newForbidden);
125+
}
126+
}
127+
128+
private void emitCsgCmp(long csg, long cmp, List<HyperEdge> edges) {
129+
RelNode child1 = dpTable.get(csg);
130+
RelNode child2 = dpTable.get(cmp);
131+
if (child1 == null || child2 == null) {
132+
throw new IllegalArgumentException(
133+
"csg and cmp were not enumerated in the previous dp process");
134+
}
135+
136+
JoinRelType joinType = hyperGraph.extractJoinType(edges);
137+
if (joinType == null) {
138+
return;
139+
}
140+
RexNode joinCond1 = hyperGraph.extractJoinCond(child1, child2, edges);
141+
RelNode newPlan1 = builder
142+
.push(child1)
143+
.push(child2)
144+
.join(joinType, joinCond1)
145+
.build();
146+
147+
// swap left and right
148+
RexNode joinCond2 = hyperGraph.extractJoinCond(child2, child1, edges);
149+
RelNode newPlan2 = builder
150+
.push(child2)
151+
.push(child1)
152+
.join(joinType, joinCond2)
153+
.build();
154+
RelNode winPlan = chooseBetterPlan(newPlan1, newPlan2);
155+
156+
RelNode oriPlan = dpTable.get(csg | cmp);
157+
if (oriPlan != null) {
158+
winPlan = chooseBetterPlan(winPlan, oriPlan);
159+
}
160+
dpTable.put(csg | cmp, winPlan);
161+
}
162+
163+
public @Nullable RelNode getBestPlan() {
164+
int size = hyperGraph.getInputs().size();
165+
long wholeGraph = LongBitmap.newBitmapBetween(0, size);
166+
return dpTable.get(wholeGraph);
167+
}
168+
169+
private RelNode chooseBetterPlan(RelNode plan1, RelNode plan2) {
170+
RelOptCost cost1 = mq.getCumulativeCost(plan1);
171+
RelOptCost cost2 = mq.getCumulativeCost(plan2);
172+
if (cost1 != null && cost2 != null) {
173+
return cost1.isLt(cost2) ? plan1 : plan2;
174+
} else if (cost1 != null) {
175+
return plan1;
176+
} else {
177+
return plan2;
178+
}
179+
}
180+
181+
}

‎core/src/main/java/org/apache/calcite/rel/rules/DphypJoinReorderRule.java

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to you under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.calcite.rel.rules;
18+
19+
import org.apache.calcite.plan.RelOptRuleCall;
20+
import org.apache.calcite.plan.RelRule;
21+
import org.apache.calcite.rel.RelNode;
22+
import org.apache.calcite.rel.core.Join;
23+
import org.apache.calcite.rex.RexBuilder;
24+
import org.apache.calcite.rex.RexNode;
25+
import org.apache.calcite.tools.RelBuilder;
26+
27+
import org.immutables.value.Value;
28+
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
32+
/** Rule that re-orders a {@link Join} tree using dphyp algorithm.
33+
*
34+
* @see CoreRules#HYPER_GRAPH_OPTIMIZE */
35+
@Value.Enclosing
36+
public class DphypJoinReorderRule
37+
extends RelRule<DphypJoinReorderRule.Config>
38+
implements TransformationRule {
39+
40+
protected DphypJoinReorderRule(Config config) {
41+
super(config);
42+
}
43+
44+
@Override public void onMatch(RelOptRuleCall call) {
45+
HyperGraph hyperGraph = call.rel(0);
46+
RelBuilder relBuilder = call.builder();
47+
// make all field name unique and convert the
48+
// HyperEdge condition from RexInputRef to RexInputFieldName
49+
hyperGraph.convertHyperEdgeCond();
50+
51+
// enumerate by Dphyp
52+
DpHyp dpHyp = new DpHyp(hyperGraph, relBuilder, call.getMetadataQuery());
53+
dpHyp.startEnumerateJoin();
54+
RelNode orderedJoin = dpHyp.getBestPlan();
55+
if (orderedJoin == null) {
56+
return;
57+
}
58+
59+
// permute field to origin order
60+
List<String> oriNames = hyperGraph.getRowType().getFieldNames();
61+
List<String> newNames = orderedJoin.getRowType().getFieldNames();
62+
List<RexNode> projects = new ArrayList<>();
63+
RexBuilder rexBuilder = hyperGraph.getCluster().getRexBuilder();
64+
for (String oriName : oriNames) {
65+
projects.add(rexBuilder.makeInputRef(orderedJoin, newNames.indexOf(oriName)));
66+
}
67+
68+
RelNode result = call.builder()
69+
.push(orderedJoin)
70+
.project(projects)
71+
.build();
72+
call.transformTo(result);
73+
}
74+
75+
/** Rule configuration. */
76+
@Value.Immutable
77+
public interface Config extends RelRule.Config {
78+
Config DEFAULT = ImmutableDphypJoinReorderRule.Config.of()
79+
.withOperandSupplier(b1 ->
80+
b1.operand(HyperGraph.class).anyInputs());
81+
82+
@Override default DphypJoinReorderRule toRule() {
83+
return new DphypJoinReorderRule(this);
84+
}
85+
}
86+
}

‎core/src/main/java/org/apache/calcite/rel/rules/HyperEdge.java

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to you under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.calcite.rel.rules;
18+
19+
import org.apache.calcite.rel.core.JoinRelType;
20+
import org.apache.calcite.rex.RexNode;
21+
22+
/**
23+
* Edge in HyperGraph, that represents a join predicate.
24+
*/
25+
public class HyperEdge {
26+
27+
private long leftNodeBits;
28+
29+
private long rightNodeBits;
30+
31+
private JoinRelType joinType;
32+
33+
private RexNode condition;
34+
35+
public HyperEdge(long leftNodeBits, long rightNodeBits, JoinRelType joinType, RexNode condition) {
36+
this.leftNodeBits = leftNodeBits;
37+
this.rightNodeBits = rightNodeBits;
38+
this.joinType = joinType;
39+
this.condition = condition;
40+
}
41+
42+
public long getNodeBitmap() {
43+
return leftNodeBits | rightNodeBits;
44+
}
45+
46+
public long getLeftNodeBitmap() {
47+
return leftNodeBits;
48+
}
49+
50+
public long getRightNodeBitmap() {
51+
return rightNodeBits;
52+
}
53+
54+
// hyperedge (u, v) is simple if |u| = |v| = 1
55+
public boolean isSimple() {
56+
boolean leftSimple = (leftNodeBits & (leftNodeBits - 1)) == 0;
57+
boolean rightSimple = (rightNodeBits & (rightNodeBits - 1)) == 0;
58+
return leftSimple && rightSimple;
59+
}
60+
61+
public JoinRelType getJoinType() {
62+
return joinType;
63+
}
64+
65+
public RexNode getCondition() {
66+
return condition;
67+
}
68+
69+
@Override public String toString() {
70+
StringBuilder sb = new StringBuilder();
71+
sb.append(LongBitmap.printBitmap(leftNodeBits))
72+
.append("——").append(joinType).append("——")
73+
.append(LongBitmap.printBitmap(rightNodeBits));
74+
return sb.toString();
75+
}
76+
77+
// before starting dphyp, replace RexInputRef to RexInputFieldName
78+
public void replaceCondition(RexNode fieldNameCond) {
79+
this.condition = fieldNameCond;
80+
}
81+
82+
}

0 commit comments

Comments
 (0)