Skip to content

Commit f39ec64

Browse files
ilaila
committed
Fix nested DISTINCT projection classification
1 parent 13c4cd5 commit f39ec64

4 files changed

Lines changed: 120 additions & 9 deletions

File tree

‎benchmark/queries/tpcc/ducklake_0319.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
-- {"operators": "DISTINCT,SUBQUERY", "complexity": "medium", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER", "ducklake": true, "openivm_verified": true}
2+
SELECT payment_count % 2 AS parity
3+
FROM (
4+
SELECT distinct_payment_count AS payment_count
5+
FROM (
6+
SELECT DISTINCT C_PAYMENT_CNT AS distinct_payment_count
7+
FROM dl.CUSTOMER
8+
)
9+
);

‎src/core/ivm_view_classifier.cpp‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,26 @@ static void BuildUpdateSemantics(DeltaViewModel &model, const PlanAnalysis &anal
391391
}
392392
}
393393

394+
static bool IsTransparentDistinctWrapper(LogicalOperatorType type) {
395+
return type == LogicalOperatorType::LOGICAL_CREATE_TABLE || type == LogicalOperatorType::LOGICAL_FILTER ||
396+
type == LogicalOperatorType::LOGICAL_ORDER_BY || type == LogicalOperatorType::LOGICAL_LIMIT ||
397+
type == LogicalOperatorType::LOGICAL_TOP_N;
398+
}
399+
400+
static bool HasOuterProjectionOverDistinct(const CreateMVPlanFacts &facts) {
401+
auto *node = facts.root;
402+
bool found_projection = false;
403+
while (node && node->children.size() == 1) {
404+
if (node->type == LogicalOperatorType::LOGICAL_PROJECTION) {
405+
found_projection = true;
406+
} else if (!IsTransparentDistinctWrapper(node->type)) {
407+
break;
408+
}
409+
node = node->children[0].get();
410+
}
411+
return found_projection && node && node->type == LogicalOperatorType::LOGICAL_DISTINCT;
412+
}
413+
394414
static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &facts, const vector<string> &output_names,
395415
idx_t visible_output_count) {
396416
const auto &analysis = facts.analysis;
@@ -411,9 +431,14 @@ static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &fa
411431
model.group_columns.size());
412432
}
413433
} else if (model.distinct_at_top) {
414-
model.group_columns = analysis.aggregate_columns;
415-
} else if (analysis.found_distinct && analysis.aggregate_columns.empty()) {
416-
model.group_columns = analysis.aggregate_columns;
434+
AddVisibleGroupNames(model.group_columns, output_names);
435+
} else if (analysis.found_distinct && !analysis.found_aggregation && HasOuterProjectionOverDistinct(facts)) {
436+
AddVisibleGroupNames(model.group_columns, output_names);
437+
if (!model.group_columns.empty()) {
438+
AddUnique(model.strategy_reasons, DeltaStrategyReason::INNER_DISTINCT_PROJECTION_RECOMPUTE);
439+
OPENIVM_DEBUG_PRINT("[CREATE MV] Inner DISTINCT below an outer projection -- using "
440+
"GROUP_RECOMPUTE\n");
441+
}
417442
} else if (group_count > 0 && group_index != DConstants::INVALID_INDEX) {
418443
model.group_columns = DeriveGroupColumnNames(facts, group_index, group_count, output_names);
419444
}
@@ -666,6 +691,8 @@ const char *DeltaStrategyReasonName(DeltaStrategyReason reason) {
666691
return "OUTER_JOIN_AGGREGATE_RECOMPUTE";
667692
case DeltaStrategyReason::OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE:
668693
return "OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE";
694+
case DeltaStrategyReason::INNER_DISTINCT_PROJECTION_RECOMPUTE:
695+
return "INNER_DISTINCT_PROJECTION_RECOMPUTE";
669696
default:
670697
return "UNKNOWN";
671698
}
@@ -861,11 +888,10 @@ bool IsDistinctAtTop(const CreateMVPlanFacts &facts, const vector<string> &outpu
861888
}
862889

863890
auto *node = facts.root;
864-
while (node && node->children.size() == 1 &&
865-
(node->type == LogicalOperatorType::LOGICAL_CREATE_TABLE ||
866-
node->type == LogicalOperatorType::LOGICAL_PROJECTION ||
867-
node->type == LogicalOperatorType::LOGICAL_FILTER || node->type == LogicalOperatorType::LOGICAL_ORDER_BY ||
868-
node->type == LogicalOperatorType::LOGICAL_LIMIT || node->type == LogicalOperatorType::LOGICAL_TOP_N)) {
891+
while (node && node->children.size() == 1) {
892+
if (!IsTransparentDistinctWrapper(node->type)) {
893+
break;
894+
}
869895
node = node->children[0].get();
870896
}
871897
if (node && node->type == LogicalOperatorType::LOGICAL_DISTINCT) {

‎src/include/core/ivm_view_classifier.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ enum class DeltaStrategyReason {
1818
REPEATED_CTE_AGGREGATE_GROUP_FALLBACK,
1919
SEMI_ANTI_AGGREGATE_GROUP_FALLBACK,
2020
OUTER_JOIN_AGGREGATE_RECOMPUTE,
21-
OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE
21+
OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE,
22+
INNER_DISTINCT_PROJECTION_RECOMPUTE
2223
};
2324

2425
enum class DeltaModelFeature {

‎test/sql/distinct.test‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,81 @@ SELECT COUNT(*) FROM (SELECT * FROM distinct_colors EXCEPT ALL SELECT DISTINCT c
4444
----
4545
0
4646

47+
# An inner DISTINCT remains inner when outer derived-table projections rename
48+
# its key and then transform it. The computed projection has bag semantics:
49+
# distinct source keys may map to duplicate parity rows.
50+
statement ok
51+
CREATE TABLE nested_distinct_input (id INT, x INT);
52+
53+
statement ok
54+
INSERT INTO nested_distinct_input VALUES (1, 1), (2, 1), (3, 2), (4, 3);
55+
56+
statement ok
57+
CREATE MATERIALIZED VIEW nested_distinct_projection AS
58+
SELECT y % 2 AS parity
59+
FROM (
60+
SELECT x AS y
61+
FROM (
62+
SELECT DISTINCT x
63+
FROM nested_distinct_input
64+
)
65+
);
66+
67+
query I
68+
SELECT type FROM openivm_views WHERE view_name = 'nested_distinct_projection';
69+
----
70+
6
71+
72+
statement ok
73+
UPDATE nested_distinct_input SET x = 4 WHERE id = 4;
74+
75+
statement ok
76+
DELETE FROM nested_distinct_input WHERE id = 1;
77+
78+
statement ok
79+
INSERT INTO nested_distinct_input VALUES (5, 5), (6, 2);
80+
81+
statement ok
82+
UPDATE nested_distinct_input SET x = 6 WHERE id = 6;
83+
84+
statement ok
85+
DELETE FROM nested_distinct_input WHERE id = 5;
86+
87+
statement ok
88+
PRAGMA refresh('nested_distinct_projection');
89+
90+
query I
91+
SELECT COUNT(*) FROM (
92+
SELECT * FROM nested_distinct_projection
93+
EXCEPT ALL
94+
SELECT y % 2
95+
FROM (
96+
SELECT x AS y
97+
FROM (
98+
SELECT DISTINCT x
99+
FROM nested_distinct_input
100+
)
101+
)
102+
);
103+
----
104+
0
105+
106+
query I
107+
SELECT COUNT(*) FROM (
108+
SELECT y % 2
109+
FROM (
110+
SELECT x AS y
111+
FROM (
112+
SELECT DISTINCT x
113+
FROM nested_distinct_input
114+
)
115+
)
116+
EXCEPT ALL
117+
SELECT * FROM nested_distinct_projection
118+
);
119+
----
120+
0
121+
47122
# Insert new value
48123
statement ok
49124
INSERT INTO colors VALUES (7, 'yellow');

0 commit comments

Comments
 (0)