Skip to content

Commit 9e83857

Browse files
staticlibsstaticlibs
committed
Use filter pushdowns from dbconnector
1 parent aac0911 commit 9e83857

4 files changed

Lines changed: 23 additions & 218 deletions

File tree

‎database-connector‎

‎src/dbconnector/CMakeLists.txt‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,11 @@ set(POSTGRES_DBCONNECTOR_PATH
44
add_library(
55
postgres_ext_dbconnector OBJECT
66
${POSTGRES_DBCONNECTOR_PATH}/src/query/query_writer.cpp
7-
# ${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_pushdown.cpp
8-
# ${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_util.cpp
9-
# ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/aggregate_optimizer.cpp
10-
# ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/optimizer_util.cpp
11-
# ${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/order_by_and_limit_optimizer.cpp
12-
)
7+
${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_pushdown.cpp
8+
${POSTGRES_DBCONNECTOR_PATH}/src/table_scan/filter_util.cpp
9+
${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/aggregate_optimizer.cpp
10+
${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/optimizer_util.cpp
11+
${POSTGRES_DBCONNECTOR_PATH}/src/optimizer/order_by_and_limit_optimizer.cpp)
1312

1413
set(ALL_OBJECT_FILES
1514
${ALL_OBJECT_FILES} $<TARGET_OBJECTS:postgres_ext_dbconnector>

‎src/include/postgres_filter_pushdown.hpp‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
#pragma once
1010

1111
#include "duckdb/planner/table_filter_set.hpp"
12-
#include "duckdb/planner/filter/expression_filter.hpp"
1312

1413
namespace duckdb {
1514

@@ -19,15 +18,8 @@ class PostgresFilterPushdown {
1918
const vector<string> &names);
2019

2120
private:
22-
static string TransformCTIDLiteral(const Value &val);
23-
static string TransformConstantFilter(const string &column_name, ExpressionType comparison_type,
24-
const Value &constant, column_t column_id);
25-
static string TransformFilter(const string &column_name, const TableFilter &filter, column_t column_id);
26-
static string TransformExpression(const string &column_name, const Expression &expr, column_t column_id);
27-
static string TransformExpressionSubject(const string &column_name, const Expression &expr);
28-
static string TransformComparison(ExpressionType type);
29-
static string CreateExpression(const string &column_name, const vector<unique_ptr<Expression>> &filters, string op,
30-
column_t column_id);
21+
// TODO
22+
// static string TransformCTIDLiteral(const Value &val);
3123
};
3224

3325
} // namespace duckdb

‎src/postgres_filter_pushdown.cpp‎

Lines changed: 15 additions & 201 deletions
Original file line numberDiff line numberDiff line change
@@ -10,210 +10,16 @@
1010
#include "duckdb/planner/filter/table_filter_functions.hpp"
1111
#include "duckdb/common/enum_util.hpp"
1212

13+
#include "dbconnector/table_scan/filter_pushdown.hpp"
14+
#include "dbconnector/table_scan/filter_util.hpp"
15+
1316
#include "postgres_utils.hpp"
1417

1518
namespace duckdb {
1619

17-
string PostgresFilterPushdown::CreateExpression(const string &column_name,
18-
const vector<unique_ptr<Expression>> &filters, string op,
19-
column_t column_id) {
20-
vector<string> filter_entries;
21-
for (auto &filter : filters) {
22-
auto filter_str = TransformExpression(column_name, *filter, column_id);
23-
if (!filter_str.empty()) {
24-
filter_entries.push_back(std::move(filter_str));
25-
}
26-
}
27-
if (filter_entries.empty()) {
28-
return string();
29-
}
30-
return "(" + StringUtil::Join(filter_entries, " " + op + " ") + ")";
31-
}
32-
33-
string PostgresFilterPushdown::TransformComparison(ExpressionType type) {
34-
switch (type) {
35-
case ExpressionType::COMPARE_EQUAL:
36-
return "=";
37-
case ExpressionType::COMPARE_NOTEQUAL:
38-
return "<>";
39-
case ExpressionType::COMPARE_LESSTHAN:
40-
return "<";
41-
case ExpressionType::COMPARE_GREATERTHAN:
42-
return ">";
43-
case ExpressionType::COMPARE_LESSTHANOREQUALTO:
44-
return "<=";
45-
case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
46-
return ">=";
47-
default:
48-
throw NotImplementedException("Unsupported expression type");
49-
}
50-
}
51-
52-
string TransformBlob(const string &val) {
53-
char const HEX_DIGITS[] = "0123456789ABCDEF";
54-
55-
string result = "'\\x";
56-
for (idx_t i = 0; i < val.size(); i++) {
57-
uint8_t byte_val = static_cast<uint8_t>(val[i]);
58-
result += HEX_DIGITS[(byte_val >> 4) & 0xf];
59-
result += HEX_DIGITS[byte_val & 0xf];
60-
}
61-
result += "'::BYTEA";
62-
return result;
63-
}
64-
65-
string TransformLiteral(const Value &val) {
66-
switch (val.type().id()) {
67-
case LogicalTypeId::BLOB:
68-
return TransformBlob(StringValue::Get(val));
69-
default:
70-
return PostgresUtils::WriteLiteral(val.ToString());
71-
}
72-
}
73-
74-
string PostgresFilterPushdown::TransformCTIDLiteral(const Value &constant) {
75-
throw InternalException("FIXME: transform ctid literal");
76-
}
77-
78-
string PostgresFilterPushdown::TransformConstantFilter(const string &column_name, ExpressionType comparison_type,
79-
const Value &constant, column_t column_id) {
80-
string constant_string;
81-
if (IsVirtualColumn(column_id)) {
82-
return "FALSE";
83-
} else {
84-
constant_string = TransformLiteral(constant);
85-
}
86-
auto operator_string = TransformComparison(comparison_type);
87-
string comparison = StringUtil::Format("%s %s %s", column_name, operator_string, constant_string);
88-
if (constant.type().id() == LogicalTypeId::VARCHAR) {
89-
comparison += " COLLATE \"C\"";
90-
}
91-
return comparison;
92-
}
93-
94-
string PostgresFilterPushdown::TransformExpressionSubject(const string &column_name, const Expression &expr) {
95-
switch (expr.GetExpressionClass()) {
96-
case ExpressionClass::BOUND_REF:
97-
case ExpressionClass::BOUND_COLUMN_REF:
98-
return column_name;
99-
case ExpressionClass::BOUND_FUNCTION: {
100-
auto &func = expr.Cast<BoundFunctionExpression>();
101-
idx_t child_idx;
102-
if (!TryGetStructExtractChildIndex(func, child_idx) || func.GetChildren().empty()) {
103-
return string();
104-
}
105-
auto parent_name = TransformExpressionSubject(column_name, *func.GetChildren()[0]);
106-
if (parent_name.empty()) {
107-
return string();
108-
}
109-
auto &struct_type = func.GetChildren()[0]->GetReturnType();
110-
if (struct_type.id() != LogicalTypeId::STRUCT || StructType::IsUnnamed(struct_type)) {
111-
return string();
112-
}
113-
auto child_name = PostgresUtils::WriteIdentifier(StructType::GetChildName(struct_type, child_idx));
114-
return "(" + parent_name + ")." + child_name;
115-
}
116-
default:
117-
return string();
118-
}
119-
}
120-
121-
string PostgresFilterPushdown::TransformExpression(const string &column_name, const Expression &expr,
122-
column_t column_id) {
123-
if (BoundComparisonExpression::IsComparison(expr)) {
124-
auto &comparison = expr.Cast<BoundFunctionExpression>();
125-
auto comparison_type = comparison.GetExpressionType();
126-
auto &left = BoundComparisonExpression::Left(comparison);
127-
auto &right = BoundComparisonExpression::Right(comparison);
128-
auto subject = TransformExpressionSubject(column_name, left);
129-
const Value *constant = nullptr;
130-
if (!subject.empty() && right.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT) {
131-
constant = &right.Cast<BoundConstantExpression>().GetValue();
132-
} else {
133-
subject = TransformExpressionSubject(column_name, right);
134-
if (!subject.empty() && left.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT) {
135-
constant = &left.Cast<BoundConstantExpression>().GetValue();
136-
comparison_type = FlipComparisonExpression(comparison_type);
137-
}
138-
}
139-
if (!constant || subject.empty()) {
140-
return string();
141-
}
142-
return TransformConstantFilter(subject, comparison_type, *constant, column_id);
143-
}
144-
145-
switch (expr.GetExpressionClass()) {
146-
case ExpressionClass::BOUND_CONJUNCTION: {
147-
auto &conjunction = expr.Cast<BoundConjunctionExpression>();
148-
switch (conjunction.GetExpressionType()) {
149-
case ExpressionType::CONJUNCTION_AND:
150-
return CreateExpression(column_name, conjunction.GetChildren(), "AND", column_id);
151-
case ExpressionType::CONJUNCTION_OR:
152-
return CreateExpression(column_name, conjunction.GetChildren(), "OR", column_id);
153-
default:
154-
return string();
155-
}
156-
}
157-
case ExpressionClass::BOUND_OPERATOR: {
158-
auto &op = expr.Cast<BoundOperatorExpression>();
159-
auto subject =
160-
op.GetChildren().empty() ? string() : TransformExpressionSubject(column_name, *op.GetChildren()[0]);
161-
switch (op.GetExpressionType()) {
162-
case ExpressionType::OPERATOR_IS_NULL:
163-
return !subject.empty() ? subject + " IS NULL" : string();
164-
case ExpressionType::OPERATOR_IS_NOT_NULL:
165-
return !subject.empty() ? subject + " IS NOT NULL" : string();
166-
case ExpressionType::COMPARE_IN: {
167-
if (subject.empty()) {
168-
return string();
169-
}
170-
string in_list;
171-
for (idx_t i = 1; i < op.GetChildren().size(); i++) {
172-
if (op.GetChildren()[i]->GetExpressionClass() != ExpressionClass::BOUND_CONSTANT) {
173-
return string();
174-
}
175-
if (!in_list.empty()) {
176-
in_list += ", ";
177-
}
178-
auto &constant = op.GetChildren()[i]->Cast<BoundConstantExpression>().GetValue();
179-
in_list += IsVirtualColumn(column_id) ? "FALSE" : TransformLiteral(constant);
180-
}
181-
return IsVirtualColumn(column_id) ? "FALSE" : subject + " IN (" + in_list + ")";
182-
}
183-
default:
184-
return string();
185-
}
186-
}
187-
case ExpressionClass::BOUND_FUNCTION: {
188-
auto &func = expr.Cast<BoundFunctionExpression>();
189-
if (func.Function().GetName() == OptionalFilterScalarFun::NAME && func.BindInfo()) {
190-
auto &data = func.BindInfo()->Cast<OptionalFilterFunctionData>();
191-
return data.child_filter_expr ? TransformExpression(column_name, *data.child_filter_expr, column_id)
192-
: string();
193-
}
194-
if (func.Function().GetName() == SelectivityOptionalFilterScalarFun::NAME && func.BindInfo()) {
195-
auto &data = func.BindInfo()->Cast<SelectivityOptionalFilterFunctionData>();
196-
return data.child_filter_expr ? TransformExpression(column_name, *data.child_filter_expr, column_id)
197-
: string();
198-
}
199-
if (func.Function().GetName() == DynamicFilterScalarFun::NAME) {
200-
return string();
201-
}
202-
return string();
203-
}
204-
default:
205-
throw InternalException("Unsupported table filter type");
206-
}
207-
}
208-
209-
string PostgresFilterPushdown::TransformFilter(const string &column_name, const TableFilter &filter,
210-
column_t column_id) {
211-
auto &expr_filter = ExpressionFilter::GetExpressionFilter(filter, "PostgresFilterPushdown::TransformFilter");
212-
return TransformExpression(column_name, *expr_filter.expr, column_id);
213-
}
214-
21520
string PostgresFilterPushdown::TransformFilters(const vector<column_t> &column_ids,
21621
optional_ptr<TableFilterSet> filters, const vector<string> &names) {
22+
using namespace dbconnector;
21723
if (!filters || !filters->HasFilters()) {
21824
// no filters
21925
return string();
@@ -225,13 +31,21 @@ string PostgresFilterPushdown::TransformFilters(const vector<column_t> &column_i
22531
if (IsVirtualColumn(column_id)) {
22632
column_name = "ctid";
22733
} else {
228-
column_name = PostgresUtils::WriteIdentifier(names[column_id]);
34+
column_name = names[column_id];
22935
}
23036
auto &filter = entry.Filter();
231-
auto filter_text = TransformFilter(column_name, filter, column_id);
37+
auto config = table_scan::FilterPushdown::CreateConfig('"', '\'', query::QuoteEscapeStyle::DOUBLE_QUOTE, "'\\x",
38+
"::BYTEA");
39+
auto filter_text = table_scan::FilterPushdown::TransformFilter(config, column_name, filter, column_id);
23240

23341
if (filter_text.empty()) {
234-
continue;
42+
if (table_scan::FilterUtil::IsInternalFilter(filter)) {
43+
continue;
44+
}
45+
throw NotImplementedException(
46+
"Unsupported filter pushdown, use 'pg_experimental_filter_pushdown=FALSE' to disable pushdowns."
47+
" Problematic filter: \"%s\"",
48+
table_scan::FilterUtil::ToString(filter));
23549
}
23650
if (!result.empty()) {
23751
result += " AND ";

0 commit comments

Comments
 (0)