Skip to content

Commit 2f3389a

Browse files
phillipleblancphillipleblanc
authored
Upgrade DuckDB to 1.3.2 (#391)
1 parent c2307be commit 2f3389a

13 files changed

Lines changed: 640 additions & 20 deletions

‎Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ datafusion-expr = { version = "47", optional = true }
2525
datafusion-physical-expr = { version = "47", optional = true }
2626
datafusion-physical-plan = { version = "47", optional = true }
2727
datafusion-proto = { version = "47", optional = true }
28-
duckdb = { version = "1.1.3", features = [
28+
duckdb = { version = "1.3.2", features = [
2929
"bundled",
3030
"r2d2",
3131
"vtab",
@@ -103,7 +103,7 @@ postgres-federation = ["postgres"]
103103

104104
[patch.crates-io]
105105
datafusion-federation = { git = "https://github.com/spiceai/datafusion-federation.git", rev = "9db74a4b360df6be1bb554c59a474a2fd4bfb7e9" } # spiceai-47
106-
duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "69ae7518ee093a1b070e9e4e6f011ef353431086" } # spiceai-1.1.3-backported-arrow-55
106+
duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "b3547f0c1b37030b623b1e03fcaea0e4e2bb753e" } # spiceai-1.3.2
107107

108108
datafusion = { git = "https://github.com/spiceai/datafusion.git", rev = "b5c62f29d2c70c5331ff50015b67b5e1cafcd578" } # spiceai-47
109109
datafusion-expr = { git = "https://github.com/spiceai/datafusion.git", rev = "b5c62f29d2c70c5331ff50015b67b5e1cafcd578" } # spiceai-47

‎src/duckdb/creator.rs‎

Lines changed: 171 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -709,7 +709,7 @@ impl TableManager {
709709
/// If there is an index defined on a single column, that column should come first in the list.
710710
/// Multi-column indexes are not considered for ordering.
711711
pub(crate) fn order_columns_by_index(&self, columns: Vec<String>) -> Vec<String> {
712-
let mut indexed_columns = Vec::new();
712+
let mut ordered_columns = Vec::new();
713713
let mut non_indexed_columns = Vec::new();
714714

715715
// Get single-column indexes
@@ -730,15 +730,15 @@ impl TableManager {
730730
// Separate columns into indexed and non-indexed
731731
for column in columns {
732732
if single_column_indexes.contains(&column) {
733-
indexed_columns.push(column);
733+
ordered_columns.push(column);
734734
} else {
735735
non_indexed_columns.push(column);
736736
}
737737
}
738738

739739
// Return indexed columns first, then non-indexed columns
740-
indexed_columns.extend(non_indexed_columns);
741-
indexed_columns
740+
ordered_columns.extend(non_indexed_columns);
741+
ordered_columns
742742
}
743743
}
744744

@@ -2105,8 +2105,8 @@ pub(crate) mod tests {
21052105
let pool = get_mem_duckdb();
21062106

21072107
let schema = Arc::new(arrow::datatypes::Schema::new(vec![
2108-
arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int64, false),
21092108
arrow::datatypes::Field::new("name", arrow::datatypes::DataType::Utf8, false),
2109+
arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int64, false),
21102110
arrow::datatypes::Field::new("age", arrow::datatypes::DataType::Int32, true),
21112111
arrow::datatypes::Field::new("status", arrow::datatypes::DataType::Utf8, false),
21122112
]));
@@ -2143,7 +2143,7 @@ pub(crate) mod tests {
21432143

21442144
tx.execute(
21452145
&format!(
2146-
r#"INSERT INTO "{table_name}" VALUES (1, 'Alice', 30, 'active'), (2, 'Bob', 25, 'inactive'), (3, 'Charlie', 35, 'active')"#,
2146+
r#"INSERT INTO "{table_name}" VALUES ('Alice', 1, 30, 'active'), ('Bob', 2, 25, 'inactive'), ('Charlie', 3, 35, 'active')"#,
21472147
table_name = table_manager.table_name()
21482148
),
21492149
[],
@@ -2152,22 +2152,175 @@ pub(crate) mod tests {
21522152

21532153
table_manager.create_view(&tx).expect("to create view");
21542154

2155-
let explain_query = format!(
2156-
"EXPLAIN SELECT * FROM {} WHERE id = 1",
2157-
table_definition.name()
2158-
);
2155+
let queries = [
2156+
format!(
2157+
"EXPLAIN ANALYZE SELECT * FROM {} WHERE id = 1",
2158+
table_definition.name()
2159+
),
2160+
format!(
2161+
"EXPLAIN ANALYZE SELECT name FROM {} WHERE id = 1",
2162+
table_definition.name()
2163+
),
2164+
];
21592165

2160-
let mut stmt = tx.prepare(&explain_query).expect("to prepare statement");
2161-
let mut rows = stmt.query([]).expect("to execute query");
2166+
for (idx, query) in queries.iter().enumerate() {
2167+
let mut stmt = tx.prepare(query).expect("to prepare statement");
2168+
let mut rows = stmt.query([]).expect("to execute query");
21622169

2163-
let mut explain_output = Vec::new();
2164-
while let Some(row) = rows.next().expect("to get next row") {
2165-
let line: String = row.get(1).expect("to get explain line");
2166-
explain_output.push(line);
2170+
let mut explain_output = Vec::new();
2171+
while let Some(row) = rows.next().expect("to get next row") {
2172+
let line: String = row.get(1).expect("to get explain line");
2173+
explain_output.push(line);
2174+
}
2175+
2176+
let explain_result = explain_output.join("\n");
2177+
2178+
insta::with_settings!({
2179+
filters => vec![
2180+
(r"Total Time: \d+\.\d+s", "Total Time: replaced"),
2181+
(r"\(\d+\.\d+s\)", "(0.00s)"),
2182+
(r"│__data_test_table_\d+│\n│\s+\d+\s+│", "│__data_test_table_redacted│\n│ redacted │"),
2183+
],
2184+
}, {
2185+
insta::assert_snapshot!(format!("explain_analyze_{idx}"), explain_result);
2186+
});
21672187
}
21682188

2169-
let explain_result = explain_output.join("\n");
2170-
insta::assert_snapshot!(explain_result);
2189+
tx.rollback().expect("should rollback transaction");
2190+
}
2191+
2192+
#[tokio::test]
2193+
async fn test_explain_analyze_with_multiple_indexes_and_view() {
2194+
let _guard = init_tracing(None);
2195+
let pool = get_mem_duckdb();
2196+
2197+
let schema = Arc::new(arrow::datatypes::Schema::new(vec![
2198+
arrow::datatypes::Field::new("name", arrow::datatypes::DataType::Utf8, false),
2199+
arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int64, false),
2200+
arrow::datatypes::Field::new("age", arrow::datatypes::DataType::Int32, true),
2201+
arrow::datatypes::Field::new("status", arrow::datatypes::DataType::Utf8, false),
2202+
]));
2203+
2204+
let table_definition = Arc::new(
2205+
TableDefinition::new(RelationName::new("test_table"), Arc::clone(&schema))
2206+
.with_indexes(vec![
2207+
(
2208+
ColumnReference::try_from("id").expect("valid column ref"),
2209+
IndexType::Enabled,
2210+
),
2211+
(
2212+
ColumnReference::try_from("age").expect("valid column ref"),
2213+
IndexType::Enabled,
2214+
),
2215+
(
2216+
ColumnReference::try_from("status").expect("valid column ref"),
2217+
IndexType::Enabled,
2218+
),
2219+
]),
2220+
);
2221+
2222+
let mut pool_conn = Arc::clone(&pool).connect_sync().expect("to get connection");
2223+
let conn = pool_conn
2224+
.as_any_mut()
2225+
.downcast_mut::<DuckDbConnection>()
2226+
.expect("to downcast to duckdb connection");
2227+
let tx = conn
2228+
.get_underlying_conn_mut()
2229+
.transaction()
2230+
.expect("should begin transaction");
2231+
2232+
let table_manager = TableManager::new(Arc::clone(&table_definition))
2233+
.with_internal(true)
2234+
.expect("to create table manager");
2235+
2236+
table_manager
2237+
.create_table(Arc::clone(&pool), &tx)
2238+
.expect("to create table");
2239+
2240+
table_manager
2241+
.create_indexes(&tx)
2242+
.expect("to create indexes");
2243+
2244+
tx.execute(
2245+
&format!(
2246+
r#"INSERT INTO "{table_name}" VALUES
2247+
('Alice', 1, 30, 'active'),
2248+
('Bob', 2, 25, 'inactive'),
2249+
('Charlie', 3, 35, 'active'),
2250+
('David', 4, 30, 'pending'),
2251+
('Eve', 5, 40, 'active')"#,
2252+
table_name = table_manager.table_name()
2253+
),
2254+
[],
2255+
)
2256+
.expect("to insert test data");
2257+
2258+
table_manager.create_view(&tx).expect("to create view");
2259+
2260+
let queries = [
2261+
// Test index on id column
2262+
format!(
2263+
"EXPLAIN ANALYZE SELECT * FROM {} WHERE id = 1",
2264+
table_definition.name()
2265+
),
2266+
format!(
2267+
"EXPLAIN ANALYZE SELECT name FROM {} WHERE id = 1",
2268+
table_definition.name()
2269+
),
2270+
format!(
2271+
"EXPLAIN ANALYZE SELECT name, status FROM {} WHERE id = 1",
2272+
table_definition.name()
2273+
),
2274+
// Test index on age column
2275+
format!(
2276+
"EXPLAIN ANALYZE SELECT * FROM {} WHERE age = 30",
2277+
table_definition.name()
2278+
),
2279+
format!(
2280+
"EXPLAIN ANALYZE SELECT name FROM {} WHERE age = 30",
2281+
table_definition.name()
2282+
),
2283+
format!(
2284+
"EXPLAIN ANALYZE SELECT id, name FROM {} WHERE age = 30",
2285+
table_definition.name()
2286+
),
2287+
// Test index on status column
2288+
format!(
2289+
"EXPLAIN ANALYZE SELECT * FROM {} WHERE status = 'active'",
2290+
table_definition.name()
2291+
),
2292+
format!(
2293+
"EXPLAIN ANALYZE SELECT name FROM {} WHERE status = 'active'",
2294+
table_definition.name()
2295+
),
2296+
format!(
2297+
"EXPLAIN ANALYZE SELECT id, age FROM {} WHERE status = 'active'",
2298+
table_definition.name()
2299+
),
2300+
];
2301+
2302+
for (idx, query) in queries.iter().enumerate() {
2303+
let mut stmt = tx.prepare(query).expect("to prepare statement");
2304+
let mut rows = stmt.query([]).expect("to execute query");
2305+
2306+
let mut explain_output = Vec::new();
2307+
while let Some(row) = rows.next().expect("to get next row") {
2308+
let line: String = row.get(1).expect("to get explain line");
2309+
explain_output.push(line);
2310+
}
2311+
2312+
let explain_result = explain_output.join("\n");
2313+
2314+
insta::with_settings!({
2315+
filters => vec![
2316+
(r"Total Time: \d+\.\d+s", "Total Time: replaced"),
2317+
(r"\(\d+\.\d+s\)", "(0.00s)"),
2318+
(r"│__data_test_table_\d+│\n│\s+\d+\s+│", "│__data_test_table_redacted│\n│ redacted │"),
2319+
],
2320+
}, {
2321+
insta::assert_snapshot!(format!("explain_analyze_multiple_indexes_{idx}"), explain_result);
2322+
});
2323+
}
21712324

21722325
tx.rollback().expect("should rollback transaction");
21732326
}

‎src/duckdb/snapshots/datafusion_table_providers__duckdb__creator__tests__explain_analyze_0.snap‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
---
2+
source: src/duckdb/creator.rs
3+
expression: explain_result
4+
---
5+
┌─────────────────────────────────────┐
6+
│┌───────────────────────────────────┐│
7+
││ Query Profiling Information ││
8+
│└───────────────────────────────────┘│
9+
└─────────────────────────────────────┘
10+
EXPLAIN ANALYZE SELECT * FROM test_table WHERE id = 1
11+
┌────────────────────────────────────────────────┐
12+
│┌──────────────────────────────────────────────┐│
13+
││ Total Time: replaced ││
14+
│└──────────────────────────────────────────────┘│
15+
└────────────────────────────────────────────────┘
16+
┌───────────────────────────┐
17+
│ QUERY │
18+
└─────────────┬─────────────┘
19+
┌─────────────┴─────────────┐
20+
│ EXPLAIN_ANALYZE │
21+
│ ──────────────────── │
22+
│ 0 Rows │
23+
│ (0.00s) │
24+
└─────────────┬─────────────┘
25+
┌─────────────┴─────────────┐
26+
│ TABLE_SCAN │
27+
│ ──────────────────── │
28+
│ Table: │
29+
│__data_test_table_redacted│
30+
│ redacted │
31+
│ │
32+
│ Type: Index Scan │
33+
│ │
34+
│ Projections: │
35+
│ id │
36+
│ name │
37+
│ age │
38+
│ status │
39+
│ │
40+
│ Filters: id=1 │
41+
│ │
42+
│ 1 Rows │
43+
│ (0.00s) │
44+
└───────────────────────────┘

‎src/duckdb/snapshots/datafusion_table_providers__duckdb__creator__tests__explain_analyze_1.snap‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
source: src/duckdb/creator.rs
3+
expression: explain_result
4+
---
5+
┌─────────────────────────────────────┐
6+
│┌───────────────────────────────────┐│
7+
││ Query Profiling Information ││
8+
│└───────────────────────────────────┘│
9+
└─────────────────────────────────────┘
10+
EXPLAIN ANALYZE SELECT name FROM test_table WHERE id = 1
11+
┌────────────────────────────────────────────────┐
12+
│┌──────────────────────────────────────────────┐│
13+
││ Total Time: replaced ││
14+
│└──────────────────────────────────────────────┘│
15+
└────────────────────────────────────────────────┘
16+
┌───────────────────────────┐
17+
│ QUERY │
18+
└─────────────┬─────────────┘
19+
┌─────────────┴─────────────┐
20+
│ EXPLAIN_ANALYZE │
21+
│ ──────────────────── │
22+
│ 0 Rows │
23+
│ (0.00s) │
24+
└─────────────┬─────────────┘
25+
┌─────────────┴─────────────┐
26+
│ TABLE_SCAN │
27+
│ ──────────────────── │
28+
│ Table: │
29+
│__data_test_table_redacted│
30+
│ redacted │
31+
│ │
32+
│ Type: Index Scan │
33+
│ Projections: name │
34+
│ Filters: id=1 │
35+
│ │
36+
│ 1 Rows │
37+
│ (0.00s) │
38+
└───────────────────────────┘

‎src/duckdb/snapshots/datafusion_table_providers__duckdb__creator__tests__explain_analyze_multiple_indexes_0.snap‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
---
2+
source: src/duckdb/creator.rs
3+
expression: explain_result
4+
---
5+
┌─────────────────────────────────────┐
6+
│┌───────────────────────────────────┐│
7+
││ Query Profiling Information ││
8+
│└───────────────────────────────────┘│
9+
└─────────────────────────────────────┘
10+
EXPLAIN ANALYZE SELECT * FROM test_table WHERE id = 1
11+
┌────────────────────────────────────────────────┐
12+
│┌──────────────────────────────────────────────┐│
13+
││ Total Time: replaced ││
14+
│└──────────────────────────────────────────────┘│
15+
└────────────────────────────────────────────────┘
16+
┌───────────────────────────┐
17+
│ QUERY │
18+
└─────────────┬─────────────┘
19+
┌─────────────┴─────────────┐
20+
│ EXPLAIN_ANALYZE │
21+
│ ──────────────────── │
22+
│ 0 Rows │
23+
│ (0.00s) │
24+
└─────────────┬─────────────┘
25+
┌─────────────┴─────────────┐
26+
│ TABLE_SCAN │
27+
│ ──────────────────── │
28+
│ Table: │
29+
│__data_test_table_redacted│
30+
│ redacted │
31+
│ │
32+
│ Type: Index Scan │
33+
│ │
34+
│ Projections: │
35+
│ id │
36+
│ age │
37+
│ status │
38+
│ name │
39+
│ │
40+
│ Filters: id=1 │
41+
│ │
42+
│ 1 Rows │
43+
│ (0.00s) │
44+
└───────────────────────────┘

0 commit comments

Comments
 (0)