Skip to content

Upgrade DuckDB to 1.3.2 #391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sgrebnov merged 5 commits into from
Jul 14, 2025
Merged

Upgrade DuckDB to 1.3.2 #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cf842df
use index ordering for columns
phillipleblanc Jul 14, 2025
29e9125
Re-order the columns to have the indexed column first
phillipleblanc Jul 14, 2025
03bbfc4
Upgrade DuckDB to 1.3.2
phillipleblanc Jul 14, 2025
4e062b2
Add test for multiple indexes
phillipleblanc Jul 14, 2025
a5a92f3
Merge branch 'spiceai' into phillip/250714-upgrade-duckdb
phillipleblanc Jul 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ datafusion-expr = { version = "47", optional = true }
datafusion-physical-expr = { version = "47", optional = true }
datafusion-physical-plan = { version = "47", optional = true }
datafusion-proto = { version = "47", optional = true }
duckdb = { version = "1.1.3", features = [
duckdb = { version = "1.3.2", features = [
"bundled",
"r2d2",
"vtab",
Expand Down Expand Up @@ -103,7 +103,7 @@ postgres-federation = ["postgres"]

[patch.crates-io]
datafusion-federation = { git = "https://github.com/spiceai/datafusion-federation.git", rev = "9db74a4b360df6be1bb554c59a474a2fd4bfb7e9" } # spiceai-47
duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "69ae7518ee093a1b070e9e4e6f011ef353431086" } # spiceai-1.1.3-backported-arrow-55
duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "b3547f0c1b37030b623b1e03fcaea0e4e2bb753e" } # spiceai-1.3.2

datafusion = { git = "https://github.com/spiceai/datafusion.git", rev = "b5c62f29d2c70c5331ff50015b67b5e1cafcd578" } # spiceai-47
datafusion-expr = { git = "https://github.com/spiceai/datafusion.git", rev = "b5c62f29d2c70c5331ff50015b67b5e1cafcd578" } # spiceai-47
Expand Down
189 changes: 171 additions & 18 deletions src/duckdb/creator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -709,7 +709,7 @@ impl TableManager {
/// If there is an index defined on a single column, that column should come first in the list.
/// Multi-column indexes are not considered for ordering.
pub(crate) fn order_columns_by_index(&self, columns: Vec<String>) -> Vec<String> {
let mut indexed_columns = Vec::new();
let mut ordered_columns = Vec::new();
let mut non_indexed_columns = Vec::new();

// Get single-column indexes
Expand All @@ -730,15 +730,15 @@ impl TableManager {
// Separate columns into indexed and non-indexed
for column in columns {
if single_column_indexes.contains(&column) {
indexed_columns.push(column);
ordered_columns.push(column);
} else {
non_indexed_columns.push(column);
}
}

// Return indexed columns first, then non-indexed columns
indexed_columns.extend(non_indexed_columns);
indexed_columns
ordered_columns.extend(non_indexed_columns);
ordered_columns
}
}

Expand Down Expand Up @@ -2105,8 +2105,8 @@ pub(crate) mod tests {
let pool = get_mem_duckdb();

let schema = Arc::new(arrow::datatypes::Schema::new(vec![
arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int64, false),
arrow::datatypes::Field::new("name", arrow::datatypes::DataType::Utf8, false),
arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int64, false),
arrow::datatypes::Field::new("age", arrow::datatypes::DataType::Int32, true),
arrow::datatypes::Field::new("status", arrow::datatypes::DataType::Utf8, false),
]));
Expand Down Expand Up @@ -2143,7 +2143,7 @@ pub(crate) mod tests {

tx.execute(
&format!(
r#"INSERT INTO "{table_name}" VALUES (1, 'Alice', 30, 'active'), (2, 'Bob', 25, 'inactive'), (3, 'Charlie', 35, 'active')"#,
r#"INSERT INTO "{table_name}" VALUES ('Alice', 1, 30, 'active'), ('Bob', 2, 25, 'inactive'), ('Charlie', 3, 35, 'active')"#,
table_name = table_manager.table_name()
),
[],
Expand All @@ -2152,22 +2152,175 @@ pub(crate) mod tests {

table_manager.create_view(&tx).expect("to create view");

let explain_query = format!(
"EXPLAIN SELECT * FROM {} WHERE id = 1",
table_definition.name()
);
let queries = [
format!(
"EXPLAIN ANALYZE SELECT * FROM {} WHERE id = 1",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT name FROM {} WHERE id = 1",
table_definition.name()
),
];

let mut stmt = tx.prepare(&explain_query).expect("to prepare statement");
let mut rows = stmt.query([]).expect("to execute query");
for (idx, query) in queries.iter().enumerate() {
let mut stmt = tx.prepare(query).expect("to prepare statement");
let mut rows = stmt.query([]).expect("to execute query");

let mut explain_output = Vec::new();
while let Some(row) = rows.next().expect("to get next row") {
let line: String = row.get(1).expect("to get explain line");
explain_output.push(line);
let mut explain_output = Vec::new();
while let Some(row) = rows.next().expect("to get next row") {
let line: String = row.get(1).expect("to get explain line");
explain_output.push(line);
}

let explain_result = explain_output.join("\n");

insta::with_settings!({
filters => vec![
(r"Total Time: \d+\.\d+s", "Total Time: replaced"),
(r"\(\d+\.\d+s\)", "(0.00s)"),
(r"│__data_test_table_\d+│\n│\s+\d+\s+│", "│__data_test_table_redacted│\n│ redacted │"),
],
}, {
insta::assert_snapshot!(format!("explain_analyze_{idx}"), explain_result);
});
}

let explain_result = explain_output.join("\n");
insta::assert_snapshot!(explain_result);
tx.rollback().expect("should rollback transaction");
}

#[tokio::test]
async fn test_explain_analyze_with_multiple_indexes_and_view() {
let _guard = init_tracing(None);
let pool = get_mem_duckdb();

let schema = Arc::new(arrow::datatypes::Schema::new(vec![
arrow::datatypes::Field::new("name", arrow::datatypes::DataType::Utf8, false),
arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int64, false),
arrow::datatypes::Field::new("age", arrow::datatypes::DataType::Int32, true),
arrow::datatypes::Field::new("status", arrow::datatypes::DataType::Utf8, false),
]));

let table_definition = Arc::new(
TableDefinition::new(RelationName::new("test_table"), Arc::clone(&schema))
.with_indexes(vec![
(
ColumnReference::try_from("id").expect("valid column ref"),
IndexType::Enabled,
),
(
ColumnReference::try_from("age").expect("valid column ref"),
IndexType::Enabled,
),
(
ColumnReference::try_from("status").expect("valid column ref"),
IndexType::Enabled,
),
]),
);

let mut pool_conn = Arc::clone(&pool).connect_sync().expect("to get connection");
let conn = pool_conn
.as_any_mut()
.downcast_mut::<DuckDbConnection>()
.expect("to downcast to duckdb connection");
let tx = conn
.get_underlying_conn_mut()
.transaction()
.expect("should begin transaction");

let table_manager = TableManager::new(Arc::clone(&table_definition))
.with_internal(true)
.expect("to create table manager");

table_manager
.create_table(Arc::clone(&pool), &tx)
.expect("to create table");

table_manager
.create_indexes(&tx)
.expect("to create indexes");

tx.execute(
&format!(
r#"INSERT INTO "{table_name}" VALUES
('Alice', 1, 30, 'active'),
('Bob', 2, 25, 'inactive'),
('Charlie', 3, 35, 'active'),
('David', 4, 30, 'pending'),
('Eve', 5, 40, 'active')"#,
table_name = table_manager.table_name()
),
[],
)
.expect("to insert test data");

table_manager.create_view(&tx).expect("to create view");

let queries = [
// Test index on id column
format!(
"EXPLAIN ANALYZE SELECT * FROM {} WHERE id = 1",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT name FROM {} WHERE id = 1",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT name, status FROM {} WHERE id = 1",
table_definition.name()
),
// Test index on age column
format!(
"EXPLAIN ANALYZE SELECT * FROM {} WHERE age = 30",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT name FROM {} WHERE age = 30",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT id, name FROM {} WHERE age = 30",
table_definition.name()
),
// Test index on status column
format!(
"EXPLAIN ANALYZE SELECT * FROM {} WHERE status = 'active'",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT name FROM {} WHERE status = 'active'",
table_definition.name()
),
format!(
"EXPLAIN ANALYZE SELECT id, age FROM {} WHERE status = 'active'",
table_definition.name()
),
];

for (idx, query) in queries.iter().enumerate() {
let mut stmt = tx.prepare(query).expect("to prepare statement");
let mut rows = stmt.query([]).expect("to execute query");

let mut explain_output = Vec::new();
while let Some(row) = rows.next().expect("to get next row") {
let line: String = row.get(1).expect("to get explain line");
explain_output.push(line);
}

let explain_result = explain_output.join("\n");

insta::with_settings!({
filters => vec![
(r"Total Time: \d+\.\d+s", "Total Time: replaced"),
(r"\(\d+\.\d+s\)", "(0.00s)"),
(r"│__data_test_table_\d+│\n│\s+\d+\s+│", "│__data_test_table_redacted│\n│ redacted │"),
],
}, {
insta::assert_snapshot!(format!("explain_analyze_multiple_indexes_{idx}"), explain_result);
});
}

tx.rollback().expect("should rollback transaction");
}
Expand Down
44 changes: 44 additions & 0 deletions ...ckdb/snapshots/datafusion_table_providers__duckdb__creator__tests__explain_analyze_0.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
---
source: src/duckdb/creator.rs
expression: explain_result
---
┌─────────────────────────────────────┐
│┌───────────────────────────────────┐│
││ Query Profiling Information ││
│└───────────────────────────────────┘│
└─────────────────────────────────────┘
EXPLAIN ANALYZE SELECT * FROM test_table WHERE id = 1
┌────────────────────────────────────────────────┐
│┌──────────────────────────────────────────────┐│
││ Total Time: replaced ││
│└──────────────────────────────────────────────┘│
└────────────────────────────────────────────────┘
┌───────────────────────────┐
│ QUERY │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ EXPLAIN_ANALYZE │
│ ──────────────────── │
│ 0 Rows │
│ (0.00s) │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ TABLE_SCAN │
│ ──────────────────── │
│ Table: │
│__data_test_table_redacted│
│ redacted │
│ │
│ Type: Index Scan │
│ │
│ Projections: │
│ id │
│ name │
│ age │
│ status │
│ │
│ Filters: id=1 │
│ │
│ 1 Rows │
│ (0.00s) │
└───────────────────────────┘
38 changes: 38 additions & 0 deletions ...ckdb/snapshots/datafusion_table_providers__duckdb__creator__tests__explain_analyze_1.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
---
source: src/duckdb/creator.rs
expression: explain_result
---
┌─────────────────────────────────────┐
│┌───────────────────────────────────┐│
││ Query Profiling Information ││
│└───────────────────────────────────┘│
└─────────────────────────────────────┘
EXPLAIN ANALYZE SELECT name FROM test_table WHERE id = 1
┌────────────────────────────────────────────────┐
│┌──────────────────────────────────────────────┐│
││ Total Time: replaced ││
│└──────────────────────────────────────────────┘│
└────────────────────────────────────────────────┘
┌───────────────────────────┐
│ QUERY │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ EXPLAIN_ANALYZE │
│ ──────────────────── │
│ 0 Rows │
│ (0.00s) │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ TABLE_SCAN │
│ ──────────────────── │
│ Table: │
│__data_test_table_redacted│
│ redacted │
│ │
│ Type: Index Scan │
│ Projections: name │
│ Filters: id=1 │
│ │
│ 1 Rows │
│ (0.00s) │
└───────────────────────────┘
44 changes: 44 additions & 0 deletions ...tafusion_table_providers__duckdb__creator__tests__explain_analyze_multiple_indexes_0.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
---
source: src/duckdb/creator.rs
expression: explain_result
---
┌─────────────────────────────────────┐
│┌───────────────────────────────────┐│
││ Query Profiling Information ││
│└───────────────────────────────────┘│
└─────────────────────────────────────┘
EXPLAIN ANALYZE SELECT * FROM test_table WHERE id = 1
┌────────────────────────────────────────────────┐
│┌──────────────────────────────────────────────┐│
││ Total Time: replaced ││
│└──────────────────────────────────────────────┘│
└────────────────────────────────────────────────┘
┌───────────────────────────┐
│ QUERY │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ EXPLAIN_ANALYZE │
│ ──────────────────── │
│ 0 Rows │
│ (0.00s) │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ TABLE_SCAN │
│ ──────────────────── │
│ Table: │
│__data_test_table_redacted│
│ redacted │
│ │
│ Type: Index Scan │
│ │
│ Projections: │
│ id │
│ age │
│ status │
│ name │
│ │
│ Filters: id=1 │
│ │
│ 1 Rows │
│ (0.00s) │
└───────────────────────────┘
Loading
Loading