Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ skip-tests-with-docker = []
# Metadata provider backends
metadata-duckdb = ["dep:duckdb"]
metadata-postgres = ["dep:sqlx", "sqlx/postgres", "sqlx/chrono"]
metadata-mysql = ["dep:sqlx", "sqlx/mysql", "sqlx/chrono"]
# Future: metadata-sqlite = ["sqlx", "sqlx/sqlite", "sqlx/chrono"]
# Future: metadata-mysql = ["sqlx", "sqlx/mysql", "sqlx/chrono"]

# Encryption support for Parquet files
encryption = ["parquet/encryption", "datafusion/parquet_encryption", "dep:base64", "dep:hex"]
31 changes: 30 additions & 1 deletion examples/basic_query.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Basic DuckLake query example with snapshot isolation
//!
//! This example demonstrates how to:
//! 1. Create a DuckLake catalog from DuckDB or PostgreSQL
//! 1. Create a DuckLake catalog from DuckDB, PostgreSQL, or MySQL
//! 2. Bind the catalog to a specific snapshot for query consistency
//! 3. Register it with DataFusion
//! 4. Execute a simple SELECT query
Expand Down Expand Up @@ -29,11 +29,20 @@
//! "postgresql://user:password@localhost:5432/postgres" \
//! "SELECT * FROM main.users"
//! ```
//!
//! With MySQL catalog (requires --features metadata-mysql):
//! ```bash
//! cargo run --example basic_query --features metadata-mysql \
//! "mysql://user:password@localhost:3306/database" \
//! "SELECT * FROM main.users"
//! ```

use datafusion::execution::runtime_env::RuntimeEnv;
use datafusion::prelude::*;
#[cfg(feature = "metadata-duckdb")]
use datafusion_ducklake::DuckdbMetadataProvider;
#[cfg(feature = "metadata-mysql")]
use datafusion_ducklake::MySqlMetadataProvider;
#[cfg(feature = "metadata-postgres")]
use datafusion_ducklake::PostgresMetadataProvider;
use datafusion_ducklake::{DuckLakeCatalog, MetadataProvider, register_ducklake_functions};
Expand All @@ -53,13 +62,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
eprintln!(
" PostgreSQL: cargo run --example basic_query --features metadata-postgres \"postgresql://...\" \"SQL\""
);
eprintln!(
" MySQL: cargo run --example basic_query --features metadata-mysql \"mysql://...\" \"SQL\""
);
exit(1);
}
let catalog_source = &args[1];
let sql = &args[2];

// Detect provider type based on input
let is_postgres = catalog_source.starts_with("postgresql://");
let is_mysql = catalog_source.starts_with("mysql://");

if is_postgres {
#[cfg(not(feature = "metadata-postgres"))]
Expand All @@ -77,6 +90,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("Current snapshot ID: {}", snapshot_id);
run_query(provider, snapshot_id, sql).await?;
}
} else if is_mysql {
#[cfg(not(feature = "metadata-mysql"))]
{
eprintln!("Error: MySQL support requires the 'metadata-mysql' feature");
eprintln!("Run with: cargo run --example basic_query --features metadata-mysql");
exit(1);
}

#[cfg(feature = "metadata-mysql")]
{
println!("Connecting to MySQL catalog: {}", catalog_source);
let provider = Arc::new(MySqlMetadataProvider::new(catalog_source).await?);
let snapshot_id = provider.get_current_snapshot()?;
println!("Current snapshot ID: {}", snapshot_id);
run_query(provider, snapshot_id, sql).await?;
}
} else {
#[cfg(feature = "metadata-duckdb")]
{
Expand Down
4 changes: 2 additions & 2 deletions src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ pub enum DuckLakeError {
#[error("DuckDB error: {0}")]
DuckDb(#[from] duckdb::Error),

/// sqlx database error (for PostgreSQL metadata provider)
#[cfg(feature = "metadata-postgres")]
/// sqlx database error (for PostgreSQL/MySQL metadata providers)
#[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql"))]
#[error("Database error: {0}")]
Sqlx(#[from] sqlx::Error),

Expand Down
4 changes: 4 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ pub mod types;
// Metadata providers (feature-gated)
#[cfg(feature = "metadata-duckdb")]
pub mod metadata_provider_duckdb;
#[cfg(feature = "metadata-mysql")]
pub mod metadata_provider_mysql;
#[cfg(feature = "metadata-postgres")]
pub mod metadata_provider_postgres;

Expand All @@ -69,5 +71,7 @@ pub use table_functions::register_ducklake_functions;
// Re-export metadata providers (feature-gated)
#[cfg(feature = "metadata-duckdb")]
pub use metadata_provider_duckdb::DuckdbMetadataProvider;
#[cfg(feature = "metadata-mysql")]
pub use metadata_provider_mysql::MySqlMetadataProvider;
#[cfg(feature = "metadata-postgres")]
pub use metadata_provider_postgres::PostgresMetadataProvider;
2 changes: 1 addition & 1 deletion src/metadata_provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ pub trait MetadataProvider: Send + Sync + std::fmt::Debug {
) -> Result<Vec<DeleteFileChange>>;
}

#[cfg(feature = "metadata-postgres")]
#[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql"))]
/// Helper function to bridge async sqlx operations to sync MetadataProvider trait
pub(crate) fn block_on<F, T>(f: F) -> T
where
Expand Down
Loading
Loading