feat: add SQLite metadata provider support

Author: shefeek-jinnah

Cargo.toml

@@ -52,7 +52,7 @@ skip-tests-with-docker = []
 metadata-duckdb = ["dep:duckdb"]
 metadata-postgres = ["dep:sqlx", "sqlx/postgres", "sqlx/chrono"]
 metadata-mysql = ["dep:sqlx", "sqlx/mysql", "sqlx/chrono"]
-# Future: metadata-sqlite = ["sqlx", "sqlx/sqlite", "sqlx/chrono"]
+metadata-sqlite = ["dep:sqlx", "sqlx/sqlite", "sqlx/chrono"]
 
 # Encryption support for Parquet files
 encryption = ["parquet/encryption", "datafusion/parquet_encryption", "dep:base64", "dep:hex"]

examples/basic_query.rs

@@ -1,7 +1,7 @@
 //! Basic DuckLake query example with snapshot isolation
 //!
 //! This example demonstrates how to:
-//! 1. Create a DuckLake catalog from DuckDB, PostgreSQL, or MySQL
+//! 1. Create a DuckLake catalog from DuckDB, PostgreSQL, MySQL, or SQLite
 //! 2. Bind the catalog to a specific snapshot for query consistency
 //! 3. Register it with DataFusion
 //! 4. Execute a simple SELECT query
@@ -36,6 +36,13 @@
 //!   "mysql://user:password@localhost:3306/database" \
 //!   "SELECT * FROM main.users"
 //! ```
+//!
+//! With SQLite catalog (requires --features metadata-sqlite):
+//! ```bash
+//! cargo run --example basic_query --features metadata-sqlite \
+//!   "sqlite:///path/to/catalog.db" \
+//!   "SELECT * FROM main.users"
+//! ```
 
 use datafusion::execution::runtime_env::RuntimeEnv;
 use datafusion::prelude::*;
@@ -45,6 +52,8 @@ use datafusion_ducklake::DuckdbMetadataProvider;
 use datafusion_ducklake::MySqlMetadataProvider;
 #[cfg(feature = "metadata-postgres")]
 use datafusion_ducklake::PostgresMetadataProvider;
+#[cfg(feature = "metadata-sqlite")]
+use datafusion_ducklake::SqliteMetadataProvider;
 use datafusion_ducklake::{DuckLakeCatalog, MetadataProvider, register_ducklake_functions};
 use object_store::ObjectStore;
 use object_store::aws::AmazonS3Builder;
@@ -65,6 +74,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         eprintln!(
             "  MySQL:      cargo run --example basic_query --features metadata-mysql \"mysql://...\" \"SQL\""
         );
+        eprintln!(
+            "  SQLite:     cargo run --example basic_query --features metadata-sqlite \"sqlite://...\" \"SQL\""
+        );
         exit(1);
     }
     let catalog_source = &args[1];
@@ -73,6 +85,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Detect provider type based on input
     let is_postgres = catalog_source.starts_with("postgresql://");
     let is_mysql = catalog_source.starts_with("mysql://");
+    let is_sqlite = catalog_source.starts_with("sqlite:");
 
     if is_postgres {
         #[cfg(not(feature = "metadata-postgres"))]
@@ -106,6 +119,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             println!("Current snapshot ID: {}", snapshot_id);
             run_query(provider, snapshot_id, sql).await?;
         }
+    } else if is_sqlite {
+        #[cfg(not(feature = "metadata-sqlite"))]
+        {
+            eprintln!("Error: SQLite support requires the 'metadata-sqlite' feature");
+            eprintln!("Run with: cargo run --example basic_query --features metadata-sqlite");
+            exit(1);
+        }
+
+        #[cfg(feature = "metadata-sqlite")]
+        {
+            println!("Connecting to SQLite catalog: {}", catalog_source);
+            let provider = Arc::new(SqliteMetadataProvider::new(catalog_source).await?);
+            let snapshot_id = provider.get_current_snapshot()?;
+            println!("Current snapshot ID: {}", snapshot_id);
+            run_query(provider, snapshot_id, sql).await?;
+        }
     } else {
         #[cfg(feature = "metadata-duckdb")]
         {

src/error.rs

@@ -18,8 +18,12 @@ pub enum DuckLakeError {
     #[error("DuckDB error: {0}")]
     DuckDb(#[from] duckdb::Error),
 
-    /// sqlx database error (for PostgreSQL/MySQL metadata providers)
-    #[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql"))]
+    /// sqlx database error (for PostgreSQL/MySQL/SQLite metadata providers)
+    #[cfg(any(
+        feature = "metadata-postgres",
+        feature = "metadata-mysql",
+        feature = "metadata-sqlite"
+    ))]
     #[error("Database error: {0}")]
     Sqlx(#[from] sqlx::Error),
 

src/lib.rs

@@ -56,6 +56,8 @@ pub mod metadata_provider_duckdb;
 pub mod metadata_provider_mysql;
 #[cfg(feature = "metadata-postgres")]
 pub mod metadata_provider_postgres;
+#[cfg(feature = "metadata-sqlite")]
+pub mod metadata_provider_sqlite;
 
 // Result type for DuckLake operations
 pub type Result<T> = std::result::Result<T, DuckLakeError>;
@@ -75,3 +77,5 @@ pub use metadata_provider_duckdb::DuckdbMetadataProvider;
 pub use metadata_provider_mysql::MySqlMetadataProvider;
 #[cfg(feature = "metadata-postgres")]
 pub use metadata_provider_postgres::PostgresMetadataProvider;
+#[cfg(feature = "metadata-sqlite")]
+pub use metadata_provider_sqlite::SqliteMetadataProvider;

src/metadata_provider.rs

@@ -405,7 +405,7 @@ pub trait MetadataProvider: Send + Sync + std::fmt::Debug {
     ) -> Result<Vec<DeleteFileChange>>;
 }
 
-#[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql"))]
+#[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql", feature = "metadata-sqlite"))]
 /// Helper function to bridge async sqlx operations to sync MetadataProvider trait
 pub(crate) fn block_on<F, T>(f: F) -> T
 where