feat: add SQLite metadata provider support (#39)

Author: shefeek-jinnah

Cargo.lock

@@ -52,7 +52,7 @@ skip-tests-with-docker = []
 metadata-duckdb = ["dep:duckdb"]
 metadata-postgres = ["dep:sqlx", "sqlx/postgres", "sqlx/chrono"]
 metadata-mysql = ["dep:sqlx", "sqlx/mysql", "sqlx/chrono"]
-# Future: metadata-sqlite = ["sqlx", "sqlx/sqlite", "sqlx/chrono"]
+metadata-sqlite = ["dep:sqlx", "sqlx/sqlite", "sqlx/chrono"]
 
 # Encryption support for Parquet files
 encryption = ["parquet/encryption", "datafusion/parquet_encryption", "dep:base64", "dep:hex"]

Cargo.toml

@@ -11,7 +11,7 @@ The goal of this project is to make DuckLake a first-class, Arrow-native lakehou
 ## Currently Supported
 
 - Read-only queries against DuckLake catalogs
-- DuckDB, PostgreSQL, and MySQL catalog backends
+- DuckDB, PostgreSQL, MySQL, and SQLite catalog backends
 - Local filesystem and S3-compatible object stores (MinIO, S3)
 - Snapshot-based consistency
 - Basic and decimal types
@@ -43,7 +43,6 @@ This project is under active development. The roadmap below reflects major areas
 ### Metadata & Catalog Improvements
 
 - Metadata caching to reduce repeated catalog lookups
-- SQLite metadata provider
 - Clear abstraction boundaries between catalog, metadata provider, and execution
 
 ### Query Planning & Performance
@@ -86,6 +85,7 @@ This project is under active development. The roadmap below reflects major areas
 | `metadata-duckdb` | DuckDB catalog backend | ✅ |
 | `metadata-postgres` | PostgreSQL catalog backend | |
 | `metadata-mysql` | MySQL catalog backend | |
+| `metadata-sqlite` | SQLite catalog backend | |
 | `encryption` | Parquet Modular Encryption (PME) support | |
 
 ```bash
@@ -98,8 +98,11 @@ cargo build --no-default-features --features metadata-postgres
 # MySQL only
 cargo build --no-default-features --features metadata-mysql
 
+# SQLite only
+cargo build --no-default-features --features metadata-sqlite
+
 # All backends
-cargo build --features metadata-postgres,metadata-mysql
+cargo build --features metadata-postgres,metadata-mysql,metadata-sqlite
 ```
 
 ### Example
@@ -115,6 +118,10 @@ cargo run --example basic_query --features metadata-postgres -- \
 # MySQL catalog
 cargo run --example basic_query --features metadata-mysql -- \
   "mysql://user:password@localhost:3306/database" "SELECT * FROM main.users"
+
+# SQLite catalog
+cargo run --example basic_query --features metadata-sqlite -- \
+  "sqlite:///path/to/catalog.db" "SELECT * FROM main.users"
 ```
 
 ### Integration

README.md

@@ -1,7 +1,7 @@
 //! Basic DuckLake query example with snapshot isolation
 //!
 //! This example demonstrates how to:
-//! 1. Create a DuckLake catalog from DuckDB, PostgreSQL, or MySQL
+//! 1. Create a DuckLake catalog from DuckDB, PostgreSQL, MySQL, or SQLite
 //! 2. Bind the catalog to a specific snapshot for query consistency
 //! 3. Register it with DataFusion
 //! 4. Execute a simple SELECT query
@@ -36,6 +36,13 @@
 //!   "mysql://user:password@localhost:3306/database" \
 //!   "SELECT * FROM main.users"
 //! ```
+//!
+//! With SQLite catalog (requires --features metadata-sqlite):
+//! ```bash
+//! cargo run --example basic_query --features metadata-sqlite \
+//!   "sqlite:///path/to/catalog.db" \
+//!   "SELECT * FROM main.users"
+//! ```
 
 use datafusion::execution::runtime_env::RuntimeEnv;
 use datafusion::prelude::*;
@@ -45,6 +52,8 @@ use datafusion_ducklake::DuckdbMetadataProvider;
 use datafusion_ducklake::MySqlMetadataProvider;
 #[cfg(feature = "metadata-postgres")]
 use datafusion_ducklake::PostgresMetadataProvider;
+#[cfg(feature = "metadata-sqlite")]
+use datafusion_ducklake::SqliteMetadataProvider;
 use datafusion_ducklake::{DuckLakeCatalog, MetadataProvider, register_ducklake_functions};
 use object_store::ObjectStore;
 use object_store::aws::AmazonS3Builder;
@@ -65,6 +74,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         eprintln!(
             "  MySQL:      cargo run --example basic_query --features metadata-mysql \"mysql://...\" \"SQL\""
         );
+        eprintln!(
+            "  SQLite:     cargo run --example basic_query --features metadata-sqlite \"sqlite://...\" \"SQL\""
+        );
         exit(1);
     }
     let catalog_source = &args[1];
@@ -73,6 +85,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Detect provider type based on input
     let is_postgres = catalog_source.starts_with("postgresql://");
     let is_mysql = catalog_source.starts_with("mysql://");
+    let is_sqlite = catalog_source.starts_with("sqlite:");
 
     if is_postgres {
         #[cfg(not(feature = "metadata-postgres"))]
@@ -106,6 +119,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             println!("Current snapshot ID: {}", snapshot_id);
             run_query(provider, snapshot_id, sql).await?;
         }
+    } else if is_sqlite {
+        #[cfg(not(feature = "metadata-sqlite"))]
+        {
+            eprintln!("Error: SQLite support requires the 'metadata-sqlite' feature");
+            eprintln!("Run with: cargo run --example basic_query --features metadata-sqlite");
+            exit(1);
+        }
+
+        #[cfg(feature = "metadata-sqlite")]
+        {
+            println!("Connecting to SQLite catalog: {}", catalog_source);
+            let provider = Arc::new(SqliteMetadataProvider::new(catalog_source).await?);
+            let snapshot_id = provider.get_current_snapshot()?;
+            println!("Current snapshot ID: {}", snapshot_id);
+            run_query(provider, snapshot_id, sql).await?;
+        }
     } else {
         #[cfg(feature = "metadata-duckdb")]
         {

examples/basic_query.rs

@@ -18,8 +18,12 @@ pub enum DuckLakeError {
     #[error("DuckDB error: {0}")]
     DuckDb(#[from] duckdb::Error),
 
-    /// sqlx database error (for PostgreSQL/MySQL metadata providers)
-    #[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql"))]
+    /// sqlx database error (for PostgreSQL/MySQL/SQLite metadata providers)
+    #[cfg(any(
+        feature = "metadata-postgres",
+        feature = "metadata-mysql",
+        feature = "metadata-sqlite"
+    ))]
     #[error("Database error: {0}")]
     Sqlx(#[from] sqlx::Error),
 

src/error.rs

@@ -56,6 +56,8 @@ pub mod metadata_provider_duckdb;
 pub mod metadata_provider_mysql;
 #[cfg(feature = "metadata-postgres")]
 pub mod metadata_provider_postgres;
+#[cfg(feature = "metadata-sqlite")]
+pub mod metadata_provider_sqlite;
 
 // Result type for DuckLake operations
 pub type Result<T> = std::result::Result<T, DuckLakeError>;
@@ -75,3 +77,5 @@ pub use metadata_provider_duckdb::DuckdbMetadataProvider;
 pub use metadata_provider_mysql::MySqlMetadataProvider;
 #[cfg(feature = "metadata-postgres")]
 pub use metadata_provider_postgres::PostgresMetadataProvider;
+#[cfg(feature = "metadata-sqlite")]
+pub use metadata_provider_sqlite::SqliteMetadataProvider;

src/lib.rs

@@ -405,7 +405,7 @@ pub trait MetadataProvider: Send + Sync + std::fmt::Debug {
     ) -> Result<Vec<DeleteFileChange>>;
 }
 
-#[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql"))]
+#[cfg(any(feature = "metadata-postgres", feature = "metadata-mysql", feature = "metadata-sqlite"))]
 /// Helper function to bridge async sqlx operations to sync MetadataProvider trait
 pub(crate) fn block_on<F, T>(f: F) -> T
 where