Enable static DuckDB extensions via Cargo in checkouts (#732)

This PR adds an experimental `bundled-cmake` build path for duckdb-rs
checkouts. It keeps the existing `bundled` feature unchanged, and adds a
higher-priority bundled backend that builds DuckDB through upstream
CMake when `bundled-cmake` is enabled.

Only in-tree static extensions such as `icu` are supported for now, but
there's a path toward future out-of-tree extension support. In fact, I
think using CMake is the only realistic path toward out-of-tree
extensions. Features like `sqlite_scanner` are already part of DuckDB's
checked-in extension configs. Reusing that upstream mechanism is much
more maintainable than inventing a parallel extension build/link system
in Rust. *Let DuckDB build DuckDB.*

*Update: For out-of-tree extensions, see
#734

Although this PR only targets checkout builds, the CMake backend is
structured so that a future crates.io-friendly variant could reuse the
same backend logic after downloading the DuckDB sources (`duckdb.tar.gz`
does not contain the full source tree).

Fixes #461

Author: mlafeldt

.github/workflows/rust.yaml

@@ -23,7 +23,7 @@ jobs:
   test:
     # - Linux uses DUCKDB_DOWNLOAD_LIB (non-bundled)
     # - Windows uses pre-downloaded archives via DUCKDB_LIB_DIR
-    # - bundled feature tested by --all-features
+    # - bundled feature covered by the Ubuntu clippy feature set below
     name: Test ${{ matrix.name }}
     strategy:
       fail-fast: true
@@ -64,7 +64,8 @@ jobs:
 
       - name: run cargo clippy
         if: matrix.os == 'ubuntu-latest'
-        run: cargo clippy --all-targets --all-features --locked -- -D warnings
+        # `bundled-cmake` is checkout-only and covered in the dedicated CMake job below.
+        run: cargo clippy --all-targets --features "buildtime_bindgen extensions-full loadable-extension modern-full vscalar vscalar-arrow vtab-full" --locked -- -D warnings
 
       - name: Dry-run release of crates
         if: matrix.os == 'ubuntu-latest'
@@ -104,6 +105,37 @@ jobs:
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
 
+  cmake:
+    name: Bundled CMake ${{ matrix.name }}
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - { name: Linux, os: ubuntu-latest }
+          - { name: macOS, os: macos-latest }
+          - { name: Windows, os: windows-latest }
+    runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      CMAKE_C_COMPILER_LAUNCHER: sccache
+      CMAKE_CXX_COMPILER_LAUNCHER: sccache
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses: mozilla-actions/sccache-action@v0.0.9
+      - name: Verify static linking
+        shell: bash
+        run: |
+          output=$(cargo run -p duckdb --example repl --no-default-features --features "bundled-cmake,icu" -- -c "from duckdb_extensions() where extension_name = 'icu';")
+          echo "$output"
+          grep -qF "STATICALLY_LINKED" <<<"$output"
+          grep -qF "(BUILT-IN)" <<<"$output"
+      - name: ICU functional test
+        run: cargo test -p duckdb --no-default-features --features "bundled-cmake,icu" --lib extension::test::test_extension_icu -- --exact
+
   msrv:
     name: MSRV Check
     runs-on: ubuntu-latest

Cargo.lock

@@ -98,10 +98,22 @@ The `duckdb` crate provides a number of Cargo features that can be enabled to ad
 - `vscalar` - Create custom scalar functions that operate on individual values or rows.
 - `vscalar-arrow` - Arrow-optimized scalar functions for vectorized operations.
 
+### File formats
+
+- `json` - Enables reading and writing JSON files. Implies `bundled`.
+- `parquet` - Enables reading and writing Parquet files. Implies `bundled`.
+
+### Bundled DuckDB extensions
+
+These extensions are only available through the CMake build backend and imply `bundled-cmake`.
+
+- `autocomplete` - DuckDB's autocomplete extension.
+- `icu` - DuckDB's ICU extension for locale-aware operations.
+- `tpch` - DuckDB's TPC-H benchmark extension.
+- `tpcds` - DuckDB's TPC-DS benchmark extension.
+
 ### Data integration
 
-- `json` - Enables reading and writing JSON files. Requires `bundled`.
-- `parquet` - Enables reading and writing Parquet files. Requires `bundled`.
 - `appender-arrow` - Efficient bulk insertion of Arrow data into DuckDB tables.
 - `polars` - Integration with Polars DataFrames.
 
@@ -114,6 +126,7 @@ The `duckdb` crate provides a number of Cargo features that can be enabled to ad
 ### Build configuration
 
 - `bundled` - Uses a bundled version of DuckDB's source code and compiles it during build. This is the simplest way to get started and avoids needing DuckDB system libraries.
+- `bundled-cmake` - *Experimental*. Builds DuckDB via its upstream CMake build system instead of `cc`. Requires a duckdb-rs checkout (not available from crates.io). See [step 2](#notes-on-building-duckdb-and-libduckdb-sys) below for details.
 - `buildtime_bindgen` - Use bindgen at build time to generate fresh bindings instead of using pre-generated ones.
 - `loadable-extension` - _Experimental_ support for creating loadable DuckDB extensions. Includes procedural macros for extension development.
 
@@ -178,7 +191,27 @@ You can adjust this behavior in a number of ways:
    duckdb = { version = "1.10501.0", features = ["bundled"] }
    ```
 
-2. When linking against a DuckDB library already on the system (so _not_ using any of the `bundled` features), you can set the `DUCKDB_LIB_DIR` environment variable to point to a directory containing the library. You can also set the `DUCKDB_INCLUDE_DIR` variable to point to the directory containing `duckdb.h`.
+2. If you use the `bundled-cmake` feature, `libduckdb-sys` will build DuckDB from the local checkout in `crates/libduckdb-sys/duckdb-sources` using upstream CMake. This keeps plain `bundled` unchanged while allowing CMake-only extensions such as `icu`.
+
+   Example:
+
+   ```toml
+   [dependencies]
+   duckdb = { git = "https://github.com/duckdb/duckdb-rs", branch = "main", features = ["bundled-cmake", "icu"] }
+   ```
+
+   Notes:
+
+   - `bundled-cmake` is *experimental* and requires a git/workspace checkout. It is not available from crates.io because the full `duckdb-sources` tree is not packaged there.
+   - `bundled-cmake` implies `bundled` (for conditional-compilation gates) but replaces the `cc` build backend with CMake. Enabling any CMake-only extension feature (e.g. `icu`) automatically activates `bundled-cmake`.
+   - `bundled-cmake` always links DuckDB's default static extensions (`core_functions` and `parquet`), so it also implies the `parquet` Cargo feature.
+   - Extension autoload/autoinstall are forced on to match the existing `bundled` backend, even though upstream CMake defaults are off.
+   - When `ninja` is on `PATH`, the Ninja generator is preferred automatically. Set `CMAKE_GENERATOR` to override.
+   - Builds DuckDB in `Release` mode by default, even in Rust debug builds, to avoid DuckDB's much slower debug/sanitizer profile. Set `DUCKDB_CMAKE_BUILD_TYPE` or `CMAKE_BUILD_TYPE` to override. `DUCKDB_CMAKE_BUILD_TYPE` takes precedence.
+   - `DUCKDB_EXTENSION_CONFIGS` is not yet supported; the build fails fast rather than producing a broken binary.
+   - Use `cargo build -vv -F bundled-cmake` to surface CMake configure/build logs.
+
+3. When linking against a DuckDB library already on the system (so _not_ using any of the `bundled` features), you can set the `DUCKDB_LIB_DIR` environment variable to point to a directory containing the library. You can also set the `DUCKDB_INCLUDE_DIR` variable to point to the directory containing `duckdb.h`.
 
    Linux example:
 
@@ -206,29 +239,31 @@ You can adjust this behavior in a number of ways:
    cargo build --examples
    ```
 
-3. Setting `DUCKDB_DOWNLOAD_LIB=1` makes the build script download pre-built DuckDB binaries from GitHub Releases. This always links against the dynamic library in the archive (setting `DUCKDB_STATIC` has no effect), and it effectively automates the manual steps above. The archives are cached in `target/duckdb-download/<target>/<version>` and that directory is automatically added to the linker search path. The downloaded version always matches the DuckDB version encoded in the `libduckdb-sys` crate version.
+4. Setting `DUCKDB_DOWNLOAD_LIB=1` makes the build script download pre-built DuckDB binaries from GitHub Releases. This always links against the dynamic library in the archive (setting `DUCKDB_STATIC` has no effect), and it effectively automates the manual steps above. The archives are cached in `target/duckdb-download/<target>/<version>` and that directory is automatically added to the linker search path. The downloaded version always matches the DuckDB version encoded in the `libduckdb-sys` crate version.
 
    ```shell
    DUCKDB_DOWNLOAD_LIB=1 cargo test
    ```
 
-4. Installing the DuckDB development packages will usually be all that is required, but
+5. Installing the DuckDB development packages will usually be all that is required, but
    the build helpers for [pkg-config](https://github.com/alexcrichton/pkg-config-rs)
    and [vcpkg](https://github.com/mcgoo/vcpkg-rs) have some additional configuration
    options. The default when using vcpkg is to dynamically link,
    which must be enabled by setting `VCPKGRS_DYNAMIC=1` environment variable before build.
 
 When none of the options above are used, the build script falls back to this discovery path and will emit the appropriate `cargo:rustc-link-lib` directives if DuckDB is found on your system.
 
-### ICU extension and the bundled feature
+### ICU extension and the bundled features
 
 When using the `bundled` feature, the ICU extension is not included due to crates.io's 10MB package size limit. This means some date/time operations (like `now() - interval '1 day'` or `ts::date` casts) will fail. You can load ICU at runtime:
 
 ```rust,ignore
 conn.execute_batch("INSTALL icu; LOAD icu;")?;
 ```
 
-Alternatively, link against libduckdb without the `bundled` feature (see build instructions above). The ICU extension will be built-in and pre-loaded in that case.
+Alternatively, link against a system libduckdb that was compiled with ICU (see build instructions above).
+
+If you are working from a duckdb-rs checkout, you can also use `bundled-cmake,icu` to compile ICU in through DuckDB's CMake build.
 
 ### Binding generation
 

README.md

@@ -19,8 +19,14 @@ name = "duckdb"
 [features]
 default = []
 bundled = ["libduckdb-sys/bundled"]
+# Warning: experimental feature
+bundled-cmake = ["bundled", "libduckdb-sys/bundled-cmake", "parquet"]
 json = ["libduckdb-sys/json", "bundled"]
 parquet = ["libduckdb-sys/parquet", "bundled"]
+autocomplete = ["libduckdb-sys/autocomplete", "bundled-cmake"]
+icu = ["libduckdb-sys/icu", "bundled-cmake"]
+tpcds = ["libduckdb-sys/tpcds", "bundled-cmake"]
+tpch = ["libduckdb-sys/tpch", "bundled-cmake"]
 vscalar = ["vtab-arrow"]
 vscalar-arrow = []
 vtab = []

crates/duckdb/Cargo.toml

@@ -2,7 +2,8 @@
 mod test {
     use crate::{Connection, Result};
 
-    // https://duckdb.org/docs/extensions/json
+    // https://duckdb.org/docs/current/data/json/overview
+    #[cfg(feature = "json")]
     #[test]
     fn test_extension_json() -> Result<()> {
         let db = Connection::open_in_memory()?;
@@ -17,7 +18,8 @@ mod test {
         Ok(())
     }
 
-    // https://duckdb.org/docs/data/parquet/overview.html
+    // https://duckdb.org/docs/current/data/parquet/overview
+    #[cfg(feature = "parquet")]
     #[test]
     fn test_extension_parquet() -> Result<()> {
         let db = Connection::open_in_memory()?;
@@ -32,6 +34,24 @@ mod test {
         Ok(())
     }
 
+    // https://duckdb.org/docs/current/core_extensions/icu
+    #[cfg(feature = "icu")]
+    #[test]
+    fn test_extension_icu() -> Result<()> {
+        let db = Connection::open_in_memory()?;
+        assert_eq!(
+            1i64,
+            db.query_row::<i64, _, _>(
+                "SELECT count(*) FROM icu_calendar_names() WHERE name = 'gregorian';",
+                [],
+                |r| r.get(0)
+            )?
+        );
+        assert!(db.query_row::<bool, _, _>("SELECT length(icu_sort_key('Ş', 'ro')) > 0;", [], |r| r.get(0))?);
+        Ok(())
+    }
+
+    #[cfg(feature = "parquet")]
     #[test]
     fn test_extension_remote_parquet() -> Result<()> {
         let db = Connection::open_in_memory()?;

crates/duckdb/src/extension.rs

@@ -120,7 +120,14 @@ mod row;
 mod statement;
 mod transaction;
 
-#[cfg(feature = "extensions-full")]
+#[cfg(any(
+    feature = "autocomplete",
+    feature = "icu",
+    feature = "json",
+    feature = "parquet",
+    feature = "tpcds",
+    feature = "tpch"
+))]
 mod extension;
 
 pub mod profiling;

crates/duckdb/src/lib.rs

@@ -17,9 +17,15 @@ rust-version = { workspace = true }
 [features]
 default = ["vcpkg", "pkg-config"]
 bundled = ["cc"]
+# Warning: experimental feature
+bundled-cmake = ["bundled", "dep:cmake", "dep:which", "parquet"]
 buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"]
 json = ["bundled"]
 parquet = ["bundled"]
+autocomplete = ["bundled-cmake"]
+icu = ["bundled-cmake"]
+tpcds = ["bundled-cmake"]
+tpch = ["bundled-cmake"]
 extensions-full = ["json", "parquet"]
 winduckdb = []
 # Warning: experimental feature
@@ -28,6 +34,7 @@ loadable-extension = ["prettyplease", "quote", "syn"]
 [build-dependencies]
 bindgen = { workspace = true, features = ["runtime"], optional = true }
 cc = { workspace = true, features = ["parallel"], optional = true }
+cmake = { version = "0.1", optional = true }
 flate2 = { workspace = true }
 pkg-config = { workspace = true, optional = true }
 prettyplease = { workspace = true, optional = true }
@@ -41,6 +48,7 @@ serde_json = { workspace = true }
 syn = { workspace = true, optional = true }
 tar = { workspace = true }
 vcpkg = { workspace = true, optional = true }
+which = { version = "8", optional = true }
 zip = { version = "6", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]