[WIP] Upgrade to V1.2.0 (duckdb#427)

* fetch extension name and version from env variable

* set error message

* fix: Support boolean lists

* test: Add boolean list roundtrip test

* Fix link formatting in CONTRIBUTING.md

* derive Debug for Savepoint

* Use ptr::write when writing uninitialized memory

* Use smaller unsafe blocks

* Rust BindData/InitData can just use Rust types

* Use unsafe blocks inside unsafe fns generated by macro

This will prevent the macro generating a warning in edition 2024

* Fix unused import

* Better safety docs for vtab methods

Contrary to the previous docs, the instances passed to these functions are *not* initialized by the caller. Rather, the called function is responsible for writing into uninitialized memory.

* Similar safety fixes in vtab tests

* Update pola-rs

Try to get the fix for pola-rs/polars#19063 in a better way to fix CI

* Remove pola-rs workaround

* Upgrade arrow-rs to 54

* Fix test  to trigger append error

* update to 1.2.0

* update polars again

* reexport polars-arrow, fix lifetime clippy

* upgrade sources, bindgen

* remove polars from default

* uh update source again?

* upgrade bindgen

* limit bindgen

* allow unsatble/deprecated c api for now

* format that shidd

* install openssl

* ok, update it and if it doesnt work we're getting rid of it

* yeet

* remove asan for now

* adjust timestamp tests for utc

* fix logicaltypeid export

* format

* add logicaltypeid export again

---------

Co-authored-by: Sam Ansmink <[email protected]>
Co-authored-by: peasee <[email protected]>
Co-authored-by: Alex Kesling <[email protected]>
Co-authored-by: Zheyu Shen <[email protected]>
Co-authored-by: Martin Pool <[email protected]>
Co-authored-by: John Allen <[email protected]>
Co-authored-by: nm <[email protected]>

Author: 8 people

.github/.codecov.yml

@@ -39,7 +39,7 @@ jobs:
         name: Download duckdb
         with:
           repository: "duckdb/duckdb"
-          tag: "v1.1.1"
+          tag: "v1.2.0"
           fileName: ${{ matrix.duckdb }}
           out-file-path: .
 
@@ -54,11 +54,6 @@ jobs:
       - run: cargo fmt --all -- --check
         if: matrix.os == 'ubuntu-latest'
 
-      # TODO: remove
-      - name: Workaround for https://github.com/pola-rs/polars/issues/19063
-        run: |
-          cargo update [email protected] --precise 2.5.0
-
       - name: run cargo clippy
         if: matrix.os == 'ubuntu-latest'
         env:
@@ -67,24 +62,7 @@ jobs:
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
         run: |
           cargo clippy --all-targets --workspace --all-features -- -D warnings -A clippy::redundant-closure
-          
-
-      - name: Run cargo-tarpaulin
-        if: matrix.os == 'ubuntu-latest'
-        uses: actions-rs/[email protected]
-        with:
-          # Intentionally omit time feature until we're on time 0.3, at which
-          # point it should be added to `bundled-full`.
-          args: '--features "buildtime_bindgen modern-full vtab-full vtab-loadable" --avoid-cfg-tarpaulin' # TODO restore to normal (https://github.com/xd009642/tarpaulin/issues/756#issuecomment-838769320)
-          version: 0.22.0
-        env:
-          DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
-          DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
-          LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
-
-      - name: Upload to codecov.io
-        if: matrix.os == 'ubuntu-latest'
-        uses: codecov/codecov-action@v1
+    
 
       # For windows
       - name: Windows extract duckdb
@@ -139,36 +117,4 @@ jobs:
           rust-version: stable
           targets: x86_64-pc-windows-msvc
 
-      - run: cargo install cargo-examples
-
-  Sanitizer:
-    name: Address Sanitizer
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      # Need nightly rust.
-      - uses: hecrj/setup-rust-action@v1
-        with:
-          rust-version: nightly
-          components: rust-src
-      - name: Tests with asan
-        env:
-          RUSTFLAGS: -Zsanitizer=address -C debuginfo=0
-          RUSTDOCFLAGS: -Zsanitizer=address
-          ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_leaks=1"
-          # Work around https://github.com/rust-lang/rust/issues/59125 by
-          # disabling backtraces. In an ideal world we'd probably suppress the
-          # leak sanitization, but we don't care about backtraces here, so long
-          # as the other tests have them.
-          RUST_BACKTRACE: "0"
-        run: |
-          # TODO switch back to modern-full once polars is fixed
-          cargo -Z build-std test --features "chrono serde_json url r2d2 uuid extensions-full" --target x86_64-unknown-linux-gnu --package duckdb
-      - name: publish crates --dry-run
-        uses: katyo/publish-crates@v2
-        with:
-          path: './'
-          args: --allow-dirty --all-features
-          dry-run: true
-          ignore-unpublished-changes: true
+      - run: cargo install cargo-examples

.github/workflows/rust.yaml

@@ -9,7 +9,7 @@ Most user should use `duckdb`, but our development may happen in both of these c
 ## Development
 
 ### duckdb-c-api
-Some features are still not implemented in the c api, so we may need pull request in the [duckdb repo]((https://github.com/duckdb/duckdb).
+Some features are still not implemented in the c api, so we may need pull request in the [duckdb repo](https://github.com/duckdb/duckdb).
 
 ```shell
 # build duckdb

CONTRIBUTING.md

@@ -7,7 +7,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.1.1"
+version = "1.2.0"
 authors = ["wangfenjin <[email protected]>"]
 edition = "2021"
 repository = "https://github.com/duckdb/duckdb-rs"
@@ -19,11 +19,11 @@ license = "MIT"
 categories = ["database"]
 
 [workspace.dependencies]
-duckdb = { version = "1.1.1", path = "crates/duckdb" }
-libduckdb-sys = { version = "1.1.1", path = "crates/libduckdb-sys" }
+duckdb = { version = "1.2.0", path = "crates/duckdb" }
+libduckdb-sys = { version = "1.2.0", path = "crates/libduckdb-sys" }
 duckdb-loadable-macros = { version = "0.1.3", path = "crates/duckdb-loadable-macros" }
 autocfg = "1.0"
-bindgen = { version = "0.69", default-features = false }
+bindgen = { version = "0.71.1", default-features = false }
 byteorder = "1.3"
 calamine = "0.22.0"
 cast = "0.3"
@@ -40,8 +40,9 @@ memchr = "2.3"
 num = { version = "0.4", default-features = false }
 num-integer = "0.1.46"
 pkg-config = "0.3.24"
-polars = "0.35.4"
-polars-core = "0.35.4"
+polars = "0.46"
+polars-core = "0.46"
+polars-arrow = "0.46"
 pretty_assertions = "1.4.0"
 prettyplease = "0.2.20"
 proc-macro2 = "1.0.56"
@@ -62,4 +63,4 @@ unicase = "2.6.0"
 url = "2.1"
 uuid = "1.0"
 vcpkg = "0.2"
-arrow = { version = "53", default-features = false }
+arrow = { version = "54", default-features = false }

Cargo.toml

@@ -8,12 +8,14 @@ use quote::quote_spanned;
 
 use darling::{ast::NestedMeta, Error, FromMeta};
 
+use std::env;
+
 /// For parsing the arguments to the duckdb_entrypoint_c_api macro
 #[derive(Debug, FromMeta)]
 struct CEntryPointMacroArgs {
     #[darling(default)]
     /// The name to be given to this extension. This name is used in the entrypoint function called by duckdb
-    ext_name: String,
+    ext_name: Option<String>,
     /// The minimum C API version this extension requires. It is recommended to set this to the lowest possible version
     /// at which your extension still compiles
     min_duckdb_version: Option<String>,
@@ -40,17 +42,22 @@ pub fn duckdb_entrypoint_c_api(attr: TokenStream, item: TokenStream) -> TokenStr
     // Set the minimum duckdb version (dev by default)
     let minimum_duckdb_version = match args.min_duckdb_version {
         Some(i) => i,
-        None => "dev".to_string(),
+        None => env::var("DUCKDB_EXTENSION_MIN_DUCKDB_VERSION").expect("Please either set env var DUCKDB_EXTENSION_MIN_DUCKDB_VERSION or pass it as an argument to the proc macro").to_string()
+    };
+
+    let extension_name = match args.ext_name {
+        Some(i) => i,
+        None => env::var("DUCKDB_EXTENSION_NAME").expect("Please either set env var DUCKDB_EXTENSION_MIN_DUCKDB_VERSION or pass it as an argument to the proc macro").to_string()
     };
 
     let ast = parse_macro_input!(item as syn::Item);
 
     match ast {
         Item::Fn(func) => {
-            let c_entrypoint = Ident::new(format!("{}_init_c_api", args.ext_name).as_str(), Span::call_site());
+            let c_entrypoint = Ident::new(format!("{}_init_c_api", extension_name).as_str(), Span::call_site());
             let prefixed_original_function = func.sig.ident.clone();
             let c_entrypoint_internal = Ident::new(
-                format!("{}_init_c_api_internal", args.ext_name).as_str(),
+                format!("{}_init_c_api_internal", extension_name).as_str(),
                 Span::call_site(),
             );
 
@@ -133,16 +140,20 @@ pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream {
                 /// Will be called by duckdb
                 #[no_mangle]
                 pub unsafe extern "C" fn #c_entrypoint(db: *mut c_void) {
-                    let connection = Connection::open_from_raw(db.cast()).expect("can't open db connection");
-                    #prefixed_original_function(connection).expect("init failed");
+                    unsafe {
+                        let connection = Connection::open_from_raw(db.cast()).expect("can't open db connection");
+                        #prefixed_original_function(connection).expect("init failed");
+                    }
                 }
 
                 /// # Safety
                 ///
                 /// Predefined function, don't need to change unless you are sure
                 #[no_mangle]
                 pub unsafe extern "C" fn #c_entrypoint_version() -> *const c_char {
-                    ffi::duckdb_library_version()
+                    unsafe {
+                        ffi::duckdb_library_version()
+                    }
                 }
 
 

crates/duckdb-loadable-macros/src/lib.rs

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb"
-version = "1.1.1"
+version = "1.2.0"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
@@ -31,7 +31,7 @@ vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
 extensions-full = ["json", "parquet", "vtab-full"]
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
 modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
-polars = ["dep:polars"]
+polars = ["dep:polars", "dep:polars-arrow"]
 # FIXME: These were added to make clippy happy: these features appear unused and should perhaps be removed
 column_decltype = []
 extra_check = []
@@ -61,6 +61,7 @@ calamine = { workspace = true, optional = true }
 num = { workspace = true, features = ["std"], optional = true }
 duckdb-loadable-macros = { workspace = true, optional = true }
 polars = { workspace = true, features = ["dtype-full"], optional = true }
+polars-arrow = { workspace = true, optional = true }
 num-integer = { workspace = true }
 
 [dev-dependencies]

crates/duckdb/Cargo.toml

@@ -1,3 +1,5 @@
+#![warn(unsafe_op_in_unsafe_fn)]
+
 extern crate duckdb;
 extern crate duckdb_loadable_macros;
 extern crate libduckdb_sys;
@@ -12,25 +14,15 @@ use libduckdb_sys as ffi;
 use std::{
     error::Error,
     ffi::{c_char, c_void, CString},
+    ptr,
 };
 
-#[repr(C)]
 struct HelloBindData {
-    name: *mut c_char,
+    name: String,
 }
 
-impl Free for HelloBindData {
-    fn free(&mut self) {
-        unsafe {
-            if self.name.is_null() {
-                return;
-            }
-            drop(CString::from_raw(self.name));
-        }
-    }
-}
+impl Free for HelloBindData {}
 
-#[repr(C)]
 struct HelloInitData {
     done: bool,
 }
@@ -45,37 +37,32 @@ impl VTab for HelloVTab {
 
     unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box<dyn std::error::Error>> {
         bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar));
-        let param = bind.get_parameter(0).to_string();
+        let name = bind.get_parameter(0).to_string();
         unsafe {
-            (*data).name = CString::new(param).unwrap().into_raw();
+            ptr::write(data, HelloBindData { name });
         }
         Ok(())
     }
 
     unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box<dyn std::error::Error>> {
         unsafe {
-            (*data).done = false;
+            ptr::write(data, HelloInitData { done: false });
         }
         Ok(())
     }
 
     unsafe fn func(func: &FunctionInfo, output: &mut DataChunkHandle) -> Result<(), Box<dyn std::error::Error>> {
-        let init_info = func.get_init_data::<HelloInitData>();
-        let bind_info = func.get_bind_data::<HelloBindData>();
+        let init_info = unsafe { func.get_init_data::<HelloInitData>().as_mut().unwrap() };
+        let bind_info = unsafe { func.get_bind_data::<HelloBindData>().as_mut().unwrap() };
 
-        unsafe {
-            if (*init_info).done {
-                output.set_len(0);
-            } else {
-                (*init_info).done = true;
-                let vector = output.flat_vector(0);
-                let name = CString::from_raw((*bind_info).name);
-                let result = CString::new(format!("Hello {}", name.to_str()?))?;
-                // Can't consume the CString
-                (*bind_info).name = CString::into_raw(name);
-                vector.insert(0, result);
-                output.set_len(1);
-            }
+        if init_info.done {
+            output.set_len(0);
+        } else {
+            init_info.done = true;
+            let vector = output.flat_vector(0);
+            let result = CString::new(format!("Hello {}", bind_info.name))?;
+            vector.insert(0, result);
+            output.set_len(1);
         }
         Ok(())
     }

crates/duckdb/examples/hello-ext/main.rs

@@ -288,20 +288,23 @@ mod test {
 
     #[test]
     fn test_appender_error() -> Result<(), crate::Error> {
+        use crate::params;
         let conn = Connection::open_in_memory()?;
         conn.execute(
             r"CREATE TABLE foo (
             foobar TEXT,
+            foobar_int INT,
             foobar_split TEXT[] AS (split(trim(foobar), ','))
             );",
             [],
         )?;
         let mut appender = conn.appender("foo")?;
-        match appender.append_row(["foo"]) {
+        match appender.append_row(params!["foo"]) {
             Err(crate::Error::DuckDBFailure(.., Some(msg))) => {
                 assert_eq!(msg, "Call to EndRow before all columns have been appended to!")
             }
-            _ => panic!("expected error"),
+            Err(err) => panic!("unexpected error: {:?}", err),
+            Ok(_) => panic!("expected an error but got Ok"),
         }
         Ok(())
     }

crates/duckdb/src/appender/mod.rs

@@ -9,6 +9,7 @@ pub struct Arrow<'stmt> {
     pub(crate) stmt: Option<&'stmt Statement<'stmt>>,
 }
 
+#[allow(clippy::needless_lifetimes)]
 impl<'stmt> Arrow<'stmt> {
     #[inline]
     pub(crate) fn new(stmt: &'stmt Statement<'stmt>) -> Arrow<'stmt> {
@@ -22,6 +23,7 @@ impl<'stmt> Arrow<'stmt> {
     }
 }
 
+#[allow(clippy::needless_lifetimes)]
 impl<'stmt> Iterator for Arrow<'stmt> {
     type Item = RecordBatch;
 
@@ -32,11 +34,13 @@ impl<'stmt> Iterator for Arrow<'stmt> {
 
 /// A handle for the resulting RecordBatch of a query in streaming
 #[must_use = "Arrow stream is lazy and will not fetch data unless consumed"]
+#[allow(clippy::needless_lifetimes)]
 pub struct ArrowStream<'stmt> {
     pub(crate) stmt: Option<&'stmt Statement<'stmt>>,
     pub(crate) schema: SchemaRef,
 }
 
+#[allow(clippy::needless_lifetimes)]
 impl<'stmt> ArrowStream<'stmt> {
     #[inline]
     pub(crate) fn new(stmt: &'stmt Statement<'stmt>, schema: SchemaRef) -> ArrowStream<'stmt> {
@@ -53,6 +57,7 @@ impl<'stmt> ArrowStream<'stmt> {
     }
 }
 
+#[allow(clippy::needless_lifetimes)]
 impl<'stmt> Iterator for ArrowStream<'stmt> {
     type Item = RecordBatch;