Initial commit for python table providers

Cargo.toml

@@ -1,132 +1,37 @@
-[package]
-name = "datafusion-table-providers"
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[workspace]
+members = [
+    "core",
+    "python",
+]
+resolver = "2"
+
+[workspace.package]
 version = "0.2.3"
 readme = "README.md"
 edition = "2021"
 repository = "https://github.com/datafusion-contrib/datafusion-table-providers"
 license = "Apache-2.0"
 description = "Extend the capabilities of DataFusion to support additional data sources via implementations of the `TableProvider` trait."
 
-[dependencies]
-arrow-flight = { version = "53", optional = true, features = [
-  "flight-sql-experimental",
-  "tls",
-] }
-arrow-odbc = { version = "14.0", optional = true }
-async-stream = { version = "0.3.6", optional = true }
-async-trait = "0.1"
-bb8 = { version = "0.8", optional = true }
-bb8-postgres = { version = "0.8", optional = true }
-bigdecimal = "0.4.6"
-byteorder = "1.5.0"
-chrono = "0.4.38"
-dashmap = "6.1.0"
+[workspace.dependencies]
 datafusion = { version = "43", default-features = false }
-datafusion-federation = { version = "0.3.1", features = [
-  "sql",
-], optional = true }
-datafusion-proto = { version = "43", optional = true }
-duckdb = { version = "1.1.1", features = [
-  "bundled",
-  "r2d2",
-  "vtab",
-  "vtab-arrow",
-  "appender-arrow",
-], optional = true }
-dyn-clone = { version = "1.0", optional = true }
-fallible-iterator = "0.3.0"
-fundu = "2.0.1"
-futures = "0.3"
-geo-types = "0.7"
-itertools = "0.13.0"
-mysql_async = { version = "0.34", features = [
-  "native-tls-tls",
-  "chrono",
-], optional = true }
-native-tls = { version = "0.2.12", optional = true }
-num-bigint = "0.4"
-odbc-api = { version = "10.0.0", optional = true }
-pem = { version = "3.0.4", optional = true }
-postgres-native-tls = { version = "0.5.0", optional = true }
-prost = { version = "0.13", optional = true }
-r2d2 = { version = "0.8.10", optional = true }
-rusqlite = { version = "0.32.1", optional = true }
-sea-query = { version = "0.32.0", features = [
-  "backend-sqlite",
-  "backend-postgres",
-  "postgres-array",
-  "with-rust_decimal",
-  "with-bigdecimal",
-  "with-time",
-  "with-chrono",
-] }
-secrecy = "0.8.0"
-serde = { version = "1.0", optional = true }
-serde_json = "1.0"
-sha2 = "0.10.8"
-snafu = "0.8.5"
-time = "0.3.36"
-tokio = { version = "1.41", features = ["macros", "fs"] }
-tokio-postgres = { version = "0.7.12", features = [
-  "with-chrono-0_4",
-  "with-uuid-1",
-  "with-serde_json-1",
-  "with-geo-types-0_7",
-], optional = true }
-tokio-rusqlite = { version = "0.6.0", optional = true }
-tonic = { version = "0.12", optional = true, features = [
-  "tls-native-roots",
-  "tls-webpki-roots",
-] }
-tracing = "0.1.40"
-trust-dns-resolver = "0.23.2"
-url = "2.5.4"
-uuid = { version = "1.11.0", optional = true }
-
-[dev-dependencies]
-anyhow = "1.0"
-bollard = "0.18.1"
-geozero = { version = "0.14.0", features = ["with-wkb"] }
-insta = { version = "1.41.1", features = ["filters"] }
-prost = { version = "0.13" }
-rand = "0.8.5"
-reqwest = "0.12.9"
-rstest = "0.23.0"
-test-log = { version = "0.2.16", features = ["trace"] }
-tokio-stream = { version = "0.1.16", features = ["net"] }
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
-
-[features]
-duckdb = [
-  "dep:duckdb",
-  "dep:r2d2",
-  "dep:uuid",
-  "dep:dyn-clone",
-  "dep:async-stream",
-]
-duckdb-federation = ["duckdb", "federation"]
-federation = ["dep:datafusion-federation"]
-flight = [
-  "dep:arrow-flight",
-  "datafusion/serde",
-  "dep:datafusion-proto",
-  "dep:serde",
-  "dep:tonic",
-]
-mysql = ["dep:mysql_async", "dep:async-stream"]
-mysql-federation = ["mysql", "federation"]
-odbc = ["dep:odbc-api", "dep:arrow-odbc", "dep:async-stream", "dep:dyn-clone"]
-odbc-federation = ["odbc", "federation"]
-postgres = [
-  "dep:tokio-postgres",
-  "dep:uuid",
-  "dep:postgres-native-tls",
-  "dep:bb8",
-  "dep:bb8-postgres",
-  "dep:native-tls",
-  "dep:pem",
-  "dep:async-stream",
-]
-postgres-federation = ["postgres", "federation"]
-sqlite = ["dep:rusqlite", "dep:tokio-rusqlite"]
-sqlite-federation = ["sqlite", "federation"]
+datafusion-ffi = { version = "43" }
+datafusion-proto = { version = "43" }
+datafusion-table-providers = { path = "core" }

core/Cargo.toml

@@ -0,0 +1,132 @@
+[package]
+name = "datafusion-table-providers"
+version = { workspace = true }
+readme = { workspace = true }
+edition = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+description = { workspace = true }
+
+[dependencies]
+arrow-flight = { version = "53", optional = true, features = [
+  "flight-sql-experimental",
+  "tls",
+] }
+arrow-odbc = { version = "14.0", optional = true }
+async-stream = { version = "0.3.6", optional = true }
+async-trait = "0.1"
+bb8 = { version = "0.8", optional = true }
+bb8-postgres = { version = "0.8", optional = true }
+bigdecimal = "0.4.6"
+byteorder = "1.5.0"
+chrono = "0.4.38"
+dashmap = "6.1.0"
+datafusion = { workspace = true }
+datafusion-federation = { version = "0.3.1", features = [
+  "sql",
+], optional = true }
+datafusion-proto = { workspace = true, optional = true }
+duckdb = { version = "1.1.1", features = [
+  "bundled",
+  "r2d2",
+  "vtab",
+  "vtab-arrow",
+  "appender-arrow",
+], optional = true }
+dyn-clone = { version = "1.0", optional = true }
+fallible-iterator = "0.3.0"
+fundu = "2.0.1"
+futures = "0.3"
+geo-types = "0.7"
+itertools = "0.13.0"
+mysql_async = { version = "0.34", features = [
+  "native-tls-tls",
+  "chrono",
+], optional = true }
+native-tls = { version = "0.2.12", optional = true }
+num-bigint = "0.4"
+odbc-api = { version = "10.0.0", optional = true }
+pem = { version = "3.0.4", optional = true }
+postgres-native-tls = { version = "0.5.0", optional = true }
+prost = { version = "0.13", optional = true }
+r2d2 = { version = "0.8.10", optional = true }
+rusqlite = { version = "0.32.1", optional = true }
+sea-query = { version = "0.32.0", features = [
+  "backend-sqlite",
+  "backend-postgres",
+  "postgres-array",
+  "with-rust_decimal",
+  "with-bigdecimal",
+  "with-time",
+  "with-chrono",
+] }
+secrecy = "0.8.0"
+serde = { version = "1.0", optional = true }
+serde_json = "1.0"
+sha2 = "0.10.8"
+snafu = "0.8.5"
+time = "0.3.36"
+tokio = { version = "1.41", features = ["macros", "fs"] }
+tokio-postgres = { version = "0.7.12", features = [
+  "with-chrono-0_4",
+  "with-uuid-1",
+  "with-serde_json-1",
+  "with-geo-types-0_7",
+], optional = true }
+tokio-rusqlite = { version = "0.6.0", optional = true }
+tonic = { version = "0.12", optional = true, features = [
+  "tls-native-roots",
+  "tls-webpki-roots",
+] }
+tracing = "0.1.40"
+trust-dns-resolver = "0.23.2"
+url = "2.5.4"
+uuid = { version = "1.11.0", optional = true }
+
+[dev-dependencies]
+anyhow = "1.0"
+bollard = "0.18.1"
+geozero = { version = "0.14.0", features = ["with-wkb"] }
+insta = { version = "1.41.1", features = ["filters"] }
+prost = { version = "0.13" }
+rand = "0.8.5"
+reqwest = "0.12.9"
+rstest = "0.23.0"
+test-log = { version = "0.2.16", features = ["trace"] }
+tokio-stream = { version = "0.1.16", features = ["net"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+
+[features]
+duckdb = [
+  "dep:duckdb",
+  "dep:r2d2",
+  "dep:uuid",
+  "dep:dyn-clone",
+  "dep:async-stream",
+]
+duckdb-federation = ["duckdb", "federation"]
+federation = ["dep:datafusion-federation"]
+flight = [
+  "dep:arrow-flight",
+  "datafusion/serde",
+  "dep:datafusion-proto",
+  "dep:serde",
+  "dep:tonic",
+]
+mysql = ["dep:mysql_async", "dep:async-stream"]
+mysql-federation = ["mysql", "federation"]
+odbc = ["dep:odbc-api", "dep:arrow-odbc", "dep:async-stream", "dep:dyn-clone"]
+odbc-federation = ["odbc", "federation"]
+postgres = [
+  "dep:tokio-postgres",
+  "dep:uuid",
+  "dep:postgres-native-tls",
+  "dep:bb8",
+  "dep:bb8-postgres",
+  "dep:native-tls",
+  "dep:pem",
+  "dep:async-stream",
+]
+postgres-federation = ["postgres", "federation"]
+sqlite = ["dep:rusqlite", "dep:tokio-rusqlite"]
+sqlite-federation = ["sqlite", "federation"]

python/.cargo/config.toml

@@ -0,0 +1,12 @@
+[target.x86_64-apple-darwin]
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]
+
+[target.aarch64-apple-darwin]
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]
+

python/.gitignore

@@ -0,0 +1,26 @@
+/venv
+.idea
+.DS_Store
+.vscode
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Python dist ignore
+dist
+
+# C extensions
+*.so
+
+# Python dist
+dist
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+venv
+.venv
+

python/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "datafusion-table-providers-python"
+version = { workspace = true }
+readme = { workspace = true }
+edition = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+description = { workspace = true }
+
+[lib]
+name = "datafusion_table_providers"
+crate-type = ["cdylib"]
+doc = false
+
+[dependencies]
+arrow = { version = "53", features = ["pyarrow"] }
+datafusion = { workspace = true, features = ["pyarrow"] }
+datafusion-ffi = { workspace = true }
+datafusion-table-providers = { workspace = true, features = ["sqlite"] }
+pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py39"] }
+tokio = { version = "1.42", features = ["macros", "rt", "rt-multi-thread", "sync"] }

python/examples/sqlite_demo.py

@@ -0,0 +1,11 @@
+from datafusion import SessionContext
+from datafusion_table_providers import sqlite
+
+ctx = SessionContext()
+pool = sqlite.SqliteTableFactory("../../core/examples/sqlite_example.db", "file", 3.0, None)
+tables = pool.tables()
+
+for t in tables:
+    ctx.register_table_provider(t, pool.get_table(t))
+    print("Checking table:", t)
+    ctx.table(t).show()