diff --git a/data/penguins.duckdb b/data/penguins.duckdb new file mode 100644 index 0000000..a7006f3 Binary files /dev/null and b/data/penguins.duckdb differ diff --git a/notebooks/datafusion-table-provider-for-adbc.ipynb b/notebooks/datafusion-table-provider-for-adbc.ipynb new file mode 100644 index 0000000..5b46175 --- /dev/null +++ b/notebooks/datafusion-table-provider-for-adbc.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "metadata": {}, + "source": [ + "[Apache DataFusion](https://datafusion.apache.org/) is an extensible query engine written in Rust that uses [Apache Arrow](https://arrow.apache.org/) as its in-memory format. One of its powerful features is table providers—an abstraction that allows DataFusion to query data from virtually any source while presenting a unified SQL interface. The [datafusion-table-providers](https://github.com/datafusion-contrib/datafusion-table-providers) crate offers ready-made table provider implementations for common data sources, including databases accessible via ADBC.\n", + "\n", + "[ADBC](https://arrow.apache.org/adbc/) (Arrow Database Connectivity) is a database-agnostic API that retrieves query results directly in Arrow format, making it an efficient bridge between DataFusion and external databases. This demo uses DuckDB for simplicity, but ADBC's real value is providing DataFusion with access to more than a dozen different data systems that lack dedicated table providers.\n", + "\n", + "In this notebook, you will:\n", + "\n", + "1. Load an ADBC driver (DuckDB) and connect to a database.\n", + "2. Create an ADBC connection pool and table factory.\n", + "3. Register an external database table as a DataFusion table provider.\n", + "4. Query the data through DataFusion's SQL interface." + ] + }, + { + "cell_type": "markdown", + "id": "33b1e776-ebb0-48ed-8284-c73e25f3c1e1", + "metadata": {}, + "source": [ + "Install the DuckDB ADBC driver with [uv](https://docs.astral.sh/uv/) and [dbc](https://docs.columnar.tech/dbc/):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0f14d77-9832-4f7a-9d3f-ffb10d93f6ef", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [], + "source": [ + "!uvx dbc install -q duckdb" + ] + }, + { + "cell_type": "markdown", + "id": "e5dc6c5f-babd-424b-81aa-bc4c0a6f983b", + "metadata": {}, + "source": [ + "Load the required crates:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cf9cf722-d25d-4287-8616-64526a8f67ed", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [], + "source": [ + ":dep adbc_core = { version = \"0.21\" }\n", + ":dep adbc_driver_manager = { version = \"0.21\" }\n", + ":dep datafusion = { version = \"52.0\", default-features = false, features = [\"sql\"] }\n", + ":dep datafusion-table-providers = { version = \"0.10.1\", features = [\"adbc\"] }" + ] + }, + { + "cell_type": "markdown", + "id": "6860554e-0d0f-400f-b52c-7247883c9d8a", + "metadata": {}, + "source": [ + "Import the necessary types:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fe14b884", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [], + "source": [ + "use adbc_core::options::{AdbcVersion, OptionDatabase};\n", + "use adbc_core::{Driver, LOAD_FLAG_DEFAULT};\n", + "use adbc_driver_manager::ManagedDriver;\n", + "use datafusion::prelude::SessionContext;\n", + "use datafusion::sql::TableReference;\n", + "use datafusion_table_providers::adbc::AdbcTableFactory;\n", + "use datafusion_table_providers::sql::db_connection_pool::adbcpool::ADBCPool;\n", + "use std::sync::Arc;" + ] + }, + { + "cell_type": "markdown", + "id": "bef3deb5-8692-4284-a38a-be9dc870994b", + "metadata": {}, + "source": [ + "Load the DuckDB ADBC driver and create a database handle:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "eb04f0f0-de9e-4b1a-8c28-cdeadc84cbf4", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [], + "source": [ + "let mut driver = ManagedDriver::load_from_name(\n", + " \"duckdb\",\n", + " None,\n", + " AdbcVersion::default(),\n", + " LOAD_FLAG_DEFAULT,\n", + " None,\n", + ")\n", + ".unwrap();\n", + "\n", + "let database = driver\n", + " .new_database_with_opts([(OptionDatabase::Uri, \"../data/penguins.duckdb\".into())])\n", + " .unwrap();" + ] + }, + { + "cell_type": "markdown", + "id": "ae93a625-425e-46eb-8dca-1a5837590b7d", + "metadata": {}, + "source": [ + "Create an ADBC connection pool and table factory:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "88ca022f-2793-49e5-894e-40fb187ccde5", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [], + "source": [ + "let adbc_pool = Arc::new(ADBCPool::new(database, None).unwrap());\n", + "let table_factory = AdbcTableFactory::new(adbc_pool.clone());" + ] + }, + { + "cell_type": "markdown", + "id": "18384331-ba9e-4381-b92d-b939114b4414", + "metadata": {}, + "source": [ + "Create a DataFusion session context:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "5041cf26-3de4-48be-a8ee-6a904c4aab02", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [], + "source": [ + "let ctx = SessionContext::new();" + ] + }, + { + "cell_type": "markdown", + "id": "6b23b8d7-4472-4bd2-a62d-40e52885b189", + "metadata": {}, + "source": [ + "Register the `penguins` table from DuckDB as a DataFusion table using the ADBC table factory:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "72fa1c20-1e19-46d7-b04b-50a5747866c3", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "None" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ctx.register_table(\n", + " \"penguins\",\n", + " table_factory\n", + " .table_provider(TableReference::bare(\"penguins\"), None)\n", + " .await\n", + " .unwrap(),\n", + ")\n", + ".unwrap()" + ] + }, + { + "cell_type": "markdown", + "id": "e34fa879-f3ed-497a-a610-7a6e74abcee3", + "metadata": {}, + "source": [ + "Execute a SQL query through DataFusion (which fetches data from DuckDB via ADBC) and display the results:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "33441f8d-32c2-473f-96a3-aef80d4c4f5a", + "metadata": { + "vscode": { + "languageId": "rust" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------+-----------+----------------+---------------+-------------------+-------------+--------+------+\n", + "| species | island | bill_length_mm | bill_depth_mm | flipper_length_mm | body_mass_g | sex | year |\n", + "+---------+-----------+----------------+---------------+-------------------+-------------+--------+------+\n", + "| Adelie | Torgersen | 39.1 | 18.7 | 181 | 3750 | male | 2007 |\n", + "| Adelie | Torgersen | 39.5 | 17.4 | 186 | 3800 | female | 2007 |\n", + "| Adelie | Torgersen | 40.3 | 18.0 | 195 | 3250 | female | 2007 |\n", + "| Adelie | Torgersen | | | | | | 2007 |\n", + "| Adelie | Torgersen | 36.7 | 19.3 | 193 | 3450 | female | 2007 |\n", + "| Adelie | Torgersen | 39.3 | 20.6 | 190 | 3650 | male | 2007 |\n", + "| Adelie | Torgersen | 38.9 | 17.8 | 181 | 3625 | female | 2007 |\n", + "| Adelie | Torgersen | 39.2 | 19.6 | 195 | 4675 | male | 2007 |\n", + "| Adelie | Torgersen | 34.1 | 18.1 | 193 | 3475 | | 2007 |\n", + "| Adelie | Torgersen | 42.0 | 20.2 | 190 | 4250 | | 2007 |\n", + "+---------+-----------+----------------+---------------+-------------------+-------------+--------+------+\n" + ] + } + ], + "source": [ + "let df = ctx\n", + " .sql(\"SELECT * FROM datafusion.public.penguins LIMIT 10\")\n", + " .await\n", + " .unwrap();\n", + "\n", + "df.show().await.unwrap();" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Rust", + "language": "rust", + "name": "rust" + }, + "language_info": { + "codemirror_mode": "rust", + "file_extension": ".rs", + "mimetype": "text/rust", + "name": "Rust", + "pygment_lexer": "rust", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/registry.json b/registry.json index 3c9ad69..dd0aed5 100644 --- a/registry.json +++ b/registry.json @@ -134,5 +134,13 @@ "authors": ["ian-cook"], "description": "Define reusable, named connection configurations in TOML files and use them to connect to databases with ADBC, just like ODBC DSNs.", "categories": ["Database Connections"] + }, + { + "title": "DataFusion Table Provider for ADBC", + "path": "notebooks/datafusion-table-provider-for-adbc.ipynb", + "date": "2026-04-17", + "authors": ["emil-sadek"], + "description": "", + "categories": ["Database Connections"] } ] diff --git a/template/index.html.j2 b/template/index.html.j2 index 84a6228..9a0ec91 100644 --- a/template/index.html.j2 +++ b/template/index.html.j2 @@ -33,14 +33,6 @@ {%- block body_footer -%} - {% include 'components/footer.html.j2' %} {% include 'components/site-footer.html.j2' %} diff --git a/template/static/style.css b/template/static/style.css index 9f67613..4f0bbf4 100644 --- a/template/static/style.css +++ b/template/static/style.css @@ -68,6 +68,15 @@ div.jp-RenderedText pre { font-style: normal; } +.jp-RenderedText pre { + overflow-x: auto; + white-space: pre !important; +} + +.jp-InputArea-editor pre { + overflow-x: auto; +} + #navbar { font-family: var(--sans-serif); font-size: 1rem;