elixir-dux
diff --git a/‎.github/workflows/ci.yml‎
Lines changed: 3 additions & 29 deletions b/‎.github/workflows/ci.yml‎
Lines changed: 3 additions & 29 deletions
diff --git a/‎.github/workflows/release.yml‎
Lines changed: 0 additions & 82 deletions b/‎.github/workflows/release.yml‎
Lines changed: 0 additions & 82 deletions
diff --git a/‎RELEASING.md‎
Lines changed: 0 additions & 70 deletions b/‎RELEASING.md‎
Lines changed: 0 additions & 70 deletions
diff --git a/‎bench/compare_backend.exs‎
Lines changed: 118 additions & 0 deletions b/‎bench/compare_backend.exs‎
Lines changed: 118 additions & 0 deletions
diff --git a/‎bench/results/adbc-migration.md‎
Lines changed: 33 additions & 0 deletions b/‎bench/results/adbc-migration.md‎
Lines changed: 33 additions & 0 deletions
diff --git a/‎bench/results/history.csv‎
Lines changed: 3 additions & 0 deletions b/‎bench/results/history.csv‎
Lines changed: 3 additions & 0 deletions
@@ -8,7 +8,6 @@ on:
 
 env:
   MIX_ENV: test
-  DUX_BUILD: "true"
 
 jobs:
   test:
@@ -32,16 +31,12 @@ jobs:
           otp-version: ${{ matrix.otp }}
           elixir-version: ${{ matrix.elixir }}
 
-      - uses: dtolnay/rust-toolchain@stable
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: native/dux
-          key: nif-${{ runner.os }}
-
       - name: Install dependencies
         run: mix deps.get
 
+      - name: Compile dependencies
+        run: mix deps.compile
+
       - name: Compile (warnings as errors)
         run: mix compile --warnings-as-errors
 
@@ -68,24 +63,3 @@ jobs:
 
       - name: Check format
         run: mix format --check-formatted
-
-  rust-lint:
-    name: Rust lint
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: native/dux
-
-      - name: Check format
-        run: cargo fmt --manifest-path=native/dux/Cargo.toml --all -- --check
-
-      - name: Clippy
-        run: cargo clippy --manifest-path=native/dux/Cargo.toml -- -Dwarnings
@@ -0,0 +1,118 @@
+defmodule Bench do
+  def small_list do
+    Enum.map(1..100, &%{id: &1, region: Enum.at(["US", "EU", "APAC"], rem(&1, 3)), amount: &1 * 10})
+  end
+
+  def medium_list do
+    Enum.map(1..10_000, &%{id: &1, region: Enum.at(["US", "EU", "APAC"], rem(&1, 3)), amount: &1 * 10})
+  end
+
+  def setup_parquet do
+    dir = Path.join(System.tmp_dir!(), "dux_bench_#{System.unique_integer([:positive])}")
+    File.mkdir_p!(dir)
+    path = Path.join(dir, "bench.parquet")
+    Dux.from_list(medium_list()) |> Dux.to_parquet(path)
+    {dir, path}
+  end
+
+  def from_list_100, do: Dux.from_list(small_list()) |> Dux.compute()
+  def from_list_10k, do: Dux.from_list(medium_list()) |> Dux.compute()
+  def from_query_10k, do: Dux.from_query("SELECT * FROM range(10000) t(x)") |> Dux.compute()
+
+  def from_parquet(path), do: Dux.from_parquet(path) |> Dux.compute()
+
+  def filter_mutate do
+    Dux.from_list(medium_list())
+    |> Dux.filter_with("amount > 5000")
+    |> Dux.mutate_with(doubled: "amount * 2")
+    |> Dux.compute()
+  end
+
+  def group_summarise do
+    Dux.from_list(medium_list())
+    |> Dux.group_by(:region)
+    |> Dux.summarise_with(total: "SUM(amount)", n: "COUNT(*)", avg: "AVG(amount)")
+    |> Dux.compute()
+  end
+
+  def full_pipeline do
+    Dux.from_list(medium_list())
+    |> Dux.filter_with("amount > 5000")
+    |> Dux.group_by(:region)
+    |> Dux.summarise_with(total: "SUM(amount)", n: "COUNT(*)")
+    |> Dux.sort_by(desc: :total)
+    |> Dux.to_rows()
+  end
+
+  def join_small do
+    left = Dux.from_list(small_list())
+    right = Dux.from_list([%{id: 1, tag: "a"}, %{id: 50, tag: "b"}, %{id: 100, tag: "c"}])
+    Dux.join(left, right, on: :id) |> Dux.compute()
+  end
+
+  def to_columns_10k, do: Dux.from_list(medium_list()) |> Dux.to_columns()
+  def to_rows_1k, do: Dux.from_list(small_list()) |> Dux.to_rows()
+end
+
+{dir, parquet_path} = Bench.setup_parquet()
+
+IO.puts("\n=== Dux Backend Benchmark ===\n")
+
+Benchee.run(
+  %{
+    "from_list(100) → compute" => &Bench.from_list_100/0,
+    "from_list(10K) → compute" => &Bench.from_list_10k/0,
+    "from_query(range 10K) → compute" => &Bench.from_query_10k/0,
+    "from_parquet → compute" => fn -> Bench.from_parquet(parquet_path) end,
+    "filter + mutate → compute" => &Bench.filter_mutate/0,
+    "group_by + summarise → compute" => &Bench.group_summarise/0,
+    "full pipeline → to_rows" => &Bench.full_pipeline/0,
+    "join (small) → compute" => &Bench.join_small/0,
+    "to_columns (10K rows)" => &Bench.to_columns_10k/0,
+    "to_rows (1K rows)" => &Bench.to_rows_1k/0
+  },
+  time: 3,
+  warmup: 1,
+  memory_time: 1,
+  print: [configuration: false]
+)
+
+IO.puts("\n=== Distributed Benchmark (2 local workers) ===\n")
+
+{:ok, w1} = Dux.Remote.Worker.start_link()
+{:ok, w2} = Dux.Remote.Worker.start_link()
+
+defmodule DistBench do
+  def local(medium) do
+    Dux.from_list(medium)
+    |> Dux.group_by(:region)
+    |> Dux.summarise_with(total: "SUM(amount)")
+    |> Dux.to_rows()
+  end
+
+  def distributed(medium, workers) do
+    Dux.from_list(medium)
+    |> Dux.distribute(workers)
+    |> Dux.group_by(:region)
+    |> Dux.summarise_with(total: "SUM(amount)")
+    |> Dux.to_rows()
+  end
+end
+
+medium = Bench.medium_list()
+
+Benchee.run(
+  %{
+    "local: group_by + summarise" => fn -> DistBench.local(medium) end,
+    "distributed(2): group_by + summarise" => fn -> DistBench.distributed(medium, [w1, w2]) end
+  },
+  time: 3,
+  warmup: 1,
+  print: [configuration: false]
+)
+
+GenServer.stop(w1)
+GenServer.stop(w2)
+File.rm_rf!(dir)
+
+IO.puts("\nDone.")
@@ -0,0 +1,33 @@
+# Benchmark: ADBC Migration (v0.1.1 NIF → ADBC)
+
+Machine: Apple M-series (arm64), macOS
+Date: 2026-03-23
+
+## Results
+
+| Operation | NIF (v0.1.1) | ADBC | Change |
+|-----------|-------------|------|--------|
+| from_query(10K) → compute | 0.10ms | 1.18ms | 12x slower |
+| from_list(100) → compute | 3.41ms | 6.04ms | 1.8x slower |
+| to_rows (1K) | 3.69ms | 6.78ms | 1.8x slower |
+| join (small) → compute | 4.24ms | 6.39ms | 1.5x slower |
+| **from_list(10K) → compute** | **3675ms** | **5.81ms** | **633x faster** |
+| **full pipeline → to_rows** | **625ms** | **4.88ms** | **128x faster** |
+| **group_by + summarise** | **655ms** | **5.81ms** | **113x faster** |
+| **filter + mutate → compute** | **3143ms** | **7.29ms** | **431x faster** |
+| **to_columns (10K)** | **3150ms** | **8.08ms** | **390x faster** |
+| distributed(2) vs local | — | 1.94x | — |
+
+## Analysis
+
+**ADBC is slower for pure SQL queries** (12x for `from_query`) because the NIF kept data
+in Arrow RecordBatches in Rust memory (no temp table creation), while ADBC ingests into
+a temp table (query + materialize + ingest).
+
+**ADBC is massively faster for `from_list` operations** (100-633x) because `from_list` in
+v0.1.1 generated a SQL `UNION ALL` for each row — a catastrophically expensive approach
+for >500 rows. ADBC uses `Adbc.Connection.ingest` which bypasses SQL entirely, going
+directly from Elixir data → Arrow columnar → DuckDB temp table.
+
+The NIF's `from_list` bottleneck affected every benchmark that used it (full pipeline,
+group_by, filter, to_columns) since the data source was always `from_list(medium_list())`.
@@ -0,0 +1,3 @@
+version,sha,date,from_query_10k_ms,from_list_100_ms,from_list_10k_ms,to_rows_1k_ms,to_columns_10k_ms,join_small_ms,full_pipeline_ms,group_summarise_ms,filter_mutate_ms,distributed_2_ms
+v0.1.1-nif,2eb9e5d,2026-03-23,0.10,3.41,3675,3.69,3150,4.24,625,655,3143,
+v0.2.0-adbc,9ae1f93,2026-03-23,1.18,6.04,5.81,6.78,8.08,6.39,4.88,5.81,7.29,9.47
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version,sha,date,from_query_10k_ms,from_list_100_ms,from_list_10k_ms,to_rows_1k_ms,to_columns_10k_ms,join_small_ms,full_pipeline_ms,group_summarise_ms,filter_mutate_ms,distributed_2_ms`
	`2`	`+v0.1.1-nif,2eb9e5d,2026-03-23,0.10,3.41,3675,3.69,3150,4.24,625,655,3143,`
	`3`	`+v0.2.0-adbc,9ae1f93,2026-03-23,1.18,6.04,5.81,6.78,8.08,6.39,4.88,5.81,7.29,9.47`