Support polars.scan_iceberg/delta (for non hybrid scan) (#24345)

xref #24247

Initially developed with Polars 2.0.0rc2.

Highlights:

* Doesn't support `pl.scan_iceberg/delta` for the hybrid scan path
* Add `deltalake`, `pyiceberg`, `sqlalchemy` dependencies for `cudf-polars[tests]` extra for unit testing
* The bulk of the changes are in `python/cudf_polars/cudf_polars/dsl/utils/lake.py` and `python/cudf_polars/cudf_polars/dsl/utils/deletions.py`

Authors:
  - Matthew Roeschke (https://github.com/mroeschke)

Approvers:
  - Matthew Murray (https://github.com/Matt711)
  - Bradley Dice (https://github.com/bdice)

URL: #24345

Author: mroeschke

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -30,6 +30,7 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
@@ -78,6 +79,7 @@ dependencies:
 - pre-commit
 - psutil
 - pyarrow>=19.0.0
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
@@ -106,6 +108,7 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-aarch64==2.28
 - typing_extensions>=4.0.0
@@ -114,4 +117,5 @@ dependencies:
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-129_arch-aarch64

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -30,6 +30,7 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
@@ -79,6 +80,7 @@ dependencies:
 - pre-commit
 - psutil
 - pyarrow>=19.0.0
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
@@ -107,6 +109,7 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-64==2.28
 - typing_extensions>=4.0.0
@@ -115,4 +118,5 @@ dependencies:
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-129_arch-x86_64

conda/environments/all_cuda-133_arch-aarch64.yaml

@@ -30,6 +30,7 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
@@ -78,6 +79,7 @@ dependencies:
 - pre-commit
 - psutil
 - pyarrow>=19.0.0
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
@@ -106,6 +108,7 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-aarch64==2.28
 - typing_extensions>=4.0.0
@@ -114,4 +117,5 @@ dependencies:
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-133_arch-aarch64

conda/environments/all_cuda-133_arch-x86_64.yaml

@@ -30,6 +30,7 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
@@ -79,6 +80,7 @@ dependencies:
 - pre-commit
 - psutil
 - pyarrow>=19.0.0
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
@@ -107,6 +109,7 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-64==2.28
 - typing_extensions>=4.0.0
@@ -115,4 +118,5 @@ dependencies:
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-133_arch-x86_64

dependencies.yaml

@@ -1296,10 +1296,24 @@ dependencies:
           # Used by ci/timeout_with_stack.py utility
           - psutil
           - *pandas
+          # cudf-polars unit tests for iceberg/delta IO
+          - deltalake>=1.6.0
+          - pyiceberg>=0.12.0
+          - sqlalchemy>=2.0.18
+          - pyarrow>=19.0.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - polars[iceberg]
+      - output_types: constraints
+        packages:
+          - polars
       - output_types: conda
         packages:
           # Used by timeout_with_stack.py utility
           - gdb
+          # TODO: Use the conda-forge package once polars-iceberg is available there
+          - pip:
+              - polars-iceberg>=0.1.0
   test_cudf_polars_tpch:
     common:
       - output_types: [conda, requirements, pyproject]