Merge branch 'main' into contains-lrb

Author: davidwendt

.agents/skills/update-pandas-testing/SKILL.md

@@ -9,11 +9,13 @@ This rule describes the procedure for updating the pandas unit testing of cudf.p
 
 ## Prerequisites
 1. Prompt the user for a development environment. Assume that the provided environment already contains cudf, the target version of pandas, and the pandas test dependencies.
-2. If the user does not provide an environment, create one from the file in `conda/environments/` that matches the CUDA version and CPU architecture of the machine (`all_cuda-<CUDA major><CUDA minor>_arch-<arch>.yaml`). These files contain pandas and the `test_python_pandas_tests` dependencies from `dependencies.yaml`, but not cudf. Build and install cudf into the new environment in editable mode.
+2. If the user does not provide an environment, create one from the file in `conda/environments/` that matches the CUDA version and CPU architecture of the machine (`all_cuda-<CUDA major><CUDA minor>_arch-<arch>.yaml`), then add the `test_python_pandas_tests` dependencies, which you generate with `rapids-dependency-file-generator` from the `pandas_tests` file key in `dependencies.yaml`. Neither file contains cudf. Build and install cudf into the new environment in editable mode.
 
     For example, to create a fictional conda development environment named "cudf-dev" on an x86_64 machine with CUDA 13
     ```bash
     conda env create -n cudf-dev -f conda/environments/all_cuda-133_arch-x86_64.yaml
+    rapids-dependency-file-generator --file-key pandas_tests --output conda --matrix "" > /tmp/pandas_tests.yaml
+    conda env update -n cudf-dev -f /tmp/pandas_tests.yaml
     conda run -n cudf-dev ./build.sh libcudf pylibcudf cudf --pydevelop
     ```
 

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -6,11 +6,9 @@ channels:
 - conda-forge
 dependencies:
 - aiobotocore>=2.2.0,<3
-- beautifulsoup4
 - binutils
 - boto3>=1.21.21
 - botocore>=1.24.21
-- bottleneck
 - breathe>=4.35.0
 - c-compiler
 - cachetools
@@ -32,21 +30,19 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
 - flatbuffers==24.3.25
-- fsspec
 - fsspec>=0.6.0
 - gcc_linux-aarch64=14.*
 - gdb
-- html5lib
 - hypothesis>=6.131.7
 - identify>=2.5.20
 - include-what-you-use==0.24.0
 - ipykernel
 - ipython
-- jinja2
 - jupyter_client
 - kvikio==26.12.*,>=0.0.0a0
 - libcufile-dev
@@ -57,9 +53,7 @@ dependencies:
 - librdkafka<2.15.0a0
 - librmm==26.12.*,>=0.0.0a0
 - libucxx==0.53.*,>=0.0.0a0
-- lxml
 - make
-- matplotlib-base
 - mmh3
 - moto>=4.0.8
 - msgpack-python
@@ -70,43 +64,33 @@ dependencies:
 - nbsphinx
 - ninja
 - notebook
-- numba
 - numba-cuda-mlir>=0.3.0
 - numba-cuda>=0.22.2,<0.31.0
 - numba>=0.60.0,<0.65.0
-- numexpr
 - numpy>=2.0,<3.0
 - numpydoc
 - nvidia-ml-py>=12
 - nvtx>=0.2.1
-- odfpy
 - openpyxl
 - packaging
 - pandas>=3.0.0,<3.1.0
 - pandoc
 - polars>=2.0,<2.2
 - pre-commit
 - psutil
-- pyarrow
 - pyarrow>=19.0.0
-- pyqt
-- pyreadstat
-- pytables
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
 - pytest-httpserver
-- pytest-reportlog
 - pytest-rerunfailures!=16.0.0
 - pytest-xdist
 - pytest<9.1.0
-- python-calamine
 - python-confluent-kafka
 - python-xxhash
 - python>=3.12
 - pytorch>=2.4.0
-- pyxlsb
-- qtpy
 - rapids-build-backend>=0.4.0,<0.5.0
 - rapids-cli
 - rapids-dask-dependency==26.12.*,>=0.0.0a0
@@ -124,16 +108,14 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-aarch64==2.28
-- tabulate
 - typing_extensions>=4.0.0
 - ucxx==0.53.*,>=0.0.0a0
-- xarray
-- xlrd
-- xlsxwriter
 - zlib>=1.2.13
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-129_arch-aarch64

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -6,11 +6,9 @@ channels:
 - conda-forge
 dependencies:
 - aiobotocore>=2.2.0,<3
-- beautifulsoup4
 - binutils
 - boto3>=1.21.21
 - botocore>=1.24.21
-- bottleneck
 - breathe>=4.35.0
 - c-compiler
 - cachetools
@@ -32,21 +30,19 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
 - flatbuffers==24.3.25
-- fsspec
 - fsspec>=0.6.0
 - gcc_linux-64=14.*
 - gdb
-- html5lib
 - hypothesis>=6.131.7
 - identify>=2.5.20
 - include-what-you-use==0.24.0
 - ipykernel
 - ipython
-- jinja2
 - jupyter_client
 - kvikio==26.12.*,>=0.0.0a0
 - libcufile-dev
@@ -57,9 +53,7 @@ dependencies:
 - librdkafka<2.15.0a0
 - librmm==26.12.*,>=0.0.0a0
 - libucxx==0.53.*,>=0.0.0a0
-- lxml
 - make
-- matplotlib-base
 - mkl<2026.2
 - mmh3
 - moto>=4.0.8
@@ -71,43 +65,33 @@ dependencies:
 - nbsphinx
 - ninja
 - notebook
-- numba
 - numba-cuda-mlir>=0.3.0
 - numba-cuda>=0.22.2,<0.31.0
 - numba>=0.60.0,<0.65.0
-- numexpr
 - numpy>=2.0,<3.0
 - numpydoc
 - nvidia-ml-py>=12
 - nvtx>=0.2.1
-- odfpy
 - openpyxl
 - packaging
 - pandas>=3.0.0,<3.1.0
 - pandoc
 - polars>=2.0,<2.2
 - pre-commit
 - psutil
-- pyarrow
 - pyarrow>=19.0.0
-- pyqt
-- pyreadstat
-- pytables
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
 - pytest-httpserver
-- pytest-reportlog
 - pytest-rerunfailures!=16.0.0
 - pytest-xdist
 - pytest<9.1.0
-- python-calamine
 - python-confluent-kafka
 - python-xxhash
 - python>=3.12
 - pytorch>=2.4.0
-- pyxlsb
-- qtpy
 - rapids-build-backend>=0.4.0,<0.5.0
 - rapids-cli
 - rapids-dask-dependency==26.12.*,>=0.0.0a0
@@ -125,16 +109,14 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-64==2.28
-- tabulate
 - typing_extensions>=4.0.0
 - ucxx==0.53.*,>=0.0.0a0
-- xarray
-- xlrd
-- xlsxwriter
 - zlib>=1.2.13
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-129_arch-x86_64

conda/environments/all_cuda-133_arch-aarch64.yaml

@@ -6,11 +6,9 @@ channels:
 - conda-forge
 dependencies:
 - aiobotocore>=2.2.0,<3
-- beautifulsoup4
 - binutils
 - boto3>=1.21.21
 - botocore>=1.24.21
-- bottleneck
 - breathe>=4.35.0
 - c-compiler
 - cachetools
@@ -32,21 +30,19 @@ dependencies:
 - cxx-compiler
 - cython>=3.2.2,<3.3.0a0
 - dask-cuda==26.12.*,>=0.0.0a0
+- deltalake>=1.6.0
 - dlpack>=0.8,<1.0
 - doxygen=1.18.0
 - fastavro>=0.22.9
 - flatbuffers==24.3.25
-- fsspec
 - fsspec>=0.6.0
 - gcc_linux-aarch64=14.*
 - gdb
-- html5lib
 - hypothesis>=6.131.7
 - identify>=2.5.20
 - include-what-you-use==0.24.0
 - ipykernel
 - ipython
-- jinja2
 - jupyter_client
 - kvikio==26.12.*,>=0.0.0a0
 - libcufile-dev
@@ -57,9 +53,7 @@ dependencies:
 - librdkafka<2.15.0a0
 - librmm==26.12.*,>=0.0.0a0
 - libucxx==0.53.*,>=0.0.0a0
-- lxml
 - make
-- matplotlib-base
 - mmh3
 - moto>=4.0.8
 - msgpack-python
@@ -70,43 +64,33 @@ dependencies:
 - nbsphinx
 - ninja
 - notebook
-- numba
 - numba-cuda-mlir>=0.3.0
 - numba-cuda>=0.22.2,<0.31.0
 - numba>=0.60.0,<0.65.0
-- numexpr
 - numpy>=2.0,<3.0
 - numpydoc
 - nvidia-ml-py>=12
 - nvtx>=0.2.1
-- odfpy
 - openpyxl
 - packaging
 - pandas>=3.0.0,<3.1.0
 - pandoc
 - polars>=2.0,<2.2
 - pre-commit
 - psutil
-- pyarrow
 - pyarrow>=19.0.0
-- pyqt
-- pyreadstat
-- pytables
+- pyiceberg>=0.12.0
 - pytest-benchmark
 - pytest-cases>=3.8.2
 - pytest-cov
 - pytest-httpserver
-- pytest-reportlog
 - pytest-rerunfailures!=16.0.0
 - pytest-xdist
 - pytest<9.1.0
-- python-calamine
 - python-confluent-kafka
 - python-xxhash
 - python>=3.12
 - pytorch>=2.10.0
-- pyxlsb
-- qtpy
 - rapids-build-backend>=0.4.0,<0.5.0
 - rapids-cli
 - rapids-dask-dependency==26.12.*,>=0.0.0a0
@@ -124,16 +108,14 @@ dependencies:
 - sphinx-remove-toctrees
 - sphinx>=8.1.0
 - sphinxcontrib-websupport
+- sqlalchemy>=2.0.18
 - streamz
 - sysroot_linux-aarch64==2.28
-- tabulate
 - typing_extensions>=4.0.0
 - ucxx==0.53.*,>=0.0.0a0
-- xarray
-- xlrd
-- xlsxwriter
 - zlib>=1.2.13
 - zstandard
 - pip:
   - nvidia-sphinx-theme
+  - polars-iceberg>=0.1.0
 name: all_cuda-133_arch-aarch64