fix(python): add pylibcudf and cudf_streaming as test deps, remap imports

Tests and examples import cudf partition/streaming APIs that moved from
rapidsmpf.integrations.cudf and rapidsmpf.streaming.cudf to
cudf_streaming.integrations and cudf_streaming.streaming respectively.

- Add depends_on_pylibcudf and depends_on_cudf_streaming to test_python
  and py_test_rapidsmpf file keys
- Remap all Python imports to cudf_streaming namespace
- Regenerate pyproject.toml via rapids-dependency-file-generator

Author: vyasr

dependencies.yaml

@@ -75,6 +75,8 @@ files:
       - py_version
       - run_rapidsmpf
       - test_python
+      - depends_on_pylibcudf
+      - depends_on_cudf_streaming
   checks:
     output: none
     includes:
@@ -168,6 +170,8 @@ files:
     includes:
       - depends_on_ray
       - test_python
+      - depends_on_pylibcudf
+      - depends_on_cudf_streaming
 channels:
   - rapidsai-nightly
   - rapidsai
@@ -616,3 +620,53 @@ dependencies:
       - output_types: [requirements, pyproject]
         packages:
           - ray>=2.55.1
+  depends_on_pylibcudf:
+    common:
+      - output_types: conda
+        packages:
+          - &pylibcudf_unsuffixed pylibcudf==26.8.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - pylibcudf-cu12==26.8.*,>=0.0.0a0
+          - matrix:
+              cuda: "13.*"
+              cuda_suffixed: "true"
+            packages:
+              - pylibcudf-cu13==26.8.*,>=0.0.0a0
+          - matrix:
+            packages:
+              - *pylibcudf_unsuffixed
+  depends_on_cudf_streaming:
+    common:
+      - output_types: conda
+        packages:
+          - &cudf_streaming_unsuffixed cudf-streaming==26.8.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - cudf-streaming-cu12==26.8.*,>=0.0.0a0
+          - matrix:
+              cuda: "13.*"
+              cuda_suffixed: "true"
+            packages:
+              - cudf-streaming-cu13==26.8.*,>=0.0.0a0
+          - matrix:
+            packages:
+              - *cudf_streaming_unsuffixed

python/rapidsmpf/pyproject.toml

@@ -38,9 +38,11 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     "cuda-core>=0.7.0",
+    "cudf-streaming==26.8.*,>=0.0.0a0",
     "cupy-cuda13x>=13.6.0,!=14.0.0,!=14.1.0",
     "numpy >=1.23,<3.0",
     "psutil",
+    "pylibcudf==26.8.*,>=0.0.0a0",
     "pytest",
     "ray>=2.55.1",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

python/rapidsmpf/rapidsmpf/examples/bulk_mpi_shuffle.py

@@ -11,6 +11,11 @@
 from typing import TYPE_CHECKING
 
 import pylibcudf as plc
+from cudf_streaming.integrations.partition import (
+    partition_and_pack,
+    unpack_and_concat,
+    unspill_partitions,
+)
 from mpi4py import MPI
 
 import rmm.mr
@@ -19,11 +24,6 @@
 import rapidsmpf.bootstrap
 import rapidsmpf.communicator.mpi
 from rapidsmpf.config import Options, get_environment_variables
-from rapidsmpf.integrations.cudf.partition import (
-    partition_and_pack,
-    unpack_and_concat,
-    unspill_partitions,
-)
 from rapidsmpf.memory.buffer import MemoryType
 from rapidsmpf.memory.buffer_resource import BufferResource
 from rapidsmpf.progress_thread import ProgressThread

python/rapidsmpf/rapidsmpf/examples/ray/bulk_ray_shuffle.py

@@ -13,14 +13,14 @@
 
 import pylibcudf as plc
 import ray
-
-import rmm.mr
-
-from rapidsmpf.integrations.cudf.partition import (
+from cudf_streaming.integrations.partition import (
     partition_and_pack,
     unpack_and_concat,
     unspill_partitions,
 )
+
+import rmm.mr
+
 from rapidsmpf.integrations.ray import RapidsMPFActor, setup_ray_ucxx_cluster
 from rapidsmpf.memory.buffer import MemoryType
 from rapidsmpf.memory.buffer_resource import BufferResource

python/rapidsmpf/rapidsmpf/examples/ray/ray_shuffle_example.py

@@ -10,14 +10,14 @@
 import numpy as np
 import pylibcudf as plc
 import ray
-
-import rmm
-
-from rapidsmpf.integrations.cudf.partition import (
+from cudf_streaming.integrations.partition import (
     partition_and_pack,
     unpack_and_concat,
     unspill_partitions,
 )
+
+import rmm
+
 from rapidsmpf.integrations.ray import RapidsMPFActor, setup_ray_ucxx_cluster
 from rapidsmpf.memory.buffer_resource import BufferResource
 from rapidsmpf.shuffler import Shuffler

python/rapidsmpf/rapidsmpf/examples/streaming/basic_example.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING
 
 import pylibcudf
+from cudf_streaming.streaming.table_chunk import TableChunk
 
 import rmm.mr
 from rmm.pylibrmm.stream import DEFAULT_STREAM
@@ -24,7 +25,6 @@
 from rapidsmpf.streaming.core.context import Context
 from rapidsmpf.streaming.core.leaf_actor import pull_from_channel, push_to_channel
 from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable

python/rapidsmpf/rapidsmpf/tests/streaming/test_allgather.py

@@ -9,15 +9,15 @@
 import numpy as np
 import pylibcudf as plc
 import pytest
+from cudf_streaming.integrations.partition import unpack_and_concat
+from cudf_streaming.streaming.table_chunk import TableChunk
 
-from rapidsmpf.integrations.cudf.partition import unpack_and_concat
 from rapidsmpf.memory.packed_data import PackedData
 from rapidsmpf.streaming.chunks.packed_data import PackedDataChunk
 from rapidsmpf.streaming.coll.allgather import AllGather, allgather
 from rapidsmpf.streaming.core.actor import define_actor, run_actor_network
 from rapidsmpf.streaming.core.leaf_actor import pull_from_channel, push_to_channel
 from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 from rapidsmpf.testing import assert_eq
 
 cudf = pytest.importorskip("cudf")

python/rapidsmpf/rapidsmpf/tests/streaming/test_define_actor.py

@@ -7,12 +7,12 @@
 
 import pylibcudf as plc
 import pytest
+from cudf_streaming.streaming.table_chunk import TableChunk
 
 from rapidsmpf.streaming.chunks.arbitrary import ArbitraryChunk
 from rapidsmpf.streaming.core.actor import define_actor, run_actor_network
 from rapidsmpf.streaming.core.leaf_actor import pull_from_channel, push_to_channel
 from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 from rapidsmpf.testing import assert_eq
 
 cudf = pytest.importorskip("cudf")

python/rapidsmpf/rapidsmpf/tests/streaming/test_fanout.py

@@ -9,12 +9,12 @@
 
 import pylibcudf as plc
 import pytest
+from cudf_streaming.streaming.table_chunk import TableChunk
 
 from rapidsmpf.streaming.core.actor import run_actor_network
 from rapidsmpf.streaming.core.fanout import FanoutPolicy, fanout
 from rapidsmpf.streaming.core.leaf_actor import pull_from_channel, push_to_channel
 from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 from rapidsmpf.testing import assert_eq
 
 cudf = pytest.importorskip("cudf")

python/rapidsmpf/rapidsmpf/tests/streaming/test_leaf_actor.py

@@ -7,11 +7,11 @@
 
 import pylibcudf as plc
 import pytest
+from cudf_streaming.streaming.table_chunk import TableChunk
 
 from rapidsmpf.streaming.core.actor import run_actor_network
 from rapidsmpf.streaming.core.leaf_actor import pull_from_channel, push_to_channel
 from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
 from rapidsmpf.testing import assert_eq
 
 cudf = pytest.importorskip("cudf")