[DO NOT MERGE] Test Python limited API compilation

.github/workflows/build-all-rapids-repos.yml

@@ -44,7 +44,17 @@ jobs:
         SCCACHE_NO_DAEMON=1 sccache --stop-server
         sccache -z;
         sccache --show-adv-stats;
-        clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules;
+        # Need Python 3.11 for the buffer protocol in the limited API
+        export PYTHON_VERSION="3.11"
+        clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env;
+        rapids-generate-scripts
+        . rapids-make-${PYTHON_PACKAGE_MANAGER}-env
+        # The latest version of this branch includes both my patches (required
+        # for RAPIDS Cython usage to work) and upstream patches required for
+        # Cython itself to work under the limited API.
+        export NO_CYTHON_COMPILE=true
+        python -m pip install git+https://github.com/cython/cython.git@limited-api-preview
+
         build-all                        \
           -v                             \
           -j$(nproc --ignore=1)          \

features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml

@@ -24,7 +24,7 @@ repos:
     - name: rmm
       sub_dir: python/rmm
       depends: [rmm]
-      args: {install: *rapids_build_backend_args}
+      args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"}
 
 - name: ucxx
   path: ucxx
@@ -35,6 +35,7 @@ repos:
       depends: [rmm]
       args: {cmake: -DUCXX_ENABLE_RMM=ON}
   python:
+      # ucxx explicitly uses functions not in the limited API, so not attempting to enable limited API builds here for now
     - name: ucxx
       sub_dir: python/ucxx
       depends: [ucxx]
@@ -55,10 +56,12 @@ repos:
       sub_dir: python/libkvikio
       depends: [kvikio]
       args: {install: *rapids_build_backend_args}
+      # Temporarily disabling limited API compilation until https://github.com/rapidsai/kvikio/pull/692 is merged
     - name: kvikio
       sub_dir: python/kvikio
       depends: [kvikio]
       args: {install: *rapids_build_backend_args}
+      #args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"}
 
 - name: cudf
   path: cudf
@@ -78,11 +81,11 @@ repos:
     - name: pylibcudf
       sub_dir: python/pylibcudf
       depends: [cudf]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: cudf
       sub_dir: python/cudf
       depends: [cudf]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: dask-cudf
       sub_dir: python/dask_cudf
       args: {install: *rapids_build_backend_args}
@@ -115,11 +118,11 @@ repos:
     - name: pylibraft
       sub_dir: python/pylibraft
       depends: [raft]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: raft-dask
       sub_dir: python/raft-dask
       depends: [ucxx, raft]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
 
 - name: cuvs
   path: cuvs
@@ -139,7 +142,7 @@ repos:
     - name: cuvs
       sub_dir: python/cuvs
       depends: [cuvs]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
 
 - name: cumlprims_mg
   path: cumlprims_mg
@@ -166,7 +169,7 @@ repos:
     - name: cuml
       sub_dir: python/cuml
       depends: [cuml]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
 
 - name: cugraph
   path: cugraph
@@ -189,10 +192,11 @@ repos:
     - name: pylibcugraph
       sub_dir: python/pylibcugraph
       depends: [cugraph]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: cugraph
       sub_dir: python/cugraph
       depends: [cugraph]
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
       args: {install: *rapids_build_backend_args}
     - name: cugraph-service-client
       sub_dir: python/cugraph-service/client
@@ -249,11 +253,12 @@ repos:
       # it doesn't have a build dir of its own. That means we can't list it as
       # a cpp repo after cuspatial (above), and we have to manually point CMake
       # to the cuspatial build dir here.
+      # Limited API compilation of cuproj currently fails due to https://github.com/cython/cython/issues/6405
       args:
         cmake: |
           -DFIND_CUPROJ_CPP=ON
           -Dcuproj_ROOT="${HOME}/cuspatial/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest"
     - name: cuspatial
       sub_dir: python/cuspatial
       depends: [cuspatial]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}