From 4b778d23c0f0d6b3b4c1b267baa35aac7568c45f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 26 Apr 2024 00:39:54 +0000 Subject: [PATCH 1/6] Enable testing of Python limited API builds --- .github/workflows/build-all-rapids-repos.yml | 14 ++++++++++++- .../opt/rapids-build-utils/manifest.yaml | 20 ++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 9f34995d..2a52b28c 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -44,7 +44,19 @@ jobs: SCCACHE_NO_DAEMON=1 sccache --stop-server sccache -z; sccache --show-adv-stats; - clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules; + # Need Python 3.11 for the buffer protocol in the limited API + export PYTHON_VERSION="3.11" + clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env; + rapids-generate-scripts + . rapids-make-${PYTHON_PACKAGE_MANAGER}-env + # The latest version of this branch includes both my patches (required + # for RAPIDS Cython usage to work) and upstream patches required for + # Cython itself to work under the limited API. + export NO_CYTHON_COMPILE=true + python -m pip install git+https://github.com/cython/cython.git@limited-api-preview + # Need to install my branch of scikit-build-core for the way I'm specifying CMAKE_CXX_FLAGS via CMAKE_ARGS to work + python -m pip install git+https://github.com/vyasr/scikit-build-core.git@fix/cmake_args_quoted_spaces + build-all \ -v \ -j$(nproc --ignore=1) \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 7adda051..76878be3 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -24,7 +24,7 @@ repos: - name: rmm sub_dir: python/rmm depends: [rmm] - args: {install: *rapids_build_backend_args} + args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"} - name: ucxx path: ucxx @@ -78,11 +78,11 @@ repos: - name: pylibcudf sub_dir: python/pylibcudf depends: [cudf] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cudf sub_dir: python/cudf depends: [cudf] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: dask-cudf sub_dir: python/dask_cudf args: {install: *rapids_build_backend_args} @@ -115,11 +115,11 @@ repos: - name: pylibraft sub_dir: python/pylibraft depends: [raft] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: raft-dask sub_dir: python/raft-dask depends: [ucxx, raft] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cuvs path: cuvs @@ -139,7 +139,7 @@ repos: - name: cuvs sub_dir: python/cuvs depends: [cuvs] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cumlprims_mg path: cumlprims_mg @@ -166,7 +166,7 @@ repos: - name: cuml sub_dir: python/cuml depends: [cuml] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cugraph path: cugraph @@ -189,10 +189,11 @@ repos: - name: pylibcugraph sub_dir: python/pylibcugraph depends: [cugraph] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cugraph sub_dir: python/cugraph depends: [cugraph] + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} args: {install: *rapids_build_backend_args} - name: cugraph-service-client sub_dir: python/cugraph-service/client @@ -249,6 +250,7 @@ repos: # it doesn't have a build dir of its own. That means we can't list it as # a cpp repo after cuspatial (above), and we have to manually point CMake # to the cuspatial build dir here. + # Limited API compilation of cuproj currently fails due to https://github.com/cython/cython/issues/6405 args: cmake: | -DFIND_CUPROJ_CPP=ON @@ -256,4 +258,4 @@ repos: - name: cuspatial sub_dir: python/cuspatial depends: [cuspatial] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} From 08dc1d513758e3ac2adf9a3c66ab2b32ce51d283 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Oct 2024 00:57:56 +0000 Subject: [PATCH 2/6] Stop cloning merged and deleted SKBC branch --- .github/workflows/build-all-rapids-repos.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 2a52b28c..dc1a64aa 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -54,8 +54,6 @@ jobs: # Cython itself to work under the limited API. export NO_CYTHON_COMPILE=true python -m pip install git+https://github.com/cython/cython.git@limited-api-preview - # Need to install my branch of scikit-build-core for the way I'm specifying CMAKE_CXX_FLAGS via CMAKE_ARGS to work - python -m pip install git+https://github.com/vyasr/scikit-build-core.git@fix/cmake_args_quoted_spaces build-all \ -v \ From 0a7c4134de01999e544a2002b36cd4292ca415c8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 28 Oct 2024 10:29:41 -0700 Subject: [PATCH 3/6] Try building ucxx and kvikio with the limited API too --- .../rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 76878be3..dc96427e 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -38,11 +38,11 @@ repos: - name: ucxx sub_dir: python/ucxx depends: [ucxx] - args: {cmake: -DFIND_UCXX_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_UCXX_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: distributed-ucxx sub_dir: python/distributed-ucxx depends: [ucxx] - args: {cmake: -DFIND_UCXX_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_UCXX_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: kvikio path: kvikio @@ -58,7 +58,7 @@ repos: - name: kvikio sub_dir: python/kvikio depends: [kvikio] - args: {install: *rapids_build_backend_args} + args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"} - name: cudf path: cudf From 6327785999efe84efd5e6b5aeb9153105ad603a3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 10 Apr 2025 11:54:49 -0700 Subject: [PATCH 4/6] Disable ucxx --- .../rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index dc96427e..a8352a56 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -35,14 +35,15 @@ repos: depends: [rmm] args: {cmake: -DUCXX_ENABLE_RMM=ON} python: + # ucxx explicitly uses functions not in the limited API, so not attempting to enable limited API builds here for now - name: ucxx sub_dir: python/ucxx depends: [ucxx] - args: {cmake: -DFIND_UCXX_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} + args: {cmake: -DFIND_UCXX_CPP=ON, install: *rapids_build_backend_args} - name: distributed-ucxx sub_dir: python/distributed-ucxx depends: [ucxx] - args: {cmake: -DFIND_UCXX_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} + args: {cmake: -DFIND_UCXX_CPP=ON, install: *rapids_build_backend_args} - name: kvikio path: kvikio From daced20a997d606671e0f35b7865b5a0814c25f6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 10 Apr 2025 11:56:28 -0700 Subject: [PATCH 5/6] test From b881a790058b1f3af90422b4f11dea19d992ddfb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 10 Apr 2025 16:35:57 -0700 Subject: [PATCH 6/6] Temporarily disable for kvikio --- .../rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index a8352a56..ae294d43 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -56,10 +56,12 @@ repos: sub_dir: python/libkvikio depends: [kvikio] args: {install: *rapids_build_backend_args} + # Temporarily disabling limited API compilation until https://github.com/rapidsai/kvikio/pull/692 is merged - name: kvikio sub_dir: python/kvikio depends: [kvikio] - args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"} + args: {install: *rapids_build_backend_args} + #args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"} - name: cudf path: cudf