Skip to content

Commit 36a53b6

Browse files
gforsythgforsyth
authored
refactor: build wheels and conda packages using Python limited API (#5419)
General testing done in #7659 Opening a new PR to not disrupt any ongoing downstream testing using artifacts from the original draft PRs. Resolves #7659 xref rapidsai/build-planning#42 Ops-Bot-Merge-Barrier: true Authors: - Gil Forsyth (https://github.com/gforsyth) Approvers: - James Lamb (https://github.com/jameslamb) URL: #5419
1 parent 332a412 commit 36a53b6

14 files changed

Lines changed: 104 additions & 26 deletions

‎.github/workflows/build.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ jobs:
5555
script: ci/build_python.sh
5656
sha: ${{ inputs.sha }}
5757
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
58+
# Build a conda package for each CUDA x ARCH x minimum supported Python version
59+
matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
5860
upload-conda:
5961
needs: [cpp-build, python-build]
6062
secrets: inherit
@@ -118,6 +120,8 @@ jobs:
118120
package-name: pylibcugraph
119121
package-type: python
120122
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
123+
# Build a wheel for each CUDA x ARCH x minimum supported Python version
124+
matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
121125
wheel-publish-pylibcugraph:
122126
needs: wheel-build-pylibcugraph
123127
secrets: inherit
@@ -129,6 +133,7 @@ jobs:
129133
date: ${{ inputs.date }}
130134
package-name: pylibcugraph
131135
package-type: python
136+
publish-wheel-search-key: pylibcugraph_wheel_python_abi3
132137
wheel-build-cugraph:
133138
needs: wheel-build-pylibcugraph
134139
secrets: inherit
@@ -143,6 +148,8 @@ jobs:
143148
package-name: cugraph
144149
package-type: python
145150
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
151+
# Build a wheel for each CUDA x ARCH x minimum supported Python version
152+
matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
146153
wheel-publish-cugraph:
147154
needs: wheel-build-cugraph
148155
secrets: inherit
@@ -154,3 +161,4 @@ jobs:
154161
date: ${{ inputs.date }}
155162
package-name: cugraph
156163
package-type: python
164+
publish-wheel-search-key: cugraph_wheel_python_abi3

‎.github/workflows/pr.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,8 @@ jobs:
243243
build_type: pull-request
244244
script: ci/build_python.sh
245245
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
246+
# Build a conda package for each CUDA x ARCH x minimum supported Python version
247+
matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
246248
conda-python-tests:
247249
needs: [conda-python-build, changed-files]
248250
secrets: inherit
@@ -298,6 +300,8 @@ jobs:
298300
package-name: pylibcugraph
299301
package-type: python
300302
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
303+
# Build a wheel for each CUDA x ARCH x minimum supported Python version
304+
matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
301305
wheel-tests-pylibcugraph:
302306
needs: [wheel-build-pylibcugraph, changed-files]
303307
secrets: inherit
@@ -318,6 +322,8 @@ jobs:
318322
package-name: cugraph
319323
package-type: python
320324
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
325+
# Build a wheel for each CUDA x ARCH x minimum supported Python version
326+
matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
321327
wheel-tests-cugraph:
322328
needs: [wheel-build-cugraph, changed-files]
323329
secrets: inherit

‎build.sh‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION.
3+
# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION.
44
# SPDX-License-Identifier: Apache-2.0
55

66
# cugraph build script
@@ -89,7 +89,11 @@ BUILD_CPP_MG_TESTS=OFF
8989
BUILD_CPP_MTMG_TESTS=OFF
9090
BUILD_ALL_GPU_ARCH=0
9191
CMAKE_GENERATOR_OPTION=(-G Ninja)
92-
PYTHON_ARGS_FOR_INSTALL=(-m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true)
92+
PYTHON_ARGS_FOR_INSTALL=(
93+
--no-build-isolation
94+
--no-deps
95+
--config-settings="rapidsai.disable-cuda=true"
96+
)
9397

9498
# Set defaults for vars that may not have been defined externally
9599
# FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback
@@ -279,13 +283,19 @@ if buildDefault || hasArg libcugraph_etl || hasArg all; then
279283
fi
280284
fi
281285

286+
# If `RAPIDS_PY_VERSION` is set, use that as the lower-bound for the stable ABI CPython version
287+
if [ -n "${RAPIDS_PY_VERSION:-}" ]; then
288+
RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}"
289+
PYTHON_ARGS_FOR_INSTALL+=("--config-settings" "skbuild.wheel.py-api=${RAPIDS_PY_API}")
290+
fi
291+
282292
# Build, and install pylibcugraph
283293
if buildDefault || hasArg pylibcugraph || hasArg all; then
284294
if hasArg --clean; then
285295
cleanPythonDir "${REPODIR}/python/pylibcugraph"
286296
else
287297
SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \
288-
python "${PYTHON_ARGS_FOR_INSTALL[@]}" "${REPODIR}/python/pylibcugraph"
298+
python -m pip install "${PYTHON_ARGS_FOR_INSTALL[@]}" "${REPODIR}/python/pylibcugraph"
289299
fi
290300
fi
291301

@@ -295,7 +305,7 @@ if buildDefault || hasArg cugraph || hasArg all; then
295305
cleanPythonDir "${REPODIR}/python/cugraph"
296306
else
297307
SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \
298-
python "${PYTHON_ARGS_FOR_INSTALL[@]}" "${REPODIR}/python/cugraph"
308+
python -m pip install "${PYTHON_ARGS_FOR_INSTALL[@]}" "${REPODIR}/python/cugraph"
299309
fi
300310
fi
301311

‎ci/build_docs.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -euo pipefail
66

77
rapids-logger "Downloading artifacts from previous jobs"
88
CPP_CHANNEL=$(rapids-download-conda-from-github cpp)
9-
PYTHON_CHANNEL=$(rapids-download-conda-from-github python)
9+
PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-package-name "conda_python" cugraph --stable --cuda "$RAPIDS_CUDA_VERSION")")
1010

1111
rapids-logger "Create test conda environment"
1212
. /opt/conda/etc/profile.d/conda.sh

‎ci/build_python.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -euo pipefail
@@ -51,3 +51,6 @@ sccache --stop-server >/dev/null 2>&1 || true
5151
# remove build_cache directory to avoid uploading the entire source tree
5252
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424
5353
rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache
54+
55+
RAPIDS_PACKAGE_NAME="$(rapids-package-name conda_python cugraph --stable --cuda)"
56+
export RAPIDS_PACKAGE_NAME

‎ci/build_wheel.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,22 @@ set -euo pipefail
66

77
package_name=$1
88
package_dir=$2
9+
shift 2
10+
11+
# Parse optional flags
12+
stable_abi=false
13+
while [[ $# -gt 0 ]]; do
14+
case "$1" in
15+
--stable)
16+
stable_abi=true
17+
shift
18+
;;
19+
*)
20+
echo "Unknown option: $1" >&2
21+
exit 1
22+
;;
23+
esac
24+
done
925

1026
source rapids-configure-sccache
1127
source rapids-date-string
@@ -28,6 +44,12 @@ RAPIDS_PIP_WHEEL_ARGS=(
2844
--no-deps
2945
--disable-pip-version-check
3046
)
47+
48+
# Add py-api setting for stable ABI builds
49+
if [[ "${stable_abi}" == "true" ]] && [[ -n "${RAPIDS_PY_API:-}" ]]; then
50+
RAPIDS_PIP_WHEEL_ARGS+=(--config-settings="skbuild.wheel.py-api=${RAPIDS_PY_API}")
51+
fi
52+
3153
# Only use --build-constraint when build isolation is enabled.
3254
#
3355
# Passing '--build-constraint' and '--no-build-isolation` together results in an error from 'pip',

‎ci/build_wheel_cugraph.sh‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,21 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
1515
#
1616
# env variable 'PIP_CONSTRAINT' is set up by rapids-init-pip. It constrains all subsequent
1717
# 'pip install', 'pip download', etc. calls (except those used in 'pip wheel', handled separately in build scripts)
18+
PYLIBCUGRAPH_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcugraph --stable --cuda "$RAPIDS_CUDA_VERSION")")
1819
LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
19-
PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python)
2020

2121
cat >> "${PIP_CONSTRAINT}" <<EOF
2222
libcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUGRAPH_WHEELHOUSE}"/libcugraph_*.whl)
2323
pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${PYLIBCUGRAPH_WHEELHOUSE}"/pylibcugraph_*.whl)
2424
EOF
2525

26-
./ci/build_wheel.sh cugraph ${package_dir}
26+
# TODO: move this variable into `ci-wheel`
27+
# Format Python limited API version string
28+
RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}"
29+
export RAPIDS_PY_API
30+
31+
./ci/build_wheel.sh cugraph ${package_dir} --stable
2732
./ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}"
33+
34+
RAPIDS_PACKAGE_NAME="$(rapids-package-name wheel_python cugraph --stable --cuda)"
35+
export RAPIDS_PACKAGE_NAME

‎ci/build_wheel_pylibcugraph.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,13 @@ cat >> "${PIP_CONSTRAINT}" <<EOF
2121
libcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUGRAPH_WHEELHOUSE}"/libcugraph_*.whl)
2222
EOF
2323

24-
./ci/build_wheel.sh pylibcugraph ${package_dir}
24+
# TODO: move this variable into `ci-wheel`
25+
# Format Python limited API version string
26+
RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}"
27+
export RAPIDS_PY_API
28+
29+
./ci/build_wheel.sh pylibcugraph ${package_dir} --stable
2530
./ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}"
31+
32+
RAPIDS_PACKAGE_NAME="$(rapids-package-name wheel_python pylibcugraph --stable --cuda)"
33+
export RAPIDS_PACKAGE_NAME

‎ci/test_notebooks.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -Eeuo pipefail
@@ -11,7 +11,7 @@ conda config --set channel_priority strict
1111

1212
rapids-logger "Downloading artifacts from previous jobs"
1313
CPP_CHANNEL=$(rapids-download-conda-from-github cpp)
14-
PYTHON_CHANNEL=$(rapids-download-conda-from-github python)
14+
PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-package-name "conda_python" cugraph --stable --cuda "$RAPIDS_CUDA_VERSION")")
1515

1616
rapids-logger "Generate notebook testing dependencies"
1717
rapids-dependency-file-generator \

‎ci/test_python.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ conda config --set channel_priority strict
1414

1515
rapids-logger "Downloading artifacts from previous jobs"
1616
CPP_CHANNEL=$(rapids-download-conda-from-github cpp)
17-
PYTHON_CHANNEL=$(rapids-download-conda-from-github python)
17+
PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-package-name "conda_python" cugraph --stable --cuda "$RAPIDS_CUDA_VERSION")")
1818

1919
rapids-logger "Generate Python testing dependencies"
2020
rapids-dependency-file-generator \

0 commit comments

Comments
 (0)