From ae89c279d2babc9208baeda36c780a69a4d9ae54 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 3 Feb 2025 12:47:04 -0500 Subject: [PATCH 01/21] feat(libcuml): port libcuml to rattler-build --- conda/recipes/libcuml/recipe.yaml | 166 ++++++++++++++++++++++++++++++ python/cuml/pyproject.toml | 2 +- python/libcuml/pyproject.toml | 2 +- 3 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 conda/recipes/libcuml/recipe.yaml diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml new file mode 100644 index 0000000000..2181375f51 --- /dev/null +++ b/conda/recipes/libcuml/recipe.yaml @@ -0,0 +1,166 @@ +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + head_rev: ${{ git.head_rev(".")[:8] }} + +recipe: + name: libcuml-split + +cache: + source: + path: ../../.. + + build: + script: + content: | + # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the + # incrementing version number in the compile line doesn't break the + # cache + set -x + export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + set +x + + ./build.sh -n libcuml prims -v --allgpuarch + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER", default="sccache") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER", default="sccache") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER", default="sccache") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR", default="Ninja") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="4") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} + SCCACHE_S3_KEY_PREFIX: libcuml-${{ env.get("RAPIDS_CONDA_ARCH") }} + + requirements: + build: + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} =${{ cuda_version }} + - cuda-version =${{ cuda_version }} + - cmake ${{ cmake_version }} + - ninja + - ${{ stdlib("c") }} + host: + - cuda-version =${{ cuda_version }} + - libcumlprims =${{ minor_version }} + - libcuvs =${{ minor_version }} + - libraft-headers =${{ minor_version }} + - librmm =${{ minor_version }} + - rapids-logger =0.1 + - treelite ${{ treelite_version }} + - if: cuda_major == "11" + then: + - cudatoolkit + - libcublas ${{ cuda11_libcublas_host_version }} + - libcublas-dev ${{ cuda11_libcublas_host_version }} + - libcufft ${{ cuda11_libcufft_host_version }} + - libcufft-dev ${{ cuda11_libcufft_host_version }} + - libcurand ${{ cuda11_libcurand_host_version }} + - libcurand-dev ${{ cuda11_libcurand_host_version }} + - libcusolver ${{ cuda11_libcusolver_host_version }} + - libcusolver-dev ${{ cuda11_libcusolver_host_version }} + - libcusparse ${{ cuda11_libcusparse_host_version }} + - libcusparse-dev ${{ cuda11_libcusparse_host_version }} + else: + - cuda-cudart-dev + - libcublas-dev + - libcufft-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + +outputs: + - package: + name: libcuml + version: ${{ version }} + build: + script: + content: | + cmake --install cpp/build + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + requirements: + build: + - cmake ${{ cmake_version }} + host: + - cuda-version =${{ cuda_version }} + - rapids-logger =0.1 + - if: cuda_major == "11" + then: cudatoolkit + else: cuda-cudart-dev + run: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - if: cuda_major == "11" + then: + - cudatoolkit + - libcublas ${{ cuda11_libcublas_run_version }} + - libcufft ${{ cuda11_libcufft_run_version }} + - libcurand ${{ cuda11_libcurand_run_version }} + - libcusolver ${{ cuda11_libcusolver_run_version }} + - libcusparse ${{ cuda11_libcusparse_run_version }} + else: + - cuda-cudart + - libcublas + - libcufft + - libcurand + - libcusolver + - libcusparse + - libcumlprims =${{ minor_version }} + - libcuvs =${{ minor_version }} + - librmm =${{ minor_version }} + - rapids-logger =0.1 + - treelite ${{ treelite_version }} + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - flatbuffers + - libcurand + - if: cuda_major == "11" + then: + - cudatoolkit + about: + homepage: ${{ load_from_file("python/libcuml/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcuml/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/libcuml/pyproject.toml").project.description }} + + - package: + name: libcuml-tests + version: ${{ version }} + build: + script: + content: | + cmake --install cpp/build --component testing + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + requirements: + build: + - cmake ${{ cmake_version }} + host: + - cuda-version =${{ cuda_version }} + - rapids-logger =0.1 + - if: cuda_major == "11" + then: cudatoolkit + else: cuda-cudart-dev + run: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - ${{ pin_subpackage("libcuml", exact=True) }} + - if: cuda_major == "11" + then: cudatoolkit + else: cuda-cudart + about: + homepage: ${{ load_from_file("python/libcuml/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcuml/pyproject.toml").project.license.text }} + summary: libcuml test & benchmark executables diff --git a/python/cuml/pyproject.toml b/python/cuml/pyproject.toml index 0804c1186b..80d4cc9baf 100644 --- a/python/cuml/pyproject.toml +++ b/python/cuml/pyproject.toml @@ -89,7 +89,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "cuda-python", diff --git a/python/libcuml/pyproject.toml b/python/libcuml/pyproject.toml index d5d043400f..3f541f4cc3 100644 --- a/python/libcuml/pyproject.toml +++ b/python/libcuml/pyproject.toml @@ -27,7 +27,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" classifiers = [ "Intended Audience :: Developers", From 0a4555c6ea0ffafd1edc71e40b3b93c8bf0f550c Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 15:39:27 -0400 Subject: [PATCH 02/21] fix(libcuml): resolve all overlinking errors --- conda/recipes/libcuml/recipe.yaml | 39 +++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 2181375f51..650ae87772 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -91,16 +91,34 @@ outputs: script: content: | cmake --install cpp/build + dynamic_linking: + overlinking_behavior: "error" string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: build: - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} host: - cuda-version =${{ cuda_version }} - rapids-logger =0.1 + - libcumlprims =${{ minor_version }} + - libcuvs =${{ minor_version }} + - treelite ${{ treelite_version }} - if: cuda_major == "11" - then: cudatoolkit - else: cuda-cudart-dev + then: + - cudatoolkit + - libcublas ${{ cuda11_libcublas_run_version }} + - libcufft ${{ cuda11_libcufft_run_version }} + - libcurand ${{ cuda11_libcurand_run_version }} + - libcusolver ${{ cuda11_libcusolver_run_version }} + - libcusparse ${{ cuda11_libcusparse_run_version }} + else: + - cuda-cudart-dev + - libcublas + - libcufft + - libcurand + - libcusolver + - libcusparse run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "11" @@ -129,6 +147,11 @@ outputs: - cuda-version - flatbuffers - libcurand + - librmm + - libcusparse + - libcufft + - libcublas + - libcusolver - if: cuda_major == "11" then: - cudatoolkit @@ -148,9 +171,11 @@ outputs: requirements: build: - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} host: - cuda-version =${{ cuda_version }} - rapids-logger =0.1 + - ${{ pin_subpackage("libcuml", exact=True) }} - if: cuda_major == "11" then: cudatoolkit else: cuda-cudart-dev @@ -160,6 +185,16 @@ outputs: - if: cuda_major == "11" then: cudatoolkit else: cuda-cudart + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - libcublas + - libcufft + - libcurand + - libcusolver + - libcusparse + - librmm about: homepage: ${{ load_from_file("python/libcuml/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libcuml/pyproject.toml").project.license.text }} From a68bed02217a14f459bea0baaf6c0d525e8ba27d Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 15:43:46 -0400 Subject: [PATCH 03/21] refactor(libcuml): remove old conda recipe and scripts --- conda/recipes/libcuml/build.sh | 4 - conda/recipes/libcuml/install_libcuml.sh | 4 - .../recipes/libcuml/install_libcuml_tests.sh | 4 - conda/recipes/libcuml/meta.yaml | 154 ------------------ 4 files changed, 166 deletions(-) delete mode 100644 conda/recipes/libcuml/build.sh delete mode 100644 conda/recipes/libcuml/install_libcuml.sh delete mode 100644 conda/recipes/libcuml/install_libcuml_tests.sh delete mode 100644 conda/recipes/libcuml/meta.yaml diff --git a/conda/recipes/libcuml/build.sh b/conda/recipes/libcuml/build.sh deleted file mode 100644 index 228f6cefda..0000000000 --- a/conda/recipes/libcuml/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -./build.sh -n libcuml prims -v --allgpuarch diff --git a/conda/recipes/libcuml/install_libcuml.sh b/conda/recipes/libcuml/install_libcuml.sh deleted file mode 100644 index 173f8cfa90..0000000000 --- a/conda/recipes/libcuml/install_libcuml.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -cmake --install cpp/build diff --git a/conda/recipes/libcuml/install_libcuml_tests.sh b/conda/recipes/libcuml/install_libcuml_tests.sh deleted file mode 100644 index 03c2304cad..0000000000 --- a/conda/recipes/libcuml/install_libcuml_tests.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -cmake --install cpp/build --component testing diff --git a/conda/recipes/libcuml/meta.yaml b/conda/recipes/libcuml/meta.yaml deleted file mode 100644 index d16a162ef1..0000000000 --- a/conda/recipes/libcuml/meta.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: libcuml-split - -source: - path: ../../.. - -build: - ignore_run_exports_from: - - {{ compiler('cuda') }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libcuml-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libcuml-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - host: - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cudatoolkit - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - - libcufft {{ cuda11_libcufft_host_version }} - - libcufft-dev {{ cuda11_libcufft_host_version }} - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - libcusolver {{ cuda11_libcusolver_host_version }} - - libcusolver-dev {{ cuda11_libcusolver_host_version }} - - libcusparse {{ cuda11_libcusparse_host_version }} - - libcusparse-dev {{ cuda11_libcusparse_host_version }} - {% else %} - - cuda-cudart-dev - - libcublas-dev - - libcufft-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev - {% endif %} - - libcumlprims ={{ minor_version }} - - libcuvs ={{ minor_version }} - - libraft-headers ={{ minor_version }} - - librmm ={{ minor_version }} - - rapids-logger =0.1 - - treelite {{ treelite_version }} - -outputs: - - name: libcuml - version: {{ version }} - script: install_libcuml.sh - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - requirements: - build: - - cmake {{ cmake_version }} - host: - - cuda-version ={{ cuda_version }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - - libcublas {{ cuda11_libcublas_run_version }} - - libcufft {{ cuda11_libcufft_run_version }} - - libcurand {{ cuda11_libcurand_run_version }} - - libcusolver {{ cuda11_libcusolver_run_version }} - - libcusparse {{ cuda11_libcusparse_run_version }} - {% else %} - - cuda-cudart - - libcublas - - libcufft - - libcurand - - libcusolver - - libcusparse - {% endif %} - - libcumlprims ={{ minor_version }} - - libcuvs ={{ minor_version }} - - librmm ={{ minor_version }} - - rapids-logger =0.1 - - treelite {{ treelite_version }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: libcuml library - - name: libcuml-tests - version: {{ version }} - script: install_libcuml_tests.sh - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - requirements: - build: - - cmake {{ cmake_version }} - host: - - cuda-version ={{ cuda_version }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart - {% endif %} - - {{ pin_subpackage('libcuml', exact=True) }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: libcuml test & benchmark executables From b4fa3f46613329899545ddd3f66e2a580dc8c609 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 15:58:14 -0400 Subject: [PATCH 04/21] fix(libcuml): disable binary prefix patching --- conda/recipes/libcuml/recipe.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 650ae87772..b77212b5cd 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -93,6 +93,11 @@ outputs: cmake --install cpp/build dynamic_linking: overlinking_behavior: "error" + prefix_detection: + ignore: + # See https://github.com/rapidsai/build-planning/issues/160 + - lib/libcuml++.so + - lib/libcuml.so string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: build: From def6cc610c6bb1aa7ac380e6782bfbf20d946ac0 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 16:09:37 -0400 Subject: [PATCH 05/21] feat(cuml): port cuml recipe to rattler-build --- conda/recipes/cuml/recipe.yaml | 111 +++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 conda/recipes/cuml/recipe.yaml diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml new file mode 100644 index 0000000000..7c5516926a --- /dev/null +++ b/conda/recipes/cuml/recipe.yaml @@ -0,0 +1,111 @@ +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} + head_rev: ${{ git.head_rev(".")[:8] }} + +package: + name: cuml + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + script: + content: | + ./build.sh cuml -v + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} + SCCACHE_S3_KEY_PREFIX: cuml-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + build: + - cmake ${{ cmake_version }} + - ninja + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - ${{ stdlib("c") }} + host: + - cuda-version =${{ cuda_version }} + - cudf =${{ minor_version }} + - cython >=3.0.0 + - libcuml =${{ version }} + - libcumlprims =${{ minor_version }} + - pip + - pylibraft =${{ minor_version }} + - python =${{ py_version }} + - raft-dask =${{ minor_version }} + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - rapids-logger =0.1 + - scikit-build-core >=0.10.0 + - treelite ${{ treelite_version }} + - if: cuda_major == "11" + then: + - cudatoolkit + - cuda-python >=11.8.5,<12.0a0 + else: + - cuda-cudart-dev + - cuda-python >=12.6.2,<13.0a0 + run: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - cudf =${{ minor_version }} + - cupy >=12.0.0 + - dask-cudf =${{ minor_version }} + - joblib >=0.11 + - libcuml =${{ version }} + - libcumlprims =${{ minor_version }} + - numpy >=1.23,<3.0a0 + - packaging + - pylibraft =${{ minor_version }} + - python + - raft-dask =${{ minor_version }} + - rapids-dask-dependency =${{ minor_version }} + - rapids-logger =0.1 + - treelite ${{ treelite_version }} + - if: cuda_major == "11" + then: + - cudatoolkit + - cuda-python >=11.8.5,<12.0a0 + else: + - cuda-cudart + - cuda-python >=12.6.2,<13.0a0 + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - if: cuda_major == "11" + then: cudatoolkit + +tests: + - python: + imports: + - cuml + pip_check: false + +about: + homepage: ${{ load_from_file("python/cuml/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cuml/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/cuml/pyproject.toml").project.description }} From 39f854eeddf3f2a96aa0a26f6416a25a74b09b38 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 16:36:18 -0400 Subject: [PATCH 06/21] feat(cuml-cpu): port cuml-cpu recipe to rattler-build --- conda/recipes/cuml-cpu/recipe.yaml | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 conda/recipes/cuml-cpu/recipe.yaml diff --git a/conda/recipes/cuml-cpu/recipe.yaml b/conda/recipes/cuml-cpu/recipe.yaml new file mode 100644 index 0000000000..30ed4e9542 --- /dev/null +++ b/conda/recipes/cuml-cpu/recipe.yaml @@ -0,0 +1,64 @@ +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} + head_rev: ${{ git.head_rev(".")[:8] }} + +package: + name: cuml-cpu + version: ${{ version }} + +source: + path: ../../.. + +build: + string: py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + script: + content: | + ./build.sh cuml-cpu -v + dynamic_linking: + overlinking_behavior: "error" + +requirements: + build: + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ stdlib("c") }} + - cmake ${{ cmake_version }} + - ninja + host: + - cython >=3.0.0 + - pip + - python =${{ py_version }} + - rapids-build-backend>=0.3.0,<0.4.0.dev0 + - scikit-build-core >=0.10.0 + run: + - numpy >=1.23,<3.0a0 + - python + - pandas + - packaging + - scikit-learn 1.5.* + - hdbscan>=0.8.39,<0.8.40 + - umap-learn=0.5.6 + - nvtx + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - if: cuda_major == "11" + then: cudatoolkit + +tests: + - python: + imports: + - cuml + pip_check: false + +about: + homepage: ${{ load_from_file("python/cuml/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cuml/pyproject.toml").project.license.text }} + summary: cuML-CPU Library From e40fb575fd97ee3fed4ade9ff7a78c1520519d66 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 16:40:27 -0400 Subject: [PATCH 07/21] refactor(cuml): remove old conda recipe and scripts --- conda/recipes/cuml/build.sh | 5 -- conda/recipes/cuml/meta.yaml | 109 ----------------------------------- 2 files changed, 114 deletions(-) delete mode 100644 conda/recipes/cuml/build.sh delete mode 100644 conda/recipes/cuml/meta.yaml diff --git a/conda/recipes/cuml/build.sh b/conda/recipes/cuml/build.sh deleted file mode 100644 index bfec44be2e..0000000000 --- a/conda/recipes/cuml/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh cuml -v diff --git a/conda/recipes/cuml/meta.yaml b/conda/recipes/cuml/meta.yaml deleted file mode 100644 index 0c97e0e56c..0000000000 --- a/conda/recipes/cuml/meta.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set py_version = environ['CONDA_PY'] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cuml - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=cuml-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=cuml-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - - cuda-python - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - {{ stdlib("c") }} - host: - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cudatoolkit - - cuda-python >=11.8.5,<12.0a0 - {% else %} - - cuda-cudart-dev - - cuda-python >=12.6.2,<13.0a0 - {% endif %} - - cudf ={{ minor_version }} - - cython >=3.0.0 - - libcuml ={{ version }} - - libcumlprims ={{ minor_version }} - - pylibraft ={{ minor_version }} - - python x.x - - raft-dask ={{ minor_version }} - - scikit-build-core >=0.10.0 - - treelite {{ treelite_version }} - - rapids-build-backend>=0.3.0,<0.4.0.dev0 - - rapids-logger =0.1 - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - - cuda-python >=11.8.5,<12.0a0 - {% else %} - - cuda-cudart - - cuda-python >=12.6.2,<13.0a0 - {% endif %} - - cudf ={{ minor_version }} - - cupy >=12.0.0 - - dask-cudf ={{ minor_version }} - - joblib >=0.11 - - libcuml ={{ version }} - - libcumlprims ={{ minor_version }} - - numpy >=1.23,<3.0a0 - - packaging - - pylibraft ={{ minor_version }} - - python x.x - - raft-dask ={{ minor_version }} - - rapids-dask-dependency ={{ minor_version }} - - treelite {{ treelite_version }} - - rapids-logger =0.1 - -test: - requirements: - - cuda-version ={{ cuda_version }} - imports: - - cuml - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: cuML library From 5abc25139b82e58cdaefc043f346e6e4a5bc39c0 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 16:40:52 -0400 Subject: [PATCH 08/21] refactor(cuml-cpu): remove old conda recipe and scripts --- conda/recipes/cuml-cpu/build.sh | 5 ---- conda/recipes/cuml-cpu/meta.yaml | 50 -------------------------------- 2 files changed, 55 deletions(-) delete mode 100644 conda/recipes/cuml-cpu/build.sh delete mode 100644 conda/recipes/cuml-cpu/meta.yaml diff --git a/conda/recipes/cuml-cpu/build.sh b/conda/recipes/cuml-cpu/build.sh deleted file mode 100644 index 41ece7d7f1..0000000000 --- a/conda/recipes/cuml-cpu/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh cuml-cpu -v diff --git a/conda/recipes/cuml-cpu/meta.yaml b/conda/recipes/cuml-cpu/meta.yaml deleted file mode 100644 index 4690f95d52..0000000000 --- a/conda/recipes/cuml-cpu/meta.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set py_version = environ['CONDA_PY'] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cuml-cpu - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - VERSION_SUFFIX - -requirements: - build: - - cmake {{ cmake_version }} - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ stdlib("c") }} - - ninja - host: - - python x.x - - scikit-build-core >=0.10.0 - - cython>=3.0.0 - - rapids-build-backend>=0.3.0,<0.4.0.dev0 - run: - - python x.x - - numpy>=1.23,<3.0a0 - - pandas - - packaging - - scikit-learn==1.5.* - - hdbscan>=0.8.39,<0.8.40 - - umap-learn=0.5.6 - - nvtx - -tests: # [linux64] - imports: # [linux64] - - cuml # [linux64] - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: cuML-CPU library From 5a4fd8b4888e544f27fe6e3a1f65bd4d51967539 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 16:47:33 -0400 Subject: [PATCH 09/21] refactor(ci): use rattler-build for cpp and python conda ci builds --- ci/build_cpp.sh | 21 +++++++++++++++++---- ci/build_python.sh | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index a6f40d25d8..3682095d5c 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -3,8 +3,6 @@ set -euo pipefail -rapids-configure-conda-channels - source rapids-configure-sccache source rapids-date-string @@ -17,9 +15,24 @@ rapids-logger "Begin cpp build" sccache --zero-stats -RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry build \ - conda/recipes/libcuml +RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) +export RAPIDS_PACKAGE_VERSION + +source rapids-rattler-channel-string + +# --no-build-id allows for caching with `sccache` +# more info is available at +# https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build +rattler-build build --recipe conda/recipes/libcuml \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats +# remove build_cache directory +rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache + rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index b8b54b04e9..1869164a89 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -3,8 +3,6 @@ set -euo pipefail -rapids-configure-conda-channels - source rapids-configure-sccache source rapids-date-string @@ -19,28 +17,52 @@ rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) +export RAPIDS_PACKAGE_VERSION + +# populates `RATTLER_CHANNELS` array +source rapids-rattler-channel-string + +rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS" + +RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") + sccache --zero-stats -# TODO: Remove `--no-test` flag once importing on a CPU -# node works correctly -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - conda/recipes/cuml +rapids-logger "Building cuml" + +# --no-build-id allows for caching with `sccache` +# more info is available at +# https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build +rattler-build build --recipe conda/recipes/cuml \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats +sccache --zero-stats # Build cuml-cpu only in CUDA 12 jobs since it only depends on python # version RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" if [[ ${RAPIDS_CUDA_MAJOR} == "12" ]]; then + rapids-logger "Building cuml-cpu" + sccache --zero-stats - RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - conda/recipes/cuml-cpu + rattler-build build --recipe conda/recipes/cuml-cpu \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats fi +# remove build_cache directory +rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache + rapids-upload-conda-to-s3 python From a60737821869f3fb16090b0283d062652307f964 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 14 Mar 2025 16:57:14 -0400 Subject: [PATCH 10/21] chore: fix shellcheck error in accel test script --- ci/accel/scikit-learn-tests/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/accel/scikit-learn-tests/run-tests.sh b/ci/accel/scikit-learn-tests/run-tests.sh index 6d429781ca..7dfd92a3fa 100755 --- a/ci/accel/scikit-learn-tests/run-tests.sh +++ b/ci/accel/scikit-learn-tests/run-tests.sh @@ -11,4 +11,4 @@ set -eu -pytest -p cuml.accel --pyargs sklearn -v $@ +pytest -p cuml.accel --pyargs sklearn -v "$@" From e9cbe40a5a4acbeeef31847b32efedb9eba7efe6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 17 Mar 2025 13:18:10 -0400 Subject: [PATCH 11/21] fix(cuml): disable import test There isn't a GPU on the build worker, so we can't do an import test here --- conda/recipes/cuml/recipe.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index 7c5516926a..2c41ce9351 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -99,12 +99,6 @@ requirements: - if: cuda_major == "11" then: cudatoolkit -tests: - - python: - imports: - - cuml - pip_check: false - about: homepage: ${{ load_from_file("python/cuml/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/cuml/pyproject.toml").project.license.text }} From 56bf912e2c81cedf6bff7b113add8e79b35bbc2f Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 17 Mar 2025 13:18:46 -0400 Subject: [PATCH 12/21] refactor(libcuml,cuml): make sccache prefix more granular --- conda/recipes/cuml/recipe.yaml | 2 +- conda/recipes/libcuml/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index 2c41ce9351..54103f83b4 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -37,7 +37,7 @@ build: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} - SCCACHE_S3_KEY_PREFIX: cuml-${{ env.get("RAPIDS_CONDA_ARCH") }} + SCCACHE_S3_KEY_PREFIX: cuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/${{ cuda_major }} requirements: build: diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index b77212b5cd..c4a05a399d 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -43,7 +43,7 @@ cache: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} - SCCACHE_S3_KEY_PREFIX: libcuml-${{ env.get("RAPIDS_CONDA_ARCH") }} + SCCACHE_S3_KEY_PREFIX: libcuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/${{ cuda_major }} requirements: build: From 2b492cdefced32bb45607b407b87946d0f214400 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 17 Mar 2025 15:43:08 -0400 Subject: [PATCH 13/21] fix(cuml-cpu): add missing secrets and env section --- conda/recipes/cuml-cpu/recipe.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cuml-cpu/recipe.yaml b/conda/recipes/cuml-cpu/recipe.yaml index 30ed4e9542..d17921ba96 100644 --- a/conda/recipes/cuml-cpu/recipe.yaml +++ b/conda/recipes/cuml-cpu/recipe.yaml @@ -17,11 +17,25 @@ source: build: string: py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" script: content: | ./build.sh cuml-cpu -v - dynamic_linking: - overlinking_behavior: "error" + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} + SCCACHE_S3_KEY_PREFIX: cuml-cpu/${{ env.get("RAPIDS_CONDA_ARCH") }} requirements: build: From fa4aa01255d89ac61685452e9dfb3955df7294ae Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 17 Mar 2025 15:46:48 -0400 Subject: [PATCH 14/21] prepend `cuda` to part of sccache key prefix --- conda/recipes/cuml/recipe.yaml | 2 +- conda/recipes/libcuml/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index 54103f83b4..b076e049ec 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -37,7 +37,7 @@ build: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} - SCCACHE_S3_KEY_PREFIX: cuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/${{ cuda_major }} + SCCACHE_S3_KEY_PREFIX: cuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} requirements: build: diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index c4a05a399d..847d4e963f 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -43,7 +43,7 @@ cache: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} - SCCACHE_S3_KEY_PREFIX: libcuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/${{ cuda_major }} + SCCACHE_S3_KEY_PREFIX: libcuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} requirements: build: From b105e7b4787f4cbf1aa116002fbd7dfb612a13c1 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 18 Mar 2025 08:56:43 -0400 Subject: [PATCH 15/21] chore: remove default values from recipe --- conda/recipes/cuml-cpu/recipe.yaml | 10 +++++----- conda/recipes/cuml/recipe.yaml | 10 +++++----- conda/recipes/libcuml/recipe.yaml | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/conda/recipes/cuml-cpu/recipe.yaml b/conda/recipes/cuml-cpu/recipe.yaml index d17921ba96..a1c289df12 100644 --- a/conda/recipes/cuml-cpu/recipe.yaml +++ b/conda/recipes/cuml-cpu/recipe.yaml @@ -30,11 +30,11 @@ build: CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: cuml-cpu/${{ env.get("RAPIDS_CONDA_ARCH") }} requirements: diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index b076e049ec..d8ce615e63 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -32,11 +32,11 @@ build: CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: cuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} requirements: diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 847d4e963f..8c85cc675f 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -33,16 +33,16 @@ cache: - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN env: - CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER", default="sccache") }} - CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER", default="sccache") }} - CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER", default="sccache") }} - CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR", default="Ninja") }} - PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="4") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="") }} + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: libcuml/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} requirements: From 099c43517bcca56ecdffbd43bdf0714d9c3ad4e8 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 19 Mar 2025 16:01:42 -0400 Subject: [PATCH 16/21] fix(libcuml): use `host_version` for host deps --- conda/recipes/libcuml/recipe.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 8c85cc675f..55b62cc92f 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -112,11 +112,11 @@ outputs: - if: cuda_major == "11" then: - cudatoolkit - - libcublas ${{ cuda11_libcublas_run_version }} - - libcufft ${{ cuda11_libcufft_run_version }} - - libcurand ${{ cuda11_libcurand_run_version }} - - libcusolver ${{ cuda11_libcusolver_run_version }} - - libcusparse ${{ cuda11_libcusparse_run_version }} + - libcublas ${{ cuda11_libcublas_host_version }} + - libcufft ${{ cuda11_libcufft_host_version }} + - libcurand ${{ cuda11_libcurand_host_version }} + - libcusolver ${{ cuda11_libcusolver_host_version }} + - libcusparse ${{ cuda11_libcusparse_host_version }} else: - cuda-cudart-dev - libcublas From 97fc0d0558783b180051c6efa7e7efc3c96af6be Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 19 Mar 2025 16:01:58 -0400 Subject: [PATCH 17/21] fix(libcuml): ignore `cudatoolkit` on cuda11 --- conda/recipes/libcuml/recipe.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 55b62cc92f..4e92628d55 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -200,6 +200,8 @@ outputs: - libcusolver - libcusparse - librmm + - if: cuda_major == "11" + then: cudatoolkit about: homepage: ${{ load_from_file("python/libcuml/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libcuml/pyproject.toml").project.license.text }} From 01c688be3d135d5d2b9a586638f071f3252af780 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Mar 2025 14:06:01 -0400 Subject: [PATCH 18/21] chore(rattler): improve comment about build cache removal --- ci/build_cpp.sh | 3 ++- ci/build_python.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 3682095d5c..542cc6aa91 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -32,7 +32,8 @@ rattler-build build --recipe conda/recipes/libcuml \ sccache --show-adv-stats -# remove build_cache directory +# remove build_cache directory to avoid uploading the entire source tree +# tracked in https://github.com/prefix-dev/rattler-build/issues/1424 rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 1869164a89..79c57dd1e2 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -62,7 +62,8 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "12" ]]; then sccache --show-adv-stats fi -# remove build_cache directory +# remove build_cache directory to avoid uploading the entire source tree +# tracked in https://github.com/prefix-dev/rattler-build/issues/1424 rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache rapids-upload-conda-to-s3 python From 5005bb097f2e6dfc8a376cfffbfe04ef3478b948 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Mar 2025 14:06:47 -0400 Subject: [PATCH 19/21] chore: remove extraneous zero-stats call --- ci/build_python.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 79c57dd1e2..2bd0178704 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -42,7 +42,6 @@ rattler-build build --recipe conda/recipes/cuml \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --zero-stats # Build cuml-cpu only in CUDA 12 jobs since it only depends on python # version From 6ad24e1f419290301ec0b18b6521bb1be1e9089b Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Mar 2025 14:07:12 -0400 Subject: [PATCH 20/21] chore(rattler): fix copyright year --- conda/recipes/cuml-cpu/recipe.yaml | 2 +- conda/recipes/cuml/recipe.yaml | 2 +- conda/recipes/libcuml/recipe.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/recipes/cuml-cpu/recipe.yaml b/conda/recipes/cuml-cpu/recipe.yaml index a1c289df12..d6a59caee2 100644 --- a/conda/recipes/cuml-cpu/recipe.yaml +++ b/conda/recipes/cuml-cpu/recipe.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. schema_version: 1 context: diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index d8ce615e63..9c6c6247d5 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. schema_version: 1 context: diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 4e92628d55..b7dcb5e999 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. schema_version: 1 context: From 9e574031fed453653efbdfd9e170b041eda126bb Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Mar 2025 14:12:17 -0400 Subject: [PATCH 21/21] fix(cuml): skip libarrow gdb file and disable prefix patching --- conda/recipes/cuml/recipe.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index 9c6c6247d5..52ec190f37 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -20,6 +20,13 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + files: + exclude: + - '*libarrow.so.*gdb.py' + prefix_detection: + # See https://github.com/rapidsai/build-planning/issues/160 + # Blanket ignore here as there are quite a few shared objects shipped in cuml + ignore_binary_files: True script: content: | ./build.sh cuml -v