ci: constrain internal builds to exact local wheels (#2510)

* ci: constrain internal builds to exact local wheels

* ci: keep CI tool tests in nightly workflow

* ci: generate local wheel constraints in workflows

Author: rwgk

.github/workflows/build-wheel.yml

@@ -150,6 +150,19 @@ jobs:
         run: |
           twine check --strict cuda_pathfinder/*.whl
 
+      - name: Constrain builds to the local cuda.pathfinder wheel
+        run: |
+          pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          mkdir -p wheel-constraints
+          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+            pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
+          else
+            pathfinder_uri="file:///host$(realpath "${pathfinder_wheels[0]}")"
+          fi
+          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt
+
       - name: Upload cuda.pathfinder build artifacts
         if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
@@ -172,6 +185,8 @@ jobs:
           output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
         env:
           CIBW_BUILD: ${{ env.CIBW_BUILD }}
+          CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"'
+          CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel'
           # TODO: remove cpython-prerelease once 3.15 is officially supported
           # Allow CPython pre-release builds (currently 3.15 / 3.15t). This is a
           # no-op for stable Python versions because CIBW_BUILD still filters
@@ -181,6 +196,8 @@ jobs:
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
+            PIP_BUILD_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-bindings.txt
+            PIP_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-bindings.txt
             CC="/host/${{ env.SCCACHE_PATH }} cc"
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
@@ -194,6 +211,8 @@ jobs:
           CIBW_ENVIRONMENT_WINDOWS: >
             CUDA_PATH="$(cygpath -w ${{ env.CUDA_PATH }})"
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
+            PIP_BUILD_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-bindings.txt)"
+            PIP_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-bindings.txt)"
           # check cache stats before leaving cibuildwheel
           CIBW_BEFORE_TEST_LINUX: >
             "/host/${{ env.SCCACHE_PATH }}" --show-adv-stats &&
@@ -226,6 +245,27 @@ jobs:
         run: |
           twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
 
+      - name: Constrain cuda.core to the local cuda.bindings wheel
+        run: |
+          pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
+          bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test "${#bindings_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          test -f "${bindings_wheels[0]}"
+          mkdir -p wheel-constraints
+          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+            pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
+            bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")"
+          else
+            pathfinder_uri="file:///host$(realpath "${pathfinder_wheels[0]}")"
+            bindings_uri="file:///host$(realpath "${bindings_wheels[0]}")"
+          fi
+          {
+            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
+            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
+          } | tee wheel-constraints/cuda-core.txt
+
       - name: Upload cuda.bindings build artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
@@ -240,6 +280,8 @@ jobs:
           output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
         env:
           CIBW_BUILD: ${{ env.CIBW_BUILD }}
+          CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"'
+          CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel'
           # TODO: remove cpython-prerelease once 3.15 is officially supported
           # Allow CPython pre-release builds (currently 3.15 / 3.15t). This is a
           # no-op for stable Python versions because CIBW_BUILD still filters
@@ -250,7 +292,8 @@ jobs:
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
             CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_CUDA_MAJOR }}
-            PIP_FIND_LINKS=/host/${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+            PIP_BUILD_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core.txt
+            PIP_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core.txt
             CC="/host/${{ env.SCCACHE_PATH }} cc"
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
@@ -265,7 +308,8 @@ jobs:
             CUDA_PATH="$(cygpath -w ${{ env.CUDA_PATH }})"
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
             CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_CUDA_MAJOR }}
-            PIP_FIND_LINKS="$(cygpath -w ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }})"
+            PIP_BUILD_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core.txt)"
+            PIP_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core.txt)"
           # check cache stats before leaving cibuildwheel
           CIBW_BEFORE_TEST_LINUX: >
             "/host${{ env.SCCACHE_PATH }}" --show-adv-stats &&
@@ -444,21 +488,45 @@ jobs:
           OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml)
           OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
           LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI")
+          PREV_BINDINGS_DIR="cuda_bindings/dist-prev"
 
           gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
           rm -rf ${OLD_BASENAME}-tests  # exclude cython test artifacts
           ls -al $OLD_BASENAME
-          mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
-          mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
+          mkdir -p "${PREV_BINDINGS_DIR}"
+          mv $OLD_BASENAME/*.whl "${PREV_BINDINGS_DIR}"
           rmdir $OLD_BASENAME
 
+      - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel
+        run: |
+          pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
+          bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test "${#bindings_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          test -f "${bindings_wheels[0]}"
+          mkdir -p wheel-constraints
+          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+            pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
+            bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")"
+          else
+            pathfinder_uri="file:///host$(realpath "${pathfinder_wheels[0]}")"
+            bindings_uri="file:///host$(realpath "${bindings_wheels[0]}")"
+          fi
+          {
+            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
+            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
+          } | tee wheel-constraints/cuda-core-prev.txt
+
       - name: Build cuda.core wheel
         uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c  # v4.1.1
         with:
           package-dir: ./cuda_core/
           output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
         env:
           CIBW_BUILD: ${{ env.CIBW_BUILD }}
+          CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"'
+          CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel'
           # TODO: remove cpython-prerelease once 3.15 is officially supported
           # Allow CPython pre-release builds (currently 3.15 / 3.15t). This is a
           # no-op for stable Python versions because CIBW_BUILD still filters
@@ -469,7 +537,8 @@ jobs:
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
             CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_PREV_CUDA_MAJOR }}
-            PIP_FIND_LINKS=/host/${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+            PIP_BUILD_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core-prev.txt
+            PIP_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core-prev.txt
             CC="/host/${{ env.SCCACHE_PATH }} cc"
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
@@ -484,7 +553,8 @@ jobs:
             CUDA_PATH="$(cygpath -w ${{ env.CUDA_PATH }})"
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
             CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_PREV_CUDA_MAJOR }}
-            PIP_FIND_LINKS="$(cygpath -w ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }})"
+            PIP_BUILD_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core-prev.txt)"
+            PIP_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core-prev.txt)"
           # check cache stats before leaving cibuildwheel
           CIBW_BEFORE_TEST_LINUX: >
             "/host${{ env.SCCACHE_PATH }}" --show-adv-stats &&

.github/workflows/coverage.yml

@@ -118,18 +118,51 @@ jobs:
         run: |
           python -m venv .venv
 
-      - name: Build cuda-pathfinder
-        run: |
-          cd cuda_pathfinder
-          ../.venv/bin/pip install -v . --group test
-
-      - name: Build cuda-bindings
-        run: |
-          cd cuda_bindings
-          ../.venv/bin/pip install -v . --group test
-
-      - name: Build cuda-core
-        run: |
+      - name: Install pip with build-constraint support
+        run: .venv/bin/python -m pip install "pip>=25.3"
+
+      - name: Build and install cuda-pathfinder wheel
+        run: |
+          .venv/bin/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/
+          .venv/bin/pip install -v ./wheels/cuda_pathfinder*.whl --group ./cuda_pathfinder/pyproject.toml:test
+
+      - name: Constrain builds to the local cuda-pathfinder wheel
+        run: |
+          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          mkdir -p wheel-constraints
+          pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
+          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt
+
+      - name: Build and install cuda-bindings wheel
+        run: |
+          export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt"
+          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
+          .venv/bin/pip wheel -v --no-deps ./cuda_bindings -w ./wheels/
+          .venv/bin/pip install -v ./wheels/cuda_bindings*.whl --group ./cuda_bindings/pyproject.toml:test
+
+      - name: Constrain cuda-core to the local cuda-bindings wheel
+        run: |
+          CUDA_MAJOR="${CUDA_VER%%.*}"
+          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
+          bindings_wheels=(wheels/cuda_bindings-"${CUDA_MAJOR}".*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test "${#bindings_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          test -f "${bindings_wheels[0]}"
+          mkdir -p wheel-constraints
+          pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
+          bindings_uri="file://$(realpath "${bindings_wheels[0]}")"
+          {
+            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
+            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
+          } | tee wheel-constraints/cuda-core.txt
+
+      - name: Build and install cuda-core
+        run: |
+          export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt"
+          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
           cd cuda_core
           ../.venv/bin/pip install -v . --group test
 
@@ -225,27 +258,48 @@ jobs:
         run: |
           python -m venv .venv
 
-      - name: Build and install cuda.pathfinder
+      - name: Build cuda.pathfinder wheel
         run: |
-          .venv/Scripts/pip install wheel setuptools Cython
+          .venv/Scripts/python -m pip install "pip>=25.3" wheel setuptools Cython
           .venv/Scripts/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/
 
+      - name: Constrain builds to the local cuda.pathfinder wheel
+        run: |
+          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          mkdir -p wheel-constraints
+          pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
+          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt
+
       - name: Build cuda.bindings wheel
         run: |
+          export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")"
+          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
           cd cuda_bindings
           ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/
 
-      # Pin cuda-bindings to the wheel built above; PIP_PRE, which is what makes
-      # that .devN wheel visible, would otherwise let a PyPI pre-release win.
+      - name: Constrain cuda.core to the local cuda.bindings wheel
+        run: |
+          CUDA_MAJOR="${CUDA_VER%%.*}"
+          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
+          bindings_wheels=(wheels/cuda_bindings-"${CUDA_MAJOR}".*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test "${#bindings_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          test -f "${bindings_wheels[0]}"
+          mkdir -p wheel-constraints
+          pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
+          bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")"
+          {
+            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
+            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
+          } | tee wheel-constraints/cuda-core.txt
+
       - name: Build cuda.core wheel
         run: |
-          export PIP_FIND_LINKS="$(pwd)/wheels"
-          export PIP_PRE=1
-          bindings_whl="$(ls ./wheels/cuda_bindings-*.whl | head -1)"
-          bindings_ver="$(basename "$bindings_whl" | cut -d- -f2)"
-          echo "cuda-bindings==${bindings_ver%%+*}" > "$GITHUB_WORKSPACE/constraints.txt"
-          cat "$GITHUB_WORKSPACE/constraints.txt"
-          export PIP_CONSTRAINT="$GITHUB_WORKSPACE/constraints.txt"
+          export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")"
+          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
           cd cuda_core
           ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/
 

.github/workflows/test-sdist-linux.yml

@@ -39,7 +39,7 @@ jobs:
           python-version: "3.12"
 
       - name: Install build tools
-        run: pip install build
+        run: python -m pip install "pip>=25.3" build
 
       # Pure Python packages -- no CTK needed.
       - name: Build cuda.pathfinder sdist and wheel-from-sdist
@@ -52,6 +52,15 @@ jobs:
           python -m build --sdist cuda_python/
           pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz
 
+      - name: Constrain builds to the local cuda.pathfinder wheel
+        run: |
+          pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          mkdir -p wheel-constraints
+          pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
+          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt
+
       # Cython packages need CTK + sccache.
       # The env vars ACTIONS_CACHE_SERVICE_V2, ACTIONS_RESULTS_URL, and ACTIONS_RUNTIME_TOKEN
       # are exposed by this action.
@@ -88,10 +97,28 @@ jobs:
           export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc)
           export CC="sccache cc"
           export CXX="sccache c++"
-          export PIP_FIND_LINKS="$(pwd)/cuda_pathfinder/dist"
+          export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt"
+          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
           python -m build --sdist cuda_bindings/
           pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz
 
+      - name: Constrain cuda.core to the local cuda.bindings wheel
+        run: |
+          CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)"
+          pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl)
+          bindings_wheels=(cuda_bindings/dist/cuda_bindings-"${CUDA_MAJOR}".*.whl)
+          test "${#pathfinder_wheels[@]}" -eq 1
+          test "${#bindings_wheels[@]}" -eq 1
+          test -f "${pathfinder_wheels[0]}"
+          test -f "${bindings_wheels[0]}"
+          mkdir -p wheel-constraints
+          pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
+          bindings_uri="file://$(realpath "${bindings_wheels[0]}")"
+          {
+            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
+            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
+          } | tee wheel-constraints/cuda-core.txt
+
       # cuda_core sdist delegates to setuptools (no CTK needed), but
       # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via
       # get_requires_for_build_wheel in build_hooks.py).
@@ -101,7 +128,8 @@ jobs:
           export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)"
           export CC="sccache cc"
           export CXX="sccache c++"
-          export PIP_FIND_LINKS="$(pwd)/cuda_bindings/dist $(pwd)/cuda_pathfinder/dist"
+          export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt"
+          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
           python -m build --sdist cuda_core/
           pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz