From 4a4cab01ef752031ba8cc12adb5ba9dc970de4ae Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 5 Aug 2026 08:37:58 -0700 Subject: [PATCH 1/3] Fix Python 3.15 validation on Windows and macOS Now that 3.15/3.15t are in the default matrix on every OS (#8427), the uv-based interpreter setup runs on Windows and macOS for the first time and breaks in three places: 1. Windows: uv lays a venv out as Scripts/, not bin/, so `source ${ENV_NAME}/bin/activate` failed outright with "conda-env-/bin/activate: No such file or directory". 2. macOS: the macos-arm64 workaround prepends ${CONDA_PREFIX}/bin to PATH. On the uv path no conda env was activated, so CONDA_PREFIX still pointed at base (python 3.14) and shadowed the venv's python3/pip3 -- the run installed and smoke-tested torch under conda base 3.14 instead of 3.15. 3. Cleanup: ENV_NAME is a conda env name on the conda path but a venv directory on the uv path, so `conda env remove -n` fails with EnvironmentLocationNotFound and, under set -e, fails the job after the tests have already passed. The conda path for 3.10-3.14t is unchanged. --- .github/scripts/validate_binaries.sh | 33 +++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 96e59fa90d..98f90b316c 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -215,11 +215,23 @@ test_cuda_device() { fi } -# Cleanup conda environment +# Cleanup the environment created for this run. +# +# ENV_NAME is a conda env name on the conda path but a venv *directory* on the +# uv path, so `conda env remove -n` would fail with EnvironmentLocationNotFound +# and, under `set -e`, fail the whole job after the tests had already passed. cleanup_conda_env() { if [[ ${TARGET_OS} != linux* ]]; then - conda deactivate - conda env remove -n "${ENV_NAME}" + if [[ ${USING_UV_VENV} == 'yes' ]]; then + # `deactivate` is a function defined by the venv activate script. + if declare -F deactivate > /dev/null; then + deactivate + fi + rm -rf "${ENV_NAME}" + else + conda deactivate + conda env remove -n "${ENV_NAME}" + fi fi } @@ -316,7 +328,9 @@ fi # the matching 3.15.0b1 interpreter with uv (from python-build-standalone) # instead of conda. A --seed venv provides pip so the rest of the flow (pip3 # install, smoke tests) is unchanged. +USING_UV_VENV="no" if [[ ${MATRIX_PYTHON_VERSION} == "3.15" || ${MATRIX_PYTHON_VERSION} == "3.15t" ]]; then + USING_UV_VENV="yes" UV_PYTHON="3.15.0b1" if [[ ${MATRIX_PYTHON_VERSION} == "3.15t" ]]; then UV_PYTHON="3.15.0b1+freethreaded" @@ -324,7 +338,12 @@ if [[ ${MATRIX_PYTHON_VERSION} == "3.15" || ${MATRIX_PYTHON_VERSION} == "3.15t" curl -LsSf https://astral.sh/uv/install.sh | sh source "${HOME}/.local/bin/env" uv venv --seed --python "${UV_PYTHON}" "${ENV_NAME}" - source "${ENV_NAME}/bin/activate" + # uv lays the venv out the platform way: Scripts/ on Windows, bin/ elsewhere. + if [[ ${TARGET_OS} == 'windows' ]]; then + source "${ENV_NAME}/Scripts/activate" + else + source "${ENV_NAME}/bin/activate" + fi else update_conda get_python_config @@ -334,7 +353,11 @@ fi # Save original PATH for macos-arm64 workaround export OLD_PATH=${PATH} -if [[ ${TARGET_OS} == 'macos-arm64' ]]; then +# This promotes the *conda* env's bin to the front of PATH. On the uv path there +# is no conda env to promote: CONDA_PREFIX still points at base (python 3.14), +# so prepending it shadows the venv's python3/pip3 and the run would install and +# validate the wrong interpreter instead of 3.15. +if [[ ${TARGET_OS} == 'macos-arm64' && ${USING_UV_VENV} == 'no' ]]; then export PATH="${CONDA_PREFIX}/bin:${PATH}" fi From 74c0c779dc50492738e042e39e7872376a53c790 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 5 Aug 2026 08:43:36 -0700 Subject: [PATCH 2/3] Provision CPython 3.15.0b4 instead of 3.15.0b1 for validation `import torch` segfaults with every published cp315 wheel under the 3.15.0b1 build. The crash is pybind11 3.0.4's gil_scoped_acquire teardown in python_tracer::init(), reached from torch._C._autograd_init(): THPAutograd_initExtension -> python_tracer::init() -> ~gil_scoped_acquire() -> dec_ref() -> PyThreadState_Clear -> SIGSEGV The same wheel imports and runs cleanly under 3.15.0b4, so bump the pin. This is what has been failing every Linux and aarch64 3.15 leg. --- .github/scripts/validate_binaries.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 98f90b316c..f819ec3ef9 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -28,7 +28,7 @@ get_python_config() { CONDA_EXTRA_PARAM=" -c conda-forge/label/python_rc -c conda-forge" ;; # Note: 3.15 / 3.15t are intentionally absent here. They are provisioned - # via uv (CPython 3.15.0b1) in the interpreter-setup branch below and + # via uv (CPython 3.15.0b4) in the interpreter-setup branch below and # never reach the conda create path that reads CONDA_EXTRA_PARAM. *) PYTHON_V=${MATRIX_PYTHON_VERSION} @@ -321,19 +321,24 @@ fi # Setup the Python environment. # -# Python 3.15 is still pre-release. The cp315/cp315t wheels are built against -# CPython 3.15.0b1 (see pytorch .ci/docker/common/install_cpython.sh), but -# conda-forge only ships 3.15.0a8 -- the pre-release ABI differs, so installing -# the wheel under the conda interpreter segfaults on "import torch". Provision -# the matching 3.15.0b1 interpreter with uv (from python-build-standalone) +# Python 3.15 is still pre-release, and conda-forge's default channel does not +# carry it, so provision the interpreter with uv (from python-build-standalone) # instead of conda. A --seed venv provides pip so the rest of the flow (pip3 # install, smoke tests) is unchanged. +# +# Pin b4, NOT b1: `import torch` segfaults under the 3.15.0b1 build. The crash is +# pybind11 3.0.4's gil_scoped_acquire teardown in python_tracer::init(), reached +# from torch._C._autograd_init(): +# THPAutograd_initExtension -> python_tracer::init() +# -> ~gil_scoped_acquire() -> dec_ref() -> PyThreadState_Clear -> SIGSEGV +# Verified with the cp315 nightly on both interpreters: b1 segfaults, b4 imports +# and runs (matmul/autograd, and 3.15t with the GIL genuinely disabled). USING_UV_VENV="no" if [[ ${MATRIX_PYTHON_VERSION} == "3.15" || ${MATRIX_PYTHON_VERSION} == "3.15t" ]]; then USING_UV_VENV="yes" - UV_PYTHON="3.15.0b1" + UV_PYTHON="3.15.0b4" if [[ ${MATRIX_PYTHON_VERSION} == "3.15t" ]]; then - UV_PYTHON="3.15.0b1+freethreaded" + UV_PYTHON="3.15.0b4+freethreaded" fi curl -LsSf https://astral.sh/uv/install.sh | sh source "${HOME}/.local/bin/env" From 11056edff580ad7eacc55d389f03728ce805b808 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 5 Aug 2026 09:09:49 -0700 Subject: [PATCH 3/3] Install cp315 numpy from the nightly index instead of building from source PyPI publishes no cp315/cp315t numpy wheels, so `pip install numpy` in the smoke-test step falls back to a source build. That is merely slow on Linux and macOS, but a hard failure on Windows: numpy/_core/src/common/simd/simd.h(4): fatal error C1083: Cannot open include file: 'stdalign.h': No such file or directory error: metadata-generation-failed download.pytorch.org/whl/nightly carries preview cp315 and cp315t numpy wheels for every platform (manylinux x86_64/aarch64, macosx arm64/x86_64, win_amd64), so pull numpy from there for 3.15 and forbid a source fallback with --only-binary=:all:. Other python versions keep using PyPI unchanged. --- .github/scripts/validate_binaries.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index f819ec3ef9..5f81459111 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -200,7 +200,19 @@ run_smoke_tests() { # For pip install also test with latest numpy if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then - pip3 install numpy --upgrade --force-reinstall + # PyPI publishes no cp315/cp315t numpy wheels, so a plain + # `pip install numpy` falls back to building from source: slow on Linux + # and macOS, and a hard failure on Windows, where MSVC cannot find + # stdalign.h ("fatal error C1083") while compiling numpy's SIMD headers. + # Preview wheels for every platform, Windows included, are on the + # nightly index -- take those instead, and never a source build. + if [[ ${MATRIX_PYTHON_VERSION} == "3.15" || ${MATRIX_PYTHON_VERSION} == "3.15t" ]]; then + pip3 install --pre numpy --upgrade --force-reinstall \ + --only-binary=:all: \ + --index-url https://download.pytorch.org/whl/nightly + else + pip3 install numpy --upgrade --force-reinstall + fi ${PYTHON_RUN} ./smoke_test/smoke_test.py ${test_suffix} ${compile_check} fi