Nightly python packages for python 3.14 (#26397)

tianleiwu · web-flow · commit 32fa4908dbd5 · 2025-11-04T09:50:16.000-08:00
Update python packaging pipeline to support python 3.14. This is to add python 3.14 wheels to both CUDA 12 and CUDA 13 nightly packaging pipelines. The CUDA 13 pipelines were added in #26420. For linux, two wheels are added for python 3.14 and 3.14t. For Windows, a wheel is added for pyhthon 3.14. Known issues: 1. onnx does not have prebuild package for python 3.14, which causes some pipelines failed. 2. onnxscript and onnx-ir depends on onnx. 3. The CI machine for Windows cuda python packaging has cuda driver 553.24, which does not support CUDA 13.0. TODO: Update cuda driver to 580 or later to test properly. Walkarounds for issues 1 and 2: * For python 3.14, we skip onnx, onnxscript and onnx-ir in requirements.txt. * Disable related tests. When onnx package supports python 3.14 in the future, we can undo the walkarounds and re-enable those tests.
diff --git a/setup.py b/setup.py
@@ -598,10 +598,10 @@ def finalize_options(self):
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 
 if enable_training or enable_training_apis:
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
@@ -1843,7 +1843,13 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
                     run_subprocess(
                         [sys.executable, "-m", "unittest", "discover", "-s", "quantization"], cwd=cwd, dll_path=dll_path
                     )
-                    if args.enable_transformers_tool_test:
+
+                    # onnx package does not support python 3.14 yet so skip the transformers tests for python 3.14.
+                    # we can remove this check when onnx package supports python 3.14.
+                    if args.enable_transformers_tool_test and (sys.version_info.major, sys.version_info.minor) < (
+                        3,
+                        14,
+                    ):
                         import google.protobuf  # noqa: PLC0415
                         import numpy  # noqa: PLC0415
 
diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
@@ -44,7 +44,10 @@ stages:
         displayName: 'Download Pipeline Artifact - Win GPU 3.13'
         artifact: 'win_gpu_wheel_3.13'
         patterns: '*.whl'
-
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.14'
+        artifact: 'win_gpu_wheel_3.14'
+        patterns: '*.whl'
       - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \;
         displayName: 'Merge files together'
 
diff --git a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml
@@ -56,3 +56,10 @@ extends:
         EP_BUILD_FLAGS: --use_dml
         EP_NAME: directml
         cmake_build_type: ${{ parameters.cmake_build_type }}
+    - template: stages/py-win-gpu-stage.yml
+      parameters:
+        MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
+        PYTHON_VERSION: '3.14'
+        EP_BUILD_FLAGS: --use_dml
+        EP_NAME: directml
+        cmake_build_type: ${{ parameters.cmake_build_type }}
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml
@@ -94,6 +94,8 @@ stages:
             PythonVersion: '3.12'
           Python313_x64:
             PythonVersion: '3.13'
+          Python314_x64:
+            PythonVersion: '3.14'
       variables:
         OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
         ExtraParam: ${{ parameters.build_py_parameters }}
@@ -138,7 +140,7 @@ stages:
           parameters:
             ${{if eq(variables['Build.SourceBranch'], 'refs/heads/main')}}:
               symbolExpiryTime: 60
-            includePublicSymbolServer: true            
+            includePublicSymbolServer: true
             symbolsArtifactName: onnxruntime_cpu_win_x64_$(PythonVersion)
             symbolsVersion: $(Build.BuildId)
             symbolProject: 'ONNX Runtime'
@@ -204,14 +206,21 @@ stages:
           extra_build_arg: ${{ parameters.build_py_parameters }}
           cmake_build_type: ${{ parameters.cmake_build_type }}
           python_version: '3.12'
-          
+
       - template: ../templates/py-macos.yml
         parameters:
           arch: 'arm64'
           extra_build_arg: ${{ parameters.build_py_parameters }}
           cmake_build_type: ${{ parameters.cmake_build_type }}
           python_version: '3.13'
 
+      - template: ../templates/py-macos.yml
+        parameters:
+          arch: 'arm64'
+          extra_build_arg: ${{ parameters.build_py_parameters }}
+          cmake_build_type: ${{ parameters.cmake_build_type }}
+          python_version: '3.14'
+
 - ${{ if eq(parameters.enable_linux_arm, true) }}:
   - stage: Python_Packaging_Linux_ARM
     dependsOn: []
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml
@@ -35,6 +35,7 @@ parameters:
     - '3.11'
     - '3.12'
     - '3.13'
+    - '3.14'
 
 stages:
     - ${{ if eq(parameters.cuda_version, '12.8') }}:
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml
@@ -50,7 +50,7 @@ stages:
     variables:
       - name: extra_build_args
         ${{ if ne(parameters.extra_build_arg, '') }}:
-          value: -x ${{ parameters.extra_build_arg }}
+          value: '-x "${{ parameters.extra_build_arg }}"'
         ${{ if eq(parameters.extra_build_arg, '') }}:
           value: ''
       - template: ../templates/common-variables.yml
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml
@@ -175,6 +175,8 @@ stages:
 
   - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
     dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
+    # Skip this stage for Python 3.14 for now until onnx package support python 3.14.
+    condition: and(succeeded(), ne('${{ parameters.PYTHON_VERSION }}', '3.14'))
     jobs:
     - job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
       workspace:
@@ -223,6 +225,8 @@ stages:
             Copy-Item -Path $(Build.sourcesDirectory)/onnxruntime/test/python/onnx_backend_test_series.py -Destination $(Agent.TempDirectory)\ort_test_data
             Copy-Item -Recurse -Path $(Build.sourcesDirectory)/onnxruntime/test/testdata -Destination $(Agent.TempDirectory)\ort_test_data
             cd $(Agent.TempDirectory)\ort_test_data
+            python -m pip install psutil py-cpuinfo py3nvml
+            python -c "import onnxruntime; onnxruntime.print_debug_info()"
             python onnx_backend_test_series.py
           workingDirectory: '$(Build.sourcesDirectory)'
           displayName: 'Run Python Tests'
diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_win_gpu_library.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_win_gpu_library.yml
@@ -27,8 +27,9 @@ steps:
           set AZCOPY_AUTO_LOGIN_TYPE=AZCLI
           azcopy.exe cp --recursive https://lotusscus.blob.core.windows.net/models/cuda_sdk/v${{ parameters.CudaVersion }} $(Agent.TempDirectory)
 
+    # Since CUDA 13.0, CUDA DLLs are  in bin\x64 folder instead of bin folder for Windows.
     - powershell: |
-        Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\bin;$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\extras\CUPTI\lib64"
+        Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\bin;$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\bin\x64;$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\extras\CUPTI\lib64"
       displayName: 'Append CUDA SDK Directory to PATH'
 
     - task: CmdLine@2
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml
@@ -27,7 +27,7 @@ parameters:
   displayName: QNN SDK version
   type: string
   default: 2.39.0.250926
-  
+
 - name: is1ES
   displayName: 'Whether the pipeline is running in 1ES'
   type: boolean
@@ -50,7 +50,7 @@ jobs:
   variables:
   - name: extra_build_args
     ${{ if ne(parameters.extra_build_arg, '') }}:
-      value: -x ${{ parameters.extra_build_arg }}
+      value: '-x "${{ parameters.extra_build_arg }}"'
     ${{ if eq(parameters.extra_build_arg, '') }}:
       value: ''
   steps:
@@ -79,4 +79,3 @@ jobs:
       arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args)
     env:
       ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk"
-
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
@@ -61,7 +61,7 @@ jobs:
   variables:
   - name: extra_build_args
     ${{ if ne(parameters.extra_build_arg, '') }}:
-      value: '-x ${{ parameters.extra_build_arg }}'
+      value: '-x "${{ parameters.extra_build_arg }}"'
     ${{ if eq(parameters.extra_build_arg, '') }}:
       value: ''
   - name: python_exe_path
@@ -89,4 +89,4 @@ jobs:
     inputs:
       targetType: filePath
       filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
-      arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path)
+      arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path)
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml b/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml
@@ -32,6 +32,8 @@ jobs:
         PythonVersion: '3.12'
       Python313:
         PythonVersion: '3.13'
+      Python314:
+        PythonVersion: '3.14'
   steps:
   - checkout: none
 
@@ -58,4 +60,3 @@ jobs:
         python3 -c "import onnxruntime as ort; print(ort.__version__)"
       workingDirectory: $(Pipeline.Workspace)/build/onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}
     displayName: Test Package Installation
-
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml
@@ -35,6 +35,8 @@ jobs:
         PythonVersion: '3.12'
       Python313_x64:
         PythonVersion: '3.13'
+      Python314_x64:
+        PythonVersion: '3.14'
   variables:
     GRADLE_OPTS: '-Dorg.gradle.daemon=false'
     VSGenerator: 'Visual Studio 17 2022'
@@ -113,5 +115,3 @@ jobs:
     inputs:
       artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion)
       targetPath: '$(Build.ArtifactStagingDirectory)'
-
-
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml
@@ -39,6 +39,8 @@ jobs:
         PythonVersion: '3.12'
       Python313_x64:
         PythonVersion: '3.13'
+      Python314_x64:
+        PythonVersion: '3.14'
   variables:
     GRADLE_OPTS: '-Dorg.gradle.daemon=false'
     VSGenerator: 'Visual Studio 17 2022'
@@ -51,9 +53,9 @@ jobs:
         parameters:
           host_cpu_arch: 'x64'
           python_version: $(PythonVersion)
-      
+
       - script: python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\windows\python\requirements.txt
-      
+
       - template: set-nightly-build-option-variable-step.yml
 
       - template: jobs/download_win_qnn_sdk.yml
diff --git a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml
@@ -21,6 +21,10 @@ steps:
         variables = {
           "PythonManylinuxDir": "/opt/python/cp313-cp313"
         }
+      elif version == "3.14":
+        variables = {
+          "PythonManylinuxDir": "/opt/python/cp314-cp314"
+        }
       else:
         raise ValueError("Unsupported Python version: '{}'".format(version))
 
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
@@ -77,8 +77,8 @@ parameters:
   type: string
   default: '12.8'
   values:
-      - 13.0
       - 12.8
+      - 13.0
 
 - name: SpecificArtifact
   displayName: Use Specific Artifact
diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh
@@ -12,9 +12,12 @@ ENABLE_CACHE=false
 PYTHON_EXES=(
   "/opt/python/cp311-cp311/bin/python3.11"
   "/opt/python/cp313-cp313/bin/python3.13"
-  "/opt/python/cp313-cp313t/bin/python3.13t"
-    "/opt/python/cp312-cp312/bin/python3.12"
+  "/opt/python/cp313-cp313t/bin/python3.13"
+  "/opt/python/cp314-cp314/bin/python3.14"
+  "/opt/python/cp314-cp314t/bin/python3.14"
+  "/opt/python/cp312-cp312/bin/python3.12"
   )
+
 while getopts "d:p:x:c:e" parameter_Option
 do case "${parameter_Option}"
 in
@@ -58,7 +61,9 @@ echo "EXTRA_ARG:"
 echo "$EXTRA_ARG"
 
 if [ "$EXTRA_ARG" != "" ]; then
-    BUILD_ARGS+=("$EXTRA_ARG")
+    # SC2206: This is intentionally unquoted to allow multiple arguments.
+    # shellcheck disable=SC2206
+    BUILD_ARGS+=($EXTRA_ARG)
 fi
 
 if [ "$ARCH" == "x86_64" ]; then
@@ -67,7 +72,8 @@ if [ "$ARCH" == "x86_64" ]; then
 fi
 
 if [ "$BUILD_DEVICE" == "GPU" ]; then
-    SHORT_CUDA_VERSION=$(echo $CUDA_VERSION | sed   's/\([[:digit:]]\+\.[[:digit:]]\+\)\.[[:digit:]]\+/\1/')
+    # Fix SC2086: Quote $CUDA_VERSION
+    SHORT_CUDA_VERSION=$(echo "$CUDA_VERSION" | sed   's/\([[:digit:]]\+\.[[:digit:]]\+\)\.[[:digit:]]\+/\1/')
     #Enable CUDA and TRT EPs.
     BUILD_ARGS+=("--use_cuda" "--use_tensorrt" "--cuda_version=$SHORT_CUDA_VERSION" "--tensorrt_home=/usr" "--cuda_home=/usr/local/cuda-$SHORT_CUDA_VERSION" "--cudnn_home=/usr/local/cuda-$SHORT_CUDA_VERSION" "--nvcc_threads=1" "--cmake_extra_defines" "CMAKE_CUDA_ARCHITECTURES=60-real;70-real;75-real;80-real;86-real;90a-real;90-virtual" "onnxruntime_USE_FPA_INTB_GEMM=OFF")
 fi
@@ -85,10 +91,20 @@ export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF"
 
 for PYTHON_EXE in "${PYTHON_EXES[@]}"
 do
+  # Check if the Python executable or its directory exists
+  if [ ! -f "$PYTHON_EXE" ]; then
+    echo "WARNING: Python executable not found at $PYTHON_EXE. Skipping this version."
+    continue
+  fi
+
+  # Recompile the entire onnxruntime from scratch for every single Python version.
+  # TODO: It might be possible to reuse some intermediate files between different Python versions to speed up the build.
   rm -rf /build/"$BUILD_CONFIG"
+
   # that's a workaround for the issue that there's no python3 in the docker image
   # like xnnpack's cmakefile, it uses pythone3 to run a external command
   python3_dir=$(dirname "$PYTHON_EXE")
+  ls "$python3_dir"
   ${PYTHON_EXE} -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/python/requirements.txt
   PATH=$python3_dir:$PATH ${PYTHON_EXE} /onnxruntime_src/tools/ci_build/build.py "${BUILD_ARGS[@]}"
   cp /build/"$BUILD_CONFIG"/dist/*.whl /build/dist
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm
diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e -x
 pushd .
-PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13")
+PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13" "/opt/python/cp314-cp314/bin/python3.14" "/opt/python/cp314-cp314t/bin/python3.14")
 CURRENT_DIR=$(pwd)
 if ! [ -x "$(command -v protoc)" ]; then
   $CURRENT_DIR/install_protobuf.sh
@@ -13,4 +13,4 @@ export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF"
 for PYTHON_EXE in "${PYTHON_EXES[@]}"
 do
   ${PYTHON_EXE} -m pip install -r requirements.txt
-done
+done
diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt
@@ -1,9 +1,10 @@
-numpy==2.2.6
+numpy==2.2.6; python_version < "3.14"
+numpy==2.3.2; python_version >= "3.14"
 mypy
 pytest
 setuptools>=68.2.2
 wheel
-onnx==1.19.1
 protobuf==4.25.8
 sympy==1.14
 flatbuffers
+onnx==1.19.1; python_version < "3.14"
diff --git a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh
@@ -1,19 +1,15 @@
 #!/bin/bash
 set -e -x
 
-INSTALL_DEPS_TRAINING=false
-INSTALL_DEPS_DISTRIBUTED_SETUP=false
 USE_CONDA=false
 
-while getopts p:d:v:tmuc parameter_Option
+while getopts p:d:c parameter_Option
 do case "${parameter_Option}"
 in
 p) PYTHON_VER=${OPTARG};;
 d) DEVICE_TYPE=${OPTARG};;
-v) CU_VER=${OPTARG};;
-t) INSTALL_DEPS_TRAINING=true;;
-m) INSTALL_DEPS_DISTRIBUTED_SETUP=true;;
 c) USE_CONDA=true;;
+*) echo "Invalid option: -${OPTARG}" >&2; exit 1 ;;
 esac
 done
 
@@ -37,4 +33,5 @@ fi
 
 export ONNX_ML=1
 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
-${PYTHON_EXE} -m pip install -r ${0/%install_python_deps\.sh/requirements\.txt}
+
+${PYTHON_EXE} -m pip install -r "${0/%install_python_deps\.sh/requirements\.txt}"
diff --git a/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt b/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt
diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh
diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt
diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt
diff --git a/tools/ci_build/github/linux/python/requirements.txt b/tools/ci_build/github/linux/python/requirements.txt
diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh
diff --git a/tools/ci_build/github/windows/python/requirements.txt b/tools/ci_build/github/windows/python/requirements.txt
diff --git a/tools/ci_build/requirements/transformers-test/requirements.txt b/tools/ci_build/requirements/transformers-test/requirements.txt

-Original file line number
+Diff line change
     - '3.11'
     - '3.12'
     - '3.13'
 +    - '3.14'
 stages:
     - ${{ if eq(parameters.cuda_version, '12.8') }}: