[CUDA] Add nightly gpu package and publish pipelines for CUDA 13 (#26420)

Add nightly gpu pipelines for CUDA 13.

Author: tianleiwu

cmake/onnxruntime_providers_cuda.cmake

@@ -222,6 +222,7 @@
 
     if(MSVC)
       target_compile_options(${target} PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:-Xcompiler /Zc:__cplusplus>")
+      target_compile_options(${target} PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:-Xcompiler /bigobj>")
     endif()
 
     onnxruntime_add_include_to_target(${target} onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} flatbuffers::flatbuffers)

tools/ci_build/github/azure-pipelines/py-cuda13-packaging-pipeline.yml

@@ -0,0 +1,52 @@
+trigger: none
+# The `resources` specify the location and version of the 1ES PT.
+resources:
+  repositories:
+  - repository: 1esPipelines
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+parameters:
+  - name: cmake_build_type
+    type: string
+    default: 'Release'
+    values:
+      - Debug
+      - Release
+      - RelWithDebInfo
+      - MinSizeRel
+
+extends:
+  # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
+  # For non-production pipelines, use "Unofficial" as defined below.
+  # For productions pipelines, use "Official".
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
+  parameters:
+    sdl:
+      componentgovernance:
+        ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)'
+        alertWarningLevel: High
+        failOnAlert: false
+        verbosity: Normal
+        timeout: 3600
+      tsa:
+        enabled: true
+      codeSignValidation:
+        enabled: true
+        break: true
+      policheck:
+        enabled: true
+        exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml'
+      codeql:
+        compiled:
+          enabled: false
+          justificationForDisabling: 'CodeQL is taking nearly 6 hours resulting in timeouts in our production pipelines'
+    pool:
+      name: 'onnxruntime-Win-CPU-VS2022-Latest'  # Name of your hosted pool
+      os: windows  # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS
+
+    stages:
+      - template: stages/py-gpu-packaging-stage.yml
+        parameters:
+          cmake_build_type: ${{ parameters.cmake_build_type }}
+          cuda_version: '13.0'

tools/ci_build/github/azure-pipelines/py-cuda13-publishing-pipeline.yml

@@ -0,0 +1,40 @@
+resources:
+  pipelines:
+  - pipeline: build
+    source: 'Python CUDA Package Test Pipeline'
+    trigger:
+      branches:
+        include:
+        - main
+    branch: main
+  repositories:
+  - repository: 1esPipelines
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+parameters:
+  - name: isReleaseBuild
+    type: boolean
+    default: false
+
+variables:
+  - name: ArtifactFeed
+    ${{ if eq(parameters.isReleaseBuild, false) }}:
+      value: ort-cuda-13-nightly
+    ${{ else }}:
+      value: onnxruntime-cuda-13
+
+extends:
+  # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
+  # For non-production pipelines, use "Unofficial" as defined below.
+  # For productions pipelines, use "Official".
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
+  parameters:
+    sdl:
+      sourceAnalysisPool:
+        name: onnxruntime-Win-CPU-VS2022-Latest
+        os: windows
+    stages:
+    - template: stages/py-cuda-publishing-stage.yml
+      parameters:
+        artifact_feed: $(ArtifactFeed)

tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml

@@ -22,6 +22,11 @@ parameters:
   default: '12.8'
   values:
    - 12.8
+   - 13.0
+
+- name: cudnn_folder
+  type: string
+  default: '9.13.0.50_cuda13'
 
 - name: PythonVersions
   type: object
@@ -32,14 +37,25 @@ parameters:
     - '3.13'
 
 stages:
-    - ${{ each python_version in parameters.PythonVersions }}:
-      - template: py-win-gpu-stage.yml
-        parameters:
-          PYTHON_VERSION: ${{ python_version }}
-          EP_NAME: gpu
-          CudaVersion: ${{ parameters.cuda_version }}
-          EP_BUILD_FLAGS: --enable_lto --use_cuda --cuda_home=$(Agent.TempDirectory)\v${{ parameters.cuda_version }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52-real;61-real;75-real;86-real;89-real;90-virtual"
-          use_tensorrt: True
+    - ${{ if eq(parameters.cuda_version, '12.8') }}:
+      - ${{ each python_version in parameters.PythonVersions }}:
+        - template: py-win-gpu-stage.yml
+          parameters:
+            PYTHON_VERSION: ${{ python_version }}
+            EP_NAME: gpu
+            CudaVersion: ${{ parameters.cuda_version }}
+            EP_BUILD_FLAGS: --enable_lto --use_cuda --cuda_home=$(Agent.TempDirectory)\v${{ parameters.cuda_version }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52-real;61-real;75-real;86-real;89-real;90-virtual"
+            use_tensorrt: True
+
+    - ${{ if eq(parameters.cuda_version, '13.0') }}:
+      - ${{ each python_version in parameters.PythonVersions }}:
+        - template: py-win-gpu-stage.yml
+          parameters:
+            PYTHON_VERSION: ${{ python_version }}
+            EP_NAME: gpu
+            CudaVersion: ${{ parameters.cuda_version }}
+            EP_BUILD_FLAGS: --enable_lto --use_cuda --cuda_home=$(Agent.TempDirectory)\v${{ parameters.cuda_version }} --cudnn_home=$(Agent.TempDirectory)\${{ parameters.cudnn_folder }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75-real;86-real;89-real;90-virtual"
+            use_tensorrt: True
 
     - template: py-linux-gpu-stage.yml
       parameters:

tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml

@@ -17,6 +17,7 @@ parameters:
    - Release
    - RelWithDebInfo
    - MinSizeRel
+
 - name: docker_base_image
   type: string
 
@@ -25,6 +26,7 @@ parameters:
   default: '12.8'
   values:
    - 12.8
+   - 13.0
 
 stages:
 - stage: Linux_py_GPU_Wheels_${{ parameters.arch }}

tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml

@@ -18,11 +18,13 @@ parameters:
     Extra parameters to pass to build.py. Don't put newlines in here.
   type: string
   default: ''
+
 - name: CudaVersion
   type: string
   default: '12.8'
   values:
     - 12.8
+    - 13.0
 
 - name: cmake_build_type
   type: string

tools/ci_build/github/azure-pipelines/templates/common-variables.yml

@@ -4,5 +4,5 @@ variables:
   # As for Debian installation, replace '-1.' by '-1+' when assigning trt version below
   linux_trt_version_cuda13: ${{ variables.cuda13_trt_version }}-1.cuda13.0
   linux_trt_version_cuda12: ${{ variables.cuda12_trt_version }}-1.cuda12.8
-  win_trt_folder_cuda13: TensorRT-${{ variables.cuda13_trt_version }}.Windows10.win10.cuda-13.0
+  win_trt_folder_cuda13: TensorRT-${{ variables.cuda13_trt_version }}.Windows.win10.cuda-13.0
   win_trt_folder_cuda12: TensorRT-${{ variables.cuda12_trt_version }}.Windows10.x86_64.cuda-12.8

tools/ci_build/github/azure-pipelines/templates/jobs/download_win_gpu_library.yml

@@ -11,12 +11,9 @@ parameters:
     values:
       - 13.0
       - 12.8
-  - name: TrtVersion
+  - name: CudnnFolder
     type: string
-    default: '10.9.0.34'
-    values:
-      - 10.9.0.34
-      - 10.13.3.9
+    default: '9.13.0.50_cuda13'
 
 steps:
   - ${{ if eq(parameters.DownloadCUDA, true) }}:
@@ -41,49 +38,60 @@ steps:
           dir $(Agent.TempDirectory)
       displayName: 'Print PATH after download CUDA SDK'
 
-  - ${{ if eq(parameters.DownloadTRT, true) }}:
     - ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        - powershell: |
-            Write-Host "##vso[task.setvariable variable=trtCudaVersion;]13.0"
-          displayName: Set trtCudaVersion
-    - ${{ if and(eq(parameters.CudaVersion, '12.8'), eq(parameters.TrtVersion, '10.13.3.9')) }}:
-        - powershell: |
-            Write-Host "##vso[task.setvariable variable=trtCudaVersion;]12.9"
-          displayName: Set trtCudaVersion
-    - ${{ if and(eq(parameters.CudaVersion, '12.8'), eq(parameters.TrtVersion, '10.9.0.34')) }}:
-        - powershell: |
-            Write-Host "##vso[task.setvariable variable=trtCudaVersion;]12.8"
-          displayName: Set trtCudaVersion
+      - task: AzureCLI@2
+        displayName: 'Download CUDNN ${{ parameters.CudnnFolder }}'
+        inputs:
+          azureSubscription: AIInfraBuildOnnxRuntimeOSS
+          scriptType: 'batch'
+          scriptLocation: 'inlineScript'
+          inlineScript: |
+            set AZCOPY_AUTO_LOGIN_TYPE=AZCLI
+            azcopy.exe cp --recursive https://lotusscus.blob.core.windows.net/models/cudnn_sdk/${{ parameters.CudnnFolder }} $(Agent.TempDirectory)
 
-    - ${{ if eq(parameters.TrtVersion, '10.9.0.34') }}:
       - powershell: |
-          Write-Host "##vso[task.setvariable variable=trtPlatformString;]Windows10.x86_64"
-        displayName: 'Set TRT platform string for 10.9.0.34'
+          Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\${{ parameters.CudnnFolder }}\bin"
+        displayName: 'Append CUDNN Directory to PATH'
 
-    - ${{ if eq(parameters.TrtVersion, '10.13.3.9') }}:
-      - powershell: |
-          Write-Host "##vso[task.setvariable variable=trtPlatformString;]Windows.win10"
-        displayName: 'Set TRT platform string for 10.13.3.9'
+      - task: CmdLine@2
+        inputs:
+          script: |
+            echo %PATH%
+            dir $(Agent.TempDirectory)
+        displayName: 'Print PATH after download CUDNN SDK'
 
+  - ${{ if eq(parameters.DownloadTRT, true) }}:
     - powershell: |
-        $trtDirName = "TensorRT-${{ parameters.TrtVersion }}.$(trtPlatformString).cuda-$(trtCudaVersion)"
-        Write-Host "TensorRT Directory Name: $trtDirName"
-        Write-Host "##vso[task.setvariable variable=trtDirName;]$trtDirName"
-      displayName: 'Construct TensorRT Directory Name'
+        # These values are copied from common-variables.yml
+        $cuda12_trt_version = '10.9.0.34'
+        $cuda13_trt_version = '10.13.3.9'
+
+        $trtFolder = ""
+        if ("${{ parameters.CudaVersion }}" -eq "13.0") {
+          $trtFolder = "TensorRT-$cuda13_trt_version.Windows.win10.cuda-13.0"
+        }
+        elseif ("${{ parameters.CudaVersion }}" -eq "12.8") {
+          $trtFolder = "TensorRT-$cuda12_trt_version.Windows10.x86_64.cuda-12.8"
+        }
+
+        Write-Host "Setting TrtFolderRuntime variable to: $trtFolder"
+        # This makes the variable available to subsequent steps in this job
+        Write-Host "##vso[task.setvariable variable=TrtFolderRuntime;]$trtFolder"
+      displayName: 'Set TensorRT Folder Variable'
 
     - task: AzureCLI@2
-      displayName: 'Download $(trtDirName)'
+      displayName: 'Download TensorRT Directory'
       inputs:
         azureSubscription: AIInfraBuildOnnxRuntimeOSS
         scriptType: 'batch'
         scriptLocation: 'inlineScript'
         inlineScript: |
           set AZCOPY_AUTO_LOGIN_TYPE=AZCLI
-          azcopy.exe cp --recursive https://lotusscus.blob.core.windows.net/models/local/$(trtDirName) $(Agent.TempDirectory)
+          azcopy.exe cp --recursive https://lotusscus.blob.core.windows.net/models/local/$(TrtFolderRuntime) $(Agent.TempDirectory)
 
     - powershell: |
-        Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\$(trtDirName)\lib"
-      displayName: 'Append $(trtDirName) Directory to PATH'
+        Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\$(TrtFolderRuntime)\lib"
+      displayName: 'Append TensorRT Directory to PATH'
 
     - task: CmdLine@2
       inputs:

tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml

@@ -21,6 +21,7 @@ parameters:
   default: '12.8'
   values:
    - 12.8
+   - 13.0
 
 # TODO: Ideally it should fetch information from the build that triggers it
 - name: cmake_build_type