Merge branch 'main' of https://github.com/microsoft/onnxruntime into sajandhy/webgpu_gqa_remove_split

Author: satyajandhyala

.config/1espt/PipelineAutobaseliningConfig.yml

@@ -5,15 +5,16 @@ pipelines:
     retail:
       source:
         credscan:
-          lastModifiedDate: 2024-10-24
+          lastModifiedDate: 2024-10-25
         policheck:
-          lastModifiedDate: 2024-10-24
+          lastModifiedDate: 2024-10-25
         eslint:
-          lastModifiedDate: 2024-10-24
+          lastModifiedDate: 2024-10-25
         psscriptanalyzer:
-          lastModifiedDate: 2024-10-24
+          lastModifiedDate: 2024-10-25
         armory:
-          lastModifiedDate: 2024-10-24
+          lastModifiedDate: 2024-10-25
+    usedNonDefaultBranch: true
   1299:
     retail:
       source:
@@ -25,10 +26,52 @@ pipelines:
           lastModifiedDate: 2024-10-25
         armory:
           lastModifiedDate: 2024-10-25
+        policheck:
+          lastModifiedDate: 2024-10-29
       binary:
         credscan:
           lastModifiedDate: 2024-10-25
         binskim:
           lastModifiedDate: 2024-10-25
         spotbugs:
           lastModifiedDate: 2024-10-25
+  1625:
+    retail:
+      source:
+        credscan:
+          lastModifiedDate: 2024-11-05
+        policheck:
+          lastModifiedDate: 2024-11-05
+        eslint:
+          lastModifiedDate: 2024-11-05
+        psscriptanalyzer:
+          lastModifiedDate: 2024-11-05
+        armory:
+          lastModifiedDate: 2024-11-05
+      binary:
+        credscan:
+          lastModifiedDate: 2024-11-13
+        binskim:
+          lastModifiedDate: 2024-11-13
+        spotbugs:
+          lastModifiedDate: 2024-11-13
+  1626:
+    retail:
+      source:
+        credscan:
+          lastModifiedDate: 2024-11-13
+        policheck:
+          lastModifiedDate: 2024-11-13
+        eslint:
+          lastModifiedDate: 2024-11-13
+        psscriptanalyzer:
+          lastModifiedDate: 2024-11-13
+        armory:
+          lastModifiedDate: 2024-11-13
+      binary:
+        credscan:
+          lastModifiedDate: 2024-11-13
+        binskim:
+          lastModifiedDate: 2024-11-13
+        spotbugs:
+          lastModifiedDate: 2024-11-13

.config/guardian/.gdnbaselines

@@ -0,0 +1,43 @@
+{
+  "properties": {
+    "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/baselines"
+  },
+  "version": "1.0.0",
+  "baselines": {
+    "default": {
+      "name": "default",
+      "createdDate": "2024-11-13 00:40:35Z",
+      "lastUpdatedDate": "2024-11-13 00:40:35Z"
+    }
+  },
+  "results": {
+    "48f03e2797fc40ecea50f878a0268947c7e13db1b2fa51aa3981246844fc4c68": {
+      "signature": "48f03e2797fc40ecea50f878a0268947c7e13db1b2fa51aa3981246844fc4c68",
+      "alternativeSignatures": [],
+      "target": "ScanTelemetry_20241113003616898.json",
+      "line": 1,
+      "memberOf": [
+        "default"
+      ],
+      "tool": "credscan",
+      "ruleId": "CSCAN-AZURE0130",
+      "createdDate": "2024-11-13 00:40:35Z",
+      "expirationDate": "2025-05-02 01:29:47Z",
+      "justification": "This error is baselined with an expiration date of 180 days from 2024-11-13 01:29:47Z"
+    },
+    "9cb6eddb3f3e886ad06cae65f5886412ff0c5fb0b96d4e943e4efa237be617b1": {
+      "signature": "9cb6eddb3f3e886ad06cae65f5886412ff0c5fb0b96d4e943e4efa237be617b1",
+      "alternativeSignatures": [],
+      "target": "ScanTelemetry_20241113111547065.json",
+      "line": 1,
+      "memberOf": [
+        "default"
+      ],
+      "tool": "credscan",
+      "ruleId": "CSCAN-AZURE0130",
+      "createdDate": "2024-11-13 11:20:17Z",
+      "expirationDate": "2025-05-02 11:55:15Z",
+      "justification": "This error is baselined with an expiration date of 180 days from 2024-11-13 11:55:15Z"
+    }
+  }
+}

.github/workflows/codeql.yml

@@ -15,6 +15,10 @@ on:
   schedule:
     - cron: '41 13 * * 0'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze

.github/workflows/lint.yml

@@ -37,22 +37,28 @@ jobs:
     # Required workflow
     name: Python format
     runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
+    permissions:
+      contents: read
+      security-events: write
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          # Version range or exact version of Python to use, using SemVer's version range syntax. Reads from .python-version if unset.
+          # Use the version configured in target-version of [tool.black] section in pyproject.toml.
           python-version: "3.10"
       - name: Setup Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           components: rustfmt
+      - name: Update PATH
+        run: |
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements-dev.txt
-          python -m pip install lintrunner lintrunner-adapters
+          set -e -x
+          python -m pip install --user -r requirements-dev.txt
           lintrunner init
       - name: Run lintrunner on all files
         run: |
@@ -81,8 +87,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+      - name: Update PATH
+        run: |
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Install ninja
-        run: python -m pip install --upgrade ninja
+        run: python -m pip install --user --upgrade ninja
       - name: Generate compile_commands.json
         run: |
           python tools/ci_build/build.py \

.github/workflows/mac.yml

@@ -117,17 +117,21 @@ jobs:
       with:
         python-version: ${{ env.python_version }}
 
+    - name: "Run Homebrew"
+      run: brew install autoconf automake autoconf-archive
+
     - name: "Run vcpkg(x64-osx)"
       uses: lukka/run-vcpkg@v11
       with:
         vcpkgDirectory: "${{ runner.temp }}/vcpkg"
-        vcpkgGitCommitId: "1de2026f28ead93ff1773e6e680387643e914ea1" # 2024.07.12
-        runVcpkgInstall: true
+        vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
+        runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
         vcpkgJsonGlob: "cmake/vcpkg.json"
         vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
       env:
         VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
         VCPKG_DEFAULT_TRIPLET: "x64-osx"
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
         # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
 
     - name: "Run compile_schema.py"
@@ -163,19 +167,22 @@ jobs:
           --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-osx" \
           --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
           --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+      env:
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
       shell: bash
 
     - name: "Run vcpkg(arm64-osx)"
       uses: lukka/run-vcpkg@v11
       with:
         vcpkgDirectory: "${{ runner.temp }}/vcpkg"
-        vcpkgGitCommitId: "1de2026f28ead93ff1773e6e680387643e914ea1" # 2024.07.12
-        runVcpkgInstall: true
+        doNotUpdateVcpkg: true
+        runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
         vcpkgJsonGlob: "cmake/vcpkg.json"
         vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
       env:
         VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
         VCPKG_DEFAULT_TRIPLET: "arm64-osx"
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
         # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
 
     - name: "Run build.py(arm64-osx)"
@@ -193,6 +200,8 @@ jobs:
           --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-osx" \
           --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
           --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+      env:
+        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
       shell: bash
 
   Objective-C-StaticAnalysis:

.github/workflows/pr_checks.yml

@@ -41,12 +41,12 @@ jobs:
 
       - name: Install dependencies and run lintrunner on all files
         run: |
-          set -e
           python -m pip install --user -r requirements-dev.txt
-          python -m pip install --user lintrunner lintrunner-adapters	  
+          python -m pip install --user lintrunner lintrunner-adapters
           lintrunner init
+          set +e
           lintrunner f --all-files -v
           exit 0
-      - uses: parkerbxyz/suggest-changes@v1
+      - uses: parkerbxyz/suggest-changes@v2
         with:
           comment: 'You can commit the suggested changes from lintrunner.'

.github/workflows/publish-csharp-apidocs.yml

@@ -20,18 +20,17 @@ permissions:
 
 jobs:
   build:
-    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
     env:
       DOCFXVERSION: 2.62.2
     steps:
     - uses: actions/checkout@v4
-    - name: Setup .NET
-      uses: actions/setup-dotnet@v4
-      with:
-        dotnet-version: 8.0.x
     - name: Install DocFX
       run: |
         dotnet tool update -g docfx
+    - name: Update PATH
+      run: |
+        Add-Content -Value "$env:USERPROFILE\.dotnet\tools" -Encoding utf8 -Path $env:GITHUB_PATH
     # NOTE: We need to restore Microsoft.ML.OnnxRuntime.csproj manually to set IncludeMobileTargets=false
     # docfx doesn't seem to be able to do that properly resulting in build errors
     - name: Restore dependencies
@@ -50,10 +49,12 @@ jobs:
     - name: Log source commit
       run: git rev-parse --short HEAD > csharp/ApiDocs/csharp/source-version.txt
     - name: Move C# docs into site
+      shell: pwsh
       run: |
-        mkdir -p _site/docs/api
-        rm -rf _site/docs/api/csharp
-        mv csharp/ApiDocs/csharp _site/docs/api/csharp
+        New-Item -Path _site/docs/api -Force -ItemType "Directory" | Out-Null
+        $OutputDirectory="_site/docs/api/csharp"
+        if (Test-Path $OutputDirectory) { Remove-Item -Recurse -Force $OutputDirectory }
+        Move-Item -Path csharp\ApiDocs\csharp -Destination $OutputDirectory
     - name: Upload docs artifact
       uses: actions/upload-artifact@v4
       with:

.github/workflows/publish-python-apidocs.yml

@@ -32,10 +32,10 @@ jobs:
           sudo apt-get install graphviz
       - name: Install dependencies
         run: |
-          python3 -m pip install --upgrade pip
+          python3 -m pip install --user --upgrade pip
           cd docs/python
-          python3 -m pip install -r requirements.txt
-          python3 -m pip install --pre onnxruntime-training -f https://download.onnxruntime.ai/onnxruntime_nightly_cpu.html
+          python3 -m pip install --user -r requirements.txt
+          python3 -m pip install --user --pre onnxruntime-training -f https://download.onnxruntime.ai/onnxruntime_nightly_cpu.html
           python3 -m pip list
       - name: Generate Python docs with Sphinx
         run: |

.github/workflows/windows.yml

@@ -56,12 +56,14 @@ jobs:
         uses: lukka/run-vcpkg@v11
         with:
           vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
-          runVcpkgInstall: true
+          vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
+          runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
           vcpkgJsonGlob: "cmake/vcpkg.json"
           vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
         env:
           VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
           VCPKG_DEFAULT_TRIPLET: "x64-windows"
+          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
           # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
 
       - name: "Run compile_schema.py"
@@ -99,19 +101,22 @@ jobs:
             --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-windows" `
             --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
             --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+        env:
+          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
         shell: pwsh
 
       - name: "Run vcpkg(arm64-windows)"
         uses: lukka/run-vcpkg@v11
         with:
           vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
           doNotUpdateVcpkg: true
-          runVcpkgInstall: true
+          runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
           vcpkgJsonGlob: "cmake/vcpkg.json"
           vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
         env:
           VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
           VCPKG_DEFAULT_TRIPLET: "arm64-windows"
+          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
           # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
 
       - name: "Run build.py(arm64-windows)"
@@ -128,4 +133,6 @@ jobs:
             --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-windows" `
             --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
             --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+        env:
+          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
         shell: pwsh

.lintrunner.toml

@@ -2,31 +2,23 @@
 # You can install the dependencies and initialize with
 #
 # ```sh
-# pip install lintrunner lintrunner-adapters
+# pip install -r requirements-lintrunner.txt
 # lintrunner init
 # ```
 #
 # This will install lintrunner on your system and download all the necessary
 # dependencies to run linters locally.
-# If you want to see what lintrunner init will install, run
-# `lintrunner init --dry-run`.
 #
-# To lint local changes:
+# To format local changes:
 #
 # ```bash
-# lintrunner
+# lintrunner -a
 # ```
 #
-# To lint all files:
+# To format all files:
 #
 # ```bash
-# lintrunner --all-files
-# ```
-#
-# To format files:
-#
-# ```bash
-# lintrunner f --all-files
+# lintrunner -a --all-files
 # ```
 #
 # To read more about lintrunner, see [wiki](https://github.com/pytorch/pytorch/wiki/lintrunner).