Skip to content

Commit 5f6a315

Browse files
author
Changming Sun
authored
Enable VCPKG in CI build (#23426)
### Description 1. Enable VCPKG flag in Windows CPU CI build pipelines. 2. Increased the min supported cmake version from 3.26 to 3.28. Because of it, drop the support for the old way of finding python by "find_package(PythonLibs)". Therefore, in build.py we no longer set "PYTHON_EXECUTABLE" cmake var when doing cmake configure. 3. Added "xnnpack-ep" as a feature for ORT's vcpkg config. 4. Added asset cache support for ORT's vcpkg build 5. Added VCPKG triplet files for Android build. 6. Set VCPKG triplet to "universal2-osx" if CMAKE_OSX_ARCHITECTURES was found in cmake extra defines. 7. Removed a small piece of code in build.py, which was for support CUDA version < 11.8. 8. Fixed an issue that CMAKE_OSX_ARCHITECTURES sometimes got specified twice when build.py invoked cmake. 9. Added more model tests to Android build. After this change, we will test all ONNX versions instead of just the latest one. 10. Fixed issues that are related to build.py's "--build_nuget" parameter. Also, enable the flag in most Windows CPU CI build jobs. 11. Removed a restriction in build.py that disallowed cross-compiling Windows ARM64 nuget package on Windows x86. ### Motivation and Context Adopt vcpkg.
1 parent e1e3f62 commit 5f6a315

File tree

206 files changed

+3843
-2438
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

206 files changed

+3843
-2438
lines changed

‎.github/workflows/mac.yml‎

Lines changed: 0 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -109,101 +109,6 @@ jobs:
109109
--apple_sysroot=iphonesimulator \
110110
--osx_arch=${{ matrix.target_arch }}
111111
112-
Vcpkg:
113-
runs-on: macos-13
114-
steps:
115-
- uses: actions/checkout@v4
116-
- uses: actions/setup-python@v5
117-
with:
118-
python-version: ${{ env.python_version }}
119-
120-
- name: "Run Homebrew"
121-
run: brew install autoconf automake autoconf-archive
122-
123-
- name: "Run vcpkg(x64-osx)"
124-
uses: lukka/run-vcpkg@v11
125-
with:
126-
vcpkgDirectory: "${{ runner.temp }}/vcpkg"
127-
vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
128-
runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
129-
vcpkgJsonGlob: "cmake/vcpkg.json"
130-
vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
131-
env:
132-
VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
133-
VCPKG_DEFAULT_TRIPLET: "x64-osx"
134-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
135-
# VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
136-
137-
- name: "Run compile_schema.py"
138-
run: |
139-
set -e -x
140-
# Runner's host triplet should be x64-osx or arm64-osx
141-
export FLATC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-osx/tools/flatbuffers"
142-
export PATH="$FLATC_DIR:$PATH"
143-
flatc --version
144-
python onnxruntime/core/flatbuffers/schema/compile_schema.py --flatc "$(which flatc)"
145-
python onnxruntime/lora/adapter_format/compile_schema.py --flatc "$(which flatc)"
146-
147-
- name: "Detect protoc"
148-
id: protoc-detect
149-
run: |
150-
export PROTOC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-osx/tools/protobuf"
151-
export PATH="$PROTOC_DIR:$PATH"
152-
protoc --version
153-
echo "protoc_path=$(which protoc)" >> "$GITHUB_OUTPUT"
154-
155-
- name: "Run build.py(x64-osx)"
156-
run: |
157-
python ./tools/ci_build/build.py \
158-
--build_dir "build/x64-osx" \
159-
--skip_submodule_sync \
160-
--skip_tests \
161-
--compile_no_warning_as_error \
162-
--parallel \
163-
--path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" \
164-
--osx_arch x86_64 \
165-
--use_vcpkg \
166-
--cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
167-
--cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-osx" \
168-
--cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
169-
--cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
170-
env:
171-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
172-
shell: bash
173-
174-
- name: "Run vcpkg(arm64-osx)"
175-
uses: lukka/run-vcpkg@v11
176-
with:
177-
vcpkgDirectory: "${{ runner.temp }}/vcpkg"
178-
doNotUpdateVcpkg: true
179-
runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
180-
vcpkgJsonGlob: "cmake/vcpkg.json"
181-
vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
182-
env:
183-
VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
184-
VCPKG_DEFAULT_TRIPLET: "arm64-osx"
185-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
186-
# VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
187-
188-
- name: "Run build.py(arm64-osx)"
189-
run: |
190-
python ./tools/ci_build/build.py \
191-
--build_dir "build/arm64-osx" \
192-
--skip_submodule_sync \
193-
--skip_tests \
194-
--compile_no_warning_as_error \
195-
--parallel \
196-
--path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" \
197-
--osx_arch arm64 \
198-
--use_vcpkg \
199-
--cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
200-
--cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-osx" \
201-
--cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
202-
--cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
203-
env:
204-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
205-
shell: bash
206-
207112
Objective-C-StaticAnalysis:
208113
runs-on: macos-14
209114

‎.github/workflows/windows.yml‎

Lines changed: 1 addition & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -41,98 +41,4 @@ jobs:
4141
4242
# The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
4343
- name: Build code
44-
run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v12.2 --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75
45-
46-
Vcpkg:
47-
runs-on: "windows-latest"
48-
steps:
49-
- uses: actions/checkout@v4
50-
- uses: actions/setup-python@v5
51-
with:
52-
python-version: '3.11.x'
53-
architecture: 'x64'
54-
55-
- name: "Run vcpkg(x64-windows)"
56-
uses: lukka/run-vcpkg@v11
57-
with:
58-
vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
59-
vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
60-
runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
61-
vcpkgJsonGlob: "cmake/vcpkg.json"
62-
vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
63-
env:
64-
VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
65-
VCPKG_DEFAULT_TRIPLET: "x64-windows"
66-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
67-
# VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
68-
69-
- name: "Run compile_schema.py"
70-
run: |
71-
# Runner's host triplet should be x64-windows or arm64-windows
72-
$FLATC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-windows/tools/flatbuffers"
73-
$env:PATH="$FLATC_DIR;$env:PATH"
74-
flatc --version
75-
$FLATC_PATH = Join-Path "$FLATC_DIR" "flatc.exe"
76-
python onnxruntime/core/flatbuffers/schema/compile_schema.py --flatc "$FLATC_PATH"
77-
python onnxruntime/lora/adapter_format/compile_schema.py --flatc "$FLATC_PATH"
78-
shell: pwsh
79-
80-
- name: "Detect protoc"
81-
id: protoc-detect
82-
run: |
83-
$PROTOC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-windows/tools/protobuf"
84-
$env:PATH="$PROTOC_DIR;$env:PATH"
85-
protoc --version
86-
$PROTOC_PATH = Join-Path "$PROTOC_DIR" "protoc.exe"
87-
"protoc_path=$PROTOC_PATH" >> $env:GITHUB_OUTPUT
88-
shell: pwsh
89-
90-
- name: "Run build.py(x64-windows)"
91-
run: |
92-
python tools\ci_build\build.py `
93-
--build_dir "cmake_build/x64-windows" `
94-
--skip_submodule_sync `
95-
--skip_tests `
96-
--compile_no_warning_as_error `
97-
--parallel `
98-
--path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" `
99-
--use_vcpkg `
100-
--cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" `
101-
--cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-windows" `
102-
--cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
103-
--cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
104-
env:
105-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
106-
shell: pwsh
107-
108-
- name: "Run vcpkg(arm64-windows)"
109-
uses: lukka/run-vcpkg@v11
110-
with:
111-
vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
112-
doNotUpdateVcpkg: true
113-
runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
114-
vcpkgJsonGlob: "cmake/vcpkg.json"
115-
vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
116-
env:
117-
VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
118-
VCPKG_DEFAULT_TRIPLET: "arm64-windows"
119-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
120-
# VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
121-
122-
- name: "Run build.py(arm64-windows)"
123-
run: |
124-
python tools\ci_build\build.py `
125-
--build_dir "cmake_build/arm64-windows" --arm64 `
126-
--skip_submodule_sync `
127-
--skip_tests `
128-
--compile_no_warning_as_error `
129-
--parallel `
130-
--path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" `
131-
--use_vcpkg `
132-
--cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" `
133-
--cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-windows" `
134-
--cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
135-
--cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
136-
env:
137-
VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
138-
shell: pwsh
44+
run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v12.2 --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75

‎cmake/CMakeLists.txt‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,10 @@
33
# Licensed under the MIT License.
44

55
# Minimum CMake required
6-
cmake_minimum_required(VERSION 3.26)
6+
cmake_minimum_required(VERSION 3.28)
77

8-
cmake_policy(SET CMP0069 NEW)
9-
set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
10-
11-
cmake_policy(SET CMP0092 NEW)
12-
cmake_policy(SET CMP0091 NEW)
13-
cmake_policy(SET CMP0117 NEW)
148
# Don't let cmake set a default value for CMAKE_CUDA_ARCHITECTURES
159
cmake_policy(SET CMP0104 OLD)
16-
17-
# Enable Hot Reload for MSVC compilers if supported.
18-
cmake_policy(SET CMP0141 NEW)
19-
2010
# Project
2111
project(onnxruntime C CXX ASM)
2212

@@ -59,8 +49,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
5949

6050
# Enable CTest
6151
enable_testing()
62-
include(Dart)
63-
52+
include(CTest)
6453
if (NOT CMAKE_BUILD_TYPE)
6554
message(STATUS "Build type not set - using RelWithDebInfo")
6655
set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose build type: Debug Release RelWithDebInfo MinSizeRel." FORCE)
@@ -968,7 +957,7 @@ if (onnxruntime_USE_QNN OR onnxruntime_USE_QNN_INTERFACE)
968957

969958
if ((NOT onnxruntime_USE_QNN_INTERFACE) AND (MSVC OR ${CMAKE_SYSTEM_NAME} STREQUAL "Linux"))
970959
file(GLOB QNN_LIB_FILES LIST_DIRECTORIES false "${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/libQnn*.so"
971-
"${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/Qnn*.dll"
960+
"${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/Qnn*.dll"
972961
"${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/libHtpPrepare.so"
973962
"${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/HtpPrepare.dll")
974963
if (${QNN_ARCH_ABI} STREQUAL "aarch64-windows-msvc" OR ${QNN_ARCH_ABI} STREQUAL "arm64x-windows-msvc")
@@ -1132,7 +1121,10 @@ function(onnxruntime_set_compile_flags target_name)
11321121
foreach(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
11331122
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY}>")
11341123
endforeach()
1135-
1124+
if(VCPKG_INSTALLED_DIR)
1125+
cmake_path(NATIVE_PATH VCPKG_INSTALLED_DIR NORMALIZE VCPKG_INSTALLED_DIR_native)
1126+
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:SHELL:/external:I ${VCPKG_INSTALLED_DIR_native}>")
1127+
endif()
11361128
foreach(onnxruntime_external_lib IN LISTS onnxruntime_EXTERNAL_LIBRARIES)
11371129
#TODO: the list contains cmake keywords like "debug". We should exclude them.
11381130
if(TARGET ${onnxruntime_external_lib})
@@ -1162,7 +1154,9 @@ function(onnxruntime_set_compile_flags target_name)
11621154
# We do not treat warnings from 3rd-party libraries as errors. In order to do that, we need to add their header files locations to /external:I.
11631155
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/experimental:external>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /experimental:external>")
11641156
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:W0>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:W0>")
1165-
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:templates->" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:templates->")
1157+
#if(NOT onnxruntime_USE_VCPKG)
1158+
# target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:templates->" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:templates->")
1159+
#endif()
11661160
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CURRENT_SOURCE_DIR}>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:I${CMAKE_CURRENT_SOURCE_DIR}>")
11671161
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CURRENT_BINARY_DIR}>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:I${CMAKE_CURRENT_BINARY_DIR}>")
11681162
if (onnxruntime_ENABLE_STATIC_ANALYSIS)
@@ -1725,6 +1719,9 @@ endif()
17251719

17261720

17271721
if (WIN32)
1722+
if (onnxruntime_USE_MIMALLOC)
1723+
list(APPEND onnxruntime_EXTERNAL_LIBRARIES mimalloc-static)
1724+
endif()
17281725
list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${SYS_PATH_LIB})
17291726
list(APPEND onnxruntime_EXTERNAL_LIBRARIES debug Dbghelp)
17301727
# In a onecore build the umbrella libs already contains references to the APIs in advapi32, so in onecore build we do not need to link to advapi32

‎cmake/deps_update_and_upload.py‎

Lines changed: 0 additions & 109 deletions
This file was deleted.

0 commit comments

Comments
 (0)