VCPKG improvements (#23688)

### Description
1. Enabling wasm build.
2. Because the above, I need to delete all the existing vcpkg triplet
files and use build.py to generate on-the-fly.
3. Updated onnx vcpkg port, to align with the latest changes in the
upstream vcpkg repo.
4. Disabling downloading protoc when onnxruntime_USE_VCPKG is true.
5. Added "-e SYSTEM_COLLECTIONURI" to all "docker run " commands, so
that in the docker instance I can use this information to determine if
the build is running in an ONNX Runtime's team ADO project.
6. Disable vcpkg asset caching if the build is not running in an ONNX
Runtime's team ADO project.
7. Add coreml-ep and dml-ep as ONNX Runtime package's features to
"cmake/vcpkg.json"
8. Add VCPKG to
tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile ,
which is used by MIGRAPHX CI build pipeline.

### Motivation and Context
This PR appears large because there were a lot of generated vcpkg
triplet files checked in to this repo. This PR deletes them. Here are
the reasons:

The official vcpkg repository has about 80 different triplets. But ONNX
Runtime has many more build variants. For example, in general, for each
platform, we need to support builds with C++ exceptions, builds without
C++ exceptions, builds with C++ RTTI, builds without C++ RTTI, linking
to static C++ runtime, linking to dynamic (shared) C++ runtime, builds
with address sanitizer, builds without address sanitizer, etc.
Therefore, a script file was created to dynamically generate the triplet
files on-the-fly.

Originally, we tried to check in all the generated files into our
repository so that people could build onnxruntime without using build.py
or any other Python scripts in the "/tools" directory. However, we
encountered an issue when adding support for WASM builds. VCPKG has a
limitation that when doing cross-compiling, the triplet file must
specify the full path of the chain-loaded toolchain file. The file needs
to be located either via environment variables (like ANDROID_NDK_HOME)
or via an absolute path. Since environment variables are hard to track,
we chose the latter approach. So the generated triplet files may contain
absolute file paths that are only valid on the current build machine.

Author: Changming Sun

cmake/CMakeLists.txt

@@ -794,6 +794,7 @@ else()
   endif()
   if (HAS_DEPRECATED_ANON_ENUM_ENUM_CONVERSION)
     list(APPEND ORT_WARNING_FLAGS -Wno-deprecated-anon-enum-enum-conversion)
+    list(APPEND ORT_WARNING_FLAGS -Wno-deprecated-enum-enum-conversion)
   endif()
   if (HAS_UNDEFINED_VAR_TEMPLATE)
     list(APPEND ORT_WARNING_FLAGS -Wno-undefined-var-template)
@@ -1085,6 +1086,16 @@ function(onnxruntime_set_compile_flags target_name)
     endif()
     if(onnxruntime_ENABLE_LAZY_TENSOR)
       target_compile_definitions(${target_name} PRIVATE ENABLE_LAZY_TENSOR)
+    endif()
+	# Enable stream for all the non-minimal build
+    if (NOT onnxruntime_MINIMAL_BUILD)
+      target_compile_definitions(${target_name} PRIVATE ORT_ENABLE_STREAM)
+    endif()
+	if (onnxruntime_REDUCED_OPS_BUILD)
+      target_compile_definitions(${target_name} PRIVATE REDUCED_OPS_BUILD)
+    endif()
+	if (onnxruntime_DISABLE_EXTERNAL_INITIALIZERS)
+      target_compile_definitions(${target_name} PRIVATE DISABLE_EXTERNAL_INITIALIZERS)
     endif()
     if (onnxruntime_ENABLE_CPU_FP16_OPS)
       target_compile_definitions(${target_name} PRIVATE ENABLE_CPU_FP16_TRAINING_OPS)

cmake/adjust_global_compile_flags.cmake

@@ -1,12 +1,6 @@
-# work around Android NDK bug which doesn't include -O flag for Release configuration
-# https://github.com/android/ndk/issues/1740
-# TODO: remove this when the NDK version(s) we support get fixed
-if (ANDROID)
-  # NB: attempting to match the effects of this fix: https://android-review.googlesource.com/c/platform/ndk/+/2168845
-  string(APPEND CMAKE_C_FLAGS_RELEASE " -O3")
-  string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O3")
-  string(APPEND CMAKE_ASM_FLAGS_RELEASE " -O3")
+# The flags set by this file are applied to 3rd-party libraries(e.g. protobuf) as well. If you want to set something that is only for ORT's code, please put it in the main CMakeLists.txt file.
 
+if (ANDROID)
   # Build shared libraries with support for 16 KB ELF alignment
   # https://source.android.com/docs/core/architecture/16kb-page-size/16kb#build-lib-16kb-alignment
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,max-page-size=16384")
@@ -101,11 +95,6 @@ if (onnxruntime_MINIMAL_BUILD)
   endif()
 endif()
 
-# Enable stream for all the non-minimal build
-if (NOT onnxruntime_MINIMAL_BUILD)
-  add_compile_definitions(ORT_ENABLE_STREAM)
-endif()
-
 if (onnxruntime_ENABLE_LTO)
     include(CheckIPOSupported)
     check_ipo_supported(RESULT ipo_enabled OUTPUT ipo_output)
@@ -117,14 +106,6 @@ if (onnxruntime_ENABLE_LTO)
     endif()
 endif()
 
-if (onnxruntime_REDUCED_OPS_BUILD)
-  add_compile_definitions(REDUCED_OPS_BUILD)
-endif()
-
-if (onnxruntime_DISABLE_EXTERNAL_INITIALIZERS)
-  add_definitions(-DDISABLE_EXTERNAL_INITIALIZERS=1)
-endif()
-
 if (onnxruntime_DISABLE_RTTI)
   add_compile_definitions(ORT_NO_RTTI)
   if (MSVC)
@@ -160,6 +141,7 @@ if (onnxruntime_DISABLE_EXCEPTIONS)
   add_compile_definitions("ORT_NO_EXCEPTIONS")
   add_compile_definitions("MLAS_NO_EXCEPTION")
   add_compile_definitions("ONNX_NO_EXCEPTIONS")
+  # The following line of code probably is not needed because the same effect can be achieved by setting the compiler flag -fno-exceptions.
   add_compile_definitions("JSON_NOEXCEPTION")  # https://json.nlohmann.me/api/macros/json_noexception/
 
   if (MSVC)
@@ -196,14 +178,6 @@ if (onnxruntime_CROSS_COMPILING)
   endif()
 endif()
 
-if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
-  check_cxx_compiler_flag(-Wno-error HAS_NOERROR)
-  if (HAS_NOERROR)
-    string(APPEND CMAKE_CXX_FLAGS " -Wno-error=attributes")
-    string(APPEND CMAKE_C_FLAGS " -Wno-error=attributes")
-  endif()
-endif()
-
 # Mark symbols to be invisible, for macOS/iOS/visionOS target only
 # Due to many dependencies have different symbol visibility settings, set global compile flags here.
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS|visionOS")
@@ -253,7 +227,6 @@ if (MSVC)
     message(FATAL_ERROR "Unknown CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
   endif()
 
-
   #Always enable exception handling, even for Windows ARM
   if (NOT onnxruntime_DISABLE_EXCEPTIONS)
     string(APPEND CMAKE_CXX_FLAGS " /EHsc")
@@ -326,9 +299,6 @@ else()
     string(APPEND CMAKE_C_FLAGS   " -g -O0 --coverage ")
   endif()
   if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
-    # suppress warnings from flatbuffers
-    string(APPEND CMAKE_CXX_FLAGS " -Wno-restrict ")
-    string(APPEND CMAKE_C_FLAGS   " -Wno-restrict ")
     if(onnxruntime_USE_XNNPACK)
       # https://github.com/google/XNNPACK/issues/7650
       string(APPEND CMAKE_C_FLAGS   " -Wno-incompatible-pointer-types ")

cmake/external/dml.cmake

@@ -100,14 +100,17 @@ else()
   endif()
 endif()
 
-onnxruntime_fetchcontent_declare(
+if (onnxruntime_USE_VCPKG)
+  find_package(directx-headers CONFIG REQUIRED)
+else()
+  onnxruntime_fetchcontent_declare(
     directx_headers
     URL ${DEP_URL_directx_headers}
     URL_HASH SHA1=${DEP_SHA1_directx_headers}
     EXCLUDE_FROM_ALL
-)
-
-FetchContent_Populate(directx_headers)
-set(directx_headers_INCLUDE_DIRS  "${directx_headers_SOURCE_DIR}/include")
+  )
 
-include_directories(BEFORE ${directx_headers_INCLUDE_DIRS})
+  FetchContent_Populate(directx_headers)
+  set(directx_headers_INCLUDE_DIRS  "${directx_headers_SOURCE_DIR}/include")
+  include_directories(BEFORE ${directx_headers_INCLUDE_DIRS})
+endif()

cmake/external/onnxruntime_external_deps.cmake

@@ -125,7 +125,7 @@ onnxruntime_fetchcontent_makeavailable(utf8_range)
 include_directories(${utf8_range_SOURCE_DIR})
 
 # Download a protoc binary from Internet if needed
-if(NOT ONNX_CUSTOM_PROTOC_EXECUTABLE)
+if(NOT ONNX_CUSTOM_PROTOC_EXECUTABLE AND NOT onnxruntime_USE_VCPKG)
   # This part of code is only for users' convenience. The code couldn't handle all cases. Users always can manually
   # download protoc from Protobuf's Github release page and pass the local path to the ONNX_CUSTOM_PROTOC_EXECUTABLE
   # variable.
@@ -292,20 +292,22 @@ onnxruntime_fetchcontent_declare(
 )
 onnxruntime_fetchcontent_makeavailable(date)
 
-onnxruntime_fetchcontent_declare(
-  mp11
-  URL ${DEP_URL_mp11}
-  URL_HASH SHA1=${DEP_SHA1_mp11}
-  EXCLUDE_FROM_ALL
-  FIND_PACKAGE_ARGS NAMES Boost
-)
-onnxruntime_fetchcontent_makeavailable(mp11)
 if(NOT TARGET Boost::mp11)
   if(onnxruntime_USE_VCPKG)
-    find_package(Boost REQUIRED)
+     find_package(Boost REQUIRED)
+     message(STATUS "Aliasing Boost::headers to Boost::mp11")
+     add_library(Boost::mp11 ALIAS Boost::headers)
+  else()
+    onnxruntime_fetchcontent_declare(
+     mp11
+     URL ${DEP_URL_mp11}
+     FIND_PACKAGE_ARGS NAMES Boost
+    )
+    onnxruntime_fetchcontent_makeavailable(mp11)    
+    if(NOT TARGET Boost::mp11)
+      add_library(Boost::mp11 ALIAS Boost::headers)
+    endif()
   endif()
-  message(STATUS "Aliasing Boost::headers to Boost::mp11")
-  add_library(Boost::mp11 ALIAS Boost::headers)
 endif()
 
 set(JSON_BuildTests OFF CACHE INTERNAL "")
@@ -626,35 +628,10 @@ endif()
 
 
 if (onnxruntime_USE_WEBGPU)
-  if (onnxruntime_CUSTOM_DAWN_SRC_PATH)
-    # use the custom dawn source path if provided
-    #
-    # specified as:
-    # build.py --use_webgpu --cmake_extra_defines "onnxruntime_CUSTOM_DAWN_SRC_PATH=<PATH_TO_DAWN_SRC_ROOT>"
-    onnxruntime_fetchcontent_declare(
-      dawn
-      SOURCE_DIR ${onnxruntime_CUSTOM_DAWN_SRC_PATH}
-      EXCLUDE_FROM_ALL
-    )
-  else()
-    onnxruntime_fetchcontent_declare(
-      dawn
-      URL ${DEP_URL_dawn}
-      URL_HASH SHA1=${DEP_SHA1_dawn}
-      # # All previous patches are merged into the upstream dawn project. We don't need to apply any patches right now.
-      # # if we need to apply patches in the future, we can uncomment the following line.
-      #
-      # The dawn.patch contains the following changes:
-      # - https://dawn-review.googlesource.com/c/dawn/+/225514
-      PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/dawn/dawn.patch
-      EXCLUDE_FROM_ALL
-    )
-  endif()
-
   set(DAWN_BUILD_SAMPLES OFF CACHE BOOL "" FORCE)
   set(DAWN_ENABLE_NULL OFF CACHE BOOL "" FORCE)
   set(DAWN_FETCH_DEPENDENCIES ON CACHE BOOL "" FORCE)
-
+  set(DAWN_BUILD_TESTS OFF CACHE BOOL "" FORCE)
   if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
     set(DAWN_EMSCRIPTEN_TOOLCHAIN "${REPO_ROOT}/cmake/external/emsdk/upstream/emscripten" CACHE STRING "" FORCE)
   else()
@@ -732,6 +709,30 @@ if (onnxruntime_USE_WEBGPU)
       set(DAWN_ENABLE_D3D11 OFF CACHE BOOL "" FORCE)
     endif()
   endif()
+  if (onnxruntime_CUSTOM_DAWN_SRC_PATH)
+    # use the custom dawn source path if provided
+    #
+    # specified as:
+    # build.py --use_webgpu --cmake_extra_defines "onnxruntime_CUSTOM_DAWN_SRC_PATH=<PATH_TO_DAWN_SRC_ROOT>"
+    onnxruntime_fetchcontent_declare(
+      dawn
+      SOURCE_DIR ${onnxruntime_CUSTOM_DAWN_SRC_PATH}
+      EXCLUDE_FROM_ALL
+    )
+  else()
+    onnxruntime_fetchcontent_declare(
+      dawn
+      URL ${DEP_URL_dawn}
+      URL_HASH SHA1=${DEP_SHA1_dawn}
+      # # All previous patches are merged into the upstream dawn project. We don't need to apply any patches right now.
+      # # if we need to apply patches in the future, we can uncomment the following line.
+      #
+      # The dawn.patch contains the following changes:
+      # - https://dawn-review.googlesource.com/c/dawn/+/225514
+      PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/dawn/dawn.patch
+      EXCLUDE_FROM_ALL
+    )
+  endif()
 
   onnxruntime_fetchcontent_makeavailable(dawn)
 
@@ -747,7 +748,7 @@ if (onnxruntime_USE_WEBGPU)
   endif()
 
   if (onnxruntime_ENABLE_PIX_FOR_WEBGPU_EP)
-    list(APPEND onnxruntime_EXTERNAL_LIBRARIES glfw webgpu_glfw)
+    list(APPEND onnxruntime_EXTERNAL_LIBRARIES webgpu_glfw glfw)
   endif()
 endif()
 

cmake/onnxruntime_providers_coreml.cmake

@@ -169,10 +169,11 @@ elseif(TARGET nlohmann_json::nlohmann_json)
 endif()
 
 if(fp16_SOURCE_DIR)
-  target_include_directories(onnxruntime_providers_coreml PRIVATE ${fp16_SOURCE_DIR}/include)
+  set(FP16_INCLUDE_DIRS ${fp16_SOURCE_DIR}/include)
 else()
-  message(FATAL_ERROR "Cannot find fp16's source code")
+  find_path(FP16_INCLUDE_DIRS "fp16.h")
 endif()
+target_include_directories(onnxruntime_providers_coreml PRIVATE ${FP16_INCLUDE_DIRS})
 
 onnxruntime_add_include_to_target(onnxruntime_providers_coreml coreml_proto)
 target_link_libraries(onnxruntime_providers_coreml PRIVATE coreml_proto)
@@ -184,26 +185,16 @@ endif()
 
 
 
-  # need to tweak the include paths to match what the coreml source code expects
-  target_include_directories(onnxruntime_providers_coreml PRIVATE
+# need to tweak the include paths to match what the coreml source code expects
+target_include_directories(onnxruntime_providers_coreml PRIVATE
                             ${coremltools_SOURCE_DIR}
                             ${coremltools_SOURCE_DIR}/mlmodel/src/
                             ${coremltools_SOURCE_DIR}/modelpackage/src/
-  )
-
-# need to tweak the include paths to match what the coreml source code expects
-target_include_directories(onnxruntime_providers_coreml PRIVATE
-                          ${fp16_SOURCE_DIR}/include
-                          ${nlohmann_json_SOURCE_DIR}/single_include/nlohmann
-                          ${coremltools_SOURCE_DIR}
-                          ${coremltools_SOURCE_DIR}/mlmodel/src/
-                          ${coremltools_SOURCE_DIR}/modelpackage/src/
 )
 
-
-  if (LINUX)
-    target_link_libraries(onnxruntime_providers_coreml PRIVATE uuid)
-  endif()
+if (LINUX)
+  target_link_libraries(onnxruntime_providers_coreml PRIVATE uuid)
+endif()
 
 
 

cmake/onnxruntime_providers_dml.cmake

@@ -11,6 +11,9 @@
   onnxruntime_add_include_to_target(onnxruntime_providers_dml
     onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} flatbuffers::flatbuffers Boost::mp11 safeint_interface ${WIL_TARGET}
   )
+  if(TARGET Microsoft::DirectX-Headers)
+    onnxruntime_add_include_to_target(onnxruntime_providers_dml Microsoft::DirectX-Headers)
+  endif()
   add_dependencies(onnxruntime_providers_dml ${onnxruntime_EXTERNAL_DEPENDENCIES})
   target_include_directories(onnxruntime_providers_dml PRIVATE
     ${ONNXRUNTIME_ROOT}

cmake/onnxruntime_webassembly.cmake

@@ -93,7 +93,9 @@ if (NOT onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
   set_property(TARGET re2 PROPERTY COMPILE_OPTIONS )
 endif()
 
-target_compile_options(onnx PRIVATE -Wno-unused-parameter -Wno-unused-variable)
+if (NOT onnxruntime_USE_VCPKG)
+  target_compile_options(onnx PRIVATE -Wno-unused-parameter -Wno-unused-variable)
+endif()
 
 if (onnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB)
     bundle_static_library(onnxruntime_webassembly
@@ -404,14 +406,21 @@ jsepDownload:_pp_")
   endif()
 
   if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-    target_link_options(onnxruntime_webassembly PRIVATE
-      # NOTE: use "SHELL:-s ASSERTIONS=2" to enable more strict assertions, which may help debugging segfaults.
-      #       However, it may be very slow.
-      # "SHELL:-s ASSERTIONS=2"
-      "SHELL:-s ASSERTIONS=1"
-      "SHELL:-s SAFE_HEAP=1"
-      "SHELL:-s STACK_OVERFLOW_CHECK=2"
-    )
+    if (CMAKE_CXX_FLAGS MATCHES "sanitize=address")
+        # The integer value below might often need be adjusted.
+        target_link_options(onnxruntime_webassembly PRIVATE "-sINITIAL_MEMORY=786432000")
+        target_link_options(onnxruntime_webassembly PRIVATE "-sASSERTIONS=2")
+    else()
+        # Enable SAFE_HEAP in debug build
+        target_link_options(onnxruntime_webassembly PRIVATE
+          # NOTE: use "SHELL:-s ASSERTIONS=2" to enable more strict assertions, which may help debugging segfaults.
+          #       However, it may be very slow.
+          # "SHELL:-s ASSERTIONS=2"
+          "SHELL:-s ASSERTIONS=1"
+          "SHELL:-s SAFE_HEAP=1"
+          "SHELL:-s STACK_OVERFLOW_CHECK=2"
+        )
+    endif()
   else()
     target_link_options(onnxruntime_webassembly PRIVATE
       "SHELL:-s ASSERTIONS=0"

cmake/vcpkg-configuration.json

@@ -2,7 +2,7 @@
     "default-registry": {
         "kind": "git",
         "repository": "https://github.com/Microsoft/vcpkg",
-        "baseline": "d504de05dcd7b55df34976be1c824324ec6bca2b"
+        "baseline": "a29711cc86340a43c054cd37b8bd2871332a01e9"
     },
     "overlay-ports": [
 	"./vcpkg-ports"

cmake/vcpkg-ports/onnx/other.patch cmake/vcpkg-ports/onnx/binskim.patchcmake/vcpkg-ports/onnx/other.patch renamed to cmake/vcpkg-ports/onnx/binskim.patch

@@ -1,16 +1,8 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index d15d97ed..a483255f 100644
+index d15d97ed..fe4bd27f 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -496,6 +496,7 @@ if (MSVC)
-   endif()
- else()
-   # On non-Windows, hide all symbols we don't need
-+  set(EXTRA_FLAGS "-Wno-unused-parameter")
-   set(ONNX_API_DEFINE "-DONNX_API=__attribute__\(\(__visibility__\(\"default\"\)\)\)")
-   set_target_properties(onnx_proto PROPERTIES CXX_VISIBILITY_PRESET hidden)
-   set_target_properties(onnx_proto PROPERTIES VISIBILITY_INLINES_HIDDEN 1)
-@@ -631,20 +632,9 @@ endif()
+@@ -631,20 +631,9 @@ endif()
  if(MSVC)
    target_compile_options(onnx_proto
                           PRIVATE /MP