Merge branch 'microsoft:main' into main

Author: orlmon01

cmake/onnxruntime.cmake

@@ -28,6 +28,7 @@ function(get_c_cxx_api_headers HEADERS_VAR)
     "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h"
     "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_float16.h"
     "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_lite_custom_op.h"
+    "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_env_config_keys.h"
     "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h"
     "${REPO_ROOT}/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h"
   )

cmake/onnxruntime_unittests.cmake

@@ -597,6 +597,7 @@ set (onnxruntime_shared_lib_test_SRC
 if (NOT onnxruntime_MINIMAL_BUILD)
   list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc)
   list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_model_builder_api.cc)
+  list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_env_creation.cc)
 endif()
 
 if(onnxruntime_RUN_ONNX_TESTS)

docs/OperatorKernels.md

@@ -653,6 +653,7 @@ Do not modify directly.*
 |ArgMin|*in* data:**T**<br> *out* reduced:**tensor(int64)**|13+|**T** = tensor(double), tensor(float), tensor(float16)|
 |||12|**T** = tensor(double), tensor(float), tensor(float16)|
 |||[1, 11]|**T** = tensor(double), tensor(float), tensor(float16)|
+|Attention|*in* Q:**T1**<br> *in* K:**T1**<br> *in* V:**T2**<br> *in* attn_mask:**U**<br> *in* past_key:**T1**<br> *in* past_value:**T2**<br> *in* nonpad_kv_seqlen:**tensor(int64)**<br> *out* Y:**T1**<br> *out* present_key:**T1**<br> *out* present_value:**T2**<br> *out* qk_matmul_output:**T1**<br><br>or<br><br>*in* Q:**T1**<br> *in* K:**T1**<br> *in* V:**T2**<br> *in* attn_mask:**U**<br> *in* past_key:**T1**<br> *in* past_value:**T2**<br> *out* Y:**T1**<br> *out* present_key:**T1**<br> *out* present_value:**T2**<br> *out* qk_matmul_output:**T1**|23+|**T1** = tensor(bfloat16), tensor(float), tensor(float16)<br/> **T2** = tensor(bfloat16), tensor(float), tensor(float16)<br/> **U** = tensor(bfloat16), tensor(bool), tensor(float), tensor(float16)|
 |AveragePool|*in* X:**T**<br> *out* Y:**T**|22+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)|
 |||[19, 21]|**T** = tensor(double), tensor(float), tensor(float16)|
 |||[11, 18]|**T** = tensor(double), tensor(float), tensor(float16)|

include/onnxruntime/core/session/environment.h

@@ -7,6 +7,7 @@
 #include <filesystem>
 #include <memory>
 #include <vector>
+#include <shared_mutex>
 #include <string>
 
 #include "core/common/common.h"
@@ -20,6 +21,7 @@
 #include "core/platform/threadpool.h"
 
 #include "core/session/abi_devices.h"
+#include "core/session/abi_key_value_pairs.h"
 #include "core/session/plugin_ep/ep_library.h"
 #include "core/session/onnxruntime_c_api.h"
 
@@ -51,11 +53,13 @@ class Environment {
     @param tp_options optional set of parameters controlling the number of intra and inter op threads for the global
     threadpools.
     @param create_global_thread_pools determine if this function will create the global threadpools or not.
+    @param config_entries Application-specified configuration entries.
   */
   static Status Create(std::unique_ptr<logging::LoggingManager> logging_manager,
                        std::unique_ptr<Environment>& environment,
                        const OrtThreadingOptions* tp_options = nullptr,
-                       bool create_global_thread_pools = false);
+                       bool create_global_thread_pools = false,
+                       const OrtKeyValuePairs* config_entries = nullptr);
 
   /**
    * Set the global threading options for the environment, if no global thread pools have been created yet.
@@ -170,14 +174,26 @@ class Environment {
   // return a shared allocator from a plugin EP or custom allocator added with RegisterAllocator
   Status GetSharedAllocator(const OrtMemoryInfo& mem_info, OrtAllocator*& allocator);
 
+  /// <summary>
+  /// Returns a copy of the configuration entries set by the application on environment creation.
+  ///
+  /// Primarily used by EP libraries to retrieve environment-level configurations, but could be used
+  /// more generally to specify global settings.
+  ///
+  /// Refer to OrtApi::CreateEnvWithOptions().
+  /// </summary>
+  /// <returns></returns>
+  OrtKeyValuePairs GetConfigEntries() const;
+
   ~Environment();
 
  private:
   ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Environment);
 
   Status Initialize(std::unique_ptr<logging::LoggingManager> logging_manager,
                     const OrtThreadingOptions* tp_options = nullptr,
-                    bool create_global_thread_pools = false);
+                    bool create_global_thread_pools = false,
+                    const OrtKeyValuePairs* config_entries = nullptr);
 
   Status RegisterAllocatorImpl(AllocatorPtr allocator);
   Status UnregisterAllocatorImpl(const OrtMemoryInfo& mem_info, bool error_if_not_found = true);
@@ -186,6 +202,13 @@ class Environment {
                                    const OrtKeyValuePairs* allocator_options, OrtAllocator** allocator,
                                    bool replace_existing);
 
+  // Inserts (or assigns) a config entry into `config_entries_`. Locks `config_entries_mutex_`.
+  void InsertOrAssignConfigEntry(std::string key, std::string value);
+
+  // Removes a config entry from `config_entries_`. Does nothing if the key does not exist.
+  // Locks `config_entries_mutex_`.
+  void RemoveConfigEntry(const std::string& key);
+
   std::unique_ptr<logging::LoggingManager> logging_manager_;
   std::unique_ptr<onnxruntime::concurrency::ThreadPool> intra_op_thread_pool_;
   std::unique_ptr<onnxruntime::concurrency::ThreadPool> inter_op_thread_pool_;
@@ -254,6 +277,20 @@ class Environment {
   DataTransferManager data_transfer_mgr_;  // plugin EP IDataTransfer instances
 
 #endif  // !defined(ORT_MINIMAL_BUILD)
+
+  // Application-specified environment configuration entries
+  // The environment may add or remove an entry on EP library registration and unregistration, respectively.
+  OrtKeyValuePairs config_entries_;
+  mutable std::shared_mutex config_entries_mutex_;  // Should be locked when accessing config_entries_
+
+  // Tracks the number of registered EP libraries that can create virtual devices.
+  // It is incremented when an EP library is registered with a name that ends in ".virtual".
+  // It is decremented when that EP library is unregistered.
+  // If it reaches 0, the config entry "allow_virtual_devices" is removed.
+  //
+  // This starts at 1 if user created an OrtEnv with the config "allow_virtual_devices" set to "1"
+  // to prevent removal of the config entry in that case.
+  size_t num_allow_virtual_device_uses_{};
 };
 
 }  // namespace onnxruntime

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -965,14 +965,6 @@ typedef OrtStatus*(ORT_API_CALL* RegisterCustomOpsFn)(OrtSessionOptions* options
  */
 typedef void (*RunAsyncCallbackFn)(void* user_data, OrtValue** outputs, size_t num_outputs, OrtStatusPtr status);
 
-/** \brief The C API
- *
- * All C API functions are defined inside this structure as pointers to functions.
- * Call OrtApiBase::GetApi to get a pointer to it
- *
- * \nosubgrouping
- */
-
 /** \addtogroup Global
  * @{
  */
@@ -1056,6 +1048,101 @@ typedef enum OrtCompiledModelCompatibility {
   OrtCompiledModelCompatibility_EP_UNSUPPORTED,
 } OrtCompiledModelCompatibility;
 
+/** \brief Configuration options for creating an OrtEnv.
+ *
+ * \note The version field must be set to ORT_API_VERSION.
+ *       This ensures forward compatibility as fields may be added in future versions.
+ *
+ * \since Version 1.24.
+ */
+typedef struct OrtEnvCreationOptions {
+  uint32_t version;  ///< Must be set to ORT_API_VERSION
+
+  /** \brief The logging severity level for the environment. Must be set to a value from OrtLoggingLevel.
+   *
+   * \note Logging messages which are less severe than the `logging_severity_level` are not emitted.
+   *
+   * \note Serves as the default logging severity level for session creation and runs.
+   *       Use ::SetSessionLogSeverityLevel() to set a logging severity level for the creation of specific session.
+   *       Use ::RunOptionsSetRunLogSeverityLevel() to set a logging severity level for a specific session run.
+   *
+   * \since Version 1.24.
+   */
+  int32_t logging_severity_level;
+
+  /** \brief The log identifier. Must be set to a valid UTF-8 null-terminated string.
+   *
+   * \note This string identifier is copied by ORT.
+   *
+   * \since Version 1.24.
+   */
+  const char* log_id;
+
+  /** \brief Optional custom logging function. May be set to NULL.
+   *
+   * \note The OrtEnvCreationOptions::custom_logging_param is provided as the first argument to this logging function.
+   *       This allows passing custom state into the logging function.
+   *
+   * \note This function is only called when a message's severity meets or exceeds the set logging severity level.
+   *
+   * \since Version 1.24.
+   */
+  OrtLoggingFunction custom_logging_function;
+
+  /** \brief Optional state to pass as the first argument to OrtEnvCreationOptions::custom_logger_function.
+   *         May be set to NULL.
+   *
+   * \since Version 1.24.
+   */
+  void* custom_logging_param;
+
+  /** \brief Optional threading options for creating an environment with global thread pools shared across sessions.
+   *         May be set to NULL.
+   *
+   * \note The OrtThreadingOptions instance is copied by ORT.
+   *
+   * \note Use OrtApi::CreateThreadingOptions() to create an instance of OrtThreadingOptions.
+   *
+   * \note Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use its own
+   *       thread pools.
+   *
+   * \since Version 1.24.
+   */
+  const OrtThreadingOptions* threading_options;
+
+  /** \brief Optional environment configuration entries represented as string key-value pairs. May be set to NULL.
+   *
+   * \note The OrtKeyValuePairs instance is copied by ORT.
+   *
+   * \note Refer to onnxruntime_env_config_keys.h for common config entry keys and their supported values.
+   *
+   * \note An application provides environment-level configuration options for execution provider libraries by
+   *       using keys with the prefix 'ep_factory.<ep_name>.'. Ex: the key 'ep_factory.my_ep.some_ep_key' represents
+   *       a key named 'some_ep_key' that is meant to be consumed by an execution provider named 'my_ep'. Refer to
+   *       the specific execution provider's documentation for valid keys and values.
+   *
+   * \note An application may separately set session-level configuration options for execution providers via other APIs
+   *       such as SessionOptionsAppendExecutionProvider_V2, which store configuration entries within OrtSessionOptions.
+   *       If an environment-level configuration conflicts with a session-level configuration, then
+   *       precedence is determined by the execution provider library itself.
+   *
+   * \since Version 1.24.
+   */
+  const OrtKeyValuePairs* config_entries;
+
+  //
+  // End of fields available in ORT 1.24
+  //
+
+} OrtEnvCreationOptions;
+
+/** \brief The C API
+ *
+ * All C API functions are defined inside this structure as pointers to functions.
+ * Call OrtApiBase::GetApi to get a pointer to it
+ *
+ * \nosubgrouping
+ */
 struct OrtApi {
   /// \name OrtStatus
   /// @{
@@ -6912,6 +6999,20 @@ struct OrtApi {
   ORT_CLASS_RELEASE(DeviceEpIncompatibilityDetails);
 
   /// @}
+
+  /** \brief Create an OrtEnv instance with the given options.
+   *
+   * \note Invoking this function will return the same instance of the environment as that returned by a previous call
+   *       to another env creation function; all arguments to this function will be ignored.
+   *
+   * \param[in] options The OrtEnvCreationOptions instance that contains creation options.
+   * \param[out] out Output parameter set to the new OrtEnv instance. Must be freed with OrtApi::ReleaseEnv.
+   *
+   * \snippet{doc} snippets.dox OrtStatus Return Value
+   *
+   * \since Version 1.24
+   */
+  ORT_API2_STATUS(CreateEnvWithOptions, _In_ const OrtEnvCreationOptions* options, _Outptr_ OrtEnv** out);
 };
 
 /*

include/onnxruntime/core/session/onnxruntime_cxx_api.h

@@ -1185,6 +1185,9 @@ struct Env : detail::Base<OrtEnv> {
   Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
       OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
 
+  /// \brief Wraps OrtApi::CreateEnvWithOptions
+  explicit Env(const OrtEnvCreationOptions* options);
+
   /// \brief C Interop Helper
   explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
 
@@ -3431,5 +3434,8 @@ struct SharedPrePackedWeightCacheImpl : Ort::detail::Base<T> {
  */
 using UnownedSharedPrePackedWeightCache =
     detail::SharedPrePackedWeightCacheImpl<Ort::detail::Unowned<OrtSharedPrePackedWeightCache>>;
+
+///< Wraps OrtEpApi::GetEnvConfigEntries()
+Ort::KeyValuePairs GetEnvConfigEntries();
 }  // namespace Ort
 #include "onnxruntime_cxx_inline.h"

include/onnxruntime/core/session/onnxruntime_cxx_inline.h

@@ -784,6 +784,15 @@ inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction loggin
   }
 }
 
+inline Env::Env(const OrtEnvCreationOptions* options) {
+  ThrowOnError(GetApi().CreateEnvWithOptions(options, &p_));
+  if (strcmp(options->log_id, "onnxruntime-node") == 0) {
+    ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
+  } else {
+    ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
+  }
+}
+
 inline Env& Env::EnableTelemetryEvents() {
   ThrowOnError(GetApi().EnableTelemetryEvents(p_));
   return *this;
@@ -3779,4 +3788,11 @@ inline Status SharedPrePackedWeightCacheImpl<T>::StoreWeightData(void** buffer_d
                                                                       num_buffers)};
 }
 }  // namespace detail
+
+inline Ort::KeyValuePairs GetEnvConfigEntries() {
+  OrtKeyValuePairs* entries = nullptr;
+  Ort::ThrowOnError(GetEpApi().GetEnvConfigEntries(&entries));
+
+  return Ort::KeyValuePairs{entries};
+}
 }  // namespace Ort

include/onnxruntime/core/session/onnxruntime_env_config_keys.h

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+// This file contains well-known keys for OrtEnv configuration entries, which may be used to configure EPs or
+// other global settings.
+// Refer to OrtEnvCreationOptions::config_entries and OrtApi::CreateEnvWithOptions.
+// This file does NOT specify all available keys as EPs may accept custom entries with the prefix "ep.<ep_name>.".
+
+// Key for a boolean option that, when enabled, allows EP factories to create virtual OrtHardwareDevice
+// instances via OrtEpApi::CreateHardwareDevice().
+//
+// This config entry is automatically set to "1" by ORT if an application registers an EP library with a registration
+// name that ends in the suffix ".virtual". See OrtApi::RegisterExecutionProviderLibrary().
+//
+// Note: A virtual OrtHardwareDevice does not represent actual hardware on the device, and is identified via the
+// metadata entry "is_virtual" with a value of "1".
+//
+// Allowed values:
+//  - "0": Default. Creation of virtual devices is not allowed.
+//         This is the assumed default value if this key is not present in the environment's configuration entries.
+//  - "1": Creation of virtual devices is allowed.
+static const char* const kOrtEnvAllowVirtualDevices = "allow_virtual_devices";

include/onnxruntime/core/session/onnxruntime_ep_c_api.h

@@ -1429,6 +1429,23 @@ struct OrtEpApi {
                   _Outptr_ OrtKernelImpl** kernel_out);
 
   ORT_CLASS_RELEASE(KernelImpl);
+
+  /** \brief Gets a new OrtKeyValuePairs instance containing a copy of all configuration entries set on the environment.
+   *
+   * \note An application provides environment-level configuration options for execution provider libraries by
+   *       using keys with the prefix 'ep_factory.<ep_name>.'. Ex: the key 'ep_factory.my_ep.some_ep_key' represents
+   *       a key named 'some_ep_key' that is meant to be consumed by an execution provider named 'my_ep'. Refer to
+   *       the specific execution provider's documentation for valid keys and values.
+   *
+   * \note Refer to onnxruntime_env_config_keys.h for common configuration entry keys and their supported values.
+   *
+   * \param[out] out Output parameter set to the OrtKeyValuePairs instance containing all configuration entries.
+   *                 Must be released via OrtApi::ReleaseKeyValuePairs.
+   *
+   * \snippet{doc} snippets.dox OrtStatus Return Value
+   * \since Version 1.24
+   */
+  ORT_API2_STATUS(GetEnvConfigEntries, _Outptr_ OrtKeyValuePairs** config_entries);
 };
 
 /**
@@ -1982,35 +1999,6 @@ struct OrtEpFactory {
                   _In_opt_ const OrtKeyValuePairs* stream_options,
                   _Outptr_ OrtSyncStreamImpl** stream);
 
-  /** \brief Set environment options on this EP factory.
-   *
-   * Environment options can be set by ORT after calling the library's 'CreateEpFactories' function to
-   * create EP factories.
-   *
-   * Supported options:
-   *   "allow_virtual_devices": Allows EP factory to specify OrtEpDevice instances that use custom
-   *      virtual OrtHardwareDevices, which can be created via OrtEpApi::CreateHardwareDevice().
-   *
-   *      A virtual OrtHardwareDevice does not represent actual hardware on the device, and is identified
-   *      via the metadata entry "is_virtual" with a value of "1".
-   *      Refer to onnxruntime_ep_device_ep_metadata_keys.h for well-known OrtHardwareDevice metadata keys.
-   *
-   *      Allowed values:
-   *      -# "0": Default. Creation of virtual devices is not allowed.
-   *      -# "1": Creation of virtual devices is allowed.
-   *
-   * \param[in] this_ptr The OrtEpFactory instance.
-   * \param[in] options The configuration options.
-   *
-   * \note Implementation of this function is optional.
-   *       An EP factory should only implement this if it needs to handle any environment options.
-   *
-   * \snippet{doc} snippets.dox OrtStatus Return Value
-   *
-   * \since Version 1.24.
-   */
-  ORT_API2_STATUS(SetEnvironmentOptions, _In_ OrtEpFactory* this_ptr, _In_ const OrtKeyValuePairs* options);
-
   /** \brief Check for known incompatibility reasons between a hardware device and this execution provider.
    *
    * This function allows an execution provider to check if a specific hardware device is compatible

onnxruntime/contrib_ops/cpu/bert/attention_parameters.h

@@ -34,6 +34,7 @@ struct AttentionParameters {
   float mask_filter_value;
   float scale;
   bool use_tf32;
+  bool is_output_bnsh;  // whether the output format is BNSH
   AttentionMaskType mask_type;
   AttentionQkvFormat qkv_format;
 };