Proposed changes to model compilation telemetry (#26804)

### Description
<!-- Describe your changes. -->
This change adjusts the compilation telemetry to span the start and stop
of the operation, with additional recording of various facets of the
compilation session and information about the result.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
These changes enable deeper understanding of compilation usage and
health, which in turn can help prioritize more investments in making
compilation better overall for users of the platform (e.g., more
reliable, performant, better tuned options / defaults, etc.).

---------

Co-authored-by: Aditya Rastogi <adityar@ntdev.microsoft.com>

Author: adrastogi

onnxruntime/core/platform/telemetry.cc

@@ -81,8 +81,34 @@ void Telemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, cons
   ORT_UNUSED_PARAMETER(captureState);
 }
 
-void Telemetry::LogCompileModel(uint32_t session_id) const {
+void Telemetry::LogCompileModelStart(uint32_t session_id,
+                                     const std::string& input_source,
+                                     const std::string& output_target,
+                                     uint32_t flags,
+                                     int graph_optimization_level,
+                                     bool embed_ep_context,
+                                     bool has_external_initializers_file,
+                                     const std::vector<std::string>& execution_provider_ids) const {
   ORT_UNUSED_PARAMETER(session_id);
+  ORT_UNUSED_PARAMETER(input_source);
+  ORT_UNUSED_PARAMETER(output_target);
+  ORT_UNUSED_PARAMETER(flags);
+  ORT_UNUSED_PARAMETER(graph_optimization_level);
+  ORT_UNUSED_PARAMETER(embed_ep_context);
+  ORT_UNUSED_PARAMETER(has_external_initializers_file);
+  ORT_UNUSED_PARAMETER(execution_provider_ids);
+}
+
+void Telemetry::LogCompileModelComplete(uint32_t session_id,
+                                        bool success,
+                                        uint32_t error_code,
+                                        uint32_t error_category,
+                                        const std::string& error_message) const {
+  ORT_UNUSED_PARAMETER(session_id);
+  ORT_UNUSED_PARAMETER(success);
+  ORT_UNUSED_PARAMETER(error_code);
+  ORT_UNUSED_PARAMETER(error_category);
+  ORT_UNUSED_PARAMETER(error_message);
 }
 
 void Telemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,

onnxruntime/core/platform/telemetry.h

@@ -66,7 +66,20 @@ class Telemetry {
                                   const std::string& loadedFrom, const std::vector<std::string>& execution_provider_ids,
                                   bool use_fp16, bool captureState) const;
 
-  virtual void LogCompileModel(uint32_t session_id) const;
+  virtual void LogCompileModelStart(uint32_t session_id,
+                                    const std::string& input_source,
+                                    const std::string& output_target,
+                                    uint32_t flags,
+                                    int graph_optimization_level,
+                                    bool embed_ep_context,
+                                    bool has_external_initializers_file,
+                                    const std::vector<std::string>& execution_provider_ids) const;
+
+  virtual void LogCompileModelComplete(uint32_t session_id,
+                                       bool success,
+                                       uint32_t error_code,
+                                       uint32_t error_category,
+                                       const std::string& error_message) const;
 
   virtual void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,
                                const char* function, uint32_t line) const;

onnxruntime/core/platform/windows/telemetry.cc

@@ -334,18 +334,67 @@ void WindowsTelemetry::LogSessionCreation(uint32_t session_id, int64_t ir_versio
   }
 }
 
-void WindowsTelemetry::LogCompileModel(uint32_t session_id) const {
+void WindowsTelemetry::LogCompileModelStart(uint32_t session_id,
+                                            const std::string& input_source,
+                                            const std::string& output_target,
+                                            uint32_t flags,
+                                            int graph_optimization_level,
+                                            bool embed_ep_context,
+                                            bool has_external_initializers_file,
+                                            const std::vector<std::string>& execution_provider_ids) const {
   if (global_register_count_ == 0 || enabled_ == false)
     return;
 
+  std::string execution_provider_string;
+  bool first = true;
+  for (const auto& ep_id : execution_provider_ids) {
+    if (first) {
+      first = false;
+    } else {
+      execution_provider_string += ',';
+    }
+    execution_provider_string += ep_id;
+  }
+
   TraceLoggingWrite(telemetry_provider_handle,
-                    "CompileModel",
+                    "CompileModelStart",
                     TraceLoggingBool(true, "UTCReplace_AppSessionGuid"),
                     TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage),
                     TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES),
+                    TraceLoggingLevel(WINEVENT_LEVEL_INFO),
                     // Telemetry info
                     TraceLoggingUInt8(0, "schemaVersion"),
-                    TraceLoggingUInt32(session_id, "sessionId"));
+                    TraceLoggingUInt32(session_id, "sessionId"),
+                    TraceLoggingString(input_source.c_str(), "inputSource"),
+                    TraceLoggingString(output_target.c_str(), "outputTarget"),
+                    TraceLoggingUInt32(flags, "flags"),
+                    TraceLoggingInt32(graph_optimization_level, "graphOptimizationLevel"),
+                    TraceLoggingBool(embed_ep_context, "embedEpContext"),
+                    TraceLoggingBool(has_external_initializers_file, "hasExternalInitializersFile"),
+                    TraceLoggingString(execution_provider_string.c_str(), "executionProviderIds"));
+}
+
+void WindowsTelemetry::LogCompileModelComplete(uint32_t session_id,
+                                               bool success,
+                                               uint32_t error_code,
+                                               uint32_t error_category,
+                                               const std::string& error_message) const {
+  if (global_register_count_ == 0 || enabled_ == false)
+    return;
+
+  TraceLoggingWrite(telemetry_provider_handle,
+                    "CompileModelComplete",
+                    TraceLoggingBool(true, "UTCReplace_AppSessionGuid"),
+                    TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage),
+                    TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES),
+                    TraceLoggingLevel(WINEVENT_LEVEL_INFO),
+                    // Telemetry info
+                    TraceLoggingUInt8(0, "schemaVersion"),
+                    TraceLoggingUInt32(session_id, "sessionId"),
+                    TraceLoggingBool(success, "success"),
+                    TraceLoggingUInt32(error_code, "errorCode"),
+                    TraceLoggingUInt32(error_category, "errorCategory"),
+                    TraceLoggingString(error_message.c_str(), "errorMessage"));
 }
 
 void WindowsTelemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,

onnxruntime/core/platform/windows/telemetry.h

@@ -59,7 +59,20 @@ class WindowsTelemetry : public Telemetry {
                           const std::string& loadedFrom, const std::vector<std::string>& execution_provider_ids,
                           bool use_fp16, bool captureState) const override;
 
-  void LogCompileModel(uint32_t session_id) const override;
+  void LogCompileModelStart(uint32_t session_id,
+                            const std::string& input_source,
+                            const std::string& output_target,
+                            uint32_t flags,
+                            int graph_optimization_level,
+                            bool embed_ep_context,
+                            bool has_external_initializers_file,
+                            const std::vector<std::string>& execution_provider_ids) const override;
+
+  void LogCompileModelComplete(uint32_t session_id,
+                               bool success,
+                               uint32_t error_code,
+                               uint32_t error_category,
+                               const std::string& error_message) const override;
 
   void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,
                        const char* function, uint32_t line) const override;

onnxruntime/core/session/model_compilation_options.cc

@@ -293,5 +293,53 @@ Status ModelCompilationOptions::Check() const {
   return Status::OK();
 }
 
+std::string ModelCompilationOptions::GetInputSourceForTelemetry() const {
+  return InputModelComesFromFile() ? "file" : "buffer";
+}
+
+std::string ModelCompilationOptions::GetOutputTargetForTelemetry() const {
+  const epctx::ModelGenOptions& options = session_options_.value.ep_context_gen_options;
+
+  if (options.TryGetOutputModelPath() != nullptr) {
+    return "file";
+  }
+  if (options.TryGetOutputModelBuffer() != nullptr) {
+    return "buffer";
+  }
+  if (options.TryGetOutputModelWriteFunc() != nullptr) {
+    return "callback";
+  }
+
+  // Default: output path will be derived from input path
+  return "file";
+}
+
+uint32_t ModelCompilationOptions::GetFlagsForTelemetry() const {
+  const epctx::ModelGenOptions& options = session_options_.value.ep_context_gen_options;
+  uint32_t flags = OrtCompileApiFlags_NONE;
+
+  if (options.error_if_output_file_exists) {
+    flags |= OrtCompileApiFlags_ERROR_IF_OUTPUT_FILE_EXISTS;
+  }
+  if (options.action_if_no_compiled_nodes == epctx::ModelGenOptions::ActionIfNoCompiledNodes::kReturnError) {
+    flags |= OrtCompileApiFlags_ERROR_IF_NO_NODES_COMPILED;
+  }
+
+  return flags;
+}
+
+int ModelCompilationOptions::GetGraphOptimizationLevelForTelemetry() const {
+  return static_cast<int>(session_options_.value.graph_optimization_level);
+}
+
+bool ModelCompilationOptions::GetEmbedEpContextForTelemetry() const {
+  return session_options_.value.ep_context_gen_options.embed_ep_context_in_model;
+}
+
+bool ModelCompilationOptions::HasExternalInitializersFileForTelemetry() const {
+  return session_options_.value.ep_context_gen_options.TryGetExternalInitializerFileInfo() != nullptr ||
+         session_options_.value.ep_context_gen_options.TryGetInitializerHandler() != nullptr;
+}
+
 }  // namespace onnxruntime
 #endif  // !defined(ORT_MINIMAL_BUILD)

onnxruntime/core/session/model_compilation_options.h

@@ -159,6 +159,44 @@ class ModelCompilationOptions {
   /// <returns>An error status if the compilation options are invalid</returns>
   Status Check() const;
 
+  // Telemetry helper methods
+
+  /// <summary>
+  /// Returns a string describing the input source type: "file" or "buffer".
+  /// </summary>
+  /// <returns>"file" or "buffer"</returns>
+  std::string GetInputSourceForTelemetry() const;
+
+  /// <summary>
+  /// Returns a string describing the output target type: "file", "buffer", or "callback".
+  /// </summary>
+  /// <returns>"file", "buffer", or "callback"</returns>
+  std::string GetOutputTargetForTelemetry() const;
+
+  /// <summary>
+  /// Returns the flags value for telemetry.
+  /// </summary>
+  /// <returns>The flags value</returns>
+  uint32_t GetFlagsForTelemetry() const;
+
+  /// <summary>
+  /// Returns the graph optimization level for telemetry.
+  /// </summary>
+  /// <returns>The graph optimization level as an integer</returns>
+  int GetGraphOptimizationLevelForTelemetry() const;
+
+  /// <summary>
+  /// Returns whether EP context embedding is enabled.
+  /// </summary>
+  /// <returns>True if EP context is embedded in model</returns>
+  bool GetEmbedEpContextForTelemetry() const;
+
+  /// <summary>
+  /// Returns whether external initializers file is configured.
+  /// </summary>
+  /// <returns>True if external initializers file is configured</returns>
+  bool HasExternalInitializersFileForTelemetry() const;
+
  private:
   void ResetInputModelSettings();
 

onnxruntime/core/session/utils.cc

@@ -9,6 +9,8 @@
 #include "core/framework/error_code_helper.h"
 #include "core/framework/execution_provider.h"
 #include "core/framework/provider_options.h"
+#include "core/platform/env.h"
+#include "core/platform/telemetry.h"
 #include "core/session/abi_session_options_impl.h"
 #include "core/session/environment.h"
 #include "core/session/inference_session.h"
@@ -388,25 +390,56 @@ namespace onnxruntime {
 Status CompileModel(const Environment& env, const ModelCompilationOptions& model_compile_options) {
   ORT_RETURN_IF_ERROR(model_compile_options.Check());
 
+  const Telemetry& telemetry_provider = Env::Default().GetTelemetryProvider();
+
   std::unique_ptr<onnxruntime::InferenceSession> session;
   const OrtSessionOptions* session_options = &model_compile_options.GetSessionOptions();
 
+  Status status;
+
   if (model_compile_options.InputModelComesFromFile()) {
     const std::filesystem::path& input_model_path = model_compile_options.GetInputModelPath();
-    ORT_RETURN_IF_ERROR(ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env,
-                                                                         input_model_path.c_str(),
-                                                                         nullptr, 0, session)));
+    status = ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env,
+                                                              input_model_path.c_str(),
+                                                              nullptr, 0, session));
   } else {
-    ORT_RETURN_IF_ERROR(
-        ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env, nullptr,
-                                                         model_compile_options.GetInputModelData(),
-                                                         model_compile_options.GetInputModelDataSize(),
-                                                         session)));
+    status = ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env, nullptr,
+                                                              model_compile_options.GetInputModelData(),
+                                                              model_compile_options.GetInputModelDataSize(),
+                                                              session));
   }
 
-  Env::Default().GetTelemetryProvider().LogCompileModel(session->GetCurrentSessionId());
-  ORT_RETURN_IF_ERROR(ToStatusAndRelease(InitializeSession(session_options, *session)));
-  return Status::OK();
+  if (!status.IsOK()) {
+    telemetry_provider.LogCompileModelComplete(
+        0,  // No session ID available
+        false,
+        static_cast<uint32_t>(status.Code()),
+        static_cast<uint32_t>(status.Category()),
+        status.ErrorMessage());
+    return status;
+  }
+
+  // Log start event now that we have the session ID and can get registered EP types
+  telemetry_provider.LogCompileModelStart(
+      session->GetCurrentSessionId(),
+      model_compile_options.GetInputSourceForTelemetry(),
+      model_compile_options.GetOutputTargetForTelemetry(),
+      model_compile_options.GetFlagsForTelemetry(),
+      model_compile_options.GetGraphOptimizationLevelForTelemetry(),
+      model_compile_options.GetEmbedEpContextForTelemetry(),
+      model_compile_options.HasExternalInitializersFileForTelemetry(),
+      session->GetRegisteredProviderTypes());
+
+  status = ToStatusAndRelease(InitializeSession(session_options, *session));
+
+  telemetry_provider.LogCompileModelComplete(
+      session->GetCurrentSessionId(),
+      status.IsOK(),
+      status.IsOK() ? 0 : static_cast<uint32_t>(status.Code()),
+      status.IsOK() ? 0 : static_cast<uint32_t>(status.Category()),
+      status.IsOK() ? "" : status.ErrorMessage());
+
+  return status;
 }
 
 Status LoadPluginOrProviderBridge(const std::string& registration_name,