NetFX-Stack-Capture- make cont profiler singleton, multi appdomain safe. Attempt unseeded capture as fast path

Author: eftiquar

src/OpenTelemetry.AutoInstrumentation.Native/cor_profiler.cpp

@@ -1232,15 +1232,25 @@ void CorProfiler::ConfigureContinuousProfiler(bool         threadSamplingEnabled
                                               unsigned int maxMemorySamplesPerMinute,
                                               unsigned int selectedThreadsSamplingInterval)
 {
-    Logger::Info("ConfigureContinuousProfiler: thread sampling enabled: ", threadSamplingEnabled,
-                 ", thread sampling interval: ", threadSamplingInterval,
-                 ", allocationSamplingEnabled: ", allocationSamplingEnabled,
-                 ", max memory samples per minute: ", maxMemorySamplesPerMinute,
-                 ", selected threads sampling interval: ", selectedThreadsSamplingInterval);
+    ContinuousProfilerParams params{threadSamplingEnabled, threadSamplingInterval, allocationSamplingEnabled,
+                                    maxMemorySamplesPerMinute, selectedThreadsSamplingInterval};
+    // Guard against multiple initialization: In .NET Framework, this method may be called
+    // once per AppDomain, but the continuous profiler is a process-level singleton.
+    // std::call_once ensures thread-safe one-time initialization across all AppDomains.
+    std::call_once(sampling_init_flag_, [this, &params]() { ConfigureContinuousProfilerInternal(params); });
+}
+
+void CorProfiler::ConfigureContinuousProfilerInternal(const ContinuousProfilerParams& params)
+{
+    Logger::Info("ConfigureContinuousProfiler: thread sampling enabled: ", params.threadSamplingEnabled,
+                 ", thread sampling interval: ", params.threadSamplingInterval,
+                 ", allocationSamplingEnabled: ", params.allocationSamplingEnabled,
+                 ", max memory samples per minute: ", params.maxMemorySamplesPerMinute,
+                 ", selected threads sampling interval: ", params.selectedThreadsSamplingInterval);
 
-    const bool selectiveSamplingConfigured = selectedThreadsSamplingInterval != 0;
+    const bool selectiveSamplingConfigured = params.selectedThreadsSamplingInterval != 0;
 
-    if (!threadSamplingEnabled && !allocationSamplingEnabled && !selectiveSamplingConfigured)
+    if (!params.threadSamplingEnabled && !params.allocationSamplingEnabled && !selectiveSamplingConfigured)
     {
         Logger::Debug("ConfigureContinuousProfiler: no sampling type configured.");
         return;
@@ -1252,26 +1262,26 @@ void CorProfiler::ConfigureContinuousProfiler(bool         threadSamplingEnabled
         return;
     }
 
-    if (threadSamplingEnabled)
+    if (params.threadSamplingEnabled)
     {
-        this->continuousProfiler->threadSamplingInterval = threadSamplingInterval;
+        this->continuousProfiler->threadSamplingInterval = params.threadSamplingInterval;
     }
     if (selectiveSamplingConfigured)
     {
-        this->continuousProfiler->selectedThreadsSamplingInterval = selectedThreadsSamplingInterval;
+        this->continuousProfiler->selectedThreadsSamplingInterval = params.selectedThreadsSamplingInterval;
         this->continuousProfiler->nextOutdatedEntriesScan         = std::chrono::steady_clock::now();
         continuous_profiler::ContinuousProfiler::InitSelectiveSamplingBuffer();
     }
 
-    if (threadSamplingEnabled || selectiveSamplingConfigured)
+    if (params.threadSamplingEnabled || selectiveSamplingConfigured)
     {
         Logger::Info("ContinuousProfiler::StartThreadSampling");
         this->continuousProfiler->StartThreadSampling();
     }
 
-    if (allocationSamplingEnabled)
+    if (params.allocationSamplingEnabled)
     {
-        this->continuousProfiler->StartAllocationSampling(maxMemorySamplesPerMinute);
+        this->continuousProfiler->StartAllocationSampling(params.maxMemorySamplesPerMinute);
     }
 }
 
@@ -1525,9 +1535,23 @@ HRESULT STDMETHODCALLTYPE CorProfiler::JITCompilationStartedOnNetFramework(Funct
     auto valid_loader_callsite = true;
     if (is_desktop_iis)
     {
-        valid_loader_callsite = module_metadata->assemblyName == WStr("System.Web") &&
-                                caller.type.name == WStr("System.Web.Compilation.BuildManager") &&
-                                caller.name == WStr("InvokePreStartInitMethods");
+        // For IIS: prefer injection into BuildManager.InvokePreStartInitMethods (application AppDomain)
+        // but also allow injection into default AppDomain for continuous profiling support
+        bool is_build_manager_injection = module_metadata->assemblyName == WStr("System.Web") &&
+                                          caller.type.name == WStr("System.Web.Compilation.BuildManager") &&
+                                          caller.name == WStr("InvokePreStartInitMethods");
+
+        bool is_default_appdomain = corlib_module_loaded && module_metadata->app_domain_id == corlib_app_domain_id;
+
+        valid_loader_callsite = is_build_manager_injection || is_default_appdomain;
+
+        if (is_default_appdomain && !is_build_manager_injection)
+        {
+            Logger::Info("JITCompilationStarted: Allowing loader injection into default AppDomain for IIS continuous "
+                         "profiling. ",
+                         "AppDomain: ", module_metadata->app_domain_id, ", Assembly: ", module_metadata->assemblyName,
+                         ", Method: ", caller.type.name, ".", caller.name);
+        }
     }
     else if (module_metadata->assemblyName == WStr("System") ||
              module_metadata->assemblyName == WStr("System.Net.Http"))

src/OpenTelemetry.AutoInstrumentation.Native/cor_profiler.h

@@ -33,6 +33,14 @@ class ContinuousProfiler;
 
 namespace trace
 {
+struct ContinuousProfilerParams
+{
+    bool         threadSamplingEnabled;
+    unsigned int threadSamplingInterval;
+    bool         allocationSamplingEnabled;
+    unsigned int maxMemorySamplesPerMinute;
+    unsigned int selectedThreadsSamplingInterval;
+};
 
 class CorProfiler : public CorProfilerBase
 {
@@ -59,6 +67,7 @@ class CorProfiler : public CorProfilerBase
 
     continuous_profiler::ContinuousProfiler* continuousProfiler;
     std::unique_ptr<continuous_profiler::IStackCaptureStrategy> stack_capture_strategy_;
+    std::once_flag sampling_init_flag_;
     HRESULT STDMETHODCALLTYPE ThreadAssignedToOSThread(ThreadID managedThreadId, DWORD osThreadId) override;
 
 
@@ -136,6 +145,7 @@ class CorProfiler : public CorProfilerBase
     //
     void InternalAddInstrumentation(WCHAR* id, CallTargetDefinition* items, int size, bool isDerived);
     bool InitThreadSampler();
+    void ConfigureContinuousProfilerInternal(const ContinuousProfilerParams& params);
 
 public:
     CorProfiler() = default;

src/OpenTelemetry.AutoInstrumentation.Native/profiler_stack_capture.cpp

@@ -10,8 +10,6 @@
 #include <dbghelp.h>
 #include <atlbase.h>
 #include <atlcom.h>
-#include <sstream>
-#include <iomanip>
 #include "logger.h"
 
 #ifndef DECLSPEC_IMPORT
@@ -219,8 +217,9 @@ static HRESULT PrepareContextForSnapshot(ThreadID           managedThreadId,
         }
     }
 
-    // Exhausted all frames without finding managed code, log it
-    trace::Logger::Warn(
+    // Exhausted all frames without finding managed code, let us log at debug level to avoid noise
+    // failure to find managed frame is expected in some scenarios (e.g., native threads)
+    trace::Logger::Debug(
         "[StackCapture] PrepareContextForSnapshot - Unable to locate managed frame in stack walk for ThreadID=",
         managedThreadId);
     return E_FAIL;
@@ -560,49 +559,54 @@ HRESULT StackCaptureEngine::CaptureStackSeeded(ThreadID             managedThrea
                                                HANDLE               threadHandle,
                                                StackCaptureContext* stackCaptureContext)
 {
+    // Try unseeded first - fast path for threads already in managed code
+    stackCaptureContext->clientParams->threadId = managedThreadId;
+    HRESULT hr                                  = profilerApi_->DoStackSnapshot(managedThreadId,
+                                                                                continuous_profiler::IStackCaptureStrategy::StackSnapshotCallbackDefault,
+                                                                                COR_PRF_SNAPSHOT_DEFAULT, stackCaptureContext->clientParams,
+                                                                                nullptr, // No seed
+                                                                                0);
+
+    if (SUCCEEDED(hr))
+    {
+        trace::Logger::Debug("[StackCapture] Unseeded capture succeeded. ThreadID=", managedThreadId);
+        return hr;
+    }
+
+    trace::Logger::Debug("[StackCapture] Unseeded failed (0x", std::hex, hr, "), attempting seeded capture...");
+
+    // Fallback: PrepareContext will check if we're at managed code before walking
     CONTEXT context      = {};
     context.ContextFlags = CONTEXT_FULL;
 
     if (!GetThreadContext(threadHandle, &context))
     {
         return E_FAIL;
     }
-
-    // Prepare context (uses SEH-protected helpers internally, no C++ RAII in that path)
-    HRESULT hr =
-        PrepareContextForSnapshot(managedThreadId, threadHandle, &context, profilerApi_.get(), &stopRequested_);
+    hr = PrepareContextForSnapshot(managedThreadId, threadHandle, &context, profilerApi_.get(), &stopRequested_);
     if (FAILED(hr))
     {
         return hr;
     }
-    stackCaptureContext->clientParams->threadId = managedThreadId;
-    hr                                          = profilerApi_->DoStackSnapshot(managedThreadId,
-                                                                                continuous_profiler::IStackCaptureStrategy::StackSnapshotCallbackDefault,
-                                                                                COR_PRF_SNAPSHOT_DEFAULT, stackCaptureContext->clientParams,
-                                                                                reinterpret_cast<BYTE*>(&context), sizeof(CONTEXT));
+
+    hr = profilerApi_->DoStackSnapshot(managedThreadId,
+                                       continuous_profiler::IStackCaptureStrategy::StackSnapshotCallbackDefault,
+                                       COR_PRF_SNAPSHOT_DEFAULT, stackCaptureContext->clientParams,
+                                       reinterpret_cast<BYTE*>(&context), sizeof(CONTEXT));
 
     if (FAILED(hr))
     {
-        trace::Logger::Error("[StackCapture] CaptureStackSeeded - DoStackSnapshot failed. HRESULT=0x", std::hex, hr,
-                             std::dec, ", ThreadID=", managedThreadId, ", SeededRIP=0x", std::hex, context.Rip,
-                             std::dec);
+        trace::Logger::Debug("[StackCapture] Seeded capture failed. HRESULT=0x", std::hex, hr,
+                             ", ThreadID=", managedThreadId);
     }
     else
     {
-        trace::Logger::Debug("[StackCapture] CaptureStackSeeded - Successfully captured stack. ThreadID=",
-                             managedThreadId);
+        trace::Logger::Debug("[StackCapture] Seeded capture succeeded. ThreadID=", managedThreadId);
     }
 
     return hr;
 }
 
-bool StackCaptureEngine::IsManagedFunction(BYTE* ip) const
-{
-    FunctionID fid = 0;
-    HRESULT    hr  = profilerApi_->GetFunctionFromIP(ip, &fid);
-    return SUCCEEDED(hr) && fid != 0;
-}
-
 bool StackCaptureEngine::WaitForCanaryThread(std::chrono::milliseconds timeout)
 {
     trace::Logger::Debug("[StackCapture] Waiting for canary thread (timeout=", timeout.count(), "ms)");

src/OpenTelemetry.AutoInstrumentation.Native/profiler_stack_capture.h

@@ -5,8 +5,6 @@
 
 #if defined(_WIN32) && defined(_M_AMD64)
 
-#include <windows.h>
-#include <vector>
 #include <functional>
 #include <memory>
 #include <thread>
@@ -147,7 +145,6 @@ namespace ProfilerStackCapture {
 
     private:
         HRESULT CaptureStackSeeded(ThreadID managedThreadId, HANDLE threadHandle, StackCaptureContext* stackCaptureContext);
-        bool IsManagedFunction(BYTE* instructionPointer) const;
         bool SafetyProbe();
 
         std::unique_ptr<IProfilerApi> profilerApi_;

src/OpenTelemetry.AutoInstrumentation/Instrumentation.cs

@@ -26,7 +26,6 @@ internal static class Instrumentation
     private static readonly Lazy<LoggerProvider?> LoggerProviderFactory = new(InitializeLoggerProvider, true);
 
     private static int _initialized;
-    private static int _continuousProfilingInitialized; // Process-wide guard for continuous profiling
     private static int _isExiting;
     private static SdkSelfDiagnosticsEventListener? _sdkEventListener;
 
@@ -113,8 +112,6 @@ public static void Initialize()
 
             if (profilerEnabled)
             {
-                // Continuous profiling is process-wide and must be initialized only once
-                // from the default AppDomain to avoid multiple canary threads
                 TryInitializeContinuousProfiling();
             }
             else
@@ -219,37 +216,13 @@ public static void Initialize()
 
     private static void TryInitializeContinuousProfiling()
     {
-#if NETFRAMEWORK
-        // Check AppDomain FIRST, before touching the flag
-        if (!AppDomain.CurrentDomain.IsDefaultAppDomain())
-        {
-            Logger.Information(
-                "Skipping continuous profiling initialization from non-default AppDomain. " +
-                "AppDomain: {0}",
-                AppDomain.CurrentDomain.FriendlyName);
-            return;  // No flag manipulation needed
-        }
-
-        Logger.Information(
-            "Initializing continuous profiling from default AppDomain: {0}",
-            AppDomain.CurrentDomain.FriendlyName);
-#endif
-
-        // NOW attempt to claim initialization rights
-        if (Interlocked.CompareExchange(ref _continuousProfilingInitialized, 1, 0) != 0)
-        {
-            Logger.Debug("Continuous profiling already initialized in this process. Skipping.");
-            return;
-        }
-
         try
         {
             InitializeSampling();
         }
         catch (Exception ex)
         {
             Logger.Error(ex, "Failed to initialize continuous profiling.");
-            Interlocked.Exchange(ref _continuousProfilingInitialized, 0);
         }
     }
 
@@ -312,17 +285,21 @@ private static void InitializeSampling()
 
         NativeMethods.ConfigureNativeContinuousProfiler(threadSamplingEnabled, threadSamplingInterval, allocationSamplingEnabled, maxMemorySamplesPerMinute, selectiveSamplingInterval);
 #if NETFRAMEWORK
-        // On .NET Framework, we need a dedicated canary thread for seeded stack walking
-        _canaryThreadManager = new CanaryThreadManager();
-        if (!_canaryThreadManager.Start(TimeSpan.FromSeconds(5)))
+        if (AppDomain.CurrentDomain.IsDefaultAppDomain())
         {
-            Logger.Error("Failed to start canary thread. Continuous profiling will not be enabled.");
-            _canaryThreadManager.Dispose();
-            _canaryThreadManager = null;
-            return;
-        }
+            // On .NET Framework, we need a dedicated canary thread for seeded stack walking
+            // host the canary thread manager in the default app domain only to avoid multiple canary threads
+            _canaryThreadManager = new CanaryThreadManager();
+            if (!_canaryThreadManager.Start(TimeSpan.FromSeconds(5)))
+            {
+                Logger.Error("Failed to start canary thread. Continuous profiling will not be enabled.");
+                _canaryThreadManager.Dispose();
+                _canaryThreadManager = null;
+                return;
+            }
 
-        Logger.Information("Canary thread started successfully for .NET Framework profiling.");
+            Logger.Information("Canary thread started successfully for .NET Framework profiling.");
+        }
 #endif
         _sampleExporter = _sampleExporterBuilder?.Build();
     }

test/IntegrationTests/ContinuousProfilerContextTrackingTests.cs

@@ -52,9 +52,16 @@ private bool AssertAllProfiles(ICollection<ExportProfilesServiceRequest> profile
                 managedThreadsWithTraceContext.Add(threadId!);
             }
         }
-
+#if NET
         Assert.True(managedThreadsWithTraceContext.Count > 1, "at least 2 distinct threads should have trace context associated.");
         Assert.True(totalSamplesWithTraceContextCount >= 3, "there should be sample with trace context in most of the batches.");
+#else
+        // for net fx, thread pool threads do not have names, hence it is not possible to uniquely
+        // identify distinct threads. We will restrict our test to ensure we have at least the main thread is reporting context
+        Assert.True(managedThreadsWithTraceContext.Count > 0, "at least one thread should have trace context associated.");
+        Assert.True(totalSamplesWithTraceContextCount > 0, "there should be at least one sample with trace context .");
+#endif
+
         return true;
     }