fix(profiling): add CPU timer health disable

Author: taegyunkim

ddtrace/internal/datadog/profiling/docs/timer_create.md

@@ -99,6 +99,9 @@ Current mitigations:
   blocks off-CPU where the per-thread CPU timer does not advance. They pass
   (no `EINTR`), which is the recorded result for the current Linux targets.
 - Run native-heavy ecosystem workloads before enabling this more broadly.
+- Permanently disable CPU-timer mode if health windows show a sustained high
+  rate of capture failures or ring overflows. This keeps pathological
+  environments from emitting failure-only timer traffic indefinitely.
 
 Required hardening before considering broader rollout:
 
@@ -120,7 +123,9 @@ Required hardening before considering broader rollout:
    thread CPU clock, it already conserves the CPU consumed during coalesced
    expirations, so overruns are not used to weight samples (doing so would
    double-count CPU). A rising overrun rate indicates the handler or interval is
-   dropping sampling resolution and is a cue to widen the interval.
+   dropping sampling resolution and is a cue to widen the interval. Sustained
+   high capture-failure or ring-overflow windows disable CPU-timer mode and are
+   reported with `health_disable_count`.
 4. Keep the 2 ms minimum interval until the native syscall and ecosystem tests
    show that a lower interval is safe. Do not expose the interval as public API
    while this remains experimental.

ddtrace/internal/datadog/profiling/stack/include/cpu_timer.hpp

@@ -31,7 +31,7 @@ struct DebugStats
     uint64_t timer_syscall_failures = 0;
     uint64_t accepted_signal_oob_tid_count = 0;
     uint64_t handler_hijack_disable_count = 0;
-    uint64_t fast_copy_conflict_count = 0;
+    uint64_t health_disable_count = 0;
     uint64_t dropped_count = 0;
     uint64_t dropped_cpu_ns = 0;
     uint64_t capture_failed_count = 0;

ddtrace/internal/datadog/profiling/stack/src/cpu_timer.cpp

@@ -75,6 +75,9 @@ constexpr uint32_t kDefaultRingCapacity = 64;
 constexpr size_t kMinAltStackSize = 128 * 1024;
 constexpr size_t kAltStackSigstkszMultiplier = 4;
 constexpr size_t kMaxSlotBytes = 256 * 1024 * 1024;
+constexpr uint64_t kHealthWindowEvents = 128;
+constexpr uint64_t kHealthBadRatioPercent = 90;
+constexpr uint32_t kHealthBadWindowsLimit = 3;
 
 int g_cookie;
 
@@ -194,6 +197,7 @@ struct CaptureState
     bool timer_deleted = false;
     bool retired = false;
 
+    std::atomic<uint64_t> published_count{ 0 };
     std::atomic<uint64_t> dropped_count{ 0 };
     std::atomic<uint64_t> dropped_cpu_ns{ 0 };
     std::atomic<uint64_t> capture_failed_count{ 0 };
@@ -202,6 +206,14 @@ struct CaptureState
     std::atomic<uint64_t> timer_overrun_total{ 0 };
     std::atomic<uint64_t> coalesced_signal_count{ 0 };
 
+    uint64_t health_seen_published_count = 0;
+    uint64_t health_seen_dropped_count = 0;
+    uint64_t health_seen_capture_failed_count = 0;
+    uint64_t health_window_published_count = 0;
+    uint64_t health_window_dropped_count = 0;
+    uint64_t health_window_capture_failed_count = 0;
+    uint32_t health_bad_window_count = 0;
+
     CaptureState(uint64_t thread_id, uint64_t tid, const char* thread_name)
       : python_thread_id(thread_id)
       , native_tid(tid)
@@ -248,7 +260,7 @@ struct EngineState
     std::atomic<uint64_t> timer_syscall_failures{ 0 };
     std::atomic<uint64_t> accepted_signal_oob_tid_count{ 0 };
     std::atomic<uint64_t> handler_hijack_disable_count{ 0 };
-    std::atomic<uint64_t> fast_copy_conflict_count{ 0 };
+    std::atomic<uint64_t> health_disable_count{ 0 };
     std::atomic<uint64_t> stage2_invalid_frame_count{ 0 };
 };
 
@@ -703,6 +715,49 @@ drain_state(EchionSampler& echion, CaptureState& state)
     }
 }
 
+uint64_t
+counter_delta(uint64_t current, uint64_t& seen)
+{
+    const uint64_t delta = current >= seen ? current - seen : 0;
+    seen = current;
+    return delta;
+}
+
+bool
+capture_state_health_should_disable(CaptureState& state)
+{
+    // AIDEV-NOTE: Health is evaluated on the sampler thread, never in the SIGPROF handler.
+    // The handler only increments lock-free counters. Timer deletion and state retirement stay
+    // on the normal control path so pathological capture failures cannot turn into signal-time
+    // locking or allocation.
+    const uint64_t published = state.published_count.load(std::memory_order_relaxed);
+    const uint64_t dropped = state.dropped_count.load(std::memory_order_relaxed);
+    const uint64_t failed = state.capture_failed_count.load(std::memory_order_relaxed);
+
+    state.health_window_published_count += counter_delta(published, state.health_seen_published_count);
+    state.health_window_dropped_count += counter_delta(dropped, state.health_seen_dropped_count);
+    state.health_window_capture_failed_count += counter_delta(failed, state.health_seen_capture_failed_count);
+
+    const uint64_t bad_events = state.health_window_dropped_count + state.health_window_capture_failed_count;
+    const uint64_t total_events = bad_events + state.health_window_published_count;
+    if (total_events < kHealthWindowEvents) {
+        return false;
+    }
+
+    const bool bad_window = bad_events * 100 >= total_events * kHealthBadRatioPercent;
+    state.health_window_published_count = 0;
+    state.health_window_dropped_count = 0;
+    state.health_window_capture_failed_count = 0;
+
+    if (!bad_window) {
+        state.health_bad_window_count = 0;
+        return false;
+    }
+
+    state.health_bad_window_count++;
+    return state.health_bad_window_count >= kHealthBadWindowsLimit;
+}
+
 void
 disable_all_timers_locked()
 {
@@ -729,6 +784,19 @@ disable_all_timers_for_hijack()
     disable_all_timers_locked();
 }
 
+void
+disable_all_timers_for_health_locked()
+{
+    if (!g_state.active.load(std::memory_order_acquire)) {
+        return;
+    }
+    g_state.health_disable_count.fetch_add(1, std::memory_order_relaxed);
+    g_state.active.store(false, std::memory_order_release);
+    g_state.replacing_wall_cpu.store(false, std::memory_order_release);
+    g_state.permanently_disabled.store(true, std::memory_order_release);
+    disable_all_timers_locked();
+}
+
 void
 disable_all_timers_for_blocked_signal()
 {
@@ -915,6 +983,7 @@ cpu_timer_signal_handler(int signo, siginfo_t* si, void* ucontext)
     }
 
     state->ring.publish_for_producer();
+    state->published_count.fetch_add(1, std::memory_order_relaxed);
     g_state.handler_active[tid].store(false, std::memory_order_seq_cst);
     errno = saved_errno;
 }
@@ -1258,6 +1327,22 @@ Engine::drain(EchionSampler& echion)
     for (auto& state : ready_to_free) {
         drain_state(echion, *state);
     }
+
+    {
+        std::lock_guard<std::mutex> lock(g_state.registry_lock);
+        bool should_disable = false;
+        if (g_state.active.load(std::memory_order_acquire)) {
+            for (auto& item : g_state.live_by_thread_id) {
+                if (capture_state_health_should_disable(*item.second)) {
+                    should_disable = true;
+                    break;
+                }
+            }
+        }
+        if (should_disable) {
+            disable_all_timers_for_health_locked();
+        }
+    }
 #else
     (void)echion;
 #endif
@@ -1313,7 +1398,7 @@ Engine::debug_stats() const
     stats.timer_syscall_failures = g_state.timer_syscall_failures.load(std::memory_order_relaxed);
     stats.accepted_signal_oob_tid_count = g_state.accepted_signal_oob_tid_count.load(std::memory_order_relaxed);
     stats.handler_hijack_disable_count = g_state.handler_hijack_disable_count.load(std::memory_order_relaxed);
-    stats.fast_copy_conflict_count = g_state.fast_copy_conflict_count.load(std::memory_order_relaxed);
+    stats.health_disable_count = g_state.health_disable_count.load(std::memory_order_relaxed);
     stats.stage2_invalid_frame_count = g_state.stage2_invalid_frame_count.load(std::memory_order_relaxed);
 
     std::lock_guard<std::mutex> lock(g_state.registry_lock);

ddtrace/internal/datadog/profiling/stack/src/stack.cpp

@@ -390,7 +390,7 @@ stack_cpu_timer_debug_stats(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)
         set_debug_stat(dict, "timer_syscall_failures", stats.timer_syscall_failures) != 0 ||
         set_debug_stat(dict, "accepted_signal_oob_tid_count", stats.accepted_signal_oob_tid_count) != 0 ||
         set_debug_stat(dict, "handler_hijack_disable_count", stats.handler_hijack_disable_count) != 0 ||
-        set_debug_stat(dict, "fast_copy_conflict_count", stats.fast_copy_conflict_count) != 0 ||
+        set_debug_stat(dict, "health_disable_count", stats.health_disable_count) != 0 ||
         set_debug_stat(dict, "dropped_count", stats.dropped_count) != 0 ||
         set_debug_stat(dict, "dropped_cpu_ns", stats.dropped_cpu_ns) != 0 ||
         set_debug_stat(dict, "capture_failed_count", stats.capture_failed_count) != 0 ||