Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
72 commits
Select commit Hold shift + click to select a range
55a7cba
chore(profiling): track signal altstack ownership
taegyunkim Jul 13, 2026
28117d5
Merge branch 'main' into taegyunkim/prof-14213-safe-copy-altstack-own…
taegyunkim Jul 13, 2026
a84b45a
docs(profiling): document signal alt stack ownership contract
taegyunkim Jul 13, 2026
44f68ad
test(profiling): cover signal alt stack ownership in teardown
taegyunkim Jul 13, 2026
e7809e7
docs(profiling): add release note for alt stack ownership fix
taegyunkim Jul 13, 2026
da11ce4
fix(profiling): don't disable an alt stack replaced after we installe…
taegyunkim Jul 13, 2026
a536d62
docs(profiling): add release note for replaced alt stack teardown fix
taegyunkim Jul 13, 2026
5984ad6
Merge branch 'main' into taegyunkim/prof-14213-safe-copy-altstack-tea…
taegyunkim Jul 14, 2026
17efe04
feat(profiling): add Linux CPU timer stack profiler
taegyunkim Jun 24, 2026
c98b4ef
fix(profiling): avoid reserved PyThreadState forward declaration
taegyunkim Jun 24, 2026
8cc3eb6
fix(profiling): keep experimental CPU timer flags local
taegyunkim Jun 24, 2026
9603962
fix(profiling): stitch async stacks for CPU timer samples
taegyunkim Jun 24, 2026
d9b5246
fix(profiling): arm CPU timers for discovered Python threads
taegyunkim Jun 24, 2026
91af746
fix(profiling): guard CPU timer thread discovery by Python version
taegyunkim Jun 24, 2026
5c91970
docs(profiling): add CPU timer profiler design notes
taegyunkim Jun 25, 2026
3bf5f31
test(profiling): add CPU timer native syscall hazard reproducers
taegyunkim Jun 26, 2026
a8f2248
docs(profiling): document CPU timer syscall hazards
taegyunkim Jun 26, 2026
cfe1a33
format
taegyunkim Jun 29, 2026
f5c4749
test(profiling): add read/readv and nanosleep CPU timer syscall hazar…
taegyunkim Jul 9, 2026
32f2ee9
feat(profiling): track CPU timer si_overrun for sampling-quality diag…
taegyunkim Jul 9, 2026
16fecdd
test(profiling): tolerate benign thread-churn races in CPU timer alts…
taegyunkim Jul 9, 2026
7ef12c1
fix(profiling): harden CPU timer signal handling
taegyunkim Jul 9, 2026
0e415b0
fix(profiling): add CPU timer health disable
taegyunkim Jul 9, 2026
0920ef5
chore(profiling): move native coverage tooling changes out
taegyunkim Jul 14, 2026
26fcf26
fix(profiling): avoid stale pthread lookup during thread discovery
taegyunkim Jul 14, 2026
ce3e77e
chore(profiling): harden CPU timer sample ring
taegyunkim Jul 14, 2026
c59c08d
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Jul 14, 2026
1d47adf
fix(profiling): require lock-free fault recovery callback
taegyunkim Jul 14, 2026
b852362
fix(profiling): preserve adaptive sampling when CPU timer is inactive
taegyunkim Jul 14, 2026
480b803
fix(profiling): decouple CPU timer drain cadence
taegyunkim Jul 15, 2026
01768b0
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Jul 15, 2026
f3de4d6
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Jul 21, 2026
ce5cd2f
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Jul 21, 2026
ab835b2
perf(profiling): allocate CPU timer TID table lazily
taegyunkim Jul 23, 2026
2a4f468
docs(profiling): clarify CPU timer version support
taegyunkim Jul 23, 2026
d537b45
feat(profiling): support CPU timer on Python 3.12
taegyunkim Jul 23, 2026
0fe59fd
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Jul 23, 2026
b818766
fix(profiling): resolve CPU timer clang-tidy findings
taegyunkim Jul 23, 2026
742b174
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Jul 23, 2026
53a6d0b
fix(profiling): keep CPU timer mode immutable
taegyunkim Jul 24, 2026
3c418d6
docs(profiling): document timer CPU profiling status
taegyunkim Jul 24, 2026
8198c85
docs(profiling): move timer CPU profile deck
taegyunkim Jul 24, 2026
df6f104
refactor(profiling): use fixed CPU sample ring storage
taegyunkim Jul 27, 2026
751cc43
fix(profiling): preserve CPU timer task identity
taegyunkim Jul 27, 2026
c258f6e
test(profiling): run suite with CPU timer sampler
taegyunkim Jul 27, 2026
d0dccbb
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Jul 27, 2026
8c8e5a2
chore(profiling): address CPU timer review feedback
taegyunkim Jul 27, 2026
7a8b8ef
docs(profiling): design timer sample attribution
taegyunkim Jul 27, 2026
a98a72b
test(profiling): cover CPU timer task ancestry
taegyunkim Jul 28, 2026
622aa53
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Jul 29, 2026
58b2f77
chore(profiling): move uWSGI assertion to focused PR
taegyunkim Jul 30, 2026
e6798b2
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Jul 30, 2026
3ed3765
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Aug 5, 2026
8380345
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Aug 7, 2026
64c5dba
docs(profiling): move CPU timer notes to experimental
taegyunkim Aug 7, 2026
8cf54e9
fix(profiling): restore CPU sample ownership
taegyunkim Aug 7, 2026
52939a8
refactor(profiling): simplify CPU timer internals
taegyunkim Aug 7, 2026
1847103
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Aug 10, 2026
db49357
perf(profiling): reduce CPU timer asyncio drain overhead
taegyunkim Aug 11, 2026
c0fef3f
fix(profiling): avoid strict-aliasing task inspection
taegyunkim Aug 12, 2026
113249d
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Aug 12, 2026
0534fad
fix(profiling): attribute CPU timer samples to greenlets
taegyunkim Aug 12, 2026
639fd1d
Merge branch 'main' into taegyunkim/prof-14213-timer-create
taegyunkim Aug 12, 2026
fd97617
refactor(profiling): simplify CPU timer internals
taegyunkim Aug 12, 2026
cf21acc
refactor(profiling): rename greenlet switch recorder
taegyunkim Aug 12, 2026
8dcec29
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Aug 13, 2026
c14d1fa
ci(profiling): extend profile test timeout
taegyunkim Aug 13, 2026
dc676cf
Revert "ci(profiling): extend profile test timeout"
taegyunkim Aug 13, 2026
7e736ea
ci(profiling): split CPU timer profile jobs
taegyunkim Aug 13, 2026
69f138f
Revert "ci(profiling): split CPU timer profile jobs"
taegyunkim Aug 13, 2026
9155126
Merge remote-tracking branch 'origin/main' into taegyunkim/prof-14213…
taegyunkim Aug 13, 2026
5c01fee
refactor(profiling): visit task identities during discovery
taegyunkim Aug 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Comment thread
taegyunkim marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ class ProfilerStats
// Total CPU time (in microseconds) spent by the sampler thread capturing samples
size_t sample_capture_cpu_time_us = 0;

// CPU time spent collecting wall samples and draining/rendering CPU timer samples.
// These components are subsets of sample_capture_cpu_time_us.
size_t wall_sample_capture_cpu_time_us = 0;
size_t cpu_timer_drain_cpu_time_us = 0;

public:
ProfilerStats() = default;
~ProfilerStats() = default;
Expand Down Expand Up @@ -108,6 +113,9 @@ class ProfilerStats
void add_sample_capture_cpu_time_us(size_t cpu_time_us);
size_t get_sample_capture_cpu_time_us() const;

void add_wall_sample_capture_cpu_time_us(size_t cpu_time_us);
void add_cpu_timer_drain_cpu_time_us(size_t cpu_time_us);

// Returns a JSON string containing relevant Profiler Stats to be included
// in the libdatadog payload.
std::string get_internal_metadata_json();
Expand Down

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I gave this a spin on the signal-bundler service over the weekend, here's a comparison (note: the before/after doesn't really make sense in terms of CPU time as it's comparing weekday data to weekend data, and this service has less stuff to process on weekends).

Image

A few things I'm noting

  • Incomplete stacks (top-right corner of the before flame graph) are completely gone, that's great 🎉
  • We don't see any Truncated Frames stacks -- I'm not sure why that is (I guess maybe the range I picked didn't have the service bootstrap, so no imports to see...)
  • For some reason, Scheduler.periodic is only visible on the flame graph from the timer-based profiler. I'm really not sure why since as far as I can tell, it doesn't change that at all. Maybe it's a ddtrace change that the previously-deployed version didn't have 🤔
  • A whole part of the flame graph is missing. I think that's just a matter of a certain chunk of logic not having been called, as I don't see why everything else except for that would work.
Image

So overall that's absolutely awesome! 🤩

What I'm a bit surprised by, however, is that the flame graph looks... just about the same as before otherwise. This service in staging is low-CPU (~2-3s CPU time per minute, according to profiles) so I would have expected the timer-based profiler to make a huge difference on correctness.
Maybe our recent changes to adaptive sampling (which we rolled out to staging/prod this week) are playing a big role in making the non-timer-based profiler almost correct? Curious if anyone has insights into this...

Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ Datadog::ProfilerStats::reset_state()
asyncio_task_count = std::nullopt;
greenlet_count = std::nullopt;
sample_capture_cpu_time_us = 0;
wall_sample_capture_cpu_time_us = 0;
cpu_timer_drain_cpu_time_us = 0;
Comment thread
taegyunkim marked this conversation as resolved.
// fast_copy_memory_* static fields are intentionally not reset (see setters).
}

Expand Down Expand Up @@ -232,6 +234,18 @@ Datadog::ProfilerStats::get_sample_capture_cpu_time_us() const
return sample_capture_cpu_time_us;
}

void
Datadog::ProfilerStats::add_wall_sample_capture_cpu_time_us(size_t cpu_time_us)
{
wall_sample_capture_cpu_time_us += cpu_time_us;
}

void
Datadog::ProfilerStats::add_cpu_timer_drain_cpu_time_us(size_t cpu_time_us)
{
cpu_timer_drain_cpu_time_us += cpu_time_us;
}

std::string
Datadog::ProfilerStats::get_internal_metadata_json()
{
Expand Down Expand Up @@ -308,6 +322,14 @@ Datadog::ProfilerStats::get_internal_metadata_json()

internal_metadata_json += R"("sample_capture_cpu_time_us": )";
append_to_string(internal_metadata_json, sample_capture_cpu_time_us);
internal_metadata_json += ",";

internal_metadata_json += R"("wall_sample_capture_cpu_time_us": )";
append_to_string(internal_metadata_json, wall_sample_capture_cpu_time_us);
internal_metadata_json += ",";

internal_metadata_json += R"("cpu_timer_drain_cpu_time_us": )";
append_to_string(internal_metadata_json, cpu_timer_drain_cpu_time_us);

internal_metadata_json += "}";

Expand Down
1 change: 1 addition & 0 deletions ddtrace/internal/datadog/profiling/stack/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ endif()
# Specify the target C-extension that we want to build
add_library(
${EXTENSION_NAME} SHARED
src/cpu_timer.cpp
src/echion/danger.cc
src/echion/frame.cc
src/echion/greenlets.cc
Expand Down
2 changes: 2 additions & 0 deletions ddtrace/internal/datadog/profiling/stack/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

from . import _stack
from ._stack import * # noqa: F403, F401 # type: ignore[assignment]
from ._stack import _cpu_timer_debug_set_fault_injection # noqa: F401
from ._stack import _cpu_timer_debug_stats # noqa: F401

is_available = True

Expand Down
12 changes: 12 additions & 0 deletions ddtrace/internal/datadog/profiling/stack/__init__.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ from ddtrace._trace import context
from ddtrace._trace import span as ddspan

# Core stack v2 functions

def start(min_interval: float = ...) -> bool: ...
def stop() -> None: ...
def is_origin_task_linking_enabled() -> bool: ...
Expand All @@ -18,13 +19,17 @@ def link_origin_task(task_id: int, task_name: str) -> None: ...
def unlink_origin_task() -> None: ...

# Sampling configuration

def set_adaptive_sampling(do_adaptive_sampling: bool = False) -> None: ...
def set_target_overhead(target_overhead: float) -> None: ...
def set_max_sampling_period(max_interval_us: int) -> None: ...
def set_adaptive_sampling_baseline(baseline_core_pct: float) -> None: ...
def set_p_stable_window_s(p_stable_window_s: int) -> None: ...
def set_p_stable_percentile(p_stable_percentile: float) -> None: ...
def set_max_threads(max_threads: int) -> None: ...
def set_cpu_timer(enabled: bool, interval_ms: int) -> None: ...
def _cpu_timer_debug_stats() -> dict[str, bool | int]: ...
def _cpu_timer_debug_set_fault_injection(enabled: bool) -> None: ...
def set_max_tasks(max_tasks: int) -> None: ...
def set_uvloop_mode(thread_id: int, uvloop_mode: bool) -> None:
"""Enable uvloop-specific stack unwinding in the native profiler for a specific thread.
Expand All @@ -41,6 +46,7 @@ def set_uvloop_mode(thread_id: int, uvloop_mode: bool) -> None:
def set_interval(new_interval: float) -> None: ...

# Memory copy strategy

def set_fast_copy(enabled: bool) -> None: ...
def is_safe_copy_failed() -> bool: ...
def fast_copy_memory_active() -> bool: ... # test introspection: is safe_memcpy active?
Expand All @@ -66,6 +72,7 @@ def segv_handler_installed() -> bool:
...

# Pause/resume sampling

def pause_sampling() -> bool | None:
"""Pause the sampling thread and wait for any in-flight sample to complete.

Expand All @@ -80,14 +87,17 @@ def pause_sampling() -> bool | None:
def resume_sampling() -> None: ...

# span <-> profile association

def link_span(span: Optional[Union[context.Context, ddspan.Span]]) -> None: ...
def _unlink_finished_span(span: ddspan.Span) -> None: ...

# Thread management

def register_thread(python_thread_id: int, native_id: int, name: str) -> None: ...
def unregister_thread(python_thread_id: int) -> None: ...

# Asyncio support

def track_asyncio_loop(thread_id: int, loop: Optional[asyncio.AbstractEventLoop]) -> None: ...
def link_tasks(parent: asyncio.Task, child: asyncio.Future) -> None: ...
def weak_link_tasks(parent: asyncio.Task, child: asyncio.Future) -> None: ...
Expand All @@ -97,6 +107,7 @@ def init_asyncio(
) -> None: ...

# Greenlet support

def track_greenlet(greenlet_id: int, name: str, frame: Union[FrameType, bool, None]) -> None: ...
def untrack_greenlet(greenlet_id: int) -> None: ...
def link_greenlets(greenlet_id: int, parent_id: int) -> None: ...
Expand All @@ -109,5 +120,6 @@ def record_greenlet_switch(
) -> None: ...

# Module attributes

is_available: bool
failure_msg: str
11 changes: 11 additions & 0 deletions ddtrace/internal/datadog/profiling/stack/_stack.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,26 @@ from typing import Sequence
from typing import Union

# Core stack v2 functions

def start(min_interval: float = ...) -> bool: ...
def stop() -> None: ...
def is_origin_task_linking_enabled() -> bool: ...

# Sampling configuration

def set_adaptive_sampling(do_adaptive_sampling: bool = False) -> None: ...
def set_target_overhead(target_overhead: float) -> None: ...
def set_max_sampling_period(max_interval_us: int) -> None: ...
def set_max_threads(max_threads: int) -> None: ...
def set_cpu_timer(enabled: bool, interval_ms: int) -> None: ...
def _cpu_timer_debug_stats() -> dict[str, bool | int]: ...
def _cpu_timer_debug_set_fault_injection(enabled: bool) -> None: ...
def set_max_tasks(max_tasks: int) -> None: ...
def set_uvloop_mode(thread_id: int, uvloop_mode: bool) -> None: ...
def set_interval(new_interval: float) -> None: ...

# Memory copy strategy

def set_fast_copy(enabled: bool) -> None: ...
def is_safe_copy_failed() -> bool: ...
def fast_copy_memory_active() -> bool: ... # test introspection: is safe_memcpy active?
Expand All @@ -35,6 +41,7 @@ def segv_handler_installed() -> bool:
...

# span <-> profile association

def link_span(
span_id: int,
local_root_span_id: int,
Expand All @@ -52,10 +59,12 @@ def link_origin_task(
def unlink_origin_task() -> None: ...

# Thread management

def register_thread(python_thread_id: int, native_id: int, name: str) -> None: ...
def unregister_thread(python_thread_id: int) -> None: ...

# Asyncio support

def track_asyncio_loop(thread_id: int, loop: Optional[asyncio.AbstractEventLoop]) -> None: ...
def link_tasks(parent: asyncio.Task[Any], child: asyncio.Future[Any]) -> None: ...
def weak_link_tasks(parent: asyncio.Task[Any], child: asyncio.Future[Any]) -> None: ...
Expand All @@ -65,6 +74,7 @@ def init_asyncio(
) -> None: ...

# Greenlet support

def track_greenlet(greenlet_id: int, name: str, frame: Union[FrameType, bool, None]) -> None: ...
def untrack_greenlet(greenlet_id: int) -> None: ...
def link_greenlets(greenlet_id: int, parent_id: int) -> None: ...
Expand All @@ -77,5 +87,6 @@ def record_greenlet_switch(
) -> None: ...

# Native call monitoring (sys.monitoring bridge)

def start_native_monitoring() -> None: ...
def stop_native_monitoring() -> None: ...
14 changes: 14 additions & 0 deletions ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,20 @@
#include <sys/types.h>
#endif

using ProfilingFaultRecover = bool (*)(int signo, siginfo_t* si, void* ucontext);

int
init_profiling_fault_handler();

bool
profiling_fault_handler_still_installed();

void
register_profiling_fault_recover(ProfilingFaultRecover recover);

void
unregister_profiling_fault_recover(ProfilingFaultRecover recover);

int
init_segv_catcher();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
#include <mach/mach.h>
#endif

#include "cpu_sample_ring.hpp"

#include <echion/errors.h>
#include <echion/greenlets.h>
#include <echion/interp.h>
Expand Down Expand Up @@ -60,7 +62,12 @@ class ThreadInfo

[[nodiscard]] Result<void> update_cpu_time();

[[nodiscard]] Result<void> sample(EchionSampler&, PyThreadState*, microsecond_t);
[[nodiscard]] Result<void> sample(EchionSampler&, PyThreadState*, microsecond_t, bool include_cpu_time = true);
void sample_cpu_timer(EchionSampler&,
PyThreadState*,
FrameStack&&,
microsecond_t,
const Datadog::CpuTimer::RawSample&);
void unwind(EchionSampler&, PyThreadState*, microsecond_t wall_time_us);

// ------------------------------------------------------------------------
Expand Down Expand Up @@ -114,20 +121,21 @@ class ThreadInfo
};

private:
using TaskAddressCallback = std::function<void(TaskObj*)>;

void reset_cycle_state() noexcept;
void render_unwound_stacks(EchionSampler&);
[[nodiscard]] Result<void> unwind_tasks(EchionSampler&, PyThreadState*, microsecond_t wall_time_us);
void unwind_greenlets(EchionSampler&, PyThreadState*, unsigned long, microsecond_t wall_time_us);
[[nodiscard]] Result<std::vector<TaskInfo::Ptr>> get_all_tasks(EchionSampler&, PyThreadState* tstate);
[[nodiscard]] Result<void> for_each_task_address(EchionSampler&,
PyThreadState* tstate,
const TaskAddressCallback& callback);
#if PY_VERSION_HEX >= 0x030e0000
[[nodiscard]] Result<void> get_tasks_from_thread_linked_list(EchionSampler& echion,
std::vector<TaskInfo::Ptr>& tasks);
[[nodiscard]] Result<void> get_tasks_from_interpreter_linked_list(EchionSampler& echion,
PyThreadState* tstate,
std::vector<TaskInfo::Ptr>& tasks);
[[nodiscard]] Result<void> get_tasks_from_linked_list(EchionSampler& echion,
uintptr_t head_addr,
std::vector<TaskInfo::Ptr>& tasks);
[[nodiscard]] Result<void> get_tasks_from_thread_linked_list(const TaskAddressCallback& callback);
[[nodiscard]] Result<void> get_tasks_from_interpreter_linked_list(PyThreadState* tstate,
const TaskAddressCallback& callback);
[[nodiscard]] Result<void> get_tasks_from_linked_list(uintptr_t head_addr, const TaskAddressCallback& callback);
#endif
};

Expand Down
Loading
Loading