Add task dump capture behind taskdump feature (#354)

* Add task dump capture behind taskdump feature

Wraps spawned futures in `TaskDumped<F>` when the `taskdump` feature is
enabled. On each poll, if the previous idle gap exceeded the configured
threshold, the frames captured at the last yield point are emitted as
`TaskDumpEvent`. Capture itself runs inside `tokio::runtime::dump::trace_with`
with a noop waker on a diagnostic re-poll, so it doesn't produce duplicate
wake or poll events.

Configured via `TracedRuntimeBuilder::with_task_dumps(TaskDumpConfig)` or
`TelemetryCoreBuilder::task_dump_config`. Capture short-circuits when the
guard is disabled, so a paused guard skips `trace_with` entirely.

Bumps tokio to 1.52 for the `taskdump` feature.

* switch ci to use private feature

* cache the timestamp on poll_start

Author: rcoh

.github/actions/rust-build/action.yml

@@ -32,8 +32,12 @@ runs:
               if [ "${{ inputs.toolchain }}" = nightly ]; then
                   rm -fv Cargo.lock
               fi
+              # tokio's `taskdump` feature only compiles on linux aarch64/x86/x86_64.
+              # On macOS drop it from --all-features; leave everything else intact.
               if [ "$RUNNER_OS" = "Linux" ]; then
                   # Raise locked-memory limit for large perf ring buffers
                   sudo prlimit --pid $$ --memlock=unlimited:unlimited
+                  cargo test --all-targets --all-features
+              else
+                  cargo test --all-targets --features __nonlinux_all_features
               fi
-              cargo test --all-targets --all-features

Cargo.lock

@@ -19,7 +19,7 @@ all-features = true
 unexpected_cfgs = { level = "warn", check-cfg = ["cfg(shuttle)"] }
 
 [dependencies]
-tokio = { version = "1.51.0", features = ["rt", "macros", "time", "rt-multi-thread", "sync", "net", "io-util"] }
+tokio = { version = "1.52.0", features = ["rt", "macros", "time", "rt-multi-thread", "sync", "net", "io-util"] }
 tokio-util = "0.7"
 arc-swap = "1"
 crossbeam-queue = "0.3"
@@ -29,6 +29,7 @@ pin-project-lite = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 smallvec = "1"
+backtrace = { version = "0.3", optional = true }
 dial9-perf-self-profile = { workspace = true, optional = true }
 dial9-trace-format = { workspace = true, features = ["serde"] }
 tracing = "0.1.44"
@@ -50,8 +51,15 @@ shuttle = { version = "0.9.1", optional = true }
 analysis = []
 cpu-profiling = ["dep:dial9-perf-self-profile"]
 _shuttle = ["dep:shuttle", "metrique-timesource/custom-timesource", "metrique-timesource/test-util"]
+## Capture async backtraces at tokio yield points. Linux-only
+## (aarch64/x86/x86_64): tokio's upstream `taskdump` feature refuses to
+## compile on other targets. Enabling this feature on an unsupported target
+## is a hard compile error from tokio.
+taskdump = ["tokio/taskdump", "dep:backtrace"]
 tracing-layer = ["dep:tracing-subscriber"]
 worker-s3 = ["dep:aws-sdk-s3-transfer-manager", "dep:aws-sdk-s3", "dep:aws-config", "dep:time"]
+## All features except platform-specific ones (taskdump). Used in CI for non-Linux targets.
+__nonlinux_all_features = ["analysis", "cpu-profiling", "tracing-layer", "worker-s3"]
 
 [dev-dependencies]
 dial9-tokio-telemetry = { path = ".", features = ["analysis", "tracing-layer", "worker-s3"] }
@@ -61,7 +69,7 @@ clap = { version = "4", features = ["derive", "env"] }
 hdrhistogram = "7"
 metrique-timesource = { version = "0.1", features = ["custom-timesource", "tokio"] }
 metrique-writer = { version = "0.1", features = ["test-util"] }
-tokio = { version = "1.51.0", features = ["test-util"] }
+tokio = { version = "1.52.0", features = ["test-util"] }
 proptest = "1"
 tempfile = "3"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
@@ -74,7 +82,7 @@ async-trait = "0.1.89"
 uuid = { version = "1", features = ["v4"] }
 
 [target.'cfg(target_os = "linux")'.dev-dependencies]
-dial9-tokio-telemetry = { path = ".", features = ["cpu-profiling", "worker-s3", "analysis", "tracing-layer"] }
+dial9-tokio-telemetry = { path = ".", features = ["cpu-profiling", "worker-s3", "analysis", "taskdump", "tracing-layer"] }
 nix = { version = "0.29", features = ["process"] }
 
 [[bench]]

dial9-tokio-telemetry/Cargo.toml

@@ -180,6 +180,7 @@ fn to_fat_event(event: &TelemetryEvent, reader: &TraceReader) -> Option<FatEvent
         }),
         TelemetryEvent::TaskSpawn { .. }
         | TelemetryEvent::TaskTerminate { .. }
+        | TelemetryEvent::TaskDump { .. }
         | TelemetryEvent::ThreadNameDef { .. }
         | TelemetryEvent::SegmentMetadata { .. }
         | TelemetryEvent::ClockSync { .. } => None,

dial9-tokio-telemetry/examples/trace_to_fat_jsonl.rs

@@ -16,6 +16,8 @@ pub mod background_task;
 pub(crate) mod metrics;
 pub(crate) mod primitives;
 pub(crate) mod rate_limit;
+#[cfg(feature = "taskdump")]
+pub(crate) mod task_dumped;
 /// Core telemetry types, recording, and trace I/O.
 pub mod telemetry;
 pub(crate) mod traced;