Skip to content

Commit 3532fda

Browse files
hellovaiclaude
andcommitted
Report where every task is stuck, and name deadlocks
When a run hangs there was nothing to go on: Ctrl-C cancelled the program, a test timeout said only that it timed out, and the shutdown leak warning named only spawn sites. Now each of them shows every unfinished task's BAML stack, what it waits for, and any cycle of tasks waiting on each other: task 11 (spawned in user.main), waiting for a key held by task 12 File "main.baml", line 46, in user.transfer File "main.baml", line 23, in user.Ledger.record task 12 (spawned in user.main), waiting for a key held by task 11 File "main.baml", line 30, in user.Ledger.audit File "main.baml", line 15, in user.Account.balance_now deadlock: task 11 waits for a key held by task 12, which waits for a key held by task 11 - `BexEngine::task_report` stops every task as a collection does, and reads each one's stack and why it parked. A task notes that reason when it parks (a sys-op, an await, spawn admission): one store, nothing recorded while it runs. The `baml.sync` key table supplies who holds each key; awaits link a task to the one settling its future. - `baml run`: the first Ctrl-C prints the report, then cancels as before. - `baml test`: a timed-out test's failure message carries the report for the test body and every task it waits on. - Shutdown: each leaked future's task is shown under the leak warning. - `Future.stack_trace()` returns where the task settling a future is right now, so a program can notice a stuck task, report it and cancel it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 7eb3cb3 commit 3532fda

19 files changed

Lines changed: 974 additions & 31 deletions

File tree

‎baml_language/crates/baml_builtins2/baml_std/baml/ns_future/future.baml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,46 @@ class Future<T, E> {
6868
function state(self) -> State throws never {
6969
$rust_function
7070
}
71+
72+
/// Where the task settling this future is right now: its stack, outermost
73+
/// frame first, standard-library frames left out. `null` once the future
74+
/// has settled, or when no task settles it.
75+
///
76+
/// Every task pauses for the moment this takes, so it suits finding out
77+
/// why a wait is taking too long, not a hot loop.
78+
///
79+
/// # Examples
80+
///
81+
/// ```baml
82+
/// let work = spawn { transfer(alice, bob, 10) };
83+
/// root.sys.sleep(root.time.Duration.from_seconds(5));
84+
/// if let trace: root.errors.StackTrace = work.stack_trace() {
85+
/// root.io.println("still running:\n" + trace.to_string());
86+
/// let _ = work.cancel();
87+
/// }
88+
/// ```
89+
function stack_trace(self) -> root.errors.StackTrace? throws never {
90+
_task_stack(self._id())
91+
}
92+
93+
/// (internal) The id the engine knows this future's task by.
94+
function _id(self) -> int throws never {
95+
$rust_function
96+
}
97+
}
98+
99+
/// (internal) What holds up the task settling future `future_id`: that
100+
/// task, every task it waits on, directly or through others, and any deadlock
101+
/// among them, as the stuck-task report prints them. Empty when no task
102+
/// settles it. The engine answers it, pausing every task to read it.
103+
function _task_report(future_id: int) -> string throws never {
104+
$rust_io_function
105+
}
106+
107+
/// (internal) The stack of the task settling future `future_id`, or `null`
108+
/// when no task does. The engine answers it, pausing every task to read it.
109+
function _task_stack(future_id: int) -> root.errors.StackTrace? throws never {
110+
$rust_io_function
71111
}
72112

73113
// BEP-034 combinators over a homogeneous array of futures (all sharing value

‎baml_language/crates/baml_builtins2/baml_std/testing/registry.baml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -607,12 +607,21 @@ function _run_test(body: TestBody, runner: TestRunner?, timeout_ms: int) -> Test
607607
if (timeout_ms > 0) {
608608
let timer: baml.future.Future<RunReport, never> = spawn {
609609
baml.sys.sleep(baml.time.Duration.from_milliseconds(timeout_ms));
610+
// Read before the race cancels the body: where it is stuck,
611+
// and every task it is waiting on.
612+
let report = baml.future._task_report(work._id());
613+
let stuck = if (report == "") {
614+
""
615+
} else {
616+
"\n" + report
617+
};
610618
RunReport {
611619
outcome: "fail",
612620
duration_ms: 0,
613621
message: "test timed out after "
614622
+ timeout_ms.to_string()
615-
+ "ms (override with --test-timeout)",
623+
+ "ms (override with --test-timeout)"
624+
+ stuck,
616625
}
617626
};
618627
racers.push(timer);

‎baml_language/crates/baml_cli/src/run_command.rs‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,32 @@ struct ScriptExpansion {
4141
extra_args: Vec<String>,
4242
}
4343

44-
/// A failed run ended the process with a panic when one escaped a root call.
45-
fn install_interrupts(rt: &tokio::runtime::Runtime) -> Result<baml_exec::Interrupts> {
44+
/// Signals cancel the root call; the first one first prints where every task
45+
/// is, so a run interrupted because it hung says why.
46+
fn install_interrupts(
47+
rt: &tokio::runtime::Runtime,
48+
engine: &Arc<BexEngine>,
49+
) -> Result<baml_exec::Interrupts> {
4650
let _guard = rt.enter();
47-
baml_exec::Interrupts::install().context("failed to install signal handlers")
51+
let engine = Arc::clone(engine);
52+
let report: baml_exec::InterruptReport = Box::new(move || {
53+
let engine = Arc::clone(&engine);
54+
Box::pin(async move { print_task_report(&engine).await })
55+
});
56+
baml_exec::Interrupts::install_with_report(Some(report))
57+
.context("failed to install signal handlers")
4858
}
4959

60+
/// Print every unfinished task's stack, what it waits for, and any deadlock.
61+
async fn print_task_report(engine: &Arc<BexEngine>) {
62+
match engine.task_report().await {
63+
Some(report) if report.is_empty() => {}
64+
Some(report) => eprint!("\ninterrupted: {report}"),
65+
None => eprintln!("\ninterrupted: could not pause every task to report where it is"),
66+
}
67+
}
68+
69+
/// A failed run ended the process with a panic when one escaped a root call.
5070
fn failure_status(engine: &BexEngine) -> bex_engine::ProcessStatus {
5171
if engine.root_panicked() {
5272
bex_engine::ProcessStatus::Panicked
@@ -776,7 +796,7 @@ impl RunArgs {
776796
let rt = tokio::runtime::Runtime::new().context("failed to create tokio runtime")?;
777797
let engine = Arc::new(engine);
778798
let output_format = output_format.unwrap_or(self.output_format);
779-
let interrupts = install_interrupts(&rt)?;
799+
let interrupts = install_interrupts(&rt, &engine)?;
780800
let log_output = self.log_output(output_format);
781801
let (call_context, logs) = Self::call_context(log_output, &interrupts);
782802
let dispatch_result = log_output.block_on(
@@ -1156,7 +1176,7 @@ impl RunArgs {
11561176
let rt = tokio::runtime::Runtime::new().context("failed to create tokio runtime")?;
11571177
let engine = Arc::new(engine);
11581178
let output_format = self.output_format;
1159-
let interrupts = install_interrupts(&rt)?;
1179+
let interrupts = install_interrupts(&rt, &engine)?;
11601180
let log_output = self.log_output(output_format);
11611181
let (call_context, logs) = Self::call_context(log_output, &interrupts);
11621182
let host_cancel = call_context.cancel.clone();

‎baml_language/crates/baml_cli/src/shutdown.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,20 @@ fn leak_message(leaks: &[bex_engine::LeakedFuture]) -> String {
105105
} else {
106106
"futures"
107107
};
108-
format!(
108+
let mut message = format!(
109109
"abandoned {} leaked background {futures} spawned in: {listed} — the owning \
110110
test or call finished without cleaning them up (--shutdown-timeout \
111111
adjusts the wait; `none` waits forever)",
112112
leaks.len()
113-
)
113+
);
114+
// Where each one was stuck when the deadline passed.
115+
for task in leaks.iter().filter_map(|leak| leak.task.as_ref()) {
116+
for line in task.to_string().lines() {
117+
message.push_str("\n ");
118+
message.push_str(line);
119+
}
120+
}
121+
message
114122
}
115123

116124
#[cfg(test)]
@@ -141,6 +149,7 @@ mod tests {
141149
fn leak_message_groups_by_origin_most_frequent_first() {
142150
let leak = |origin: &str| bex_engine::LeakedFuture {
143151
origin: origin.into(),
152+
task: None,
144153
};
145154
let msg = leak_message(&[
146155
leak("user.b.serve"),

‎baml_language/crates/baml_cli/src/snapshots/baml_cli__describe_command_tests__render_builtin_package_listing.snap‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -105,15 +105,15 @@ class baml.fs.Lock <builtin>/baml/ns_fs/fs.baml:2
105105
function baml.fs.lock <builtin>/baml/ns_fs/fs.baml:265
106106
enum baml.future.State <builtin>/baml/ns_future/future.baml:19
107107
class baml.future.Future <builtin>/baml/ns_future/future.baml:26
108-
class baml.future.Success <builtin>/baml/ns_future/future.baml:79
109-
class baml.future.Failure <builtin>/baml/ns_future/future.baml:84
110-
class baml.future.Panicked <builtin>/baml/ns_future/future.baml:89
111-
function baml.future.all_settled <builtin>/baml/ns_future/future.baml:101
112-
function baml.future.all <builtin>/baml/ns_future/future.baml:136
113-
class baml.future.AllFailed <builtin>/baml/ns_future/future.baml:175
114-
function baml.future.race <builtin>/baml/ns_future/future.baml:193
115-
function baml.future.any <builtin>/baml/ns_future/future.baml:209
116-
function baml.future.with_timeout <builtin>/baml/ns_future/future.baml:294
108+
class baml.future.Success <builtin>/baml/ns_future/future.baml:119
109+
class baml.future.Failure <builtin>/baml/ns_future/future.baml:124
110+
class baml.future.Panicked <builtin>/baml/ns_future/future.baml:129
111+
function baml.future.all_settled <builtin>/baml/ns_future/future.baml:141
112+
function baml.future.all <builtin>/baml/ns_future/future.baml:176
113+
class baml.future.AllFailed <builtin>/baml/ns_future/future.baml:215
114+
function baml.future.race <builtin>/baml/ns_future/future.baml:233
115+
function baml.future.any <builtin>/baml/ns_future/future.baml:249
116+
function baml.future.with_timeout <builtin>/baml/ns_future/future.baml:334
117117
class baml.glob.ScanOptions <builtin>/baml/ns_glob/glob.baml:2
118118
class baml.glob.Glob <builtin>/baml/ns_glob/glob.baml:18
119119
function baml.glob.new <builtin>/baml/ns_glob/glob.baml:41

‎baml_language/crates/baml_cli/tests/signal_e2e.rs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,84 @@ fn pack_signal_cancels_main_and_runs_defer() {
134134
}
135135
}
136136

137+
/// Two tasks that each hold the key the other wants. Each takes its first
138+
/// key and waits until the other has its own before asking for the next, so
139+
/// the deadlock forms every time; `ready` appears once it has.
140+
const DEADLOCK: &str = r#"
141+
class Log {
142+
entries: string[],
143+
}
144+
145+
function pause() -> void {
146+
baml.sys.sleep(baml.time.Duration.from_milliseconds(5))
147+
}
148+
149+
function cross(first: Log, second: Log, started: Log) -> void {
150+
baml.sync.atomic(first, () -> void {
151+
started.entries.push("in");
152+
while (started.entries.length() < 2) {
153+
pause();
154+
}
155+
baml.sync.atomic(second, () -> void { });
156+
})
157+
}
158+
159+
function main() -> void {
160+
let a = Log { entries: [] };
161+
let b = Log { entries: [] };
162+
let started = Log { entries: [] };
163+
let x = spawn { cross(a, b, started) };
164+
let y = spawn { cross(b, a, started) };
165+
while (started.entries.length() < 2) {
166+
pause();
167+
}
168+
baml.sys.sleep(baml.time.Duration.from_milliseconds(100));
169+
baml.fs.write("ready", "") catch_all (_) { _ => null };
170+
await x;
171+
await y;
172+
}
173+
"#;
174+
175+
/// The first signal prints where every task is and names the deadlock before
176+
/// it cancels the root call; the process then dies by the signal.
177+
#[test]
178+
fn first_signal_reports_where_each_task_is() {
179+
let tmp = tempfile::tempdir().unwrap();
180+
common::write_project(tmp.path(), DEADLOCK);
181+
let mut child = Command::new(common::baml_cli())
182+
.args(["run", "main"])
183+
.current_dir(tmp.path())
184+
.env("BAML_HOME", common::shared_baml_home())
185+
.env("BAML_CLI_ALLOW_DIRECT", "1")
186+
.env("DEV_BAML_CLI_DISABLE_AGENT_DETECTION", "1")
187+
.stdout(Stdio::null())
188+
.stderr(Stdio::piped())
189+
.spawn()
190+
.expect("spawn baml-cli");
191+
wait_for_file(&tmp.path().join("ready"), &mut child);
192+
send(&child, libc::SIGINT);
193+
let status = wait_with_timeout(&mut child);
194+
let mut stderr = String::new();
195+
std::io::Read::read_to_string(&mut child.stderr.take().unwrap(), &mut stderr).unwrap();
196+
assert_eq!(
197+
status.signal(),
198+
Some(libc::SIGINT),
199+
"got {status}; stderr:\n{stderr}"
200+
);
201+
for expected in [
202+
"interrupted: 3 tasks still running:",
203+
"(called from the host), awaiting task ",
204+
", waiting for a key held by task ",
205+
"in user.cross",
206+
"deadlock: task ",
207+
] {
208+
assert!(
209+
stderr.contains(expected),
210+
"missing {expected:?} in:\n{stderr}"
211+
);
212+
}
213+
}
214+
137215
/// A second signal while the root call is still unwinding ends the process
138216
/// at once instead of waiting for a slow `defer`.
139217
#[test]

‎baml_language/crates/baml_exec/src/interrupt.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
// cancelled, the host finishes engine shutdown and then dies by the same
77
// signal, so the parent sees an ordinary signal death (shell status 128+n).
88
//
9+
// Before cancelling, the first signal runs the host's report hook, if it
10+
// installed one (`baml run` prints where every task is stuck), so the report
11+
// shows the program as it was, not as it unwinds.
12+
//
913
// A second signal, or any signal once the root call has finished (the
1014
// engine shutdown wait), terminates immediately. Once installed the
1115
// handlers stay for the life of the process: the runtime cannot hand the
@@ -22,6 +26,10 @@ use std::sync::{
2226

2327
use bex_engine::CancellationToken;
2428

29+
/// What the first signal runs before it cancels the root call.
30+
pub type InterruptReport =
31+
Box<dyn Fn() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync>;
32+
2533
/// A termination signal the host received.
2634
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2735
pub struct Interrupt(i32);
@@ -52,6 +60,13 @@ impl Interrupts {
5260
/// Install the handlers. Must be called inside a Tokio runtime with the
5361
/// IO driver enabled.
5462
pub fn install() -> std::io::Result<Self> {
63+
Self::install_with_report(None)
64+
}
65+
66+
/// [`Self::install`], running `report` on the first signal before it
67+
/// cancels the root call. A second signal while `report` runs still
68+
/// terminates at once.
69+
pub fn install_with_report(report: Option<InterruptReport>) -> std::io::Result<Self> {
5570
let token = CancellationToken::new();
5671
let state = Arc::new(State {
5772
received: AtomicI32::new(0),
@@ -71,6 +86,12 @@ impl Interrupts {
7186
if !first {
7287
Interrupt(signal).terminate();
7388
}
89+
if let Some(report) = &report {
90+
tokio::select! {
91+
() = report() => {}
92+
second = signals.recv() => Interrupt(second).terminate(),
93+
}
94+
}
7495
listener_token.cancel();
7596
}
7697
});

‎baml_language/crates/baml_exec/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ pub use dispatch::{
4242
};
4343
pub use envelope::{PACK_SECTION_NAME, PackEnvelope, PackMode, TargetEntry};
4444
#[cfg(not(target_arch = "wasm32"))]
45-
pub use interrupt::{Interrupt, Interrupts};
45+
pub use interrupt::{Interrupt, InterruptReport, Interrupts};
4646
pub use json_coerce::load_json_source;
4747
pub use launch_context::{
4848
CONTEXT_ENV, context_source_from_env, ensure_single_stdin_consumer, load_launch_context,

0 commit comments

Comments
 (0)