Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,46 @@ class Future<T, E> {
function state(self) -> State throws never {
$rust_function
}

/// Where the task settling this future is right now: its stack, outermost
/// frame first, standard-library frames left out. `null` once the future
/// has settled, or when no task settles it.
///
/// Every task pauses for the moment this takes, so it suits finding out
/// why a wait is taking too long, not a hot loop.
///
/// # Examples
///
/// ```baml
/// let work = spawn { transfer(alice, bob, 10) };
/// root.sys.sleep(root.time.Duration.from_seconds(5));
/// if let trace: root.errors.StackTrace = work.stack_trace() {
/// root.io.println("still running:\n" + trace.to_string());
/// let _ = work.cancel();
/// }
/// ```
function stack_trace(self) -> root.errors.StackTrace? throws never {
_task_stack(self._id())
}

/// (internal) The id the engine knows this future's task by.
function _id(self) -> int throws never {
$rust_function
}
}

/// (internal) What holds up the task settling future `future_id`: that
/// task, every task it waits on, directly or through others, and any deadlock
/// among them, as the stuck-task report prints them. Empty when no task
/// settles it. The engine answers it, pausing every task to read it.
function _task_report(future_id: int) -> string throws never {
$rust_io_function
}

/// (internal) The stack of the task settling future `future_id`, or `null`
/// when no task does. The engine answers it, pausing every task to read it.
function _task_stack(future_id: int) -> root.errors.StackTrace? throws never {
$rust_io_function
}

// BEP-034 combinators over a homogeneous array of futures (all sharing value
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -607,12 +607,21 @@ function _run_test(body: TestBody, runner: TestRunner?, timeout_ms: int) -> Test
if (timeout_ms > 0) {
let timer: baml.future.Future<RunReport, never> = spawn {
baml.sys.sleep(baml.time.Duration.from_milliseconds(timeout_ms));
// Read before the race cancels the body: where it is stuck,
// and every task it is waiting on.
let report = baml.future._task_report(work._id());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
let stuck = if (report == "") {
""
} else {
"\n" + report
};
RunReport {
outcome: "fail",
duration_ms: 0,
message: "test timed out after "
+ timeout_ms.to_string()
+ "ms (override with --test-timeout)",
+ "ms (override with --test-timeout)"
+ stuck,
}
};
racers.push(timer);
Expand Down
30 changes: 25 additions & 5 deletions baml_language/crates/baml_cli/src/run_command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,32 @@ struct ScriptExpansion {
extra_args: Vec<String>,
}

/// A failed run ended the process with a panic when one escaped a root call.
fn install_interrupts(rt: &tokio::runtime::Runtime) -> Result<baml_exec::Interrupts> {
/// Signals cancel the root call; the first one first prints where every task
/// is, so a run interrupted because it hung says why.
fn install_interrupts(
rt: &tokio::runtime::Runtime,
engine: &Arc<BexEngine>,
) -> Result<baml_exec::Interrupts> {
let _guard = rt.enter();
baml_exec::Interrupts::install().context("failed to install signal handlers")
let engine = Arc::clone(engine);
let report: baml_exec::InterruptReport = Box::new(move || {
let engine = Arc::clone(&engine);
Box::pin(async move { print_task_report(&engine).await })
});
baml_exec::Interrupts::install_with_report(Some(report))
.context("failed to install signal handlers")
}

/// Print every unfinished task's stack, what it waits for, and any deadlock.
async fn print_task_report(engine: &Arc<BexEngine>) {
match engine.task_report().await {
Some(report) if report.is_empty() => {}
Some(report) => eprint!("\ninterrupted: {report}"),
None => eprintln!("\ninterrupted: could not pause every task to report where it is"),
}
}

/// A failed run ended the process with a panic when one escaped a root call.
fn failure_status(engine: &BexEngine) -> bex_engine::ProcessStatus {
if engine.root_panicked() {
bex_engine::ProcessStatus::Panicked
Expand Down Expand Up @@ -776,7 +796,7 @@ impl RunArgs {
let rt = tokio::runtime::Runtime::new().context("failed to create tokio runtime")?;
let engine = Arc::new(engine);
let output_format = output_format.unwrap_or(self.output_format);
let interrupts = install_interrupts(&rt)?;
let interrupts = install_interrupts(&rt, &engine)?;
let log_output = self.log_output(output_format);
let (call_context, logs) = Self::call_context(log_output, &interrupts);
let dispatch_result = log_output.block_on(
Expand Down Expand Up @@ -1156,7 +1176,7 @@ impl RunArgs {
let rt = tokio::runtime::Runtime::new().context("failed to create tokio runtime")?;
let engine = Arc::new(engine);
let output_format = self.output_format;
let interrupts = install_interrupts(&rt)?;
let interrupts = install_interrupts(&rt, &engine)?;
let log_output = self.log_output(output_format);
let (call_context, logs) = Self::call_context(log_output, &interrupts);
let host_cancel = call_context.cancel.clone();
Expand Down
13 changes: 11 additions & 2 deletions baml_language/crates/baml_cli/src/shutdown.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,20 @@ fn leak_message(leaks: &[bex_engine::LeakedFuture]) -> String {
} else {
"futures"
};
format!(
let mut message = format!(
"abandoned {} leaked background {futures} spawned in: {listed} — the owning \
test or call finished without cleaning them up (--shutdown-timeout \
adjusts the wait; `none` waits forever)",
leaks.len()
)
);
// Where each one was stuck when the deadline passed.
for task in leaks.iter().filter_map(|leak| leak.task.as_ref()) {
for line in task.to_string().lines() {
message.push_str("\n ");
message.push_str(line);
}
}
message
}

#[cfg(test)]
Expand Down Expand Up @@ -141,6 +149,7 @@ mod tests {
fn leak_message_groups_by_origin_most_frequent_first() {
let leak = |origin: &str| bex_engine::LeakedFuture {
origin: origin.into(),
task: None,
};
let msg = leak_message(&[
leak("user.b.serve"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,15 +105,15 @@ class baml.fs.Lock <builtin>/baml/ns_fs/fs.baml:2
function baml.fs.lock <builtin>/baml/ns_fs/fs.baml:265
enum baml.future.State <builtin>/baml/ns_future/future.baml:19
class baml.future.Future <builtin>/baml/ns_future/future.baml:26
class baml.future.Success <builtin>/baml/ns_future/future.baml:79
class baml.future.Failure <builtin>/baml/ns_future/future.baml:84
class baml.future.Panicked <builtin>/baml/ns_future/future.baml:89
function baml.future.all_settled <builtin>/baml/ns_future/future.baml:101
function baml.future.all <builtin>/baml/ns_future/future.baml:136
class baml.future.AllFailed <builtin>/baml/ns_future/future.baml:175
function baml.future.race <builtin>/baml/ns_future/future.baml:193
function baml.future.any <builtin>/baml/ns_future/future.baml:209
function baml.future.with_timeout <builtin>/baml/ns_future/future.baml:294
class baml.future.Success <builtin>/baml/ns_future/future.baml:119
class baml.future.Failure <builtin>/baml/ns_future/future.baml:124
class baml.future.Panicked <builtin>/baml/ns_future/future.baml:129
function baml.future.all_settled <builtin>/baml/ns_future/future.baml:141
function baml.future.all <builtin>/baml/ns_future/future.baml:176
class baml.future.AllFailed <builtin>/baml/ns_future/future.baml:215
function baml.future.race <builtin>/baml/ns_future/future.baml:233
function baml.future.any <builtin>/baml/ns_future/future.baml:249
function baml.future.with_timeout <builtin>/baml/ns_future/future.baml:334
class baml.glob.ScanOptions <builtin>/baml/ns_glob/glob.baml:2
class baml.glob.Glob <builtin>/baml/ns_glob/glob.baml:18
function baml.glob.new <builtin>/baml/ns_glob/glob.baml:41
Expand Down
78 changes: 78 additions & 0 deletions baml_language/crates/baml_cli/tests/signal_e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,84 @@ fn pack_signal_cancels_main_and_runs_defer() {
}
}

/// Two tasks that each hold the key the other wants. Each takes its first
/// key and waits until the other has its own before asking for the next, so
/// the deadlock forms every time; `ready` appears once it has.
const DEADLOCK: &str = r#"
class Log {
entries: string[],
}

function pause() -> void {
baml.sys.sleep(baml.time.Duration.from_milliseconds(5))
}

function cross(first: Log, second: Log, started: Log) -> void {
baml.sync.atomic(first, () -> void {
started.entries.push("in");
while (started.entries.length() < 2) {
pause();
}
baml.sync.atomic(second, () -> void { });
})
}

function main() -> void {
let a = Log { entries: [] };
let b = Log { entries: [] };
let started = Log { entries: [] };
let x = spawn { cross(a, b, started) };
let y = spawn { cross(b, a, started) };
while (started.entries.length() < 2) {
pause();
}
baml.sys.sleep(baml.time.Duration.from_milliseconds(100));
baml.fs.write("ready", "") catch_all (_) { _ => null };
await x;
await y;
}
"#;

/// The first signal prints where every task is and names the deadlock before
/// it cancels the root call; the process then dies by the signal.
#[test]
fn first_signal_reports_where_each_task_is() {
let tmp = tempfile::tempdir().unwrap();
common::write_project(tmp.path(), DEADLOCK);
let mut child = Command::new(common::baml_cli())
.args(["run", "main"])
.current_dir(tmp.path())
.env("BAML_HOME", common::shared_baml_home())
.env("BAML_CLI_ALLOW_DIRECT", "1")
.env("DEV_BAML_CLI_DISABLE_AGENT_DETECTION", "1")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("spawn baml-cli");
wait_for_file(&tmp.path().join("ready"), &mut child);
send(&child, libc::SIGINT);
let status = wait_with_timeout(&mut child);
let mut stderr = String::new();
std::io::Read::read_to_string(&mut child.stderr.take().unwrap(), &mut stderr).unwrap();
assert_eq!(
status.signal(),
Some(libc::SIGINT),
"got {status}; stderr:\n{stderr}"
);
for expected in [
"interrupted: 3 tasks still running:",
"(called from the host), awaiting task ",
", waiting for a key held by task ",
"in user.cross",
"deadlock: task ",
] {
assert!(
stderr.contains(expected),
"missing {expected:?} in:\n{stderr}"
);
}
}

/// A second signal while the root call is still unwinding ends the process
/// at once instead of waiting for a slow `defer`.
#[test]
Expand Down
21 changes: 21 additions & 0 deletions baml_language/crates/baml_exec/src/interrupt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
// cancelled, the host finishes engine shutdown and then dies by the same
// signal, so the parent sees an ordinary signal death (shell status 128+n).
//
// Before cancelling, the first signal runs the host's report hook, if it
// installed one (`baml run` prints where every task is stuck), so the report
// shows the program as it was, not as it unwinds.
//
// A second signal, or any signal once the root call has finished (the
// engine shutdown wait), terminates immediately. Once installed the
// handlers stay for the life of the process: the runtime cannot hand the
Expand All @@ -22,6 +26,10 @@ use std::sync::{

use bex_engine::CancellationToken;

/// What the first signal runs before it cancels the root call.
pub type InterruptReport =
Box<dyn Fn() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync>;

/// A termination signal the host received.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Interrupt(i32);
Expand Down Expand Up @@ -52,6 +60,13 @@ impl Interrupts {
/// Install the handlers. Must be called inside a Tokio runtime with the
/// IO driver enabled.
pub fn install() -> std::io::Result<Self> {
Self::install_with_report(None)
}

/// [`Self::install`], running `report` on the first signal before it
/// cancels the root call. A second signal while `report` runs still
/// terminates at once.
pub fn install_with_report(report: Option<InterruptReport>) -> std::io::Result<Self> {
let token = CancellationToken::new();
let state = Arc::new(State {
received: AtomicI32::new(0),
Expand All @@ -71,6 +86,12 @@ impl Interrupts {
if !first {
Interrupt(signal).terminate();
}
if let Some(report) = &report {
tokio::select! {
() = report() => {}
second = signals.recv() => Interrupt(second).terminate(),
}
}
listener_token.cancel();
}
});
Expand Down
2 changes: 1 addition & 1 deletion baml_language/crates/baml_exec/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ pub use dispatch::{
};
pub use envelope::{PACK_SECTION_NAME, PackEnvelope, PackMode, TargetEntry};
#[cfg(not(target_arch = "wasm32"))]
pub use interrupt::{Interrupt, Interrupts};
pub use interrupt::{Interrupt, InterruptReport, Interrupts};
pub use json_coerce::load_json_source;
pub use launch_context::{
CONTEXT_ENV, context_source_from_env, ensure_single_stdin_consumer, load_launch_context,
Expand Down
Loading
Loading