Add VM serial console logging for fast-cycle diagnosis

config/src/lib.rs

@@ -3,7 +3,8 @@ use thiserror::*;
 pub mod firecracker;
 pub mod manager;
 
-pub const DEFAULT_BOOT_ARGS: &str = "random.trust_cpu=on reboot=k panic=1 pci=off";
+pub const DEFAULT_BOOT_ARGS: &str =
+    "random.trust_cpu=on reboot=k panic=1 pci=off console=ttyS0";
 pub const NETWORK_MAGIC_MAC_START: &str = "06:00";
 pub const NETWORK_MASK_SHORT: u8 = 30;
 pub const NETWORK_MAX_ALLOCATIONS: u8 = 200;

manager/src/instance.rs

@@ -8,7 +8,7 @@ use config::{
 };
 use github::GitHub;
 use rand::distributions::{Alphanumeric, DistString};
-use std::{fs, os::unix::process::CommandExt, path::PathBuf, process::Command};
+use std::{fs, io::Read, os::unix::process::CommandExt, path::PathBuf, process::Command};
 use tracing::{debug, info, instrument, warn};
 use util::mount;
 
@@ -93,6 +93,26 @@ impl Instance {
         self.work_dir.as_std_path().join("firecracker.pid")
     }
 
+    pub fn console_log_path(&self) -> Utf8PathBuf {
+        self.work_dir.join("console.log")
+    }
+
+    /// Return the last `n` lines of the VM serial console log, or an empty string if unavailable.
+    pub fn read_console_tail(&self, n: usize) -> String {
+        let path = self.console_log_path();
+        let mut file = match fs::File::open(&path) {
+            Ok(f) => f,
+            Err(_) => return String::new(),
+        };
+        let mut content = String::new();
+        if file.read_to_string(&mut content).is_err() {
+            return String::new();
+        }
+        let lines: Vec<&str> = content.lines().collect();
+        let start = lines.len().saturating_sub(n);
+        lines[start..].join("\n")
+    }
+
     #[instrument(skip(self), fields(role = %self.role, idx = self.idx))]
     pub fn setup(&mut self) -> Result<()> {
         debug!(work_dir = %self.work_dir, "creating work directory");
@@ -208,12 +228,13 @@ impl Instance {
         self.setup_run()?;
 
         debug!(role = %self.role, idx = self.idx, "spawning Firecracker");
+        let console_file = fs::File::create(self.console_log_path())?;
         let child = unsafe {
             Command::new("firecracker")
                 .args(["--config-file", "config.json", "--no-api"])
                 .current_dir(&self.work_dir)
                 .stdin(std::process::Stdio::null())
-                .stdout(std::process::Stdio::null())
+                .stdout(console_file)
                 .stderr(std::process::Stdio::piped())
                 .pre_exec(|| {
                     if libc::setsid() == -1 {

manager/src/slot.rs

@@ -4,7 +4,7 @@ use opentelemetry::KeyValue;
 use std::io::Read;
 use std::path::Path;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use tracing::{error, info, instrument, warn};
 
 use crate::{instance::Instance, network::NetworkAllocation};
@@ -75,6 +75,7 @@ pub fn run_slot(config: Arc<ManagerConfig>, role: &Role, idx: usize) -> Result<(
                     error!(role = %role.name, idx, pid, error = %e, "failed to write PID file");
                 }
 
+                let cycle_start = Instant::now();
                 let status = {
                     let _running = tracing::info_span!(
                         "vm_running",
@@ -86,6 +87,7 @@ pub fn run_slot(config: Arc<ManagerConfig>, role: &Role, idx: usize) -> Result<(
                     .entered();
                     child.wait()
                 };
+                let cycle_secs = cycle_start.elapsed().as_secs();
 
                 match status {
                     Ok(status) => {
@@ -99,6 +101,7 @@ pub fn run_slot(config: Arc<ManagerConfig>, role: &Role, idx: usize) -> Result<(
                             idx,
                             cycle = cycle_n,
                             exit_code = status.code(),
+                            duration_secs = cycle_secs,
                             result = result_str,
                             "Firecracker exited"
                         );
@@ -109,10 +112,26 @@ pub fn run_slot(config: Arc<ManagerConfig>, role: &Role, idx: usize) -> Result<(
                                     idx,
                                     cycle = cycle_n,
                                     exit_code = status.code(),
-                                    "firecracker: {}", line
+                                    "firecracker stderr: {}", line
                                 );
                             }
                         }
+                        // Log console tail for short cycles — these indicate a failure inside the VM
+                        if cycle_secs < 60 {
+                            let tail = instance.read_console_tail(40);
+                            if !tail.is_empty() {
+                                warn!(
+                                    role = %role.name,
+                                    idx,
+                                    cycle = cycle_n,
+                                    duration_secs = cycle_secs,
+                                    "VM cycled quickly — console tail follows"
+                                );
+                                for line in tail.lines() {
+                                    warn!(role = %role.name, idx, cycle = cycle_n, "console: {}", line);
+                                }
+                            }
+                        }
                         jobs_counter.add(
                             1,
                             &[