Merge pull request #548 from Dstack-TEE/feat/vmm-serial-log-history

feat(vmm): preserve serial logs across VM restarts

Author: kvinwang

.github/workflows/docker-build-check.yml

@@ -30,7 +30,7 @@ jobs:
           platforms: linux/amd64
           provenance: false
           build-args: |
-            DSTACK_REV=${{ github.sha }}
+            DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
             DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
 
       - name: Build KMS contracts
@@ -55,7 +55,7 @@ jobs:
           platforms: linux/amd64
           provenance: false
           build-args: |
-            DSTACK_REV=${{ github.sha }}
+            DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
 
   verifier:
     runs-on: ubuntu-latest
@@ -74,4 +74,4 @@ jobs:
           platforms: linux/amd64
           provenance: false
           build-args: |
-            DSTACK_REV=${{ github.sha }}
+            DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}

vmm/src/app.rs

@@ -242,6 +242,11 @@ impl App {
                     fs::remove_file(path)?;
                 }
             }
+            // Append current serial.log to serial.history.log before QEMU truncates it.
+            rotate_serial_log(&work_dir, self.config.cvm.serial_history_max_bytes);
+            // Add boot separator to stdout/stderr (they are opened in append mode).
+            append_boot_separator(&work_dir.stdout_file());
+            append_boot_separator(&work_dir.stderr_file());
 
             let devices = self.try_allocate_gpus(&vm_config.manifest)?;
             let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?;
@@ -1051,6 +1056,64 @@ impl App {
     }
 }
 
+/// Append a boot separator line with timestamp to an append-mode log file.
+fn append_boot_separator(path: &std::path::Path) {
+    use std::io::Write;
+    if !path.exists() {
+        return;
+    }
+    let Ok(mut file) = std::fs::OpenOptions::new().append(true).open(path) else {
+        return;
+    };
+    let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now());
+    let _ = writeln!(file, "\n===== boot @ {timestamp} =====\n");
+}
+
+/// Append current serial.log into serial.history.log with a boot separator,
+/// then truncate history if it exceeds `max_bytes`.
+fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) {
+    use std::io::Write;
+
+    let serial = work_dir.serial_file();
+    if !serial.exists() {
+        return;
+    }
+    let Ok(content) = fs::read(&serial) else {
+        return;
+    };
+    if content.is_empty() {
+        return;
+    }
+    let history = work_dir.serial_history_file();
+    let Ok(mut file) = std::fs::OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&history)
+    else {
+        return;
+    };
+    let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now());
+    let _ = writeln!(file, "\n===== boot @ {timestamp} =====\n");
+    let _ = file.write_all(&content);
+    drop(file);
+
+    // Truncate from the front if history exceeds max_bytes.
+    if let Ok(meta) = fs::metadata(&history) {
+        if meta.len() > max_bytes {
+            if let Ok(data) = fs::read(&history) {
+                let skip = data.len() - max_bytes as usize;
+                // Find the next newline after skip point to avoid cutting mid-line.
+                let start = data[skip..]
+                    .iter()
+                    .position(|&b| b == b'\n')
+                    .map(|p| skip + p + 1)
+                    .unwrap_or(skip);
+                let _ = fs::write(&history, &data[start..]);
+            }
+        }
+    }
+}
+
 pub(crate) fn make_sys_config(cfg: &Config, manifest: &Manifest) -> Result<String> {
     let image_path = cfg.image_path.join(&manifest.image);
     let image = Image::load(image_path).context("Failed to load image info")?;

vmm/src/app/qemu.rs

@@ -1042,6 +1042,10 @@ impl VmWorkDir {
         self.workdir.join("serial.log")
     }
 
+    pub fn serial_history_file(&self) -> PathBuf {
+        self.workdir.join("serial.history.log")
+    }
+
     pub fn serial_pty(&self) -> PathBuf {
         self.workdir.join("serial.pty")
     }

vmm/src/config.rs

@@ -203,6 +203,13 @@ pub struct CvmConfig {
     /// SMBIOS product information for cloud environment detection
     #[serde(default)]
     pub product: ProductConfig,
+
+    /// Max size in bytes for serial.history.log (default 4MB).
+    /// Previous boot serial logs are appended here before each restart.
+    /// Accepts human-readable sizes like "4MB", "512KB".
+    #[serde(default = "default_serial_history_max_bytes")]
+    #[serde(with = "size_parser::human_size")]
+    pub serial_history_max_bytes: u64,
 }
 
 /// SMBIOS product information configuration.
@@ -452,6 +459,10 @@ pub struct KeyProviderConfig {
     pub port: u16,
 }
 
+fn default_serial_history_max_bytes() -> u64 {
+    4 * 1024 * 1024 // 4MB
+}
+
 const CLIENT_CONF_PATH: &str = "/etc/dstack/client.conf";
 fn read_qemu_path_from_client_conf() -> Option<PathBuf> {
     #[derive(Debug, Deserialize)]