Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions rust/crates/rqd/src/frame/running_frame.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use std::{
fmt::Display,
path::Path,
process::ExitStatus,
sync::atomic::{AtomicBool, Ordering},
sync::{Arc, RwLock},
};
use std::{process::Stdio, thread};
Expand Down Expand Up @@ -59,6 +60,9 @@ pub struct RunningFrame {
pub entrypoint_file_path: String,
state: RwLock<FrameState>,
should_remove_from_cache: RwLock<bool>,
#[serde(skip_serializing)]
#[serde(skip_deserializing)]
stats_frozen: AtomicBool,
}

#[derive(Serialize, Deserialize, Debug)]
Expand Down Expand Up @@ -177,6 +181,7 @@ impl RunningFrame {
launch_thread_handle: None,
})),
should_remove_from_cache: RwLock::new(false),
stats_frozen: AtomicBool::new(false),
}
}

Expand Down Expand Up @@ -226,6 +231,11 @@ impl RunningFrame {
}

pub fn update_frame_stats(&self, proc_stats: ProcessStats) {
// Don't update stats if they've been frozen (e.g., when frame is being killed for OOM)
if self.stats_frozen.load(Ordering::SeqCst) {
return;
}

self.frame_stats
.write()
.unwrap_or_else(|poisoned| poisoned.into_inner())
Expand Down Expand Up @@ -1041,6 +1051,8 @@ impl RunningFrame {

// Replace snapshot config with the new config:
frame.config = config;
// Initialize stats_frozen (skipped during deserialization)
frame.stats_frozen = AtomicBool::new(false);

let pid = frame.pid();

Expand Down Expand Up @@ -1258,6 +1270,17 @@ Render Frame Completed
.read()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}

/// Freezes the frame statistics to prevent further updates.
///
/// This is typically called when a frame is being killed for OOM (out of memory),
/// to capture the accurate memory measurement at the moment of kill detection
/// and prevent corruption from reading zombie/dying processes.
///
/// Once frozen, calls to `update_frame_stats()` will be ignored.
pub fn freeze_stats(&self) {
self.stats_frozen.store(true, Ordering::SeqCst);
}
}

#[cfg(test)]
Expand Down
4 changes: 4 additions & 0 deletions rust/crates/rqd/src/system/machine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,10 @@ impl MachineMonitor {
// Attempt to kill selected frames.
// Logic will ignore kill errors and try again on the next iteration
for frame in frames_to_kill {
// Freeze stats before killing to capture accurate memory measurement.
// This prevents corruption from reading zombie/dying processes after kill signal.
frame.freeze_stats();

if let Ok(manager) = manager::instance().await {
info!("Requesting a kill for {}", &frame);
let kill_result = manager
Expand Down
Loading