fix(container-runner): read cgroup v1 memory usage under gvisor

Author: abcxff

container-runner/src/monitor.rs

@@ -1,18 +1,33 @@
 //! Periodic instance resource monitor.
 //!
 //! Opt-in via the [`ENABLE_ENV`] environment variable. When enabled it samples
-//! memory and CPU usage every [`SAMPLE_INTERVAL`] and logs them, so memory
+//! memory and CPU usage every [`sample_interval`] and logs them, so memory
 //! growth toward the limit (and the OOM that follows) is visible in the logs at
 //! fine granularity. Disabled by default so nothing is logged unless explicitly
 //! turned on.
 //!
-//! Two counter sources are supported, picked at startup:
-//!   - **cgroup v2** under `/sys/fs/cgroup` (real Linux). Exact against the
-//!     container limits.
-//!   - **`/proc`** (`/proc/meminfo`, `/proc/stat`), the fallback for a gVisor
-//!     sandbox, which does not expose cgroup v2. Values reflect the sandbox and
-//!     are approximate.
-//! If neither is readable the monitor logs once and disables itself.
+//! Memory and CPU are detected independently, because the gVisor sandbox
+//! exposes cgroup v1 memory but no cgroup v2 or cgroup v1 CPU accounting, so the
+//! two counters legitimately come from different sources.
+//!
+//! Only memory *usage* is reported, not a limit or percentage: under gVisor both
+//! the cgroup `memory.limit_in_bytes` and `/proc/meminfo` report the sandbox size
+//! rather than the container's configured limit, so any percentage would be
+//! misleading.
+//!
+//! Memory sources, in preference order:
+//!   - **cgroup v2** `memory.current` (real Linux). Exact.
+//!   - **cgroup v1** `memory/memory.usage_in_bytes` (gVisor sandbox).
+//!     Container-wide usage (all processes plus page cache).
+//!   - **`/proc/meminfo`** last resort. Under gVisor this reflects the whole
+//!     sandbox, not the container, so it is only an approximation.
+//!
+//! CPU sources, in preference order:
+//!   - **cgroup v2** `cpu.stat`.
+//!   - **`/proc/stat`**, the gVisor sandbox fallback (sandbox-wide, approximate).
+//!
+//! If neither a memory nor a CPU source is readable the monitor logs once and
+//! disables itself.
 
 use std::time::{Duration, Instant};
 
@@ -23,13 +38,21 @@ use tokio::time::{MissedTickBehavior, interval};
 /// `true`, `yes`, and `on` (case-insensitive).
 const ENABLE_ENV: &str = "RIVET_LOG_RESOURCE_USAGE";
 
-/// How often to sample and log resource usage.
-const SAMPLE_INTERVAL: Duration = Duration::from_millis(500);
+/// Default cadence for sampling and logging resource usage, when
+/// [`INTERVAL_ENV`] is unset.
+const DEFAULT_SAMPLE_INTERVAL: Duration = Duration::from_millis(500);
 
-const MEMORY_CURRENT: &str = "/sys/fs/cgroup/memory.current";
-const MEMORY_MAX: &str = "/sys/fs/cgroup/memory.max";
-const CPU_STAT: &str = "/sys/fs/cgroup/cpu.stat";
-const CPU_MAX: &str = "/sys/fs/cgroup/cpu.max";
+/// Environment variable overriding the sample/log interval, in milliseconds.
+/// Unset, unparseable, or `0` falls back to [`DEFAULT_SAMPLE_INTERVAL`].
+const INTERVAL_ENV: &str = "RIVET_RESOURCE_USAGE_INTERVAL_MS";
+
+/// cgroup v2 memory usage (real Linux).
+const MEMORY_CURRENT_V2: &str = "/sys/fs/cgroup/memory.current";
+/// cgroup v1 memory usage (gVisor sandbox). Container-wide, includes page
+/// cache. The sibling `memory.limit_in_bytes` is deliberately not read: under
+/// gVisor it reports the sandbox size, not the configured limit.
+const MEMORY_USAGE_V1: &str = "/sys/fs/cgroup/memory/memory.usage_in_bytes";
+const CPU_STAT_V2: &str = "/sys/fs/cgroup/cpu.stat";
 
 const PROC_MEMINFO: &str = "/proc/meminfo";
 const PROC_STAT: &str = "/proc/stat";
@@ -38,20 +61,41 @@ const PROC_STAT: &str = "/proc/stat";
 /// and gVisor.
 const USER_HZ: u64 = 100;
 
-/// Where the monitor reads counters from.
+/// Where the monitor reads memory counters from.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum MemSource {
+	/// cgroup v2 `memory.current` (real Linux).
+	CgroupV2,
+	/// cgroup v1 `memory.usage_in_bytes` (gVisor sandbox).
+	CgroupV1,
+	/// `/proc/meminfo` (sandbox-wide approximation).
+	Proc,
+}
+
+impl MemSource {
+	fn label(self) -> &'static str {
+		match self {
+			MemSource::CgroupV2 => "cgroup_v2",
+			MemSource::CgroupV1 => "cgroup_v1",
+			MemSource::Proc => "proc",
+		}
+	}
+}
+
+/// Where the monitor reads CPU counters from.
 #[derive(Clone, Copy, PartialEq, Eq)]
-enum Source {
-	/// cgroup v2 under `/sys/fs/cgroup` (real Linux).
+enum CpuSource {
+	/// cgroup v2 `cpu.stat` (real Linux).
 	CgroupV2,
-	/// `/proc` (gVisor sandbox, which does not expose cgroup v2).
+	/// `/proc/stat` (gVisor sandbox, which does not expose cgroup CPU).
 	Proc,
 }
 
-impl Source {
+impl CpuSource {
 	fn label(self) -> &'static str {
 		match self {
-			Source::CgroupV2 => "cgroup_v2",
-			Source::Proc => "proc",
+			CpuSource::CgroupV2 => "cgroup_v2",
+			CpuSource::Proc => "proc",
 		}
 	}
 }
@@ -63,8 +107,24 @@ pub fn spawn_resource_monitor() {
 	if !monitor_enabled() {
 		return;
 	}
-	tracing::info!(interval_ms = SAMPLE_INTERVAL.as_millis() as u64, "resource monitor enabled");
-	tokio::spawn(run_monitor());
+	let sample_interval = sample_interval();
+	tracing::info!(
+		interval_ms = sample_interval.as_millis() as u64,
+		"resource monitor enabled"
+	);
+	tokio::spawn(run_monitor(sample_interval));
+}
+
+/// The configured sample/log interval: [`INTERVAL_ENV`] in milliseconds when set
+/// to a positive integer, otherwise [`DEFAULT_SAMPLE_INTERVAL`].
+fn sample_interval() -> Duration {
+	match std::env::var(INTERVAL_ENV)
+		.ok()
+		.and_then(|value| value.trim().parse::<u64>().ok())
+	{
+		Some(ms) if ms > 0 => Duration::from_millis(ms),
+		_ => DEFAULT_SAMPLE_INTERVAL,
+	}
 }
 
 fn monitor_enabled() -> bool {
@@ -84,66 +144,83 @@ pub fn enabled() -> bool {
 	monitor_enabled()
 }
 
-/// The counter source the monitor would use right now: `"cgroup_v2"`, `"proc"`,
-/// or `"none"` when neither is readable. Exposed for the actor-scoped status log.
+/// The memory counter source the monitor would use right now: `"cgroup_v2"`,
+/// `"cgroup_v1"`, `"proc"`, or `"none"`. Exposed for the actor-scoped status log.
+/// Memory is the monitor's headline signal, so this reports the memory source.
 pub fn sampling_source() -> &'static str {
-	match detect_source() {
+	match detect_mem_source() {
 		Some(source) => source.label(),
 		None => "none",
 	}
 }
 
-/// Pick the counter source, preferring exact cgroup v2 over the `/proc` fallback.
-fn detect_source() -> Option<Source> {
-	if read_u64(MEMORY_CURRENT).is_some() && read_cgroup_cpu_usage_usec().is_some() {
-		Some(Source::CgroupV2)
-	} else if read_proc_memory().is_some() && read_proc_cpu_busy_usec().is_some() {
-		Some(Source::Proc)
+/// Pick the memory source, preferring exact cgroup v2, then cgroup v1 (gVisor
+/// sandbox), then the sandbox-wide `/proc/meminfo` approximation.
+fn detect_mem_source() -> Option<MemSource> {
+	if read_u64(MEMORY_CURRENT_V2).is_some() {
+		Some(MemSource::CgroupV2)
+	} else if read_u64(MEMORY_USAGE_V1).is_some() {
+		Some(MemSource::CgroupV1)
+	} else if read_proc_memory().is_some() {
+		Some(MemSource::Proc)
 	} else {
 		None
 	}
 }
 
-async fn run_monitor() {
-	let Some(source) = detect_source() else {
+/// Pick the CPU source, preferring cgroup v2 over the `/proc/stat` fallback.
+fn detect_cpu_source() -> Option<CpuSource> {
+	if read_cgroup_cpu_usage_usec().is_some() {
+		Some(CpuSource::CgroupV2)
+	} else if read_proc_cpu_busy_usec().is_some() {
+		Some(CpuSource::Proc)
+	} else {
+		None
+	}
+}
+
+async fn run_monitor(sample_interval: Duration) {
+	let mem_source = detect_mem_source();
+	let cpu_source = detect_cpu_source();
+	if mem_source.is_none() && cpu_source.is_none() {
 		tracing::warn!(
 			cgroup_dir = "/sys/fs/cgroup",
 			proc_meminfo = PROC_MEMINFO,
 			proc_stat = PROC_STAT,
-			"resource monitor disabled: neither cgroup v2 nor /proc counters are readable"
+			"resource monitor disabled: no readable memory or cpu counters"
 		);
 		return;
-	};
-	tracing::info!(source = source.label(), "resource monitor sampling");
+	}
+	tracing::info!(
+		mem_source = mem_source.map(MemSource::label).unwrap_or("none"),
+		cpu_source = cpu_source.map(CpuSource::label).unwrap_or("none"),
+		"resource monitor sampling"
+	);
 
-	let mut ticker = interval(SAMPLE_INTERVAL);
+	let mut ticker = interval(sample_interval);
 	// A slow sample must not make the monitor try to catch up with a burst of
 	// back-to-back ticks; just resume on the next boundary.
 	ticker.set_missed_tick_behavior(MissedTickBehavior::Skip);
 	// The first tick fires immediately; consume it so the first logged line
 	// already covers a full interval of CPU time.
 	ticker.tick().await;
 
-	// Limits are fixed for the instance lifetime, so read them once.
-	let mem_limit_mib = memory_limit_bytes(source).map(bytes_to_mib);
-	let cpu_limit_cores = cpu_limit_cores(source);
-
-	let mut prev_cpu_usec = cpu_busy_usec(source);
+	let mut prev_cpu_usec = cpu_source.and_then(cpu_busy_usec);
 	let mut prev_at = Instant::now();
 
 	loop {
 		ticker.tick().await;
 		let now = Instant::now();
 		let elapsed = now.duration_since(prev_at);
 
-		let cpu_now_usec = cpu_busy_usec(source);
+		let cpu_now_usec = cpu_source.and_then(cpu_busy_usec);
 		// CPU time used over the interval, divided by wall time, is the number of
 		// vCPU cores consumed (1.0 == one core fully busy).
 		let cpu_cores = match (prev_cpu_usec, cpu_now_usec) {
 			(Some(prev), Some(cur)) if !elapsed.is_zero() => {
-				cur.saturating_sub(prev) as f64 / elapsed.as_micros() as f64
+				Some(cur.saturating_sub(prev) as f64 / elapsed.as_micros() as f64)
 			}
-			_ => 0.0,
+			_ => None,
 		};
 		prev_cpu_usec = cpu_now_usec;
 		prev_at = now;
@@ -156,61 +233,35 @@ async fn run_monitor() {
 			continue;
 		}
 
-		let mem_used_mib = memory_used_bytes(source).map(bytes_to_mib);
-		let mem_pct = match (mem_used_mib, mem_limit_mib) {
-			(Some(used), Some(limit)) if limit > 0.0 => Some(used / limit * 100.0),
-			_ => None,
-		};
-		let cpu_pct = match cpu_limit_cores {
-			Some(limit) if limit > 0.0 => Some(cpu_cores / limit * 100.0),
-			_ => None,
-		};
+		let mem_used_mib = mem_source.and_then(memory_used_bytes).map(bytes_to_mib);
 
 		for actor_id in actor_ids {
 			tracing::info!(
 				actor_id = %actor_id,
-				source = source.label(),
+				mem_source = mem_source.map(MemSource::label).unwrap_or("none"),
+				cpu_source = cpu_source.map(CpuSource::label).unwrap_or("none"),
 				mem_used_mib = ?mem_used_mib,
-				mem_limit_mib = ?mem_limit_mib,
-				mem_pct = ?mem_pct,
-				cpu_cores,
-				cpu_limit_cores = ?cpu_limit_cores,
-				cpu_pct = ?cpu_pct,
+				cpu_cores = ?cpu_cores,
 				"instance resource usage"
 			);
 		}
 	}
 }
 
 /// Current memory usage in bytes for the selected source.
-fn memory_used_bytes(source: Source) -> Option<u64> {
+fn memory_used_bytes(source: MemSource) -> Option<u64> {
 	match source {
-		Source::CgroupV2 => read_u64(MEMORY_CURRENT),
-		Source::Proc => read_proc_memory().map(|(used, _limit)| used),
-	}
-}
-
-/// Memory limit in bytes for the selected source, or `None` when unlimited.
-fn memory_limit_bytes(source: Source) -> Option<u64> {
-	match source {
-		Source::CgroupV2 => read_cgroup_memory_max(),
-		Source::Proc => read_proc_memory().map(|(_used, limit)| limit),
+		MemSource::CgroupV2 => read_u64(MEMORY_CURRENT_V2),
+		MemSource::CgroupV1 => read_u64(MEMORY_USAGE_V1),
+		MemSource::Proc => read_proc_memory(),
 	}
 }
 
 /// Cumulative busy CPU time in microseconds for the selected source.
-fn cpu_busy_usec(source: Source) -> Option<u64> {
-	match source {
-		Source::CgroupV2 => read_cgroup_cpu_usage_usec(),
-		Source::Proc => read_proc_cpu_busy_usec(),
-	}
-}
-
-/// CPU limit in vCPU cores for the selected source, or `None` when unknown.
-fn cpu_limit_cores(source: Source) -> Option<f64> {
+fn cpu_busy_usec(source: CpuSource) -> Option<u64> {
 	match source {
-		Source::CgroupV2 => read_cgroup_cpu_limit_cores(),
-		Source::Proc => read_proc_cpu_count(),
+		CpuSource::CgroupV2 => read_cgroup_cpu_usage_usec(),
+		CpuSource::Proc => read_proc_cpu_busy_usec(),
 	}
 }
 
@@ -220,49 +271,21 @@ fn read_u64(path: &str) -> Option<u64> {
 	std::fs::read_to_string(path).ok()?.trim().parse().ok()
 }
 
-/// cgroup v2 memory limit in bytes, or `None` when unlimited (`memory.max` is
-/// `"max"`).
-fn read_cgroup_memory_max() -> Option<u64> {
-	let raw = std::fs::read_to_string(MEMORY_MAX).ok()?;
-	let raw = raw.trim();
-	if raw == "max" {
-		None
-	} else {
-		raw.parse().ok()
-	}
-}
-
 /// Cumulative CPU time consumed by the cgroup, in microseconds, from the
 /// `usage_usec` line of `cpu.stat`.
 fn read_cgroup_cpu_usage_usec() -> Option<u64> {
-	let stat = std::fs::read_to_string(CPU_STAT).ok()?;
+	let stat = std::fs::read_to_string(CPU_STAT_V2).ok()?;
 	stat.lines()
 		.find_map(|line| line.strip_prefix("usage_usec "))
 		.and_then(|value| value.trim().parse().ok())
 }
 
-/// cgroup v2 CPU limit in vCPU cores from `cpu.max` (`"<quota> <period>"`), or
-/// `None` when unlimited (`quota` is `"max"`).
-fn read_cgroup_cpu_limit_cores() -> Option<f64> {
-	let raw = std::fs::read_to_string(CPU_MAX).ok()?;
-	let mut parts = raw.split_whitespace();
-	let quota = parts.next()?;
-	let period: f64 = parts.next()?.parse().ok()?;
-	if quota == "max" || period <= 0.0 {
-		return None;
-	}
-	let quota: f64 = quota.parse().ok()?;
-	Some(quota / period)
-}
-
-/// `(used_bytes, total_bytes)` from `/proc/meminfo`. Used is `MemTotal -
-/// MemAvailable`; total doubles as the limit under gVisor, where it reflects the
-/// sandbox memory.
-fn read_proc_memory() -> Option<(u64, u64)> {
+/// Used memory in bytes from `/proc/meminfo` (`MemTotal - MemAvailable`). Under
+/// gVisor this reflects the whole sandbox, not the container.
+fn read_proc_memory() -> Option<u64> {
 	let total_kb = read_meminfo_kb("MemTotal")?;
 	let available_kb = read_meminfo_kb("MemAvailable")?;
-	let used_kb = total_kb.saturating_sub(available_kb);
-	Some((used_kb * 1024, total_kb * 1024))
+	Some(total_kb.saturating_sub(available_kb) * 1024)
 }
 
 /// Value in kB of a `/proc/meminfo` key such as `"MemTotal"`.
@@ -298,17 +321,6 @@ fn read_proc_cpu_busy_usec() -> Option<u64> {
 	Some(busy * (1_000_000 / USER_HZ))
 }
 
-/// Number of vCPUs from the per-CPU (`cpu0`, `cpu1`, ...) lines of `/proc/stat`.
-fn read_proc_cpu_count() -> Option<f64> {
-	let content = std::fs::read_to_string(PROC_STAT).ok()?;
-	// The aggregate line is `"cpu "` (trailing space); per-CPU lines are `"cpu0"`.
-	let count = content
-		.lines()
-		.filter(|line| line.starts_with("cpu") && !line.starts_with("cpu "))
-		.count();
-	(count > 0).then_some(count as f64)
-}
-
 fn bytes_to_mib(bytes: u64) -> f64 {
 	bytes as f64 / (1024.0 * 1024.0)
 }