Skip to content

Commit 9cfcde1

Browse files
committed
sync: measure what materialization costs in resident memory
An ignored measurement test, in the style of manifest_decode_cost. It is not a guard and it asserts nothing. Run it with: cargo test --release -- --ignored --nocapture materialize_resident_cost It exists because it REFUTED the hypothesis it was written to confirm. On 2026-08-19 the daemon showed 2.52 GB RSS with 2.2 GB resident and dirty in empty large-allocation regions, and I expected the 70.2 MB re-read per pass to be what retained it. It is not. 200 passes over three 23 MB files is 14 GB of churn and RSS does not move: corpus: 3 files, 70.0 MB RSS MB: start 75, after 200 re-reading passes 75, after 200 cached passes 75 So the read is not the retention driver, and the fixes in #56 and #57 should be weighted on CPU alone. The live behaviour was allocator retention of FREED pages. It was reclaimable: RSS fell from 2.52 GB to 709 MB within 90 s when an unrelated release build created memory pressure, with no restart. Why the daemon reaches a 2.5 GB high-water mark when this reproduction plateaus at 75 MB is UNRESOLVED. The untested hypothesis is per-thread large-block caches across the daemon's 14 threads against this single-threaded loop.
1 parent 09b8df4 commit 9cfcde1

1 file changed

Lines changed: 118 additions & 0 deletions

File tree

src/sync/engine.rs

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2157,6 +2157,124 @@ mod tests {
21572157
}
21582158
}
21592159

2160+
/// What the materialize re-read costs in RESIDENT MEMORY, measured.
2161+
///
2162+
/// Ignored by default because it is a measurement, not a guard. Run it with
2163+
/// `cargo test --release -- --ignored --nocapture materialize_resident_cost`.
2164+
///
2165+
/// Sized to the live `st2-declarations-default` entry observed on
2166+
/// 2026-08-19: three files of about 23 MB, 70.2 MB total, passed over once
2167+
/// or twice a second. On 19 August the daemon's RSS sat at 2.52 GB with
2168+
/// 2.2 GB resident and dirty in EMPTY large-allocation regions, while RSS
2169+
/// was flat under 12.5 GB of churn per 90 s. That is retention, and this
2170+
/// measures whether the read is what feeds it.
2171+
#[test]
2172+
#[ignore = "measurement, not a guard"]
2173+
fn materialize_resident_cost_with_and_without_the_cache() {
2174+
fn rss_kb() -> u64 {
2175+
let out = std::process::Command::new("ps")
2176+
.args(["-o", "rss=", "-p", &std::process::id().to_string()])
2177+
.output()
2178+
.expect("ps");
2179+
String::from_utf8_lossy(&out.stdout).trim().parse().unwrap_or(0)
2180+
}
2181+
2182+
let dir = tempfile::tempdir().unwrap();
2183+
let root = dir.path().to_path_buf();
2184+
std::fs::create_dir_all(&root).unwrap();
2185+
2186+
// Three files the size of the real build artifacts.
2187+
let mut node = SyncNode::new(Author([3u8; 32]));
2188+
let blob = vec![0xABu8; 23_346_256];
2189+
for name in ["a.bin", "b.bin", "c.bin"] {
2190+
node.local_write(name, &blob, 1_700_000_000, 0);
2191+
}
2192+
drop(blob);
2193+
2194+
let mut observed = HashMap::new();
2195+
materialize_tracked(
2196+
&mut node,
2197+
&root,
2198+
SyncPolicy::Catalog.rules(),
2199+
&HashMap::new(),
2200+
&mut observed,
2201+
&HashMap::new(),
2202+
None,
2203+
)
2204+
.unwrap();
2205+
2206+
// The cache a scan would have recorded for these three files.
2207+
let mut cache = HashMap::new();
2208+
let mut total = 0u64;
2209+
for (rel, meta) in node
2210+
.manifest()
2211+
.present_paths()
2212+
.map(|(r, m)| (r.clone(), *m))
2213+
.collect::<Vec<_>>()
2214+
{
2215+
let path = root.join(&rel);
2216+
let disk = std::fs::metadata(&path).unwrap();
2217+
let (secs, nanos) = mtime_of_metadata(&disk);
2218+
total += disk.len();
2219+
cache.insert(
2220+
rel,
2221+
ScanCacheEntry {
2222+
size: disk.len(),
2223+
mtime_secs: secs,
2224+
mtime_nanos: nanos,
2225+
hash: meta.hash,
2226+
},
2227+
);
2228+
}
2229+
println!("corpus: {} files, {:.1} MB", cache.len(), total as f64 / 1e6);
2230+
2231+
const PASSES: usize = 200;
2232+
2233+
let before = rss_kb();
2234+
for _ in 0..PASSES {
2235+
let mut obs = HashMap::new();
2236+
materialize_tracked(
2237+
&mut node,
2238+
&root,
2239+
SyncPolicy::Catalog.rules(),
2240+
&HashMap::new(),
2241+
&mut obs,
2242+
&HashMap::new(),
2243+
None,
2244+
)
2245+
.unwrap();
2246+
}
2247+
let after_reads = rss_kb();
2248+
2249+
for _ in 0..PASSES {
2250+
let mut obs = HashMap::new();
2251+
materialize_tracked(
2252+
&mut node,
2253+
&root,
2254+
SyncPolicy::Catalog.rules(),
2255+
&HashMap::new(),
2256+
&mut obs,
2257+
&cache,
2258+
None,
2259+
)
2260+
.unwrap();
2261+
}
2262+
let after_cached = rss_kb();
2263+
2264+
println!(
2265+
"RSS MB: start {:.0}, after {} re-reading passes {:.0}, after {} cached passes {:.0}",
2266+
before as f64 / 1024.0,
2267+
PASSES,
2268+
after_reads as f64 / 1024.0,
2269+
PASSES,
2270+
after_cached as f64 / 1024.0
2271+
);
2272+
println!(
2273+
"bytes read: re-reading path {:.1} GB, cached path 0.0 GB",
2274+
(total as f64 * PASSES as f64) / 1e9
2275+
);
2276+
}
2277+
21602278
/// What the sweep is worth, measured rather than asserted.
21612279
///
21622280
/// Ignored by default because it is a measurement, not a guard. Run it with

0 commit comments

Comments
 (0)