Skip to content

Commit bef9130

Browse files
committed
address comments
Signed-off-by: chang-ning <spiderpower02@gmail.com>
1 parent fee4014 commit bef9130

2 files changed

Lines changed: 78 additions & 44 deletions

File tree

src/sampler.rs

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,43 @@ use std::time::{Duration, Instant};
88

99
use crate::{net, nvlink, stat, xgmi};
1010

11+
/// One subsystem's reading stamped right at its own read, so rate math
12+
/// never absorbs another subsystem's latency (e.g. a driver init earlier
13+
/// in the same pass).
14+
pub struct Sample<T> {
15+
pub data: T,
16+
pub taken_at: Instant,
17+
}
18+
19+
fn sample<T>(read: impl FnOnce() -> std::io::Result<T>) -> Option<Sample<T>> {
20+
let taken_at = Instant::now();
21+
read().ok().map(|data| Sample { data, taken_at })
22+
}
23+
1124
/// Everything one sampling pass produces. Raw counters only; delta/rate
1225
/// math stays on the UI thread where the previous snapshot lives.
1326
/// Each field is None when its read failed, so one failing subsystem
1427
/// (e.g. a kernel without rdma netlink) never blocks the others.
1528
pub struct Snapshot {
16-
pub stats: Option<Vec<stat::PortStat>>,
17-
pub ifstats: Option<Vec<net::IfStats>>,
18-
pub nvlink: Option<Vec<nvlink::NvLinkSnapshot>>,
19-
pub xgmi: Option<Vec<xgmi::XgmiSnapshot>>,
29+
pub stats: Option<Sample<Vec<stat::PortStat>>>,
30+
pub ifstats: Option<Sample<Vec<net::IfStats>>>,
31+
pub nvlink: Option<Sample<Vec<nvlink::NvLinkSnapshot>>>,
32+
pub xgmi: Option<Sample<Vec<xgmi::XgmiSnapshot>>>,
2033
pub processes: Option<Vec<stat::ProcessRdmaInfo>>,
34+
/// Pass start; used for the duplicate guard, staleness, and trace ts.
2135
pub taken_at: Instant,
2236
}
2337

2438
fn collect() -> Snapshot {
25-
// Stamp before the reads: the UI derives rates from taken_at deltas, so
26-
// a slow pass (e.g. first-time driver init) must not inflate elapsed.
2739
let taken_at = Instant::now();
2840
let processes = stat::read_all_qps()
2941
.ok()
3042
.map(|qps| stat::aggregate_by_process(&qps));
3143
Snapshot {
32-
stats: stat::read_all_stats().ok(),
33-
ifstats: net::read_all_ifstats().ok(),
34-
nvlink: nvlink::read_all_nvlink_stats().ok(),
35-
xgmi: xgmi::read_all_xgmi_stats().ok(),
44+
stats: sample(stat::read_all_stats),
45+
ifstats: sample(net::read_all_ifstats),
46+
nvlink: sample(nvlink::read_all_nvlink_stats),
47+
xgmi: sample(xgmi::read_all_xgmi_stats),
3648
processes,
3749
taken_at,
3850
}

src/tui/app.rs

Lines changed: 56 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -494,43 +494,45 @@ impl App {
494494
return;
495495
}
496496
}
497-
// Per-subsystem elapsed: measured to that subsystem's last success,
498-
// so a skipped tick diffs cumulative counters over the true span.
499-
let since = |at: Option<Instant>| at.map(|a| snap.taken_at.duration_since(a).as_secs_f64());
500-
501-
if let Some(stats) = snap.stats {
502-
self.rdma_rows = match since(self.prev_stats_at) {
503-
Some(e) => compute_throughputs(&self.prev_stats, &stats, e),
504-
None => compute_throughputs(&stats, &stats, 1.0), // baseline
497+
// Per-subsystem elapsed: each sample carries its own read-adjacent
498+
// timestamp, measured to that subsystem's last success, so neither
499+
// another subsystem's latency nor a skipped tick can skew the span.
500+
let since =
501+
|now: Instant, at: Option<Instant>| at.map(|a| now.duration_since(a).as_secs_f64());
502+
503+
if let Some(s) = snap.stats {
504+
self.rdma_rows = match since(s.taken_at, self.prev_stats_at) {
505+
Some(e) => compute_throughputs(&self.prev_stats, &s.data, e),
506+
None => compute_throughputs(&s.data, &s.data, 1.0), // baseline
505507
};
506-
self.prev_stats = stats;
507-
self.prev_stats_at = Some(snap.taken_at);
508+
self.prev_stats = s.data;
509+
self.prev_stats_at = Some(s.taken_at);
508510
}
509511

510-
if let Some(nvlink) = snap.nvlink {
511-
self.nvlink_rows = match since(self.prev_nvlink_at) {
512-
Some(e) => compute_nvlink_throughputs(&self.prev_nvlink, &nvlink, e),
513-
None => compute_nvlink_throughputs(&nvlink, &nvlink, 1.0), // baseline
512+
if let Some(s) = snap.nvlink {
513+
self.nvlink_rows = match since(s.taken_at, self.prev_nvlink_at) {
514+
Some(e) => compute_nvlink_throughputs(&self.prev_nvlink, &s.data, e),
515+
None => compute_nvlink_throughputs(&s.data, &s.data, 1.0), // baseline
514516
};
515-
self.prev_nvlink = nvlink;
516-
self.prev_nvlink_at = Some(snap.taken_at);
517+
self.prev_nvlink = s.data;
518+
self.prev_nvlink_at = Some(s.taken_at);
517519
}
518520

519-
if let Some(xgmi) = snap.xgmi {
520-
self.xgmi_rows = match since(self.prev_xgmi_at) {
521-
Some(e) => compute_xgmi_throughputs(&self.prev_xgmi, &xgmi, e),
522-
None => compute_xgmi_throughputs(&xgmi, &xgmi, 1.0), // baseline
521+
if let Some(s) = snap.xgmi {
522+
self.xgmi_rows = match since(s.taken_at, self.prev_xgmi_at) {
523+
Some(e) => compute_xgmi_throughputs(&self.prev_xgmi, &s.data, e),
524+
None => compute_xgmi_throughputs(&s.data, &s.data, 1.0), // baseline
523525
};
524-
self.prev_xgmi = xgmi;
525-
self.prev_xgmi_at = Some(snap.taken_at);
526+
self.prev_xgmi = s.data;
527+
self.prev_xgmi_at = Some(s.taken_at);
526528
}
527529

528-
if let Some(ifstats) = snap.ifstats {
529-
if let Some(e) = since(self.prev_ifstats_at) {
530-
self.net_rate = net::compute_net_rate(&self.prev_ifstats, &ifstats, e);
530+
if let Some(s) = snap.ifstats {
531+
if let Some(e) = since(s.taken_at, self.prev_ifstats_at) {
532+
self.net_rate = net::compute_net_rate(&self.prev_ifstats, &s.data, e);
531533
}
532-
self.prev_ifstats = ifstats;
533-
self.prev_ifstats_at = Some(snap.taken_at);
534+
self.prev_ifstats = s.data;
535+
self.prev_ifstats_at = Some(s.taken_at);
534536
}
535537

536538
if let Some(processes) = snap.processes {
@@ -2288,12 +2290,16 @@ mod apply_snapshot_tests {
22882290
}
22892291
}
22902292

2293+
fn sample<T>(data: T, taken_at: Instant) -> Option<crate::sampler::Sample<T>> {
2294+
Some(crate::sampler::Sample { data, taken_at })
2295+
}
2296+
22912297
fn snapshot(stats: Vec<PortStat>, taken_at: Instant) -> Snapshot {
22922298
Snapshot {
2293-
stats: Some(stats),
2294-
ifstats: Some(Vec::new()),
2295-
nvlink: Some(Vec::new()),
2296-
xgmi: Some(Vec::new()),
2299+
stats: sample(stats, taken_at),
2300+
ifstats: sample(Vec::new(), taken_at),
2301+
nvlink: sample(Vec::new(), taken_at),
2302+
xgmi: sample(Vec::new(), taken_at),
22972303
processes: Some(Vec::new()),
22982304
taken_at,
22992305
}
@@ -2311,14 +2317,30 @@ mod apply_snapshot_tests {
23112317
};
23122318
Snapshot {
23132319
stats: None, // e.g. kernel without rdma netlink support
2314-
ifstats: Some(Vec::new()),
2315-
nvlink: Some(vec![gpu]),
2316-
xgmi: Some(Vec::new()),
2320+
ifstats: sample(Vec::new(), taken_at),
2321+
nvlink: sample(vec![gpu], taken_at),
2322+
xgmi: sample(Vec::new(), taken_at),
23172323
processes: Some(Vec::new()),
23182324
taken_at,
23192325
}
23202326
}
23212327

2328+
#[test]
2329+
fn rates_use_per_subsystem_read_timestamps() {
2330+
// A slow pass must not skew a subsystem read later in it: the GPU
2331+
// sample's own timestamp, not the pass start, drives its elapsed.
2332+
let mut app = App::new();
2333+
let t0 = Instant::now();
2334+
let mut first = gpu_snapshot(0, t0);
2335+
// First pass was slow: GPU counters actually read 2s after pass start.
2336+
first.nvlink.as_mut().unwrap().taken_at = t0 + Duration::from_secs(2);
2337+
app.apply_snapshot(first);
2338+
// Second pass, fast: read at t0+4s -> true GPU window is 2s, not 4s.
2339+
app.apply_snapshot(gpu_snapshot(1_000_000_000, t0 + Duration::from_secs(4)));
2340+
// 1e9 bytes * 8 / 2 s / 1e9 = 4.0 Gbps (2.0 would mean pass-start skew)
2341+
assert!((app.throughputs[0].tx_gbps - 4.0).abs() < 1e-9);
2342+
}
2343+
23222344
#[test]
23232345
fn first_snapshot_populates_rows_with_zero_rates() {
23242346
let mut app = App::new();

0 commit comments

Comments
 (0)