Skip to content

Commit b027e10

Browse files
crazyguitarcrazyguitar
authored
feat: per-GPU health gauges on NVLink/XGMI tabs and capacity-based bar colors (#36)
1 parent 6de7882 commit b027e10

7 files changed

Lines changed: 492 additions & 35 deletions

File tree

‎src/gpu.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
//! Shared per-GPU health metrics, filled by the NVML (nvlink) and amdsmi
2+
//! (xgmi) readers and rendered as the GPU tabs' gauge strip.
3+
4+
/// Live values; each field is None when its read failed or is unsupported.
5+
#[derive(Clone, Debug, Default, PartialEq)]
6+
pub struct GpuMetrics {
7+
pub util_pct: Option<u32>,
8+
pub vram_used_mb: Option<u64>,
9+
pub vram_total_mb: Option<u64>,
10+
pub temp_c: Option<i64>,
11+
pub power_w: Option<f64>,
12+
pub clock_mhz: Option<u32>,
13+
}
14+
15+
impl GpuMetrics {
16+
/// True when no reading is available at all; readers report None then,
17+
/// so the UI skips the gauge line instead of rendering all dashes.
18+
pub fn is_empty(&self) -> bool {
19+
*self == Self::default()
20+
}
21+
}

‎src/main.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
mod gpu;
12
mod net;
23
mod netlink;
34
mod nvlink;

‎src/nvlink.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use std::collections::HashMap;
99
use std::io;
1010
use std::sync::{LazyLock, Mutex, OnceLock};
1111

12+
use nvml_wrapper::enum_wrappers::device::{Clock, TemperatureSensor};
1213
use nvml_wrapper::enum_wrappers::nv_link::{ErrorCounter, IntDeviceType};
1314
use nvml_wrapper::enums::device::SampleValue;
1415
use nvml_wrapper::structs::device::FieldId;
@@ -132,6 +133,8 @@ fn link_meta(
132133
pub struct NvLinkSnapshot {
133134
pub gpu_index: u32,
134135
pub gpu_name: String,
136+
/// Live GPU health (util/vram/temp/power/clock) for the gauge strip.
137+
pub metrics: Option<crate::gpu::GpuMetrics>,
135138
/// Total number of NVLink ports exposed by the GPU (active + inactive).
136139
pub link_count: u32,
137140
/// Negotiated aggregate speed in Gbps, derived from the per-link
@@ -210,6 +213,22 @@ pub fn read_all_nvlink_stats() -> io::Result<Vec<NvLinkSnapshot>> {
210213
Ok(snapshots)
211214
}
212215

216+
/// Live GPU health via safe NVML wrappers; absent readings become None.
217+
fn read_gpu_metrics(device: &nvml_wrapper::Device<'_>) -> crate::gpu::GpuMetrics {
218+
let mem = device.memory_info().ok();
219+
crate::gpu::GpuMetrics {
220+
util_pct: device.utilization_rates().ok().map(|u| u.gpu),
221+
vram_used_mb: mem.as_ref().map(|m| m.used / (1024 * 1024)),
222+
vram_total_mb: mem.as_ref().map(|m| m.total / (1024 * 1024)),
223+
temp_c: device
224+
.temperature(TemperatureSensor::Gpu)
225+
.ok()
226+
.map(|t| t as i64),
227+
power_w: device.power_usage().ok().map(|mw| mw as f64 / 1000.0),
228+
clock_mhz: device.clock_info(Clock::Graphics).ok(),
229+
}
230+
}
231+
213232
fn read_device_snapshot(nvml: &Nvml, idx: u32) -> Option<NvLinkSnapshot> {
214233
let device = nvml.device_by_index(idx).ok()?;
215234
let gpu_name = device.name().unwrap_or_default();
@@ -262,6 +281,7 @@ fn read_device_snapshot(nvml: &Nvml, idx: u32) -> Option<NvLinkSnapshot> {
262281
Some(NvLinkSnapshot {
263282
gpu_index: idx,
264283
gpu_name,
284+
metrics: Some(read_gpu_metrics(&device)).filter(|m| !m.is_empty()),
265285
link_count,
266286
link_gbps: if total_speed_gbps > 0.0 {
267287
Some(total_speed_gbps)
@@ -440,6 +460,7 @@ mod tests {
440460
let snap = NvLinkSnapshot {
441461
gpu_index: 0,
442462
gpu_name: "test".to_string(),
463+
metrics: None,
443464
link_count: 4,
444465
link_gbps: None,
445466
tx_bytes: None,
@@ -494,6 +515,7 @@ mod tests {
494515
let snap = NvLinkSnapshot {
495516
gpu_index: 0,
496517
gpu_name: "test".to_string(),
518+
metrics: None,
497519
link_count: 0,
498520
link_gbps: None,
499521
tx_bytes: None,

‎src/tui/app.rs‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ pub struct XgmiThroughputMeta {
6060
pub gpu_name: String,
6161
pub active_links: u32,
6262
pub links: Vec<crate::xgmi::XgmiLinkSnapshot>,
63+
/// Live GPU health (util/vram/temp/power/clock) for the gauge strip
64+
/// and detail pane.
65+
pub metrics: Option<crate::gpu::GpuMetrics>,
6366
}
6467

6568
/// Per-GPU NVLink metadata. The TUI uses this to show per-link details
@@ -75,6 +78,9 @@ pub struct NvLinkThroughputMeta {
7578
pub gpu_name: String,
7679
pub active_links: u32,
7780
pub links: Vec<crate::nvlink::LinkSnapshot>,
81+
/// Live GPU health (util/vram/temp/power/clock) for the gauge strip
82+
/// and detail pane.
83+
pub metrics: Option<crate::gpu::GpuMetrics>,
7884
}
7985

8086
#[derive(Clone, Debug)]
@@ -401,13 +407,15 @@ const HISTORY_LEN: usize = 60;
401407
pub struct DeviceHistory {
402408
pub tx: Vec<f64>,
403409
pub rx: Vec<f64>,
410+
pub util: Vec<f64>,
404411
}
405412

406413
impl DeviceHistory {
407414
fn new() -> Self {
408415
Self {
409416
tx: Vec::with_capacity(HISTORY_LEN),
410417
rx: Vec::with_capacity(HISTORY_LEN),
418+
util: Vec::with_capacity(HISTORY_LEN),
411419
}
412420
}
413421

@@ -419,6 +427,13 @@ impl DeviceHistory {
419427
self.tx.push(tx);
420428
self.rx.push(rx);
421429
}
430+
431+
fn push_util(&mut self, util: f64) {
432+
if self.util.len() >= HISTORY_LEN {
433+
self.util.remove(0);
434+
}
435+
self.util.push(util);
436+
}
422437
}
423438

424439
#[derive(Clone)]
@@ -597,10 +612,20 @@ impl App {
597612

598613
fn update_history(&mut self) {
599614
for t in &self.throughputs {
600-
self.history
615+
let util = t
616+
.nvlink
617+
.as_ref()
618+
.and_then(|m| m.metrics.as_ref())
619+
.or_else(|| t.xgmi.as_ref().and_then(|m| m.metrics.as_ref()))
620+
.and_then(|m| m.util_pct);
621+
let entry = self
622+
.history
601623
.entry(t.dev_name.clone())
602-
.or_insert_with(DeviceHistory::new)
603-
.push(t.tx_gbps, t.rx_gbps);
624+
.or_insert_with(DeviceHistory::new);
625+
entry.push(t.tx_gbps, t.rx_gbps);
626+
if let Some(u) = util {
627+
entry.push_util(u as f64);
628+
}
604629
}
605630
}
606631

@@ -1207,6 +1232,7 @@ fn compute_nvlink_throughputs(
12071232
gpu_name: gpu.gpu_name.clone(),
12081233
active_links: active,
12091234
links,
1235+
metrics: gpu.metrics.clone(),
12101236
}),
12111237
xgmi: None,
12121238
class: DeviceClass::Nvlink,
@@ -1306,6 +1332,7 @@ fn compute_xgmi_throughputs(
13061332
gpu_name: gpu.gpu_name.clone(),
13071333
active_links: active,
13081334
links,
1335+
metrics: gpu.metrics.clone(),
13091336
}),
13101337
class: DeviceClass::Xgmi,
13111338
});
@@ -1339,6 +1366,7 @@ mod xgmi_tests {
13391366
link_gbps: Some(512.0 * active as f64),
13401367
correctable_errors: Some(0),
13411368
uncorrectable_errors: Some(0),
1369+
metrics: None,
13421370
links,
13431371
}
13441372
}
@@ -1494,6 +1522,7 @@ mod nvlink_tests {
14941522
let prev = vec![NvLinkSnapshot {
14951523
gpu_index: 0,
14961524
gpu_name: "H100".to_string(),
1525+
metrics: None,
14971526
link_count: 2,
14981527
link_gbps: Some(100.0),
14991528
tx_bytes: None,
@@ -1506,6 +1535,7 @@ mod nvlink_tests {
15061535
let curr = vec![NvLinkSnapshot {
15071536
gpu_index: 0,
15081537
gpu_name: "H100".to_string(),
1538+
metrics: None,
15091539
link_count: 3,
15101540
link_gbps: Some(150.0),
15111541
tx_bytes: None,
@@ -1581,6 +1611,7 @@ mod nvlink_tests {
15811611
let prev = vec![NvLinkSnapshot {
15821612
gpu_index: 0,
15831613
gpu_name: "H100".to_string(),
1614+
metrics: None,
15841615
link_count: 2,
15851616
link_gbps: Some(100.0),
15861617
tx_bytes: None,
@@ -1593,6 +1624,7 @@ mod nvlink_tests {
15931624
let curr = vec![NvLinkSnapshot {
15941625
gpu_index: 0,
15951626
gpu_name: "H100".to_string(),
1627+
metrics: None,
15961628
link_count: 2,
15971629
link_gbps: Some(100.0),
15981630
tx_bytes: None,
@@ -1649,6 +1681,7 @@ mod nvlink_tests {
16491681
let prev = vec![NvLinkSnapshot {
16501682
gpu_index: 0,
16511683
gpu_name: "H100".to_string(),
1684+
metrics: None,
16521685
link_count: 2,
16531686
link_gbps: Some(100.0),
16541687
tx_bytes: None,
@@ -1661,6 +1694,7 @@ mod nvlink_tests {
16611694
let curr = vec![NvLinkSnapshot {
16621695
gpu_index: 0,
16631696
gpu_name: "H100".to_string(),
1697+
metrics: None,
16641698
link_count: 2,
16651699
link_gbps: Some(100.0),
16661700
tx_bytes: None,
@@ -1692,6 +1726,7 @@ mod nvlink_tests {
16921726
let curr = vec![NvLinkSnapshot {
16931727
gpu_index: 1,
16941728
gpu_name: "A100".to_string(),
1729+
metrics: None,
16951730
link_count: 1,
16961731
link_gbps: Some(50.0),
16971732
tx_bytes: None,
@@ -1719,6 +1754,7 @@ mod nvlink_tests {
17191754
let prev = vec![NvLinkSnapshot {
17201755
gpu_index: 0,
17211756
gpu_name: "H100".to_string(),
1757+
metrics: None,
17221758
link_count: 2,
17231759
link_gbps: Some(100.0),
17241760
tx_bytes: None,
@@ -1728,6 +1764,7 @@ mod nvlink_tests {
17281764
let curr_first = vec![NvLinkSnapshot {
17291765
gpu_index: 0,
17301766
gpu_name: "H100".to_string(),
1767+
metrics: None,
17311768
link_count: 3,
17321769
link_gbps: Some(150.0),
17331770
tx_bytes: None,
@@ -1752,6 +1789,7 @@ mod nvlink_tests {
17521789
let curr_second = vec![NvLinkSnapshot {
17531790
gpu_index: 0,
17541791
gpu_name: "H100".to_string(),
1792+
metrics: None,
17551793
link_count: 3,
17561794
link_gbps: Some(150.0),
17571795
tx_bytes: None,
@@ -1791,6 +1829,7 @@ mod nvlink_tests {
17911829
let prev = vec![NvLinkSnapshot {
17921830
gpu_index: 0,
17931831
gpu_name: "H100".to_string(),
1832+
metrics: None,
17941833
link_count: 1,
17951834
link_gbps: Some(50.0),
17961835
tx_bytes: None,
@@ -1800,6 +1839,7 @@ mod nvlink_tests {
18001839
let curr = vec![NvLinkSnapshot {
18011840
gpu_index: 0,
18021841
gpu_name: "H100".to_string(),
1842+
metrics: None,
18031843
link_count: 1,
18041844
link_gbps: Some(50.0),
18051845
tx_bytes: None,
@@ -1838,6 +1878,7 @@ mod nvlink_tests {
18381878
let prev = vec![NvLinkSnapshot {
18391879
gpu_index: 2,
18401880
gpu_name: "B200".to_string(),
1881+
metrics: None,
18411882
link_count: 1,
18421883
link_gbps: Some(100.0),
18431884
tx_bytes: None,
@@ -1852,6 +1893,7 @@ mod nvlink_tests {
18521893
let curr = vec![NvLinkSnapshot {
18531894
gpu_index: 2,
18541895
gpu_name: "B200".to_string(),
1896+
metrics: None,
18551897
link_count: 1,
18561898
link_gbps: Some(100.0),
18571899
tx_bytes: None,
@@ -1878,6 +1920,7 @@ mod nvlink_tests {
18781920
let prev_a = vec![NvLinkSnapshot {
18791921
gpu_index: 0,
18801922
gpu_name: "H100".to_string(),
1923+
metrics: None,
18811924
link_count: 3,
18821925
link_gbps: Some(150.0),
18831926
tx_bytes: None,
@@ -1887,6 +1930,7 @@ mod nvlink_tests {
18871930
let curr_a = vec![NvLinkSnapshot {
18881931
gpu_index: 0,
18891932
gpu_name: "H100".to_string(),
1933+
metrics: None,
18901934
link_count: 3,
18911935
link_gbps: Some(150.0),
18921936
tx_bytes: None,
@@ -1906,6 +1950,7 @@ mod nvlink_tests {
19061950
let curr_b = vec![NvLinkSnapshot {
19071951
gpu_index: 0,
19081952
gpu_name: "H100".to_string(),
1953+
metrics: None,
19091954
link_count: 3,
19101955
link_gbps: Some(150.0),
19111956
tx_bytes: None,
@@ -1946,6 +1991,7 @@ mod nvlink_tests {
19461991
let prev = vec![NvLinkSnapshot {
19471992
gpu_index: 0,
19481993
gpu_name: "H100".to_string(),
1994+
metrics: None,
19491995
link_count: 2,
19501996
link_gbps: Some(100.0),
19511997
tx_bytes: Some(10_000_000_000),
@@ -1958,6 +2004,7 @@ mod nvlink_tests {
19582004
let curr = vec![NvLinkSnapshot {
19592005
gpu_index: 0,
19602006
gpu_name: "H100".to_string(),
2007+
metrics: None,
19612008
link_count: 2,
19622009
link_gbps: Some(100.0),
19632010
tx_bytes: Some(15_000_000_000),
@@ -2010,6 +2057,7 @@ mod nvlink_tests {
20102057
gpu_name: "H100".to_string(),
20112058
active_links: 2,
20122059
links: Vec::new(),
2060+
metrics: None,
20132061
}),
20142062
xgmi: None,
20152063
class: DeviceClass::Nvlink,
@@ -2065,6 +2113,7 @@ mod nvlink_tests {
20652113
gpu_name: "H100".to_string(),
20662114
active_links: 3,
20672115
links: Vec::new(),
2116+
metrics: None,
20682117
}),
20692118
xgmi: None,
20702119
class: DeviceClass::Nvlink,
@@ -2309,6 +2358,10 @@ mod apply_snapshot_tests {
23092358
let gpu = crate::nvlink::NvLinkSnapshot {
23102359
gpu_index: 0,
23112360
gpu_name: "test-gpu".to_string(),
2361+
metrics: Some(crate::gpu::GpuMetrics {
2362+
util_pct: Some(42),
2363+
..Default::default()
2364+
}),
23122365
link_count: 0,
23132366
link_gbps: None,
23142367
tx_bytes: Some(tx_bytes),
@@ -2325,6 +2378,17 @@ mod apply_snapshot_tests {
23252378
}
23262379
}
23272380

2381+
#[test]
2382+
fn gpu_util_flows_into_history() {
2383+
let mut app = App::new();
2384+
let t0 = Instant::now();
2385+
app.apply_snapshot(gpu_snapshot(0, t0));
2386+
app.apply_snapshot(gpu_snapshot(1_000, t0 + Duration::from_secs(2)));
2387+
let h = app.history.get("nvidia0").expect("history for nvidia0");
2388+
assert_eq!(h.util.len(), 2);
2389+
assert_eq!(h.util[0], 42.0);
2390+
}
2391+
23282392
#[test]
23292393
fn rates_use_per_subsystem_read_timestamps() {
23302394
// A slow pass must not skew a subsystem read later in it: the GPU

0 commit comments

Comments
 (0)