Skip to content

feat: per-GPU health gauges on NVLink/XGMI tabs and capacity-based bar colors #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
crazyguitar merged 3 commits into from
Jul 12, 2026
Merged

feat: per-GPU health gauges on NVLink/XGMI tabs and capacity-based bar colors #36

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a7523d3
add gpu metrics
crazyguitar Jul 11, 2026
9ec99b3
address issues
crazyguitar Jul 11, 2026
1a58db7
updte ui
crazyguitar Jul 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/gpu.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//! Shared per-GPU health metrics, filled by the NVML (nvlink) and amdsmi
//! (xgmi) readers and rendered as the GPU tabs' gauge strip.

/// Live values; each field is None when its read failed or is unsupported.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct GpuMetrics {
pub util_pct: Option<u32>,
pub vram_used_mb: Option<u64>,
pub vram_total_mb: Option<u64>,
pub temp_c: Option<i64>,
pub power_w: Option<f64>,
pub clock_mhz: Option<u32>,
}
1 change: 1 addition & 0 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
mod gpu;
mod net;
mod netlink;
mod nvlink;
Expand Down
22 changes: 22 additions & 0 deletions src/nvlink.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use std::collections::HashMap;
use std::io;
use std::sync::{LazyLock, Mutex, OnceLock};

use nvml_wrapper::enum_wrappers::device::{Clock, TemperatureSensor};
use nvml_wrapper::enum_wrappers::nv_link::{ErrorCounter, IntDeviceType};
use nvml_wrapper::enums::device::SampleValue;
use nvml_wrapper::structs::device::FieldId;
Expand Down Expand Up @@ -132,6 +133,8 @@ fn link_meta(
pub struct NvLinkSnapshot {
pub gpu_index: u32,
pub gpu_name: String,
/// Live GPU health (util/vram/temp/power/clock) for the gauge strip.
pub metrics: Option<crate::gpu::GpuMetrics>,
/// Total number of NVLink ports exposed by the GPU (active + inactive).
pub link_count: u32,
/// Negotiated aggregate speed in Gbps, derived from the per-link
Expand Down Expand Up @@ -210,6 +213,22 @@ pub fn read_all_nvlink_stats() -> io::Result<Vec<NvLinkSnapshot>> {
Ok(snapshots)
}

/// Live GPU health via safe NVML wrappers; absent readings become None.
fn read_gpu_metrics(device: &nvml_wrapper::Device<'_>) -> crate::gpu::GpuMetrics {
let mem = device.memory_info().ok();
crate::gpu::GpuMetrics {
util_pct: device.utilization_rates().ok().map(|u| u.gpu),
vram_used_mb: mem.as_ref().map(|m| m.used / (1024 * 1024)),
vram_total_mb: mem.as_ref().map(|m| m.total / (1024 * 1024)),
temp_c: device
.temperature(TemperatureSensor::Gpu)
.ok()
.map(|t| t as i64),
power_w: device.power_usage().ok().map(|mw| mw as f64 / 1000.0),
clock_mhz: device.clock_info(Clock::Graphics).ok(),
}
}

fn read_device_snapshot(nvml: &Nvml, idx: u32) -> Option<NvLinkSnapshot> {
let device = nvml.device_by_index(idx).ok()?;
let gpu_name = device.name().unwrap_or_default();
Expand Down Expand Up @@ -262,6 +281,7 @@ fn read_device_snapshot(nvml: &Nvml, idx: u32) -> Option<NvLinkSnapshot> {
Some(NvLinkSnapshot {
gpu_index: idx,
gpu_name,
metrics: Some(read_gpu_metrics(&device)),
Comment thread
crazyguitar marked this conversation as resolved.
Outdated
link_count,
link_gbps: if total_speed_gbps > 0.0 {
Some(total_speed_gbps)
Expand Down Expand Up @@ -440,6 +460,7 @@ mod tests {
let snap = NvLinkSnapshot {
gpu_index: 0,
gpu_name: "test".to_string(),
metrics: None,
link_count: 4,
link_gbps: None,
tx_bytes: None,
Expand Down Expand Up @@ -494,6 +515,7 @@ mod tests {
let snap = NvLinkSnapshot {
gpu_index: 0,
gpu_name: "test".to_string(),
metrics: None,
link_count: 0,
link_gbps: None,
tx_bytes: None,
Expand Down
70 changes: 67 additions & 3 deletions src/tui/app.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ pub struct XgmiThroughputMeta {
pub gpu_name: String,
pub active_links: u32,
pub links: Vec<crate::xgmi::XgmiLinkSnapshot>,
/// Live GPU health (util/vram/temp/power/clock) for the gauge strip
/// and detail pane.
pub metrics: Option<crate::gpu::GpuMetrics>,
}

/// Per-GPU NVLink metadata. The TUI uses this to show per-link details
Expand All @@ -75,6 +78,9 @@ pub struct NvLinkThroughputMeta {
pub gpu_name: String,
pub active_links: u32,
pub links: Vec<crate::nvlink::LinkSnapshot>,
/// Live GPU health (util/vram/temp/power/clock) for the gauge strip
/// and detail pane.
pub metrics: Option<crate::gpu::GpuMetrics>,
}

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -401,13 +407,15 @@ const HISTORY_LEN: usize = 60;
pub struct DeviceHistory {
pub tx: Vec<f64>,
pub rx: Vec<f64>,
pub util: Vec<f64>,
}

impl DeviceHistory {
fn new() -> Self {
Self {
tx: Vec::with_capacity(HISTORY_LEN),
rx: Vec::with_capacity(HISTORY_LEN),
util: Vec::with_capacity(HISTORY_LEN),
}
}

Expand All @@ -419,6 +427,13 @@ impl DeviceHistory {
self.tx.push(tx);
self.rx.push(rx);
}

fn push_util(&mut self, util: f64) {
if self.util.len() >= HISTORY_LEN {
self.util.remove(0);
}
self.util.push(util);
}
}

#[derive(Clone)]
Expand Down Expand Up @@ -597,10 +612,20 @@ impl App {

fn update_history(&mut self) {
for t in &self.throughputs {
self.history
let util = t
.nvlink
.as_ref()
.and_then(|m| m.metrics.as_ref())
.or_else(|| t.xgmi.as_ref().and_then(|m| m.metrics.as_ref()))
.and_then(|m| m.util_pct);
let entry = self
.history
.entry(t.dev_name.clone())
.or_insert_with(DeviceHistory::new)
.push(t.tx_gbps, t.rx_gbps);
.or_insert_with(DeviceHistory::new);
entry.push(t.tx_gbps, t.rx_gbps);
if let Some(u) = util {
entry.push_util(u as f64);
}
}
}

Expand Down Expand Up @@ -1207,6 +1232,7 @@ fn compute_nvlink_throughputs(
gpu_name: gpu.gpu_name.clone(),
active_links: active,
links,
metrics: gpu.metrics.clone(),
}),
xgmi: None,
class: DeviceClass::Nvlink,
Expand Down Expand Up @@ -1306,6 +1332,7 @@ fn compute_xgmi_throughputs(
gpu_name: gpu.gpu_name.clone(),
active_links: active,
links,
metrics: gpu.metrics.clone(),
}),
class: DeviceClass::Xgmi,
});
Expand Down Expand Up @@ -1339,6 +1366,7 @@ mod xgmi_tests {
link_gbps: Some(512.0 * active as f64),
correctable_errors: Some(0),
uncorrectable_errors: Some(0),
metrics: None,
links,
}
}
Expand Down Expand Up @@ -1494,6 +1522,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: None,
Expand All @@ -1506,6 +1535,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 3,
link_gbps: Some(150.0),
tx_bytes: None,
Expand Down Expand Up @@ -1581,6 +1611,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: None,
Expand All @@ -1593,6 +1624,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: None,
Expand Down Expand Up @@ -1649,6 +1681,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: None,
Expand All @@ -1661,6 +1694,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: None,
Expand Down Expand Up @@ -1692,6 +1726,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 1,
gpu_name: "A100".to_string(),
metrics: None,
link_count: 1,
link_gbps: Some(50.0),
tx_bytes: None,
Expand Down Expand Up @@ -1719,6 +1754,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: None,
Expand All @@ -1728,6 +1764,7 @@ mod nvlink_tests {
let curr_first = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 3,
link_gbps: Some(150.0),
tx_bytes: None,
Expand All @@ -1752,6 +1789,7 @@ mod nvlink_tests {
let curr_second = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 3,
link_gbps: Some(150.0),
tx_bytes: None,
Expand Down Expand Up @@ -1791,6 +1829,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 1,
link_gbps: Some(50.0),
tx_bytes: None,
Expand All @@ -1800,6 +1839,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 1,
link_gbps: Some(50.0),
tx_bytes: None,
Expand Down Expand Up @@ -1838,6 +1878,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 2,
gpu_name: "B200".to_string(),
metrics: None,
link_count: 1,
link_gbps: Some(100.0),
tx_bytes: None,
Expand All @@ -1852,6 +1893,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 2,
gpu_name: "B200".to_string(),
metrics: None,
link_count: 1,
link_gbps: Some(100.0),
tx_bytes: None,
Expand All @@ -1878,6 +1920,7 @@ mod nvlink_tests {
let prev_a = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 3,
link_gbps: Some(150.0),
tx_bytes: None,
Expand All @@ -1887,6 +1930,7 @@ mod nvlink_tests {
let curr_a = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 3,
link_gbps: Some(150.0),
tx_bytes: None,
Expand All @@ -1906,6 +1950,7 @@ mod nvlink_tests {
let curr_b = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 3,
link_gbps: Some(150.0),
tx_bytes: None,
Expand Down Expand Up @@ -1946,6 +1991,7 @@ mod nvlink_tests {
let prev = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: Some(10_000_000_000),
Expand All @@ -1958,6 +2004,7 @@ mod nvlink_tests {
let curr = vec![NvLinkSnapshot {
gpu_index: 0,
gpu_name: "H100".to_string(),
metrics: None,
link_count: 2,
link_gbps: Some(100.0),
tx_bytes: Some(15_000_000_000),
Expand Down Expand Up @@ -2010,6 +2057,7 @@ mod nvlink_tests {
gpu_name: "H100".to_string(),
active_links: 2,
links: Vec::new(),
metrics: None,
}),
xgmi: None,
class: DeviceClass::Nvlink,
Expand Down Expand Up @@ -2065,6 +2113,7 @@ mod nvlink_tests {
gpu_name: "H100".to_string(),
active_links: 3,
links: Vec::new(),
metrics: None,
}),
xgmi: None,
class: DeviceClass::Nvlink,
Expand Down Expand Up @@ -2309,6 +2358,10 @@ mod apply_snapshot_tests {
let gpu = crate::nvlink::NvLinkSnapshot {
gpu_index: 0,
gpu_name: "test-gpu".to_string(),
metrics: Some(crate::gpu::GpuMetrics {
util_pct: Some(42),
..Default::default()
}),
link_count: 0,
link_gbps: None,
tx_bytes: Some(tx_bytes),
Expand All @@ -2325,6 +2378,17 @@ mod apply_snapshot_tests {
}
}

#[test]
fn gpu_util_flows_into_history() {
let mut app = App::new();
let t0 = Instant::now();
app.apply_snapshot(gpu_snapshot(0, t0));
app.apply_snapshot(gpu_snapshot(1_000, t0 + Duration::from_secs(2)));
let h = app.history.get("nvidia0").expect("history for nvidia0");
assert_eq!(h.util.len(), 2);
assert_eq!(h.util[0], 42.0);
}

#[test]
fn rates_use_per_subsystem_read_timestamps() {
// A slow pass must not skew a subsystem read later in it: the GPU
Expand Down
Loading