Skip to content

Commit 982b418

Browse files
committed
address race condition
Signed-off-by: chang-ning <spiderpower02@gmail.com>
1 parent bef9130 commit 982b418

3 files changed

Lines changed: 20 additions & 7 deletions

File tree

src/nvlink.rs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,13 +167,20 @@ const LINK_SPEED_FIELD_IDS: [u32; 12] = [
167167

168168
/// dlopen + nvmlInit once, kept for the process lifetime (nvmlShutdown is
169169
/// never called; process exit tears it down). A failed init retries on the
170-
/// next poll so a driver that loads after startup is still picked up.
170+
/// next poll; the lock serializes init so racing callers can never build
171+
/// and drop a second Nvml instance.
171172
fn nvml() -> Option<&'static Nvml> {
172173
static INSTANCE: OnceLock<Nvml> = OnceLock::new();
174+
static INIT: Mutex<()> = Mutex::new(());
173175
if let Some(n) = INSTANCE.get() {
174176
return Some(n);
175177
}
176-
Nvml::init().ok().map(|n| INSTANCE.get_or_init(|| n))
178+
let _guard = INIT.lock().unwrap_or_else(|e| e.into_inner());
179+
if let Some(n) = INSTANCE.get() {
180+
return Some(n);
181+
}
182+
let n = Nvml::init().ok()?;
183+
Some(INSTANCE.get_or_init(|| n))
177184
}
178185

179186
/// Read NVLink statistics for every GPU in the system.

src/sampler.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ impl Sampler {
7272
pub fn spawn(interval: Duration) -> Self {
7373
let shared = Arc::new(Shared {
7474
slot: Mutex::new(None),
75-
interval_ms: AtomicU64::new(interval.as_millis() as u64),
75+
interval_ms: AtomicU64::new(interval_to_ms(interval)),
7676
stop: AtomicBool::new(false),
7777
died: Mutex::new(None),
7878
});
@@ -102,7 +102,7 @@ impl Sampler {
102102
pub fn set_interval(&self, interval: Duration) {
103103
self.shared
104104
.interval_ms
105-
.store(interval.as_millis() as u64, Ordering::Relaxed);
105+
.store(interval_to_ms(interval), Ordering::Relaxed);
106106
}
107107

108108
/// Ask the thread to exit. Detach, never join: a thread stuck inside a
@@ -120,6 +120,12 @@ impl Drop for Sampler {
120120
}
121121
}
122122

123+
/// Interval as stored millis, clamped to 1ms: a zero value would make the
124+
/// sampling loop spin with no sleep at all.
125+
fn interval_to_ms(interval: Duration) -> u64 {
126+
(interval.as_millis() as u64).max(1)
127+
}
128+
123129
/// Render a `catch_unwind` payload (typically &str or String) for the UI.
124130
fn panic_message(payload: Box<dyn std::any::Any + Send>) -> String {
125131
if let Some(s) = payload.downcast_ref::<&str>() {

src/trace.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ impl Recorder {
3838
}
3939

4040
/// Record one interval's metrics, timestamped relative to record start.
41-
/// `taken_at` is when the counters were read (sampler-side), so trace
42-
/// spacing is immune to UI queue and event-poll latency; a sample taken
43-
/// before recording started saturates to ts 0.
41+
/// `taken_at` is the sampling pass start (sampler-side), so trace spacing
42+
/// is immune to UI queue and event-poll latency; a sample taken before
43+
/// recording started saturates to ts 0.
4444
pub fn push(&mut self, taken_at: Instant, ports: Vec<PortMetrics>) {
4545
let ts_us = taken_at.saturating_duration_since(self.start).as_micros() as u64;
4646
self.samples.push(TraceSample { ts_us, ports });

0 commit comments

Comments
 (0)