@@ -41,6 +41,9 @@ type component struct {
4141}
4242
4343func New (gpudInstance * components.GPUdInstance ) (components.Component , error ) {
44+ if gpudInstance == nil {
45+ return nil , errors .New ("gpud instance is nil" )
46+ }
4447 cctx , ccancel := context .WithCancel (gpudInstance .RootCtx )
4548 c := & component {
4649 ctx : cctx ,
@@ -51,6 +54,7 @@ func New(gpudInstance *components.GPUdInstance) (components.Component, error) {
5154 nvmlInstance : gpudInstance .NVMLInstance ,
5255 getTemperatureFunc : GetTemperature ,
5356 }
57+
5458 return c , nil
5559}
5660
@@ -150,7 +154,12 @@ func (c *component) Check() components.CheckResult {
150154 return cr
151155 }
152156
153- tempThresholdExceeded := make ([]string , 0 )
157+ marginThreshold := GetDefaultThresholds ()
158+
159+ gpuTempThresholdExceeded := make ([]string , 0 )
160+ hbmTempThresholdExceeded := make ([]string , 0 )
161+ marginThresholdExceeded := make ([]string , 0 )
162+
154163 devs := c .nvmlInstance .Devices ()
155164 for uuid , dev := range devs {
156165 temp , err := c .getTemperatureFunc (uuid , dev )
@@ -182,22 +191,60 @@ func (c *component) Check() components.CheckResult {
182191 log .Logger .Warnw (cr .reason , "uuid" , uuid , "error" , cr .err )
183192 return cr
184193 }
185- cr .Temperatures = append (cr .Temperatures , temp )
186194
187- // same logic as DCGM "VerifyHBMTemperature" that alerts "DCGM_FR_TEMP_VIOLATION",
188- // use "DCGM_FI_DEV_MEM_MAX_OP_TEMP" to get the max HBM temperature threshold "NVML_TEMPERATURE_THRESHOLD_MEM_MAX"
189- if temp .ThresholdCelsiusMemMax > 0 && temp .CurrentCelsiusGPUCore > temp .ThresholdCelsiusMemMax {
190- tempThresholdExceeded = append (tempThresholdExceeded ,
191- fmt .Sprintf ("%s current temperature is %d °C exceeding the HBM temperature threshold %d °C" ,
195+ // nvmlDeviceGetMarginTemperature reports the thermal margin to the nearest slowdown
196+ // threshold as defined by NVML. NVML does not specify GPU core vs HBM; it is
197+ // whichever slowdown threshold is nearest (driver-defined).
198+ // ref. https://docs.nvidia.com/deploy/nvml-api/group__nvmlDeviceQueries.html#group__nvmlDeviceQueries_1g42db93dc04fc99d253eadc2037a5232d
199+ if temp .ThresholdCelsiusSlowdown > 0 && temp .MarginTemperatureSupported {
200+ // margin left less than the threshold, indicating the GPU is approaching the slowdown threshold
201+ // e.g.,
202+ // 5°C margin left means the GPU is approaching the slowdown threshold at 82°C
203+ // when the slowdown threshold is 87°C
204+ if temp .ThresholdCelsiusSlowdownMargin <= marginThreshold .CelsiusSlowdownMargin {
205+ // e.g.,
206+ // 5°C margin left, when we set the threshold to 10°C
207+ // so that we can alert in advance before it's too late,
208+ // before the slowdown already happened
209+ marginThresholdExceeded = append (marginThresholdExceeded ,
210+ fmt .Sprintf ("%s has only %d °C margin left to slowdown (threshold %d °C)" ,
211+ uuid ,
212+ temp .ThresholdCelsiusSlowdownMargin ,
213+ marginThreshold .CelsiusSlowdownMargin ,
214+ ),
215+ )
216+ }
217+ }
218+
219+ if temp .ThresholdCelsiusGPUMax > 0 && temp .CurrentCelsiusGPUCore > temp .ThresholdCelsiusGPUMax {
220+ gpuTempThresholdExceeded = append (gpuTempThresholdExceeded ,
221+ fmt .Sprintf ("%s current temperature is %d °C exceeding the threshold %d °C" ,
192222 uuid ,
193223 temp .CurrentCelsiusGPUCore ,
224+ temp .ThresholdCelsiusGPUMax ,
225+ ),
226+ )
227+ }
228+
229+ // same logic as DCGM "VerifyHBMTemperature" that alerts "DCGM_FR_TEMP_VIOLATION",
230+ // use "DCGM_FI_DEV_MEM_MAX_OP_TEMP" to get the max HBM temperature threshold "NVML_TEMPERATURE_THRESHOLD_MEM_MAX"
231+ if temp .ThresholdCelsiusMemMax > 0 && temp .HBMTemperatureSupported && temp .CurrentCelsiusHBM > temp .ThresholdCelsiusMemMax {
232+ hbmTempThresholdExceeded = append (hbmTempThresholdExceeded ,
233+ fmt .Sprintf ("%s HBM temperature is %d °C exceeding the threshold %d °C" ,
234+ uuid ,
235+ temp .CurrentCelsiusHBM ,
194236 temp .ThresholdCelsiusMemMax ,
195237 ),
196238 )
197239 }
198240
241+ cr .Temperatures = append (cr .Temperatures , temp )
242+
199243 metricCurrentCelsius .With (prometheus.Labels {"uuid" : uuid }).Set (float64 (temp .CurrentCelsiusGPUCore ))
244+ metricCurrentHBMCelsius .With (prometheus.Labels {"uuid" : uuid }).Set (float64 (temp .CurrentCelsiusHBM ))
200245 metricThresholdSlowdownCelsius .With (prometheus.Labels {"uuid" : uuid }).Set (float64 (temp .ThresholdCelsiusSlowdown ))
246+ metricThresholdMemMaxCelsius .With (prometheus.Labels {"uuid" : uuid }).Set (float64 (temp .ThresholdCelsiusMemMax ))
247+ metricMarginCelsius .With (prometheus.Labels {"uuid" : uuid }).Set (float64 (temp .ThresholdCelsiusSlowdownMargin ))
201248
202249 slowdownPct , err := temp .GetUsedPercentSlowdown ()
203250 if err != nil {
@@ -208,14 +255,26 @@ func (c *component) Check() components.CheckResult {
208255 return cr
209256 }
210257 metricSlowdownUsedPercent .With (prometheus.Labels {"uuid" : uuid }).Set (slowdownPct )
258+
259+ memMaxPct := 0.0
260+ if temp .ThresholdCelsiusMemMax > 0 && temp .HBMTemperatureSupported {
261+ memMaxPct = float64 (temp .CurrentCelsiusHBM ) / float64 (temp .ThresholdCelsiusMemMax ) * 100
262+ }
263+ metricMemMaxUsedPercent .With (prometheus.Labels {"uuid" : uuid }).Set (memMaxPct )
211264 }
212265
213- if len (tempThresholdExceeded ) == 0 {
266+ if len (marginThresholdExceeded ) > 0 {
267+ cr .health = apiv1 .HealthStateTypeDegraded
268+ cr .reason = fmt .Sprintf ("margin threshold exceeded: %s" , strings .Join (marginThresholdExceeded , ", " ))
269+ } else if len (gpuTempThresholdExceeded ) > 0 {
270+ cr .health = apiv1 .HealthStateTypeDegraded
271+ cr .reason = fmt .Sprintf ("GPU temperature anomalies detected: %s" , strings .Join (gpuTempThresholdExceeded , ", " ))
272+ } else if len (hbmTempThresholdExceeded ) > 0 {
273+ cr .health = apiv1 .HealthStateTypeDegraded
274+ cr .reason = fmt .Sprintf ("HBM temperature anomalies detected: %s" , strings .Join (hbmTempThresholdExceeded , ", " ))
275+ } else {
214276 cr .health = apiv1 .HealthStateTypeHealthy
215277 cr .reason = fmt .Sprintf ("all %d GPU(s) were checked, no temperature issue found" , len (devs ))
216- } else {
217- cr .health = apiv1 .HealthStateTypeUnhealthy
218- cr .reason = fmt .Sprintf ("exceeded HBM temperature thresholds: %s" , strings .Join (tempThresholdExceeded , ", " ))
219278 }
220279
221280 return cr
@@ -253,14 +312,24 @@ func (cr *checkResult) String() string {
253312
254313 buf := bytes .NewBuffer (nil )
255314 table := tablewriter .NewWriter (buf )
256- table .SetHeader ([]string {"GPU UUID" , "GPU Bus ID" , "Current temp" , "HBM temp threshold" , "Used % " })
315+ table .SetHeader ([]string {"GPU UUID" , "GPU Bus ID" , "GPU temp" , "HBM temp" , "HBM temp threshold""HBM used %" , "Margin to slowdown "
257316 for _ , temp := range cr .Temperatures {
317+ hbmTemp := "n/a"
318+ if temp .HBMTemperatureSupported {
319+ hbmTemp = fmt .Sprintf ("%d °C" , temp .CurrentCelsiusHBM )
320+ }
321+ marginTemp := "n/a"
322+ if temp .MarginTemperatureSupported {
323+ marginTemp = fmt .Sprintf ("%d °C" , temp .ThresholdCelsiusSlowdownMargin )
324+ }
258325 table .Append ([]string {
259326 temp .UUID ,
260327 temp .BusID ,
261328 fmt .Sprintf ("%d °C" , temp .CurrentCelsiusGPUCore ),
329+ hbmTemp ,
262330 fmt .Sprintf ("%d °C" , temp .ThresholdCelsiusMemMax ),
263331 fmt .Sprintf ("%s %%" , temp .UsedPercentMemMax ),
332+ marginTemp ,
264333 })
265334 }
266335 table .Render ()
0 commit comments