fix: modified gpu-health-monitor to correctly categorize for fatal and non-fatal errorcodes per healthwatch (#972)

Author: deesharma24

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/configmap.yaml

@@ -35,19 +35,4 @@ data:
     [cli]
     EnabledEventProcessors = PlatformConnectorEventProcessor
 
-    [DCGMHealthConditionsCategorizationMapping]
-    DCGM_HEALTH_WATCH_THERMAL=NonFatal
-    DCGM_HEALTH_WATCH_POWER=NonFatal
-    DCGM_HEALTH_WATCH_NVLINK=Fatal
-    DCGM_HEALTH_WATCH_SM=Fatal
-    DCGM_HEALTH_WATCH_MEM=Fatal
-    DCGM_HEALTH_WATCH_INFOROM=Fatal
-    DCGM_HEALTH_WATCH_MCU=Fatal
-    DCGM_HEALTH_WATCH_DRIVER=Fatal
-    DCGM_HEALTH_WATCH_NVSWITCH_FATAL=Fatal
-    DCGM_HEALTH_WATCH_NVSWITCH_NONFATAL=NonFatal
-    DCGM_HEALTH_WATCH_PCIE=Fatal
-    DCGM_HEALTH_WATCH_PMU=Fatal
-    DCGM_HEALTH_WATCH_ALL=Fatal
-
 {{ (.Files.Glob "files/dcgmerrorsmapping.csv").AsConfig | indent 2 }}

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-3.x.yaml

@@ -27,10 +27,11 @@ spec:
       {{- include "gpu-health-monitor.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
+        {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
       labels:
         {{- include "gpu-health-monitor.selectorLabels" . | nindent 8 }}
     spec:

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-4.x.yaml

@@ -27,10 +27,11 @@ spec:
       {{- include "gpu-health-monitor.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
+        {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
       labels:
         {{- include "gpu-health-monitor.selectorLabels" . | nindent 8 }}
     spec:

distros/kubernetes/nvsentinel/templates/daemonset.yaml

@@ -29,10 +29,11 @@ spec:
       {{- include "nvsentinel.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.platformConnector.podAnnotations }}
       annotations:
+        {{- with .Values.platformConnector.podAnnotations }}
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
       labels:
         {{- include "nvsentinel.selectorLabels" . | nindent 8 }}
     spec:

health-monitors/gpu-health-monitor/gpu_health_monitor/cli.py

@@ -32,7 +32,6 @@ def _init_event_processor(
     exit: Event,
     dcgm_errors_info_dict: dict[str, str],
     state_file_path: str,
-    dcgm_health_conditions_categorization_mapping_config: dict[str, str],
     metadata_path: str,
     processing_strategy: platformconnector_pb2.ProcessingStrategy,
 ):
@@ -45,7 +44,6 @@ def _init_event_processor(
                 exit=exit,
                 dcgm_errors_info_dict=dcgm_errors_info_dict,
                 state_file_path=state_file_path,
-                dcgm_health_conditions_categorization_mapping_config=dcgm_health_conditions_categorization_mapping_config,
                 metadata_path=metadata_path,
                 processing_strategy=processing_strategy,
             )
@@ -106,7 +104,6 @@ def cli(
         sys.exit(1)
 
     dcgm_errors_info_dict: dict[str, str] = {}
-    dcgm_health_conditions_categorization_mapping_config = config["DCGMHealthConditionsCategorizationMapping"]
 
     # Initialize structured JSON logging
     # Version is read from package metadata (set at build time via poetry version)
@@ -143,7 +140,6 @@ def cli(
                 exit,
                 dcgm_errors_info_dict,
                 state_file_path,
-                dcgm_health_conditions_categorization_mapping_config,
                 metadata_path,
                 processing_strategy_value,
             )

health-monitors/gpu-health-monitor/gpu_health_monitor/logger.py

@@ -99,6 +99,7 @@ def set_default_structured_logger_with_level(module: str, version: str, level: s
                 structlog.stdlib.ExtraAdder(),
                 structlog.processors.TimeStamper(fmt="iso"),
                 inject_module_version,
+                structlog.processors.format_exc_info,
             ],
             processors=[
                 structlog.stdlib.ProcessorFormatter.remove_processors_meta,

health-monitors/gpu-health-monitor/gpu_health_monitor/platform_connector/metrics.py

@@ -30,6 +30,6 @@
 
 dcgm_health_active_events = Gauge(
     "dcgm_health_active_events",
-    "Total number of active health events at any given time by severity",
-    labelnames=["event_type", "gpu_id", "severity"],
+    "Active health events by watch type and GPU",
+    labelnames=["event_type", "gpu_id"],
 )

health-monitors/gpu-health-monitor/gpu_health_monitor/platform_connector/platform_connector.py

@@ -33,9 +33,12 @@
 
 
 @dataclasses.dataclass
-class CachedEntityState:
-    isFatal: bool
-    isHealthy: bool
+class EntityCacheEntry:
+    active_errors: set[str] = dataclasses.field(default_factory=set)
+
+    @property
+    def is_healthy(self) -> bool:
+        return not self.active_errors
 
 
 class PlatformConnectorEventProcessor(dcgmtypes.CallbackInterface):
@@ -46,7 +49,6 @@ def __init__(
         exit: Event,
         dcgm_errors_info_dict: dict[str, str],
         state_file_path: str,
-        dcgm_health_conditions_categorization_mapping_config: dict[str, str],
         metadata_path: str,
         processing_strategy: platformconnector_pb2.ProcessingStrategy,
     ) -> None:
@@ -60,8 +62,7 @@ def __init__(
         self.state_file_path = state_file_path
         self.node_bootid_path = "/proc/sys/kernel/random/boot_id"
         self.old_bootid = self.read_old_system_bootid_from_state_file()
-        self.entity_cache: dict[str, CachedEntityState] = {}
-        self.dcgm_health_conditions_categorization_mapping_config = dcgm_health_conditions_categorization_mapping_config
+        self.entity_cache: dict[str, EntityCacheEntry] = {}
         self._metadata_reader = MetadataReader(metadata_path)
         self._processing_strategy = processing_strategy
 
@@ -94,7 +95,8 @@ def clear_dcgm_connectivity_failure(self, timestamp: Timestamp) -> None:
         check_name = "GpuDcgmConnectivityFailure"
 
         key = self._build_cache_key(check_name, "DCGM", "ALL")
-        if key not in self.entity_cache or not self.entity_cache[key].isHealthy:
+        entry = self.entity_cache.get(key)
+        if entry is None or not entry.is_healthy:
             event_metadata = {}
             chassis_serial = self._metadata_reader.get_chassis_serial()
             if chassis_serial:
@@ -121,10 +123,9 @@ def clear_dcgm_connectivity_failure(self, timestamp: Timestamp) -> None:
         if len(health_events):
             try:
                 if self.send_health_event_with_retries(health_events):
-                    # Only update cache after successful send
-                    self.entity_cache[key] = CachedEntityState(isFatal=False, isHealthy=True)
-                    log.info(f"Updated cache for key {key} after successful send")
-                    metrics.dcgm_health_active_events.labels(event_type=check_name, gpu_id="", severity="fatal").set(0)
+                    self.entity_cache[key] = EntityCacheEntry()
+                    log.info(f"Updated cache for key {key} with value {self.entity_cache[key]} after successful send")
+                    metrics.dcgm_health_active_events.labels(event_type=check_name, gpu_id="").set(0)
             except Exception as e:
                 log.error(f"Exception while sending DCGM connectivity restored events: {e}")
                 raise
@@ -142,8 +143,8 @@ def health_event_occurred(self, health_details: dict[str, dcgmtypes.HealthDetail
 
             health_events = []
             # Collect pending cache and metric updates to apply only after successful send
-            pending_cache_updates: dict[str, CachedEntityState] = {}
-            pending_metric_updates: list[tuple[str, int, str, int]] = []  # (event_type, gpu_id, severity, value)
+            pending_cache_updates: dict[str, EntityCacheEntry] = {}
+            pending_metric_updates: list[tuple[str, int, int]] = []  # (event_type, gpu_id, value)
 
             for watch_name, details in health_details.items():
                 check_name = self._convert_dcgm_watch_name_to_check_name(watch_name)
@@ -178,27 +179,16 @@ def health_event_occurred(self, health_details: dict[str, dcgmtypes.HealthDetail
 
                         entities_impacted_supports_component_reset = pci_address and gpu_uuid
 
+                        recommended_action = self.get_recommended_action_from_dcgm_error_map(failure_details.code)
+                        isHealthy = False
+                        isFatal = recommended_action != platformconnector_pb2.NONE
                         key = self._build_cache_key(check_name, entity.entityType, entity.entityValue)
-                        isFatal = False
-                        isHealthy = True
-                        if details.status == dcgmtypes.HealthStatus.PASS:
-                            isFatal = False
-                            isHealthy = True
-                        else:
-                            isFatal = (
-                                False
-                                if self.dcgm_health_conditions_categorization_mapping_config[watch_name] == "NonFatal"
-                                else True
-                            )
-                            isHealthy = False
-                        if (
-                            key not in self.entity_cache
-                            or self.entity_cache[key].isFatal != isFatal
-                            or self.entity_cache[key].isHealthy != isHealthy
-                        ):
-                            # Defer cache update until after successful send
-                            pending_cache_updates[key] = CachedEntityState(isFatal=isFatal, isHealthy=isHealthy)
-                            recommended_action = self.get_recommended_action_from_dcgm_error_map(failure_details.code)
+
+                        entry = self.entity_cache.get(key)
+                        if entry is None or failure_details.code not in entry.active_errors:
+                            existing_errors = set(entry.active_errors) if entry else set()
+                            existing_errors.add(failure_details.code)
+                            pending_cache_updates[key] = EntityCacheEntry(active_errors=existing_errors)
 
                             # The COMPONENT_RESET recommended action requires that the GPU_UUID is present on the
                             # unhealthy HealthEvent. Sending an event with COMPONENT_RESET that is missing the GPU_UUID
@@ -244,13 +234,7 @@ def health_event_occurred(self, health_details: dict[str, dcgmtypes.HealthDetail
                                     processingStrategy=self._processing_strategy,
                                 )
                             )
-                            severity = (
-                                "non_fatal"
-                                if self.dcgm_health_conditions_categorization_mapping_config[watch_name] == "NonFatal"
-                                else "fatal"
-                            )
-                            # Defer metric update until after successful send
-                            pending_metric_updates.append((check_name, gpu_id, severity, 1))
+                            pending_metric_updates.append((check_name, gpu_id, 1))
                     else:
 
                         entity = platformconnector_pb2.Entity(entityType=self._component_class, entityValue=str(gpu_id))
@@ -270,63 +254,49 @@ def health_event_occurred(self, health_details: dict[str, dcgmtypes.HealthDetail
                             )
 
                         key = self._build_cache_key(check_name, entity.entityType, entity.entityValue)
-                        if (
-                            key not in self.entity_cache
-                            or self.entity_cache[key].isFatal
-                            or not self.entity_cache[key].isHealthy
-                        ):
-                            # Don't send health events for non-fatal health conditions when they are healthy
-                            # they will get published as node conditions which we don't want to do to have
-                            # consistency in the health events publishing logic
-                            if self.dcgm_health_conditions_categorization_mapping_config[watch_name] == "NonFatal":
-                                log.debug(f"Skipping non-fatal health event for watch {watch_name}")
-                                # For non-fatal events that are not sent, update cache immediately
-                                # (no race condition since nothing is being sent)
-                                self.entity_cache[key] = CachedEntityState(isFatal=False, isHealthy=True)
-                                log.info(f"Updated cache for key {key} with value {self.entity_cache[key]}")
-                                metrics.dcgm_health_active_events.labels(
-                                    event_type=check_name, gpu_id=gpu_id, severity="non_fatal"
-                                ).set(0)
-                            else:
-                                # Defer cache update until after successful send
-                                pending_cache_updates[key] = CachedEntityState(isFatal=False, isHealthy=True)
-                                event_metadata = {}
-                                chassis_serial = self._metadata_reader.get_chassis_serial()
-                                if chassis_serial:
-                                    event_metadata["chassis_serial"] = chassis_serial
-
-                                health_events.append(
-                                    platformconnector_pb2.HealthEvent(
-                                        version=self._version,
-                                        agent=self._agent,
-                                        componentClass=self._component_class,
-                                        checkName=check_name,
-                                        generatedTimestamp=timestamp,
-                                        isFatal=False,
-                                        isHealthy=True,
-                                        errorCode=[],
-                                        entitiesImpacted=entities_impacted,
-                                        message=f"GPU {self._get_dcgm_watch(watch_name)} watch reported no errors",
-                                        recommendedAction=platformconnector_pb2.NONE,
-                                        nodeName=self._node_name,
-                                        metadata=event_metadata,
-                                        processingStrategy=self._processing_strategy,
-                                    )
+                        entry = self.entity_cache.get(key)
+
+                        if entry is None or not entry.is_healthy:
+                            had_errors = entry is not None and not entry.is_healthy
+                            pending_cache_updates[key] = EntityCacheEntry()
+
+                            event_metadata = {}
+                            chassis_serial = self._metadata_reader.get_chassis_serial()
+                            if chassis_serial:
+                                event_metadata["chassis_serial"] = chassis_serial
+
+                            health_events.append(
+                                platformconnector_pb2.HealthEvent(
+                                    version=self._version,
+                                    agent=self._agent,
+                                    componentClass=self._component_class,
+                                    checkName=check_name,
+                                    generatedTimestamp=timestamp,
+                                    isFatal=False,
+                                    isHealthy=True,
+                                    errorCode=[],
+                                    entitiesImpacted=entities_impacted,
+                                    message=f"GPU {self._get_dcgm_watch(watch_name)} watch reported no errors",
+                                    recommendedAction=platformconnector_pb2.NONE,
+                                    nodeName=self._node_name,
+                                    metadata=event_metadata,
+                                    processingStrategy=self._processing_strategy,
                                 )
-                                # Defer metric update until after successful send
-                                pending_metric_updates.append((check_name, gpu_id, "fatal", 0))
+                            )
+                            if had_errors:
+                                pending_metric_updates.append((check_name, gpu_id, 0))
             log.debug(f"dcgm health event is {health_events}")
             if len(health_events):
                 try:
                     if self.send_health_event_with_retries(health_events):
                         # Only update cache and metrics after successful send
                         for key, state in pending_cache_updates.items():
                             self.entity_cache[key] = state
-                            log.info(f"Updated cache for key {key} with value {state} after successful send")
-                        for event_type, gpu_id, severity, value in pending_metric_updates:
-                            metrics.dcgm_health_active_events.labels(
-                                event_type=event_type, gpu_id=gpu_id, severity=severity
-                            ).set(value)
+                            log.info(
+                                f"Updated cache for key {key} with value {self.entity_cache[key]} after successful send"
+                            )
+                        for event_type, gpu_id, value in pending_metric_updates:
+                            metrics.dcgm_health_active_events.labels(event_type=event_type, gpu_id=gpu_id).set(value)
                 except Exception as e:
                     log.error(f"Exception while sending health events: {e}")
 
@@ -376,7 +346,8 @@ def dcgm_connectivity_failed(self) -> None:
             health_events = []
             check_name = "GpuDcgmConnectivityFailure"
             key = self._build_cache_key(check_name, "DCGM", "ALL")
-            if key not in self.entity_cache or self.entity_cache[key].isHealthy:
+            entry = self.entity_cache.get(key)
+            if entry is None or entry.is_healthy:
                 event_metadata = {}
                 chassis_serial = self._metadata_reader.get_chassis_serial()
                 if chassis_serial:
@@ -403,12 +374,11 @@ def dcgm_connectivity_failed(self) -> None:
             if len(health_events):
                 try:
                     if self.send_health_event_with_retries(health_events):
-                        # Only update cache after successful send
-                        self.entity_cache[key] = CachedEntityState(isFatal=True, isHealthy=False)
-                        log.info(f"Updated cache for key {key} after successful send")
-                        metrics.dcgm_health_active_events.labels(
-                            event_type=check_name, gpu_id="", severity="fatal"
-                        ).set(1)
+                        self.entity_cache[key] = EntityCacheEntry(active_errors={"DCGM_CONNECTIVITY_ERROR"})
+                        log.info(
+                            f"Updated cache for key {key} with value {self.entity_cache[key]} after successful send"
+                        )
+                        metrics.dcgm_health_active_events.labels(event_type=check_name, gpu_id="").set(1)
                 except Exception as e:
                     log.error(f"Exception while sending DCGM connectivity failure events: {e}")
                     raise