feat(gfd): enhance device health check with compute capability probe

Improve GFD's device health checking to catch devices in degraded
states (e.g., after XID errors). Previously, devices might pass the
GetName() check but fail during actual labeling. Now uses two probes:

1. GetName() - catches completely dead devices
2. GetCudaComputeCapability() - catches degraded devices

This prevents scenarios where XID errors leave devices in a state where
basic queries succeed but complex queries fail, which would cause
partial label generation and unnecessary warnings.

Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>

Author: ArangoGutierrez

cmd/gpu-feature-discovery/main.go

@@ -254,10 +254,29 @@ func (d *gfd) run(sigs chan os.Signal) (bool, error) {
 	}()
 
 	timestampLabeler := lm.NewTimestampLabeler(d.config)
+
+	// Stats for tracking labeling health
+	totalRuns := 0
+	failedRuns := 0
+	lastSuccessTime := time.Now()
+
 rerun:
+	totalRuns++
 	loopLabelers, err := lm.NewLabelers(d.manager, d.vgpu, d.config)
 	if err != nil {
-		return false, err
+		failedRuns++
+		klog.Errorf("GFD: Error creating labelers (run %d, failures %d): %v",
+			totalRuns, failedRuns, err)
+
+		// In oneshot mode, fail immediately (for testing and CI)
+		if *d.config.Flags.GFD.Oneshot {
+			return false, err
+		}
+
+		// Otherwise, sleep and retry
+		klog.Info("Sleeping for ", *d.config.Flags.GFD.SleepInterval)
+		time.Sleep(time.Duration(*d.config.Flags.GFD.SleepInterval))
+		goto rerun
 	}
 
 	labelers := lm.Merge(
@@ -267,18 +286,52 @@ rerun:
 
 	labels, err := labelers.Labels()
 	if err != nil {
-		return false, fmt.Errorf("error generating labels: %v", err)
+		failedRuns++
+		klog.Errorf("GFD: Error generating labels (run %d, failures %d): %v",
+			totalRuns, failedRuns, err)
+
+		// In oneshot mode, fail immediately (for testing and CI)
+		if *d.config.Flags.GFD.Oneshot {
+			return false, err
+		}
+
+		// Otherwise, sleep and retry
+		klog.Info("Sleeping for ", *d.config.Flags.GFD.SleepInterval)
+		time.Sleep(time.Duration(*d.config.Flags.GFD.SleepInterval))
+		goto rerun
 	}
 
 	if len(labels) <= 1 {
 		klog.Warning("No labels generated from any source")
 	}
 
+	// Log stats periodically (every 10 successful runs)
+	if totalRuns%10 == 0 {
+		klog.Infof("GFD Stats: runs=%d, failures=%d, success_rate=%.1f%%, last_success=%v",
+			totalRuns, failedRuns, float64(totalRuns-failedRuns)/float64(totalRuns)*100,
+			time.Since(lastSuccessTime))
+	}
+
 	klog.Info("Creating Labels")
 	if err := d.labelOutputer.Output(labels); err != nil {
-		return false, err
+		failedRuns++
+		klog.Errorf("GFD: Error outputting labels (run %d, failures %d): %v",
+			totalRuns, failedRuns, err)
+
+		// In oneshot mode, fail immediately
+		if *d.config.Flags.GFD.Oneshot {
+			return false, err
+		}
+
+		// Otherwise, sleep and retry
+		klog.Info("Sleeping for ", *d.config.Flags.GFD.SleepInterval)
+		time.Sleep(time.Duration(*d.config.Flags.GFD.SleepInterval))
+		goto rerun
 	}
 
+	// Update last success time
+	lastSuccessTime = time.Now()
+
 	if *d.config.Flags.GFD.Oneshot {
 		return false, nil
 	}

internal/lm/imex.go

@@ -17,7 +17,6 @@
 package lm
 
 import (
-	"fmt"
 	"strings"
 
 	"k8s.io/klog/v2"
@@ -48,16 +47,19 @@ func getFabricIDs(devices []resource.Device) (string, string, error) {
 	for i, device := range devices {
 		isFabricAttached, err := device.IsFabricAttached()
 		if err != nil {
-			return "", "", fmt.Errorf("error checking imex capability: %v", err)
+			// Skip unhealthy devices when checking fabric attachment
+			klog.V(2).Infof("Skipping device %d for IMEX capability check: %v", i, err)
+			continue
 		}
 		if !isFabricAttached {
 			continue
 		}
 
 		clusterUUID, cliqueID, err := device.GetFabricIDs()
 		if err != nil {
-
-			return "", "", fmt.Errorf("error getting fabric IDs: %w", err)
+			// Skip devices that fail fabric ID retrieval
+			klog.V(2).Infof("Skipping device %d, error getting fabric IDs: %v", i, err)
+			continue
 		}
 
 		uniqueClusterUUIDs[clusterUUID] = append(uniqueClusterUUIDs[clusterUUID], i)

internal/lm/nvml.go

@@ -50,6 +50,23 @@ func NewDeviceLabeler(manager resource.Manager, config *spec.Config) (Labeler, e
 		return empty{}, nil
 	}
 
+	// Filter to only healthy devices - skip those that fail basic health
+	// checks. This prevents GFD from crashing when a device goes unhealthy.
+	totalDevices := len(devices)
+	healthyDevices := resource.FilterHealthyDevices(devices)
+	if len(healthyDevices) < totalDevices {
+		klog.Warningf("GFD: %d/%d devices failed health check, continuing with healthy devices",
+			totalDevices-len(healthyDevices), totalDevices)
+	}
+
+	if len(healthyDevices) == 0 {
+		klog.Warning("GFD: No healthy devices available for labeling")
+		return empty{}, nil
+	}
+
+	// Use healthyDevices for all subsequent labeling
+	devices = healthyDevices
+
 	machineTypeLabeler, err := newMachineTypeLabeler(*config.Flags.GFD.MachineTypeFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct machine type labeler: %v", err)
@@ -160,7 +177,9 @@ func newMigCapabilityLabeler(manager resource.Manager) (Labeler, error) {
 	for _, d := range devices {
 		isMigCapable, err = d.IsMigCapable()
 		if err != nil {
-			return nil, fmt.Errorf("error getting mig capability: %v", err)
+			// Skip unhealthy devices, continue checking others
+			klog.V(2).Infof("Skipping device for MIG capability check: %v", err)
+			continue
 		}
 		if isMigCapable {
 			break
@@ -201,7 +220,9 @@ func isMPSCapable(manager resource.Manager) (bool, error) {
 	for _, d := range devices {
 		isMigEnabled, err := d.IsMigEnabled()
 		if err != nil {
-			return false, fmt.Errorf("failed to check if device is MIG-enabled: %w", err)
+			// Skip unhealthy devices when checking MPS capability
+			klog.V(2).Infof("Skipping device for MPS capability check: %v", err)
+			continue
 		}
 		if isMigEnabled {
 			return false, fmt.Errorf("%w for mig devices", errMPSSharingNotSupported)
@@ -247,10 +268,12 @@ func getModeForClasses(classes []uint32) string {
 
 func getDeviceClasses(devices []resource.Device) ([]uint32, error) {
 	seenClasses := make(map[uint32]bool)
-	for _, d := range devices {
+	for i, d := range devices {
 		class, err := d.GetPCIClass()
 		if err != nil {
-			return nil, err
+			// Skip unhealthy devices when determining GPU mode
+			klog.V(2).Infof("Skipping device %d for PCI class check: %v", i, err)
+			continue
 		}
 		seenClasses[class] = true
 	}

internal/lm/resource.go

@@ -41,9 +41,17 @@ func NewGPUResourceLabeler(config *spec.Config, device resource.Device, count in
 		return empty{}, nil
 	}
 
+	// Check if device is healthy before querying
+	if err := resource.CheckDeviceHealth(device); err != nil {
+		klog.Warningf("Skipping unhealthy device in GPU resource labeler: %v", err)
+		return empty{}, nil
+	}
+
 	model, err := device.GetName()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get device model: %v", err)
+		// If health check passed but GetName fails, log and skip
+		klog.Warningf("Device health check passed but GetName failed: %v", err)
+		return empty{}, nil
 	}
 
 	totalMemoryMiB, err := device.GetTotalMemoryMiB()
@@ -55,7 +63,9 @@ func NewGPUResourceLabeler(config *spec.Config, device resource.Device, count in
 
 	architectureLabels, err := newArchitectureLabels(resourceLabeler, device)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create architecture labels: %v", err)
+		// Don't crash on architecture label failure, log and continue
+		klog.Warningf("Failed to create architecture labels: %v", err)
+		architectureLabels = make(Labels)
 	}
 
 	memoryLabeler := (Labeler)(&empty{})
@@ -78,25 +88,35 @@ func NewMIGResourceLabeler(resourceName spec.ResourceName, config *spec.Config,
 		return empty{}, nil
 	}
 
+	// Check if device is healthy before querying
+	if err := resource.CheckDeviceHealth(device); err != nil {
+		klog.Warningf("Skipping unhealthy MIG device in resource labeler: %v", err)
+		return empty{}, nil
+	}
+
 	parent, err := device.GetDeviceHandleFromMigDeviceHandle()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get parent of MIG device: %v", err)
+		klog.Warningf("Failed to get parent of MIG device, skipping: %v", err)
+		return empty{}, nil
 	}
 	model, err := parent.GetName()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get device model: %v", err)
+		klog.Warningf("Failed to get device model, skipping: %v", err)
+		return empty{}, nil
 	}
 
 	migProfile, err := device.GetName()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get MIG profile name: %v", err)
+		klog.Warningf("Failed to get MIG profile name, skipping: %v", err)
+		return empty{}, nil
 	}
 
 	resourceLabeler := newResourceLabeler(resourceName, config)
 
 	attributeLabels, err := newMigAttributeLabels(resourceLabeler, device)
 	if err != nil {
-		return nil, fmt.Errorf("faled to get MIG attribute labels: %v", err)
+		klog.Warningf("Failed to get MIG attribute labels: %v", err)
+		attributeLabels = make(Labels)
 	}
 
 	labelers := Merge(
@@ -252,7 +272,9 @@ func (rl resourceLabeler) replicationInfo() *spec.ReplicatedResource {
 func newMigAttributeLabels(rl resourceLabeler, device resource.Device) (Labels, error) {
 	attributes, err := device.GetAttributes()
 	if err != nil {
-		return nil, fmt.Errorf("unable to get attributes of MIG device: %v", err)
+		// Return empty labels instead of crashing
+		klog.Warningf("Unable to get attributes of MIG device, skipping: %v", err)
+		return make(Labels), nil
 	}
 
 	labels := rl.labels(attributes)
@@ -263,7 +285,9 @@ func newMigAttributeLabels(rl resourceLabeler, device resource.Device) (Labels,
 func newArchitectureLabels(rl resourceLabeler, device resource.Device) (Labels, error) {
 	computeMajor, computeMinor, err := device.GetCudaComputeCapability()
 	if err != nil {
-		return nil, fmt.Errorf("failed to determine CUDA compute capability: %v", err)
+		// Return empty labels instead of error - allows labeling to continue
+		klog.Warningf("Failed to determine CUDA compute capability, skipping architecture labels: %v", err)
+		return make(Labels), nil
 	}
 
 	if computeMajor == 0 {

internal/resource/device_health.go

@@ -0,0 +1,112 @@
+/**
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package resource
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	"k8s.io/klog/v2"
+)
+
+// DeviceUnhealthyError indicates that a device is unhealthy and cannot be
+// queried. This error type allows GFD to skip unhealthy devices gracefully
+// rather than crashing.
+type DeviceUnhealthyError struct {
+	DeviceID string
+	Reason   string
+	NVMLErr  nvml.Return
+}
+
+func (e *DeviceUnhealthyError) Error() string {
+	if e.NVMLErr != nvml.SUCCESS {
+		return fmt.Sprintf("device %s unhealthy: %v (NVML: %v)",
+			e.DeviceID, e.Reason, e.NVMLErr)
+	}
+	return fmt.Sprintf("device %s unhealthy: %v", e.DeviceID, e.Reason)
+}
+
+// IsUnhealthyNVMLError categorizes NVML errors to determine if they
+// indicate an unhealthy device vs a transient or fatal error.
+func IsUnhealthyNVMLError(ret nvml.Return) bool {
+	switch ret {
+	case nvml.SUCCESS:
+		return false
+	case nvml.ERROR_GPU_IS_LOST,
+		nvml.ERROR_UNINITIALIZED,
+		nvml.ERROR_INVALID_ARGUMENT,
+		nvml.ERROR_NOT_FOUND,
+		nvml.ERROR_NO_PERMISSION:
+		return true
+	default:
+		// For unknown errors, be conservative and treat as unhealthy
+		// This prevents GFD from crashing on unexpected errors
+		return true
+	}
+}
+
+// CheckDeviceHealth performs a health check on a device to verify it's
+// queryable. This uses two probes to catch both completely dead devices
+// and devices in degraded states (e.g., after XID errors). This prevents
+// GFD from attempting to label devices that will fail during query.
+func CheckDeviceHealth(d Device) error {
+	// Probe 1: Basic identity check
+	_, err := d.GetName()
+	if err != nil {
+		return &DeviceUnhealthyError{
+			Reason:  "GetName failed",
+			NVMLErr: toNVMLReturn(err),
+		}
+	}
+
+	// Probe 2: Compute capability check (catches devices in degraded state)
+	// Many XID errors leave the device in a state where GetName() works but
+	// more complex queries fail. This probe catches those scenarios.
+	_, _, err = d.GetCudaComputeCapability()
+	if err != nil {
+		return &DeviceUnhealthyError{
+			Reason:  "GetCudaComputeCapability failed",
+			NVMLErr: toNVMLReturn(err),
+		}
+	}
+
+	return nil
+}
+
+// toNVMLReturn extracts the NVML return code from an error
+func toNVMLReturn(err error) nvml.Return {
+	if nvmlErr, ok := err.(nvml.Return); ok {
+		return nvmlErr
+	}
+	return nvml.ERROR_UNKNOWN
+}
+
+// FilterHealthyDevices filters a list of devices to only include those
+// that pass basic health checks. Unhealthy devices are logged as warnings
+// but don't cause GFD to crash.
+func FilterHealthyDevices(devices []Device) []Device {
+	var healthy []Device
+	for i, d := range devices {
+		if err := CheckDeviceHealth(d); err != nil {
+			// Log warning but continue
+			klog.Warningf("Skipping unhealthy device at index %d: %v", i, err)
+			continue
+		}
+		healthy = append(healthy, d)
+	}
+	return healthy
+}