[LEP-3777] fix(nvidia): do not exit on NVML init error (instead returns Unhealthy)

components/accelerator/nvidia/clock-speed/component.go

@@ -130,6 +130,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/clock-speed/component_test.go

@@ -78,6 +78,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 // TestData_GetError tests the getError method of Data
 func TestData_GetError(t *testing.T) {
 	// Test nil data

components/accelerator/nvidia/ecc/component.go

@@ -132,6 +132,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/ecc/component_test.go

@@ -85,6 +85,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 // mockNvmlInstanceNVMLNotLoaded is a special mock implementation that always returns false for NVMLExists
 type mockNvmlInstanceNVMLNotLoaded struct {
 	*mockNVMLInstance

components/accelerator/nvidia/fabric-manager/component.go

@@ -297,6 +297,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"
@@ -528,6 +541,8 @@ type checkResult struct {
 
 	// tracks the healthy evaluation result of the last check
 	health apiv1.HealthStateType
+	// tracks the suggested actions for the last check
+	suggestedActions *apiv1.SuggestedActions
 	// tracks the reason of the last check
 	reason string
 }

components/accelerator/nvidia/fabric-manager/component_test.go

@@ -691,6 +691,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 func TestComponentCheck_NVMLInstance(t *testing.T) {
 	tests := []struct {
 		name                    string

components/accelerator/nvidia/gpm/component.go

@@ -176,6 +176,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/gpm/component_test.go

@@ -83,6 +83,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 // customMockNVMLInstance implements the nvml.InstanceV2 interface with customizable NVMLExists behavior
 type customMockNVMLInstance struct {
 	devs       map[string]device.Device
@@ -141,6 +145,10 @@ func (m *customMockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *customMockNVMLInstance) InitError() error {
+	return nil
+}
+
 // createMockGPMComponent creates a component with mocked functions for testing
 func createMockGPMComponent(
 	ctx context.Context,

components/accelerator/nvidia/gpu-counts/component.go

@@ -175,6 +175,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/gpu-counts/component_test.go

@@ -83,6 +83,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 // mockRebootEventStore implements pkghost.RebootEventStore for testing
 type mockRebootEventStore struct {
 	events eventstore.Events

components/accelerator/nvidia/hw-slowdown/component.go

@@ -202,6 +202,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/hw-slowdown/component_test.go

@@ -82,6 +82,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 // Helper function to create a mock NVML instance with specified devices
 func createMockNVMLInstance(devices map[string]device.Device) *mockNVMLInstance {
 	return &mockNVMLInstance{

components/accelerator/nvidia/infiniband/component.go

@@ -5,6 +5,7 @@ package infiniband
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"os"
 	"sort"
 	"strings"
@@ -286,6 +287,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/infiniband/component_test.go

@@ -519,6 +519,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 func mockGetThresholds() types.ExpectedPortStates {
 	return types.ExpectedPortStates{
 		AtLeastPorts: 1,

components/accelerator/nvidia/memory/component.go

@@ -133,6 +133,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/memory/component_test.go

@@ -84,6 +84,10 @@ func (m *mockNvmlInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNvmlInstance) InitError() error {
+	return nil
+}
+
 // MockMemoryComponent creates a component with mocked functions for testing
 func MockMemoryComponent(
 	ctx context.Context,

components/accelerator/nvidia/nccl/component.go

@@ -152,6 +152,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"
@@ -201,6 +214,8 @@ type checkResult struct {
 
 	// tracks the healthy evaluation result of the last check
 	health apiv1.HealthStateType
+	// tracks the suggested actions for the last check
+	suggestedActions *apiv1.SuggestedActions
 	// tracks the reason of the last check
 	reason string
 }

components/accelerator/nvidia/nccl/component_test.go

@@ -152,6 +152,11 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return args.Error(0)
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	args := m.Called()
+	return args.Error(0)
+}
+
 // TestCheck tests the Check method in various scenarios
 func TestCheck(t *testing.T) {
 	t.Parallel()
@@ -169,6 +174,7 @@ func TestCheck(t *testing.T) {
 	t.Run("nvml exists but no product name", func(t *testing.T) {
 		mockNvml := new(mockNVMLInstance)
 		mockNvml.On("NVMLExists").Return(true)
+		mockNvml.On("InitError").Return(nil)
 		mockNvml.On("ProductName").Return("")
 
 		comp := &component{
@@ -196,6 +202,7 @@ func TestCheck(t *testing.T) {
 	t.Run("nil readAllKmsg", func(t *testing.T) {
 		mockNvml := new(mockNVMLInstance)
 		mockNvml.On("NVMLExists").Return(true)
+		mockNvml.On("InitError").Return(nil)
 		mockNvml.On("ProductName").Return("Test GPU")
 		mockNvml.On("Devices").Return(map[string]device.Device{})
 		mockNvml.On("GetMemoryErrorManagementCapabilities").Return(nvidiaproduct.MemoryErrorManagementCapabilities{})
@@ -213,6 +220,7 @@ func TestCheck(t *testing.T) {
 	t.Run("readAllKmsg returns error", func(t *testing.T) {
 		mockNvml := new(mockNVMLInstance)
 		mockNvml.On("NVMLExists").Return(true)
+		mockNvml.On("InitError").Return(nil)
 		mockNvml.On("ProductName").Return("Test GPU")
 		mockNvml.On("Devices").Return(map[string]device.Device{})
 		mockNvml.On("GetMemoryErrorManagementCapabilities").Return(nvidiaproduct.MemoryErrorManagementCapabilities{})
@@ -235,6 +243,7 @@ func TestCheck(t *testing.T) {
 	t.Run("no matching messages", func(t *testing.T) {
 		mockNvml := new(mockNVMLInstance)
 		mockNvml.On("NVMLExists").Return(true)
+		mockNvml.On("InitError").Return(nil)
 		mockNvml.On("ProductName").Return("Test GPU")
 		mockNvml.On("Devices").Return(map[string]device.Device{})
 		mockNvml.On("GetMemoryErrorManagementCapabilities").Return(nvidiaproduct.MemoryErrorManagementCapabilities{})
@@ -261,6 +270,7 @@ func TestCheck(t *testing.T) {
 	t.Run("with matching messages", func(t *testing.T) {
 		mockNvml := new(mockNVMLInstance)
 		mockNvml.On("NVMLExists").Return(true)
+		mockNvml.On("InitError").Return(nil)
 		mockNvml.On("ProductName").Return("Test GPU")
 		mockNvml.On("Devices").Return(map[string]device.Device{})
 		mockNvml.On("GetMemoryErrorManagementCapabilities").Return(nvidiaproduct.MemoryErrorManagementCapabilities{})
@@ -572,6 +582,7 @@ func TestCheck_NVML_NoProductName(t *testing.T) {
 
 	mockNvml := new(mockNVMLInstance)
 	mockNvml.On("NVMLExists").Return(true)
+	mockNvml.On("InitError").Return(nil)
 	mockNvml.On("ProductName").Return("")
 
 	comp := &component{

components/accelerator/nvidia/nvlink/component.go

@@ -137,6 +137,19 @@ func (c *component) Check() components.CheckResult {
 		cr.reason = "NVIDIA NVML library is not loaded"
 		return cr
 	}
+	// Check for NVML initialization errors first.
+	// This handles cases like "error getting device handle for index 'N': Unknown Error"
+	// which corresponds to nvidia-smi showing "Unable to determine the device handle for GPU".
+	if err := c.nvmlInstance.InitError(); err != nil {
+		cr.health = apiv1.HealthStateTypeUnhealthy
+		cr.reason = fmt.Sprintf("NVML initialization error: %v", err)
+		cr.suggestedActions = &apiv1.SuggestedActions{
+			RepairActions: []apiv1.RepairActionType{
+				apiv1.RepairActionTypeRebootSystem,
+			},
+		}
+		return cr
+	}
 	if c.nvmlInstance.ProductName() == "" {
 		cr.health = apiv1.HealthStateTypeHealthy
 		cr.reason = "NVIDIA NVML is loaded but GPU is not detected (missing product name)"

components/accelerator/nvidia/nvlink/component_test.go

@@ -81,6 +81,10 @@ func (m *mockNVMLInstance) Shutdown() error {
 	return nil
 }
 
+func (m *mockNVMLInstance) InitError() error {
+	return nil
+}
+
 // mockNVMLInstanceNVMLNotExists is a special mock for the case where NVMLExists returns false
 type mockNVMLInstanceNVMLNotExists struct {
 	mockNVMLInstance