[LEP-3777] feat(nvidia): add --nvml-device-get-devices-error flag for testing NVML init failures (#1196)

This adds a hidden testing flag to simulate Device().GetDevices()
failures,
enabling integration testing of the erroredInstance path introduced in:
#1180

The flag simulates the "Unable to determine the device handle for GPU:
Unknown Error"
scenario that occurs when NVML loads but device enumeration fails (e.g.,
Xid 79).
When enabled, gpud continues running but all nvidia components report
unhealthy.

Signed-off-by: Gyuho Lee <gyuhol@nvidia.com>

Signed-off-by: Gyuho Lee <gyuhol@nvidia.com>

Author: gyuho

cmd/gpud/command/command.go

@@ -300,6 +300,11 @@ sudo rm /etc/systemd/system/gpud.service
 					Usage:  "(testing purposes) set the comma-separated gpu uuids to return GPU fabric health summary unhealthy (nvml.GPU_FABRIC_HEALTH_SUMMARY_UNHEALTHY). NOTE: Only works on multi-GPU NVSwitch systems (H100-SXM, H200-SXM, GB200). Ignored on PCIe variants and single-GPU systems.",
 					Hidden: true, // only for testing
 				},
+				cli.BoolFlag{
+					Name:   "nvml-device-get-devices-error",
+					Usage:  "(testing purposes) simulate NVML Device().GetDevices() failure returning 'Unable to determine the device handle for GPU: Unknown Error'. Use to test gpud behavior when NVML loads but device enumeration fails (e.g., Xid 79).",
+					Hidden: true, // only for testing
+				},
 			},
 		},
 		{

cmd/gpud/run/command.go

@@ -221,6 +221,10 @@ func Command(cliContext *cli.Context) error {
 	// (e.g., set "H100-SXM" on H100-PCIe to enable fabric state failure injection testing)
 	gpuProductNameOverride := cliContext.String("gpu-product-name")
 
+	// NVML device enumeration error injection for testing
+	// When enabled, Device().GetDevices() returns an error simulating Xid 79 or similar failures
+	nvmlDeviceGetDevicesError := cliContext.Bool("nvml-device-get-devices-error")
+
 	ibExcludedDevices := parseInfinibandExcludeDevices(ibExcludeDevicesStr)
 	if len(ibExcludedDevices) > 0 {
 		log.Logger.Infow("excluding infiniband devices from monitoring", "devices", ibExcludedDevices)
@@ -241,6 +245,7 @@ func Command(cliContext *cli.Context) error {
 			GPUUUIDsWithGPURequiresReset:                  gpuUUIDsWithGPURequiresReset,
 			GPUUUIDsWithFabricStateHealthSummaryUnhealthy: gpuUUIDsWithFabricStateHealthSummaryUnhealthy,
 			GPUProductNameOverride:                        gpuProductNameOverride,
+			NVMLDeviceGetDevicesError:                     nvmlDeviceGetDevicesError,
 		}),
 	}
 

components/registry.go

@@ -60,6 +60,12 @@ type FailureInjector struct {
 	// (e.g., H100-PCIe) doesn't support fabric state monitoring.
 	// Set to "H100-SXM" or "H200-SXM" to simulate a fabric-capable system.
 	GPUProductNameOverride string
+
+	// NVMLDeviceGetDevicesError when true simulates Device().GetDevices() failure.
+	// This is useful for testing the "Unable to determine the device handle for GPU: Unknown Error"
+	// scenario that occurs when NVML library loads but device enumeration fails (e.g., Xid 79).
+	// ref. https://github.com/leptonai/gpud/pull/1180
+	NVMLDeviceGetDevicesError bool
 }
 
 // InitFunc is the function that initializes a component.

pkg/nvidia-query/nvml/instance.go

@@ -28,6 +28,13 @@ type FailureInjectorConfig struct {
 	// When set, this affects FabricStateSupported(), FabricManagerSupported(),
 	// and memory management capabilities detection.
 	GPUProductNameOverride string
+
+	// NVMLDeviceGetDevicesError when true simulates Device().GetDevices() failure.
+	// This is useful for testing the "Unable to determine the device handle for GPU: Unknown Error"
+	// scenario that occurs when NVML library loads but device enumeration fails (e.g., Xid 79).
+	// When enabled, gpud continues running but all nvidia components report unhealthy.
+	// ref. https://github.com/leptonai/gpud/pull/1180
+	NVMLDeviceGetDevicesError bool
 }
 
 var _ Instance = &instance{}
@@ -133,11 +140,22 @@ func refreshNVMLAndExit(ctx context.Context) {
 	}
 }
 
+// ErrDeviceGetDevicesInjected is the error returned when NVMLDeviceGetDevicesError is enabled.
+// This simulates the "Unable to determine the device handle for GPU: Unknown Error" scenario.
+var ErrDeviceGetDevicesInjected = errors.New("error getting device handle for index '0': Unknown Error (injected for testing)")
+
 // newInstance creates a new instance of the NVML library.
 // If NVML is not installed, it returns no-op nvml instance.
 // The "refreshNVML" function is only called when the NVML library is not found.
 func newInstance(refreshCtx context.Context, refreshNVML func(context.Context), failureInjector *FailureInjectorConfig) (Instance, error) {
-	nvmlLib, err := nvmllib.New()
+	// Build library options for failure injection
+	var libOpts []nvmllib.OpOption
+	if failureInjector != nil && failureInjector.NVMLDeviceGetDevicesError {
+		log.Logger.Warnw("NVML Device().GetDevices() error injection enabled for testing")
+		libOpts = append(libOpts, nvmllib.WithDeviceGetDevicesError(ErrDeviceGetDevicesInjected))
+	}
+
+	nvmlLib, err := nvmllib.New(libOpts...)
 	if err != nil {
 		if errors.Is(err, nvmllib.ErrNVMLNotFound) {
 			if refreshNVML != nil {

pkg/nvidia-query/nvml/instance_test.go

@@ -17,3 +17,90 @@ func TestInstanceV2(t *testing.T) {
 	}
 	t.Logf("instance mem cap %+v", inst.GetMemoryErrorManagementCapabilities())
 }
+
+func TestNewWithFailureInjector_NVMLDeviceGetDevicesError(t *testing.T) {
+	// Test that enabling NVMLDeviceGetDevicesError returns an erroredInstance
+	inst, err := NewWithFailureInjector(&FailureInjectorConfig{
+		NVMLDeviceGetDevicesError: true,
+	})
+
+	if errors.Is(err, nvmllib.ErrNVMLNotFound) {
+		t.Skipf("nvml not installed, skipping")
+	}
+
+	// Should not return an error from NewWithFailureInjector - instead, it returns an erroredInstance
+	if err != nil {
+		t.Fatalf("failed to create instance: %v", err)
+	}
+
+	// If NVML is not installed, the function returns a noOpInstance (NVMLExists=false)
+	// which is expected - the error injection only works when NVML is actually loaded
+	if !inst.NVMLExists() {
+		t.Skipf("nvml not installed (noOpInstance returned), skipping")
+	}
+
+	// InitError() should return our injected error
+	initErr := inst.InitError()
+	if initErr == nil {
+		t.Fatal("expected InitError() to return the injected error, got nil")
+	}
+
+	// Verify it's our injected error
+	if !errors.Is(initErr, ErrDeviceGetDevicesInjected) {
+		t.Fatalf("expected InitError() to return ErrDeviceGetDevicesInjected, got: %v", initErr)
+	}
+
+	// Devices should be nil for erroredInstance
+	if inst.Devices() != nil {
+		t.Fatalf("expected Devices() to return nil for erroredInstance, got: %v", inst.Devices())
+	}
+
+	t.Logf("successfully tested NVMLDeviceGetDevicesError injection: %v", initErr)
+}
+
+func TestErroredInstance(t *testing.T) {
+	// Test the erroredInstance directly
+	testErr := errors.New("test error")
+	inst := NewErrored(testErr)
+
+	// NVMLExists returns true because the library loaded
+	if !inst.NVMLExists() {
+		t.Error("expected NVMLExists() to return true for erroredInstance")
+	}
+
+	// InitError returns the error
+	if inst.InitError() != testErr {
+		t.Errorf("expected InitError() to return %v, got %v", testErr, inst.InitError())
+	}
+
+	// All data methods return nil/zero values
+	if inst.Devices() != nil {
+		t.Error("expected Devices() to return nil for erroredInstance")
+	}
+	if inst.Library() != nil {
+		t.Error("expected Library() to return nil for erroredInstance")
+	}
+	if inst.ProductName() != "" {
+		t.Error("expected ProductName() to return empty string for erroredInstance")
+	}
+	if inst.DriverVersion() != "" {
+		t.Error("expected DriverVersion() to return empty string for erroredInstance")
+	}
+	if inst.DriverMajor() != 0 {
+		t.Error("expected DriverMajor() to return 0 for erroredInstance")
+	}
+	if inst.CUDAVersion() != "" {
+		t.Error("expected CUDAVersion() to return empty string for erroredInstance")
+	}
+	if inst.FabricManagerSupported() {
+		t.Error("expected FabricManagerSupported() to return false for erroredInstance")
+	}
+	if inst.FabricStateSupported() {
+		t.Error("expected FabricStateSupported() to return false for erroredInstance")
+	}
+
+	// Shutdown should not error
+	if err := inst.Shutdown(); err != nil {
+		t.Errorf("expected Shutdown() to return nil, got %v", err)
+	}
+}

pkg/nvidia-query/nvml/lib/lib.go

@@ -62,6 +62,7 @@ func createLibrary(opts ...OpOption) Library {
 		devices:                                options.devicesToReturn,
 		getRemappedRowsForAllDevs:              options.devGetRemappedRowsForAllDevs,
 		getCurrentClocksEventReasonsForAllDevs: options.devGetCurrentClocksEventReasonsForAllDevs,
+		getDevicesError:                        options.devGetDevicesError,
 	}
 
 	infoOpts := []nvinfo.Option{
@@ -90,9 +91,15 @@ type devInterface struct {
 	devices                                []nvlibdevice.Device
 	getRemappedRowsForAllDevs              func() (int, int, bool, bool, nvml.Return)
 	getCurrentClocksEventReasonsForAllDevs func() (uint64, nvml.Return)
+	getDevicesError                        error
 }
 
 func (d *devInterface) GetDevices() ([]nvlibdevice.Device, error) {
+	// Check for injected error first (for testing device enumeration failures)
+	if d.getDevicesError != nil {
+		return nil, d.getDevicesError
+	}
+
 	devs := d.devices
 
 	var err error

pkg/nvidia-query/nvml/lib/lib_test.go

@@ -1,6 +1,7 @@
 package lib
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
@@ -13,3 +14,18 @@ func Test_createLibrary(t *testing.T) {
 	)
 	assert.Equal(t, nv.NVML().Init(), nvml.SUCCESS)
 }
+
+func Test_WithDeviceGetDevicesError(t *testing.T) {
+	testErr := errors.New("error getting device handle for index '0': Unknown Error (injected for testing)")
+
+	nv := createLibrary(
+		WithInitReturn(nvml.SUCCESS),
+		WithDeviceGetDevicesError(testErr),
+	)
+
+	// GetDevices should return the injected error
+	devices, err := nv.Device().GetDevices()
+	assert.Nil(t, devices)
+	assert.Equal(t, testErr, err)
+	assert.ErrorIs(t, err, testErr)
+}

pkg/nvidia-query/nvml/lib/options.go

@@ -18,6 +18,10 @@ type Op struct {
 
 	// ref. https://docs.nvidia.com/deploy/nvml-api/group__nvmlDeviceQueries.html#group__nvmlDeviceQueries_1g7e505374454a0d4fc7339b6c885656d6
 	devGetCurrentClocksEventReasonsForAllDevs func() (uint64, nvml.Return)
+
+	// devGetDevicesError is the error to return from Device().GetDevices().
+	// Used for testing NVML device enumeration failure scenarios.
+	devGetDevicesError error
 }
 
 type OpOption func(*Op)
@@ -77,3 +81,12 @@ func WithDeviceGetCurrentClocksEventReasonsForAllDevs(f func() (uint64, nvml.Ret
 		op.devGetCurrentClocksEventReasonsForAllDevs = f
 	}
 }
+
+// WithDeviceGetDevicesError specifies the error to return from Device().GetDevices().
+// This is used for testing NVML device enumeration failure scenarios, such as when
+// nvidia-smi shows "Unable to determine the device handle for GPU: Unknown Error".
+func WithDeviceGetDevicesError(err error) OpOption {
+	return func(op *Op) {
+		op.devGetDevicesError = err
+	}
+}

pkg/server/server.go

@@ -255,13 +255,15 @@ func New(ctx context.Context, auditLogger log.AuditLogger, config *lepconfig.Con
 	if config.FailureInjector != nil && (len(config.FailureInjector.GPUUUIDsWithGPULost) > 0 ||
 		len(config.FailureInjector.GPUUUIDsWithGPURequiresReset) > 0 ||
 		len(config.FailureInjector.GPUUUIDsWithFabricStateHealthSummaryUnhealthy) > 0 ||
-		config.FailureInjector.GPUProductNameOverride != "") {
+		config.FailureInjector.GPUProductNameOverride != "" ||
+		config.FailureInjector.NVMLDeviceGetDevicesError) {
 		// If failure injector is configured for NVML-level errors or product name override, use it
 		nvmlInstance, err = nvidianvml.NewWithFailureInjector(&nvidianvml.FailureInjectorConfig{
 			GPUUUIDsWithGPULost:                           config.FailureInjector.GPUUUIDsWithGPULost,
 			GPUUUIDsWithGPURequiresReset:                  config.FailureInjector.GPUUUIDsWithGPURequiresReset,
 			GPUUUIDsWithFabricStateHealthSummaryUnhealthy: config.FailureInjector.GPUUUIDsWithFabricStateHealthSummaryUnhealthy,
 			GPUProductNameOverride:                        config.FailureInjector.GPUProductNameOverride,
+			NVMLDeviceGetDevicesError:                     config.FailureInjector.NVMLDeviceGetDevicesError,
 		})
 	} else {
 		nvmlInstance, err = nvidianvml.NewWithExitOnSuccessfulLoad(ctx)