[LEP-4267] fix(xid): detect fallen-off-bus logs as XID 79 (#1209)

Some new GPU drivers now emitting in multi-line

```
$ journalctl -k | grep -C 20 fallen

Feb 14 05:03:41 000-000-00000-1-gpu-1 systemd[1]: systemd-pcrmachine.service - TPM2 PCR Machine ID Measurement was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Feb 14 05:03:41 000-000-00000-1-gpu-1 kernel: mlx5_core 0000:ab:00.0 mlx5_5: Port: 1 Link INIT
Feb 14 05:03:41 000-000-00000-1-gpu-1 kernel: workqueue: ib_cache_event_task [ib_core] hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
Feb 14 05:03:41 000-000-00000-1-gpu-1 systemd[1]: Starting systemd-remount-fs.service - Remount Root and Kernel File Systems...
Feb 14 05:03:41 000-000-00000-1-gpu-1 systemd[1]: systemd-tpm2-setup-early.service - TPM2 SRK Setup (Early) was skipped because of an unmet condition check (ConditionSecurity=measured-uki).
Feb 14 05:03:41 000-000-00000-1-gpu-1 systemd[1]: Starting systemd-udev-trigger.service - Coldplug All udev Devices...
Feb 14 05:03:41 000-000-00000-1-gpu-1 systemd[1]: Started systemd-journald.service - Journal Service.
Feb 14 05:03:41 000-000-00000-1-gpu-1 kernel: EXT4-fs (md127p3): re-mounted 0f3cd65a-338b-4574-bba3-ecef214e1917 r/w. Quota mode: none.
Feb 14 05:03:41 000-000-00000-1-gpu-1 kernel: mlx5_core 0000:ab:00.0 mlx5_5: Port: 1 Link ACTIVE
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: bridge: filtering via arp/ip/ip6tables is no longer available by default. Update your scripts to load br_netfilter if you need this.
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: Bridge firewalling registered
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: knem 1.1.4.90mlnx3: initialized
Feb 14 05:03:42 000-000-00000-1-gpu-1 systemd-journald[2382]: Received client request to flush runtime journal.
Feb 14 05:03:42 000-000-00000-1-gpu-1 systemd-journald[2382]: /var/log/journal/c0f88c9a436e4570b7183c73a833c48c/system.journal: Journal file uses a different sequence number ID, rotating.
Feb 14 05:03:42 000-000-00000-1-gpu-1 systemd-journald[2382]: Rotating system journal.
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: nvidia-nvlink: Nvlink Core is being initialized, major device number 236
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: 
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: nvidia 0000:18:00.0: enabling device (0140 -> 0142)

Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: NVRM: The NVIDIA GPU 0000:18:00.0
                                              NVRM: (PCI ID: 10de:2901) installed in this system has
                                              NVRM: fallen off the bus and is not responding to commands.

Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: nvidia: probe of 0000:18:00.0 failed with error -1
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: nvidia 0000:29:00.0: enabling device (0140 -> 0142)
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: nvidia 0000:3a:00.0: enabling device (0140 -> 0142)
Feb 14 05:03:42 000-000-00000-1-gpu-1 kernel: nvidia 0000:4b:00.0: enabling device (0140 -> 0142)
```

---------

Signed-off-by: Gyuho Lee <gyuhol@nvidia.com>

Author: gyuho

Dockerfile

@@ -13,7 +13,7 @@ ARG CUDA_VERSION="12.4.1"
 # ==============================================================================
 # Stage 1: Builder - Build gpud binary and vendor Go dependencies
 # ==============================================================================
-FROM golang:1.24.7 AS builder
+FROM golang:1.25.7 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 ARG BUILDPLATFORM

README.md

@@ -35,10 +35,10 @@ To install from the official release on Linux amd64 (x86_64) machine:
 curl -fsSL https://pkg.gpud.dev/install.sh | sh
 ```
 
-To specify a version:
+To install the latest published version explicitly:
 
 ```bash
-curl -fsSL https://pkg.gpud.dev/install.sh | sh -s v0.10.0
+curl -fsSL https://pkg.gpud.dev/install.sh | sh -s $(curl -fsSL https://pkg.gpud.dev/unstable_latest.txt)
 ```
 
 The install script also currently support other architectures (e.g., arm64) and OSes (e.g., macOS).

components/accelerator/nvidia/xid/component_test.go

@@ -575,6 +575,37 @@ func TestCheck(t *testing.T) {
 		assert.Equal(t, 31, data.FoundErrors[0].Xid)
 	})
 
+	t.Run("with fallen-off-bus fallback format without explicit Xid", func(t *testing.T) {
+		mockedNVML := createMockNVMLInstance()
+		gpudInstance := &components.GPUdInstance{
+			RootCtx:      ctx,
+			NVMLInstance: mockedNVML,
+		}
+
+		comp, err := New(gpudInstance)
+		assert.NoError(t, err)
+
+		c := comp.(*component)
+		c.readAllKmsg = func(ctx context.Context) ([]kmsg.Message, error) {
+			return []kmsg.Message{
+				{
+					Timestamp: metav1.NewTime(time.Now()),
+					Message: "NVRM: The NVIDIA GPU 0000:18:00.0\n" +
+						"NVRM: (PCI ID: 10de:2901) installed in this system has\n" +
+						"NVRM: fallen off the bus and is not responding to commands.",
+				},
+			}, nil
+		}
+
+		result := comp.Check()
+		data := result.(*checkResult)
+		assert.Len(t, data.FoundErrors, 1)
+		assert.Equal(t, 79, data.FoundErrors[0].Xid)
+		assert.Equal(t, "PCI:0000:18:00", data.FoundErrors[0].DeviceUUID)
+		assert.Equal(t, apiv1.HealthStateTypeUnhealthy, result.HealthStateType())
+		assert.Contains(t, result.Summary(), "matched 1 xid errors from 1 kmsg(s)")
+	})
+
 	t.Run("with XID 63 and 64 errors that should be skipped when row remapping is supported", func(t *testing.T) {
 		// Using a properly implemented mock with row remapping support
 		mockedNVML := createMockNVMLInstanceWithRowRemapping()
@@ -1293,6 +1324,80 @@ func TestHandleEventChannel(t *testing.T) {
 	wg.Wait()
 }
 
+func TestStartWithFallenOffBusFallback(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	dbRW, dbRO, cleanup := sqlite.OpenTestDB(t)
+	defer cleanup()
+
+	store, err := eventstore.New(dbRW, dbRO, DefaultRetentionPeriod)
+	assert.NoError(t, err)
+
+	rebootEventStore := pkghost.NewRebootEventStore(store)
+	mockedNVML := createMockNVMLInstance()
+
+	gpudInstance := &components.GPUdInstance{
+		RootCtx:          ctx,
+		EventStore:       store,
+		RebootEventStore: rebootEventStore,
+		NVMLInstance:     mockedNVML,
+	}
+
+	comp, err := New(gpudInstance)
+	assert.NoError(t, err)
+	assert.NotNil(t, comp)
+
+	c := comp.(*component)
+	if c.eventBucket == nil {
+		c.eventBucket, err = store.Bucket(Name)
+		assert.NoError(t, err)
+	}
+
+	kmsgCh := make(chan kmsg.Message, 10)
+	go c.start(kmsgCh, 100*time.Millisecond)
+	defer func() {
+		assert.NoError(t, comp.Close())
+	}()
+
+	kmsgCh <- kmsg.Message{
+		Timestamp: metav1.NewTime(time.Now()),
+		Message: "NVRM: The NVIDIA GPU 0000:18:00.0\n" +
+			"NVRM: (PCI ID: 10de:2901) installed in this system has\n" +
+			"NVRM: fallen off the bus and is not responding to commands.",
+	}
+
+	deadline := time.Now().Add(5 * time.Second)
+	foundXid79 := false
+	xidRegex := regexp.MustCompile(`XID (\d+)`)
+	for time.Now().Before(deadline) {
+		events, err := comp.Events(ctx, time.Now().Add(-1*time.Hour))
+		assert.NoError(t, err)
+
+		for _, event := range events {
+			if matches := xidRegex.FindStringSubmatch(event.Message); len(matches) > 1 {
+				xidCode, err := strconv.Atoi(matches[1])
+				assert.NoError(t, err)
+				if xidCode == 79 {
+					foundXid79 = true
+					break
+				}
+			}
+		}
+		if foundXid79 {
+			break
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	assert.True(t, foundXid79, "expected synthetic XID 79 event to be stored from fallen-off-bus fallback format")
+
+	states := comp.LastHealthStates()
+	assert.NotEmpty(t, states)
+	assert.Equal(t, apiv1.HealthStateTypeUnhealthy, states[0].Health)
+}
+
 func TestCheckResult_getError(t *testing.T) {
 	tests := []struct {
 		name     string

components/accelerator/nvidia/xid/kmsg.go

@@ -26,11 +26,25 @@ const (
 	// Example: NVRM: Xid (PCI:0018:01:00): 149, NETIR_LINK_EVT Fatal XC0 i0 Link 08 (0x004505c6 0x00000000 ...)
 	// Reference: https://docs.nvidia.com/deploy/xid-errors/analyzing-xid-catalog.html
 	RegexNVRMXidExtended = `NVRM: Xid \(PCI:([0-9a-fA-F:]+)\): (\d+)(?:, pid=(\d+), name=([^,]+))?, ([A-Z_]+(?:/[A-Z_]+)?)\s+(Nonfatal|Fatal)\s+(XC[01])\s+(i\d+)\s+Link\s+(-?\d+)\s+\((0x[0-9a-fA-F]+)\s+(0x[0-9a-fA-F]+)(?:\s+(0x[0-9a-fA-F]+))?(?:\s+(0x[0-9a-fA-F]+))?(?:\s+(0x[0-9a-fA-F]+))?(?:\s+(0x[0-9a-fA-F]+))?`
+
+	// Fallback signatures for "GPU has fallen off the bus" logs that do not include
+	// explicit "NVRM: Xid (...)" text.
+	//
+	// Multiline example:
+	// NVRM: The NVIDIA GPU 0000:18:00.0
+	// NVRM: ... fallen off the bus and is not responding to commands.
+	RegexNVRMFallenOffBusMultiline = `(?s)NVRM:\s+The NVIDIA GPU ((?:[0-9a-fA-F]{4}:)?[0-9a-fA-F]{2}:[0-9a-fA-F]{2})\.0.*?fallen off the bus and is not responding to commands\.`
+
+	// Single-line example:
+	// NVRM: GPU 0000:18:00.0: GPU has fallen off the bus.
+	RegexNVRMFallenOffBusSingleLine = `NVRM:\s+GPU ((?:[0-9a-fA-F]{4}:)?[0-9a-fA-F]{2}:[0-9a-fA-F]{2})\.0:\s+GPU has fallen off the bus\.?`
 )
 
 var (
-	compiledRegexNVRMXidCombined = regexp.MustCompile(RegexNVRMXidCombined)
-	compiledRegexNVRMXidExtended = regexp.MustCompile(RegexNVRMXidExtended)
+	compiledRegexNVRMXidCombined            = regexp.MustCompile(RegexNVRMXidCombined)
+	compiledRegexNVRMXidExtended            = regexp.MustCompile(RegexNVRMXidExtended)
+	compiledRegexNVRMFallenOffBusMultiline  = regexp.MustCompile(RegexNVRMFallenOffBusMultiline)
+	compiledRegexNVRMFallenOffBusSingleLine = regexp.MustCompile(RegexNVRMFallenOffBusSingleLine)
 )
 
 const (
@@ -197,18 +211,56 @@ func Match(line string) *XidError {
 	}
 
 	extractedID, deviceUUID := ExtractNVRMXidInfo(line)
-	if extractedID == 0 {
-		return nil
+	if extractedID != 0 {
+		detail, ok := GetDetail(extractedID)
+		if !ok {
+			return nil
+		}
+		return &XidError{
+			Xid:        extractedID,
+			DeviceUUID: deviceUUID,
+			Detail:     detail,
+		}
 	}
-	detail, ok := GetDetail(extractedID)
-	if !ok {
-		return nil
+
+	if xid, fallbackDeviceUUID := extractFallenOffBusXidInfo(line); xid != 0 {
+		// NVIDIA docs map "GPU has fallen off the bus" to XID 79 even when newer
+		// driver logs omit the explicit "Xid (...): 79" prefix.
+		detail, ok := GetDetail(xid)
+		if !ok {
+			return nil
+		}
+		return &XidError{
+			Xid:        xid,
+			DeviceUUID: fallbackDeviceUUID,
+			Detail:     detail,
+		}
+	}
+
+	return nil
+}
+
+func extractFallenOffBusXidInfo(line string) (int, string) {
+	if match := compiledRegexNVRMFallenOffBusSingleLine.FindStringSubmatch(line); len(match) == 2 {
+		return 79, normalizePCIBDF(match[1])
+	}
+
+	if match := compiledRegexNVRMFallenOffBusMultiline.FindStringSubmatch(line); len(match) == 2 {
+		return 79, normalizePCIBDF(match[1])
+	}
+
+	return 0, ""
+}
+
+func normalizePCIBDF(deviceUUID string) string {
+	normalized := strings.TrimSpace(strings.TrimPrefix(deviceUUID, "PCI:"))
+	if strings.Count(normalized, ":") == 1 {
+		normalized = "0000:" + normalized
 	}
-	return &XidError{
-		Xid:        extractedID,
-		DeviceUUID: deviceUUID,
-		Detail:     detail,
+	if normalized == "" {
+		return ""
 	}
+	return "PCI:" + normalized
 }
 
 // MessageToInject represents a synthetic kernel message snippet and its log priority.

components/accelerator/nvidia/xid/kmsg_test.go

@@ -118,6 +118,34 @@ func TestMatch(t *testing.T) {
 			expectedXid:    79,
 			expectedDevice: "PCI:0000:05:00",
 		},
+		{
+			name: "multiline fallen-off-bus fallback without explicit Xid",
+			input: "NVRM: The NVIDIA GPU 0000:18:00.0\n" +
+				"NVRM: (PCI ID: 10de:2901) installed in this system has\n" +
+				"NVRM: fallen off the bus and is not responding to commands.",
+			expectNil:      false,
+			expectedXid:    79,
+			expectedDevice: "PCI:0000:18:00",
+		},
+		{
+			name:           "single-line fallen-off-bus fallback without explicit Xid",
+			input:          "NVRM: GPU 0000:29:00.0: GPU has fallen off the bus.",
+			expectNil:      false,
+			expectedXid:    79,
+			expectedDevice: "PCI:0000:29:00",
+		},
+		{
+			name:           "single-line fallen-off-bus fallback normalizes missing PCI domain",
+			input:          "NVRM: GPU 18:00.0: GPU has fallen off the bus.",
+			expectNil:      false,
+			expectedXid:    79,
+			expectedDevice: "PCI:0000:18:00",
+		},
+		{
+			name:      "fallen-off-bus text without valid BDF is ignored",
+			input:     "NVRM: The NVIDIA GPU invalid-bdf.0\nNVRM: fallen off the bus and is not responding to commands.",
+			expectNil: true,
+		},
 		{
 			name:           "valid XID error without PCI prefix",
 			input:          "[...] NVRM: Xid (0000:03:00): 14, Channel 00000001",
@@ -257,6 +285,74 @@ func TestMatchDmesgWithXid119(t *testing.T) {
 	}
 }
 
+func TestMatchReturnsNilWhenXidDetailMissing(t *testing.T) {
+	t.Parallel()
+
+	result := Match("NVRM: Xid (PCI:0000:05:00): 99999, unknown error")
+	require.Nil(t, result)
+}
+
+func TestMatchReturnsNilWhenFallbackDetailMissing(t *testing.T) {
+	originalDetails := details
+
+	clonedDetails := make(map[int]Detail, len(details))
+	for xid, detail := range details {
+		clonedDetails[xid] = detail
+	}
+	delete(clonedDetails, 79)
+	details = clonedDetails
+
+	t.Cleanup(func() {
+		details = originalDetails
+	})
+
+	result := Match("NVRM: GPU 0000:29:00.0: GPU has fallen off the bus.")
+	require.Nil(t, result)
+}
+
+func TestNormalizePCIBDF(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "already normalized",
+			input:    "PCI:0000:29:00",
+			expected: "PCI:0000:29:00",
+		},
+		{
+			name:     "adds domain when missing",
+			input:    "18:00",
+			expected: "PCI:0000:18:00",
+		},
+		{
+			name:     "trims whitespace and prefix",
+			input:    "PCI:0000:18:00  ",
+			expected: "PCI:0000:18:00",
+		},
+		{
+			name:     "empty input",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "empty after PCI prefix removal",
+			input:    "PCI:",
+			expected: "",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, normalizePCIBDF(tt.input))
+		})
+	}
+}
+
 func TestGetMessageToInject(t *testing.T) {
 	t.Parallel()
 

docs/INSTALL.md

@@ -7,10 +7,10 @@ curl -fsSL https://pkg.gpud.dev/install.sh | sh
 gpud up
 ```
 
-To specify a version
+To install the latest published version explicitly:
 
 ```bash
-curl -fsSL https://pkg.gpud.dev/install.sh | sh -s v0.10.0
+curl -fsSL https://pkg.gpud.dev/install.sh | sh -s $(curl -fsSL https://pkg.gpud.dev/unstable_latest.txt)
 ```
 
 Then open [localhost:15132](https://localhost:15132) for the local web UI.

docs/TUTORIALS.md

@@ -7,7 +7,7 @@ What is [GPUd](https://github.com/leptonai/gpud)? GPUd is designed to ensure GPU
 *(see [Install GPUd](./INSTALL.md) for more and [release page](https://github.com/leptonai/gpud/releases) for latest versions)*
 
 ```bash
-curl -fsSL https://pkg.gpud.dev/install.sh | sh -s v0.9.1
+curl -fsSL https://pkg.gpud.dev/install.sh | sh -s $(curl -fsSL https://pkg.gpud.dev/unstable_latest.txt)
 
 gpud --version
 ```
@@ -34,7 +34,7 @@ sudo systemctl stop gpud.service
 sudo rm -f /usr/sbin/gpud
 sudo rm -f /usr/local/bin/gpud
 
-curl -fsSL https://pkg.gpud.dev/install.sh | sh -s v0.10.0
+curl -fsSL https://pkg.gpud.dev/install.sh | sh -s $(curl -fsSL https://pkg.gpud.dev/unstable_latest.txt)
 which gpud
 
 # (optional) if you already installed an old GPUd

go.mod

@@ -1,6 +1,6 @@
 module github.com/leptonai/gpud
 
-go 1.24.7
+go 1.25.7
 
 require (
 	github.com/NVIDIA/go-nvlib v0.8.1