test(nvlink): cover post-RX dedup paths

Author: gyuho

components/accelerator/nvidia/nvlink/kmsg.go

@@ -7,9 +7,10 @@ const (
 	// update an NVLink post-RX-detection link mask.
 	EventNamePostRxDetectFailure = "nvlink_post_rx_detect_failure"
 
-	// NVIDIA's published GH100 kernel source emits these errors when updating a
-	// post-RX-detection link mask fails. The source does not classify this error
-	// as a driver hang.
+	// NVIDIA's published GH100 discovery path calls
+	// knvlinkUpdatePostRxDetectLinkMask and reports the peer-mask failure when
+	// that update fails. Some driver builds also emit the paired update failure.
+	// The source does not classify this error as a driver hang.
 	// ref. https://github.com/NVIDIA/open-gpu-kernel-modules/blob/452cec62d827034798072827d3866d1881662b77/src/nvidia/src/kernel/gpu/nvlink/arch/hopper/kernel_nvlink_gh100.c#L173-L249
 	RegexPostRxDetectFailureKMessage = `NVRM: (?:knvlinkUpdatePostRxDetectLinkMask_[A-Za-z0-9_]+: Failed to update Rx Detect Link mask!|knvlinkDiscoverPostRxDetLinks_[A-Za-z0-9_]+: Getting peer[0-9]+(?:'s)? postRxDetLinkMask failed!)`
 

components/accelerator/nvidia/nvlink/kmsg_test.go

@@ -17,6 +17,11 @@ func TestMatch(t *testing.T) {
 			line:  "NVRM: knvlinkDiscoverPostRxDetLinks_GH100: Getting peer0's postRxDetLinkMask failed!",
 			match: true,
 		},
+		{
+			name:  "incident link-mask update signature",
+			line:  "NVRM: knvlinkUpdatePostRxDetectLinkMask_IMPL: Failed to update Rx Detect Link mask!",
+			match: true,
+		},
 		{
 			name:  "other GPU architecture and peer",
 			line:  "kernel: NVRM: knvlinkDiscoverPostRxDetLinks_GB100_REV2: Getting peer17 postRxDetLinkMask failed!",

components/accelerator/nvidia/nvlink/mock_component_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"os"
+	"reflect"
 	"runtime"
 	"testing"
 	"time"
@@ -27,13 +28,15 @@ func TestNew_WithEventStoreAsRoot_KmsgSyncer_WithMockey(t *testing.T) {
 
 		bucket := &memoryEventBucket{}
 		var match kmsg.MatchFunc
+		var syncerOpts []kmsg.OpOption
 		mockey.Mock(kmsg.NewSyncer).To(func(
 			_ context.Context,
 			matchFunc kmsg.MatchFunc,
 			eventBucket eventstore.Bucket,
-			_ ...kmsg.OpOption,
+			opts ...kmsg.OpOption,
 		) (*kmsg.Syncer, error) {
 			match = matchFunc
+			syncerOpts = opts
 			assert.Same(t, bucket, eventBucket)
 			return &kmsg.Syncer{}, nil
 		}).Build()
@@ -48,6 +51,8 @@ func TestNew_WithEventStoreAsRoot_KmsgSyncer_WithMockey(t *testing.T) {
 		require.NotNil(t, c.kmsgSyncer)
 		assert.Nil(t, c.readAllKmsg, "daemon components should rely on the kmsg syncer")
 		require.NotNil(t, match)
+		require.Len(t, syncerOpts, 1)
+		assert.Equal(t, int(defaultKmsgEventDedupWindow.Seconds()), getKmsgCacheKeyTruncateSeconds(t, syncerOpts[0]))
 
 		name, message := match("NVRM: knvlinkDiscoverPostRxDetLinks_GH100: Getting peer0's postRxDetLinkMask failed!")
 		assert.Equal(t, EventNamePostRxDetectFailure, name)
@@ -63,6 +68,17 @@ func TestNew_WithEventStoreAsRoot_KmsgSyncer_WithMockey(t *testing.T) {
 	})
 }
 
+func getKmsgCacheKeyTruncateSeconds(t *testing.T, opt kmsg.OpOption) int {
+	t.Helper()
+	op := &kmsg.Op{}
+	opt(op)
+
+	v := reflect.ValueOf(op).Elem().FieldByName("cacheKeyTruncateSeconds")
+	require.True(t, v.IsValid(), "cacheKeyTruncateSeconds field must exist")
+	require.Equal(t, reflect.Int, v.Kind(), "cacheKeyTruncateSeconds must be int")
+	return int(v.Int())
+}
+
 func TestNew_WithEventStoreAsRoot_KmsgSyncerError_WithMockey(t *testing.T) {
 	mockey.PatchConvey("New cleans up when the kmsg syncer cannot be created", t, func() {
 		mockey.Mock(os.Geteuid).Return(0).Build()

components/accelerator/nvidia/nvlink/post_rx_detect_failure_test.go

@@ -213,7 +213,9 @@ func TestCheckDetectsAndDeduplicatesPostRxDetectFailureForOneShotScan(t *testing
 		{Timestamp: metav1.NewTime(baseTime.Add(8*time.Second + 3827*time.Microsecond)), Message: "[6619723.539539] NVRM: knvlinkDiscoverPostRxDetLinks_GH100: Getting peer0's postRxDetLinkMask failed!"},
 	}
 	for _, message := range reportedKmsgs {
-		assert.True(t, HasPostRxDetectFailure(message.Message), message.Message)
+		eventName, eventMessage := Match(message.Message)
+		assert.Equal(t, EventNamePostRxDetectFailure, eventName, message.Message)
+		assert.Equal(t, postRxDetectFailureMessage, eventMessage, message.Message)
 	}
 	c.readAllKmsg = func(context.Context) ([]kmsg.Message, error) {
 		return reportedKmsgs, nil
@@ -228,6 +230,12 @@ func TestCheckDetectsAndDeduplicatesPostRxDetectFailureForOneShotScan(t *testing
 	state := result.HealthStates()[0]
 	require.NotNil(t, state.SuggestedActions)
 	assert.Equal(t, []apiv1.RepairActionType{apiv1.RepairActionTypeRebootSystem}, state.SuggestedActions.RepairActions)
+
+	reportedKmsgs = append(reportedKmsgs, kmsg.Message{
+		Timestamp: metav1.NewTime(baseTime.Add(defaultKmsgEventDedupWindow)),
+		Message:   "NVRM: knvlinkUpdatePostRxDetectLinkMask_IMPL: Failed to update Rx Detect Link mask!",
+	})
+	assert.Equal(t, "matched 2 kmsg(s)", c.Check().String(), "a later dedup window should report the failure again")
 }
 
 func TestCheckHandlesOneShotKmsgReadResult(t *testing.T) {