feat: add e2e tests

Signed-off-by: Tanisha goyal <[email protected]>

Author: tanishagoyal2

distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml

@@ -216,7 +216,10 @@ postgresql:
 
               -- Metadata
               created_at TIMESTAMPTZ DEFAULT NOW(),
-              updated_at TIMESTAMPTZ DEFAULT NOW()
+              updated_at TIMESTAMPTZ DEFAULT NOW(),
+
+              -- Event handling strategy
+              processing_strategy VARCHAR(50)
           );
 
           -- Indexes for health_events

docs/postgresql-schema.sql

@@ -103,7 +103,10 @@ CREATE TABLE IF NOT EXISTS health_events (
 
     -- Metadata
     created_at TIMESTAMPTZ DEFAULT NOW(),
-    updated_at TIMESTAMPTZ DEFAULT NOW()
+    updated_at TIMESTAMPTZ DEFAULT NOW(),
+
+    -- Event handling strategy
+    processing_strategy VARCHAR(50)
 );
 
 -- Indexes for health_events

fault-quarantine/pkg/evaluator/rule_evaluator_test.go

@@ -260,6 +260,7 @@ func TestRoundTrip(t *testing.T) {
 			"nanos":   float64(eventTime.GetNanos()),
 		},
 		"nodeName":            "test-node",
+		"processingStrategy":  float64(0),
 		"quarantineOverrides": nil,
 		"drainOverrides":      nil,
 	}

tests/fault_quarantine_test.go

@@ -23,6 +23,7 @@ import (
 
 	"tests/helpers"
 
+	"github.com/nvidia/nvsentinel/data-models/pkg/protos"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
@@ -228,3 +229,67 @@ func TestPreCordonedNodeHandling(t *testing.T) {
 
 	testEnv.Test(t, feature.Feature())
 }
+
+func TestFaultQuarantineWithProcessingStrategy(t *testing.T) {
+	feature := features.New("TestFaultQuarantineWithProcessingStrategy").
+		WithLabel("suite", "fault-quarantine-with-processing-strategy")
+
+	var testCtx *helpers.QuarantineTestContext
+
+	feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client, err := c.NewClient()
+		require.NoError(t, err)
+
+		var newCtx context.Context
+		newCtx, testCtx = helpers.SetupQuarantineTest(ctx, t, c, "data/managed-by-nvsentinel-configmap.yaml")
+
+		err = helpers.SetNodeManagedByNVSentinel(newCtx, client, testCtx.NodeName, true)
+		require.NoError(t, err)
+
+		return newCtx
+	})
+
+	feature.Assess("Check that node is not quarantined for STORE_ONLY events", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client, err := c.NewClient()
+		require.NoError(t, err)
+
+		event := helpers.NewHealthEvent(testCtx.NodeName).
+			WithErrorCode("79").
+			WithMessage("XID error occurred").
+			WithProcessingStrategy(int(protos.ProcessingStrategy_STORE_ONLY))
+		helpers.SendHealthEvent(ctx, t, event)
+
+		helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{
+			ExpectCordoned:   false,
+			ExpectAnnotation: false,
+		})
+
+		return ctx
+	})
+
+	feature.Assess("Check that node is quarantined for EXECUTE_REMEDIATION events", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client, err := c.NewClient()
+		require.NoError(t, err)
+
+		event := helpers.NewHealthEvent(testCtx.NodeName).
+			WithErrorCode("79").
+			WithMessage("XID error occurred").
+			WithProcessingStrategy(int(protos.ProcessingStrategy_EXECUTE_REMEDIATION))
+		helpers.SendHealthEvent(ctx, t, event)
+
+		helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{
+			ExpectCordoned:   true,
+			ExpectAnnotation: true,
+		})
+
+		return ctx
+	})
+
+	feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		helpers.SendHealthyEvent(ctx, t, testCtx.NodeName)
+
+		return helpers.TeardownQuarantineTest(ctx, t, c)
+	})
+
+	testEnv.Test(t, feature.Feature())
+}

tests/helpers/healthevent.go

@@ -45,6 +45,7 @@ type HealthEventTemplate struct {
 	Metadata            map[string]string    `json:"metadata,omitempty"`
 	QuarantineOverrides *QuarantineOverrides `json:"quarantineOverrides,omitempty"`
 	NodeName            string               `json:"nodeName"`
+	ProcessingStrategy  int                  `json:"processingStrategy,omitempty"`
 }
 
 type EntityImpacted struct {
@@ -149,6 +150,11 @@ func (h *HealthEventTemplate) WithRecommendedAction(action int) *HealthEventTemp
 	return h
 }
 
+func (h *HealthEventTemplate) WithProcessingStrategy(strategy int) *HealthEventTemplate {
+	h.ProcessingStrategy = strategy
+	return h
+}
+
 func (h *HealthEventTemplate) WriteToTempFile() (string, error) {
 	tempFile, err := os.CreateTemp("", "health-event-*.json")
 	if err != nil {

tests/helpers/kube.go

@@ -384,6 +384,30 @@ func WaitForNodeEvent(ctx context.Context, t *testing.T, c klient.Client, nodeNa
 	}, EventuallyWaitTimeout, WaitInterval, "node %s should have event %v", nodeName, expectedEvent)
 }
 
+func EnsureNodeEventNotPresent(ctx context.Context, t *testing.T,
+	c klient.Client, nodeName string, eventType, eventReason string) {
+	t.Helper()
+
+	require.Never(t, func() bool {
+		events, err := GetNodeEvents(ctx, c, nodeName, eventType)
+		if err != nil {
+			t.Logf("failed to get events for node %s: %v", nodeName, err)
+			return false
+		}
+
+		for _, event := range events.Items {
+			if event.Type == eventType && event.Reason == eventReason {
+				t.Logf("node %s has event %v", nodeName, event)
+				return true
+			}
+		}
+
+		t.Logf("node %s does not have event %v", nodeName, eventType)
+
+		return false
+	}, NeverWaitTimeout, WaitInterval, "node %s should not have event %v", nodeName, eventType, eventReason)
+}
+
 // SelectTestNodeFromUnusedPool selects an available test node from the cluster.
 // Prefers uncordoned nodes but will fall back to the first node if none are available.
 func SelectTestNodeFromUnusedPool(ctx context.Context, t *testing.T, client klient.Client) string {

tests/platform-connector_test.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+import (
+	"context"
+	"testing"
+	"tests/helpers"
+
+	"github.com/nvidia/nvsentinel/data-models/pkg/protos"
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+type PlatformConnectorTestContext struct {
+	NodeName        string
+	ConfigMapBackup []byte
+	TestNamespace   string
+}
+
+func TestPlatformConnectorWithProcessingStrategy(t *testing.T) {
+	feature := features.New("TestPlatformConnector").
+		WithLabel("suite", "platform-connector")
+
+	var testCtx *PlatformConnectorTestContext
+
+	feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client, err := c.NewClient()
+		require.NoError(t, err)
+
+		nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client)
+
+		testCtx = &PlatformConnectorTestContext{
+			NodeName: nodeName,
+		}
+
+		return ctx
+	})
+
+	feature.Assess("Check that platform connector is not applying node events/conditions for STORE_ONLY events", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client, err := c.NewClient()
+		require.NoError(t, err)
+
+		event := helpers.NewHealthEvent(testCtx.NodeName).
+			WithErrorCode(helpers.ERRORCODE_79).
+			WithMessage("XID error occurred").
+			WithProcessingStrategy(int(protos.ProcessingStrategy_STORE_ONLY))
+		helpers.SendHealthEvent(ctx, t, event)
+
+		t.Logf("Node %s should not have condition SysLogsXIDError", testCtx.NodeName)
+		helpers.EnsureNodeConditionNotPresent(ctx, t, client, testCtx.NodeName, "SysLogsXIDError")
+
+		event = helpers.NewHealthEvent(testCtx.NodeName).
+			WithErrorCode(helpers.ERRORCODE_31).
+			WithMessage("XID error occurred").
+			WithFatal(false).
+			WithProcessingStrategy(int(protos.ProcessingStrategy_STORE_ONLY))
+		helpers.SendHealthEvent(ctx, t, event)
+
+		t.Logf("Node %s should not have event SysLogsXIDError", testCtx.NodeName)
+		helpers.EnsureNodeEventNotPresent(ctx, t, client, testCtx.NodeName, "SysLogsXIDError", "SysLogsXIDErrorIsNotHealthy")
+
+		event = helpers.NewHealthEvent(testCtx.NodeName).
+			WithErrorCode(helpers.ERRORCODE_79).
+			WithMessage("XID error occurred").
+			WithProcessingStrategy(int(protos.ProcessingStrategy_EXECUTE_REMEDIATION))
+		helpers.SendHealthEvent(ctx, t, event)
+
+		t.Logf("Node %s should have condition SysLogsXIDError", testCtx.NodeName)
+		helpers.CheckNodeConditionExists(ctx, client, testCtx.NodeName, "SysLogsXIDError", "SysLogsXIDErrorIsNotHealthy")
+
+		event = helpers.NewHealthEvent(testCtx.NodeName).
+			WithErrorCode(helpers.ERRORCODE_31).
+			WithMessage("XID error occurred").
+			WithFatal(false).
+			WithProcessingStrategy(int(protos.ProcessingStrategy_EXECUTE_REMEDIATION))
+		helpers.SendHealthEvent(ctx, t, event)
+
+		t.Logf("Node %s should have event SysLogsXIDError", testCtx.NodeName)
+		helpers.CheckNodeEventExists(ctx, client, testCtx.NodeName, "SysLogsXIDError", "SysLogsXIDErrorIsNotHealthy")
+
+		return ctx
+	})
+
+	feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		helpers.SendHealthyEvent(ctx, t, testCtx.NodeName)
+		return ctx
+	})
+
+	testEnv.Test(t, feature.Feature())
+}