feat: add event handling strategy changes in csp-health-monitor

data-models/protobufs/health_event.proto

@@ -29,6 +29,14 @@ message HealthEvents {
   repeated HealthEvent events = 2;
 }
 
+// ProcessingStrategy defines how downstream modules should handle the event.
+// EXECUTE_REMEDIATION: normal behavior; downstream modules may update cluster state.
+// STORE_ONLY: observability-only behavior; event should be persisted/exported but should not modify cluster resources.
+enum ProcessingStrategy {
+  EXECUTE_REMEDIATION = 0;
+  STORE_ONLY = 1;
+}
+
 enum RecommendedAction {
   NONE = 0;
   COMPONENT_RESET = 2;
@@ -66,6 +74,7 @@ message HealthEvent {
   string nodeName = 13;
   BehaviourOverrides quarantineOverrides = 14;
   BehaviourOverrides drainOverrides = 15;
+  ProcessingStrategy processingStrategy = 16;
 }
 
 message BehaviourOverrides {

distros/kubernetes/nvsentinel/charts/csp-health-monitor/templates/deployment.yaml

@@ -158,6 +158,7 @@ spec:
           - "--database-client-cert-mount-path={{ .Values.clientCertMountPath }}"
           - "--uds-path=/run/nvsentinel/nvsentinel.sock"
           - "--metrics-port=2113"
+          - "--processing-strategy={{ .Values.processingStrategy }}"
           resources:
             {{- toYaml .Values.quarantineTriggerEngine.resources | default .Values.resources | nindent 12 }}
           ports:

distros/kubernetes/nvsentinel/charts/csp-health-monitor/values.yaml

@@ -49,6 +49,13 @@ tolerations: []
 
 podAnnotations: {}
 
+# Processing strategy for health events
+# valid values: EXECUTE_REMEDIATION, STORE_ONLY
+# default: EXECUTE_REMEDIATION
+# EXECUTE_REMEDIATION: normal behavior; downstream modules may update cluster state.
+# STORE_ONLY: observability-only behavior; event should be persisted/exported but should not modify cluster resources (i.e., no node conditions, no quarantine, no drain, no remediation).
+processingStrategy: EXECUTE_REMEDIATION
+
 # Log verbosity level for the main CSP health monitor container (e.g. "debug", "info", "warn", "error")
 logLevel: info
 

event-exporter/pkg/transformer/cloudevents.go

@@ -63,6 +63,7 @@ func ToCloudEvent(event *pb.HealthEvent, metadata map[string]string) (*CloudEven
 		"entitiesImpacted":   entities,
 		"generatedTimestamp": timestamp,
 		"nodeName":           event.NodeName,
+		"processingStrategy": event.ProcessingStrategy.String(),
 	}
 
 	if len(event.Metadata) > 0 {

event-exporter/pkg/transformer/cloudevents_test.go

@@ -66,6 +66,7 @@ func TestToCloudEvent(t *testing.T) {
 					Force: false,
 					Skip:  true,
 				},
+				ProcessingStrategy: pb.ProcessingStrategy_STORE_ONLY,
 			},
 			metadata: map[string]string{
 				"cluster":     "prod-cluster-1",
@@ -102,6 +103,9 @@ func TestToCloudEvent(t *testing.T) {
 				if healthEvent["recommendedAction"] != "RESTART_VM" {
 					t.Errorf("recommendedAction = %v, want %v", healthEvent["recommendedAction"], "RESTART_VM")
 				}
+				if healthEvent["processingStrategy"] != "STORE_ONLY" {
+					t.Errorf("processingStrategy = %v, want STORE_ONLY", healthEvent["processingStrategy"])
+				}
 
 				entities := healthEvent["entitiesImpacted"].([]map[string]any)
 				if len(entities) != 2 {

fault-quarantine/pkg/evaluator/rule_evaluator_test.go

@@ -260,6 +260,7 @@ func TestRoundTrip(t *testing.T) {
 			"nanos":   float64(eventTime.GetNanos()),
 		},
 		"nodeName":            "test-node",
+		"processingStrategy":  float64(0),
 		"quarantineOverrides": nil,
 		"drainOverrides":      nil,
 	}

fault-quarantine/pkg/initializer/init.go

@@ -63,7 +63,7 @@ func InitializeAll(ctx context.Context, params InitializationParams) (*Component
 	}
 
 	builder := client.GetPipelineBuilder()
-	pipeline := builder.BuildAllHealthEventInsertsPipeline()
+	pipeline := builder.BuildProcessableHealthEventInsertsPipeline()
 
 	var tomlCfg config.TomlConfig
 	if err := configmanager.LoadTOMLConfig(params.TomlConfigPath, &tomlCfg); err != nil {

health-monitors/csp-health-monitor/cmd/maintenance-notifier/main.go

@@ -58,6 +58,7 @@ type appConfig struct {
 	udsPath                     string
 	databaseClientCertMountPath string
 	metricsPort                 string
+	processingStrategy          string
 }
 
 func parseFlags() *appConfig {
@@ -71,6 +72,8 @@ func parseFlags() *appConfig {
 		"Directory where database client tls.crt, tls.key, and ca.crt are mounted.",
 	)
 	flag.StringVar(&cfg.metricsPort, "metrics-port", defaultMetricsPortSidecar, "Port for the sidecar Prometheus metrics.")
+	flag.StringVar(&cfg.processingStrategy, "processing-strategy", "EXECUTE_REMEDIATION",
+		"Event processing strategy: EXECUTE_REMEDIATION or STORE_ONLY")
 
 	// Parse flags after initialising klog
 	flag.Parse()
@@ -213,7 +216,15 @@ func run() error {
 			return fmt.Errorf("kubernetes client setup failed: %w", err)
 		}
 
-		engine := trigger.NewEngine(cfg, store, platformConnectorClient, k8sClient)
+		value, ok := pb.ProcessingStrategy_value[appCfg.processingStrategy]
+		if !ok {
+			return fmt.Errorf("invalid processingStrategy %q (expected EXECUTE_REMEDIATION or STORE_ONLY)",
+				appCfg.processingStrategy)
+		}
+
+		slog.Info("Event handling strategy configured", "processingStrategy", appCfg.processingStrategy)
+
+		engine := trigger.NewEngine(cfg, store, platformConnectorClient, k8sClient, pb.ProcessingStrategy(value))
 
 		slog.Info("Trigger engine starting...")
 		engine.Start(gCtx)

health-monitors/csp-health-monitor/pkg/triggerengine/trigger.go

@@ -56,13 +56,14 @@ const (
 // Engine polls the datastore for maintenance events and forwards the
 // corresponding health signals to NVSentinel through the UDS connector.
 type Engine struct {
-	store           datastore.Store
-	udsClient       pb.PlatformConnectorClient
-	config          *config.Config
-	pollInterval    time.Duration
-	k8sClient       kubernetes.Interface
-	monitoredNodes  sync.Map // Track which nodes are currently being monitored
-	monitorInterval time.Duration
+	store              datastore.Store
+	udsClient          pb.PlatformConnectorClient
+	config             *config.Config
+	pollInterval       time.Duration
+	k8sClient          kubernetes.Interface
+	monitoredNodes     sync.Map // Track which nodes are currently being monitored
+	monitorInterval    time.Duration
+	processingStrategy pb.ProcessingStrategy
 }
 
 // NewEngine constructs a ready-to-run Engine instance.
@@ -71,14 +72,16 @@ func NewEngine(
 	store datastore.Store,
 	udsClient pb.PlatformConnectorClient,
 	k8sClient kubernetes.Interface,
+	processingStrategy pb.ProcessingStrategy,
 ) *Engine {
 	return &Engine{
-		config:          cfg,
-		store:           store,
-		udsClient:       udsClient,
-		pollInterval:    time.Duration(cfg.MaintenanceEventPollIntervalSeconds) * time.Second,
-		k8sClient:       k8sClient,
-		monitorInterval: defaultMonitorInterval,
+		config:             cfg,
+		store:              store,
+		udsClient:          udsClient,
+		pollInterval:       time.Duration(cfg.MaintenanceEventPollIntervalSeconds) * time.Second,
+		k8sClient:          k8sClient,
+		monitorInterval:    defaultMonitorInterval,
+		processingStrategy: processingStrategy,
 	}
 }
 
@@ -343,13 +346,14 @@ func (e *Engine) mapMaintenanceEventToHealthEvent(
 	}
 
 	healthEvent := &pb.HealthEvent{
-		Agent:             "csp-health-monitor", // Consistent agent name
-		ComponentClass:    event.ResourceType,   // e.g., "EC2", "gce_instance"
-		CheckName:         "CSPMaintenance",     // Consistent check name
-		IsFatal:           isFatal,
-		IsHealthy:         isHealthy,
-		Message:           message,
-		RecommendedAction: pb.RecommendedAction(actionEnum),
+		Agent:              "csp-health-monitor", // Consistent agent name
+		ComponentClass:     event.ResourceType,   // e.g., "EC2", "gce_instance"
+		CheckName:          "CSPMaintenance",     // Consistent check name
+		IsFatal:            isFatal,
+		IsHealthy:          isHealthy,
+		ProcessingStrategy: e.processingStrategy,
+		Message:            message,
+		RecommendedAction:  pb.RecommendedAction(actionEnum),
 		EntitiesImpacted: []*pb.Entity{
 			{
 				EntityType:  event.ResourceType,