Add NodeNetworkInterfaceDown alert

Signed-off-by: João Vilaça <[email protected]>

Author: machadovilaca

hack/prom-rule-ci/observability-prom-rules-tests.yaml

@@ -70,3 +70,52 @@ tests:
               operator_health_impact: "none"
               kubernetes_operator_part_of: "kubevirt"
               kubernetes_operator_component: "cnv-observability"
+
+  - interval: 1m
+    input_series:
+      - series: 'node_network_up{instance="n1.cnv.redhat.com", device="eno0"}'
+        values: "1+0x10 0+0x10"
+      - series: 'node_network_up{instance="n2.cnv.redhat.com", device="eno0"}'
+        values: "1+0x10 0+0x10"
+      - series: 'node_network_up{instance="n2.cnv.redhat.com", device="eno1"}'
+        values: "0+0x20"
+
+    alert_rule_test:
+      - eval_time: 8m
+        alertname: NodeNetworkInterfaceDown
+        exp_alerts:
+          - exp_annotations:
+              summary: "Network interfaces are down"
+              description: "1 network devices have been down on instance n2.cnv.redhat.com for more than 5 minutes."
+              runbook_url: "https://kubevirt.io/monitoring/runbooks/NodeNetworkInterfaceDown"
+            exp_labels:
+              instance: "n2.cnv.redhat.com"
+              severity: "warning"
+              operator_health_impact: "none"
+              kubernetes_operator_part_of: "kubevirt"
+              kubernetes_operator_component: "cnv-observability"
+
+      - eval_time: 18m
+        alertname: NodeNetworkInterfaceDown
+        exp_alerts:
+          - exp_annotations:
+              summary: "Network interfaces are down"
+              description: "2 network devices have been down on instance n2.cnv.redhat.com for more than 5 minutes."
+              runbook_url: "https://kubevirt.io/monitoring/runbooks/NodeNetworkInterfaceDown"
+            exp_labels:
+              instance: "n2.cnv.redhat.com"
+              severity: "warning"
+              operator_health_impact: "none"
+              kubernetes_operator_part_of: "kubevirt"
+              kubernetes_operator_component: "cnv-observability"
+
+          - exp_annotations:
+              summary: "Network interfaces are down"
+              description: "1 network devices have been down on instance n1.cnv.redhat.com for more than 5 minutes."
+              runbook_url: "https://kubevirt.io/monitoring/runbooks/NodeNetworkInterfaceDown"
+            exp_labels:
+              instance: "n1.cnv.redhat.com"
+              severity: "warning"
+              operator_health_impact: "none"
+              kubernetes_operator_part_of: "kubevirt"
+              kubernetes_operator_component: "cnv-observability"

pkg/monitoring/observability/rules/alerts/cluster_alerts.go

@@ -34,5 +34,18 @@ func clusterAlerts() []promv1.Rule {
 				"operator_health_impact": "none",
 			},
 		},
+		{
+			Alert: "NodeNetworkInterfaceDown",
+			Expr:  intstr.FromString("count by (instance) (node_network_up{device!~\"veth.+|tunbr\"} == 0) > 0"),
+			For:   ptr.To(promv1.Duration("5m")),
+			Annotations: map[string]string{
+				"summary":     "Network interfaces are down",
+				"description": "{{ $value }} network devices have been down on instance {{ $labels.instance }} for more than 5 minutes.",
+			},
+			Labels: map[string]string{
+				"severity":               "warning",
+				"operator_health_impact": "none",
+			},
+		},
 	}
 }