fix(alert-tuning): [SPRE-5878] suppress EtcdHighNumberOfLeaderChanges during cluster upgrades

rhobs/alerting/data_plane/prometheus.cluster_capacity_alerts.yaml

@@ -79,7 +79,9 @@ spec:
 
       - alert: EtcdHighNumberOfLeaderChanges
         expr: |
-          increase(etcd_server_leader_changes_seen_total{namespace="openshift-etcd"}[10m]) > 2
+          (increase(etcd_server_leader_changes_seen_total{namespace="openshift-etcd"}[10m]) > 2)
+          unless on (source_cluster)
+          cluster_upgrade_ongoing
         for: 1m
         labels:
           severity: warning

rhobs/alerting/data_plane/prometheus.cluster_ongoing_updrade_alert.yaml

@@ -12,9 +12,12 @@ spec:
       rules:
         - alert: ClusterUpgradeOngoing
           expr: >-
-            (aus_cluster_version_remaining_soak_days <= -2 >= -3)
-            * on(cluster_uuid) group_left(cluster_name, current_version)
-            aus_cluster_upgrade_policy_info{org_name="RHTAP"}
+            label_replace(
+              (aus_cluster_version_remaining_soak_days <= -2 >= -3)
+              * on(cluster_uuid) group_left(cluster_name, current_version)
+              aus_cluster_upgrade_policy_info{org_name="RHTAP"},
+              "source_cluster", "$1", "cluster_name", "(.*)"
+            )
           for: 1m
           labels:
             severity: info

rhobs/recording/cluster_upgrade_recording_rules.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: rhtap-cluster-upgrade
+  labels:
+    tenant: rhtap
+spec:
+  groups:
+    - name: cluster_upgrade
+      interval: 1m
+      rules:
+        - record: cluster_upgrade_ongoing
+          expr: |
+            max by (source_cluster) (
+              ALERTS{alertname="ClusterUpgradeOngoing", alertstate="firing"}
+            )

test/promql/tests/data_plane/cluster_capacity_test.yaml

@@ -140,7 +140,7 @@ tests:
 
   - interval: 1m
     input_series:
-      # Etcd No Leader failures within threshold limit, so it will be alerted.
+      # Leader changes exceed threshold, no upgrade in progress — alert fires.
       - series: 'etcd_server_leader_changes_seen_total{namespace="openshift-etcd",source_cluster="cluster01"}'
         values: '3+2x60'
 
@@ -162,8 +162,46 @@ tests:
 
   - interval: 1m
     input_series:
-      # Etcd has Leader within threshold limit, so it will not be alerted.
-      - series: 'etcd_server_leader_changes_seen_total{source_cluster="cluster01"}'
+      # Leader changes exceed threshold BUT a cluster upgrade is ongoing for the same cluster.
+      # The unless operator suppresses the alert — upgrade-induced re-elections are expected behaviour.
+      - series: 'etcd_server_leader_changes_seen_total{namespace="openshift-etcd",source_cluster="cluster01"}'
+        values: '3+2x60'
+      - series: 'cluster_upgrade_ongoing{source_cluster="cluster01"}'
+        values: '1+0x60'
+
+    alert_rule_test:
+      - eval_time: 11m
+        alertname: EtcdHighNumberOfLeaderChanges
+
+  - interval: 1m
+    input_series:
+      # Leader changes exceed threshold on cluster01, but the upgrade is ongoing on a DIFFERENT cluster (cluster02).
+      # Suppression is per-cluster — cluster01 alert must still fire.
+      - series: 'etcd_server_leader_changes_seen_total{namespace="openshift-etcd",source_cluster="cluster01"}'
+        values: '3+2x60'
+      - series: 'cluster_upgrade_ongoing{source_cluster="cluster02"}'
+        values: '1+0x60'
+
+    alert_rule_test:
+      - eval_time: 11m
+        alertname: EtcdHighNumberOfLeaderChanges
+        exp_alerts:
+          - exp_labels:
+              component: etcd
+              namespace: openshift-etcd
+              severity: warning
+              source_cluster: cluster01
+            exp_annotations:
+              summary: 'Etcd high number of leader changes'
+              description: 'Etcd leader changed more than 2 times during 10 minutes in cluster cluster01.'
+              alert_routing_key: spreandinfra
+              team: o11y
+              slo: 'false'
+
+  - interval: 1m
+    input_series:
+      # Counter is stable — no leader changes, no alert.
+      - series: 'etcd_server_leader_changes_seen_total{namespace="openshift-etcd",source_cluster="cluster01"}'
         values: '1+0x60'
 
     alert_rule_test:

test/promql/tests/data_plane/cluster_upgrade_ongoing_alert_test.yaml

@@ -22,6 +22,7 @@ tests:
               cluster_uuid: "uuid-1"
               cluster_name: "prod-cluster"
               current_version: "4.14.1"
+              source_cluster: "prod-cluster"
               alertname: ClusterUpgradeOngoing
             exp_annotations:
               summary: "Cluster prod-cluster is currently undergoing an upgrade."

test/promql/tests/recording/cluster_upgrade_recording_rules_test.yaml

@@ -0,0 +1,29 @@
+evaluation_interval: 1m
+
+rule_files:
+  - cluster_upgrade_recording_rules.yaml
+
+tests:
+  - name: ClusterUpgradeOngoingSignalFiring
+    interval: 1m
+    input_series:
+      - series: 'ALERTS{alertname="ClusterUpgradeOngoing",alertstate="firing",source_cluster="cluster01",severity="info"}'
+        values: '1+0x10'
+
+    promql_expr_test:
+      - expr: cluster_upgrade_ongoing
+        eval_time: 5m
+        exp_samples:
+          - labels: '{__name__="cluster_upgrade_ongoing", source_cluster="cluster01"}'
+            value: 1
+
+  - name: ClusterUpgradeOngoingSignalPendingNotCaptured
+    interval: 1m
+    input_series:
+      - series: 'ALERTS{alertname="ClusterUpgradeOngoing",alertstate="pending",source_cluster="cluster01",severity="info"}'
+        values: '1+0x10'
+
+    promql_expr_test:
+      - expr: cluster_upgrade_ongoing
+        eval_time: 5m
+        exp_samples: []