feat: only alert on the latest e2etestrun

On-behalf-of: @SAP rene.schuenemann@sap.com
Signed-off-by: René Schünemann (SAP) <rene.schuenemann@sap.com>

Author: reshnm

component-settings.yaml

@@ -6,7 +6,7 @@ KUSTOMIZATIONS_LOCATION_PREFIX: ${REPOSITORY_CONTEXT}/observability-stack/kustom
 CERT_MANAGER_VERSION: "v1.20.2"
 
 # metrics-operator
-METRICS_OPERATOR_VERSION: "v0.10.1"
+METRICS_OPERATOR_VERSION: "v0.11.0"
 
 # open telemetry operator and collector
 OPEN_TELEMETRY_OPERATOR_CHART_VERSION: "0.110.0"

kustomizations/metrics/platform/e2e/metric-e2e-testrun.yaml

@@ -12,6 +12,9 @@ spec:
   interval: "1m"
   dataSinkRef:
     name: default
+  valueFrom:
+    fieldPath: "metadata.creationTimestamp"
+    type: timestamp
   projections:
     - name: tr_name
       fieldPath: "metadata.name"

kustomizations/metrics/platform/e2e/rule-e2e-testrun.yaml

@@ -8,17 +8,20 @@ metadata:
 spec:
   groups:
     - name: e2e.testrun
-      interval: 5m
+      interval: 30s
       rules:
         - alert: E2ETestRunConditionFailed
           expr: |
-            co_e2e_testruns{tr_tc_create_project_run_status="False"}
-            or co_e2e_testruns{tr_tc_create_project_cleanup_status="False"}
-            or co_e2e_testruns{tr_tc_create_workspace_run_status="False"}
-            or co_e2e_testruns{tr_tc_create_workspace_cleanup_status="False"}
-            or co_e2e_testruns{tr_tc_create_mcpv2_run_status="False"}
-            or co_e2e_testruns{tr_tc_create_mcpv2_cleanup_status="False"}
-          for: 1m
+            (
+              co_e2e_testruns{tr_tc_create_project_run_status="False"}
+              or co_e2e_testruns{tr_tc_create_project_cleanup_status="False"}
+              or co_e2e_testruns{tr_tc_create_workspace_run_status="False"}
+              or co_e2e_testruns{tr_tc_create_workspace_cleanup_status="False"}
+              or co_e2e_testruns{tr_tc_create_mcpv2_run_status="False"}
+              or co_e2e_testruns{tr_tc_create_mcpv2_cleanup_status="False"}
+            ) and on(cluster, tr_name)
+            topk by(cluster) (1, max by(cluster, tr_name) (co_e2e_testruns))
+          for: 10s
           labels:
             severity: warning
             component: e2e