prometheus: add an alert for CronJob owned Job failure and another for non Cronjob owned Job

Author: ezekiel-alexrod

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 ### Enhancements
 
+- Add alerts for a CronJob owned Job failure and another one for
+  non-CronJob owned Job failure 
+  (PR[#4584](https://github.com/scality/metalk8s/pull/4584))
+
 - Bump Kubernetes version to
   [1.30.11](https://github.com/kubernetes/kubernetes/releases/tag/v1.30.11)
   (PR[#4578](https://github.com/scality/metalk8s/pull/4578))

charts/drop-prometheus-rules.yaml

@@ -16,6 +16,7 @@ node-exporter:
   - NodeFileDescriptorLimit
 kubernetes-apps:
   - KubeJobNotCompleted
+  - KubeJobFailed
 # workaround: this fires upon install
 # revert the entire commit after the fix is merged
 etcd:

salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml

@@ -9,7 +9,7 @@ spec:
     replicas: 1
   config:
     retention_time: "10d"
-    retention_size: "0"  # "0" to disable size-based retention
+    retention_size: "0" # "0" to disable size-based retention
     enable_admin_api: false
     serviceMonitor:
       kubelet:
@@ -19,37 +19,43 @@ spec:
       kube_job_not_completed:
         warning:
           hours: 24 # Hours of job active before we trigger alert
+      kube_cronjob_owned_job_failed:
+        warning:
+          minutes: 5 # Minutes of job active before we trigger alert
+      kube_job_failed:
+        warning:
+          minutes: 5 # Minutes of job active before we trigger alert
     node_exporter:
       node_filesystem_space_filling_up:
         warning:
-          hours: 24  # Hours before there is no space left
-          threshold: 40  # Min space left to trigger prediction
+          hours: 24 # Hours before there is no space left
+          threshold: 40 # Min space left to trigger prediction
         critical:
           hours: 4
           threshold: 20
       node_filesystem_almost_out_of_space:
         warning:
-          available: 20  # Percentage of free space left
+          available: 20 # Percentage of free space left
         critical:
           available: 12
       node_filesystem_files_filling_up:
         warning:
-          hours: 24  # Hours before there is no inode left
-          threshold: 40  # Min space left to trigger prediction
+          hours: 24 # Hours before there is no inode left
+          threshold: 40 # Min space left to trigger prediction
         critical:
           hours: 4
           threshold: 20
       node_filesystem_almost_out_of_files:
         warning:
-          available: 15  # Percentage of free inodes left
+          available: 15 # Percentage of free inodes left
         critical:
           available: 8
       node_network_receive_errors:
         warning:
-          error_rate: 0.01  # Rate of receive errors for the last 2m
+          error_rate: 0.01 # Rate of receive errors for the last 2m
       node_network_transmit_errors:
         warning:
-          error_rate: 0.01  # Rate of transmit errors for the last 2m
+          error_rate: 0.01 # Rate of transmit errors for the last 2m
       node_high_number_conntrack_entries_used:
         warning:
           threshold: 0.75

salt/metalk8s/addons/prometheus-operator/deployed/chart.sls

@@ -73953,16 +73953,6 @@ spec:
       for: 15m
       labels:
         severity: warning
-    - alert: KubeJobFailed
-      annotations:
-        description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to
-          complete. Removing failed job after investigation should clear this alert.
-        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
-        summary: Job failed to complete.
-      expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"}  > 0
-      for: 15m
-      labels:
-        severity: warning
     - alert: KubeHpaReplicasMismatch
       annotations:
         description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }}

salt/metalk8s/addons/prometheus-operator/deployed/kube-alerts-rules.sls

@@ -37,4 +37,36 @@ spec:
         kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > ({% endraw %}{{ rules.kube_apps.kube_job_not_completed.warning.hours }}{% raw %} * 60 * 60)
       labels:
         severity: warning
+    - alert: KubeCronJobOwnedJobFailed
+      annotations:
+        description: Job {{ $labels.job_name }} created by CronJob {{ $labels.namespace }}/{{ $labels.cronjob_name }} failed to complete.
+          Check the logs of the Job and the CronJob state to understand the failure.
+          Removing failed job after investigation should clear this alert.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecronjobownedjobfailed
+        summary: Job owned by CronJob failed to complete.
+      for: {% endraw %}{{ rules.kube_apps.kube_cronjob_owned_job_failed.warning.minutes }}{% raw %}m
+      expr: |-
+        (
+          kube_job_status_failed{job_name!=""} == 1
+          and on(job_name, namespace)
+          kube_job_owner{owner_kind="CronJob", job_name!=""}
+        )
+        * on(job_name, namespace) group_left(owner_name)
+        kube_job_owner{owner_kind="CronJob", job_name!=""}
+      labels:
+        severity: warning
+    - alert: KubeJobFailed
+      annotations:
+        description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.
+          Check the logs of the Job to understand the failure.
+          Removing failed job after investigation should clear this alert.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
+        summary: Job failed to complete.
+      expr: |-
+        kube_job_status_failed{job_name!=""} == 1
+        unless on(job_name, namespace)
+        kube_job_owner{owner_kind="CronJob", job_name!=""}
+      for: {% endraw %}{{ rules.kube_apps.kube_job_failed.warning.minutes }}{% raw %}m
+      labels:
+        severity: warning
 {%- endraw %}