fix: General alert changes and solidtime fix

Author: Gianni Stubbe

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml

@@ -13,6 +13,12 @@ groups:
         for: 10m
         annotations:
           summary: "Endpoint {{ $labels.node_name }} has high latency: {{ $value }}"
+      
+      - alert: High packet loss
+        expr: (1 - avg_over_time(probe_success[15m])) * 100 > 5
+        for: 10m
+        annotations:
+          summary: "Endpoint {{ $labels.node_name }} has high packet loss: {{ $value }}"
 
       - alert: Non running containers
         expr: engine_daemon_container_states_containers{state!="running"} > 0

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml

@@ -12,15 +12,17 @@ groups:
           description: "PBS job is not returning a healthy state."
 
       - alert: PBS - Verification Not Successful
-        expr: pbs_snapshot_vm_last_verify == 0
+        expr: >-
+          (max_over_time(pbs_snapshot_vm_last_verify[2d]) == 0)
+          and (pbs_snapshot_vm_count > 1)
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
+          summary: "No successfully verified PBS snapshot for {{ $labels.vm_name }} in the last two days"
           description: >-
-            The latest snapshot for {{ $labels.vm_name }} on
-            {{ $labels.node_name }} is not verified successfully.
+            No successfully verified snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} has been observed in the last two days.
 
       - alert: PBS - Local Backup Stale
         expr: >-

roles/grafana/tasks/main.yml

@@ -41,7 +41,7 @@
         && Header(`X-Authelia-Bypass`, `{{ authelia_bypass_header }}`)"
       traefik.http.routers.grafana-bypass.tls: "true"
       traefik.http.routers.grafana-bypass.tls.certresolver: "le"
-      traefik.http.routers.grafana-bypass.service: "netbox-service"
+      traefik.http.routers.grafana-bypass.service: "grafana-service"
       traefik.http.routers.grafana-bypass.priority: "100"
       # SHARED
       traefik.http.services.grafana-service.loadbalancer.server.port: "3000"

roles/prometheus/templates/alertmanager.yml.j2

@@ -6,6 +6,23 @@ route:
   group_by: ['site']
   repeat_interval: 10000h
 
+inhibit_rules:
+  - source_matchers:
+      - alertname = "Device offline"
+    target_matchers:
+      - alertname = "Probe Failed"
+    equal: ['node_name']
+  - source_matchers:
+      - alertname = "Probe Failed"
+    target_matchers:
+      - alertname = "High packet loss"
+    equal: ['node_name']
+  - source_matchers:
+      - alertname = "Probe Failed"
+    target_matchers:
+      - alertname = "High latency"
+    equal: ['node_name']
+
 receivers:
   - name: default
     # N8N to matrix

roles/solidtime/tasks/main.yml

@@ -111,6 +111,7 @@
     env:
       CONTAINER_MODE: http
       AUTO_DB_MIGRATE: "true"
+      APP_URL: https://time.unitix.be
     env_file: "{{ docker.dir.config }}/solidtime/laravel.env"
     state: healthy
     pull: true