personal-ansible/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml at 0a3aed1f7ec9f8293ccc5cf0d305da601b21000b · 33Fraise33/personal-ansible · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
---
groups:
  - name: pbs.rules
    rules:
      - alert: PBS - Down
        expr: pbs_up == 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "PBS Offline"
          description: "PBS job is not returning a healthy state."

      - alert: PBS - Verification Not Successful
        expr: >-
          (max_over_time(pbs_snapshot_vm_last_verify[2d]) == 0)
          and (pbs_snapshot_vm_count > 1)
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "No successfully verified PBS snapshot for {{ $labels.vm_name }} in the last two days"
          description: >-
            No successfully verified snapshot for {{ $labels.vm_name }} on
            {{ $labels.node_name }} has been observed in the last two days.

      - alert: PBS - Local Backup Stale
        expr: >-
          (time() - pbs_snapshot_vm_last_timestamp > 172800)
          and on (site_slug, namespace)
          label_replace(
            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
            "namespace", "$1", "site_slug", "(.+)"
          )
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "PBS backup for {{ $labels.vm_name }} is older than two days"
          description: >-
            The latest local snapshot for {{ $labels.vm_name }} on
            {{ $labels.node_name }} is older than two days.

      - alert: PBS - Sync Backup Stale
        expr: >-
          (time() - pbs_snapshot_vm_last_timestamp > 172800)
          unless on (site_slug, namespace)
          label_replace(
            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
            "namespace", "$1", "site_slug", "(.+)"
          )
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "PBS sync for {{ $labels.vm_name }} is older than two days"
          description: >-
            The latest remote snapshot for {{ $labels.vm_name }} on
            {{ $labels.node_name }} is older than two days.

      - alert: PBS - Sync Missing
        expr: >-
          count by (site_slug, node_name, instance) (pbs_snapshot_vm_last_timestamp)
          unless on (site_slug, node_name, instance)
          count by (site_slug, node_name, instance) (
            pbs_snapshot_vm_last_timestamp
            unless on (site_slug, namespace)
            label_replace(
              count by (site_slug) (pbs_snapshot_vm_last_timestamp),
              "namespace", "$1", "site_slug", "(.+)"
            )
          )
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "PBS has no remote snapshots"
          description: >-
            PBS {{ $labels.node_name }} has local snapshots but no snapshots in
            a remote namespace.