fix: Further PBS monitoring fixes

Author: Gianni Stubbe

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml

@@ -2,29 +2,77 @@
 groups:
   - name: pbs.rules
     rules:
-      - alert: PBS - Backup Failed
-        expr: pbs_task_status{task_type="backup",status=~"(?i:failed|error)"} == 1
+      - alert: PBS - Down
+        expr: pbs_up == 0
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS backup failed on {{ $labels.node_name }}"
-          description: "Backup {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+          summary: "PBS Offline"
+          description: "PBS job is not returning a healthy state."
 
-      - alert: PBS - Verification Failed
-        expr: pbs_task_status{task_type="verify",status=~"(?i:failed|error)"} == 1
+      - alert: PBS - Verification Not Successful
+        expr: pbs_snapshot_vm_last_verify == 0
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS verification failed on {{ $labels.node_name }}"
-          description: "Verification {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+          summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
+          description: >-
+            The latest snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is not verified successfully.
 
-      - alert: PBS - Sync Failed
-        expr: pbs_task_status{task_type="sync",status=~"(?i:failed|error)"} == 1
+      - alert: PBS - Local Backup Stale
+        expr: >-
+          (time() - pbs_snapshot_vm_last_timestamp > 172800)
+          and on (site_slug, namespace)
+          label_replace(
+            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+            "namespace", "$1", "site_slug", "(.+)"
+          )
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS sync failed on {{ $labels.node_name }}"
-          description: "Sync {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+          summary: "PBS backup for {{ $labels.vm_name }} is older than two days"
+          description: >-
+            The latest local snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is older than two days.
+
+      - alert: PBS - Sync Backup Stale
+        expr: >-
+          (time() - pbs_snapshot_vm_last_timestamp > 172800)
+          unless on (site_slug, namespace)
+          label_replace(
+            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+            "namespace", "$1", "site_slug", "(.+)"
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS sync for {{ $labels.vm_name }} is older than two days"
+          description: >-
+            The latest remote snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is older than two days.
+
+      - alert: PBS - Sync Missing
+        expr: >-
+          count by (site_slug, node_name, instance) (pbs_snapshot_vm_last_timestamp)
+          unless on (site_slug, node_name, instance)
+          count by (site_slug, node_name, instance) (
+            pbs_snapshot_vm_last_timestamp
+            unless on (site_slug, namespace)
+            label_replace(
+              count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+              "namespace", "$1", "site_slug", "(.+)"
+            )
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS has no remote snapshots"
+          description: >-
+            PBS {{ $labels.node_name }} has local snapshots but no snapshots in
+            a remote namespace.

playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2

@@ -264,25 +264,33 @@ scrape_configs:
 
 {% if prometheus.features.pbs_exporter | default(false) %}
   - job_name: 'pbs_exporter'
-    metrics_path: /probe
+    metrics_path: /metrics
     http_sd_configs:
       - <<: *netbox_auth
         url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
     relabel_configs:
-      - source_labels: [__meta_netbox_name]
+      - source_labels: [__meta_netbox_primary_ip]
+        regex: '([0-9.]+)'
+        replacement: 'https://$1:8007'
+        target_label: __param_target
+      - source_labels: [__meta_netbox_primary_ip]
+        regex: '([0-9a-fA-F:]*:[0-9a-fA-F:]+)'
+        replacement: 'https://[$1]:8007'
         target_label: __param_target
       - source_labels: [__meta_netbox_primary_ip]
         target_label: instance
       - source_labels: [__meta_netbox_name]
         target_label: node_name
       - source_labels: [__meta_netbox_site]
         target_label: site
+      - source_labels: [__meta_netbox_site_slug]
+        target_label: site_slug
       - source_labels: [__meta_netbox_tenant]
         target_label: tenant
       - source_labels: [__meta_netbox_role_slug]
         target_label: device_role
       - target_label: __address__
-        replacement: "prometheus-pbs_exporter:9569"
+        replacement: "prometheus-pbs_exporter:10019"
 
 {% endif %}
 

playbooks/host_vars/beokpdcort01.yml

@@ -106,15 +106,15 @@ firewall:
           ip: "@MONITORING"
         destination:
           protocol: tcp
-          port: "{ 80,443,9090,9100,9323,9342 }"  # http,https,federate,node_exporter,docker_exporter,frr_exporter
+          port: "{ 80,443,8007,9090,9100,9323,9342 }"  # http,https,pbs,federate,node_exporter,docker_exporter,frr_exporter
         counter: true
       - comment: "Allow from MONITORING"
         type: ip6
         source:
           ip: "@v6_MONITORING"
         destination:
           protocol: tcp
-          port: "{ 80,443,9090,9100,9323,9342 }"  # http,https,node_exporter,docker_exporter,frr_exporter
+          port: "{ 80,443,8007,9090,9100,9323,9342 }"  # http,https,pbs,node_exporter,docker_exporter,frr_exporter
         counter: true
       # DNS
       - comment: "Allow Adguard to Recursive"

playbooks/host_vars/beokpdcovm01.yml

@@ -3,18 +3,86 @@ dawarich_photon_api_host: photon.frai.se
 dawarich_photon_api_port: "443"
 dawarich_photon_api_use_https: "true"
 
+adguard_hostname: dns1.frai.se
+
+docker:
+  dir:
+    config: /data/docker-config
+    data: /data/docker
+  networks:
+    monitoring:
+      driver_options:
+        parent: ens19.1001
+      hosts:
+        loki: 94
+      ipam:
+        - gateway: 10.10.48.65
+          subnet: 10.10.48.64/27
+        - gateway: 2001:678:1080:1001::1
+          subnet: 2001:678:1080:1001::/64
+      ipv6: true
+      type: ipvlan
+    networking:
+      driver_options:
+        parent: ens19.1004
+      hosts:
+        uisp: 189
+        unifi: 190
+      ipam:
+        - gateway: 10.10.48.161
+          subnet: 10.10.48.160/27
+        - gateway: 2001:678:1080:1004::1
+          subnet: 2001:678:1080:1004::/64
+      ipv6: true
+      type: ipvlan
+    services:
+      driver_options:
+        parent: ens19.1002
+      hosts:
+        adguard: 124
+        traefik: 125
+        unbound: 123
+      ipam:
+        - gateway: 10.10.48.97
+          subnet: 10.10.48.96/27
+        - gateway: 2001:678:1080:1102::1
+          subnet: 2001:678:1080:1102::/64
+      ipv6: true
+      type: ipvlan
+  users:
+    - fraise
+
+mounts:
+  - fs: ext4
+    mountpoint: /data/
+    source: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2
+
+netbox_interface: true
+
+portainer:
+  network: services
+  type: portainer-ce
+
 prometheus:
   features:
+    adguard: true
+    alertmanager: true
+    blackbox: true
+    graphite: true
+    ipmi: true
+    snmp: true
+    speedtest: true
     pbs_exporter: true
 
-# Each target is keyed by the PBS VM's NetBox name. The target must be the
-# endpoint accepted by the PBS exporter; token secrets belong in Vault.
-# pbs_exporter_targets:
-#   - netbox_name: pbs.example
-#     target: https://pbs.example:8007
-#     api_token_id: prometheus@pbs!exporter
-#     api_token_secret: !vault |
-#       $ANSIBLE_VAULT;1.1;AES256
+# Shared secret for the prometheus@pbs!exporter token configured on every PBS.
+pbs_exporter_api_token: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  66653733653366643937326266313733396333356535656563306634613739333964316434303736
+  3530373137643934333034373738366631353838666663660a326662613839346663383832626538
+  39613466356665323864376164323835396661666137313039396436373736386331306362363137
+  3233313231613636330a613430353262356234386366346534386462376362636136316130306238
+  31386434316431383361613835323464626661313530383834306231623764393032313932343938
+  3934663937333435663532346261663261643038343063303030
 
 cloudflared:
   docker_network: services

roles/prometheus/tasks/pbs_exporter.yml

@@ -1,91 +1,29 @@
 ---
-- name: Validate PBS exporter targets list
+- name: Validate PBS exporter API token
   ansible.builtin.assert:
     that:
-      - pbs_exporter_targets is defined
-      - pbs_exporter_targets is sequence
-      - pbs_exporter_targets is not string
-      - pbs_exporter_targets | length > 0
-    fail_msg: pbs_exporter_targets must be a non-empty inventory list.
-  no_log: true
-
-- name: Validate PBS exporter targets
-  ansible.builtin.assert:
-    that:
-      - item.netbox_name is defined
-      - item.netbox_name is string
-      - item.netbox_name | length > 0
-      - item.target is defined
-      - item.target is string
-      - item.target | length > 0
-      - item.api_token_id is defined
-      - item.api_token_id is string
-      - item.api_token_id | length > 0
-      - item.api_token_secret is defined
-      - item.api_token_secret is string
-      - item.api_token_secret | length > 0
-    fail_msg: >-
-      Every PBS exporter target must define netbox_name, target, api_token_id,
-      and api_token_secret.
-  loop: "{{ pbs_exporter_targets }}"
-  loop_control:
-    label: hidden
-  no_log: true
-
-- name: Validate unique PBS exporter target selectors
-  ansible.builtin.assert:
-    that:
-      - >-
-        pbs_exporter_targets | map(attribute='netbox_name') | unique | list | length
-        == pbs_exporter_targets | length
-      - >-
-        pbs_exporter_targets | map(attribute='target') | unique | list | length
-        == pbs_exporter_targets | length
-    fail_msg: PBS exporter NetBox names and targets must be unique.
-  no_log: true
-
-- name: Create PBS exporter config directory
-  ansible.builtin.file:
-    path: "{{ docker.dir.config }}/prometheus_pbs_exporter"
-    state: directory
-    owner: root
-    group: root
-    mode: "0700"
-  no_log: true
-
-- name: Render PBS exporter targets
-  ansible.builtin.template:
-    src: pbs_targets.yml.j2
-    dest: "{{ docker.dir.config }}/prometheus_pbs_exporter/targets.yml"
-    owner: root
-    group: root
-    mode: "0600"
-  no_log: true
-
-- name: Remove legacy per-target PBS exporters
-  community.docker.docker_container:
-    name: "prometheus-pbs-exporter-{{ item.name }}"
-    state: absent
-  loop: "{{ pbs_exporter_instances | default([]) }}"
-  loop_control:
-    label: hidden
+      - pbs_exporter_api_token is defined
+      - pbs_exporter_api_token is string
+      - pbs_exporter_api_token | length > 0
+    fail_msg: pbs_exporter_api_token must be supplied as an inventory variable.
   no_log: true
 
 - name: Setup Prometheus PBS exporter
   community.docker.docker_container:
     name: prometheus-pbs_exporter
-    image: "{{ pbs_exporter_image | default('natrontech/pbs-exporter:latest') }}"
+    image: ghcr.io/natrontech/pbs-exporter:latest
+    user: "65534:65534"
     networks:
       - name: monitoring
     dns_servers: "{{ dns.servers }}"
-    volumes:
-      - "{{ docker.dir.config }}/prometheus_pbs_exporter/targets.yml:/config/targets.yml:ro"
-    command: "--config.file=/config/targets.yml"
+    env:
+      PBS_USERNAME: pbs_exporter@pbs
+      PBS_API_TOKEN_NAME: pbs_exporter_token
+      PBS_API_TOKEN: "{{ pbs_exporter_api_token }}"
+      PBS_INSECURE: "true"
     restart: true
     restart_policy: always
     state: started
     pull: true
     labels:
       traefik.enable: "false"
-      com.unitix.prometheus.pbs-exporter: "true"
-  no_log: true