Add PBS exporter and failure alerts

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml

@@ -0,0 +1,78 @@
+---
+groups:
+  - name: pbs.rules
+    rules:
+      - alert: PBS - Down
+        expr: pbs_up == 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS Offline"
+          description: "PBS job is not returning a healthy state."
+
+      - alert: PBS - Verification Not Successful
+        expr: pbs_snapshot_vm_last_verify == 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
+          description: >-
+            The latest snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is not verified successfully.
+
+      - alert: PBS - Local Backup Stale
+        expr: >-
+          (time() - pbs_snapshot_vm_last_timestamp > 172800)
+          and on (site_slug, namespace)
+          label_replace(
+            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+            "namespace", "$1", "site_slug", "(.+)"
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS backup for {{ $labels.vm_name }} is older than two days"
+          description: >-
+            The latest local snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is older than two days.
+
+      - alert: PBS - Sync Backup Stale
+        expr: >-
+          (time() - pbs_snapshot_vm_last_timestamp > 172800)
+          unless on (site_slug, namespace)
+          label_replace(
+            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+            "namespace", "$1", "site_slug", "(.+)"
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS sync for {{ $labels.vm_name }} is older than two days"
+          description: >-
+            The latest remote snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is older than two days.
+
+      - alert: PBS - Sync Missing
+        expr: >-
+          count by (site_slug, node_name, instance) (pbs_snapshot_vm_last_timestamp)
+          unless on (site_slug, node_name, instance)
+          count by (site_slug, node_name, instance) (
+            pbs_snapshot_vm_last_timestamp
+            unless on (site_slug, namespace)
+            label_replace(
+              count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+              "namespace", "$1", "site_slug", "(.+)"
+            )
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS has no remote snapshots"
+          description: >-
+            PBS {{ $labels.node_name }} has local snapshots but no snapshots in
+            a remote namespace.

playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2

@@ -262,6 +262,38 @@ scrape_configs:
       - targets:
           - "prometheus-adguard_exporter:9618"
 
+{% if prometheus.features.pbs_exporter | default(false) %}
+  - job_name: 'pbs_exporter'
+    metrics_path: /metrics
+    http_sd_configs:
+      - <<: *netbox_auth
+        url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
+    relabel_configs:
+      - source_labels: [__meta_netbox_primary_ip]
+        regex: '([0-9.]+)'
+        replacement: 'https://$1:8007'
+        target_label: __param_target
+      - source_labels: [__meta_netbox_primary_ip]
+        regex: '([0-9a-fA-F:]*:[0-9a-fA-F:]+)'
+        replacement: 'https://[$1]:8007'
+        target_label: __param_target
+      - source_labels: [__meta_netbox_primary_ip]
+        target_label: instance
+      - source_labels: [__meta_netbox_name]
+        target_label: node_name
+      - source_labels: [__meta_netbox_site]
+        target_label: site
+      - source_labels: [__meta_netbox_site_slug]
+        target_label: site_slug
+      - source_labels: [__meta_netbox_tenant]
+        target_label: tenant
+      - source_labels: [__meta_netbox_role_slug]
+        target_label: device_role
+      - target_label: __address__
+        replacement: "prometheus-pbs_exporter:10019"
+
+{% endif %}
+
   - job_name: "hass"
     metrics_path: /api/prometheus
     authorization:

playbooks/host_vars/beokpdcort01.yml

@@ -106,15 +106,15 @@ firewall:
           ip: "@MONITORING"
         destination:
           protocol: tcp
-          port: "{ 80,443,9090,9100,9323,9342 }"  # http,https,federate,node_exporter,docker_exporter,frr_exporter
+          port: "{ 80,443,8007,9090,9100,9323,9342 }"  # http,https,pbs,federate,node_exporter,docker_exporter,frr_exporter
         counter: true
       - comment: "Allow from MONITORING"
         type: ip6
         source:
           ip: "@v6_MONITORING"
         destination:
           protocol: tcp
-          port: "{ 80,443,9090,9100,9323,9342 }"  # http,https,node_exporter,docker_exporter,frr_exporter
+          port: "{ 80,443,8007,9090,9100,9323,9342 }"  # http,https,pbs,node_exporter,docker_exporter,frr_exporter
         counter: true
       # DNS
       - comment: "Allow Adguard to Recursive"

playbooks/host_vars/beokpdcovm01.yml

@@ -3,6 +3,87 @@ dawarich_photon_api_host: photon.frai.se
 dawarich_photon_api_port: "443"
 dawarich_photon_api_use_https: "true"
 
+adguard_hostname: dns1.frai.se
+
+docker:
+  dir:
+    config: /data/docker-config
+    data: /data/docker
+  networks:
+    monitoring:
+      driver_options:
+        parent: ens19.1001
+      hosts:
+        loki: 94
+      ipam:
+        - gateway: 10.10.48.65
+          subnet: 10.10.48.64/27
+        - gateway: 2001:678:1080:1001::1
+          subnet: 2001:678:1080:1001::/64
+      ipv6: true
+      type: ipvlan
+    networking:
+      driver_options:
+        parent: ens19.1004
+      hosts:
+        uisp: 189
+        unifi: 190
+      ipam:
+        - gateway: 10.10.48.161
+          subnet: 10.10.48.160/27
+        - gateway: 2001:678:1080:1004::1
+          subnet: 2001:678:1080:1004::/64
+      ipv6: true
+      type: ipvlan
+    services:
+      driver_options:
+        parent: ens19.1002
+      hosts:
+        adguard: 124
+        traefik: 125
+        unbound: 123
+      ipam:
+        - gateway: 10.10.48.97
+          subnet: 10.10.48.96/27
+        - gateway: 2001:678:1080:1102::1
+          subnet: 2001:678:1080:1102::/64
+      ipv6: true
+      type: ipvlan
+  users:
+    - fraise
+
+mounts:
+  - fs: ext4
+    mountpoint: /data/
+    source: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2
+
+netbox_interface: true
+
+portainer:
+  network: services
+  type: portainer-ce
+
+prometheus:
+  features:
+    adguard: true
+    alertmanager: true
+    blackbox: true
+    graphite: true
+    ipmi: true
+    snmp: true
+    speedtest: true
+    pbs_exporter: true
+
+# Shared secret for the prometheus@pbs!exporter token configured on every PBS.
+pbs_exporter_api_token: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  66653733653366643937326266313733396333356535656563306634613739333964316434303736
+  3530373137643934333034373738366631353838666663660a326662613839346663383832626538
+  39613466356665323864376164323835396661666137313039396436373736386331306362363137
+  3233313231613636330a613430353262356234386366346534386462376362636136316130306238
+  31386434316431383361613835323464626661313530383834306231623764393032313932343938
+  3934663937333435663532346261663261643038343063303030
+
 cloudflared:
   docker_network: services
   service_token: !vault |

roles/prometheus/tasks/main.yml

@@ -27,6 +27,11 @@
   tags: adguard, adguard_exporter
   when: prometheus.features.adguard | default (false)
 
+- name: Setup PBS exporter
+  ansible.builtin.import_tasks: pbs_exporter.yml
+  tags: pbs, pbs_exporter, pbs-exporter
+  when: prometheus.features.pbs_exporter | default(false)
+
 - name: Setup alertmanager
   ansible.builtin.import_tasks: alertmanager.yml
   tags: alertmanager, alerts, alert

roles/prometheus/tasks/pbs_exporter.yml

@@ -0,0 +1,29 @@
+---
+- name: Validate PBS exporter API token
+  ansible.builtin.assert:
+    that:
+      - pbs_exporter_api_token is defined
+      - pbs_exporter_api_token is string
+      - pbs_exporter_api_token | length > 0
+    fail_msg: pbs_exporter_api_token must be supplied as an inventory variable.
+  no_log: true
+
+- name: Setup Prometheus PBS exporter
+  community.docker.docker_container:
+    name: prometheus-pbs_exporter
+    image: ghcr.io/natrontech/pbs-exporter:latest
+    user: "65534:65534"
+    networks:
+      - name: monitoring
+    dns_servers: "{{ dns.servers }}"
+    env:
+      PBS_USERNAME: pbs_exporter@pbs
+      PBS_API_TOKEN_NAME: pbs_exporter_token
+      PBS_API_TOKEN: "{{ pbs_exporter_api_token }}"
+      PBS_INSECURE: "true"
+    restart: true
+    restart_policy: always
+    state: started
+    pull: true
+    labels:
+      traefik.enable: "false"