Add PBS exporter and failure alerts

Co-authored-by: 33Fraise33 <33Fraise33@users.noreply.github.com>

Author: opencode-agent[bot]

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml

@@ -0,0 +1,30 @@
+---
+groups:
+  - name: pbs.rules
+    rules:
+      - alert: PBS backup failed
+        expr: pbs_task_status{task_type="backup",status=~"(?i:failed|error)"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS backup failed on {{ $labels.node_name }}"
+          description: "Backup {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+
+      - alert: PBS verification failed
+        expr: pbs_task_status{task_type="verify",status=~"(?i:failed|error)"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS verification failed on {{ $labels.node_name }}"
+          description: "Verification {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+
+      - alert: PBS sync failed
+        expr: pbs_task_status{task_type="sync",status=~"(?i:failed|error)"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS sync failed on {{ $labels.node_name }}"
+          description: "Sync {{ $labels.task }} for datastore {{ $labels.datastore }} failed."

playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2

@@ -262,6 +262,27 @@ scrape_configs:
       - targets:
           - "prometheus-adguard_exporter:9618"
 
+  - job_name: 'pbs_exporter'
+    metrics_path: /probe
+    http_sd_configs:
+      - <<: *netbox_auth
+        url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
+    relabel_configs:
+      - source_labels: [__meta_netbox_primary_ip]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - source_labels: [__meta_netbox_name]
+        target_label: node_name
+      - source_labels: [__meta_netbox_site]
+        target_label: site
+      - source_labels: [__meta_netbox_tenant]
+        target_label: tenant
+      - source_labels: [__meta_netbox_role_slug]
+        target_label: device_role
+      - target_label: __address__
+        replacement: "prometheus-pbs_exporter:9569"
+
   - job_name: "hass"
     metrics_path: /api/prometheus
     authorization:

playbooks/host_vars/beokpdcovm01.yml

@@ -3,6 +3,10 @@ dawarich_photon_api_host: photon.frai.se
 dawarich_photon_api_port: "443"
 dawarich_photon_api_use_https: "true"
 
+prometheus:
+  features:
+    pbs_exporter: true
+
 cloudflared:
   docker_network: services
   service_token: !vault |

roles/prometheus/tasks/main.yml

@@ -27,6 +27,11 @@
   tags: adguard, adguard_exporter
   when: prometheus.features.adguard | default (false)
 
+- name: Setup PBS exporter
+  ansible.builtin.import_tasks: pbs_exporter.yml
+  tags: pbs, pbs_exporter, pbs-exporter
+  when: prometheus.features.pbs_exporter | default(false)
+
 - name: Setup alertmanager
   ansible.builtin.import_tasks: alertmanager.yml
   tags: alertmanager, alerts, alert

roles/prometheus/tasks/pbs_exporter.yml

@@ -0,0 +1,31 @@
+---
+- name: Validate PBS exporter credentials
+  ansible.builtin.assert:
+    that:
+      - pbs_exporter_api_token_id is defined
+      - pbs_exporter_api_token_id | length > 0
+      - pbs_exporter_api_token_secret is defined
+      - pbs_exporter_api_token_secret | length > 0
+    fail_msg: >-
+      pbs_exporter_api_token_id and pbs_exporter_api_token_secret must be
+      supplied as inventory variables.
+  no_log: true
+
+- name: Setup Prometheus PBS exporter
+  community.docker.docker_container:
+    name: prometheus-pbs_exporter
+    image: "{{ pbs_exporter_image | default('natrontech/pbs-exporter:latest') }}"
+    networks:
+      - name: monitoring
+    dns_servers: "{{ dns.servers }}"
+    env:
+      PBS_TOKEN_ID: "{{ pbs_exporter_api_token_id }}"
+      PBS_TOKEN_SECRET: "{{ pbs_exporter_api_token_secret }}"
+      PBS_TLS_INSECURE: "true"
+    restart: true
+    restart_policy: always
+    state: started
+    pull: true
+    labels:
+      traefik.enable: "false"
+  no_log: true