From c5ba9895bb51cb53205f35751f5e8eb82159054b Mon Sep 17 00:00:00 2001
From: "opencode-agent[bot]" <opencode-agent[bot]@users.noreply.github.com>
Date: Sun, 2 Aug 2026 08:39:00 +0000
Subject: [PATCH 1/4] Add PBS exporter and failure alerts

Co-authored-by: 33Fraise33 <33Fraise33@users.noreply.github.com>
---
 .../files/prometheus/alerts.d/pbs.rules.yml   | 30 ++++++++++++++++++
 .../templates/prometheus/prometheus.yml.j2    | 21 +++++++++++++
 playbooks/host_vars/beokpdcovm01.yml          |  4 +++
 roles/prometheus/tasks/main.yml               |  5 +++
 roles/prometheus/tasks/pbs_exporter.yml       | 31 +++++++++++++++++++
 5 files changed, 91 insertions(+)
 create mode 100644 playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
 create mode 100644 roles/prometheus/tasks/pbs_exporter.yml

diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
new file mode 100644
index 0000000..94247f2
--- /dev/null
+++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
@@ -0,0 +1,30 @@
+---
+groups:
+  - name: pbs.rules
+    rules:
+      - alert: PBS backup failed
+        expr: pbs_task_status{task_type="backup",status=~"(?i:failed|error)"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS backup failed on {{ $labels.node_name }}"
+          description: "Backup {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+
+      - alert: PBS verification failed
+        expr: pbs_task_status{task_type="verify",status=~"(?i:failed|error)"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS verification failed on {{ $labels.node_name }}"
+          description: "Verification {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+
+      - alert: PBS sync failed
+        expr: pbs_task_status{task_type="sync",status=~"(?i:failed|error)"} == 1
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS sync failed on {{ $labels.node_name }}"
+          description: "Sync {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
diff --git a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
index 03f389c..562fa73 100644
--- a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
+++ b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
@@ -262,6 +262,27 @@ scrape_configs:
       - targets:
           - "prometheus-adguard_exporter:9618"
 
+  - job_name: 'pbs_exporter'
+    metrics_path: /probe
+    http_sd_configs:
+      - <<: *netbox_auth
+        url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
+    relabel_configs:
+      - source_labels: [__meta_netbox_primary_ip]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - source_labels: [__meta_netbox_name]
+        target_label: node_name
+      - source_labels: [__meta_netbox_site]
+        target_label: site
+      - source_labels: [__meta_netbox_tenant]
+        target_label: tenant
+      - source_labels: [__meta_netbox_role_slug]
+        target_label: device_role
+      - target_label: __address__
+        replacement: "prometheus-pbs_exporter:9569"
+
   - job_name: "hass"
     metrics_path: /api/prometheus
     authorization:
diff --git a/playbooks/host_vars/beokpdcovm01.yml b/playbooks/host_vars/beokpdcovm01.yml
index 31a0d40..bae1724 100644
--- a/playbooks/host_vars/beokpdcovm01.yml
+++ b/playbooks/host_vars/beokpdcovm01.yml
@@ -3,6 +3,10 @@ dawarich_photon_api_host: photon.frai.se
 dawarich_photon_api_port: "443"
 dawarich_photon_api_use_https: "true"
 
+prometheus:
+  features:
+    pbs_exporter: true
+
 cloudflared:
   docker_network: services
   service_token: !vault |
diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml
index 412f76e..46930ca 100644
--- a/roles/prometheus/tasks/main.yml
+++ b/roles/prometheus/tasks/main.yml
@@ -27,6 +27,11 @@
   tags: adguard, adguard_exporter
   when: prometheus.features.adguard | default (false)
 
+- name: Setup PBS exporter
+  ansible.builtin.import_tasks: pbs_exporter.yml
+  tags: pbs, pbs_exporter, pbs-exporter
+  when: prometheus.features.pbs_exporter | default(false)
+
 - name: Setup alertmanager
   ansible.builtin.import_tasks: alertmanager.yml
   tags: alertmanager, alerts, alert
diff --git a/roles/prometheus/tasks/pbs_exporter.yml b/roles/prometheus/tasks/pbs_exporter.yml
new file mode 100644
index 0000000..4c1b959
--- /dev/null
+++ b/roles/prometheus/tasks/pbs_exporter.yml
@@ -0,0 +1,31 @@
+---
+- name: Validate PBS exporter credentials
+  ansible.builtin.assert:
+    that:
+      - pbs_exporter_api_token_id is defined
+      - pbs_exporter_api_token_id | length > 0
+      - pbs_exporter_api_token_secret is defined
+      - pbs_exporter_api_token_secret | length > 0
+    fail_msg: >-
+      pbs_exporter_api_token_id and pbs_exporter_api_token_secret must be
+      supplied as inventory variables.
+  no_log: true
+
+- name: Setup Prometheus PBS exporter
+  community.docker.docker_container:
+    name: prometheus-pbs_exporter
+    image: "{{ pbs_exporter_image | default('natrontech/pbs-exporter:latest') }}"
+    networks:
+      - name: monitoring
+    dns_servers: "{{ dns.servers }}"
+    env:
+      PBS_TOKEN_ID: "{{ pbs_exporter_api_token_id }}"
+      PBS_TOKEN_SECRET: "{{ pbs_exporter_api_token_secret }}"
+      PBS_TLS_INSECURE: "true"
+    restart: true
+    restart_policy: always
+    state: started
+    pull: true
+    labels:
+      traefik.enable: "false"
+  no_log: true

From e900502266dc552e74fde3fe66258e86d2cdd86c Mon Sep 17 00:00:00 2001
From: "opencode-agent[bot]" <opencode-agent[bot]@users.noreply.github.com>
Date: Sun, 2 Aug 2026 12:15:35 +0000
Subject: [PATCH 2/4] Multi-instance PBS exporter support

Co-authored-by: 33Fraise33 <33Fraise33@users.noreply.github.com>
---
 .../files/prometheus/alerts.d/pbs.rules.yml   |  6 +-
 .../templates/prometheus/prometheus.yml.j2    | 12 +++-
 roles/prometheus/tasks/pbs_exporter.yml       | 64 ++++++++++++++++---
 3 files changed, 67 insertions(+), 15 deletions(-)

diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
index 94247f2..22091fe 100644
--- a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
+++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
@@ -2,7 +2,7 @@
 groups:
   - name: pbs.rules
     rules:
-      - alert: PBS backup failed
+      - alert: PBSBackupFailed
         expr: pbs_task_status{task_type="backup",status=~"(?i:failed|error)"} == 1
         for: 10m
         labels:
@@ -11,7 +11,7 @@ groups:
           summary: "PBS backup failed on {{ $labels.node_name }}"
           description: "Backup {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
 
-      - alert: PBS verification failed
+      - alert: PBSVerificationFailed
         expr: pbs_task_status{task_type="verify",status=~"(?i:failed|error)"} == 1
         for: 10m
         labels:
@@ -20,7 +20,7 @@ groups:
           summary: "PBS verification failed on {{ $labels.node_name }}"
           description: "Verification {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
 
-      - alert: PBS sync failed
+      - alert: PBSSyncFailed
         expr: pbs_task_status{task_type="sync",status=~"(?i:failed|error)"} == 1
         for: 10m
         labels:
diff --git a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
index 562fa73..2aba4f4 100644
--- a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
+++ b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
@@ -262,12 +262,17 @@ scrape_configs:
       - targets:
           - "prometheus-adguard_exporter:9618"
 
-  - job_name: 'pbs_exporter'
+{% if prometheus.features.pbs_exporter | default(false) %}
+{% for pbs_instance in pbs_exporter_instances | default([]) %}
+  - job_name: 'pbs_exporter_{{ pbs_instance.name }}'
     metrics_path: /probe
     http_sd_configs:
       - <<: *netbox_auth
         url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
     relabel_configs:
+      - source_labels: [__meta_netbox_name]
+        regex: '^{{ pbs_instance.netbox_name | regex_escape }}$'
+        action: keep
       - source_labels: [__meta_netbox_primary_ip]
         target_label: __param_target
       - source_labels: [__param_target]
@@ -281,7 +286,10 @@ scrape_configs:
       - source_labels: [__meta_netbox_role_slug]
         target_label: device_role
       - target_label: __address__
-        replacement: "prometheus-pbs_exporter:9569"
+        replacement: "prometheus-pbs-exporter-{{ pbs_instance.name }}:9569"
+
+{% endfor %}
+{% endif %}
 
   - job_name: "hass"
     metrics_path: /api/prometheus
diff --git a/roles/prometheus/tasks/pbs_exporter.yml b/roles/prometheus/tasks/pbs_exporter.yml
index 4c1b959..c111cfb 100644
--- a/roles/prometheus/tasks/pbs_exporter.yml
+++ b/roles/prometheus/tasks/pbs_exporter.yml
@@ -1,26 +1,66 @@
 ---
-- name: Validate PBS exporter credentials
+- name: Validate PBS exporter instances list
   ansible.builtin.assert:
     that:
-      - pbs_exporter_api_token_id is defined
-      - pbs_exporter_api_token_id | length > 0
-      - pbs_exporter_api_token_secret is defined
-      - pbs_exporter_api_token_secret | length > 0
+      - pbs_exporter_instances is defined
+      - pbs_exporter_instances is sequence
+      - pbs_exporter_instances is not string
+      - pbs_exporter_instances | length > 0
     fail_msg: >-
-      pbs_exporter_api_token_id and pbs_exporter_api_token_secret must be
-      supplied as inventory variables.
+      pbs_exporter_instances must be a non-empty inventory list.
   no_log: true
 
-- name: Setup Prometheus PBS exporter
+- name: Validate PBS exporter instances
+  ansible.builtin.assert:
+    that:
+      - item.name is defined
+      - item.name is string
+      - item.name is match('^[a-z0-9][a-z0-9-]*$')
+      - item.netbox_name is defined
+      - item.netbox_name is string
+      - item.netbox_name | length > 0
+      - item.api_token_id is defined
+      - item.api_token_id is string
+      - item.api_token_id | length > 0
+      - item.api_token_secret is defined
+      - item.api_token_secret is string
+      - item.api_token_secret | length > 0
+    fail_msg: >-
+      Every PBS exporter instance must define a lowercase name, netbox_name,
+      api_token_id, and api_token_secret.
+  loop: "{{ pbs_exporter_instances }}"
+  loop_control:
+    label: hidden
+  no_log: true
+
+- name: Validate unique PBS exporter instance selectors
+  ansible.builtin.assert:
+    that:
+      - >-
+        pbs_exporter_instances | map(attribute='name') | unique | list | length
+        == pbs_exporter_instances | length
+      - >-
+        pbs_exporter_instances | map(attribute='netbox_name') | unique | list | length
+        == pbs_exporter_instances | length
+    fail_msg: PBS exporter instance names and NetBox names must be unique.
+  no_log: true
+
+- name: Remove legacy Prometheus PBS exporter
   community.docker.docker_container:
     name: prometheus-pbs_exporter
+    state: absent
+  no_log: true
+
+- name: Setup Prometheus PBS exporters
+  community.docker.docker_container:
+    name: "prometheus-pbs-exporter-{{ item.name }}"
     image: "{{ pbs_exporter_image | default('natrontech/pbs-exporter:latest') }}"
     networks:
       - name: monitoring
     dns_servers: "{{ dns.servers }}"
     env:
-      PBS_TOKEN_ID: "{{ pbs_exporter_api_token_id }}"
-      PBS_TOKEN_SECRET: "{{ pbs_exporter_api_token_secret }}"
+      PBS_TOKEN_ID: "{{ item.api_token_id }}"
+      PBS_TOKEN_SECRET: "{{ item.api_token_secret }}"
       PBS_TLS_INSECURE: "true"
     restart: true
     restart_policy: always
@@ -28,4 +68,8 @@
     pull: true
     labels:
       traefik.enable: "false"
+      com.unitix.prometheus.pbs-exporter: "true"
+  loop: "{{ pbs_exporter_instances }}"
+  loop_control:
+    label: hidden
   no_log: true

From 6428baf4e8120baf631d2319262341034f4649bd Mon Sep 17 00:00:00 2001
From: "opencode-agent[bot]" <opencode-agent[bot]@users.noreply.github.com>
Date: Sun, 2 Aug 2026 14:50:04 +0000
Subject: [PATCH 3/4] Single PBS exporter with target config

Co-authored-by: 33Fraise33 <33Fraise33@users.noreply.github.com>
---
 .../files/prometheus/alerts.d/pbs.rules.yml   |  6 +-
 .../templates/prometheus/prometheus.yml.j2    | 11 +--
 playbooks/host_vars/beokpdcovm01.yml          |  9 +++
 roles/prometheus/tasks/pbs_exporter.yml       | 78 +++++++++++--------
 roles/prometheus/templates/pbs_targets.yml.j2 |  9 +++
 5 files changed, 71 insertions(+), 42 deletions(-)
 create mode 100644 roles/prometheus/templates/pbs_targets.yml.j2

diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
index 22091fe..17b3afc 100644
--- a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
+++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
@@ -2,7 +2,7 @@
 groups:
   - name: pbs.rules
     rules:
-      - alert: PBSBackupFailed
+      - alert: PBS - Backup Failed
         expr: pbs_task_status{task_type="backup",status=~"(?i:failed|error)"} == 1
         for: 10m
         labels:
@@ -11,7 +11,7 @@ groups:
           summary: "PBS backup failed on {{ $labels.node_name }}"
           description: "Backup {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
 
-      - alert: PBSVerificationFailed
+      - alert: PBS - Verification Failed
         expr: pbs_task_status{task_type="verify",status=~"(?i:failed|error)"} == 1
         for: 10m
         labels:
@@ -20,7 +20,7 @@ groups:
           summary: "PBS verification failed on {{ $labels.node_name }}"
           description: "Verification {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
 
-      - alert: PBSSyncFailed
+      - alert: PBS - Sync Failed
         expr: pbs_task_status{task_type="sync",status=~"(?i:failed|error)"} == 1
         for: 10m
         labels:
diff --git a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
index 2aba4f4..2db0036 100644
--- a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
+++ b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
@@ -263,19 +263,15 @@ scrape_configs:
           - "prometheus-adguard_exporter:9618"
 
 {% if prometheus.features.pbs_exporter | default(false) %}
-{% for pbs_instance in pbs_exporter_instances | default([]) %}
-  - job_name: 'pbs_exporter_{{ pbs_instance.name }}'
+  - job_name: 'pbs_exporter'
     metrics_path: /probe
     http_sd_configs:
       - <<: *netbox_auth
         url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
     relabel_configs:
       - source_labels: [__meta_netbox_name]
-        regex: '^{{ pbs_instance.netbox_name | regex_escape }}$'
-        action: keep
-      - source_labels: [__meta_netbox_primary_ip]
         target_label: __param_target
-      - source_labels: [__param_target]
+      - source_labels: [__meta_netbox_primary_ip]
         target_label: instance
       - source_labels: [__meta_netbox_name]
         target_label: node_name
@@ -286,9 +282,8 @@ scrape_configs:
       - source_labels: [__meta_netbox_role_slug]
         target_label: device_role
       - target_label: __address__
-        replacement: "prometheus-pbs-exporter-{{ pbs_instance.name }}:9569"
+        replacement: "prometheus-pbs_exporter:9569"
 
-{% endfor %}
 {% endif %}
 
   - job_name: "hass"
diff --git a/playbooks/host_vars/beokpdcovm01.yml b/playbooks/host_vars/beokpdcovm01.yml
index bae1724..20abb9a 100644
--- a/playbooks/host_vars/beokpdcovm01.yml
+++ b/playbooks/host_vars/beokpdcovm01.yml
@@ -7,6 +7,15 @@ prometheus:
   features:
     pbs_exporter: true
 
+# Each target is keyed by the PBS VM's NetBox name. The target must be the
+# endpoint accepted by the PBS exporter; token secrets belong in Vault.
+# pbs_exporter_targets:
+#   - netbox_name: pbs.example
+#     target: https://pbs.example:8007
+#     api_token_id: prometheus@pbs!exporter
+#     api_token_secret: !vault |
+#       $ANSIBLE_VAULT;1.1;AES256
+
 cloudflared:
   docker_network: services
   service_token: !vault |
diff --git a/roles/prometheus/tasks/pbs_exporter.yml b/roles/prometheus/tasks/pbs_exporter.yml
index c111cfb..129b094 100644
--- a/roles/prometheus/tasks/pbs_exporter.yml
+++ b/roles/prometheus/tasks/pbs_exporter.yml
@@ -1,24 +1,23 @@
 ---
-- name: Validate PBS exporter instances list
+- name: Validate PBS exporter targets list
   ansible.builtin.assert:
     that:
-      - pbs_exporter_instances is defined
-      - pbs_exporter_instances is sequence
-      - pbs_exporter_instances is not string
-      - pbs_exporter_instances | length > 0
-    fail_msg: >-
-      pbs_exporter_instances must be a non-empty inventory list.
+      - pbs_exporter_targets is defined
+      - pbs_exporter_targets is sequence
+      - pbs_exporter_targets is not string
+      - pbs_exporter_targets | length > 0
+    fail_msg: pbs_exporter_targets must be a non-empty inventory list.
   no_log: true
 
-- name: Validate PBS exporter instances
+- name: Validate PBS exporter targets
   ansible.builtin.assert:
     that:
-      - item.name is defined
-      - item.name is string
-      - item.name is match('^[a-z0-9][a-z0-9-]*$')
       - item.netbox_name is defined
       - item.netbox_name is string
       - item.netbox_name | length > 0
+      - item.target is defined
+      - item.target is string
+      - item.target | length > 0
       - item.api_token_id is defined
       - item.api_token_id is string
       - item.api_token_id | length > 0
@@ -26,42 +25,62 @@
       - item.api_token_secret is string
       - item.api_token_secret | length > 0
     fail_msg: >-
-      Every PBS exporter instance must define a lowercase name, netbox_name,
-      api_token_id, and api_token_secret.
-  loop: "{{ pbs_exporter_instances }}"
+      Every PBS exporter target must define netbox_name, target, api_token_id,
+      and api_token_secret.
+  loop: "{{ pbs_exporter_targets }}"
   loop_control:
     label: hidden
   no_log: true
 
-- name: Validate unique PBS exporter instance selectors
+- name: Validate unique PBS exporter target selectors
   ansible.builtin.assert:
     that:
       - >-
-        pbs_exporter_instances | map(attribute='name') | unique | list | length
-        == pbs_exporter_instances | length
+        pbs_exporter_targets | map(attribute='netbox_name') | unique | list | length
+        == pbs_exporter_targets | length
       - >-
-        pbs_exporter_instances | map(attribute='netbox_name') | unique | list | length
-        == pbs_exporter_instances | length
-    fail_msg: PBS exporter instance names and NetBox names must be unique.
+        pbs_exporter_targets | map(attribute='target') | unique | list | length
+        == pbs_exporter_targets | length
+    fail_msg: PBS exporter NetBox names and targets must be unique.
+  no_log: true
+
+- name: Create PBS exporter config directory
+  ansible.builtin.file:
+    path: "{{ docker.dir.config }}/prometheus_pbs_exporter"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
   no_log: true
 
-- name: Remove legacy Prometheus PBS exporter
+- name: Render PBS exporter targets
+  ansible.builtin.template:
+    src: pbs_targets.yml.j2
+    dest: "{{ docker.dir.config }}/prometheus_pbs_exporter/targets.yml"
+    owner: root
+    group: root
+    mode: "0600"
+  no_log: true
+
+- name: Remove legacy per-target PBS exporters
   community.docker.docker_container:
-    name: prometheus-pbs_exporter
+    name: "prometheus-pbs-exporter-{{ item.name }}"
     state: absent
+  loop: "{{ pbs_exporter_instances | default([]) }}"
+  loop_control:
+    label: hidden
   no_log: true
 
-- name: Setup Prometheus PBS exporters
+- name: Setup Prometheus PBS exporter
   community.docker.docker_container:
-    name: "prometheus-pbs-exporter-{{ item.name }}"
+    name: prometheus-pbs_exporter
     image: "{{ pbs_exporter_image | default('natrontech/pbs-exporter:latest') }}"
     networks:
       - name: monitoring
     dns_servers: "{{ dns.servers }}"
-    env:
-      PBS_TOKEN_ID: "{{ item.api_token_id }}"
-      PBS_TOKEN_SECRET: "{{ item.api_token_secret }}"
-      PBS_TLS_INSECURE: "true"
+    volumes:
+      - "{{ docker.dir.config }}/prometheus_pbs_exporter/targets.yml:/config/targets.yml:ro"
+    command: "--config.file=/config/targets.yml"
     restart: true
     restart_policy: always
     state: started
@@ -69,7 +88,4 @@
     labels:
       traefik.enable: "false"
       com.unitix.prometheus.pbs-exporter: "true"
-  loop: "{{ pbs_exporter_instances }}"
-  loop_control:
-    label: hidden
   no_log: true
diff --git a/roles/prometheus/templates/pbs_targets.yml.j2 b/roles/prometheus/templates/pbs_targets.yml.j2
new file mode 100644
index 0000000..cef79f6
--- /dev/null
+++ b/roles/prometheus/templates/pbs_targets.yml.j2
@@ -0,0 +1,9 @@
+---
+targets:
+{% for pbs_target in pbs_exporter_targets %}
+  - name: "{{ pbs_target.netbox_name }}"
+    target: "{{ pbs_target.target }}"
+    token_id: "{{ pbs_target.api_token_id }}"
+    token_secret: "{{ pbs_target.api_token_secret }}"
+    tls_insecure: true
+{% endfor %}

From 9a73c7195af0418c372c911daeda90bc67e9625b Mon Sep 17 00:00:00 2001
From: Gianni Stubbe <git@frai.se>
Date: Sun, 2 Aug 2026 22:25:06 +0200
Subject: [PATCH 4/4] fix: Further PBS monitoring fixes

---
 .../files/prometheus/alerts.d/pbs.rules.yml   | 72 +++++++++++++---
 .../templates/prometheus/prometheus.yml.j2    | 14 ++-
 playbooks/host_vars/beokpdcort01.yml          |  4 +-
 playbooks/host_vars/beokpdcovm01.yml          | 84 ++++++++++++++++--
 roles/prometheus/tasks/pbs_exporter.yml       | 86 +++----------------
 roles/prometheus/templates/pbs_targets.yml.j2 |  9 --
 6 files changed, 161 insertions(+), 108 deletions(-)
 delete mode 100644 roles/prometheus/templates/pbs_targets.yml.j2

diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
index 17b3afc..b74413d 100644
--- a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
+++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml
@@ -2,29 +2,77 @@
 groups:
   - name: pbs.rules
     rules:
-      - alert: PBS - Backup Failed
-        expr: pbs_task_status{task_type="backup",status=~"(?i:failed|error)"} == 1
+      - alert: PBS - Down
+        expr: pbs_up == 0
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS backup failed on {{ $labels.node_name }}"
-          description: "Backup {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+          summary: "PBS Offline"
+          description: "PBS job is not returning a healthy state."
 
-      - alert: PBS - Verification Failed
-        expr: pbs_task_status{task_type="verify",status=~"(?i:failed|error)"} == 1
+      - alert: PBS - Verification Not Successful
+        expr: pbs_snapshot_vm_last_verify == 0
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS verification failed on {{ $labels.node_name }}"
-          description: "Verification {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+          summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
+          description: >-
+            The latest snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is not verified successfully.
 
-      - alert: PBS - Sync Failed
-        expr: pbs_task_status{task_type="sync",status=~"(?i:failed|error)"} == 1
+      - alert: PBS - Local Backup Stale
+        expr: >-
+          (time() - pbs_snapshot_vm_last_timestamp > 172800)
+          and on (site_slug, namespace)
+          label_replace(
+            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+            "namespace", "$1", "site_slug", "(.+)"
+          )
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS sync failed on {{ $labels.node_name }}"
-          description: "Sync {{ $labels.task }} for datastore {{ $labels.datastore }} failed."
+          summary: "PBS backup for {{ $labels.vm_name }} is older than two days"
+          description: >-
+            The latest local snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is older than two days.
+
+      - alert: PBS - Sync Backup Stale
+        expr: >-
+          (time() - pbs_snapshot_vm_last_timestamp > 172800)
+          unless on (site_slug, namespace)
+          label_replace(
+            count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+            "namespace", "$1", "site_slug", "(.+)"
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS sync for {{ $labels.vm_name }} is older than two days"
+          description: >-
+            The latest remote snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} is older than two days.
+
+      - alert: PBS - Sync Missing
+        expr: >-
+          count by (site_slug, node_name, instance) (pbs_snapshot_vm_last_timestamp)
+          unless on (site_slug, node_name, instance)
+          count by (site_slug, node_name, instance) (
+            pbs_snapshot_vm_last_timestamp
+            unless on (site_slug, namespace)
+            label_replace(
+              count by (site_slug) (pbs_snapshot_vm_last_timestamp),
+              "namespace", "$1", "site_slug", "(.+)"
+            )
+          )
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "PBS has no remote snapshots"
+          description: >-
+            PBS {{ $labels.node_name }} has local snapshots but no snapshots in
+            a remote namespace.
diff --git a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
index 2db0036..e5b90c8 100644
--- a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
+++ b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2
@@ -264,12 +264,18 @@ scrape_configs:
 
 {% if prometheus.features.pbs_exporter | default(false) %}
   - job_name: 'pbs_exporter'
-    metrics_path: /probe
+    metrics_path: /metrics
     http_sd_configs:
       - <<: *netbox_auth
         url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
     relabel_configs:
-      - source_labels: [__meta_netbox_name]
+      - source_labels: [__meta_netbox_primary_ip]
+        regex: '([0-9.]+)'
+        replacement: 'https://$1:8007'
+        target_label: __param_target
+      - source_labels: [__meta_netbox_primary_ip]
+        regex: '([0-9a-fA-F:]*:[0-9a-fA-F:]+)'
+        replacement: 'https://[$1]:8007'
         target_label: __param_target
       - source_labels: [__meta_netbox_primary_ip]
         target_label: instance
@@ -277,12 +283,14 @@ scrape_configs:
         target_label: node_name
       - source_labels: [__meta_netbox_site]
         target_label: site
+      - source_labels: [__meta_netbox_site_slug]
+        target_label: site_slug
       - source_labels: [__meta_netbox_tenant]
         target_label: tenant
       - source_labels: [__meta_netbox_role_slug]
         target_label: device_role
       - target_label: __address__
-        replacement: "prometheus-pbs_exporter:9569"
+        replacement: "prometheus-pbs_exporter:10019"
 
 {% endif %}
 
diff --git a/playbooks/host_vars/beokpdcort01.yml b/playbooks/host_vars/beokpdcort01.yml
index c475dde..fc56316 100644
--- a/playbooks/host_vars/beokpdcort01.yml
+++ b/playbooks/host_vars/beokpdcort01.yml
@@ -106,7 +106,7 @@ firewall:
           ip: "@MONITORING"
         destination:
           protocol: tcp
-          port: "{ 80,443,9090,9100,9323,9342 }"  # http,https,federate,node_exporter,docker_exporter,frr_exporter
+          port: "{ 80,443,8007,9090,9100,9323,9342 }"  # http,https,pbs,federate,node_exporter,docker_exporter,frr_exporter
         counter: true
       - comment: "Allow from MONITORING"
         type: ip6
@@ -114,7 +114,7 @@ firewall:
           ip: "@v6_MONITORING"
         destination:
           protocol: tcp
-          port: "{ 80,443,9090,9100,9323,9342 }"  # http,https,node_exporter,docker_exporter,frr_exporter
+          port: "{ 80,443,8007,9090,9100,9323,9342 }"  # http,https,pbs,node_exporter,docker_exporter,frr_exporter
         counter: true
       # DNS
       - comment: "Allow Adguard to Recursive"
diff --git a/playbooks/host_vars/beokpdcovm01.yml b/playbooks/host_vars/beokpdcovm01.yml
index 20abb9a..3caaf8a 100644
--- a/playbooks/host_vars/beokpdcovm01.yml
+++ b/playbooks/host_vars/beokpdcovm01.yml
@@ -3,18 +3,86 @@ dawarich_photon_api_host: photon.frai.se
 dawarich_photon_api_port: "443"
 dawarich_photon_api_use_https: "true"
 
+adguard_hostname: dns1.frai.se
+
+docker:
+  dir:
+    config: /data/docker-config
+    data: /data/docker
+  networks:
+    monitoring:
+      driver_options:
+        parent: ens19.1001
+      hosts:
+        loki: 94
+      ipam:
+        - gateway: 10.10.48.65
+          subnet: 10.10.48.64/27
+        - gateway: 2001:678:1080:1001::1
+          subnet: 2001:678:1080:1001::/64
+      ipv6: true
+      type: ipvlan
+    networking:
+      driver_options:
+        parent: ens19.1004
+      hosts:
+        uisp: 189
+        unifi: 190
+      ipam:
+        - gateway: 10.10.48.161
+          subnet: 10.10.48.160/27
+        - gateway: 2001:678:1080:1004::1
+          subnet: 2001:678:1080:1004::/64
+      ipv6: true
+      type: ipvlan
+    services:
+      driver_options:
+        parent: ens19.1002
+      hosts:
+        adguard: 124
+        traefik: 125
+        unbound: 123
+      ipam:
+        - gateway: 10.10.48.97
+          subnet: 10.10.48.96/27
+        - gateway: 2001:678:1080:1102::1
+          subnet: 2001:678:1080:1102::/64
+      ipv6: true
+      type: ipvlan
+  users:
+    - fraise
+
+mounts:
+  - fs: ext4
+    mountpoint: /data/
+    source: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2
+
+netbox_interface: true
+
+portainer:
+  network: services
+  type: portainer-ce
+
 prometheus:
   features:
+    adguard: true
+    alertmanager: true
+    blackbox: true
+    graphite: true
+    ipmi: true
+    snmp: true
+    speedtest: true
     pbs_exporter: true
 
-# Each target is keyed by the PBS VM's NetBox name. The target must be the
-# endpoint accepted by the PBS exporter; token secrets belong in Vault.
-# pbs_exporter_targets:
-#   - netbox_name: pbs.example
-#     target: https://pbs.example:8007
-#     api_token_id: prometheus@pbs!exporter
-#     api_token_secret: !vault |
-#       $ANSIBLE_VAULT;1.1;AES256
+# Shared secret for the prometheus@pbs!exporter token configured on every PBS.
+pbs_exporter_api_token: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  66653733653366643937326266313733396333356535656563306634613739333964316434303736
+  3530373137643934333034373738366631353838666663660a326662613839346663383832626538
+  39613466356665323864376164323835396661666137313039396436373736386331306362363137
+  3233313231613636330a613430353262356234386366346534386462376362636136316130306238
+  31386434316431383361613835323464626661313530383834306231623764393032313932343938
+  3934663937333435663532346261663261643038343063303030
 
 cloudflared:
   docker_network: services
diff --git a/roles/prometheus/tasks/pbs_exporter.yml b/roles/prometheus/tasks/pbs_exporter.yml
index 129b094..ac210ac 100644
--- a/roles/prometheus/tasks/pbs_exporter.yml
+++ b/roles/prometheus/tasks/pbs_exporter.yml
@@ -1,91 +1,29 @@
 ---
-- name: Validate PBS exporter targets list
+- name: Validate PBS exporter API token
   ansible.builtin.assert:
     that:
-      - pbs_exporter_targets is defined
-      - pbs_exporter_targets is sequence
-      - pbs_exporter_targets is not string
-      - pbs_exporter_targets | length > 0
-    fail_msg: pbs_exporter_targets must be a non-empty inventory list.
-  no_log: true
-
-- name: Validate PBS exporter targets
-  ansible.builtin.assert:
-    that:
-      - item.netbox_name is defined
-      - item.netbox_name is string
-      - item.netbox_name | length > 0
-      - item.target is defined
-      - item.target is string
-      - item.target | length > 0
-      - item.api_token_id is defined
-      - item.api_token_id is string
-      - item.api_token_id | length > 0
-      - item.api_token_secret is defined
-      - item.api_token_secret is string
-      - item.api_token_secret | length > 0
-    fail_msg: >-
-      Every PBS exporter target must define netbox_name, target, api_token_id,
-      and api_token_secret.
-  loop: "{{ pbs_exporter_targets }}"
-  loop_control:
-    label: hidden
-  no_log: true
-
-- name: Validate unique PBS exporter target selectors
-  ansible.builtin.assert:
-    that:
-      - >-
-        pbs_exporter_targets | map(attribute='netbox_name') | unique | list | length
-        == pbs_exporter_targets | length
-      - >-
-        pbs_exporter_targets | map(attribute='target') | unique | list | length
-        == pbs_exporter_targets | length
-    fail_msg: PBS exporter NetBox names and targets must be unique.
-  no_log: true
-
-- name: Create PBS exporter config directory
-  ansible.builtin.file:
-    path: "{{ docker.dir.config }}/prometheus_pbs_exporter"
-    state: directory
-    owner: root
-    group: root
-    mode: "0700"
-  no_log: true
-
-- name: Render PBS exporter targets
-  ansible.builtin.template:
-    src: pbs_targets.yml.j2
-    dest: "{{ docker.dir.config }}/prometheus_pbs_exporter/targets.yml"
-    owner: root
-    group: root
-    mode: "0600"
-  no_log: true
-
-- name: Remove legacy per-target PBS exporters
-  community.docker.docker_container:
-    name: "prometheus-pbs-exporter-{{ item.name }}"
-    state: absent
-  loop: "{{ pbs_exporter_instances | default([]) }}"
-  loop_control:
-    label: hidden
+      - pbs_exporter_api_token is defined
+      - pbs_exporter_api_token is string
+      - pbs_exporter_api_token | length > 0
+    fail_msg: pbs_exporter_api_token must be supplied as an inventory variable.
   no_log: true
 
 - name: Setup Prometheus PBS exporter
   community.docker.docker_container:
     name: prometheus-pbs_exporter
-    image: "{{ pbs_exporter_image | default('natrontech/pbs-exporter:latest') }}"
+    image: ghcr.io/natrontech/pbs-exporter:latest
+    user: "65534:65534"
     networks:
       - name: monitoring
     dns_servers: "{{ dns.servers }}"
-    volumes:
-      - "{{ docker.dir.config }}/prometheus_pbs_exporter/targets.yml:/config/targets.yml:ro"
-    command: "--config.file=/config/targets.yml"
+    env:
+      PBS_USERNAME: pbs_exporter@pbs
+      PBS_API_TOKEN_NAME: pbs_exporter_token
+      PBS_API_TOKEN: "{{ pbs_exporter_api_token }}"
+      PBS_INSECURE: "true"
     restart: true
     restart_policy: always
     state: started
     pull: true
     labels:
       traefik.enable: "false"
-      com.unitix.prometheus.pbs-exporter: "true"
-  no_log: true
diff --git a/roles/prometheus/templates/pbs_targets.yml.j2 b/roles/prometheus/templates/pbs_targets.yml.j2
deleted file mode 100644
index cef79f6..0000000
--- a/roles/prometheus/templates/pbs_targets.yml.j2
+++ /dev/null
@@ -1,9 +0,0 @@
----
-targets:
-{% for pbs_target in pbs_exporter_targets %}
-  - name: "{{ pbs_target.netbox_name }}"
-    target: "{{ pbs_target.target }}"
-    token_id: "{{ pbs_target.api_token_id }}"
-    token_secret: "{{ pbs_target.api_token_secret }}"
-    tls_insecure: true
-{% endfor %}