Skip to content

Commit 68b85e9

Browse files
33Fraise3333Fraise33
authored
Merge pull request #17 from 33Fraise33/opencode/issue16-20260802083648
Add PBS exporter and failure alerts
2 parents 4201f46 + 9a73c71 commit 68b85e9

6 files changed

Lines changed: 227 additions & 2 deletions

File tree

‎playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
---
2+
groups:
3+
- name: pbs.rules
4+
rules:
5+
- alert: PBS - Down
6+
expr: pbs_up == 0
7+
for: 10m
8+
labels:
9+
severity: critical
10+
annotations:
11+
summary: "PBS Offline"
12+
description: "PBS job is not returning a healthy state."
13+
14+
- alert: PBS - Verification Not Successful
15+
expr: pbs_snapshot_vm_last_verify == 0
16+
for: 10m
17+
labels:
18+
severity: critical
19+
annotations:
20+
summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
21+
description: >-
22+
The latest snapshot for {{ $labels.vm_name }} on
23+
{{ $labels.node_name }} is not verified successfully.
24+
25+
- alert: PBS - Local Backup Stale
26+
expr: >-
27+
(time() - pbs_snapshot_vm_last_timestamp > 172800)
28+
and on (site_slug, namespace)
29+
label_replace(
30+
count by (site_slug) (pbs_snapshot_vm_last_timestamp),
31+
"namespace", "$1", "site_slug", "(.+)"
32+
)
33+
for: 10m
34+
labels:
35+
severity: critical
36+
annotations:
37+
summary: "PBS backup for {{ $labels.vm_name }} is older than two days"
38+
description: >-
39+
The latest local snapshot for {{ $labels.vm_name }} on
40+
{{ $labels.node_name }} is older than two days.
41+
42+
- alert: PBS - Sync Backup Stale
43+
expr: >-
44+
(time() - pbs_snapshot_vm_last_timestamp > 172800)
45+
unless on (site_slug, namespace)
46+
label_replace(
47+
count by (site_slug) (pbs_snapshot_vm_last_timestamp),
48+
"namespace", "$1", "site_slug", "(.+)"
49+
)
50+
for: 10m
51+
labels:
52+
severity: critical
53+
annotations:
54+
summary: "PBS sync for {{ $labels.vm_name }} is older than two days"
55+
description: >-
56+
The latest remote snapshot for {{ $labels.vm_name }} on
57+
{{ $labels.node_name }} is older than two days.
58+
59+
- alert: PBS - Sync Missing
60+
expr: >-
61+
count by (site_slug, node_name, instance) (pbs_snapshot_vm_last_timestamp)
62+
unless on (site_slug, node_name, instance)
63+
count by (site_slug, node_name, instance) (
64+
pbs_snapshot_vm_last_timestamp
65+
unless on (site_slug, namespace)
66+
label_replace(
67+
count by (site_slug) (pbs_snapshot_vm_last_timestamp),
68+
"namespace", "$1", "site_slug", "(.+)"
69+
)
70+
)
71+
for: 10m
72+
labels:
73+
severity: critical
74+
annotations:
75+
summary: "PBS has no remote snapshots"
76+
description: >-
77+
PBS {{ $labels.node_name }} has local snapshots but no snapshots in
78+
a remote namespace.

‎playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,38 @@ scrape_configs:
262262
- targets:
263263
- "prometheus-adguard_exporter:9618"
264264

265+
{% if prometheus.features.pbs_exporter | default(false) %}
266+
- job_name: 'pbs_exporter'
267+
metrics_path: /metrics
268+
http_sd_configs:
269+
- <<: *netbox_auth
270+
url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs
271+
relabel_configs:
272+
- source_labels: [__meta_netbox_primary_ip]
273+
regex: '([0-9.]+)'
274+
replacement: 'https://$1:8007'
275+
target_label: __param_target
276+
- source_labels: [__meta_netbox_primary_ip]
277+
regex: '([0-9a-fA-F:]*:[0-9a-fA-F:]+)'
278+
replacement: 'https://[$1]:8007'
279+
target_label: __param_target
280+
- source_labels: [__meta_netbox_primary_ip]
281+
target_label: instance
282+
- source_labels: [__meta_netbox_name]
283+
target_label: node_name
284+
- source_labels: [__meta_netbox_site]
285+
target_label: site
286+
- source_labels: [__meta_netbox_site_slug]
287+
target_label: site_slug
288+
- source_labels: [__meta_netbox_tenant]
289+
target_label: tenant
290+
- source_labels: [__meta_netbox_role_slug]
291+
target_label: device_role
292+
- target_label: __address__
293+
replacement: "prometheus-pbs_exporter:10019"
294+
295+
{% endif %}
296+
265297
- job_name: "hass"
266298
metrics_path: /api/prometheus
267299
authorization:

‎playbooks/host_vars/beokpdcort01.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,15 +106,15 @@ firewall:
106106
ip: "@MONITORING"
107107
destination:
108108
protocol: tcp
109-
port: "{ 80,443,9090,9100,9323,9342 }" # http,https,federate,node_exporter,docker_exporter,frr_exporter
109+
port: "{ 80,443,8007,9090,9100,9323,9342 }" # http,https,pbs,federate,node_exporter,docker_exporter,frr_exporter
110110
counter: true
111111
- comment: "Allow from MONITORING"
112112
type: ip6
113113
source:
114114
ip: "@v6_MONITORING"
115115
destination:
116116
protocol: tcp
117-
port: "{ 80,443,9090,9100,9323,9342 }" # http,https,node_exporter,docker_exporter,frr_exporter
117+
port: "{ 80,443,8007,9090,9100,9323,9342 }" # http,https,pbs,node_exporter,docker_exporter,frr_exporter
118118
counter: true
119119
# DNS
120120
- comment: "Allow Adguard to Recursive"

‎playbooks/host_vars/beokpdcovm01.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,87 @@ dawarich_photon_api_host: photon.frai.se
33
dawarich_photon_api_port: "443"
44
dawarich_photon_api_use_https: "true"
55

6+
adguard_hostname: dns1.frai.se
7+
8+
docker:
9+
dir:
10+
config: /data/docker-config
11+
data: /data/docker
12+
networks:
13+
monitoring:
14+
driver_options:
15+
parent: ens19.1001
16+
hosts:
17+
loki: 94
18+
ipam:
19+
- gateway: 10.10.48.65
20+
subnet: 10.10.48.64/27
21+
- gateway: 2001:678:1080:1001::1
22+
subnet: 2001:678:1080:1001::/64
23+
ipv6: true
24+
type: ipvlan
25+
networking:
26+
driver_options:
27+
parent: ens19.1004
28+
hosts:
29+
uisp: 189
30+
unifi: 190
31+
ipam:
32+
- gateway: 10.10.48.161
33+
subnet: 10.10.48.160/27
34+
- gateway: 2001:678:1080:1004::1
35+
subnet: 2001:678:1080:1004::/64
36+
ipv6: true
37+
type: ipvlan
38+
services:
39+
driver_options:
40+
parent: ens19.1002
41+
hosts:
42+
adguard: 124
43+
traefik: 125
44+
unbound: 123
45+
ipam:
46+
- gateway: 10.10.48.97
47+
subnet: 10.10.48.96/27
48+
- gateway: 2001:678:1080:1102::1
49+
subnet: 2001:678:1080:1102::/64
50+
ipv6: true
51+
type: ipvlan
52+
users:
53+
- fraise
54+
55+
mounts:
56+
- fs: ext4
57+
mountpoint: /data/
58+
source: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2
59+
60+
netbox_interface: true
61+
62+
portainer:
63+
network: services
64+
type: portainer-ce
65+
66+
prometheus:
67+
features:
68+
adguard: true
69+
alertmanager: true
70+
blackbox: true
71+
graphite: true
72+
ipmi: true
73+
snmp: true
74+
speedtest: true
75+
pbs_exporter: true
76+
77+
# Shared secret for the prometheus@pbs!exporter token configured on every PBS.
78+
pbs_exporter_api_token: !vault |
79+
$ANSIBLE_VAULT;1.1;AES256
80+
66653733653366643937326266313733396333356535656563306634613739333964316434303736
81+
3530373137643934333034373738366631353838666663660a326662613839346663383832626538
82+
39613466356665323864376164323835396661666137313039396436373736386331306362363137
83+
3233313231613636330a613430353262356234386366346534386462376362636136316130306238
84+
31386434316431383361613835323464626661313530383834306231623764393032313932343938
85+
3934663937333435663532346261663261643038343063303030
86+
687
cloudflared:
788
docker_network: services
889
service_token: !vault |

‎roles/prometheus/tasks/main.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@
2727
tags: adguard, adguard_exporter
2828
when: prometheus.features.adguard | default (false)
2929

30+
- name: Setup PBS exporter
31+
ansible.builtin.import_tasks: pbs_exporter.yml
32+
tags: pbs, pbs_exporter, pbs-exporter
33+
when: prometheus.features.pbs_exporter | default(false)
34+
3035
- name: Setup alertmanager
3136
ansible.builtin.import_tasks: alertmanager.yml
3237
tags: alertmanager, alerts, alert

‎roles/prometheus/tasks/pbs_exporter.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
- name: Validate PBS exporter API token
3+
ansible.builtin.assert:
4+
that:
5+
- pbs_exporter_api_token is defined
6+
- pbs_exporter_api_token is string
7+
- pbs_exporter_api_token | length > 0
8+
fail_msg: pbs_exporter_api_token must be supplied as an inventory variable.
9+
no_log: true
10+
11+
- name: Setup Prometheus PBS exporter
12+
community.docker.docker_container:
13+
name: prometheus-pbs_exporter
14+
image: ghcr.io/natrontech/pbs-exporter:latest
15+
user: "65534:65534"
16+
networks:
17+
- name: monitoring
18+
dns_servers: "{{ dns.servers }}"
19+
env:
20+
PBS_USERNAME: pbs_exporter@pbs
21+
PBS_API_TOKEN_NAME: pbs_exporter_token
22+
PBS_API_TOKEN: "{{ pbs_exporter_api_token }}"
23+
PBS_INSECURE: "true"
24+
restart: true
25+
restart_policy: always
26+
state: started
27+
pull: true
28+
labels:
29+
traefik.enable: "false"

0 commit comments

Comments
 (0)