diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml new file mode 100644 index 0000000..b74413d --- /dev/null +++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml @@ -0,0 +1,78 @@ +--- +groups: + - name: pbs.rules + rules: + - alert: PBS - Down + expr: pbs_up == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "PBS Offline" + description: "PBS job is not returning a healthy state." + + - alert: PBS - Verification Not Successful + expr: pbs_snapshot_vm_last_verify == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully" + description: >- + The latest snapshot for {{ $labels.vm_name }} on + {{ $labels.node_name }} is not verified successfully. + + - alert: PBS - Local Backup Stale + expr: >- + (time() - pbs_snapshot_vm_last_timestamp > 172800) + and on (site_slug, namespace) + label_replace( + count by (site_slug) (pbs_snapshot_vm_last_timestamp), + "namespace", "$1", "site_slug", "(.+)" + ) + for: 10m + labels: + severity: critical + annotations: + summary: "PBS backup for {{ $labels.vm_name }} is older than two days" + description: >- + The latest local snapshot for {{ $labels.vm_name }} on + {{ $labels.node_name }} is older than two days. + + - alert: PBS - Sync Backup Stale + expr: >- + (time() - pbs_snapshot_vm_last_timestamp > 172800) + unless on (site_slug, namespace) + label_replace( + count by (site_slug) (pbs_snapshot_vm_last_timestamp), + "namespace", "$1", "site_slug", "(.+)" + ) + for: 10m + labels: + severity: critical + annotations: + summary: "PBS sync for {{ $labels.vm_name }} is older than two days" + description: >- + The latest remote snapshot for {{ $labels.vm_name }} on + {{ $labels.node_name }} is older than two days. + + - alert: PBS - Sync Missing + expr: >- + count by (site_slug, node_name, instance) (pbs_snapshot_vm_last_timestamp) + unless on (site_slug, node_name, instance) + count by (site_slug, node_name, instance) ( + pbs_snapshot_vm_last_timestamp + unless on (site_slug, namespace) + label_replace( + count by (site_slug) (pbs_snapshot_vm_last_timestamp), + "namespace", "$1", "site_slug", "(.+)" + ) + ) + for: 10m + labels: + severity: critical + annotations: + summary: "PBS has no remote snapshots" + description: >- + PBS {{ $labels.node_name }} has local snapshots but no snapshots in + a remote namespace. diff --git a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 index 03f389c..e5b90c8 100644 --- a/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 +++ b/playbooks/host_files/beokpdcovm01/templates/prometheus/prometheus.yml.j2 @@ -262,6 +262,38 @@ scrape_configs: - targets: - "prometheus-adguard_exporter:9618" +{% if prometheus.features.pbs_exporter | default(false) %} + - job_name: 'pbs_exporter' + metrics_path: /metrics + http_sd_configs: + - <<: *netbox_auth + url: https://netbox.unitix.be/api/plugins/prometheus-sd/virtual-machines/?status=active&tag=pbs + relabel_configs: + - source_labels: [__meta_netbox_primary_ip] + regex: '([0-9.]+)' + replacement: 'https://$1:8007' + target_label: __param_target + - source_labels: [__meta_netbox_primary_ip] + regex: '([0-9a-fA-F:]*:[0-9a-fA-F:]+)' + replacement: 'https://[$1]:8007' + target_label: __param_target + - source_labels: [__meta_netbox_primary_ip] + target_label: instance + - source_labels: [__meta_netbox_name] + target_label: node_name + - source_labels: [__meta_netbox_site] + target_label: site + - source_labels: [__meta_netbox_site_slug] + target_label: site_slug + - source_labels: [__meta_netbox_tenant] + target_label: tenant + - source_labels: [__meta_netbox_role_slug] + target_label: device_role + - target_label: __address__ + replacement: "prometheus-pbs_exporter:10019" + +{% endif %} + - job_name: "hass" metrics_path: /api/prometheus authorization: diff --git a/playbooks/host_vars/beokpdcort01.yml b/playbooks/host_vars/beokpdcort01.yml index c475dde..fc56316 100644 --- a/playbooks/host_vars/beokpdcort01.yml +++ b/playbooks/host_vars/beokpdcort01.yml @@ -106,7 +106,7 @@ firewall: ip: "@MONITORING" destination: protocol: tcp - port: "{ 80,443,9090,9100,9323,9342 }" # http,https,federate,node_exporter,docker_exporter,frr_exporter + port: "{ 80,443,8007,9090,9100,9323,9342 }" # http,https,pbs,federate,node_exporter,docker_exporter,frr_exporter counter: true - comment: "Allow from MONITORING" type: ip6 @@ -114,7 +114,7 @@ firewall: ip: "@v6_MONITORING" destination: protocol: tcp - port: "{ 80,443,9090,9100,9323,9342 }" # http,https,node_exporter,docker_exporter,frr_exporter + port: "{ 80,443,8007,9090,9100,9323,9342 }" # http,https,pbs,node_exporter,docker_exporter,frr_exporter counter: true # DNS - comment: "Allow Adguard to Recursive" diff --git a/playbooks/host_vars/beokpdcovm01.yml b/playbooks/host_vars/beokpdcovm01.yml index 31a0d40..3caaf8a 100644 --- a/playbooks/host_vars/beokpdcovm01.yml +++ b/playbooks/host_vars/beokpdcovm01.yml @@ -3,6 +3,87 @@ dawarich_photon_api_host: photon.frai.se dawarich_photon_api_port: "443" dawarich_photon_api_use_https: "true" +adguard_hostname: dns1.frai.se + +docker: + dir: + config: /data/docker-config + data: /data/docker + networks: + monitoring: + driver_options: + parent: ens19.1001 + hosts: + loki: 94 + ipam: + - gateway: 10.10.48.65 + subnet: 10.10.48.64/27 + - gateway: 2001:678:1080:1001::1 + subnet: 2001:678:1080:1001::/64 + ipv6: true + type: ipvlan + networking: + driver_options: + parent: ens19.1004 + hosts: + uisp: 189 + unifi: 190 + ipam: + - gateway: 10.10.48.161 + subnet: 10.10.48.160/27 + - gateway: 2001:678:1080:1004::1 + subnet: 2001:678:1080:1004::/64 + ipv6: true + type: ipvlan + services: + driver_options: + parent: ens19.1002 + hosts: + adguard: 124 + traefik: 125 + unbound: 123 + ipam: + - gateway: 10.10.48.97 + subnet: 10.10.48.96/27 + - gateway: 2001:678:1080:1102::1 + subnet: 2001:678:1080:1102::/64 + ipv6: true + type: ipvlan + users: + - fraise + +mounts: + - fs: ext4 + mountpoint: /data/ + source: /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2 + +netbox_interface: true + +portainer: + network: services + type: portainer-ce + +prometheus: + features: + adguard: true + alertmanager: true + blackbox: true + graphite: true + ipmi: true + snmp: true + speedtest: true + pbs_exporter: true + +# Shared secret for the prometheus@pbs!exporter token configured on every PBS. +pbs_exporter_api_token: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 66653733653366643937326266313733396333356535656563306634613739333964316434303736 + 3530373137643934333034373738366631353838666663660a326662613839346663383832626538 + 39613466356665323864376164323835396661666137313039396436373736386331306362363137 + 3233313231613636330a613430353262356234386366346534386462376362636136316130306238 + 31386434316431383361613835323464626661313530383834306231623764393032313932343938 + 3934663937333435663532346261663261643038343063303030 + cloudflared: docker_network: services service_token: !vault | diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index 412f76e..46930ca 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -27,6 +27,11 @@ tags: adguard, adguard_exporter when: prometheus.features.adguard | default (false) +- name: Setup PBS exporter + ansible.builtin.import_tasks: pbs_exporter.yml + tags: pbs, pbs_exporter, pbs-exporter + when: prometheus.features.pbs_exporter | default(false) + - name: Setup alertmanager ansible.builtin.import_tasks: alertmanager.yml tags: alertmanager, alerts, alert diff --git a/roles/prometheus/tasks/pbs_exporter.yml b/roles/prometheus/tasks/pbs_exporter.yml new file mode 100644 index 0000000..ac210ac --- /dev/null +++ b/roles/prometheus/tasks/pbs_exporter.yml @@ -0,0 +1,29 @@ +--- +- name: Validate PBS exporter API token + ansible.builtin.assert: + that: + - pbs_exporter_api_token is defined + - pbs_exporter_api_token is string + - pbs_exporter_api_token | length > 0 + fail_msg: pbs_exporter_api_token must be supplied as an inventory variable. + no_log: true + +- name: Setup Prometheus PBS exporter + community.docker.docker_container: + name: prometheus-pbs_exporter + image: ghcr.io/natrontech/pbs-exporter:latest + user: "65534:65534" + networks: + - name: monitoring + dns_servers: "{{ dns.servers }}" + env: + PBS_USERNAME: pbs_exporter@pbs + PBS_API_TOKEN_NAME: pbs_exporter_token + PBS_API_TOKEN: "{{ pbs_exporter_api_token }}" + PBS_INSECURE: "true" + restart: true + restart_policy: always + state: started + pull: true + labels: + traefik.enable: "false"