Updated wud & alerting

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml

@@ -13,6 +13,12 @@ groups:
         for: 10m
         annotations:
           summary: "Endpoint {{ $labels.node_name }} has high latency: {{ $value }}"
+
+      - alert: High packet loss
+        expr: (1 - avg_over_time(probe_success[15m])) * 100 > 5
+        for: 10m
+        annotations:
+          summary: "Endpoint {{ $labels.node_name }} has high packet loss: {{ $value }}"
 
       - alert: Non running containers
         expr: engine_daemon_container_states_containers{state!="running"} > 0

playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml

@@ -12,15 +12,17 @@ groups:
           description: "PBS job is not returning a healthy state."
 
       - alert: PBS - Verification Not Successful
-        expr: pbs_snapshot_vm_last_verify == 0
+        expr: >-
+          (max_over_time(pbs_snapshot_vm_last_verify[2d]) == 0)
+          and (pbs_snapshot_vm_count > 1)
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
+          summary: "No successfully verified PBS snapshot for {{ $labels.vm_name }} in the last two days"
           description: >-
-            The latest snapshot for {{ $labels.vm_name }} on
-            {{ $labels.node_name }} is not verified successfully.
+            No successfully verified snapshot for {{ $labels.vm_name }} on
+            {{ $labels.node_name }} has been observed in the last two days.
 
       - alert: PBS - Local Backup Stale
         expr: >-

playbooks/host_vars/bejbegiavm01.yml

@@ -76,3 +76,14 @@ cloudflared:
     30386539343235616237363535376530383364626331343330316231353864653562656562343463
     31666565393963633964623339373133393536393130356330633034373566326331653363646237
     36353962383230376465323564323332666239663064333963306364663862383363
+
+wud_registry_logins:
+  WUD_REGISTRY_HUB_PUBLIC_LOGIN: "bejbegiavm01fraise"
+  WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault |
+    $ANSIBLE_VAULT;1.1;AES256
+    31666334373731396164393162643864636131346263646464366331663435346536366339303330
+    3436323835393166383734333833383832363537306533660a633439633835313734316631626663
+    63303732623837366534646230323432376662636331326164363834303365326161626363613439
+    3665393664643462650a313465386539613466323265393233386338356139303634373964636439
+    35383364646335663264373637396663303664386662393534353233653038386462306564353963
+    3239386537656236653761633933303664373235313235346439

playbooks/host_vars/beokpdcovm01.yml

@@ -74,6 +74,27 @@ prometheus:
     speedtest: true
     pbs_exporter: true
 
+wud_registry_logins:
+  WUD_REGISTRY_HUB_PUBLIC_LOGIN: "beokpdcovm01fraise"
+  WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault |
+    $ANSIBLE_VAULT;1.1;AES256
+    34393966326332633062333833636435396163666532613262383638656433626335323738613861
+    6564343166376364393636363933363736633932376261640a663361613739613462633636386232
+    61366434323833306433326633336263303034343331313863373436326464636531303632313035
+    3764333733353834610a356266383962663036323533616566613839623031373832356638313261
+    37396264393634663462326136616361313961643465376363303137333466633231
+  WUD_REGISTRY_CUSTOM_SOLIDTIME_URL: https://registry.on-premise.solidtime.io
+  WUD_REGISTRY_CUSTOM_SOLIDTIME_LOGIN: op-72971212
+  WUD_REGISTRY_CUSTOM_SOLIDTIME_PASSWORD: "{{ solidtime_registry_pass }}"
+
+solidtime_registry_pass: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  62663437373265363363663064313033636363303738396239303465353764323931376464633436
+  3730303138336335313436343533373734373065326632620a303262373066313161313362303830
+  65353265633836303563353139336362653862663632373861326162623666363932613964353764
+  6562653834393365330a646463626231616463386163636333343937383333363161326130373431
+  37326332376631623636333231316562376264633538313761646161666439366161
+
 # Shared secret for the prometheus@pbs!exporter token configured on every PBS.
 pbs_exporter_api_token: !vault |
   $ANSIBLE_VAULT;1.1;AES256

playbooks/host_vars/beokpdcovm02.yml

@@ -12,3 +12,14 @@ firewall:
   sets: "{{ firewall_sets_all + firewall_sets_internal }}"
   filter:
     input: "{{ firewall_filter_input_host + firewall_filter_input_all }}"
+
+wud_registry_logins:
+  WUD_REGISTRY_HUB_PUBLIC_LOGIN: "beokpdcovm02fraise"
+  WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault |
+    $ANSIBLE_VAULT;1.1;AES256
+    35323362373531626336656335323935636535616136626430306536383838336633613135303766
+    6138376134383738656337316432373265303339386662360a653935303935313937393665643463
+    37643364373530666332353335306332663366626132323436653062396365323633616636323934
+    6665613936343031380a313063376434393333363331313734653534383737373364316463313563
+    34336631623463343038326436383036623765303161316139626161643233353966323931653732
+    6133616537383263346363633361616631616363356364633830

roles/grafana/tasks/main.yml

@@ -41,7 +41,7 @@
         && Header(`X-Authelia-Bypass`, `{{ authelia_bypass_header }}`)"
       traefik.http.routers.grafana-bypass.tls: "true"
       traefik.http.routers.grafana-bypass.tls.certresolver: "le"
-      traefik.http.routers.grafana-bypass.service: "netbox-service"
+      traefik.http.routers.grafana-bypass.service: "grafana-service"
       traefik.http.routers.grafana-bypass.priority: "100"
       # SHARED
       traefik.http.services.grafana-service.loadbalancer.server.port: "3000"

roles/n8n/tasks/main.yml

@@ -40,12 +40,11 @@
       DB_POSTGRESDB_HOST: postgres
       DB_POSTGRESDB_USER: "{{ postgres_user }}"
       DB_POSTGRESDB_PASSWORD: "{{ postgres_password }}"
-      WEBHOOK_URL: https://n8n.frai.se
+      N8N_WEBHOOK_URL: https://n8n.frai.se
       N8N_HOST: n8n.frai.se
       N8N_EDITOR_BASE_URL: https://n8n.frai.se
       N8N_PROXY_HOPS: "1"
       N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: "true"
-      N8N_RUNNERS_ENABLED: "true"
     state: started
     restart: true
     restart_policy: always

roles/prometheus/templates/alertmanager.yml.j2

@@ -6,6 +6,23 @@ route:
   group_by: ['site']
   repeat_interval: 10000h
 
+inhibit_rules:
+  - source_matchers:
+      - alertname = "Device offline"
+    target_matchers:
+      - alertname = "Probe Failed"
+    equal: ['node_name']
+  - source_matchers:
+      - alertname = "Probe Failed"
+    target_matchers:
+      - alertname = "High packet loss"
+    equal: ['node_name']
+  - source_matchers:
+      - alertname = "Probe Failed"
+    target_matchers:
+      - alertname = "High latency"
+    equal: ['node_name']
+
 receivers:
   - name: default
     # N8N to matrix

roles/solidtime/tasks/main.yml

@@ -111,6 +111,7 @@
     env:
       CONTAINER_MODE: http
       AUTO_DB_MIGRATE: "true"
+      APP_URL: https://time.unitix.be
     env_file: "{{ docker.dir.config }}/solidtime/laravel.env"
     state: healthy
     pull: true

roles/solidtime/vars/main.yml

@@ -9,13 +9,6 @@ solidtime_postgres_pass: !vault |
   3766366136313366650a303332336436376364623435323064313863373963366466356530396233
   62333737313562376661643130373662663264666165313863633838663336666236373232383462
   6331396231356534653163373661343539636434333537303461
-solidtime_registry_pass: !vault |
-  $ANSIBLE_VAULT;1.1;AES256
-  62663437373265363363663064313033636363303738396239303465353764323931376464633436
-  3730303138336335313436343533373734373065326632620a303262373066313161313362303830
-  65353265633836303563353139336362653862663632373861326162623666363932613964353764
-  6562653834393365330a646463626231616463386163636333343937383333363161326130373431
-  37326332376631623636333231316562376264633538313761646161666439366161
 solidtime_app_key: !vault |
   $ANSIBLE_VAULT;1.1;AES256
   61656330666232333336363462363036393763623364646336333332343964343333356564633635

roles/wud/defaults/main.yml

@@ -0,0 +1,21 @@
+---
+wud_n8n_path: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  31356663383334663234346364346436656238656435653431323063663037323264643534626266
+  6334333334376462643934393837323166396536636434650a393637636664306366623661336338
+  63623364613730616135616536333139343333336164626362623336636562306466383138666333
+  6139373834343739310a663736383737613361353038633831323864663266376364636231306531
+  64316463366365636362613438653832666537363938303465633535316362636633633535356163
+  3035623535313032376633383465393932393462333139343738
+# IN N8N Set: "Bearer <Token>" as value header auth value and "Authorization" as header name
+wud_n8n_bearer: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  30323135326331336566653234343463646134306434316264353837666663333239333565643135
+  3131626664633333633762383661336362613961343161610a383631353034383138313865336563
+  66343433313962343039323264386336633037663763333535356363633939663135613666306566
+  3734366631336464390a333766326565663334363765363635363161646539346565643931393334
+  38383936653031646434323965646333393239623764653463666464323439633537393862636233
+  3761376266373563303166353436643830666331393866376239
+wud_registry_logins:
+  WUD_REGISTRY_HUB_PUBLIC_LOGIN: "{{ docker_login.username }}"
+  WUD_REGISTRY_HUB_PUBLIC_PASSWORD: "{{ docker_login.password }}"

roles/wud/tasks/main.yml

@@ -14,10 +14,19 @@
       timeout: 10s
       retries: 3
       start_period: 10s
-    env:
+    env: "{{ static_dict | combine(wud_registry_logins) | combine (wud_extra_registries | default({}) ) }}"
+    restart: true
+    state: healthy
+    restart_policy: always
+    pull: true
+    labels:
+      traefik.enable: "false"
+  vars:
+    static_dict:
       WUD_SERVER_PORT: "9103"
       WUD_WATCHER_LOCAL_CRON: "33 4 * * *"
-      WUD_WATCHER_LOCAL_WATCHDIGESTDEFAULT: "true"
+      WUD_REGISTRY_HUB_PUBLIC_WATCHDIGEST: "true"
+      WUD_REGISTRY_HUB_PUBLIC_SUPPRESSDIGESTWATCHWARNING: "true"
       WUD_TRIGGER_DOCKER_UPGRADEANY_AUTO: "true"
       WUD_TRIGGER_DOCKER_UPGRADEANY_PRUNE: "true"
       WUD_TRIGGER_HTTP_NOTIFYANY_AUTO: "true"
@@ -26,9 +35,3 @@
       WUD_TRIGGER_HTTP_NOTIFYANY_SIMPLEBODY: "{{ inventory_hostname }}"
       WUD_TRIGGER_HTTP_NOTIFYANY_AUTH_TYPE: BEARER
       WUD_TRIGGER_HTTP_NOTIFYANY_AUTH_BEARER: "{{ wud_n8n_bearer }}"
-    restart: true
-    state: healthy
-    restart_policy: always
-    pull: true
-    labels:
-      traefik.enable: "false"