diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml index c773a33..612d248 100644 --- a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml +++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml @@ -13,6 +13,12 @@ groups: for: 10m annotations: summary: "Endpoint {{ $labels.node_name }} has high latency: {{ $value }}" + + - alert: High packet loss + expr: (1 - avg_over_time(probe_success[15m])) * 100 > 5 + for: 10m + annotations: + summary: "Endpoint {{ $labels.node_name }} has high packet loss: {{ $value }}" - alert: Non running containers expr: engine_daemon_container_states_containers{state!="running"} > 0 diff --git a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml index b74413d..3535c2a 100644 --- a/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml +++ b/playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml @@ -12,15 +12,17 @@ groups: description: "PBS job is not returning a healthy state." - alert: PBS - Verification Not Successful - expr: pbs_snapshot_vm_last_verify == 0 + expr: >- + (max_over_time(pbs_snapshot_vm_last_verify[2d]) == 0) + and (pbs_snapshot_vm_count > 1) for: 10m labels: severity: critical annotations: - summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully" + summary: "No successfully verified PBS snapshot for {{ $labels.vm_name }} in the last two days" description: >- - The latest snapshot for {{ $labels.vm_name }} on - {{ $labels.node_name }} is not verified successfully. + No successfully verified snapshot for {{ $labels.vm_name }} on + {{ $labels.node_name }} has been observed in the last two days. - alert: PBS - Local Backup Stale expr: >- diff --git a/playbooks/host_vars/bejbegiavm01.yml b/playbooks/host_vars/bejbegiavm01.yml index 6a75c4e..4243967 100644 --- a/playbooks/host_vars/bejbegiavm01.yml +++ b/playbooks/host_vars/bejbegiavm01.yml @@ -76,3 +76,14 @@ cloudflared: 30386539343235616237363535376530383364626331343330316231353864653562656562343463 31666565393963633964623339373133393536393130356330633034373566326331653363646237 36353962383230376465323564323332666239663064333963306364663862383363 + +wud_registry_logins: + WUD_REGISTRY_HUB_PUBLIC_LOGIN: "bejbegiavm01fraise" + WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31666334373731396164393162643864636131346263646464366331663435346536366339303330 + 3436323835393166383734333833383832363537306533660a633439633835313734316631626663 + 63303732623837366534646230323432376662636331326164363834303365326161626363613439 + 3665393664643462650a313465386539613466323265393233386338356139303634373964636439 + 35383364646335663264373637396663303664386662393534353233653038386462306564353963 + 3239386537656236653761633933303664373235313235346439 \ No newline at end of file diff --git a/playbooks/host_vars/beokpdcovm01.yml b/playbooks/host_vars/beokpdcovm01.yml index 3caaf8a..3152412 100644 --- a/playbooks/host_vars/beokpdcovm01.yml +++ b/playbooks/host_vars/beokpdcovm01.yml @@ -74,6 +74,27 @@ prometheus: speedtest: true pbs_exporter: true +wud_registry_logins: + WUD_REGISTRY_HUB_PUBLIC_LOGIN: "beokpdcovm01fraise" + WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 34393966326332633062333833636435396163666532613262383638656433626335323738613861 + 6564343166376364393636363933363736633932376261640a663361613739613462633636386232 + 61366434323833306433326633336263303034343331313863373436326464636531303632313035 + 3764333733353834610a356266383962663036323533616566613839623031373832356638313261 + 37396264393634663462326136616361313961643465376363303137333466633231 + WUD_REGISTRY_CUSTOM_SOLIDTIME_URL: https://registry.on-premise.solidtime.io + WUD_REGISTRY_CUSTOM_SOLIDTIME_LOGIN: op-72971212 + WUD_REGISTRY_CUSTOM_SOLIDTIME_PASSWORD: "{{ solidtime_registry_pass }}" + +solidtime_registry_pass: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 62663437373265363363663064313033636363303738396239303465353764323931376464633436 + 3730303138336335313436343533373734373065326632620a303262373066313161313362303830 + 65353265633836303563353139336362653862663632373861326162623666363932613964353764 + 6562653834393365330a646463626231616463386163636333343937383333363161326130373431 + 37326332376631623636333231316562376264633538313761646161666439366161 + # Shared secret for the prometheus@pbs!exporter token configured on every PBS. pbs_exporter_api_token: !vault | $ANSIBLE_VAULT;1.1;AES256 diff --git a/playbooks/host_vars/beokpdcovm02.yml b/playbooks/host_vars/beokpdcovm02.yml index dec4542..a4fcc93 100644 --- a/playbooks/host_vars/beokpdcovm02.yml +++ b/playbooks/host_vars/beokpdcovm02.yml @@ -12,3 +12,14 @@ firewall: sets: "{{ firewall_sets_all + firewall_sets_internal }}" filter: input: "{{ firewall_filter_input_host + firewall_filter_input_all }}" + +wud_registry_logins: + WUD_REGISTRY_HUB_PUBLIC_LOGIN: "beokpdcovm02fraise" + WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 35323362373531626336656335323935636535616136626430306536383838336633613135303766 + 6138376134383738656337316432373265303339386662360a653935303935313937393665643463 + 37643364373530666332353335306332663366626132323436653062396365323633616636323934 + 6665613936343031380a313063376434393333363331313734653534383737373364316463313563 + 34336631623463343038326436383036623765303161316139626161643233353966323931653732 + 6133616537383263346363633361616631616363356364633830 \ No newline at end of file diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 231ec76..3aff5e3 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -41,7 +41,7 @@ && Header(`X-Authelia-Bypass`, `{{ authelia_bypass_header }}`)" traefik.http.routers.grafana-bypass.tls: "true" traefik.http.routers.grafana-bypass.tls.certresolver: "le" - traefik.http.routers.grafana-bypass.service: "netbox-service" + traefik.http.routers.grafana-bypass.service: "grafana-service" traefik.http.routers.grafana-bypass.priority: "100" # SHARED traefik.http.services.grafana-service.loadbalancer.server.port: "3000" diff --git a/roles/n8n/tasks/main.yml b/roles/n8n/tasks/main.yml index 880081e..350d118 100644 --- a/roles/n8n/tasks/main.yml +++ b/roles/n8n/tasks/main.yml @@ -40,12 +40,11 @@ DB_POSTGRESDB_HOST: postgres DB_POSTGRESDB_USER: "{{ postgres_user }}" DB_POSTGRESDB_PASSWORD: "{{ postgres_password }}" - WEBHOOK_URL: https://n8n.frai.se + N8N_WEBHOOK_URL: https://n8n.frai.se N8N_HOST: n8n.frai.se N8N_EDITOR_BASE_URL: https://n8n.frai.se N8N_PROXY_HOPS: "1" N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: "true" - N8N_RUNNERS_ENABLED: "true" state: started restart: true restart_policy: always diff --git a/roles/prometheus/templates/alertmanager.yml.j2 b/roles/prometheus/templates/alertmanager.yml.j2 index c2fa6a7..8da6883 100644 --- a/roles/prometheus/templates/alertmanager.yml.j2 +++ b/roles/prometheus/templates/alertmanager.yml.j2 @@ -6,6 +6,23 @@ route: group_by: ['site'] repeat_interval: 10000h +inhibit_rules: + - source_matchers: + - alertname = "Device offline" + target_matchers: + - alertname = "Probe Failed" + equal: ['node_name'] + - source_matchers: + - alertname = "Probe Failed" + target_matchers: + - alertname = "High packet loss" + equal: ['node_name'] + - source_matchers: + - alertname = "Probe Failed" + target_matchers: + - alertname = "High latency" + equal: ['node_name'] + receivers: - name: default # N8N to matrix diff --git a/roles/solidtime/tasks/main.yml b/roles/solidtime/tasks/main.yml index 9c947f8..3f8b7f2 100644 --- a/roles/solidtime/tasks/main.yml +++ b/roles/solidtime/tasks/main.yml @@ -111,6 +111,7 @@ env: CONTAINER_MODE: http AUTO_DB_MIGRATE: "true" + APP_URL: https://time.unitix.be env_file: "{{ docker.dir.config }}/solidtime/laravel.env" state: healthy pull: true diff --git a/roles/solidtime/vars/main.yml b/roles/solidtime/vars/main.yml index e87eaa0..f9d1bf7 100644 --- a/roles/solidtime/vars/main.yml +++ b/roles/solidtime/vars/main.yml @@ -9,13 +9,6 @@ solidtime_postgres_pass: !vault | 3766366136313366650a303332336436376364623435323064313863373963366466356530396233 62333737313562376661643130373662663264666165313863633838663336666236373232383462 6331396231356534653163373661343539636434333537303461 -solidtime_registry_pass: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 62663437373265363363663064313033636363303738396239303465353764323931376464633436 - 3730303138336335313436343533373734373065326632620a303262373066313161313362303830 - 65353265633836303563353139336362653862663632373861326162623666363932613964353764 - 6562653834393365330a646463626231616463386163636333343937383333363161326130373431 - 37326332376631623636333231316562376264633538313761646161666439366161 solidtime_app_key: !vault | $ANSIBLE_VAULT;1.1;AES256 61656330666232333336363462363036393763623364646336333332343964343333356564633635 diff --git a/roles/wud/defaults/main.yml b/roles/wud/defaults/main.yml new file mode 100644 index 0000000..4f05cad --- /dev/null +++ b/roles/wud/defaults/main.yml @@ -0,0 +1,21 @@ +--- +wud_n8n_path: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31356663383334663234346364346436656238656435653431323063663037323264643534626266 + 6334333334376462643934393837323166396536636434650a393637636664306366623661336338 + 63623364613730616135616536333139343333336164626362623336636562306466383138666333 + 6139373834343739310a663736383737613361353038633831323864663266376364636231306531 + 64316463366365636362613438653832666537363938303465633535316362636633633535356163 + 3035623535313032376633383465393932393462333139343738 +# IN N8N Set: "Bearer " as value header auth value and "Authorization" as header name +wud_n8n_bearer: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 30323135326331336566653234343463646134306434316264353837666663333239333565643135 + 3131626664633333633762383661336362613961343161610a383631353034383138313865336563 + 66343433313962343039323264386336633037663763333535356363633939663135613666306566 + 3734366631336464390a333766326565663334363765363635363161646539346565643931393334 + 38383936653031646434323965646333393239623764653463666464323439633537393862636233 + 3761376266373563303166353436643830666331393866376239 +wud_registry_logins: + WUD_REGISTRY_HUB_PUBLIC_LOGIN: "{{ docker_login.username }}" + WUD_REGISTRY_HUB_PUBLIC_PASSWORD: "{{ docker_login.password }}" diff --git a/roles/wud/tasks/main.yml b/roles/wud/tasks/main.yml index 1288262..4a50fa9 100644 --- a/roles/wud/tasks/main.yml +++ b/roles/wud/tasks/main.yml @@ -14,10 +14,19 @@ timeout: 10s retries: 3 start_period: 10s - env: + env: "{{ static_dict | combine(wud_registry_logins) | combine (wud_extra_registries | default({}) ) }}" + restart: true + state: healthy + restart_policy: always + pull: true + labels: + traefik.enable: "false" + vars: + static_dict: WUD_SERVER_PORT: "9103" WUD_WATCHER_LOCAL_CRON: "33 4 * * *" - WUD_WATCHER_LOCAL_WATCHDIGESTDEFAULT: "true" + WUD_REGISTRY_HUB_PUBLIC_WATCHDIGEST: "true" + WUD_REGISTRY_HUB_PUBLIC_SUPPRESSDIGESTWATCHWARNING: "true" WUD_TRIGGER_DOCKER_UPGRADEANY_AUTO: "true" WUD_TRIGGER_DOCKER_UPGRADEANY_PRUNE: "true" WUD_TRIGGER_HTTP_NOTIFYANY_AUTO: "true" @@ -26,9 +35,3 @@ WUD_TRIGGER_HTTP_NOTIFYANY_SIMPLEBODY: "{{ inventory_hostname }}" WUD_TRIGGER_HTTP_NOTIFYANY_AUTH_TYPE: BEARER WUD_TRIGGER_HTTP_NOTIFYANY_AUTH_BEARER: "{{ wud_n8n_bearer }}" - restart: true - state: healthy - restart_policy: always - pull: true - labels: - traefik.enable: "false" diff --git a/roles/wud/vars/main.yml b/roles/wud/vars/main.yml deleted file mode 100644 index e97e97b..0000000 --- a/roles/wud/vars/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -wud_n8n_path: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 31356663383334663234346364346436656238656435653431323063663037323264643534626266 - 6334333334376462643934393837323166396536636434650a393637636664306366623661336338 - 63623364613730616135616536333139343333336164626362623336636562306466383138666333 - 6139373834343739310a663736383737613361353038633831323864663266376364636231306531 - 64316463366365636362613438653832666537363938303465633535316362636633633535356163 - 3035623535313032376633383465393932393462333139343738 -wud_n8n_bearer: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 34393530336232336361386434653763303431346662613136346135666530306233333432356666 - 6464303831363064386437663933383439396633323133320a306133613936333938323839656434 - 37383963626566383434366364336435333564666130633062326632646539626439663865343535 - 6337373533356636300a366535343331323638303636343433316136383933323164373239333235 - 62613965386232393034626336663536393530306338383236656565373835373762636239383833 - 3065623738383231336639333838326331303966656662336162