Skip to content

Commit 2525c50

Browse files
33Fraise3333Fraise33
authored
Merge pull request #22 from 33Fraise33/fix/snmp_alerting
Updated wud & alerting
2 parents 977dfda + 0a3aed1 commit 2525c50

13 files changed

Lines changed: 107 additions & 39 deletions

File tree

‎playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/generic.rules.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,12 @@ groups:
1313
for: 10m
1414
annotations:
1515
summary: "Endpoint {{ $labels.node_name }} has high latency: {{ $value }}"
16+
17+
- alert: High packet loss
18+
expr: (1 - avg_over_time(probe_success[15m])) * 100 > 5
19+
for: 10m
20+
annotations:
21+
summary: "Endpoint {{ $labels.node_name }} has high packet loss: {{ $value }}"
1622

1723
- alert: Non running containers
1824
expr: engine_daemon_container_states_containers{state!="running"} > 0

‎playbooks/host_files/beokpdcovm01/files/prometheus/alerts.d/pbs.rules.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,17 @@ groups:
1212
description: "PBS job is not returning a healthy state."
1313

1414
- alert: PBS - Verification Not Successful
15-
expr: pbs_snapshot_vm_last_verify == 0
15+
expr: >-
16+
(max_over_time(pbs_snapshot_vm_last_verify[2d]) == 0)
17+
and (pbs_snapshot_vm_count > 1)
1618
for: 10m
1719
labels:
1820
severity: critical
1921
annotations:
20-
summary: "PBS snapshot for {{ $labels.vm_name }} is not verified successfully"
22+
summary: "No successfully verified PBS snapshot for {{ $labels.vm_name }} in the last two days"
2123
description: >-
22-
The latest snapshot for {{ $labels.vm_name }} on
23-
{{ $labels.node_name }} is not verified successfully.
24+
No successfully verified snapshot for {{ $labels.vm_name }} on
25+
{{ $labels.node_name }} has been observed in the last two days.
2426
2527
- alert: PBS - Local Backup Stale
2628
expr: >-

‎playbooks/host_vars/bejbegiavm01.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,14 @@ cloudflared:
7676
30386539343235616237363535376530383364626331343330316231353864653562656562343463
7777
31666565393963633964623339373133393536393130356330633034373566326331653363646237
7878
36353962383230376465323564323332666239663064333963306364663862383363
79+
80+
wud_registry_logins:
81+
WUD_REGISTRY_HUB_PUBLIC_LOGIN: "bejbegiavm01fraise"
82+
WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault |
83+
$ANSIBLE_VAULT;1.1;AES256
84+
31666334373731396164393162643864636131346263646464366331663435346536366339303330
85+
3436323835393166383734333833383832363537306533660a633439633835313734316631626663
86+
63303732623837366534646230323432376662636331326164363834303365326161626363613439
87+
3665393664643462650a313465386539613466323265393233386338356139303634373964636439
88+
35383364646335663264373637396663303664386662393534353233653038386462306564353963
89+
3239386537656236653761633933303664373235313235346439

‎playbooks/host_vars/beokpdcovm01.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,27 @@ prometheus:
7474
speedtest: true
7575
pbs_exporter: true
7676

77+
wud_registry_logins:
78+
WUD_REGISTRY_HUB_PUBLIC_LOGIN: "beokpdcovm01fraise"
79+
WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault |
80+
$ANSIBLE_VAULT;1.1;AES256
81+
34393966326332633062333833636435396163666532613262383638656433626335323738613861
82+
6564343166376364393636363933363736633932376261640a663361613739613462633636386232
83+
61366434323833306433326633336263303034343331313863373436326464636531303632313035
84+
3764333733353834610a356266383962663036323533616566613839623031373832356638313261
85+
37396264393634663462326136616361313961643465376363303137333466633231
86+
WUD_REGISTRY_CUSTOM_SOLIDTIME_URL: https://registry.on-premise.solidtime.io
87+
WUD_REGISTRY_CUSTOM_SOLIDTIME_LOGIN: op-72971212
88+
WUD_REGISTRY_CUSTOM_SOLIDTIME_PASSWORD: "{{ solidtime_registry_pass }}"
89+
90+
solidtime_registry_pass: !vault |
91+
$ANSIBLE_VAULT;1.1;AES256
92+
62663437373265363363663064313033636363303738396239303465353764323931376464633436
93+
3730303138336335313436343533373734373065326632620a303262373066313161313362303830
94+
65353265633836303563353139336362653862663632373861326162623666363932613964353764
95+
6562653834393365330a646463626231616463386163636333343937383333363161326130373431
96+
37326332376631623636333231316562376264633538313761646161666439366161
97+
7798
# Shared secret for the prometheus@pbs!exporter token configured on every PBS.
7899
pbs_exporter_api_token: !vault |
79100
$ANSIBLE_VAULT;1.1;AES256

‎playbooks/host_vars/beokpdcovm02.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,14 @@ firewall:
1212
sets: "{{ firewall_sets_all + firewall_sets_internal }}"
1313
filter:
1414
input: "{{ firewall_filter_input_host + firewall_filter_input_all }}"
15+
16+
wud_registry_logins:
17+
WUD_REGISTRY_HUB_PUBLIC_LOGIN: "beokpdcovm02fraise"
18+
WUD_REGISTRY_HUB_PUBLIC_PASSWORD: !vault |
19+
$ANSIBLE_VAULT;1.1;AES256
20+
35323362373531626336656335323935636535616136626430306536383838336633613135303766
21+
6138376134383738656337316432373265303339386662360a653935303935313937393665643463
22+
37643364373530666332353335306332663366626132323436653062396365323633616636323934
23+
6665613936343031380a313063376434393333363331313734653534383737373364316463313563
24+
34336631623463343038326436383036623765303161316139626161643233353966323931653732
25+
6133616537383263346363633361616631616363356364633830

‎roles/grafana/tasks/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
&& Header(`X-Authelia-Bypass`, `{{ authelia_bypass_header }}`)"
4242
traefik.http.routers.grafana-bypass.tls: "true"
4343
traefik.http.routers.grafana-bypass.tls.certresolver: "le"
44-
traefik.http.routers.grafana-bypass.service: "netbox-service"
44+
traefik.http.routers.grafana-bypass.service: "grafana-service"
4545
traefik.http.routers.grafana-bypass.priority: "100"
4646
# SHARED
4747
traefik.http.services.grafana-service.loadbalancer.server.port: "3000"

‎roles/n8n/tasks/main.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,11 @@
4040
DB_POSTGRESDB_HOST: postgres
4141
DB_POSTGRESDB_USER: "{{ postgres_user }}"
4242
DB_POSTGRESDB_PASSWORD: "{{ postgres_password }}"
43-
WEBHOOK_URL: https://n8n.frai.se
43+
N8N_WEBHOOK_URL: https://n8n.frai.se
4444
N8N_HOST: n8n.frai.se
4545
N8N_EDITOR_BASE_URL: https://n8n.frai.se
4646
N8N_PROXY_HOPS: "1"
4747
N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: "true"
48-
N8N_RUNNERS_ENABLED: "true"
4948
state: started
5049
restart: true
5150
restart_policy: always

‎roles/prometheus/templates/alertmanager.yml.j2‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,23 @@ route:
66
group_by: ['site']
77
repeat_interval: 10000h
88

9+
inhibit_rules:
10+
- source_matchers:
11+
- alertname = "Device offline"
12+
target_matchers:
13+
- alertname = "Probe Failed"
14+
equal: ['node_name']
15+
- source_matchers:
16+
- alertname = "Probe Failed"
17+
target_matchers:
18+
- alertname = "High packet loss"
19+
equal: ['node_name']
20+
- source_matchers:
21+
- alertname = "Probe Failed"
22+
target_matchers:
23+
- alertname = "High latency"
24+
equal: ['node_name']
25+
926
receivers:
1027
- name: default
1128
# N8N to matrix

‎roles/solidtime/tasks/main.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
env:
112112
CONTAINER_MODE: http
113113
AUTO_DB_MIGRATE: "true"
114+
APP_URL: https://time.unitix.be
114115
env_file: "{{ docker.dir.config }}/solidtime/laravel.env"
115116
state: healthy
116117
pull: true

‎roles/solidtime/vars/main.yml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,6 @@ solidtime_postgres_pass: !vault |
99
3766366136313366650a303332336436376364623435323064313863373963366466356530396233
1010
62333737313562376661643130373662663264666165313863633838663336666236373232383462
1111
6331396231356534653163373661343539636434333537303461
12-
solidtime_registry_pass: !vault |
13-
$ANSIBLE_VAULT;1.1;AES256
14-
62663437373265363363663064313033636363303738396239303465353764323931376464633436
15-
3730303138336335313436343533373734373065326632620a303262373066313161313362303830
16-
65353265633836303563353139336362653862663632373861326162623666363932613964353764
17-
6562653834393365330a646463626231616463386163636333343937383333363161326130373431
18-
37326332376631623636333231316562376264633538313761646161666439366161
1912
solidtime_app_key: !vault |
2013
$ANSIBLE_VAULT;1.1;AES256
2114
61656330666232333336363462363036393763623364646336333332343964343333356564633635

0 commit comments

Comments
 (0)