add alertmanager

Author: Kroner

.github/workflows/main.yaml

@@ -21,9 +21,7 @@ jobs:
           POSTGRES_DB=test
           POSTGRES_HOST=postgres
           POSTGRES_PORT=5432
-          # Добавьте другие переменные, если нужно
           EOF
-          cat .env  # Для проверки (опционально)
           
       - name: Install Docker Compose (v2)
         run: |
@@ -96,4 +94,6 @@ jobs:
           pwd
           mkdir -p roles/app/defaults
           echo -e "\npostgres_password: ${{ secrets.POSTGRES_PASS }}" >> roles/app/defaults/main.yaml
+          echo -e "\ntelegram_bot_token: ${{ secrets.BOT_TOKEN}}" >> roles/app/defaults/main.yaml
+          echo -e "\ntelegram_chat_id: ${{ secrets.BOT_CHAT_ID }}" >> roles/app/defaults/main.yaml
           ansible-playbook playbook.yaml

alertmanager/alertmanager.yml.template

@@ -0,0 +1,40 @@
+route:
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 1h 
+
+  # Дефолтный получатель (если ни один route не сработал)
+  receiver: 'telegram-warning'
+
+  # Правила маршрутизации (проверяются сверху вниз!)
+  routes:
+  - match:
+      severity: 'critical'
+    receiver: 'telegram-critical'
+    continue: true
+
+  - match:
+      severity: 'warning'
+    receiver: 'telegram-warning'
+
+receivers:
+- name: 'telegram-critical'
+  telegram_configs:
+  - api_url: "https://api.telegram.org"
+    bot_token: "$TELEGRAM_BOT_TOKEN"
+    chat_id: $TELEGRAM_CHAT_ID
+    message: "🚨 CRITICAL: {{ .CommonAnnotations.summary }}"
+    parse_mode: "HTML"
+
+- name: 'telegram-warning'
+  telegram_configs:
+  - api_url: "https://api.telegram.org"
+    bot_token: "$TELEGRAM_BOT_TOKEN"
+    chat_id: $TELEGRAM_CHAT_ID
+    message: |
+      {{ if eq .Labels.severity "warning" }}
+      ⚠️ WARNING: {{ .CommonAnnotations.summary }}
+      {{ else }}
+      ℹ️ INFO: {{ .CommonAnnotations.summary }}
+      {{ end }}
+    parse_mode: "HTML"

ansible/defaults/main.yaml

@@ -1 +1,3 @@
-postgres_password: "postgres"
+postgres_password: "postgres"
+telegram_bot_token: "none"
+telegram_chat_id: "12345678"

ansible/roles/app/tasks/main.yaml

@@ -21,8 +21,32 @@
       POSTGRES_DB=messages
       POSTGRES_HOST=postgres
       POSTGRES_PORT=5432
+      TELEGRAM_BOT_TOKEN={{ telegram_bot_token }}
+      TELEGRAM_CHAT_ID={{ telegram_chat_id }}
     mode: '0644'
 
+- name: Ensure alertmanager directory exists
+  ansible.builtin.file:
+    path: "{{ app_dir }}/alertmanager"
+    state: directory
+    mode: '0755'
+
+- name: Install gettext for envsubst
+  become: yes
+  apt:
+    name: gettext-base
+    state: present
+  when: ansible_os_family == 'Debian'
+
+- name: Generate alertmanager.yml from template
+  ansible.builtin.shell: |
+    envsubst < "{{ app_dir }}/alertmanager/alertmanager.yml.template" > "{{ app_dir }}/alertmanager/alertmanager.yml"
+  environment:
+    TELEGRAM_BOT_TOKEN: "{{ telegram_bot_token }}"
+    TELEGRAM_CHAT_ID: "{{ telegram_chat_id }}"
+  args:
+    executable: /bin/sh
+
 - name: Run Docker Compose
   command: docker compose up -d --force-recreate 
   args:

docker-compose.yaml

@@ -38,7 +38,7 @@ services:
     ports:
       - "9090:9090"  # Веб-интерфейс Prometheus
     volumes:
-      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml  # Конфиг
+      - ./prometheus/:/etc/prometheus/  # Конфиr|Алерты
     networks:
       - app-network
     depends_on:
@@ -56,7 +56,36 @@ services:
     depends_on:
       - prometheus  # Grafana зависит от Prometheus
 
+  # Добавляем node_exporter
+  node_exporter:
+    image: prom/node-exporter:latest
+    container_name: node_exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+    ports:
+      - '9100:9100'
+    networks:
+      - app-network
+    pid: "host"  # Важно для доступа к системным метрикам
 
+  alertmanager:
+    container_name: alertmanager
+    image: prom/alertmanager:latest
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+    ports:
+      - "9093:9093"
+    networks:
+      - app-network
+    depends_on:
+      - prometheus 
+    
 # Объявляем сети и тома
 volumes:
   postgres_data:  # Том для данных Postgres (не удаляется при перезапуске)

prometheus/alert.rules.yml

@@ -0,0 +1,19 @@
+groups:
+- name: host_alerts
+  rules:
+  - alert: HighCPU
+    expr: instance:node_cpu_usage:percentage > 90
+    for: 20s
+    labels:
+      severity: "critical"
+    annotations:
+      summary: "High CPU usage ({{ $value }}%) on {{ $labels.instance }}"
+      description: "CPU превысил 90% более чем на 2 минут."
+
+  - alert: HighMemory
+    expr: instance:node_memory_usage:percentage > 85
+    for: 2m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "High RAM usage ({{ $value }}%) on {{ $labels.instance }}"

prometheus/prometheus.yml

@@ -1,7 +1,22 @@
 global:
-  scrape_interval: 15s  # Частота сбора метрик
+  scrape_interval: 15s        # Как часто собирать метрики
+  evaluation_interval: 20s    # Как часто проверять правила (alert/record)
+
+rule_files:
+  - '/etc/prometheus/record.rules.yml'  # Record Rules
+  - '/etc/prometheus/alert.rules.yml'
+
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: ['alertmanager:9093']
 
 scrape_configs:
   - job_name: "app"
     static_configs:
-      - targets: ["app:8080"]  # Имя сервиса из docker-compose + порт метрик
+      - targets: ["app:8080"] 
+
+  - job_name: 'node_exporter'
+    scrape_interval: 5s 
+    static_configs:
+      - targets: ['node_exporter:9100']

prometheus/record.rules.yml

@@ -0,0 +1,13 @@
+groups:
+- name: host_metrics
+  interval: 15s
+  rules:
+  - record: instance:node_cpu_usage:percentage
+    expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)
+    labels:
+      metric_type: "cpu"
+
+  - record: instance:node_memory_usage:percentage
+    expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100
+    labels:
+      metric_type: "memory"