health checks for everything work

Author: trzysiek

docker-compose.yml

@@ -1,13 +1,19 @@
 version: '3.8'
 
+######################
+# APPS (microservices)
+# - service1
+# - service2
+# - health-monitor
+
 services:
   service1:
     build: ./service1
     container_name: service1
     ports:
       - "5001:5001"
     networks:
-      - microservices-network
+      - grafana_network
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
       interval: 30s
@@ -20,7 +26,7 @@ services:
     ports:
       - "5002:5002"
     networks:
-      - microservices-network
+      - grafana_network
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5002/health"]
       interval: 30s
@@ -33,7 +39,7 @@ services:
     ports:
       - "5000:5000"
     networks:
-      - microservices-network
+      - grafana_network
     depends_on:
       - service1
       - service2
@@ -43,6 +49,102 @@ services:
       - MAX_RETRIES=3      # Max consecutive failures before marking as down
     restart: unless-stopped
 
+######################
+# MONITORING
+# - loki
+# - prometheus
+# - grafana
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--storage.tsdb.retention.time=200h'
+      - '--web.enable-lifecycle'
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9090/-/healthy"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    networks:
+      - grafana_network
+
+  grafana:
+    image: grafana/grafana:12.0.2
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_USERS_ALLOW_SIGN_UP=false
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    depends_on:
+      - prometheus
+    networks:
+      - grafana_network
+
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./loki:/etc/loki
+      - loki_data:/loki
+    command: -config.file=/etc/loki/loki.yml
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3100/ready"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    networks:
+      - grafana_network
+
+  # Example microservice with metrics endpoint
+  metrics_generator:
+    image: prom/node-exporter:latest
+    container_name: example_metrics
+    ports:
+      - "9100:9100"
+    networks:
+      - grafana_network
+
+  log-generator:
+    build:
+      context: ./log-generator
+      dockerfile: Dockerfile
+    container_name: log-generator
+    networks:
+      - grafana_network
+    depends_on:
+      - loki
+
+volumes:
+  grafana_data:
+    driver: local
+  prometheus_data:
+    driver: local
+  loki_data:
+    driver: local
+
 networks:
-  microservices-network:
+  grafana_network:
     driver: bridge

health-monitor/monitor.py

@@ -22,6 +22,27 @@
         'last_check': None,
         'last_response_time': None,
         'consecutive_failures': 0
+    },
+    'prometheus': {
+        'url': 'http://prometheus:9090/-/healthy',
+        'status': 'unknown',
+        'last_check': None,
+        'last_response_time': None,
+        'consecutive_failures': 0
+    },
+    'grafana': {
+        'url': 'http://grafana:3000/api/health',
+        'status': 'unknown',
+        'last_check': None,
+        'last_response_time': None,
+        'consecutive_failures': 0
+    },
+    'loki': {
+        'url': 'http://loki:3100/ready',
+        'status': 'unknown',
+        'last_check': None,
+        'last_response_time': None,
+        'consecutive_failures': 0
     }
 }