feat(observability): add Prometheus + Grafana compose profile

.env.example

@@ -9,5 +9,8 @@ ACME_EMAIL=admin@example.com
 POSTGRES_USER=devops
 POSTGRES_PASSWORD=devops
 
+# Observability (docker compose --profile observe up)
+GRAFANA_PASSWORD=admin
+
 # GenAI
 LLM_API_KEY=

README.md

@@ -68,6 +68,7 @@ pixi run pre-commit-install
 ├── infra/
 │   ├── helm/                   # Helm chart scaffold
 │   ├── compose/                # Compose overrides
+│   ├── monitoring/             # Prometheus + Grafana config (observe profile)
 │   └── postgres/               # Dev DB init scripts
 ├── docs/
 │   └── submissions/
@@ -112,6 +113,16 @@ docker compose up --build
 
 Starts all services plus shared infrastructure (Postgres, NATS). Service env vars (`DATABASE_URL`, `NATS_URL`) are pre-wired.
 
+Enable observability stack (opt-in):
+
+```bash
+docker compose --profile observe up
+```
+
+- Prometheus: `http://localhost:9090` - configured to scrape all 9 services + NATS; service targets require a Prometheus-compatible `/metrics` endpoint to appear `UP`
+- Grafana: `http://localhost:3001` - Prometheus pre-wired as default datasource; dashboards persist via `grafana_data` volume
+- Set `GRAFANA_PASSWORD` in `.env` (default: `admin`)
+
 - Production compose with Traefik + TLS: `docker-compose.prod.yml`
 - Override the image tag: `IMAGE_TAG=v0.1.0 docker compose up`
 

docker-compose.yml

@@ -153,5 +153,31 @@ services:
         condition: service_started
     restart: unless-stopped
 
+  # ── Observability (opt-in: docker compose --profile observe up) ───────────
+
+  prometheus:
+    image: prom/prometheus:v3.3.1
+    profiles: [observe]
+    volumes:
+      - ./infra/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    ports:
+      - "9090:9090"
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.6.1
+    profiles: [observe]
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./infra/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+    ports:
+      - "3001:3000"
+    depends_on:
+      - prometheus
+    restart: unless-stopped
+
 volumes:
   postgres_data:
+  grafana_data:

infra/monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true

infra/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,41 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: gateway
+    static_configs:
+      - targets: ["gateway:8080"]
+
+  - job_name: incident-service
+    static_configs:
+      - targets: ["incident-service:8081"]
+
+  - job_name: event-service
+    static_configs:
+      - targets: ["event-service:8082"]
+
+  - job_name: rule-engine
+    static_configs:
+      - targets: ["rule-engine:8083"]
+
+  - job_name: user-service
+    static_configs:
+      - targets: ["user-service:8084"]
+
+  - job_name: notification-service
+    static_configs:
+      - targets: ["notification-service:8085"]
+
+  - job_name: webhook-service
+    static_configs:
+      - targets: ["webhook-service:8086"]
+
+  - job_name: genai-service
+    static_configs:
+      - targets: ["genai-service:8087"]
+
+  - job_name: nats
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["nats:8222"]