Skip to content

Commit 4b718f2

Browse files
imol-aiimol-ai
authored
Merge pull request #130 from AET-DevOps26/feat/grafana-logs
Grafana logs, alerting, and dashboards
2 parents 48fa722 + b49b29e commit 4b718f2

10 files changed

Lines changed: 1132 additions & 12 deletions

File tree

‎.github/workflows/deploy-k8s.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ jobs:
6060
# HELP_SERVICE_KEY — Gemini API key for py-help-service
6161
# RECIPE_SERVICE_KEY — Gemini API key for py-recipe-service
6262
# GRAFANA_ADMIN_PASSWORD — Grafana admin password
63+
# DISCORD_WEBHOOK_URL — Discord webhook URL for Grafana alert notifications
6364
# KUBE_CONFIG — base64-encoded kubeconfig (base64 -w0 on Linux)
6465
- name: Upsert K8s secrets
6566
run: |
@@ -97,6 +98,10 @@ jobs:
9798
--from-literal=admin-password="${{ secrets.GRAFANA_ADMIN_PASSWORD }}" \
9899
--dry-run=client -o yaml | kubectl apply -f -
99100
101+
kubectl create secret generic grafana-discord-secret -n monitoring \
102+
--from-literal=webhook-url="${{ secrets.DISCORD_WEBHOOK_URL }}" \
103+
--dry-run=client -o yaml | kubectl apply -f -
104+
100105
- name: Deploy PostgreSQL (StatefulSet)
101106
run: kubectl apply -f infra/k8s/postgres/
102107

@@ -114,10 +119,18 @@ jobs:
114119
- name: Deploy monitoring
115120
run: kubectl apply -f infra/k8s/monitoring/
116121

122+
- name: Restart monitoring to reload ConfigMap-based config
123+
run: |
124+
kubectl rollout restart deployment/grafana -n monitoring
125+
kubectl rollout restart deployment/loki -n monitoring
126+
kubectl rollout restart deployment/alloy -n monitoring
127+
117128
- name: Wait for monitoring rollouts
118129
run: |
119130
kubectl rollout status deployment/prometheus -n monitoring --timeout=300s
120131
kubectl rollout status deployment/grafana -n monitoring --timeout=180s
132+
kubectl rollout status deployment/loki -n monitoring --timeout=180s
133+
kubectl rollout status deployment/alloy -n monitoring --timeout=120s
121134
122135
- name: Restart deployments to pull latest images
123136
if: github.event_name != 'push'

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Coverage reports: https://aet-devops26.github.io/team-devsecops/
1010

1111
API scheme (Swagger UI): https://devsecops.stud.k8s.aet.cit.tum.de/swagger-ui/index.html
1212

13-
Monitoring (Grafana): https://devsecops.stud.k8s.aet.cit.tum.de/grafana
13+
Monitoring (Grafana): https://grafana.devsecops.stud.k8s.aet.cit.tum.de
1414
## Local development
1515

1616
The full stack runs under Docker Compose with live-reload:

‎infra/k8s/monitoring/alloy.yaml‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: alloy
6+
namespace: monitoring
7+
---
8+
# Namespace-scoped Role in monitoring — grants Alloy permission to read pod
9+
# metadata and stream logs via the K8s API within this namespace.
10+
apiVersion: rbac.authorization.k8s.io/v1
11+
kind: Role
12+
metadata:
13+
name: alloy-log-reader
14+
namespace: monitoring
15+
rules:
16+
- apiGroups: [""]
17+
resources: ["pods", "pods/log"]
18+
verbs: ["get", "list", "watch"]
19+
---
20+
apiVersion: rbac.authorization.k8s.io/v1
21+
kind: RoleBinding
22+
metadata:
23+
name: alloy-log-reader
24+
namespace: monitoring
25+
roleRef:
26+
apiGroup: rbac.authorization.k8s.io
27+
kind: Role
28+
name: alloy-log-reader
29+
subjects:
30+
- kind: ServiceAccount
31+
name: alloy
32+
namespace: monitoring
33+
---
34+
# Same Role in app namespace — the ServiceAccount lives in monitoring but
35+
# RoleBindings can reference subjects from any namespace.
36+
apiVersion: rbac.authorization.k8s.io/v1
37+
kind: Role
38+
metadata:
39+
name: alloy-log-reader
40+
namespace: app
41+
rules:
42+
- apiGroups: [""]
43+
resources: ["pods", "pods/log"]
44+
verbs: ["get", "list", "watch"]
45+
---
46+
apiVersion: rbac.authorization.k8s.io/v1
47+
kind: RoleBinding
48+
metadata:
49+
name: alloy-log-reader
50+
namespace: app
51+
roleRef:
52+
apiGroup: rbac.authorization.k8s.io
53+
kind: Role
54+
name: alloy-log-reader
55+
subjects:
56+
- kind: ServiceAccount
57+
name: alloy
58+
namespace: monitoring
59+
---
60+
apiVersion: v1
61+
kind: ConfigMap
62+
metadata:
63+
name: alloy-config
64+
namespace: monitoring
65+
data:
66+
config.alloy: |
67+
// Discover pods only in the namespaces we have permission to read.
68+
// Scoping discovery here avoids needing cluster-wide ClusterRole.
69+
discovery.kubernetes "pods" {
70+
role = "pod"
71+
namespaces {
72+
names = ["app", "monitoring"]
73+
}
74+
}
75+
76+
// Map pod metadata to Loki stream labels
77+
discovery.relabel "pods" {
78+
targets = discovery.kubernetes.pods.targets
79+
80+
rule {
81+
source_labels = ["__meta_kubernetes_namespace"]
82+
target_label = "namespace"
83+
}
84+
rule {
85+
source_labels = ["__meta_kubernetes_pod_name"]
86+
target_label = "pod"
87+
}
88+
rule {
89+
source_labels = ["__meta_kubernetes_pod_container_name"]
90+
target_label = "container"
91+
}
92+
rule {
93+
source_labels = ["__meta_kubernetes_pod_label_app"]
94+
target_label = "app"
95+
}
96+
// Grafana Logs Drilldown follows OTel convention and groups by service_name
97+
rule {
98+
source_labels = ["__meta_kubernetes_pod_label_app"]
99+
target_label = "service_name"
100+
}
101+
}
102+
103+
// Tail logs from each discovered pod via the K8s log streaming API
104+
loki.source.kubernetes "pods" {
105+
targets = discovery.relabel.pods.output
106+
forward_to = [loki.write.local.receiver]
107+
}
108+
109+
// Push collected logs to Loki
110+
loki.write "local" {
111+
endpoint {
112+
url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
113+
}
114+
}
115+
---
116+
# Single Deployment (not DaemonSet) — loki.source.kubernetes reads logs via the
117+
# K8s API, so one pod is sufficient regardless of how many nodes the cluster has.
118+
apiVersion: apps/v1
119+
kind: Deployment
120+
metadata:
121+
name: alloy
122+
namespace: monitoring
123+
spec:
124+
replicas: 1
125+
revisionHistoryLimit: 2
126+
selector:
127+
matchLabels:
128+
app: alloy
129+
template:
130+
metadata:
131+
labels:
132+
app: alloy
133+
spec:
134+
serviceAccountName: alloy
135+
containers:
136+
- name: alloy
137+
image: grafana/alloy:v1.7.1
138+
args:
139+
- run
140+
- /etc/alloy/config.alloy
141+
- --storage.path=/var/lib/alloy/data
142+
- --server.http.listen-addr=0.0.0.0:12345
143+
ports:
144+
- name: http
145+
containerPort: 12345
146+
readinessProbe:
147+
httpGet:
148+
path: /-/ready
149+
port: http
150+
initialDelaySeconds: 10
151+
periodSeconds: 10
152+
failureThreshold: 6
153+
volumeMounts:
154+
- name: config
155+
mountPath: /etc/alloy
156+
- name: data
157+
mountPath: /var/lib/alloy/data
158+
resources:
159+
requests:
160+
cpu: 30m
161+
memory: 64Mi
162+
limits:
163+
cpu: 100m
164+
memory: 128Mi
165+
volumes:
166+
- name: config
167+
configMap:
168+
name: alloy-config
169+
- name: data
170+
emptyDir: {}

0 commit comments

Comments
 (0)