diff --git a/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml b/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml index bef22a93e..205dd083e 100644 --- a/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml @@ -52,13 +52,13 @@ data: receivers: - name: "drop" - {{- if eq .Values.alertmanager.notificationChannel "teams" }} + {{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "teams") }} - name: "teams-notifications" webhook_configs: - url: {{ .Values.alertmanager.teams.webhookUrl | quote }} send_resolved: true {{- end }} - {{- if eq .Values.alertmanager.notificationChannel "ches" }} + {{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} - name: "ches-notifications" webhook_configs: - url: {{ .Values.alertmanager.ches.adapterUrl | quote }} diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml index d3dc1f7f4..4b3bb0bd4 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml index 15f0671c7..3df8ba9dd 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml index 5f6b76c08..458982a46 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml index a5638a2a6..3e4522c9f 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml index f67d66ce7..3527a0d10 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: v1 kind: Service metadata: diff --git a/deployments/openshift/helm/plg/templates/grafana-deployment.yaml b/deployments/openshift/helm/plg/templates/grafana-deployment.yaml index 7d337265e..89dd12b76 100644 --- a/deployments/openshift/helm/plg/templates/grafana-deployment.yaml +++ b/deployments/openshift/helm/plg/templates/grafana-deployment.yaml @@ -6,6 +6,8 @@ metadata: {{- include "plg.grafana.labels" . | nindent 4 }} spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: {{- include "plg.grafana.selectorLabels" . | nindent 6 }} diff --git a/deployments/openshift/helm/plg/templates/loki-configmap.yaml b/deployments/openshift/helm/plg/templates/loki-configmap.yaml index cc4fb24f4..f8586cb1b 100644 --- a/deployments/openshift/helm/plg/templates/loki-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/loki-configmap.yaml @@ -35,6 +35,12 @@ data: limits_config: retention_period: {{ include "plg.loki.retentionPeriod" . }} allow_structured_metadata: true + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 8 + + ingester: + chunk_idle_period: 30m + chunk_target_size: 1536000 compactor: working_directory: /loki/compactor diff --git a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml index 443598d00..7438f8647 100644 --- a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml @@ -67,3 +67,10 @@ data: regex: (.*) target_label: __address__ replacement: $1:{{ .Values.prometheus.scrapeTargets.temporalWorker.port }} + + - job_name: "loki" + metrics_path: "/metrics" + scrape_interval: {{ .Values.prometheus.scrapeInterval }} + static_configs: + - targets: + - "{{ include "plg.loki.fullname" . }}:{{ .Values.loki.httpPort }}" diff --git a/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml b/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml new file mode 100644 index 000000000..b4e2f8995 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "plg.prometheus.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "plg.prometheus.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "plg.prometheus.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml b/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml index ed6402d5a..6e2a0c5b4 100644 --- a/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml +++ b/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml @@ -19,6 +19,7 @@ spec: checksum/rules: {{ include (print $.Template.BasePath "/prometheus-rules-configmap.yaml") . | sha256sum }} checksum/alertmanager-config: {{ include (print $.Template.BasePath "/alertmanager-configmap.yaml") . | sha256sum }} spec: + serviceAccountName: {{ include "plg.prometheus.fullname" . }} containers: - name: prometheus image: "{{ .Values.prometheus.image.repository }}:{{ .Values.prometheus.image.tag }}" diff --git a/deployments/openshift/helm/plg/values-openshift.yaml b/deployments/openshift/helm/plg/values-openshift.yaml index 8f3903102..faf7f9c4c 100644 --- a/deployments/openshift/helm/plg/values-openshift.yaml +++ b/deployments/openshift/helm/plg/values-openshift.yaml @@ -9,7 +9,7 @@ loki: memory: "512Mi" cpu: "500m" limits: - memory: "1Gi" + memory: "2Gi" cpu: "500m" prometheus: diff --git a/deployments/openshift/kustomize/base/backend-services/deployment.yml b/deployments/openshift/kustomize/base/backend-services/deployment.yml index 3247d03ea..37681c9a8 100644 --- a/deployments/openshift/kustomize/base/backend-services/deployment.yml +++ b/deployments/openshift/kustomize/base/backend-services/deployment.yml @@ -351,10 +351,10 @@ spec: readOnly: true resources: requests: - memory: "32Mi" + memory: "64Mi" cpu: "50m" limits: - memory: "64Mi" + memory: "128Mi" cpu: "100m" volumes: - name: storage diff --git a/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml b/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml index c5e5e0642..495e072e2 100644 --- a/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml +++ b/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml @@ -9,7 +9,7 @@ data: /var/log/app/backend.log { copytruncate rotate 5 - size 50M + size 10M missingok notifempty } diff --git a/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml b/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml index 0dcff9192..6c075cf18 100644 --- a/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml +++ b/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml @@ -9,7 +9,7 @@ data: /var/log/app/worker.log { copytruncate rotate 5 - size 50M + size 10M missingok notifempty } diff --git a/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml b/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml index 3b8813ea1..c0005ac8f 100644 --- a/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml +++ b/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml @@ -238,10 +238,10 @@ spec: readOnly: true resources: requests: - memory: "32Mi" + memory: "64Mi" cpu: "50m" limits: - memory: "64Mi" + memory: "128Mi" cpu: "100m" volumes: - name: storage diff --git a/docs-md/LOKI_HELM_CHART.md b/docs-md/LOKI_HELM_CHART.md index c6d214f7f..fe48ad2c8 100644 --- a/docs-md/LOKI_HELM_CHART.md +++ b/docs-md/LOKI_HELM_CHART.md @@ -35,7 +35,7 @@ deployments/openshift/helm/plg/ | `loki.storageClassName` | Storage class (empty = cluster default) | `""` | | `loki.resources.requests.memory` | Memory request | `256Mi` | | `loki.resources.requests.cpu` | CPU request | `500m` | -| `loki.resources.limits.memory` | Memory limit | `256Mi` | +| `loki.resources.limits.memory` | Memory limit | `256Mi` (OpenShift override: `2Gi`) | | `loki.resources.limits.cpu` | CPU limit | `500m` | | `loki.httpPort` | HTTP listen port | `3100` | @@ -71,6 +71,34 @@ To change the retention period, override `loki.retentionDays`: helm upgrade plg ./deployments/openshift/helm/plg --set loki.retentionDays=14 ``` +## Ingestion Rate Limits + +The OpenShift deployment configures ingestion limits to apply back-pressure before Loki exhausts its memory ceiling: + +| Setting | Value | Purpose | +|---------|-------|---------| +| `ingestion_rate_mb` | 4 | Sustained ingest rate per tenant (MB/s) | +| `ingestion_burst_size_mb` | 8 | Burst allowance above the sustained rate | +| `chunk_idle_period` | 30m | Flush idle chunks to disk after this interval | +| `chunk_target_size` | 1536000 (~1.5MB) | Flush a chunk to disk once it reaches this size | + +Without these limits, Loki accumulates unbounded in-memory chunks when ingestion outpaces disk flushes, eventually reaching the memory ceiling and being OOMKilled. The ingestion limits cause Promtail to receive a `429 Too Many Requests` response and retry, rather than Loki silently growing until it is killed. + +## Prometheus Metrics + +Loki exposes a `/metrics` endpoint on its HTTP port. The PLG Prometheus instance scrapes it via a dedicated `loki` scrape job configured in `prometheus-configmap.yaml`. Use `{job="loki"}` as the label selector in queries, for example: + +```promql +# Current RSS memory usage +process_resident_memory_bytes{job="loki"} + +# In-memory ingester chunks +loki_ingester_memory_chunks + +# Ingest rate (bytes/sec) +rate(loki_distributor_bytes_received_total[5m]) +``` + ## Deployment ### OpenShift diff --git a/docs-md/PLG_DEPLOYMENT_INTEGRATION.md b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md index ac879f3fe..4e23886d4 100644 --- a/docs-md/PLG_DEPLOYMENT_INTEGRATION.md +++ b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md @@ -74,6 +74,22 @@ The PLG deployment is completely independent of the Kustomize-based application - No Kustomize base or overlay files are modified for PLG - If PLG deployment fails, the application deployment is unaffected +## Prometheus RBAC + +Prometheus uses Kubernetes pod service discovery (`kubernetes_sd_configs`) to scrape `backend-services` and `temporal-worker` pods by IP across replicas. This requires permission to list and watch pods in the namespace. + +The chart creates a dedicated `ServiceAccount`, `Role` (pods: `get`/`list`/`watch`), and `RoleBinding` for the Prometheus StatefulSet. These are scoped to the release namespace only. The `default` service account is not used — it has no pod-list permissions on OpenShift. + +## ches-adapter and Alertmanager + +The ches-adapter Deployment, Service, PodDisruptionBudget, and sidecar ConfigMaps are only rendered when **both** `alertmanager.notificationsEnabled: true` and `alertmanager.notificationChannel: "ches"`. When notifications are disabled (the default), none of these resources are created and no ches-adapter image is required. + +Similarly, the `ches-notifications` and `teams-notifications` receivers in the Alertmanager config are only emitted when `notificationsEnabled: true`. This prevents Alertmanager from failing to start due to empty webhook URL validation when notifications are off. + +## Grafana Storage + +Grafana stores its SQLite database and alert history on a `ReadWriteOnce` PersistentVolumeClaim. The Deployment uses `strategy: Recreate` rather than the default `RollingUpdate`, so the old pod is terminated before the new one starts. This prevents `Multi-Attach` errors caused by two pods competing for the same RWO volume during an upgrade. + ## Accessing Grafana Grafana is not exposed via an OpenShift Route. Access it via port-forwarding: diff --git a/docs-md/PROMTAIL_SIDECARS.md b/docs-md/PROMTAIL_SIDECARS.md index fa04a311d..b67921503 100644 --- a/docs-md/PROMTAIL_SIDECARS.md +++ b/docs-md/PROMTAIL_SIDECARS.md @@ -22,14 +22,16 @@ Each application pod includes a Promtail sidecar container that: ## Resource Limits -All Promtail sidecars use minimal resource allocations: +Promtail sidecars on PVC-backed services (`backend-services`, `temporal-worker`) use the following allocations: | Resource | Request | Limit | |----------|---------|-------| -| Memory | 32Mi | 64Mi | +| Memory | 64Mi | 128Mi | | CPU | 50m | 100m | -These defaults are sized for low-traffic environments. To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`. +The memory limit is set to 128Mi to prevent OOMKills caused by Loki backpressure. When Loki is under memory pressure it slows ingest responses, causing Promtail to buffer pending log batches in-heap. A 64Mi limit was insufficient in practice and caused CrashLoopBackOff on both `backend-services` and `temporal-worker` pods. + +To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`. ## Configuration