From 6acf72c0b1057be948e4c3337c651d4b8adfdb8c Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 10:49:37 -0700 Subject: [PATCH 1/9] increase loki and promtail memory --- deployments/openshift/helm/plg/values-openshift.yaml | 2 +- .../openshift/kustomize/base/backend-services/deployment.yml | 4 ++-- .../kustomize/base/temporal/temporal-worker-deployment.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/openshift/helm/plg/values-openshift.yaml b/deployments/openshift/helm/plg/values-openshift.yaml index 8f3903102..faf7f9c4c 100644 --- a/deployments/openshift/helm/plg/values-openshift.yaml +++ b/deployments/openshift/helm/plg/values-openshift.yaml @@ -9,7 +9,7 @@ loki: memory: "512Mi" cpu: "500m" limits: - memory: "1Gi" + memory: "2Gi" cpu: "500m" prometheus: diff --git a/deployments/openshift/kustomize/base/backend-services/deployment.yml b/deployments/openshift/kustomize/base/backend-services/deployment.yml index 3247d03ea..37681c9a8 100644 --- a/deployments/openshift/kustomize/base/backend-services/deployment.yml +++ b/deployments/openshift/kustomize/base/backend-services/deployment.yml @@ -351,10 +351,10 @@ spec: readOnly: true resources: requests: - memory: "32Mi" + memory: "64Mi" cpu: "50m" limits: - memory: "64Mi" + memory: "128Mi" cpu: "100m" volumes: - name: storage diff --git a/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml b/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml index 3b8813ea1..c0005ac8f 100644 --- a/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml +++ b/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml @@ -238,10 +238,10 @@ spec: readOnly: true resources: requests: - memory: "32Mi" + memory: "64Mi" cpu: "50m" limits: - memory: "64Mi" + memory: "128Mi" cpu: "100m" volumes: - name: storage From 825bcebe8348929fcbc2d7e1bddfadca016026d5 Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 10:49:54 -0700 Subject: [PATCH 2/9] smaller rotation values for more frequent sending --- .../kustomize/base/backend-services/logrotate-configmap.yml | 2 +- .../openshift/kustomize/base/temporal/logrotate-configmap.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml b/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml index c5e5e0642..495e072e2 100644 --- a/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml +++ b/deployments/openshift/kustomize/base/backend-services/logrotate-configmap.yml @@ -9,7 +9,7 @@ data: /var/log/app/backend.log { copytruncate rotate 5 - size 50M + size 10M missingok notifempty } diff --git a/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml b/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml index 0dcff9192..6c075cf18 100644 --- a/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml +++ b/deployments/openshift/kustomize/base/temporal/logrotate-configmap.yml @@ -9,7 +9,7 @@ data: /var/log/app/worker.log { copytruncate rotate 5 - size 50M + size 10M missingok notifempty } From 956d10d548667052496ec6302a830a5e406630e1 Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 11:20:38 -0700 Subject: [PATCH 3/9] prometheus metrics for loki --- .../helm/plg/templates/prometheus-configmap.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml index 443598d00..ce85a9454 100644 --- a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml @@ -67,3 +67,12 @@ data: regex: (.*) target_label: __address__ replacement: $1:{{ .Values.prometheus.scrapeTargets.temporalWorker.port }} + + - job_name: "loki" + metrics_path: "/metrics" + scrape_interval: {{ .Values.prometheus.scrapeInterval }} + static_configs: + - targets: + - "{{ include "plg.loki.fullname" . }}:{{ .Values.loki.httpPort }}" + labels: + container: loki From 28590d7d30c020684b56cebeedff8cd1df866caf Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 11:21:03 -0700 Subject: [PATCH 4/9] prometheus service account --- .../helm/plg/templates/prometheus-rbac.yaml | 32 +++++++++++++++++++ .../plg/templates/prometheus-statefulset.yaml | 1 + 2 files changed, 33 insertions(+) create mode 100644 deployments/openshift/helm/plg/templates/prometheus-rbac.yaml diff --git a/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml b/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml new file mode 100644 index 000000000..c1e46db8a --- /dev/null +++ b/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "plg.prometheus.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "plg.prometheus.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml b/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml index ed6402d5a..6e2a0c5b4 100644 --- a/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml +++ b/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml @@ -19,6 +19,7 @@ spec: checksum/rules: {{ include (print $.Template.BasePath "/prometheus-rules-configmap.yaml") . | sha256sum }} checksum/alertmanager-config: {{ include (print $.Template.BasePath "/alertmanager-configmap.yaml") . | sha256sum }} spec: + serviceAccountName: {{ include "plg.prometheus.fullname" . }} containers: - name: prometheus image: "{{ .Values.prometheus.image.repository }}:{{ .Values.prometheus.image.tag }}" From 0364ca2ac66782332ee40e9098bb44cfb91f70ec Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 11:59:54 -0700 Subject: [PATCH 5/9] self review round one --- .../openshift/helm/plg/templates/loki-configmap.yaml | 6 ++++++ .../openshift/helm/plg/templates/prometheus-configmap.yaml | 2 -- .../openshift/helm/plg/templates/prometheus-rbac.yaml | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/deployments/openshift/helm/plg/templates/loki-configmap.yaml b/deployments/openshift/helm/plg/templates/loki-configmap.yaml index cc4fb24f4..f8586cb1b 100644 --- a/deployments/openshift/helm/plg/templates/loki-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/loki-configmap.yaml @@ -35,6 +35,12 @@ data: limits_config: retention_period: {{ include "plg.loki.retentionPeriod" . }} allow_structured_metadata: true + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 8 + + ingester: + chunk_idle_period: 30m + chunk_target_size: 1536000 compactor: working_directory: /loki/compactor diff --git a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml index ce85a9454..7438f8647 100644 --- a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml @@ -74,5 +74,3 @@ data: static_configs: - targets: - "{{ include "plg.loki.fullname" . }}:{{ .Values.loki.httpPort }}" - labels: - container: loki diff --git a/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml b/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml index c1e46db8a..b4e2f8995 100644 --- a/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml +++ b/deployments/openshift/helm/plg/templates/prometheus-rbac.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "plg.prometheus.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "plg.prometheus.labels" . | nindent 4 }} --- From 1afcd039d9c21bf5e39e104a065df62b79352a3c Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 12:35:17 -0700 Subject: [PATCH 6/9] grafana recreate strategy --- .../openshift/helm/plg/templates/grafana-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployments/openshift/helm/plg/templates/grafana-deployment.yaml b/deployments/openshift/helm/plg/templates/grafana-deployment.yaml index 7d337265e..89dd12b76 100644 --- a/deployments/openshift/helm/plg/templates/grafana-deployment.yaml +++ b/deployments/openshift/helm/plg/templates/grafana-deployment.yaml @@ -6,6 +6,8 @@ metadata: {{- include "plg.grafana.labels" . | nindent 4 }} spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: {{- include "plg.grafana.selectorLabels" . | nindent 6 }} From 1df8edba124ea5ce74dd9900e6afe8976f30d298 Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 12:35:46 -0700 Subject: [PATCH 7/9] alermanager only create receiver if notifications enabled --- .../openshift/helm/plg/templates/alertmanager-configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml b/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml index bef22a93e..205dd083e 100644 --- a/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/alertmanager-configmap.yaml @@ -52,13 +52,13 @@ data: receivers: - name: "drop" - {{- if eq .Values.alertmanager.notificationChannel "teams" }} + {{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "teams") }} - name: "teams-notifications" webhook_configs: - url: {{ .Values.alertmanager.teams.webhookUrl | quote }} send_resolved: true {{- end }} - {{- if eq .Values.alertmanager.notificationChannel "ches" }} + {{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} - name: "ches-notifications" webhook_configs: - url: {{ .Values.alertmanager.ches.adapterUrl | quote }} From 847d0fc0c35053a5854b41095d8f107a6ec2b89c Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 12:50:23 -0700 Subject: [PATCH 8/9] ches only if notificaitons enabled --- .../openshift/helm/plg/templates/ches-adapter-deployment.yaml | 2 +- .../helm/plg/templates/ches-adapter-logrotate-configmap.yaml | 2 +- deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml | 2 +- .../helm/plg/templates/ches-adapter-promtail-configmap.yaml | 2 +- .../openshift/helm/plg/templates/ches-adapter-service.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml index d3dc1f7f4..4b3bb0bd4 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-deployment.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml index 15f0671c7..3df8ba9dd 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-logrotate-configmap.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml index 5f6b76c08..458982a46 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-pdb.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml index a5638a2a6..3e4522c9f 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-promtail-configmap.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml b/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml index f67d66ce7..3527a0d10 100644 --- a/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml +++ b/deployments/openshift/helm/plg/templates/ches-adapter-service.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.alertmanager.notificationChannel "ches" }} +{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }} apiVersion: v1 kind: Service metadata: From 8bfbe0666bb720d99b21cc8eb504c96811247262 Mon Sep 17 00:00:00 2001 From: dbarkowsky Date: Mon, 15 Jun 2026 13:00:55 -0700 Subject: [PATCH 9/9] documentatino update --- docs-md/LOKI_HELM_CHART.md | 30 ++++++++++++++++++++++++++- docs-md/PLG_DEPLOYMENT_INTEGRATION.md | 16 ++++++++++++++ docs-md/PROMTAIL_SIDECARS.md | 8 ++++--- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs-md/LOKI_HELM_CHART.md b/docs-md/LOKI_HELM_CHART.md index c6d214f7f..fe48ad2c8 100644 --- a/docs-md/LOKI_HELM_CHART.md +++ b/docs-md/LOKI_HELM_CHART.md @@ -35,7 +35,7 @@ deployments/openshift/helm/plg/ | `loki.storageClassName` | Storage class (empty = cluster default) | `""` | | `loki.resources.requests.memory` | Memory request | `256Mi` | | `loki.resources.requests.cpu` | CPU request | `500m` | -| `loki.resources.limits.memory` | Memory limit | `256Mi` | +| `loki.resources.limits.memory` | Memory limit | `256Mi` (OpenShift override: `2Gi`) | | `loki.resources.limits.cpu` | CPU limit | `500m` | | `loki.httpPort` | HTTP listen port | `3100` | @@ -71,6 +71,34 @@ To change the retention period, override `loki.retentionDays`: helm upgrade plg ./deployments/openshift/helm/plg --set loki.retentionDays=14 ``` +## Ingestion Rate Limits + +The OpenShift deployment configures ingestion limits to apply back-pressure before Loki exhausts its memory ceiling: + +| Setting | Value | Purpose | +|---------|-------|---------| +| `ingestion_rate_mb` | 4 | Sustained ingest rate per tenant (MB/s) | +| `ingestion_burst_size_mb` | 8 | Burst allowance above the sustained rate | +| `chunk_idle_period` | 30m | Flush idle chunks to disk after this interval | +| `chunk_target_size` | 1536000 (~1.5MB) | Flush a chunk to disk once it reaches this size | + +Without these limits, Loki accumulates unbounded in-memory chunks when ingestion outpaces disk flushes, eventually reaching the memory ceiling and being OOMKilled. The ingestion limits cause Promtail to receive a `429 Too Many Requests` response and retry, rather than Loki silently growing until it is killed. + +## Prometheus Metrics + +Loki exposes a `/metrics` endpoint on its HTTP port. The PLG Prometheus instance scrapes it via a dedicated `loki` scrape job configured in `prometheus-configmap.yaml`. Use `{job="loki"}` as the label selector in queries, for example: + +```promql +# Current RSS memory usage +process_resident_memory_bytes{job="loki"} + +# In-memory ingester chunks +loki_ingester_memory_chunks + +# Ingest rate (bytes/sec) +rate(loki_distributor_bytes_received_total[5m]) +``` + ## Deployment ### OpenShift diff --git a/docs-md/PLG_DEPLOYMENT_INTEGRATION.md b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md index ac879f3fe..4e23886d4 100644 --- a/docs-md/PLG_DEPLOYMENT_INTEGRATION.md +++ b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md @@ -74,6 +74,22 @@ The PLG deployment is completely independent of the Kustomize-based application - No Kustomize base or overlay files are modified for PLG - If PLG deployment fails, the application deployment is unaffected +## Prometheus RBAC + +Prometheus uses Kubernetes pod service discovery (`kubernetes_sd_configs`) to scrape `backend-services` and `temporal-worker` pods by IP across replicas. This requires permission to list and watch pods in the namespace. + +The chart creates a dedicated `ServiceAccount`, `Role` (pods: `get`/`list`/`watch`), and `RoleBinding` for the Prometheus StatefulSet. These are scoped to the release namespace only. The `default` service account is not used — it has no pod-list permissions on OpenShift. + +## ches-adapter and Alertmanager + +The ches-adapter Deployment, Service, PodDisruptionBudget, and sidecar ConfigMaps are only rendered when **both** `alertmanager.notificationsEnabled: true` and `alertmanager.notificationChannel: "ches"`. When notifications are disabled (the default), none of these resources are created and no ches-adapter image is required. + +Similarly, the `ches-notifications` and `teams-notifications` receivers in the Alertmanager config are only emitted when `notificationsEnabled: true`. This prevents Alertmanager from failing to start due to empty webhook URL validation when notifications are off. + +## Grafana Storage + +Grafana stores its SQLite database and alert history on a `ReadWriteOnce` PersistentVolumeClaim. The Deployment uses `strategy: Recreate` rather than the default `RollingUpdate`, so the old pod is terminated before the new one starts. This prevents `Multi-Attach` errors caused by two pods competing for the same RWO volume during an upgrade. + ## Accessing Grafana Grafana is not exposed via an OpenShift Route. Access it via port-forwarding: diff --git a/docs-md/PROMTAIL_SIDECARS.md b/docs-md/PROMTAIL_SIDECARS.md index fa04a311d..b67921503 100644 --- a/docs-md/PROMTAIL_SIDECARS.md +++ b/docs-md/PROMTAIL_SIDECARS.md @@ -22,14 +22,16 @@ Each application pod includes a Promtail sidecar container that: ## Resource Limits -All Promtail sidecars use minimal resource allocations: +Promtail sidecars on PVC-backed services (`backend-services`, `temporal-worker`) use the following allocations: | Resource | Request | Limit | |----------|---------|-------| -| Memory | 32Mi | 64Mi | +| Memory | 64Mi | 128Mi | | CPU | 50m | 100m | -These defaults are sized for low-traffic environments. To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`. +The memory limit is set to 128Mi to prevent OOMKills caused by Loki backpressure. When Loki is under memory pressure it slows ingest responses, causing Promtail to buffer pending log batches in-heap. A 64Mi limit was insufficient in practice and caused CrashLoopBackOff on both `backend-services` and `temporal-worker` pods. + +To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`. ## Configuration