bcgov · dbarkowsky · Jun 16, 2026 · Jun 15, 2026 · Jun 15, 2026 · Jun 15, 2026
@@ -52,13 +52,13 @@ data:
 
     receivers:
       - name: "drop"
-      {{- if eq .Values.alertmanager.notificationChannel "teams" }}
+      {{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "teams") }}
       - name: "teams-notifications"
         webhook_configs:
           - url: {{ .Values.alertmanager.teams.webhookUrl | quote }}
             send_resolved: true
       {{- end }}
-      {{- if eq .Values.alertmanager.notificationChannel "ches" }}
+      {{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }}
       - name: "ches-notifications"
         webhook_configs:
           - url: {{ .Values.alertmanager.ches.adapterUrl | quote }}

@@ -1,4 +1,4 @@
-{{- if eq .Values.alertmanager.notificationChannel "ches" }}
+{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:

@@ -1,4 +1,4 @@
-{{- if eq .Values.alertmanager.notificationChannel "ches" }}
+{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

@@ -1,4 +1,4 @@
-{{- if eq .Values.alertmanager.notificationChannel "ches" }}
+{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }}
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:

@@ -1,4 +1,4 @@
-{{- if eq .Values.alertmanager.notificationChannel "ches" }}
+{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

@@ -1,4 +1,4 @@
-{{- if eq .Values.alertmanager.notificationChannel "ches" }}
+{{- if and .Values.alertmanager.notificationsEnabled (eq .Values.alertmanager.notificationChannel "ches") }}
 apiVersion: v1
 kind: Service
 metadata:

@@ -6,6 +6,8 @@ metadata:
     {{- include "plg.grafana.labels" . | nindent 4 }}
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       {{- include "plg.grafana.selectorLabels" . | nindent 6 }}

@@ -35,6 +35,12 @@ data:
     limits_config:
       retention_period: {{ include "plg.loki.retentionPeriod" . }}
       allow_structured_metadata: true
+      ingestion_rate_mb: 4
+      ingestion_burst_size_mb: 8
+
+    ingester:
+      chunk_idle_period: 30m
+      chunk_target_size: 1536000
 
     compactor:
       working_directory: /loki/compactor

@@ -67,3 +67,10 @@ data:
             regex: (.*)
             target_label: __address__
             replacement: $1:{{ .Values.prometheus.scrapeTargets.temporalWorker.port }}
+
+      - job_name: "loki"
+        metrics_path: "/metrics"
+        scrape_interval: {{ .Values.prometheus.scrapeInterval }}
+        static_configs:
+          - targets:
+              - "{{ include "plg.loki.fullname" . }}:{{ .Values.loki.httpPort }}"
@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "plg.prometheus.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "plg.prometheus.labels" . | nindent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "plg.prometheus.fullname" . }}
+  labels:
+    {{- include "plg.prometheus.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "plg.prometheus.fullname" . }}
+  labels:
+    {{- include "plg.prometheus.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "plg.prometheus.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "plg.prometheus.fullname" . }}
+    namespace: {{ .Release.Namespace }}
@@ -19,6 +19,7 @@ spec:
         checksum/rules: {{ include (print $.Template.BasePath "/prometheus-rules-configmap.yaml") . | sha256sum }}
         checksum/alertmanager-config: {{ include (print $.Template.BasePath "/alertmanager-configmap.yaml") . | sha256sum }}
     spec:
+      serviceAccountName: {{ include "plg.prometheus.fullname" . }}
       containers:
         - name: prometheus
           image: "{{ .Values.prometheus.image.repository }}:{{ .Values.prometheus.image.tag }}"

@@ -9,7 +9,7 @@ loki:
       memory: "512Mi"
       cpu: "500m"
     limits:
-      memory: "1Gi"
+      memory: "2Gi"
       cpu: "500m"
 
 prometheus:

@@ -351,10 +351,10 @@ spec:
           readOnly: true
         resources:
           requests:
-            memory: "32Mi"
+            memory: "64Mi"
             cpu: "50m"
           limits:
-            memory: "64Mi"
+            memory: "128Mi"
             cpu: "100m"
       volumes:
       - name: storage

@@ -9,7 +9,7 @@ data:
     /var/log/app/backend.log {
         copytruncate
         rotate 5
-        size 50M
+        size 10M
         missingok
         notifempty
     }
@@ -9,7 +9,7 @@ data:
     /var/log/app/worker.log {
         copytruncate
         rotate 5
-        size 50M
+        size 10M
         missingok
         notifempty
     }
@@ -238,10 +238,10 @@ spec:
               readOnly: true
           resources:
             requests:
-              memory: "32Mi"
+              memory: "64Mi"
               cpu: "50m"
             limits:
-              memory: "64Mi"
+              memory: "128Mi"
               cpu: "100m"
       volumes:
         - name: storage

@@ -35,7 +35,7 @@ deployments/openshift/helm/plg/
 | `loki.storageClassName` | Storage class (empty = cluster default) | `""` |
 | `loki.resources.requests.memory` | Memory request | `256Mi` |
 | `loki.resources.requests.cpu` | CPU request | `500m` |
-| `loki.resources.limits.memory` | Memory limit | `256Mi` |
+| `loki.resources.limits.memory` | Memory limit | `256Mi` (OpenShift override: `2Gi`) |
 | `loki.resources.limits.cpu` | CPU limit | `500m` |
 | `loki.httpPort` | HTTP listen port | `3100` |
 
@@ -71,6 +71,34 @@ To change the retention period, override `loki.retentionDays`:
 helm upgrade plg ./deployments/openshift/helm/plg --set loki.retentionDays=14
 ```
 
+## Ingestion Rate Limits
+
+The OpenShift deployment configures ingestion limits to apply back-pressure before Loki exhausts its memory ceiling:
+
+| Setting | Value | Purpose |
+|---------|-------|---------|
+| `ingestion_rate_mb` | 4 | Sustained ingest rate per tenant (MB/s) |
+| `ingestion_burst_size_mb` | 8 | Burst allowance above the sustained rate |
+| `chunk_idle_period` | 30m | Flush idle chunks to disk after this interval |
+| `chunk_target_size` | 1536000 (~1.5MB) | Flush a chunk to disk once it reaches this size |
+
+Without these limits, Loki accumulates unbounded in-memory chunks when ingestion outpaces disk flushes, eventually reaching the memory ceiling and being OOMKilled. The ingestion limits cause Promtail to receive a `429 Too Many Requests` response and retry, rather than Loki silently growing until it is killed.
+
+## Prometheus Metrics
+
+Loki exposes a `/metrics` endpoint on its HTTP port. The PLG Prometheus instance scrapes it via a dedicated `loki` scrape job configured in `prometheus-configmap.yaml`. Use `{job="loki"}` as the label selector in queries, for example:
+
+```promql
+# Current RSS memory usage
+process_resident_memory_bytes{job="loki"}
+
+# In-memory ingester chunks
+loki_ingester_memory_chunks
+
+# Ingest rate (bytes/sec)
+rate(loki_distributor_bytes_received_total[5m])
+```
+
 ## Deployment
 
 ### OpenShift

@@ -74,6 +74,22 @@ The PLG deployment is completely independent of the Kustomize-based application
 - No Kustomize base or overlay files are modified for PLG
 - If PLG deployment fails, the application deployment is unaffected
 
+## Prometheus RBAC
+
+Prometheus uses Kubernetes pod service discovery (`kubernetes_sd_configs`) to scrape `backend-services` and `temporal-worker` pods by IP across replicas. This requires permission to list and watch pods in the namespace.
+
+The chart creates a dedicated `ServiceAccount`, `Role` (pods: `get`/`list`/`watch`), and `RoleBinding` for the Prometheus StatefulSet. These are scoped to the release namespace only. The `default` service account is not used — it has no pod-list permissions on OpenShift.
+
+## ches-adapter and Alertmanager
+
+The ches-adapter Deployment, Service, PodDisruptionBudget, and sidecar ConfigMaps are only rendered when **both** `alertmanager.notificationsEnabled: true` and `alertmanager.notificationChannel: "ches"`. When notifications are disabled (the default), none of these resources are created and no ches-adapter image is required.
+
+Similarly, the `ches-notifications` and `teams-notifications` receivers in the Alertmanager config are only emitted when `notificationsEnabled: true`. This prevents Alertmanager from failing to start due to empty webhook URL validation when notifications are off.
+
+## Grafana Storage
+
+Grafana stores its SQLite database and alert history on a `ReadWriteOnce` PersistentVolumeClaim. The Deployment uses `strategy: Recreate` rather than the default `RollingUpdate`, so the old pod is terminated before the new one starts. This prevents `Multi-Attach` errors caused by two pods competing for the same RWO volume during an upgrade.
+
 ## Accessing Grafana
 
 Grafana is not exposed via an OpenShift Route. Access it via port-forwarding:

@@ -22,14 +22,16 @@ Each application pod includes a Promtail sidecar container that:
 
 ## Resource Limits
 
-All Promtail sidecars use minimal resource allocations:
+Promtail sidecars on PVC-backed services (`backend-services`, `temporal-worker`) use the following allocations:
 
 | Resource | Request | Limit |
 |----------|---------|-------|
-| Memory | 32Mi | 64Mi |
+| Memory | 64Mi | 128Mi |
 | CPU | 50m | 100m |
 
-These defaults are sized for low-traffic environments. To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`.
+The memory limit is set to 128Mi to prevent OOMKills caused by Loki backpressure. When Loki is under memory pressure it slows ingest responses, causing Promtail to buffer pending log batches in-heap. A 64Mi limit was insufficient in practice and caused CrashLoopBackOff on both `backend-services` and `temporal-worker` pods.
+
+To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`.
 
 ## Configuration