feat: implement US-010 - Integrate PLG Deployment with GitHub Actions and Scripts

strukalex · claude · strukalex · commit 372bdcab0277 · 2026-03-17T16:42:00.000-07:00
Add PLG Helm deployment to GitHub Actions workflow and oc-deploy.sh script.
PLG environment variables configurable per overlay. PLG is a separate Helm
release that does not affect existing Kustomize deployment.

Co-Authored-By: Claude Opus 4.6 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/build-apps.yml b/.github/workflows/build-apps.yml
@@ -162,6 +162,58 @@ jobs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
+  deploy-plg:
+    name: Deploy PLG Stack
+    needs: [build-apps, get-environment]
+    if: ${{ always() && (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') }}
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ needs.get-environment.outputs.environment }}
+    env:
+      OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}
+      OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }}
+      OPENSHIFT_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE }}
+      GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: v3.17.0
+
+      - name: Install oc CLI
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4"
+
+      - name: Login to OpenShift
+        run: |
+          oc login "${{ env.OPENSHIFT_SERVER }}" \
+            --token="${{ env.OPENSHIFT_TOKEN }}" \
+            --insecure-skip-tls-verify=true
+
+      - name: Deploy PLG Helm Chart
+        run: |
+          CHART_DIR="deployments/openshift/helm/plg"
+
+          # Use environment-specific defaults; secrets override via GitHub environment
+          GRAFANA_PWD="${{ env.GRAFANA_ADMIN_PASSWORD }}"
+          if [ -z "${GRAFANA_PWD}" ]; then
+            GRAFANA_PWD="admin"
+          fi
+
+          helm upgrade --install plg "${CHART_DIR}" \
+            --namespace "${{ env.OPENSHIFT_NAMESPACE }}" \
+            -f "${CHART_DIR}/values-openshift.yaml" \
+            --set "grafana.adminPassword=${GRAFANA_PWD}" \
+            --wait --timeout 120s
+
+          echo "PLG stack deployed successfully."
+
   # trigger_migration:
   #   needs: [metadata, build-apps]
   #   if: ${{ always() && needs.build-apps.result == 'success' && needs.metadata.outputs.changed-apps != '[]' && needs.metadata.outputs.changed-apps != '' }}
diff --git a/deployments/openshift/config/dev.env.example b/deployments/openshift/config/dev.env.example
@@ -115,3 +115,17 @@ THROTTLE_AUTH_TTL_MS=60000
 THROTTLE_AUTH_LIMIT=10
 THROTTLE_AUTH_REFRESH_TTL_MS=60000
 THROTTLE_AUTH_REFRESH_LIMIT=5
+
+# -----------------------------------------------------------------------------
+# PLG Monitoring Stack (Prometheus, Loki, Grafana)
+# -----------------------------------------------------------------------------
+# Grafana admin password. Change this from the default for any shared environment.
+GRAFANA_ADMIN_PASSWORD=admin
+# Loki log retention period in days.
+LOKI_RETENTION_DAYS=30
+# Loki PVC storage size.
+LOKI_PVC_SIZE=10Gi
+# Prometheus PVC storage size.
+PROMETHEUS_PVC_SIZE=10Gi
+# Prometheus scrape interval for all targets.
+METRICS_SCRAPE_INTERVAL=15s
diff --git a/deployments/openshift/config/prod.env.example b/deployments/openshift/config/prod.env.example
@@ -115,3 +115,17 @@ THROTTLE_AUTH_TTL_MS=60000
 THROTTLE_AUTH_LIMIT=5
 THROTTLE_AUTH_REFRESH_TTL_MS=60000
 THROTTLE_AUTH_REFRESH_LIMIT=3
+
+# -----------------------------------------------------------------------------
+# PLG Monitoring Stack (Prometheus, Loki, Grafana)
+# -----------------------------------------------------------------------------
+# Grafana admin password. Use a strong password in production.
+GRAFANA_ADMIN_PASSWORD=change-me-in-production
+# Loki log retention period in days.
+LOKI_RETENTION_DAYS=30
+# Loki PVC storage size.
+LOKI_PVC_SIZE=10Gi
+# Prometheus PVC storage size.
+PROMETHEUS_PVC_SIZE=10Gi
+# Prometheus scrape interval for all targets.
+METRICS_SCRAPE_INTERVAL=15s
diff --git a/docs-md/PLG_DEPLOYMENT_INTEGRATION.md b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md
@@ -0,0 +1,94 @@
+# PLG Deployment Integration
+
+## Overview
+
+The PLG (Prometheus, Loki, Grafana) monitoring stack is deployed as a separate Helm release alongside the application. It does not modify or interfere with the existing Kustomize-based application deployment. PLG deployment is integrated into both the GitHub Actions CI/CD pipeline and the local deployment scripts.
+
+## Deployment Methods
+
+### GitHub Actions (CI/CD)
+
+The `build-apps.yml` workflow includes a `deploy-plg` job that runs after application images are built. This job:
+
+1. Checks out the repository to access the Helm chart at `deployments/openshift/helm/plg/`
+2. Installs the Helm and `oc` CLIs
+3. Authenticates to OpenShift using environment secrets
+4. Runs `helm upgrade --install` with the OpenShift values file
+
+The job runs regardless of whether application images were built (it depends on `build-apps` succeeding or being skipped), ensuring the PLG stack stays up to date even when no application code changed.
+
+#### Required GitHub Environment Secrets
+
+| Secret | Description |
+|--------|-------------|
+| `OPENSHIFT_SERVER` | OpenShift API server URL |
+| `OPENSHIFT_TOKEN` | Service account token for the target namespace |
+| `OPENSHIFT_NAMESPACE` | Target namespace (e.g., `fd34fb-dev`) |
+| `GRAFANA_ADMIN_PASSWORD` | Grafana admin password (falls back to `admin` if unset) |
+
+### Local Deployment (`oc-deploy.sh`)
+
+The `scripts/oc-deploy.sh` script deploys the PLG stack as **Step 7**, between applying the Kustomize overlay (Step 6) and creating instance secrets (Step 8). This step:
+
+1. Reads PLG-specific configuration from the environment profile (`dev.env` or `prod.env`)
+2. Derives instance-specific Prometheus scrape targets from the Kustomize instance name
+3. Runs `helm upgrade --install` with environment-specific values passed via `--set` flags
+
+If the `helm` CLI is not installed, the PLG step is skipped with a warning. The application deployment continues normally.
+
+#### Instance-Specific Helm Release
+
+Each application instance gets its own PLG Helm release named `<instance>-plg`. Prometheus scrape targets are configured to point at the instance-specific Kubernetes service names (e.g., `my-instance-backend-services`, `my-instance-temporal`).
+
+### Teardown (`oc-teardown.sh`)
+
+The `scripts/oc-teardown.sh` script includes a step (3b) that uninstalls the PLG Helm release for the instance being torn down. If Helm is not installed or no PLG release exists, the step is skipped gracefully.
+
+## Environment Configuration
+
+PLG-specific variables are configured in the same environment profile files used by the application (`deployments/openshift/config/<env>.env`). They follow the existing config merge pattern: profile defaults can be overridden by instance-specific files.
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin login password |
+| `LOKI_RETENTION_DAYS` | `30` | Log retention period in days |
+| `LOKI_PVC_SIZE` | `10Gi` | Persistent volume size for Loki data |
+| `PROMETHEUS_PVC_SIZE` | `10Gi` | Persistent volume size for Prometheus TSDB |
+| `METRICS_SCRAPE_INTERVAL` | `15s` | How often Prometheus scrapes targets |
+
+These variables are read by `oc-deploy.sh` via the `config-loader.sh` library and passed to Helm as `--set` overrides on top of the `values-openshift.yaml` base.
+
+## Separation from Kustomize
+
+The PLG deployment is completely independent of the Kustomize-based application deployment:
+
+- PLG resources are managed by Helm, not Kustomize
+- PLG uses its own labels (`app.kubernetes.io/managed-by: Helm`, `app.kubernetes.io/part-of: plg`)
+- No Kustomize base or overlay files are modified for PLG
+- If PLG deployment fails, the application deployment is unaffected
+
+## Accessing Grafana
+
+Grafana is not exposed via an OpenShift Route. Access it via port-forwarding:
+
+```bash
+# For instance-specific deployments (via oc-deploy.sh)
+oc port-forward svc/<instance>-plg-grafana 3001:3001 -n <namespace>
+
+# For CI-deployed PLG (single release per namespace)
+oc port-forward svc/plg-grafana 3001:3001 -n <namespace>
+```
+
+Then open `http://localhost:3001` and log in with `admin` / `<GRAFANA_ADMIN_PASSWORD>`.
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `deployments/openshift/helm/plg/` | PLG Helm chart (templates, values) |
+| `deployments/openshift/helm/plg/values-openshift.yaml` | OpenShift-specific value overrides |
+| `deployments/openshift/config/dev.env.example` | Dev environment config template (includes PLG variables) |
+| `deployments/openshift/config/prod.env.example` | Prod environment config template (includes PLG variables) |
+| `.github/workflows/build-apps.yml` | CI workflow with `deploy-plg` job |
+| `scripts/oc-deploy.sh` | Local deployment script (Step 7: PLG) |
+| `scripts/oc-teardown.sh` | Teardown script (Step 3b: PLG uninstall) |
diff --git a/docs-md/openshift-deployment/ENVIRONMENT_CONFIGURATION.md b/docs-md/openshift-deployment/ENVIRONMENT_CONFIGURATION.md
@@ -170,6 +170,16 @@ These values are derived automatically by the deploy script — do not set them
 | `THROTTLE_AUTH_REFRESH_TTL_MS` | Token refresh rate limit window |
 | `THROTTLE_AUTH_REFRESH_LIMIT` | Max refresh requests per IP (stricter in prod) |
 
+### PLG Monitoring Stack
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin login password |
+| `LOKI_RETENTION_DAYS` | `30` | Log retention period in days |
+| `LOKI_PVC_SIZE` | `10Gi` | Persistent volume size for Loki data |
+| `PROMETHEUS_PVC_SIZE` | `10Gi` | Persistent volume size for Prometheus TSDB |
+| `METRICS_SCRAPE_INTERVAL` | `15s` | How often Prometheus scrapes targets |
+
 ## How Secrets Reach the Pods
 
 The deploy script creates per-instance OpenShift Secrets from values in the env file. Each instance gets its own copy.
diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/README.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/README.md
@@ -57,7 +57,7 @@ After implementing the user story check it off at the bottom of this file.
 ### Phase 4 — Deployment Integration (local + OpenShift)
 - [x] **US-008** (Create docker-compose.monitoring.yml with Promtail, Loki, Prometheus, Grafana)
 - [x] **US-009** (Add Promtail sidecar containers to all OpenShift application pods)
-- [ ] **US-010** (Integrate PLG Helm deployment into GitHub Actions workflow and /scripts)
+- [x] **US-010** (Integrate PLG Helm deployment into GitHub Actions workflow and /scripts)
 
 ### Phase 5 — Grafana Dashboards
 - [ ] **US-011** (Application Overview dashboard — request rate, error rate, latency, active sessions)
diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-010-openshift-deployment-integration.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-010-openshift-deployment-integration.md
@@ -6,22 +6,22 @@
 
 ## Acceptance Criteria
 
-- [ ] **Scenario 1**: GitHub Actions workflow deploys PLG Helm chart
+- [x] **Scenario 1**: GitHub Actions workflow deploys PLG Helm chart
     - **Given** the existing GitHub Actions workflow builds and deploys the application
     - **When** the workflow runs
     - **Then** it also deploys the PLG Helm chart to the same namespace as the application
 
-- [ ] **Scenario 2**: Local deployment scripts deploy PLG
+- [x] **Scenario 2**: Local deployment scripts deploy PLG
     - **Given** the existing deployment scripts in `/scripts` handle application deployment
     - **When** a developer runs the deployment scripts locally
     - **Then** the PLG Helm chart is also deployed to the target namespace
 
-- [ ] **Scenario 3**: PLG environment variables configurable per overlay
+- [x] **Scenario 3**: PLG environment variables configurable per overlay
     - **Given** environment-specific configuration exists for dev, test, and prod
     - **When** deploying to a specific environment
     - **Then** PLG-specific variables (`GRAFANA_ADMIN_PASSWORD`, `LOKI_RETENTION_DAYS`, `LOKI_PVC_SIZE`, `PROMETHEUS_PVC_SIZE`, `METRICS_SCRAPE_INTERVAL`) are sourced from the environment's configuration
 
-- [ ] **Scenario 4**: PLG deployment does not affect existing Kustomize deployment
+- [x] **Scenario 4**: PLG deployment does not affect existing Kustomize deployment
     - **Given** the application is deployed via Kustomize
     - **When** the PLG Helm chart is deployed
     - **Then** the existing Kustomize resources are not modified or disrupted
diff --git a/scripts/oc-deploy.sh b/scripts/oc-deploy.sh
@@ -474,9 +474,56 @@ oc apply -k "${OVERLAY_DIR}" -n "${NAMESPACE}" || {
 log_info "Resources applied successfully."
 
 # ============================================================
-# Step 7: Create/update instance secrets
+# Step 7: Deploy PLG monitoring stack (Helm)
 # ============================================================
-log_step "Step 7: Creating instance secrets"
+log_step "Step 7: Deploying PLG monitoring stack"
+
+if ! command -v helm &>/dev/null; then
+  log_error "'helm' CLI is not installed. Install Helm to deploy the PLG monitoring stack."
+  log_error "Skipping PLG deployment — the application will still work without it."
+else
+  PLG_CHART_DIR="${PROJECT_ROOT}/deployments/openshift/helm/plg"
+  PLG_RELEASE_NAME="${INSTANCE_NAME}-plg"
+
+  # Read PLG-specific configuration with defaults
+  GRAFANA_ADMIN_PASSWORD=$(get_config "GRAFANA_ADMIN_PASSWORD" 2>/dev/null || echo "admin")
+  LOKI_RETENTION_DAYS=$(get_config "LOKI_RETENTION_DAYS" 2>/dev/null || echo "30")
+  LOKI_PVC_SIZE=$(get_config "LOKI_PVC_SIZE" 2>/dev/null || echo "10Gi")
+  PROMETHEUS_PVC_SIZE=$(get_config "PROMETHEUS_PVC_SIZE" 2>/dev/null || echo "10Gi")
+  METRICS_SCRAPE_INTERVAL=$(get_config "METRICS_SCRAPE_INTERVAL" 2>/dev/null || echo "15s")
+
+  # Scrape targets use instance-prefixed service names (Kustomize namePrefix adds <instance>-)
+  BACKEND_SERVICES_HOST=$(get_resource_name "${INSTANCE_NAME}" "backend-services")
+  TEMPORAL_HOST=$(get_resource_name "${INSTANCE_NAME}" "temporal")
+
+  log_info "PLG release name: ${PLG_RELEASE_NAME}"
+  log_info "Helm chart: ${PLG_CHART_DIR}"
+  log_info "Loki retention: ${LOKI_RETENTION_DAYS} days, PVC: ${LOKI_PVC_SIZE}"
+  log_info "Prometheus PVC: ${PROMETHEUS_PVC_SIZE}, scrape interval: ${METRICS_SCRAPE_INTERVAL}"
+
+  helm upgrade --install "${PLG_RELEASE_NAME}" "${PLG_CHART_DIR}" \
+    --namespace "${NAMESPACE}" \
+    -f "${PLG_CHART_DIR}/values-openshift.yaml" \
+    --set "grafana.adminPassword=${GRAFANA_ADMIN_PASSWORD}" \
+    --set "loki.retentionDays=${LOKI_RETENTION_DAYS}" \
+    --set "loki.pvcSize=${LOKI_PVC_SIZE}" \
+    --set "prometheus.pvcSize=${PROMETHEUS_PVC_SIZE}" \
+    --set "prometheus.scrapeInterval=${METRICS_SCRAPE_INTERVAL}" \
+    --set "prometheus.scrapeTargets.backendServices.host=${BACKEND_SERVICES_HOST}" \
+    --set "prometheus.scrapeTargets.temporalServer.host=${TEMPORAL_HOST}" \
+    --wait --timeout 120s || {
+    log_error "Failed to deploy PLG monitoring stack."
+    log_error "The application deployment is unaffected. PLG can be deployed manually later."
+    log_error "  helm upgrade --install ${PLG_RELEASE_NAME} ${PLG_CHART_DIR} -n ${NAMESPACE} -f ${PLG_CHART_DIR}/values-openshift.yaml"
+  }
+
+  log_info "PLG monitoring stack deployed successfully."
+fi
+
+# ============================================================
+# Step 8: Create/update instance secrets
+# ============================================================
+log_step "Step 8: Creating instance secrets"
 
 # Secrets are read from the same env file loaded in Step 3 (via get_config).
 # No separate secrets file is needed.
@@ -524,9 +571,9 @@ oc label secret "${WORKER_SECRET_NAME}" \
 log_info "Instance secrets created successfully."
 
 # ============================================================
-# Step 8: Wait for rollout completion
+# Step 9: Wait for rollout completion
 # ============================================================
-log_step "Step 8: Waiting for rollout completion"
+log_step "Step 9: Waiting for rollout completion"
 
 DEPLOYMENT_SERVICES=("backend-services" "frontend" "temporal" "temporal-ui" "temporal-worker")
 
@@ -547,9 +594,9 @@ done
 log_info "All deployments rolled out successfully."
 
 # ============================================================
-# Step 9: Print access URLs
+# Step 10: Print access URLs
 # ============================================================
-log_step "Step 9: Deployment Complete"
+log_step "Step 10: Deployment Complete"
 
 FRONTEND_ROUTE="https://${INSTANCE_NAME}-frontend-${NAMESPACE}.${CLUSTER_DOMAIN}"
 BACKEND_ROUTE="https://${INSTANCE_NAME}-backend-${NAMESPACE}.${CLUSTER_DOMAIN}"
@@ -571,6 +618,10 @@ To access Temporal UI (not publicly exposed):
   oc port-forward deployment/${INSTANCE_NAME}-temporal-ui 8080:8080 -n ${NAMESPACE}
   Then open http://localhost:8080
 
+To access Grafana (not publicly exposed):
+  oc port-forward svc/${INSTANCE_NAME}-plg-grafana 3001:3001 -n ${NAMESPACE}
+  Then open http://localhost:3001 (admin / <GRAFANA_ADMIN_PASSWORD>)
+
 To tear down this instance:
   ./scripts/oc-teardown.sh --instance ${INSTANCE_NAME}
 
diff --git a/scripts/oc-teardown.sh b/scripts/oc-teardown.sh
@@ -174,6 +174,24 @@ done
 
 log_info "All instance resources deleted."
 
+# ============================================================
+# Step 3b: Uninstall PLG Helm release
+# ============================================================
+PLG_RELEASE_NAME="${INSTANCE_NAME}-plg"
+
+if command -v helm &>/dev/null; then
+  if helm status "${PLG_RELEASE_NAME}" -n "${NAMESPACE}" &>/dev/null; then
+    log_info "Uninstalling PLG Helm release: ${PLG_RELEASE_NAME}"
+    helm uninstall "${PLG_RELEASE_NAME}" -n "${NAMESPACE}" || {
+      log_error "Failed to uninstall PLG Helm release '${PLG_RELEASE_NAME}'. Continuing with teardown."
+    }
+  else
+    log_info "No PLG Helm release '${PLG_RELEASE_NAME}' found — skipping."
+  fi
+else
+  log_info "Helm CLI not installed — skipping PLG release cleanup."
+fi
+
 # ============================================================
 # Step 4: Verify deletion
 # ============================================================
