Merge pull request #71 from bcgov/pr/plg-monitoring-stack

PR 3/4: PLG Monitoring Stack - Metrics, Helm Charts & Promtail Sidecars

Author: alex-struk

.claude/skills/requirements-refiner/SKILL.md

@@ -10,6 +10,7 @@ description: "Requirements Refiner: Iteratively questions the user to clarify va
 2. **Iterative Elicitation**:
 - Identify gaps, ambiguities, or assumptions.
 - Ask a set of numbered clarifying questions.
+- For **each question**, provide 2–4 concrete answer options labeled (a), (b), (c), etc., and mark one as **"Recommended"** with a brief rationale. The user can pick an option, combine options, or provide their own answer.
 - Wait for the user's response.
 - Repeat this step until the requirements are clear and complete.
 3. **Consolidate**:
@@ -25,5 +26,22 @@ description: "Requirements Refiner: Iteratively questions the user to clarify va
 ## Key Behaviors
 - **Iterative Approach**: Do not rush to the final output. Prioritize clarity over speed.
 - **Probe Deeply**: Ask about edge cases, error states, and user roles.
+- **Suggest, Don't Just Ask**: Every clarifying question must include concrete options with a recommended choice. Base recommendations on the project context, industry best practices, and the elicitation standards. This helps the user make faster decisions and reduces back-and-forth.
 - **Datetime-stamped Folders**: Use the current UTC time in `YYYYMMDDHHmmss` format as the folder prefix.
 - **Output Format**: The final output must be saved as `feature-docs/{YYYYMMDDHHmmss}-{feature-slug}/REQUIREMENTS.md`.
+
+## Question Format Example
+
+```
+1. **Who should be able to trigger this workflow?**
+   (a) Only admin users
+   (b) Any authenticated user
+   (c) Both authenticated users and external API consumers
+   → **Recommended: (b)** — Most workflows in this system are user-initiated; restricting to admins adds friction without clear security benefit.
+
+2. **How should the system handle partial failures?**
+   (a) Fail the entire operation and roll back
+   (b) Continue processing remaining items and report failures at the end
+   (c) Retry failed items up to N times, then report
+   → **Recommended: (c)** — Retries with a cap balance reliability with predictable completion times.
+```

.github/workflows/build-apps.yml

@@ -162,6 +162,58 @@ jobs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
+  deploy-plg:
+    name: Deploy PLG Stack
+    needs: [build-apps, get-environment]
+    if: ${{ always() && (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') }}
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ needs.get-environment.outputs.environment }}
+    env:
+      OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}
+      OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }}
+      OPENSHIFT_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE }}
+      GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: v3.17.0
+
+      - name: Install oc CLI
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4"
+
+      - name: Login to OpenShift
+        run: |
+          oc login "${{ env.OPENSHIFT_SERVER }}" \
+            --token="${{ env.OPENSHIFT_TOKEN }}" \
+            --insecure-skip-tls-verify=true
+
+      - name: Deploy PLG Helm Chart
+        run: |
+          CHART_DIR="deployments/openshift/helm/plg"
+
+          # Use environment-specific defaults; secrets override via GitHub environment
+          GRAFANA_PWD="${{ env.GRAFANA_ADMIN_PASSWORD }}"
+          if [ -z "${GRAFANA_PWD}" ]; then
+            GRAFANA_PWD="admin"
+          fi
+
+          helm upgrade --install plg "${CHART_DIR}" \
+            --namespace "${{ env.OPENSHIFT_NAMESPACE }}" \
+            -f "${CHART_DIR}/values-openshift.yaml" \
+            --set "grafana.adminPassword=${GRAFANA_PWD}" \
+            --wait --timeout 120s
+
+          echo "PLG stack deployed successfully."
+
   # trigger_migration:
   #   needs: [metadata, build-apps]
   #   if: ${{ always() && needs.build-apps.result == 'success' && needs.metadata.outputs.changed-apps != '[]' && needs.metadata.outputs.changed-apps != '' }}

.vscode/tasks.json

@@ -149,6 +149,30 @@
         "runOn": "folderOpen"
       },
       "problemMatcher": []
+    },
+    {
+      "label": "monitoring: docker up",
+      "type": "shell",
+      "command": "npm",
+      "args": ["run", "dev:monitoring"],
+      "options": {
+        "cwd": "${workspaceFolder}"
+      },
+      "problemMatcher": [],
+      "presentation": {
+        "panel": "new",
+        "reveal": "always"
+      }
+    },
+    {
+      "label": "Dev: all + monitoring",
+      "dependsOn": [
+        "Dev: prerequisites",
+        "monitoring: docker up",
+        "Dev: runtime"
+      ],
+      "dependsOrder": "sequence",
+      "problemMatcher": []
     }
   ]
 }

apps/backend-services/package.json

@@ -61,6 +61,7 @@
     "passport-jwt": "^4.0.1",
     "pg": "^8.16.3",
     "prisma": "7.2.0",
+    "prom-client": "^15.1.3",
     "rxjs": "^7.8.2",
     "uuid": "13.0.0"
   },

apps/backend-services/src/api-key/api-key.service.spec.ts

@@ -190,7 +190,7 @@ describe("ApiKeyService", () => {
 
       const result = await service.validateApiKey(validKey);
 
-      expect(result).toEqual({ groupId: "group-test" });
+      expect(result).toEqual({ groupId: "group-test", keyPrefix: "testkey1" });
       expect(mockApiKeyDbService.findApiKeysByPrefix).toHaveBeenCalledWith(
         "testkey1",
       );

apps/backend-services/src/api-key/api-key.service.ts

@@ -5,6 +5,7 @@ import {
   ApiKeyInfoDto,
   GeneratedApiKeyDto,
 } from "@/api-key/dto/api-key-info.dto";
+import type { ValidatedApiKey } from "@/auth/types";
 import { AppLoggerService } from "@/logging/app-logger.service";
 import { ApiKeyDbService } from "./api-key-db.service";
 
@@ -95,7 +96,7 @@ export class ApiKeyService {
     return this.generateApiKey(userId, groupId);
   }
 
-  async validateApiKey(key: string): Promise<{ groupId: string } | null> {
+  async validateApiKey(key: string): Promise<ValidatedApiKey | null> {
     // Extract prefix from the incoming key for indexed lookup
     const prefix = key.substring(0, 8);
 
@@ -109,7 +110,7 @@ export class ApiKeyService {
         // Update last_used timestamp
         await this.apiKeyDb.updateApiKeyLastUsed(apiKey.id);
 
-        return { groupId: apiKey.group_id };
+        return { groupId: apiKey.group_id, keyPrefix: apiKey.key_prefix };
       }
     }
 

apps/backend-services/src/app.module.ts

@@ -16,6 +16,7 @@ import { GroupModule } from "./group/group.module";
 import { HitlModule } from "./hitl/hitl.module";
 import { LabelingModule } from "./labeling/labeling.module";
 import { LoggingModule } from "./logging/logging.module";
+import { MetricsModule } from "./metrics/metrics.module";
 import { OcrModule } from "./ocr/ocr.module";
 import { QueueModule } from "./queue/queue.module";
 import { TemporalModule } from "./temporal/temporal.module";
@@ -63,6 +64,7 @@ import { WorkflowModule } from "./workflow/workflow.module";
     AzureModule,
     BootstrapModule,
     GroupModule,
+    MetricsModule,
   ],
   providers: [
     {

apps/backend-services/src/auth/api-key-auth.guard.spec.ts

@@ -8,7 +8,6 @@ import { Reflector } from "@nestjs/core";
 import { Test, TestingModule } from "@nestjs/testing";
 import { ApiKeyService } from "../api-key/api-key.service";
 import { ApiKeyAuthGuard } from "./api-key-auth.guard";
-import { IdentityOptions } from "./identity.decorator";
 
 describe("ApiKeyAuthGuard", () => {
   let guard: ApiKeyAuthGuard;
@@ -80,7 +79,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should return true if user is already authenticated", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
 
     const context = createMockExecutionContext({}, { sub: "testuser" });
     const result = await guard.canActivate(context);
@@ -92,7 +91,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should throw UnauthorizedException if no API key header and no authenticated user", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
 
     const context = createMockExecutionContext({});
 
@@ -105,7 +104,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should return true if no API key header but user is already authenticated", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
 
     const context = createMockExecutionContext({}, { sub: "testuser" });
     const result = await guard.canActivate(context);
@@ -117,7 +116,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should throw UnauthorizedException for invalid API key", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
     mockApiKeyService.validateApiKey.mockResolvedValue(null);
 
     const context = createMockExecutionContext({ "x-api-key": "invalidkey" });
@@ -128,12 +127,13 @@ describe("ApiKeyAuthGuard", () => {
     expect(apiKeyService.validateApiKey).toHaveBeenCalledWith("invalidkey");
   });
 
-  it("should set apiKeyGroupId for valid API key", async () => {
+  it("should set apiKey for valid API key", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
     mockApiKeyService.validateApiKey.mockResolvedValue({
       groupId: "group-abc",
+      keyPrefix: "aBcDeFgH",
     });
 
     const mockRequest: Record<string, unknown> = {
@@ -152,14 +152,17 @@ describe("ApiKeyAuthGuard", () => {
 
     expect(result).toBe(true);
     expect(mockRequest.user).toBeUndefined();
-    expect(mockRequest.apiKeyGroupId).toBe("group-abc");
+    expect(mockRequest.apiKey).toEqual({
+      groupId: "group-abc",
+      keyPrefix: "aBcDeFgH",
+    });
   });
 
   describe("failed-attempt throttling", () => {
     beforeEach(() => {
       (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
         allowApiKey: true,
-      } as IdentityOptions);
+      });
       mockApiKeyService.validateApiKey.mockResolvedValue(null);
     });
 
@@ -249,6 +252,7 @@ describe("ApiKeyAuthGuard", () => {
       // Successful validation should reset the counter
       mockApiKeyService.validateApiKey.mockResolvedValueOnce({
         groupId: "group-reset",
+        keyPrefix: "validkey",
       });
 
       const mockRequest = {

apps/backend-services/src/auth/api-key-auth.guard.ts

@@ -93,10 +93,10 @@ export class ApiKeyAuthGuard implements CanActivate, OnModuleDestroy {
     // Successful validation — reset failure counter for this IP
     this.failedAttempts.delete(clientIp);
 
-    // Attach the API key's group_id for use by IdentityGuard and downstream
+    // Attach the validated API key for use by IdentityGuard and downstream
     // service-layer authorization helpers. The key is group-scoped; there is
     // no user identity to apply.
-    request.apiKeyGroupId = keyInfo.groupId;
+    request.apiKey = keyInfo;
 
     return true;
   }

apps/backend-services/src/auth/guard-composition.spec.ts

@@ -47,8 +47,8 @@ const JWT_ADMIN = {
   email: "admin@example.com",
 };
 
-const API_KEY_USER = { groupId: "group-user" };
-const API_KEY_ADMIN = { groupId: "group-admin" };
+const API_KEY_USER = { groupId: "group-user", keyPrefix: "test-pre" };
+const API_KEY_ADMIN = { groupId: "group-admin", keyPrefix: "test-pre" };
 
 // ---------------------------------------------------------------------------
 // Stub: replaces Passport JWT validation with a simple token check.
@@ -170,7 +170,7 @@ describe("Guard Composition Integration", () => {
     jest.clearAllMocks();
 
     mockApiKeyService.validateApiKey.mockImplementation(
-      (key: string): Promise<{ groupId: string } | null> => {
+      (key: string): Promise<{ groupId: string; keyPrefix: string } | null> => {
         if (key === VALID_API_KEY) return Promise.resolve(API_KEY_USER);
         if (key === "valid-admin-api-key")
           return Promise.resolve(API_KEY_ADMIN);