PR 3/4: PLG Monitoring Stack - Metrics, Helm Charts & Promtail Sidecars

.claude/skills/requirements-refiner/SKILL.md

@@ -10,6 +10,7 @@ description: "Requirements Refiner: Iteratively questions the user to clarify va
 2. **Iterative Elicitation**:
 - Identify gaps, ambiguities, or assumptions.
 - Ask a set of numbered clarifying questions.
+- For **each question**, provide 2–4 concrete answer options labeled (a), (b), (c), etc., and mark one as **"Recommended"** with a brief rationale. The user can pick an option, combine options, or provide their own answer.
 - Wait for the user's response.
 - Repeat this step until the requirements are clear and complete.
 3. **Consolidate**:
@@ -25,5 +26,22 @@ description: "Requirements Refiner: Iteratively questions the user to clarify va
 ## Key Behaviors
 - **Iterative Approach**: Do not rush to the final output. Prioritize clarity over speed.
 - **Probe Deeply**: Ask about edge cases, error states, and user roles.
+- **Suggest, Don't Just Ask**: Every clarifying question must include concrete options with a recommended choice. Base recommendations on the project context, industry best practices, and the elicitation standards. This helps the user make faster decisions and reduces back-and-forth.
 - **Datetime-stamped Folders**: Use the current UTC time in `YYYYMMDDHHmmss` format as the folder prefix.
 - **Output Format**: The final output must be saved as `feature-docs/{YYYYMMDDHHmmss}-{feature-slug}/REQUIREMENTS.md`.
+
+## Question Format Example
+
+```
+1. **Who should be able to trigger this workflow?**
+   (a) Only admin users
+   (b) Any authenticated user
+   (c) Both authenticated users and external API consumers
+   → **Recommended: (b)** — Most workflows in this system are user-initiated; restricting to admins adds friction without clear security benefit.
+
+2. **How should the system handle partial failures?**
+   (a) Fail the entire operation and roll back
+   (b) Continue processing remaining items and report failures at the end
+   (c) Retry failed items up to N times, then report
+   → **Recommended: (c)** — Retries with a cap balance reliability with predictable completion times.
+```

.github/workflows/build-apps.yml

@@ -162,24 +162,76 @@
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
+  deploy-plg:
+    name: Deploy PLG Stack
+    needs: [build-apps, get-environment]
+    if: ${{ always() && (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') }}
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ needs.get-environment.outputs.environment }}
+    env:
+      OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}
+      OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }}
+      OPENSHIFT_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE }}
+      GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: v3.17.0
+
+      - name: Install oc CLI
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4"
+
+      - name: Login to OpenShift
+        run: |
+          oc login "${{ env.OPENSHIFT_SERVER }}" \
+            --token="${{ env.OPENSHIFT_TOKEN }}" \
+            --insecure-skip-tls-verify=true
+
+      - name: Deploy PLG Helm Chart
+        run: |
+          CHART_DIR="deployments/openshift/helm/plg"
+
+          # Use environment-specific defaults; secrets override via GitHub environment
+          GRAFANA_PWD="${{ env.GRAFANA_ADMIN_PASSWORD }}"
+          if [ -z "${GRAFANA_PWD}" ]; then
+            GRAFANA_PWD="admin"
+          fi
+
+          helm upgrade --install plg "${CHART_DIR}" \
+            --namespace "${{ env.OPENSHIFT_NAMESPACE }}" \
+            -f "${CHART_DIR}/values-openshift.yaml" \
+            --set "grafana.adminPassword=${GRAFANA_PWD}" \
+            --wait --timeout 120s
+
+          echo "PLG stack deployed successfully."
+
   # trigger_migration:
   #   needs: [metadata, build-apps]
   #   if: ${{ always() && needs.build-apps.result == 'success' && needs.metadata.outputs.changed-apps != '[]' && needs.metadata.outputs.changed-apps != '' }}
   #   runs-on: ubuntu-latest
   #   steps:
   #     - name: Trigger Migration Workflow
   #       uses: actions/github-script@v7
   #       with:
   #         github-token: ${{ secrets.GITHUB_TOKEN }}
   #         script: |
   #           await github.rest.actions.createWorkflowDispatch({
   #             owner: context.repo.owner,
   #             repo: context.repo.repo,
   #             workflow_id: 'migrate-db.yml',
   #             ref: '${{ github.ref }}',
   #             inputs: {
   #               sha: '${{ github.sha }}',
   #               changed_apps: '${{ needs.metadata.outputs.changed-apps }}'
   #             }
   #           });
 

.vscode/tasks.json

@@ -149,6 +149,30 @@
         "runOn": "folderOpen"
       },
       "problemMatcher": []
+    },
+    {
+      "label": "monitoring: docker up",
+      "type": "shell",
+      "command": "npm",
+      "args": ["run", "dev:monitoring"],
+      "options": {
+        "cwd": "${workspaceFolder}"
+      },
+      "problemMatcher": [],
+      "presentation": {
+        "panel": "new",
+        "reveal": "always"
+      }
+    },
+    {
+      "label": "Dev: all + monitoring",
+      "dependsOn": [
+        "Dev: prerequisites",
+        "monitoring: docker up",
+        "Dev: runtime"
+      ],
+      "dependsOrder": "sequence",
+      "problemMatcher": []
     }
   ]
 }

apps/backend-services/package.json

@@ -61,6 +61,7 @@
     "passport-jwt": "^4.0.1",
     "pg": "^8.16.3",
     "prisma": "7.2.0",
+    "prom-client": "^15.1.3",
     "rxjs": "^7.8.2",
     "uuid": "13.0.0"
   },

apps/backend-services/src/api-key/api-key.service.spec.ts

@@ -190,7 +190,7 @@ describe("ApiKeyService", () => {
 
       const result = await service.validateApiKey(validKey);
 
-      expect(result).toEqual({ groupId: "group-test" });
+      expect(result).toEqual({ groupId: "group-test", keyPrefix: "testkey1" });
       expect(mockApiKeyDbService.findApiKeysByPrefix).toHaveBeenCalledWith(
         "testkey1",
       );

apps/backend-services/src/api-key/api-key.service.ts

@@ -95,7 +95,9 @@ export class ApiKeyService {
     return this.generateApiKey(userId, groupId);
   }
 
-  async validateApiKey(key: string): Promise<{ groupId: string } | null> {
+  async validateApiKey(
+    key: string,
+  ): Promise<{ groupId: string; keyPrefix: string } | null> {
     // Extract prefix from the incoming key for indexed lookup
     const prefix = key.substring(0, 8);
 
@@ -109,7 +111,7 @@ export class ApiKeyService {
         // Update last_used timestamp
         await this.apiKeyDb.updateApiKeyLastUsed(apiKey.id);
 
-        return { groupId: apiKey.group_id };
+        return { groupId: apiKey.group_id, keyPrefix: apiKey.key_prefix };
       }
     }
 

apps/backend-services/src/app.module.ts

@@ -16,6 +16,7 @@ import { GroupModule } from "./group/group.module";
 import { HitlModule } from "./hitl/hitl.module";
 import { LabelingModule } from "./labeling/labeling.module";
 import { LoggingModule } from "./logging/logging.module";
+import { MetricsModule } from "./metrics/metrics.module";
 import { OcrModule } from "./ocr/ocr.module";
 import { QueueModule } from "./queue/queue.module";
 import { TemporalModule } from "./temporal/temporal.module";
@@ -63,6 +64,7 @@ import { WorkflowModule } from "./workflow/workflow.module";
     AzureModule,
     BootstrapModule,
     GroupModule,
+    MetricsModule,
   ],
   providers: [
     {

apps/backend-services/src/auth/api-key-auth.guard.spec.ts

@@ -8,7 +8,6 @@ import { Reflector } from "@nestjs/core";
 import { Test, TestingModule } from "@nestjs/testing";
 import { ApiKeyService } from "../api-key/api-key.service";
 import { ApiKeyAuthGuard } from "./api-key-auth.guard";
-import { IdentityOptions } from "./identity.decorator";
 
 describe("ApiKeyAuthGuard", () => {
   let guard: ApiKeyAuthGuard;
@@ -80,7 +79,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should return true if user is already authenticated", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
 
     const context = createMockExecutionContext({}, { sub: "testuser" });
     const result = await guard.canActivate(context);
@@ -92,7 +91,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should throw UnauthorizedException if no API key header and no authenticated user", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
 
     const context = createMockExecutionContext({});
 
@@ -105,7 +104,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should return true if no API key header but user is already authenticated", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
 
     const context = createMockExecutionContext({}, { sub: "testuser" });
     const result = await guard.canActivate(context);
@@ -117,7 +116,7 @@ describe("ApiKeyAuthGuard", () => {
   it("should throw UnauthorizedException for invalid API key", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
     mockApiKeyService.validateApiKey.mockResolvedValue(null);
 
     const context = createMockExecutionContext({ "x-api-key": "invalidkey" });
@@ -128,12 +127,13 @@ describe("ApiKeyAuthGuard", () => {
     expect(apiKeyService.validateApiKey).toHaveBeenCalledWith("invalidkey");
   });
 
-  it("should set apiKeyGroupId for valid API key", async () => {
+  it("should set apiKeyGroupId and apiKeyPrefix for valid API key", async () => {
     (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
       allowApiKey: true,
-    } as IdentityOptions);
+    });
     mockApiKeyService.validateApiKey.mockResolvedValue({
       groupId: "group-abc",
+      keyPrefix: "aBcDeFgH",
     });
 
     const mockRequest: Record<string, unknown> = {
@@ -153,13 +153,14 @@ describe("ApiKeyAuthGuard", () => {
     expect(result).toBe(true);
     expect(mockRequest.user).toBeUndefined();
     expect(mockRequest.apiKeyGroupId).toBe("group-abc");
+    expect(mockRequest.apiKeyPrefix).toBe("aBcDeFgH");
   });
 
   describe("failed-attempt throttling", () => {
     beforeEach(() => {
       (reflector.getAllAndOverride as jest.Mock).mockReturnValue({
         allowApiKey: true,
-      } as IdentityOptions);
+      });
       mockApiKeyService.validateApiKey.mockResolvedValue(null);
     });
 
@@ -249,6 +250,7 @@ describe("ApiKeyAuthGuard", () => {
       // Successful validation should reset the counter
       mockApiKeyService.validateApiKey.mockResolvedValueOnce({
         groupId: "group-reset",
+        keyPrefix: "validkey",
       });
 
       const mockRequest = {

apps/backend-services/src/auth/api-key-auth.guard.ts

@@ -97,6 +97,7 @@ export class ApiKeyAuthGuard implements CanActivate, OnModuleDestroy {
     // service-layer authorization helpers. The key is group-scoped; there is
     // no user identity to apply.
     request.apiKeyGroupId = keyInfo.groupId;
+    request.apiKeyPrefix = keyInfo.keyPrefix;
 
     return true;
   }

apps/backend-services/src/auth/types.ts

@@ -35,6 +35,8 @@ declare module "express" {
     user?: User;
     /** Set by ApiKeyAuthGuard when a valid API key is used. */
     apiKeyGroupId?: string;
+    /** Set by ApiKeyAuthGuard — the stored key prefix for audit logging. */
+    apiKeyPrefix?: string;
     /**
      * Set by IdentityGuard after authentication succeeds.
      * Contains the normalised requestor identity for downstream authorization.

apps/backend-services/src/logging/app-logger.service.spec.ts

@@ -121,6 +121,77 @@ describe("AppLoggerService", () => {
     });
   });
 
+  describe("with sessionId in request context", () => {
+    beforeEach(() =>
+      mockGetRequestContext.mockReturnValue({
+        requestId: "req-1",
+        userId: "user-1",
+        sessionId: "session-abc-123",
+      }),
+    );
+
+    it("includes sessionId in merged context", () => {
+      service.log("test message");
+      expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", {
+        requestId: "req-1",
+        userId: "user-1",
+        sessionId: "session-abc-123",
+      });
+    });
+  });
+
+  describe("with clientIp in request context", () => {
+    beforeEach(() =>
+      mockGetRequestContext.mockReturnValue({
+        requestId: "req-3",
+        clientIp: "203.0.113.50",
+      }),
+    );
+
+    it("includes clientIp in merged context", () => {
+      service.log("test message");
+      expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", {
+        requestId: "req-3",
+        clientIp: "203.0.113.50",
+      });
+    });
+  });
+
+  describe("with apiKeyId in request context", () => {
+    beforeEach(() =>
+      mockGetRequestContext.mockReturnValue({
+        requestId: "req-5",
+        apiKeyId: "aBcDeFgH",
+      }),
+    );
+
+    it("includes apiKeyId in merged context", () => {
+      service.log("test message");
+      expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", {
+        requestId: "req-5",
+        apiKeyId: "aBcDeFgH",
+      });
+    });
+  });
+
+  describe("omits falsy optional fields", () => {
+    beforeEach(() =>
+      mockGetRequestContext.mockReturnValue({
+        requestId: "req-6",
+        sessionId: undefined,
+        apiKeyId: undefined,
+        clientIp: undefined,
+      }),
+    );
+
+    it("does not include undefined optional fields", () => {
+      service.log("test message");
+      expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", {
+        requestId: "req-6",
+      });
+    });
+  });
+
   describe("static getLogLevel", () => {
     it("exposes the getLogLevel function", () => {
       expect(AppLoggerService.getLogLevel).toBeDefined();

apps/backend-services/src/logging/app-logger.service.ts

@@ -18,6 +18,9 @@ export class AppLoggerService {
     return {
       ...(ctx?.requestId && { requestId: ctx.requestId }),
       ...(ctx?.userId && { userId: ctx.userId }),
+      ...(ctx?.sessionId && { sessionId: ctx.sessionId }),
+      ...(ctx?.apiKeyId && { apiKeyId: ctx.apiKeyId }),
+      ...(ctx?.clientIp && { clientIp: ctx.clientIp }),
       ...context,
     };
   }