diff --git a/.claude/skills/requirements-refiner/SKILL.md b/.claude/skills/requirements-refiner/SKILL.md index 47ed2b6ce..b7fe2c098 100644 --- a/.claude/skills/requirements-refiner/SKILL.md +++ b/.claude/skills/requirements-refiner/SKILL.md @@ -10,6 +10,7 @@ description: "Requirements Refiner: Iteratively questions the user to clarify va 2. **Iterative Elicitation**: - Identify gaps, ambiguities, or assumptions. - Ask a set of numbered clarifying questions. +- For **each question**, provide 2–4 concrete answer options labeled (a), (b), (c), etc., and mark one as **"Recommended"** with a brief rationale. The user can pick an option, combine options, or provide their own answer. - Wait for the user's response. - Repeat this step until the requirements are clear and complete. 3. **Consolidate**: @@ -25,5 +26,22 @@ description: "Requirements Refiner: Iteratively questions the user to clarify va ## Key Behaviors - **Iterative Approach**: Do not rush to the final output. Prioritize clarity over speed. - **Probe Deeply**: Ask about edge cases, error states, and user roles. +- **Suggest, Don't Just Ask**: Every clarifying question must include concrete options with a recommended choice. Base recommendations on the project context, industry best practices, and the elicitation standards. This helps the user make faster decisions and reduces back-and-forth. - **Datetime-stamped Folders**: Use the current UTC time in `YYYYMMDDHHmmss` format as the folder prefix. - **Output Format**: The final output must be saved as `feature-docs/{YYYYMMDDHHmmss}-{feature-slug}/REQUIREMENTS.md`. + +## Question Format Example + +``` +1. **Who should be able to trigger this workflow?** + (a) Only admin users + (b) Any authenticated user + (c) Both authenticated users and external API consumers + → **Recommended: (b)** — Most workflows in this system are user-initiated; restricting to admins adds friction without clear security benefit. + +2. **How should the system handle partial failures?** + (a) Fail the entire operation and roll back + (b) Continue processing remaining items and report failures at the end + (c) Retry failed items up to N times, then report + → **Recommended: (c)** — Retries with a cap balance reliability with predictable completion times. +``` diff --git a/.github/workflows/build-apps.yml b/.github/workflows/build-apps.yml index 32a01332e..da80e2945 100644 --- a/.github/workflows/build-apps.yml +++ b/.github/workflows/build-apps.yml @@ -162,6 +162,58 @@ jobs: rm -rf /tmp/.buildx-cache mv /tmp/.buildx-cache-new /tmp/.buildx-cache + deploy-plg: + name: Deploy PLG Stack + needs: [build-apps, get-environment] + if: ${{ always() && (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') }} + runs-on: ubuntu-latest + environment: + name: ${{ needs.get-environment.outputs.environment }} + env: + OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} + OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} + OPENSHIFT_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE }} + GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: v3.17.0 + + - name: Install oc CLI + uses: redhat-actions/openshift-tools-installer@v1 + with: + oc: "4" + + - name: Login to OpenShift + run: | + oc login "${{ env.OPENSHIFT_SERVER }}" \ + --token="${{ env.OPENSHIFT_TOKEN }}" \ + --insecure-skip-tls-verify=true + + - name: Deploy PLG Helm Chart + run: | + CHART_DIR="deployments/openshift/helm/plg" + + # Use environment-specific defaults; secrets override via GitHub environment + GRAFANA_PWD="${{ env.GRAFANA_ADMIN_PASSWORD }}" + if [ -z "${GRAFANA_PWD}" ]; then + GRAFANA_PWD="admin" + fi + + helm upgrade --install plg "${CHART_DIR}" \ + --namespace "${{ env.OPENSHIFT_NAMESPACE }}" \ + -f "${CHART_DIR}/values-openshift.yaml" \ + --set "grafana.adminPassword=${GRAFANA_PWD}" \ + --wait --timeout 120s + + echo "PLG stack deployed successfully." + # trigger_migration: # needs: [metadata, build-apps] # if: ${{ always() && needs.build-apps.result == 'success' && needs.metadata.outputs.changed-apps != '[]' && needs.metadata.outputs.changed-apps != '' }} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 5d56d730d..c82640ace 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -149,6 +149,30 @@ "runOn": "folderOpen" }, "problemMatcher": [] + }, + { + "label": "monitoring: docker up", + "type": "shell", + "command": "npm", + "args": ["run", "dev:monitoring"], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "presentation": { + "panel": "new", + "reveal": "always" + } + }, + { + "label": "Dev: all + monitoring", + "dependsOn": [ + "Dev: prerequisites", + "monitoring: docker up", + "Dev: runtime" + ], + "dependsOrder": "sequence", + "problemMatcher": [] } ] } diff --git a/apps/backend-services/package.json b/apps/backend-services/package.json index 5f29b97ff..43f392ad3 100644 --- a/apps/backend-services/package.json +++ b/apps/backend-services/package.json @@ -61,6 +61,7 @@ "passport-jwt": "^4.0.1", "pg": "^8.16.3", "prisma": "7.2.0", + "prom-client": "^15.1.3", "rxjs": "^7.8.2", "uuid": "13.0.0" }, diff --git a/apps/backend-services/src/api-key/api-key.service.spec.ts b/apps/backend-services/src/api-key/api-key.service.spec.ts index 876e516a2..98f929d8e 100644 --- a/apps/backend-services/src/api-key/api-key.service.spec.ts +++ b/apps/backend-services/src/api-key/api-key.service.spec.ts @@ -190,7 +190,7 @@ describe("ApiKeyService", () => { const result = await service.validateApiKey(validKey); - expect(result).toEqual({ groupId: "group-test" }); + expect(result).toEqual({ groupId: "group-test", keyPrefix: "testkey1" }); expect(mockApiKeyDbService.findApiKeysByPrefix).toHaveBeenCalledWith( "testkey1", ); diff --git a/apps/backend-services/src/api-key/api-key.service.ts b/apps/backend-services/src/api-key/api-key.service.ts index 157d12677..4b2bf6edf 100644 --- a/apps/backend-services/src/api-key/api-key.service.ts +++ b/apps/backend-services/src/api-key/api-key.service.ts @@ -5,6 +5,7 @@ import { ApiKeyInfoDto, GeneratedApiKeyDto, } from "@/api-key/dto/api-key-info.dto"; +import type { ValidatedApiKey } from "@/auth/types"; import { AppLoggerService } from "@/logging/app-logger.service"; import { ApiKeyDbService } from "./api-key-db.service"; @@ -95,7 +96,7 @@ export class ApiKeyService { return this.generateApiKey(userId, groupId); } - async validateApiKey(key: string): Promise<{ groupId: string } | null> { + async validateApiKey(key: string): Promise { // Extract prefix from the incoming key for indexed lookup const prefix = key.substring(0, 8); @@ -109,7 +110,7 @@ export class ApiKeyService { // Update last_used timestamp await this.apiKeyDb.updateApiKeyLastUsed(apiKey.id); - return { groupId: apiKey.group_id }; + return { groupId: apiKey.group_id, keyPrefix: apiKey.key_prefix }; } } diff --git a/apps/backend-services/src/app.module.ts b/apps/backend-services/src/app.module.ts index d78482140..0b988ab94 100644 --- a/apps/backend-services/src/app.module.ts +++ b/apps/backend-services/src/app.module.ts @@ -16,6 +16,7 @@ import { GroupModule } from "./group/group.module"; import { HitlModule } from "./hitl/hitl.module"; import { LabelingModule } from "./labeling/labeling.module"; import { LoggingModule } from "./logging/logging.module"; +import { MetricsModule } from "./metrics/metrics.module"; import { OcrModule } from "./ocr/ocr.module"; import { QueueModule } from "./queue/queue.module"; import { TemporalModule } from "./temporal/temporal.module"; @@ -63,6 +64,7 @@ import { WorkflowModule } from "./workflow/workflow.module"; AzureModule, BootstrapModule, GroupModule, + MetricsModule, ], providers: [ { diff --git a/apps/backend-services/src/auth/api-key-auth.guard.spec.ts b/apps/backend-services/src/auth/api-key-auth.guard.spec.ts index aa3d8fc40..cec545c67 100644 --- a/apps/backend-services/src/auth/api-key-auth.guard.spec.ts +++ b/apps/backend-services/src/auth/api-key-auth.guard.spec.ts @@ -8,7 +8,6 @@ import { Reflector } from "@nestjs/core"; import { Test, TestingModule } from "@nestjs/testing"; import { ApiKeyService } from "../api-key/api-key.service"; import { ApiKeyAuthGuard } from "./api-key-auth.guard"; -import { IdentityOptions } from "./identity.decorator"; describe("ApiKeyAuthGuard", () => { let guard: ApiKeyAuthGuard; @@ -80,7 +79,7 @@ describe("ApiKeyAuthGuard", () => { it("should return true if user is already authenticated", async () => { (reflector.getAllAndOverride as jest.Mock).mockReturnValue({ allowApiKey: true, - } as IdentityOptions); + }); const context = createMockExecutionContext({}, { sub: "testuser" }); const result = await guard.canActivate(context); @@ -92,7 +91,7 @@ describe("ApiKeyAuthGuard", () => { it("should throw UnauthorizedException if no API key header and no authenticated user", async () => { (reflector.getAllAndOverride as jest.Mock).mockReturnValue({ allowApiKey: true, - } as IdentityOptions); + }); const context = createMockExecutionContext({}); @@ -105,7 +104,7 @@ describe("ApiKeyAuthGuard", () => { it("should return true if no API key header but user is already authenticated", async () => { (reflector.getAllAndOverride as jest.Mock).mockReturnValue({ allowApiKey: true, - } as IdentityOptions); + }); const context = createMockExecutionContext({}, { sub: "testuser" }); const result = await guard.canActivate(context); @@ -117,7 +116,7 @@ describe("ApiKeyAuthGuard", () => { it("should throw UnauthorizedException for invalid API key", async () => { (reflector.getAllAndOverride as jest.Mock).mockReturnValue({ allowApiKey: true, - } as IdentityOptions); + }); mockApiKeyService.validateApiKey.mockResolvedValue(null); const context = createMockExecutionContext({ "x-api-key": "invalidkey" }); @@ -128,12 +127,13 @@ describe("ApiKeyAuthGuard", () => { expect(apiKeyService.validateApiKey).toHaveBeenCalledWith("invalidkey"); }); - it("should set apiKeyGroupId for valid API key", async () => { + it("should set apiKey for valid API key", async () => { (reflector.getAllAndOverride as jest.Mock).mockReturnValue({ allowApiKey: true, - } as IdentityOptions); + }); mockApiKeyService.validateApiKey.mockResolvedValue({ groupId: "group-abc", + keyPrefix: "aBcDeFgH", }); const mockRequest: Record = { @@ -152,14 +152,17 @@ describe("ApiKeyAuthGuard", () => { expect(result).toBe(true); expect(mockRequest.user).toBeUndefined(); - expect(mockRequest.apiKeyGroupId).toBe("group-abc"); + expect(mockRequest.apiKey).toEqual({ + groupId: "group-abc", + keyPrefix: "aBcDeFgH", + }); }); describe("failed-attempt throttling", () => { beforeEach(() => { (reflector.getAllAndOverride as jest.Mock).mockReturnValue({ allowApiKey: true, - } as IdentityOptions); + }); mockApiKeyService.validateApiKey.mockResolvedValue(null); }); @@ -249,6 +252,7 @@ describe("ApiKeyAuthGuard", () => { // Successful validation should reset the counter mockApiKeyService.validateApiKey.mockResolvedValueOnce({ groupId: "group-reset", + keyPrefix: "validkey", }); const mockRequest = { diff --git a/apps/backend-services/src/auth/api-key-auth.guard.ts b/apps/backend-services/src/auth/api-key-auth.guard.ts index 63bf79e8f..c1c4e4968 100644 --- a/apps/backend-services/src/auth/api-key-auth.guard.ts +++ b/apps/backend-services/src/auth/api-key-auth.guard.ts @@ -93,10 +93,10 @@ export class ApiKeyAuthGuard implements CanActivate, OnModuleDestroy { // Successful validation — reset failure counter for this IP this.failedAttempts.delete(clientIp); - // Attach the API key's group_id for use by IdentityGuard and downstream + // Attach the validated API key for use by IdentityGuard and downstream // service-layer authorization helpers. The key is group-scoped; there is // no user identity to apply. - request.apiKeyGroupId = keyInfo.groupId; + request.apiKey = keyInfo; return true; } diff --git a/apps/backend-services/src/auth/guard-composition.spec.ts b/apps/backend-services/src/auth/guard-composition.spec.ts index 8095c84c7..2fbb00dcf 100644 --- a/apps/backend-services/src/auth/guard-composition.spec.ts +++ b/apps/backend-services/src/auth/guard-composition.spec.ts @@ -47,8 +47,8 @@ const JWT_ADMIN = { email: "admin@example.com", }; -const API_KEY_USER = { groupId: "group-user" }; -const API_KEY_ADMIN = { groupId: "group-admin" }; +const API_KEY_USER = { groupId: "group-user", keyPrefix: "test-pre" }; +const API_KEY_ADMIN = { groupId: "group-admin", keyPrefix: "test-pre" }; // --------------------------------------------------------------------------- // Stub: replaces Passport JWT validation with a simple token check. @@ -170,7 +170,7 @@ describe("Guard Composition Integration", () => { jest.clearAllMocks(); mockApiKeyService.validateApiKey.mockImplementation( - (key: string): Promise<{ groupId: string } | null> => { + (key: string): Promise<{ groupId: string; keyPrefix: string } | null> => { if (key === VALID_API_KEY) return Promise.resolve(API_KEY_USER); if (key === "valid-admin-api-key") return Promise.resolve(API_KEY_ADMIN); diff --git a/apps/backend-services/src/auth/identity.guard.spec.ts b/apps/backend-services/src/auth/identity.guard.spec.ts index fc04edad1..199f426d1 100644 --- a/apps/backend-services/src/auth/identity.guard.spec.ts +++ b/apps/backend-services/src/auth/identity.guard.spec.ts @@ -102,7 +102,7 @@ describe("IdentityGuard", () => { ); const request: Record = { // No request.user — API key auth does not set a user object - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, }; const result = await identityGuard.canActivate(createContext(request)); @@ -120,7 +120,7 @@ describe("IdentityGuard", () => { groupService as unknown as GroupService, ); const request: Record = { - apiKeyGroupId: "specific-group-id", + apiKey: { groupId: "specific-group-id", keyPrefix: "aBcDeFgH" }, }; await identityGuard.canActivate(createContext(request)); @@ -130,7 +130,7 @@ describe("IdentityGuard", () => { ).toBeUndefined(); }); - it("should prefer API key path over JWT path when apiKeyGroupId is set and @Identity is present", async () => { + it("should prefer API key path over JWT path when apiKey is set and @Identity is present", async () => { const identityGuard = new IdentityGuard( createReflectorWithIdentity({ allowApiKey: true }), groupService as unknown as GroupService, @@ -138,7 +138,7 @@ describe("IdentityGuard", () => { // Edge case: both present (should not happen in practice, but guard should be deterministic) const request: Record = { user: { sub: "some-user" }, - apiKeyGroupId: "group-id", + apiKey: { groupId: "group-id", keyPrefix: "aBcDeFgH" }, }; await identityGuard.canActivate(createContext(request)); @@ -158,7 +158,9 @@ describe("IdentityGuard", () => { createReflectorWithIdentity({ allowApiKey: true }), groupService as unknown as GroupService, ); - const request: Record = { apiKeyGroupId: "group-123" }; + const request: Record = { + apiKey: { groupId: "group-123", keyPrefix: "aBcDeFgH" }, + }; await identityGuard.canActivate(createContext(request)); @@ -172,7 +174,9 @@ describe("IdentityGuard", () => { createReflectorWithIdentity({ allowApiKey: true }), groupService as unknown as GroupService, ); - const request: Record = { apiKeyGroupId: "group-123" }; + const request: Record = { + apiKey: { groupId: "group-123", keyPrefix: "aBcDeFgH" }, + }; await identityGuard.canActivate(createContext(request)); @@ -184,7 +188,9 @@ describe("IdentityGuard", () => { it("should throw ForbiddenException when @Identity is absent and request uses an API key", async () => { // Default guard has no @Identity in reflector mock - const request: Record = { apiKeyGroupId: "group-123" }; + const request: Record = { + apiKey: { groupId: "group-123", keyPrefix: "aBcDeFgH" }, + }; await expect(guard.canActivate(createContext(request))).rejects.toThrow( ForbiddenException, @@ -447,7 +453,7 @@ describe("IdentityGuard", () => { groupService as unknown as GroupService, ); const request: Record = { - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, }; await expect( @@ -769,7 +775,7 @@ describe("IdentityGuard", () => { groupService as unknown as GroupService, ); const request: Record = { - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, }; await expect( @@ -783,7 +789,7 @@ describe("IdentityGuard", () => { groupService as unknown as GroupService, ); const request: Record = { - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, }; await expect( @@ -797,7 +803,7 @@ describe("IdentityGuard", () => { groupService as unknown as GroupService, ); const request: Record = { - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, }; const result = await identityGuard.canActivate(createContext(request)); @@ -845,7 +851,7 @@ describe("IdentityGuard", () => { groupService as unknown as GroupService, ); const request: Record = { - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, params: { groupId: "group-abc" }, }; @@ -904,7 +910,7 @@ describe("IdentityGuard", () => { it("should throw ForbiddenException when @Identity is absent and request carries an API key", async () => { // Without @Identity, API key requests are always denied const request: Record = { - apiKeyGroupId: "group-abc", + apiKey: { groupId: "group-abc", keyPrefix: "aBcDeFgH" }, }; await expect(guard.canActivate(createContext(request))).rejects.toThrow( diff --git a/apps/backend-services/src/auth/identity.guard.ts b/apps/backend-services/src/auth/identity.guard.ts index db4b9289c..a3215a6ba 100644 --- a/apps/backend-services/src/auth/identity.guard.ts +++ b/apps/backend-services/src/auth/identity.guard.ts @@ -32,7 +32,7 @@ export { ROLE_ORDER }; * `resolvedIdentity.userId` is set with no DB queries. * - **API key path**: When the {@link Identity} decorator is present on the handler, * `resolvedIdentity.isSystemAdmin` is set to `false` and `resolvedIdentity.groupRoles` - * is populated using `request.apiKeyGroupId` (set by `ApiKeyAuthGuard`) as the key + * is populated using `request.apiKey.groupId` (set by `ApiKeyAuthGuard`) as the key * with a default role of `GroupRole.MEMBER`. When the decorator is absent, a base * identity object is set without enrichment. No database queries are made. * @@ -71,7 +71,7 @@ export class IdentityGuard implements CanActivate { IdentityOptions | undefined >(IDENTITY_KEY, [context.getHandler(), context.getClass()]); - if (request.apiKeyGroupId) { + if (request.apiKey) { if (identityOptions !== undefined) { // Reject API key requests unless the endpoint explicitly opts in. if (!identityOptions.allowApiKey) { @@ -83,7 +83,7 @@ export class IdentityGuard implements CanActivate { // No database queries required; the key is group-scoped. request.resolvedIdentity = { isSystemAdmin: false, - groupRoles: { [request.apiKeyGroupId]: GroupRole.MEMBER }, + groupRoles: { [request.apiKey.groupId]: GroupRole.MEMBER }, }; } else { // Api-key was not explicity allowed. It it denied by default. diff --git a/apps/backend-services/src/auth/types.ts b/apps/backend-services/src/auth/types.ts index b74cb1f67..0535148a5 100644 --- a/apps/backend-services/src/auth/types.ts +++ b/apps/backend-services/src/auth/types.ts @@ -30,11 +30,21 @@ export interface ResolvedIdentity { groupRoles?: Record; } +/** + * Shape returned by ApiKeyService.validateApiKey and attached to the request + * by ApiKeyAuthGuard. Mirrors the optional `user` property so both auth + * paths expose their credential via a single object. + */ +export interface ValidatedApiKey { + groupId: string; + keyPrefix: string; +} + declare module "express" { interface Request { user?: User; /** Set by ApiKeyAuthGuard when a valid API key is used. */ - apiKeyGroupId?: string; + apiKey?: ValidatedApiKey; /** * Set by IdentityGuard after authentication succeeds. * Contains the normalised requestor identity for downstream authorization. diff --git a/apps/backend-services/src/logging/app-logger.service.spec.ts b/apps/backend-services/src/logging/app-logger.service.spec.ts index 1b1046cd7..ea2ff3629 100644 --- a/apps/backend-services/src/logging/app-logger.service.spec.ts +++ b/apps/backend-services/src/logging/app-logger.service.spec.ts @@ -121,6 +121,77 @@ describe("AppLoggerService", () => { }); }); + describe("with sessionId in request context", () => { + beforeEach(() => + mockGetRequestContext.mockReturnValue({ + requestId: "req-1", + userId: "user-1", + sessionId: "session-abc-123", + }), + ); + + it("includes sessionId in merged context", () => { + service.log("test message"); + expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", { + requestId: "req-1", + userId: "user-1", + sessionId: "session-abc-123", + }); + }); + }); + + describe("with clientIp in request context", () => { + beforeEach(() => + mockGetRequestContext.mockReturnValue({ + requestId: "req-3", + clientIp: "203.0.113.50", + }), + ); + + it("includes clientIp in merged context", () => { + service.log("test message"); + expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", { + requestId: "req-3", + clientIp: "203.0.113.50", + }); + }); + }); + + describe("with apiKeyId in request context", () => { + beforeEach(() => + mockGetRequestContext.mockReturnValue({ + requestId: "req-5", + apiKeyId: "aBcDeFgH", + }), + ); + + it("includes apiKeyId in merged context", () => { + service.log("test message"); + expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", { + requestId: "req-5", + apiKeyId: "aBcDeFgH", + }); + }); + }); + + describe("omits falsy optional fields", () => { + beforeEach(() => + mockGetRequestContext.mockReturnValue({ + requestId: "req-6", + sessionId: undefined, + apiKeyId: undefined, + clientIp: undefined, + }), + ); + + it("does not include undefined optional fields", () => { + service.log("test message"); + expect(mockLoggerMethods.info).toHaveBeenCalledWith("test message", { + requestId: "req-6", + }); + }); + }); + describe("static getLogLevel", () => { it("exposes the getLogLevel function", () => { expect(AppLoggerService.getLogLevel).toBeDefined(); diff --git a/apps/backend-services/src/logging/app-logger.service.ts b/apps/backend-services/src/logging/app-logger.service.ts index 111a38ac7..5c71da167 100644 --- a/apps/backend-services/src/logging/app-logger.service.ts +++ b/apps/backend-services/src/logging/app-logger.service.ts @@ -18,6 +18,9 @@ export class AppLoggerService { return { ...(ctx?.requestId && { requestId: ctx.requestId }), ...(ctx?.userId && { userId: ctx.userId }), + ...(ctx?.sessionId && { sessionId: ctx.sessionId }), + ...(ctx?.apiKeyId && { apiKeyId: ctx.apiKeyId }), + ...(ctx?.clientIp && { clientIp: ctx.clientIp }), ...context, }; } diff --git a/apps/backend-services/src/logging/logging.middleware.spec.ts b/apps/backend-services/src/logging/logging.middleware.spec.ts index 68e9fa8da..1554f4397 100644 --- a/apps/backend-services/src/logging/logging.middleware.spec.ts +++ b/apps/backend-services/src/logging/logging.middleware.spec.ts @@ -1,4 +1,5 @@ import type { NextFunction, Request, Response } from "express"; +import type { Socket } from "net"; import { AppLoggerService } from "./app-logger.service"; import { LoggingMiddleware } from "./logging.middleware"; import { requestContext } from "./request-context"; @@ -23,7 +24,6 @@ const mockLogger = { describe("LoggingMiddleware", () => { let middleware: LoggingMiddleware; - let mockReq: Partial; let mockRes: Partial; let mockNext: NextFunction; @@ -31,22 +31,33 @@ describe("LoggingMiddleware", () => { jest.clearAllMocks(); middleware = new LoggingMiddleware(mockLogger); - mockReq = { headers: {} } as Partial; mockRes = { setHeader: jest.fn() } as Partial; mockNext = jest.fn(); mockRun.mockImplementation((_store, callback) => callback()); }); + function createMockRequest( + headers: Record = {}, + remoteAddress?: string, + ): Request { + return { + headers, + socket: { remoteAddress } as Socket, + } as unknown as Request; + } + it("sets a UUID request-id header on the request", () => { - middleware.use(mockReq as Request, mockRes as Response, mockNext); - const requestId = mockReq.headers?.["x-request-id"]; + const req = createMockRequest({}, "127.0.0.1"); + middleware.use(req, mockRes as Response, mockNext); + const requestId = req.headers["x-request-id"]; expect(typeof requestId).toBe("string"); expect((requestId as string).length).toBeGreaterThan(0); }); it("sets the x-request-id header on the response", () => { - middleware.use(mockReq as Request, mockRes as Response, mockNext); + const req = createMockRequest({}, "127.0.0.1"); + middleware.use(req, mockRes as Response, mockNext); expect(mockRes.setHeader).toHaveBeenCalledWith( "x-request-id", expect.any(String), @@ -54,7 +65,8 @@ describe("LoggingMiddleware", () => { }); it("runs the context store and calls next", () => { - middleware.use(mockReq as Request, mockRes as Response, mockNext); + const req = createMockRequest({}, "127.0.0.1"); + middleware.use(req, mockRes as Response, mockNext); expect(mockRun).toHaveBeenCalledWith( expect.objectContaining({ requestId: expect.any(String) }), expect.any(Function), @@ -63,8 +75,9 @@ describe("LoggingMiddleware", () => { }); it("assigns the same requestId to request header and context store", () => { - middleware.use(mockReq as Request, mockRes as Response, mockNext); - const headerRequestId = mockReq.headers?.["x-request-id"]; + const req = createMockRequest({}, "127.0.0.1"); + middleware.use(req, mockRes as Response, mockNext); + const headerRequestId = req.headers["x-request-id"]; const storeArg = mockRun.mock.calls[0][0]; expect(storeArg.requestId).toBe(headerRequestId); }); @@ -72,10 +85,76 @@ describe("LoggingMiddleware", () => { it("generates unique requestIds across calls", () => { const ids = new Set(); for (let i = 0; i < 5; i++) { - const req = { headers: {} } as unknown as Request; + const req = createMockRequest({}, "127.0.0.1"); middleware.use(req, mockRes as Response, mockNext); ids.add(req.headers["x-request-id"] as string); } expect(ids.size).toBe(5); }); + + describe("clientIp extraction", () => { + it("extracts clientIp from X-Forwarded-For header (first IP)", () => { + const req = createMockRequest( + { "x-forwarded-for": "203.0.113.50, 70.41.3.18, 150.172.238.178" }, + "127.0.0.1", + ); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBe("203.0.113.50"); + }); + + it("trims whitespace from X-Forwarded-For first IP", () => { + const req = createMockRequest( + { "x-forwarded-for": " 10.0.0.1 , 10.0.0.2" }, + "127.0.0.1", + ); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBe("10.0.0.1"); + }); + + it("extracts clientIp from single-entry X-Forwarded-For header", () => { + const req = createMockRequest( + { "x-forwarded-for": "192.168.1.1" }, + "127.0.0.1", + ); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBe("192.168.1.1"); + }); + + it("falls back to X-Real-IP when X-Forwarded-For is absent", () => { + const req = createMockRequest({ "x-real-ip": "10.0.0.5" }, "127.0.0.1"); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBe("10.0.0.5"); + }); + + it("falls back to socket remoteAddress when no proxy headers present", () => { + const req = createMockRequest({}, "::1"); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBe("::1"); + }); + + it("prefers X-Forwarded-For over X-Real-IP", () => { + const req = createMockRequest( + { + "x-forwarded-for": "203.0.113.50", + "x-real-ip": "10.0.0.5", + }, + "127.0.0.1", + ); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBe("203.0.113.50"); + }); + + it("sets clientIp to undefined when no headers and no socket address", () => { + const req = createMockRequest({}, undefined); + middleware.use(req, mockRes as Response, mockNext); + const storeArg = mockRun.mock.calls[0][0]; + expect(storeArg.clientIp).toBeUndefined(); + }); + }); }); diff --git a/apps/backend-services/src/logging/logging.middleware.ts b/apps/backend-services/src/logging/logging.middleware.ts index caa0f3e83..470b5732f 100644 --- a/apps/backend-services/src/logging/logging.middleware.ts +++ b/apps/backend-services/src/logging/logging.middleware.ts @@ -16,9 +16,25 @@ export class LoggingMiddleware implements NestMiddleware { req.headers[REQUEST_ID_HEADER] = requestId; res.setHeader(REQUEST_ID_HEADER, requestId); - const store = { requestId }; + const clientIp = this.extractClientIp(req); + + const store = { requestId, clientIp }; requestContext.run(store, () => { next(); }); } + + private extractClientIp(req: Request): string | undefined { + const xForwardedFor = req.headers["x-forwarded-for"]; + if (typeof xForwardedFor === "string" && xForwardedFor) { + return xForwardedFor.split(",")[0].trim(); + } + + const xRealIp = req.headers["x-real-ip"]; + if (typeof xRealIp === "string" && xRealIp) { + return xRealIp; + } + + return req.socket.remoteAddress; + } } diff --git a/apps/backend-services/src/logging/request-context.ts b/apps/backend-services/src/logging/request-context.ts index 852ad67a3..602917a4f 100644 --- a/apps/backend-services/src/logging/request-context.ts +++ b/apps/backend-services/src/logging/request-context.ts @@ -3,6 +3,9 @@ import { AsyncLocalStorage } from "async_hooks"; export interface RequestContextData { requestId: string; userId?: string; + sessionId?: string; + apiKeyId?: string; + clientIp?: string; } export const requestContext = new AsyncLocalStorage(); diff --git a/apps/backend-services/src/logging/request-logging.interceptor.spec.ts b/apps/backend-services/src/logging/request-logging.interceptor.spec.ts index a509c9cbe..266412e7a 100644 --- a/apps/backend-services/src/logging/request-logging.interceptor.spec.ts +++ b/apps/backend-services/src/logging/request-logging.interceptor.spec.ts @@ -57,49 +57,142 @@ describe("RequestLoggingInterceptor", () => { expect(typeof req._loggingStartTime).toBe("number"); }); - it("does not set userId when store is null", () => { - mockGetStore.mockReturnValue(undefined); - const req = makeRequest({ - resolvedIdentity: { userId: "u-1" }, - } as Partial); - const ctx = makeContext(req); - const next: CallHandler = { handle: () => of(undefined) }; - // expect no error - expect(() => interceptor.intercept(ctx, next).subscribe()).not.toThrow(); - }); + describe("userId enrichment", () => { + it("does not set userId when store is null", () => { + mockGetStore.mockReturnValue(undefined); + const req = makeRequest({ + resolvedIdentity: { userId: "u-1" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + expect(() => interceptor.intercept(ctx, next).subscribe()).not.toThrow(); + }); - it("does not set userId when resolvedIdentity is absent", () => { - const store = { requestId: "req-1" }; - mockGetStore.mockReturnValue(store); - const req = makeRequest(); - const ctx = makeContext(req); - const next: CallHandler = { handle: () => of(undefined) }; - interceptor.intercept(ctx, next).subscribe(); - expect(store).not.toHaveProperty("userId"); + it("does not set userId when resolvedIdentity is absent", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest(); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).not.toHaveProperty("userId"); + }); + + it("sets userId from resolvedIdentity.userId when present", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + resolvedIdentity: { userId: "user-abc" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).toHaveProperty("userId", "user-abc"); + }); + + it("does not set userId when resolvedIdentity lacks userId key", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + resolvedIdentity: { groupIds: [] }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).not.toHaveProperty("userId"); + }); }); - it("sets userId from resolvedIdentity.userId when present", () => { - const store = { requestId: "req-1" }; - mockGetStore.mockReturnValue(store); - const req = makeRequest({ - resolvedIdentity: { userId: "user-abc" }, - } as Partial); - const ctx = makeContext(req); - const next: CallHandler = { handle: () => of(undefined) }; - interceptor.intercept(ctx, next).subscribe(); - expect(store).toHaveProperty("userId", "user-abc"); + describe("sessionId enrichment", () => { + it("sets sessionId from user.session_state when present", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + user: { sub: "user-1", session_state: "keycloak-session-uuid-123" }, + resolvedIdentity: { userId: "user-1" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).toHaveProperty("sessionId", "keycloak-session-uuid-123"); + }); + + it("does not set sessionId when user has no session_state", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + user: { sub: "user-1" }, + resolvedIdentity: { userId: "user-1" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).not.toHaveProperty("sessionId"); + }); + + it("does not set sessionId when user is undefined", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest(); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).not.toHaveProperty("sessionId"); + }); + + it("does not set sessionId when session_state is an empty string", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + user: { sub: "user-1", session_state: "" }, + resolvedIdentity: { userId: "user-1" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).not.toHaveProperty("sessionId"); + }); }); - it("does not set userId when resolvedIdentity lacks userId key", () => { - const store = { requestId: "req-1" }; - mockGetStore.mockReturnValue(store); - const req = makeRequest({ - resolvedIdentity: { groupIds: [] }, - } as Partial); - const ctx = makeContext(req); - const next: CallHandler = { handle: () => of(undefined) }; - interceptor.intercept(ctx, next).subscribe(); - expect(store).not.toHaveProperty("userId"); + describe("apiKeyId enrichment", () => { + it("sets apiKeyId from apiKey.keyPrefix when present", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + apiKey: { groupId: "group-123", keyPrefix: "aBcDeFgH" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).toHaveProperty("apiKeyId", "aBcDeFgH"); + }); + + it("omits sessionId when request is authenticated via API key", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + apiKey: { groupId: "group-123", keyPrefix: "aBcDeFgH" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).toHaveProperty("apiKeyId", "aBcDeFgH"); + expect(store).not.toHaveProperty("sessionId"); + }); + + it("omits apiKeyId when request is authenticated via JWT", () => { + const store = { requestId: "req-1" }; + mockGetStore.mockReturnValue(store); + const req = makeRequest({ + user: { sub: "user-1", session_state: "keycloak-session-uuid-456" }, + resolvedIdentity: { userId: "user-1" }, + } as Partial); + const ctx = makeContext(req); + const next: CallHandler = { handle: () => of(undefined) }; + interceptor.intercept(ctx, next).subscribe(); + expect(store).toHaveProperty("sessionId", "keycloak-session-uuid-456"); + expect(store).not.toHaveProperty("apiKeyId"); + }); }); describe("logRequest", () => { diff --git a/apps/backend-services/src/logging/request-logging.interceptor.ts b/apps/backend-services/src/logging/request-logging.interceptor.ts index 1ba7b21c9..95558315d 100644 --- a/apps/backend-services/src/logging/request-logging.interceptor.ts +++ b/apps/backend-services/src/logging/request-logging.interceptor.ts @@ -36,6 +36,15 @@ export class RequestLoggingInterceptor implements NestInterceptor { if (userId) store.userId = userId; } + if (store && request.apiKey) { + store.apiKeyId = request.apiKey.keyPrefix; + } else if (store && request.user) { + const sessionState = request.user.session_state; + if (typeof sessionState === "string" && sessionState) { + store.sessionId = sessionState; + } + } + return next.handle().pipe( tap({ next: () => this.logRequest(request, context.getType()), diff --git a/apps/backend-services/src/metrics/metrics.controller.spec.ts b/apps/backend-services/src/metrics/metrics.controller.spec.ts new file mode 100644 index 000000000..4b6478849 --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.controller.spec.ts @@ -0,0 +1,87 @@ +import { ForbiddenException } from "@nestjs/common"; +import { Test } from "@nestjs/testing"; +import type { Request, Response } from "express"; +import { MetricsController } from "./metrics.controller"; +import { MetricsService } from "./metrics.service"; + +describe("MetricsController", () => { + let controller: MetricsController; + let metricsService: MetricsService; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + controllers: [MetricsController], + providers: [MetricsService], + }).compile(); + + controller = module.get(MetricsController); + metricsService = module.get(MetricsService); + metricsService.onModuleInit(); + }); + + function createMockRequest( + headers: Record = {}, + ): Request { + return { headers } as unknown as Request; + } + + function createMockResponse(): Response { + const res = { + set: jest.fn().mockReturnThis(), + send: jest.fn().mockReturnThis(), + }; + return res as unknown as Response; + } + + it("should be defined", () => { + expect(controller).toBeDefined(); + }); + + it("should return metrics for in-cluster requests (no X-Forwarded-Host)", async () => { + const req = createMockRequest({}); + const res = createMockResponse(); + + await controller.getMetrics(req, res); + + expect(res.set).toHaveBeenCalledWith( + "Content-Type", + expect.stringContaining("text/plain"), + ); + expect(res.send).toHaveBeenCalledWith( + expect.stringContaining("http_requests_total"), + ); + }); + + it("should throw ForbiddenException when X-Forwarded-Host is present", async () => { + const req = createMockRequest({ + "x-forwarded-host": "example.apps.openshift.com", + }); + const res = createMockResponse(); + + await expect(controller.getMetrics(req, res)).rejects.toThrow( + ForbiddenException, + ); + }); + + it("should include Node.js runtime metrics", async () => { + const req = createMockRequest({}); + const res = createMockResponse(); + + await controller.getMetrics(req, res); + + const metricsOutput = (res.send as jest.Mock).mock.calls[0][0] as string; + expect(metricsOutput).toContain("nodejs_"); + }); + + it("should include RED metric definitions in output", async () => { + const req = createMockRequest({}); + const res = createMockResponse(); + + await controller.getMetrics(req, res); + + const metricsOutput = (res.send as jest.Mock).mock.calls[0][0] as string; + expect(metricsOutput).toContain("http_requests_total"); + expect(metricsOutput).toContain("http_request_duration_seconds"); + expect(metricsOutput).toContain("http_request_errors_total"); + }); +}); diff --git a/apps/backend-services/src/metrics/metrics.controller.ts b/apps/backend-services/src/metrics/metrics.controller.ts new file mode 100644 index 000000000..dbaec9996 --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.controller.ts @@ -0,0 +1,29 @@ +import { Controller, ForbiddenException, Get, Req, Res } from "@nestjs/common"; +import { ApiExcludeController } from "@nestjs/swagger"; +import type { Request, Response } from "express"; +import { Public } from "@/auth/public.decorator"; +import { MetricsService } from "./metrics.service"; + +@ApiExcludeController() +@Controller() +export class MetricsController { + constructor(private readonly metricsService: MetricsService) {} + + @Public() + @Get("metrics") + async getMetrics(@Req() req: Request, @Res() res: Response): Promise { + // Block external access: when the request arrives via the OpenShift Route, + // the router injects X-Forwarded-Host. In-cluster Prometheus scrapes + // directly via the Service, so this header is absent. + const forwardedHost = req.headers["x-forwarded-host"]; + if (forwardedHost) { + throw new ForbiddenException( + "Metrics endpoint is not accessible externally", + ); + } + + const metrics = await this.metricsService.getMetrics(); + res.set("Content-Type", this.metricsService.getContentType()); + res.send(metrics); + } +} diff --git a/apps/backend-services/src/metrics/metrics.middleware.spec.ts b/apps/backend-services/src/metrics/metrics.middleware.spec.ts new file mode 100644 index 000000000..4d76148bb --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.middleware.spec.ts @@ -0,0 +1,205 @@ +import type { Request, Response } from "express"; +import { MetricsMiddleware } from "./metrics.middleware"; +import { MetricsService } from "./metrics.service"; + +describe("MetricsMiddleware", () => { + let middleware: MetricsMiddleware; + let metricsService: MetricsService; + + beforeEach(() => { + metricsService = new MetricsService(); + middleware = new MetricsMiddleware(metricsService); + }); + + function createMockRequest(overrides: Partial = {}): Request { + return { + method: "GET", + path: "/test", + route: undefined, + ...overrides, + } as unknown as Request; + } + + function createMockResponse(): Response & { + triggerFinish: () => void; + } { + const listeners: Record void>> = {}; + return { + statusCode: 200, + on(event: string, callback: () => void) { + if (!listeners[event]) { + listeners[event] = []; + } + listeners[event].push(callback); + return this; + }, + triggerFinish() { + for (const cb of listeners["finish"] ?? []) { + cb(); + } + }, + } as unknown as Response & { triggerFinish: () => void }; + } + + it("should be defined", () => { + expect(middleware).toBeDefined(); + }); + + it("should call next for non-metrics paths", () => { + const req = createMockRequest(); + const res = createMockResponse(); + const next = jest.fn(); + + middleware.use(req, res, next); + + expect(next).toHaveBeenCalled(); + }); + + it("should skip metrics collection for /metrics path", () => { + const req = createMockRequest({ path: "/metrics" }); + const res = createMockResponse(); + const next = jest.fn(); + + const incSpy = jest.spyOn(metricsService.httpRequestsTotal, "inc"); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(next).toHaveBeenCalled(); + expect(incSpy).not.toHaveBeenCalled(); + }); + + it("should increment httpRequestsTotal on response finish", () => { + const req = createMockRequest({ method: "POST", path: "/api/data" }); + const res = createMockResponse(); + res.statusCode = 201; + const next = jest.fn(); + + const incSpy = jest.spyOn(metricsService.httpRequestsTotal, "inc"); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(incSpy).toHaveBeenCalledWith({ + method: "POST", + path: "/api/data", + status_code: "201", + }); + }); + + it("should observe duration in httpRequestDurationSeconds on response finish", () => { + const req = createMockRequest(); + const res = createMockResponse(); + const next = jest.fn(); + + const observeSpy = jest.spyOn( + metricsService.httpRequestDurationSeconds, + "observe", + ); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(observeSpy).toHaveBeenCalledWith( + { method: "GET", path: "/test" }, + expect.any(Number), + ); + }); + + it("should increment httpRequestErrorsTotal for 4xx status codes", () => { + const req = createMockRequest(); + const res = createMockResponse(); + res.statusCode = 404; + const next = jest.fn(); + + const errorIncSpy = jest.spyOn( + metricsService.httpRequestErrorsTotal, + "inc", + ); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(errorIncSpy).toHaveBeenCalledWith({ + method: "GET", + path: "/test", + status_code: "404", + }); + }); + + it("should increment httpRequestErrorsTotal for 5xx status codes", () => { + const req = createMockRequest(); + const res = createMockResponse(); + res.statusCode = 500; + const next = jest.fn(); + + const errorIncSpy = jest.spyOn( + metricsService.httpRequestErrorsTotal, + "inc", + ); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(errorIncSpy).toHaveBeenCalledWith({ + method: "GET", + path: "/test", + status_code: "500", + }); + }); + + it("should NOT increment httpRequestErrorsTotal for 2xx status codes", () => { + const req = createMockRequest(); + const res = createMockResponse(); + res.statusCode = 200; + const next = jest.fn(); + + const errorIncSpy = jest.spyOn( + metricsService.httpRequestErrorsTotal, + "inc", + ); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(errorIncSpy).not.toHaveBeenCalled(); + }); + + it("should NOT increment httpRequestErrorsTotal for 3xx status codes", () => { + const req = createMockRequest(); + const res = createMockResponse(); + res.statusCode = 302; + const next = jest.fn(); + + const errorIncSpy = jest.spyOn( + metricsService.httpRequestErrorsTotal, + "inc", + ); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(errorIncSpy).not.toHaveBeenCalled(); + }); + + it("should use route path when available instead of raw path", () => { + const req = createMockRequest({ + method: "GET", + path: "/api/documents/123", + route: { path: "/api/documents/:id" } as unknown as Request["route"], + }); + const res = createMockResponse(); + const next = jest.fn(); + + const incSpy = jest.spyOn(metricsService.httpRequestsTotal, "inc"); + + middleware.use(req, res, next); + res.triggerFinish(); + + expect(incSpy).toHaveBeenCalledWith({ + method: "GET", + path: "/api/documents/:id", + status_code: "200", + }); + }); +}); diff --git a/apps/backend-services/src/metrics/metrics.middleware.ts b/apps/backend-services/src/metrics/metrics.middleware.ts new file mode 100644 index 000000000..c269f8932 --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.middleware.ts @@ -0,0 +1,50 @@ +import type { NestMiddleware } from "@nestjs/common"; +import { Injectable } from "@nestjs/common"; +import type { NextFunction, Request, Response } from "express"; +import { MetricsService } from "./metrics.service"; + +const METRICS_PATH = "/metrics"; + +@Injectable() +export class MetricsMiddleware implements NestMiddleware { + constructor(private readonly metricsService: MetricsService) {} + + use(req: Request, res: Response, next: NextFunction): void { + // Exclude the /metrics endpoint itself from being counted + if (req.path === METRICS_PATH) { + next(); + return; + } + + const startTime = process.hrtime.bigint(); + + res.on("finish", () => { + const durationNs = Number(process.hrtime.bigint() - startTime); + const durationSeconds = durationNs / 1e9; + const method = req.method; + const path = req.route?.path ?? req.path; + const statusCode = res.statusCode.toString(); + + this.metricsService.httpRequestsTotal.inc({ + method, + path, + status_code: statusCode, + }); + + this.metricsService.httpRequestDurationSeconds.observe( + { method, path }, + durationSeconds, + ); + + if (res.statusCode >= 400) { + this.metricsService.httpRequestErrorsTotal.inc({ + method, + path, + status_code: statusCode, + }); + } + }); + + next(); + } +} diff --git a/apps/backend-services/src/metrics/metrics.module.ts b/apps/backend-services/src/metrics/metrics.module.ts new file mode 100644 index 000000000..5cacace95 --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.module.ts @@ -0,0 +1,19 @@ +import { + type MiddlewareConsumer, + Module, + type NestModule, +} from "@nestjs/common"; +import { MetricsController } from "./metrics.controller"; +import { MetricsMiddleware } from "./metrics.middleware"; +import { MetricsService } from "./metrics.service"; + +@Module({ + controllers: [MetricsController], + providers: [MetricsService, MetricsMiddleware], + exports: [MetricsService], +}) +export class MetricsModule implements NestModule { + configure(consumer: MiddlewareConsumer): void { + consumer.apply(MetricsMiddleware).forRoutes("*"); + } +} diff --git a/apps/backend-services/src/metrics/metrics.service.spec.ts b/apps/backend-services/src/metrics/metrics.service.spec.ts new file mode 100644 index 000000000..27c350aad --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.service.spec.ts @@ -0,0 +1,94 @@ +import { Test } from "@nestjs/testing"; +import { MetricsService } from "./metrics.service"; + +describe("MetricsService", () => { + let service: MetricsService; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [MetricsService], + }).compile(); + + service = module.get(MetricsService); + service.onModuleInit(); + }); + + it("should be defined", () => { + expect(service).toBeDefined(); + }); + + describe("getMetrics", () => { + it("should return metrics string including default Node.js metrics", async () => { + const metrics = await service.getMetrics(); + expect(typeof metrics).toBe("string"); + // Default metrics include process and nodejs prefixed metrics + expect(metrics).toContain("nodejs_"); + }); + + it("should include http_requests_total metric definition", async () => { + const metrics = await service.getMetrics(); + expect(metrics).toContain("http_requests_total"); + }); + + it("should include http_request_duration_seconds metric definition", async () => { + const metrics = await service.getMetrics(); + expect(metrics).toContain("http_request_duration_seconds"); + }); + + it("should include http_request_errors_total metric definition", async () => { + const metrics = await service.getMetrics(); + expect(metrics).toContain("http_request_errors_total"); + }); + }); + + describe("getContentType", () => { + it("should return a valid content type", () => { + const contentType = service.getContentType(); + expect(contentType).toContain("text/plain"); + }); + }); + + describe("httpRequestsTotal", () => { + it("should increment counter with labels", async () => { + service.httpRequestsTotal.inc({ + method: "GET", + path: "/test", + status_code: "200", + }); + + const metrics = await service.getMetrics(); + expect(metrics).toContain( + 'http_requests_total{method="GET",path="/test",status_code="200"} 1', + ); + }); + }); + + describe("httpRequestErrorsTotal", () => { + it("should increment error counter with labels", async () => { + service.httpRequestErrorsTotal.inc({ + method: "POST", + path: "/fail", + status_code: "500", + }); + + const metrics = await service.getMetrics(); + expect(metrics).toContain( + 'http_request_errors_total{method="POST",path="/fail",status_code="500"} 1', + ); + }); + }); + + describe("httpRequestDurationSeconds", () => { + it("should observe duration with labels", async () => { + service.httpRequestDurationSeconds.observe( + { method: "GET", path: "/test" }, + 0.5, + ); + + const metrics = await service.getMetrics(); + expect(metrics).toContain("http_request_duration_seconds_bucket"); + expect(metrics).toContain("http_request_duration_seconds_count"); + expect(metrics).toContain("http_request_duration_seconds_sum"); + }); + }); +}); diff --git a/apps/backend-services/src/metrics/metrics.service.ts b/apps/backend-services/src/metrics/metrics.service.ts new file mode 100644 index 000000000..eb4a311a1 --- /dev/null +++ b/apps/backend-services/src/metrics/metrics.service.ts @@ -0,0 +1,53 @@ +import { Injectable, type OnModuleInit } from "@nestjs/common"; +import { + Counter, + collectDefaultMetrics, + Histogram, + Registry, +} from "prom-client"; + +@Injectable() +export class MetricsService implements OnModuleInit { + private readonly registry: Registry; + readonly httpRequestsTotal: Counter; + readonly httpRequestErrorsTotal: Counter; + readonly httpRequestDurationSeconds: Histogram; + + constructor() { + this.registry = new Registry(); + + this.httpRequestsTotal = new Counter({ + name: "http_requests_total", + help: "Total number of HTTP requests", + labelNames: ["method", "path", "status_code"] as const, + registers: [this.registry], + }); + + this.httpRequestErrorsTotal = new Counter({ + name: "http_request_errors_total", + help: "Total number of HTTP requests resulting in 4xx or 5xx status codes", + labelNames: ["method", "path", "status_code"] as const, + registers: [this.registry], + }); + + this.httpRequestDurationSeconds = new Histogram({ + name: "http_request_duration_seconds", + help: "Duration of HTTP requests in seconds", + labelNames: ["method", "path"] as const, + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], + registers: [this.registry], + }); + } + + onModuleInit(): void { + collectDefaultMetrics({ register: this.registry }); + } + + async getMetrics(): Promise { + return this.registry.metrics(); + } + + getContentType(): string { + return this.registry.contentType; + } +} diff --git a/deployments/local/docker-compose.monitoring.yml b/deployments/local/docker-compose.monitoring.yml new file mode 100644 index 000000000..5f2e1bde0 --- /dev/null +++ b/deployments/local/docker-compose.monitoring.yml @@ -0,0 +1,73 @@ +services: + loki: + image: grafana/loki:3.4.0 + container_name: ai-doc-intelligence-loki + restart: unless-stopped + command: -config.file=/etc/loki/loki.yaml + volumes: + - ./loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + ports: + - "3100:3100" + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + promtail: + image: grafana/promtail:3.4.0 + container_name: ai-doc-intelligence-promtail + restart: unless-stopped + command: -config.file=/etc/promtail/promtail-config.yml + volumes: + - ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + loki: + condition: service_healthy + + prometheus: + image: prom/prometheus:v3.2.1 + container_name: ai-doc-intelligence-prometheus + restart: unless-stopped + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=15d" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + ports: + - "9090:9090" + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9090/-/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 10s + + grafana: + image: grafana/grafana:11.5.2 + container_name: ai-doc-intelligence-grafana + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_AUTH_ANONYMOUS_ENABLED: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + ports: + - "3001:3000" + depends_on: + loki: + condition: service_healthy + prometheus: + condition: service_healthy + +volumes: + loki_data: + prometheus_data: diff --git a/deployments/local/grafana/provisioning/datasources/datasources.yml b/deployments/local/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 000000000..977265ed8 --- /dev/null +++ b/deployments/local/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,14 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: false + editable: false diff --git a/deployments/local/loki/loki.yaml b/deployments/local/loki/loki.yaml new file mode 100644 index 000000000..52caa2b7a --- /dev/null +++ b/deployments/local/loki/loki.yaml @@ -0,0 +1,40 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 720h + allow_structured_metadata: true + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/deployments/local/prometheus/prometheus.yml b/deployments/local/prometheus/prometheus.yml new file mode 100644 index 000000000..e88d0f31e --- /dev/null +++ b/deployments/local/prometheus/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "backend-services" + metrics_path: "/metrics" + scrape_interval: 15s + static_configs: + - targets: + - "host.docker.internal:3002" + + - job_name: "temporal-server" + metrics_path: "/metrics" + scrape_interval: 15s + static_configs: + - targets: + - "temporal:9090" diff --git a/deployments/local/promtail/promtail-config.yml b/deployments/local/promtail/promtail-config.yml new file mode 100644 index 000000000..6f72b7ef9 --- /dev/null +++ b/deployments/local/promtail/promtail-config.yml @@ -0,0 +1,26 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + # Use the container name as the "container" label + - source_labels: ["__meta_docker_container_name"] + regex: "/(.*)" + target_label: "container" + # Use the compose service name as the "service" label + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + target_label: "service" + # Use the compose project as the "project" label + - source_labels: ["__meta_docker_container_label_com_docker_compose_project"] + target_label: "project" diff --git a/deployments/openshift/config/dev.env.example b/deployments/openshift/config/dev.env.example index d75489a8c..cf181747f 100644 --- a/deployments/openshift/config/dev.env.example +++ b/deployments/openshift/config/dev.env.example @@ -115,3 +115,17 @@ THROTTLE_AUTH_TTL_MS=60000 THROTTLE_AUTH_LIMIT=10 THROTTLE_AUTH_REFRESH_TTL_MS=60000 THROTTLE_AUTH_REFRESH_LIMIT=5 + +# ----------------------------------------------------------------------------- +# PLG Monitoring Stack (Prometheus, Loki, Grafana) +# ----------------------------------------------------------------------------- +# Grafana admin password. Change this from the default for any shared environment. +GRAFANA_ADMIN_PASSWORD=admin +# Loki log retention period in days. +LOKI_RETENTION_DAYS=30 +# Loki PVC storage size. +LOKI_PVC_SIZE=10Gi +# Prometheus PVC storage size. +PROMETHEUS_PVC_SIZE=10Gi +# Prometheus scrape interval for all targets. +METRICS_SCRAPE_INTERVAL=15s diff --git a/deployments/openshift/config/prod.env.example b/deployments/openshift/config/prod.env.example index 7a24aea78..9b28ad645 100644 --- a/deployments/openshift/config/prod.env.example +++ b/deployments/openshift/config/prod.env.example @@ -115,3 +115,17 @@ THROTTLE_AUTH_TTL_MS=60000 THROTTLE_AUTH_LIMIT=5 THROTTLE_AUTH_REFRESH_TTL_MS=60000 THROTTLE_AUTH_REFRESH_LIMIT=3 + +# ----------------------------------------------------------------------------- +# PLG Monitoring Stack (Prometheus, Loki, Grafana) +# ----------------------------------------------------------------------------- +# Grafana admin password. Use a strong password in production. +GRAFANA_ADMIN_PASSWORD=change-me-in-production +# Loki log retention period in days. +LOKI_RETENTION_DAYS=30 +# Loki PVC storage size. +LOKI_PVC_SIZE=10Gi +# Prometheus PVC storage size. +PROMETHEUS_PVC_SIZE=10Gi +# Prometheus scrape interval for all targets. +METRICS_SCRAPE_INTERVAL=15s diff --git a/deployments/openshift/helm/plg/Chart.yaml b/deployments/openshift/helm/plg/Chart.yaml new file mode 100644 index 000000000..71a1fb292 --- /dev/null +++ b/deployments/openshift/helm/plg/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: plg +description: PLG (Prometheus, Loki, Grafana) observability stack +type: application +version: 0.1.0 +appVersion: "3.4.0" diff --git a/deployments/openshift/helm/plg/templates/_helpers.tpl b/deployments/openshift/helm/plg/templates/_helpers.tpl new file mode 100644 index 000000000..ede4f3619 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/_helpers.tpl @@ -0,0 +1,113 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "plg.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "plg.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "plg.labels" -}} +helm.sh/chart: {{ include "plg.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: plg +{{- end }} + +{{/* +Loki labels. +*/}} +{{- define "plg.loki.labels" -}} +{{ include "plg.labels" . }} +app.kubernetes.io/name: loki +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: log-aggregation +{{- end }} + +{{/* +Loki selector labels. +*/}} +{{- define "plg.loki.selectorLabels" -}} +app.kubernetes.io/name: loki +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Loki fullname. +*/}} +{{- define "plg.loki.fullname" -}} +{{- printf "%s-loki" (include "plg.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Convert retention days to hours for Loki configuration. +*/}} +{{- define "plg.loki.retentionPeriod" -}} +{{- printf "%dh" (mul .Values.loki.retentionDays 24) }} +{{- end }} + +{{/* +Prometheus labels. +*/}} +{{- define "plg.prometheus.labels" -}} +{{ include "plg.labels" . }} +app.kubernetes.io/name: prometheus +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: metrics +{{- end }} + +{{/* +Prometheus selector labels. +*/}} +{{- define "plg.prometheus.selectorLabels" -}} +app.kubernetes.io/name: prometheus +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Prometheus fullname. +*/}} +{{- define "plg.prometheus.fullname" -}} +{{- printf "%s-prometheus" (include "plg.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Grafana labels. +*/}} +{{- define "plg.grafana.labels" -}} +{{ include "plg.labels" . }} +app.kubernetes.io/name: grafana +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: visualization +{{- end }} + +{{/* +Grafana selector labels. +*/}} +{{- define "plg.grafana.selectorLabels" -}} +app.kubernetes.io/name: grafana +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Grafana fullname. +*/}} +{{- define "plg.grafana.fullname" -}} +{{- printf "%s-grafana" (include "plg.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/deployments/openshift/helm/plg/templates/grafana-configmap.yaml b/deployments/openshift/helm/plg/templates/grafana-configmap.yaml new file mode 100644 index 000000000..e7835fb41 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/grafana-configmap.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "plg.grafana.fullname" . }}-config + labels: + {{- include "plg.grafana.labels" . | nindent 4 }} +data: + grafana.ini: | + [server] + http_port = {{ .Values.grafana.httpPort }} + root_url = %(protocol)s://%(domain)s:%(http_port)s/ + + [security] + admin_user = {{ .Values.grafana.adminUser }} + + [analytics] + reporting_enabled = false + check_for_updates = false + + [users] + allow_sign_up = false + + [auth] + disable_login_form = false + + [paths] + provisioning = /etc/grafana/provisioning diff --git a/deployments/openshift/helm/plg/templates/grafana-datasources-configmap.yaml b/deployments/openshift/helm/plg/templates/grafana-datasources-configmap.yaml new file mode 100644 index 000000000..f6e44f2ba --- /dev/null +++ b/deployments/openshift/helm/plg/templates/grafana-datasources-configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "plg.grafana.fullname" . }}-datasources + labels: + {{- include "plg.grafana.labels" . | nindent 4 }} +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://{{ include "plg.prometheus.fullname" . }}:{{ .Values.prometheus.httpPort }} + isDefault: true + editable: false + - name: Loki + type: loki + access: proxy + url: http://{{ include "plg.loki.fullname" . }}:{{ .Values.loki.httpPort }} + isDefault: false + editable: false diff --git a/deployments/openshift/helm/plg/templates/grafana-deployment.yaml b/deployments/openshift/helm/plg/templates/grafana-deployment.yaml new file mode 100644 index 000000000..b917e69e4 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/grafana-deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "plg.grafana.fullname" . }} + labels: + {{- include "plg.grafana.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "plg.grafana.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "plg.grafana.labels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/grafana-configmap.yaml") . | sha256sum }} + checksum/datasources: {{ include (print $.Template.BasePath "/grafana-datasources-configmap.yaml") . | sha256sum }} + spec: + securityContext: + fsGroup: 472 + containers: + - name: grafana + image: "{{ .Values.grafana.image.repository }}:{{ .Values.grafana.image.tag }}" + imagePullPolicy: {{ .Values.grafana.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.grafana.httpPort }} + protocol: TCP + env: + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.grafana.adminPassword | quote }} + readinessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 30 + periodSeconds: 15 + resources: + {{- toYaml .Values.grafana.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/grafana/grafana.ini + subPath: grafana.ini + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + volumes: + - name: config + configMap: + name: {{ include "plg.grafana.fullname" . }}-config + - name: datasources + configMap: + name: {{ include "plg.grafana.fullname" . }}-datasources diff --git a/deployments/openshift/helm/plg/templates/grafana-service.yaml b/deployments/openshift/helm/plg/templates/grafana-service.yaml new file mode 100644 index 000000000..b47a4daf2 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/grafana-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "plg.grafana.fullname" . }} + labels: + {{- include "plg.grafana.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.grafana.httpPort }} + targetPort: http + protocol: TCP + selector: + {{- include "plg.grafana.selectorLabels" . | nindent 4 }} diff --git a/deployments/openshift/helm/plg/templates/loki-configmap.yaml b/deployments/openshift/helm/plg/templates/loki-configmap.yaml new file mode 100644 index 000000000..cc4fb24f4 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/loki-configmap.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "plg.loki.fullname" . }}-config + labels: + {{- include "plg.loki.labels" . | nindent 4 }} +data: + loki.yaml: | + auth_enabled: false + + server: + http_listen_port: {{ .Values.loki.httpPort }} + + common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + + schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + limits_config: + retention_period: {{ include "plg.loki.retentionPeriod" . }} + allow_structured_metadata: true + + compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + + analytics: + reporting_enabled: false diff --git a/deployments/openshift/helm/plg/templates/loki-service.yaml b/deployments/openshift/helm/plg/templates/loki-service.yaml new file mode 100644 index 000000000..2f635b0b8 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/loki-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "plg.loki.fullname" . }} + labels: + {{- include "plg.loki.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.loki.httpPort }} + targetPort: http + protocol: TCP + selector: + {{- include "plg.loki.selectorLabels" . | nindent 4 }} diff --git a/deployments/openshift/helm/plg/templates/loki-statefulset.yaml b/deployments/openshift/helm/plg/templates/loki-statefulset.yaml new file mode 100644 index 000000000..4fbf7e57a --- /dev/null +++ b/deployments/openshift/helm/plg/templates/loki-statefulset.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "plg.loki.fullname" . }} + labels: + {{- include "plg.loki.labels" . | nindent 4 }} +spec: + replicas: 1 + serviceName: {{ include "plg.loki.fullname" . }} + selector: + matchLabels: + {{- include "plg.loki.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "plg.loki.labels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/loki-configmap.yaml") . | sha256sum }} + spec: + securityContext: + fsGroup: 10001 + containers: + - name: loki + image: "{{ .Values.loki.image.repository }}:{{ .Values.loki.image.tag }}" + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - -config.file=/etc/loki/loki.yaml + - -target=all + ports: + - name: http + containerPort: {{ .Values.loki.httpPort }} + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 15 + resources: + {{- toYaml .Values.loki.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/loki + - name: data + mountPath: /loki + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + volumes: + - name: config + configMap: + name: {{ include "plg.loki.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "plg.loki.selectorLabels" . | nindent 10 }} + spec: + accessModes: + - ReadWriteOnce + {{- if .Values.loki.storageClassName }} + storageClassName: {{ .Values.loki.storageClassName }} + {{- end }} + resources: + requests: + storage: {{ .Values.loki.pvcSize }} diff --git a/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml new file mode 100644 index 000000000..990e6de95 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/prometheus-configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "plg.prometheus.fullname" . }}-config + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +data: + prometheus.yml: | + global: + scrape_interval: {{ .Values.prometheus.scrapeInterval }} + evaluation_interval: {{ .Values.prometheus.scrapeInterval }} + + scrape_configs: + - job_name: "backend-services" + metrics_path: "/metrics" + scrape_interval: {{ .Values.prometheus.scrapeInterval }} + static_configs: + - targets: + - "{{ .Values.prometheus.scrapeTargets.backendServices.host }}:{{ .Values.prometheus.scrapeTargets.backendServices.port }}" + + - job_name: "temporal-server" + metrics_path: "/metrics" + scrape_interval: {{ .Values.prometheus.scrapeInterval }} + static_configs: + - targets: + - "{{ .Values.prometheus.scrapeTargets.temporalServer.host }}:{{ .Values.prometheus.scrapeTargets.temporalServer.port }}" diff --git a/deployments/openshift/helm/plg/templates/prometheus-service.yaml b/deployments/openshift/helm/plg/templates/prometheus-service.yaml new file mode 100644 index 000000000..ed40ac64d --- /dev/null +++ b/deployments/openshift/helm/plg/templates/prometheus-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.prometheus.httpPort }} + targetPort: http + protocol: TCP + selector: + {{- include "plg.prometheus.selectorLabels" . | nindent 4 }} diff --git a/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml b/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml new file mode 100644 index 000000000..c080a33c9 --- /dev/null +++ b/deployments/openshift/helm/plg/templates/prometheus-statefulset.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "plg.prometheus.fullname" . }} + labels: + {{- include "plg.prometheus.labels" . | nindent 4 }} +spec: + replicas: 1 + serviceName: {{ include "plg.prometheus.fullname" . }} + selector: + matchLabels: + {{- include "plg.prometheus.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "plg.prometheus.labels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/prometheus-configmap.yaml") . | sha256sum }} + spec: + securityContext: + fsGroup: 65534 + containers: + - name: prometheus + image: "{{ .Values.prometheus.image.repository }}:{{ .Values.prometheus.image.tag }}" + imagePullPolicy: {{ .Values.prometheus.image.pullPolicy }} + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time={{ .Values.prometheus.retentionDays }}d + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - name: http + containerPort: {{ .Values.prometheus.httpPort }} + protocol: TCP + readinessProbe: + httpGet: + path: /-/ready + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /-/healthy + port: http + initialDelaySeconds: 30 + periodSeconds: 15 + resources: + {{- toYaml .Values.prometheus.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + volumes: + - name: config + configMap: + name: {{ include "plg.prometheus.fullname" . }}-config + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "plg.prometheus.selectorLabels" . | nindent 10 }} + spec: + accessModes: + - ReadWriteOnce + {{- if .Values.prometheus.storageClassName }} + storageClassName: {{ .Values.prometheus.storageClassName }} + {{- end }} + resources: + requests: + storage: {{ .Values.prometheus.pvcSize }} diff --git a/deployments/openshift/helm/plg/values-local.yaml b/deployments/openshift/helm/plg/values-local.yaml new file mode 100644 index 000000000..96f5978f3 --- /dev/null +++ b/deployments/openshift/helm/plg/values-local.yaml @@ -0,0 +1,37 @@ +# PLG Stack - Local Docker Environment Overrides + +loki: + # Smaller storage for local development. + pvcSize: 2Gi + + # Shorter retention for local development. + retentionDays: 7 + + resources: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "256Mi" + cpu: "500m" + +prometheus: + # Smaller storage for local development. + pvcSize: 2Gi + + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + +grafana: + resources: + requests: + memory: "128Mi" + cpu: "125m" + limits: + memory: "256Mi" + cpu: "250m" diff --git a/deployments/openshift/helm/plg/values-openshift.yaml b/deployments/openshift/helm/plg/values-openshift.yaml new file mode 100644 index 000000000..41ab99e79 --- /dev/null +++ b/deployments/openshift/helm/plg/values-openshift.yaml @@ -0,0 +1,33 @@ +# PLG Stack - OpenShift Environment Overrides + +loki: + pvcSize: 10Gi + retentionDays: 30 + + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "256Mi" + cpu: "500m" + +prometheus: + pvcSize: 10Gi + + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "500m" + +grafana: + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "256Mi" + cpu: "250m" diff --git a/deployments/openshift/helm/plg/values.yaml b/deployments/openshift/helm/plg/values.yaml new file mode 100644 index 000000000..212d64ec4 --- /dev/null +++ b/deployments/openshift/helm/plg/values.yaml @@ -0,0 +1,102 @@ +# PLG Stack - Default Values +# Override per environment via values-local.yaml, values-openshift.yaml, or --set flags. + +loki: + image: + repository: grafana/loki + tag: "3.4.0" + pullPolicy: IfNotPresent + + # Retention period in days. Loki compactor deletes chunks older than this. + retentionDays: 30 + + # PVC size for Loki data storage. + pvcSize: 10Gi + + # Storage class for the PVC. Empty string uses the cluster default. + storageClassName: "" + + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "256Mi" + cpu: "500m" + + # Port Loki listens on. + httpPort: 3100 + + # NDJSON fields to extract from log lines via pipeline stages. + # These match the fields produced by @ai-di/shared-logging. + ndjsonFields: + - timestamp + - level + - service + - requestId + - userId + - sessionId + - clientIp + - method + - path + - statusCode + - durationMs + +prometheus: + image: + repository: prom/prometheus + tag: "v3.2.1" + pullPolicy: IfNotPresent + + # Data retention period in days. + retentionDays: 15 + + # PVC size for Prometheus TSDB storage. + pvcSize: 10Gi + + # Storage class for the PVC. Empty string uses the cluster default. + storageClassName: "" + + # Scrape interval for all targets. + scrapeInterval: "15s" + + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "500m" + + # Port Prometheus listens on. + httpPort: 9090 + + # Scrape targets reference service names within the same namespace. + scrapeTargets: + backendServices: + host: "backend-services" + port: 3002 + temporalServer: + host: "temporal" + port: 9090 + +grafana: + image: + repository: grafana/grafana + tag: "11.5.2" + pullPolicy: IfNotPresent + + # Admin credentials. Override GRAFANA_ADMIN_PASSWORD via --set or environment values. + adminUser: admin + adminPassword: "admin" + + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "256Mi" + cpu: "250m" + + # Port Grafana listens on. + httpPort: 3001 diff --git a/deployments/openshift/kustomize/base/backend-services/deployment.yml b/deployments/openshift/kustomize/base/backend-services/deployment.yml index 8668b577f..ebf67e201 100644 --- a/deployments/openshift/kustomize/base/backend-services/deployment.yml +++ b/deployments/openshift/kustomize/base/backend-services/deployment.yml @@ -288,6 +288,24 @@ spec: limits: memory: "64Mi" cpu: "50m" + - name: promtail + image: grafana/promtail:3.4.2 + args: + - -config.file=/etc/promtail/promtail.yaml + volumeMounts: + - name: logs + mountPath: /var/log/app + readOnly: true + - name: promtail-config + mountPath: /etc/promtail + readOnly: true + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "64Mi" + cpu: "100m" volumes: - name: storage persistentVolumeClaim: @@ -300,4 +318,6 @@ spec: name: backend-services-logrotate - name: logrotate-state emptyDir: {} - + - name: promtail-config + configMap: + name: backend-services-promtail diff --git a/deployments/openshift/kustomize/base/backend-services/kustomization.yml b/deployments/openshift/kustomize/base/backend-services/kustomization.yml index 9785dda0a..2c17c705d 100644 --- a/deployments/openshift/kustomize/base/backend-services/kustomization.yml +++ b/deployments/openshift/kustomize/base/backend-services/kustomization.yml @@ -9,6 +9,7 @@ resources: - pvc.yml - pvc-logs.yml - logrotate-configmap.yml + - promtail-configmap.yml - networkpolicy.yml commonLabels: app: backend-services diff --git a/deployments/openshift/kustomize/base/backend-services/promtail-configmap.yml b/deployments/openshift/kustomize/base/backend-services/promtail-configmap.yml new file mode 100644 index 000000000..3b13702c4 --- /dev/null +++ b/deployments/openshift/kustomize/base/backend-services/promtail-configmap.yml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backend-services-promtail + labels: + app: backend-services +data: + promtail.yaml: | + server: + http_listen_port: 0 + grpc_listen_port: 0 + positions: + filename: /tmp/positions.yaml + clients: + - url: http://loki:3100/loki/api/v1/push + scrape_configs: + - job_name: backend-services + static_configs: + - targets: + - localhost + labels: + service: backend-services + __path__: /var/log/app/*.log diff --git a/deployments/openshift/kustomize/base/backend-services/route.yml b/deployments/openshift/kustomize/base/backend-services/route.yml index a650d5636..12554fb81 100644 --- a/deployments/openshift/kustomize/base/backend-services/route.yml +++ b/deployments/openshift/kustomize/base/backend-services/route.yml @@ -4,6 +4,12 @@ metadata: name: backend-services labels: app: backend-services + annotations: + # Block /metrics from being accessible via the public Route. + # Prometheus scrapes /metrics via the in-cluster Service directly. + # The haproxy.router.openshift.io/deny list uses path-based ACLs + # to return 403 for matching request paths. + haproxy.router.openshift.io/deny-list: /metrics spec: to: kind: Service diff --git a/deployments/openshift/kustomize/base/crunchydb/kustomization.yml b/deployments/openshift/kustomize/base/crunchydb/kustomization.yml index 1e25e73a9..35c362060 100644 --- a/deployments/openshift/kustomize/base/crunchydb/kustomization.yml +++ b/deployments/openshift/kustomize/base/crunchydb/kustomization.yml @@ -2,5 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - postgrescluster.yml + - promtail-configmap.yml commonLabels: app.kubernetes.io/part-of: crunchydb \ No newline at end of file diff --git a/deployments/openshift/kustomize/base/crunchydb/postgrescluster.yml b/deployments/openshift/kustomize/base/crunchydb/postgrescluster.yml index 7e3b02257..b327de401 100644 --- a/deployments/openshift/kustomize/base/crunchydb/postgrescluster.yml +++ b/deployments/openshift/kustomize/base/crunchydb/postgrescluster.yml @@ -22,6 +22,28 @@ spec: requests: cpu: '5m' memory: '10Mi' + containers: + - name: promtail + image: grafana/promtail:3.4.2 + args: + - -config.file=/etc/promtail/promtail.yaml + volumeMounts: + - name: promtail-config + mountPath: /etc/promtail + readOnly: true + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "64Mi" + cpu: "100m" + volumes: + - name: promtail-config + projected: + sources: + - configMap: + name: postgresql-promtail backups: pgbackrest: global: @@ -69,6 +91,12 @@ spec: min_wal_size: 32MB max_wal_size: 64MB # default is 1GB max_slot_wal_keep_size: 128MB # default is -1, allowing unlimited wal growth when replicas fall behind + logging_collector: 'on' + log_directory: 'log' + log_filename: 'postgresql.log' + log_truncate_on_rotation: 'on' + log_rotation_age: '1440' + log_rotation_size: '50000' # proxy: # pgBouncer: # replicas: 2 diff --git a/deployments/openshift/kustomize/base/crunchydb/promtail-configmap.yml b/deployments/openshift/kustomize/base/crunchydb/promtail-configmap.yml new file mode 100644 index 000000000..3adb6fbba --- /dev/null +++ b/deployments/openshift/kustomize/base/crunchydb/promtail-configmap.yml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql-promtail + labels: + app.kubernetes.io/part-of: crunchydb +data: + promtail.yaml: | + server: + http_listen_port: 0 + grpc_listen_port: 0 + positions: + filename: /tmp/positions.yaml + clients: + - url: http://loki:3100/loki/api/v1/push + scrape_configs: + - job_name: postgresql + static_configs: + - targets: + - localhost + labels: + service: postgresql + __path__: /pgdata/pg16/log/*.log diff --git a/deployments/openshift/kustomize/base/frontend/deployment.yml b/deployments/openshift/kustomize/base/frontend/deployment.yml index bf10f2712..ad406865f 100644 --- a/deployments/openshift/kustomize/base/frontend/deployment.yml +++ b/deployments/openshift/kustomize/base/frontend/deployment.yml @@ -18,10 +18,19 @@ spec: - name: frontend image: artifacts.developer.gov.bc.ca/kfd3-fd34fb-local/frontend:main-latest imagePullPolicy: Always + command: ["sh", "-c"] + args: + - | + ln -sf /var/log/app/access.log /var/log/nginx/access.log + ln -sf /var/log/app/error.log /var/log/nginx/error.log + exec nginx -g 'daemon off;' ports: - containerPort: 8080 name: http protocol: TCP + volumeMounts: + - name: logs + mountPath: /var/log/app resources: requests: memory: "64Mi" @@ -45,3 +54,27 @@ spec: periodSeconds: 5 timeoutSeconds: 3 failureThreshold: 3 + - name: promtail + image: grafana/promtail:3.4.2 + args: + - -config.file=/etc/promtail/promtail.yaml + volumeMounts: + - name: logs + mountPath: /var/log/app + readOnly: true + - name: promtail-config + mountPath: /etc/promtail + readOnly: true + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "64Mi" + cpu: "100m" + volumes: + - name: logs + emptyDir: {} + - name: promtail-config + configMap: + name: frontend-promtail diff --git a/deployments/openshift/kustomize/base/frontend/kustomization.yml b/deployments/openshift/kustomize/base/frontend/kustomization.yml index 4cda9cdbe..282d7c07b 100644 --- a/deployments/openshift/kustomize/base/frontend/kustomization.yml +++ b/deployments/openshift/kustomize/base/frontend/kustomization.yml @@ -4,6 +4,7 @@ resources: - deployment.yml - service.yml - route.yml + - promtail-configmap.yml - networkpolicy.yml commonLabels: app: frontend diff --git a/deployments/openshift/kustomize/base/frontend/promtail-configmap.yml b/deployments/openshift/kustomize/base/frontend/promtail-configmap.yml new file mode 100644 index 000000000..796082c08 --- /dev/null +++ b/deployments/openshift/kustomize/base/frontend/promtail-configmap.yml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: frontend-promtail + labels: + app: frontend +data: + promtail.yaml: | + server: + http_listen_port: 0 + grpc_listen_port: 0 + positions: + filename: /tmp/positions.yaml + clients: + - url: http://loki:3100/loki/api/v1/push + scrape_configs: + - job_name: frontend + static_configs: + - targets: + - localhost + labels: + service: frontend + __path__: /var/log/app/*.log diff --git a/deployments/openshift/kustomize/base/temporal/kustomization.yml b/deployments/openshift/kustomize/base/temporal/kustomization.yml index 9725c2526..998dcc548 100644 --- a/deployments/openshift/kustomize/base/temporal/kustomization.yml +++ b/deployments/openshift/kustomize/base/temporal/kustomization.yml @@ -14,5 +14,7 @@ resources: - temporal-worker-deployment.yml - pvc-logs.yml - logrotate-configmap.yml + - promtail-configmap-worker.yml + - promtail-configmap-server.yml commonLabels: app.kubernetes.io/part-of: temporal diff --git a/deployments/openshift/kustomize/base/temporal/promtail-configmap-server.yml b/deployments/openshift/kustomize/base/temporal/promtail-configmap-server.yml new file mode 100644 index 000000000..95dadf6e7 --- /dev/null +++ b/deployments/openshift/kustomize/base/temporal/promtail-configmap-server.yml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: temporal-server-promtail + labels: + app: temporal +data: + promtail.yaml: | + server: + http_listen_port: 0 + grpc_listen_port: 0 + positions: + filename: /tmp/positions.yaml + clients: + - url: http://loki:3100/loki/api/v1/push + scrape_configs: + - job_name: temporal-server + static_configs: + - targets: + - localhost + labels: + service: temporal-server + __path__: /var/log/app/*.log diff --git a/deployments/openshift/kustomize/base/temporal/promtail-configmap-worker.yml b/deployments/openshift/kustomize/base/temporal/promtail-configmap-worker.yml new file mode 100644 index 000000000..a909c492e --- /dev/null +++ b/deployments/openshift/kustomize/base/temporal/promtail-configmap-worker.yml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: temporal-worker-promtail + labels: + app: temporal-worker +data: + promtail.yaml: | + server: + http_listen_port: 0 + grpc_listen_port: 0 + positions: + filename: /tmp/positions.yaml + clients: + - url: http://loki:3100/loki/api/v1/push + scrape_configs: + - job_name: temporal-worker + static_configs: + - targets: + - localhost + labels: + service: temporal-worker + __path__: /var/log/app/*.log diff --git a/deployments/openshift/kustomize/base/temporal/temporal-server-deployment.yml b/deployments/openshift/kustomize/base/temporal/temporal-server-deployment.yml index 0077c57dc..5fe60dac5 100644 --- a/deployments/openshift/kustomize/base/temporal/temporal-server-deployment.yml +++ b/deployments/openshift/kustomize/base/temporal/temporal-server-deployment.yml @@ -18,6 +18,11 @@ spec: # Holds config_template.yaml + generated docker.yaml so main container sees both (entrypoint needs config_template.yaml). - name: temporal-config emptyDir: {} + - name: logs + emptyDir: {} + - name: promtail-config + configMap: + name: temporal-server-promtail initContainers: # Populate temporal-config with config_template.yaml and env-filled docker.yaml so entrypoint finds both. - name: config-init @@ -132,10 +137,14 @@ spec: - containerPort: 7233 name: grpc protocol: TCP + command: ["sh", "-c"] + args: ["/etc/temporal/entrypoint.sh 2>&1 | tee -a /var/log/app/temporal-server.log"] # Full config dir from config-init (config_template.yaml + docker.yaml). volumeMounts: - name: temporal-config mountPath: /etc/temporal/config + - name: logs + mountPath: /var/log/app env: # Enable TLS for SQL persistence (Crunchy Postgres pg_hba is hostssl-only). - name: SQL_TLS_ENABLED @@ -185,3 +194,21 @@ spec: periodSeconds: 5 timeoutSeconds: 3 failureThreshold: 3 + - name: promtail + image: grafana/promtail:3.4.2 + args: + - -config.file=/etc/promtail/promtail.yaml + volumeMounts: + - name: logs + mountPath: /var/log/app + readOnly: true + - name: promtail-config + mountPath: /etc/promtail + readOnly: true + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "64Mi" + cpu: "100m" diff --git a/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml b/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml index a2fff8c82..3edf23d4a 100644 --- a/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml +++ b/deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml @@ -164,6 +164,24 @@ spec: limits: memory: "64Mi" cpu: "50m" + - name: promtail + image: grafana/promtail:3.4.2 + args: + - -config.file=/etc/promtail/promtail.yaml + volumeMounts: + - name: logs + mountPath: /var/log/app + readOnly: true + - name: promtail-config + mountPath: /etc/promtail + readOnly: true + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "64Mi" + cpu: "100m" volumes: - name: storage persistentVolumeClaim: @@ -176,3 +194,6 @@ spec: name: temporal-worker-logrotate - name: logrotate-state emptyDir: {} + - name: promtail-config + configMap: + name: temporal-worker-promtail diff --git a/docs-md/GRAFANA_HELM_CHART.md b/docs-md/GRAFANA_HELM_CHART.md new file mode 100644 index 000000000..4122bb6ba --- /dev/null +++ b/docs-md/GRAFANA_HELM_CHART.md @@ -0,0 +1,109 @@ +# Grafana Helm Chart + +Grafana is deployed as part of the PLG (Prometheus, Loki, Grafana) observability stack via a standalone Helm chart located at `deployments/openshift/helm/plg/`. + +## Chart Structure + +``` +deployments/openshift/helm/plg/ + Chart.yaml # Chart metadata + values.yaml # Default values + values-local.yaml # Local Docker environment overrides + values-openshift.yaml # OpenShift environment overrides + templates/ + _helpers.tpl # Template helper functions + grafana-configmap.yaml # Grafana server configuration (grafana.ini) + grafana-datasources-configmap.yaml # Pre-provisioned data sources (Prometheus + Loki) + grafana-deployment.yaml # Grafana Deployment (stateless) + grafana-service.yaml # ClusterIP Service for Grafana +``` + +## Configurable Values + +| Value | Description | Default | +|-------|-------------|---------| +| `grafana.image.repository` | Grafana container image | `grafana/grafana` | +| `grafana.image.tag` | Grafana image tag | `11.5.2` | +| `grafana.adminUser` | Grafana admin username | `admin` | +| `grafana.adminPassword` | Grafana admin password (override via `GRAFANA_ADMIN_PASSWORD`) | `admin` | +| `grafana.resources.requests.memory` | Memory request | `256Mi` | +| `grafana.resources.requests.cpu` | CPU request | `250m` | +| `grafana.resources.limits.memory` | Memory limit | `256Mi` | +| `grafana.resources.limits.cpu` | CPU limit | `250m` | +| `grafana.httpPort` | HTTP listen port | `3001` | + +## Pre-Configured Data Sources + +Grafana is provisioned with two data sources that are available immediately after deployment, with no manual setup required: + +### Prometheus + +- **Name**: Prometheus +- **Type**: `prometheus` +- **URL**: Resolved from the Prometheus service within the same Helm release +- **Default**: Yes (used as the default data source for metric queries) + +### Loki + +- **Name**: Loki +- **Type**: `loki` +- **URL**: Resolved from the Loki service within the same Helm release + +Both data sources use the `proxy` access mode, meaning Grafana proxies requests to the backend services. Data sources are marked as non-editable to prevent drift from the provisioned configuration. + +## Authentication + +Grafana uses username/password authentication. The admin credentials are configurable via Helm values: + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + --set grafana.adminPassword= +``` + +Sign-up is disabled. Only the configured admin account can log in by default. + +## Network Access + +Grafana is deployed as a ClusterIP service and is not exposed via an OpenShift Route. Developers access it via port-forwarding, following the same pattern used for the Temporal UI: + +```bash +kubectl port-forward svc/-plg-grafana 3001:3001 -n +``` + +Then open `http://localhost:3001` in a browser. + +## Deployment + +### OpenShift + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + -f ./deployments/openshift/helm/plg/values-openshift.yaml \ + --set grafana.adminPassword= \ + -n +``` + +### Local Development + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + -f ./deployments/openshift/helm/plg/values-local.yaml +``` + +### Custom Overrides + +Any value can be overridden via `--set` flags: + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + --set grafana.adminPassword=mysecret \ + --set grafana.resources.limits.memory=512Mi +``` + +## Architecture Notes + +- Grafana runs as a single-replica Deployment (not a StatefulSet) because it does not require persistent storage for this use case. +- Data sources are provisioned via Grafana's file-based provisioning mechanism using ConfigMaps mounted into `/etc/grafana/provisioning/datasources`. +- The admin password is passed via the `GF_SECURITY_ADMIN_PASSWORD` environment variable. +- Config changes trigger automatic pod restarts via `checksum/config` and `checksum/datasources` annotations on the Deployment pod template. +- The existing Kustomize deployment for the application is not modified. PLG is a separate Helm release. diff --git a/docs-md/LOCAL_MONITORING_STACK.md b/docs-md/LOCAL_MONITORING_STACK.md new file mode 100644 index 000000000..63ca45a1a --- /dev/null +++ b/docs-md/LOCAL_MONITORING_STACK.md @@ -0,0 +1,94 @@ +# Local PLG Monitoring Stack + +The project includes an opt-in PLG (Prometheus, Loki, Grafana) monitoring stack for local development. It runs alongside the core Docker Compose services (PostgreSQL, MinIO) without affecting them. + +## Quick Start + +Start the monitoring stack: + +```bash +npm run dev:monitoring +``` + +Or start it alongside the core stack: + +```bash +docker compose -f apps/backend-services/docker-compose.yml -f deployments/local/docker-compose.monitoring.yml up -d +``` + +Stop the monitoring stack: + +```bash +npm run dev:monitoring:down +``` + +View monitoring container logs: + +```bash +npm run dev:monitoring:logs +``` + +## Services and Ports + +| Service | URL | Description | +|------------|------------------------|------------------------------------------| +| Grafana | http://localhost:3001 | Dashboards and log/metric exploration | +| Prometheus | http://localhost:9090 | Metrics storage and querying | +| Loki | http://localhost:3100 | Log aggregation | + +## Default Credentials + +- **Grafana**: `admin` / `admin` (override via `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` environment variables) + +## Architecture + +### Components + +- **Loki** (grafana/loki:3.4.0) - Receives and stores logs. Configured with filesystem storage and 30-day retention. +- **Promtail** (grafana/promtail:3.4.0) - Discovers running Docker containers via the Docker socket and forwards their stdout logs to Loki. Adds `service`, `container`, and `project` labels automatically. +- **Prometheus** (prom/prometheus:v3.2.1) - Scrapes metrics from backend-services (`host.docker.internal:3002/metrics`) and the Temporal server (`temporal:9090/metrics`). Data is retained for 15 days. +- **Grafana** (grafana/grafana:11.5.2) - Pre-configured with Prometheus and Loki data sources for querying metrics and logs. + +### Data Persistence + +Loki and Prometheus data is stored in named Docker volumes (`loki_data` and `prometheus_data`). Data survives container restarts and `docker compose down`. To clear all monitoring data: + +```bash +docker compose -f deployments/local/docker-compose.monitoring.yml down -v +``` + +### Log Collection + +Promtail auto-discovers all running Docker containers by mounting `/var/run/docker.sock`. It applies the following labels to each log stream: + +- `container` - The Docker container name +- `service` - The Docker Compose service name +- `project` - The Docker Compose project name + +### Grafana Data Sources + +Grafana is provisioned with two data sources on startup: + +- **Prometheus** (default) - Points to the local Prometheus instance +- **Loki** - Points to the local Loki instance + +No manual configuration is required. + +## Configuration Files + +| File | Purpose | +|---------------------------------------------------------------|-------------------------------| +| `deployments/local/docker-compose.monitoring.yml` | Docker Compose service definitions | +| `deployments/local/loki/loki.yaml` | Loki server configuration | +| `deployments/local/prometheus/prometheus.yml` | Prometheus scrape targets | +| `deployments/local/promtail/promtail-config.yml` | Promtail log discovery rules | +| `deployments/local/grafana/provisioning/datasources/datasources.yml` | Grafana data source provisioning | + +## VS Code Integration + +Two VS Code tasks are available: + +- **monitoring: docker up** - Starts the monitoring stack +- **Dev: all + monitoring** - Starts the full development environment including the monitoring stack + +The existing **Dev: all** task is unchanged and does not include monitoring. diff --git a/docs-md/LOGGING.md b/docs-md/LOGGING.md index a03e7be8d..2366ed84b 100644 --- a/docs-md/LOGGING.md +++ b/docs-md/LOGGING.md @@ -1,6 +1,6 @@ # Logging (shared package) -The `@ai-di/shared-logging` package provides structured logging used by `backend-services` and the Temporal worker. See feature spec in `feature-docs/007-logging-system` (if present). When `NODE_ENV=development`, the shared logger emits human-readable one-line format instead of NDJSON; otherwise it emits NDJSON. Set `NODE_ENV=development` in each app's `.env` for local dev (e.g. both `apps/backend-services` and `apps/temporal`); use no spaces around `=` so dotenv sets the variable correctly. In development, set `LOG_PRETTY_CONTEXT=1` (or `true`/`yes`) to pretty-print context JSON on multiple lines; when unset, context stays inline. +The `@ai-di/shared-logging` package provides structured NDJSON logging used by `backend-services` and the Temporal worker. See feature spec in `feature-docs/007-logging-system` (if present). ## Package location @@ -8,8 +8,8 @@ The `@ai-di/shared-logging` package provides structured logging used by `backend ## Usage -- **Backend:** `AppLoggerService` wraps `createLogger("backend-services")` (see `apps/backend-services`). Request-scoped `requestId` and `userId` are merged into every log via middleware and request context. The `requestId` is always generated server-side (a new UUID per request); any client-supplied `x-request-id` header is ignored so logs and audit cannot be confused by reused IDs. In development, `LoggingInterceptor` (registered in `LoggingModule`) logs each HTTP request/response as NDJSON (method, path, statusCode, durationMs, and at debug level query, params, body). -- **Temporal worker:** `createLogger("temporal-worker")` and `createActivityLogger(activityName, context)` (see `apps/temporal/src/logger.ts`). Activities that receive `requestId` in workflow input should pass it in `context` so logs can be traced by requestId across backend and worker. **SDK internal logs** (e.g. "Activity failed", "Workflow failed") are always routed through the same shared logger via a custom Runtime logger (`apps/temporal/src/temporal-runtime-logger.ts`), so output format (pretty in dev, NDJSON in prod) and `LOG_LEVEL` are consistent for both SDK and application logs. +- **Backend:** `AppLoggerService` wraps `createLogger("backend-services")` (see `apps/backend-services`). Request-scoped `requestId`, `userId`, `sessionId`, `apiKeyId`, and `clientIp` are merged into every log via middleware and request context. The `requestId` is always generated server-side (a new UUID per request); any client-supplied `x-request-id` header is ignored so logs and audit cannot be confused by reused IDs. The `sessionId` is extracted from the Keycloak JWT `session_state` claim (via `req.user.session_state`) in the `RequestLoggingInterceptor` and stored in `AsyncLocalStorage`. For API key-authenticated requests (validated by `ApiKeyAuthGuard`), the `apiKeyId` field is set to the stored key prefix (first 8 characters) instead of `sessionId`; the two fields are mutually exclusive. The full API key value is never written to log output. The `clientIp` is extracted by the `LoggingMiddleware` using the priority: `X-Forwarded-For` (first entry) > `X-Real-IP` > `req.socket.remoteAddress`. On OpenShift, the client IP arrives via `X-Forwarded-For` due to reverse proxy/ingress; locally, `req.socket.remoteAddress` is used as fallback. For unauthenticated requests, both `sessionId` and `apiKeyId` are omitted from log output. In development, `LoggingInterceptor` (registered in `LoggingModule`) logs each HTTP request/response as NDJSON (method, path, statusCode, durationMs, and at debug level query, params, body). +- **Temporal worker:** `createLogger("temporal-worker")` and `createActivityLogger(activityName, context)` (see `apps/temporal/src/logger.ts`). Activities that receive `requestId` in workflow input should pass it in `context` so logs can be traced by requestId across backend and worker. **SDK internal logs** (e.g. "Activity failed", "Workflow failed") are routed through the same shared logger via a custom Runtime logger and native log forwarding (`apps/temporal/src/temporal-runtime-logger.ts`), so all worker process output is NDJSON and respects `LOG_LEVEL`. ## Testing diff --git a/docs-md/LOKI_HELM_CHART.md b/docs-md/LOKI_HELM_CHART.md new file mode 100644 index 000000000..c6d214f7f --- /dev/null +++ b/docs-md/LOKI_HELM_CHART.md @@ -0,0 +1,108 @@ +# Loki Helm Chart + +Loki is deployed as part of the PLG (Prometheus, Loki, Grafana) observability stack via a standalone Helm chart located at `deployments/openshift/helm/plg/`. + +## Chart Structure + +``` +deployments/openshift/helm/plg/ + Chart.yaml # Chart metadata + values.yaml # Default values + values-local.yaml # Local Docker environment overrides + values-openshift.yaml # OpenShift environment overrides + templates/ + _helpers.tpl # Template helper functions + loki-configmap.yaml # Loki server configuration + loki-statefulset.yaml # Loki StatefulSet deployment + loki-service.yaml # ClusterIP Service for Loki + prometheus-configmap.yaml # Prometheus server configuration + prometheus-statefulset.yaml # Prometheus StatefulSet deployment + prometheus-service.yaml # ClusterIP Service for Prometheus + grafana-configmap.yaml # Grafana server configuration + grafana-datasources-configmap.yaml # Pre-provisioned data sources + grafana-deployment.yaml # Grafana Deployment + grafana-service.yaml # ClusterIP Service for Grafana +``` + +## Configurable Values + +| Value | Description | Default | +|-------|-------------|---------| +| `loki.image.repository` | Loki container image | `grafana/loki` | +| `loki.image.tag` | Loki image tag | `3.4.0` | +| `loki.retentionDays` | Log retention period in days | `30` | +| `loki.pvcSize` | PVC storage size | `10Gi` | +| `loki.storageClassName` | Storage class (empty = cluster default) | `""` | +| `loki.resources.requests.memory` | Memory request | `256Mi` | +| `loki.resources.requests.cpu` | CPU request | `500m` | +| `loki.resources.limits.memory` | Memory limit | `256Mi` | +| `loki.resources.limits.cpu` | CPU limit | `500m` | +| `loki.httpPort` | HTTP listen port | `3100` | + +## NDJSON Log Parsing + +Loki is configured to work with the NDJSON structured logs produced by `@ai-di/shared-logging`. Loki natively parses JSON log lines, making the following fields queryable via LogQL: + +- `timestamp` (ISO 8601) +- `level` (debug, info, warn, error) +- `service` (e.g., "backend-services", "temporal-worker") +- `requestId` (UUID) +- `userId` (from resolved identity) +- `sessionId` (from Keycloak session_state) +- `clientIp` (client IP address) +- `method`, `path`, `statusCode` (HTTP request details) +- `durationMs` (request duration) + +LogQL queries can extract these fields using the `json` parser: + +```logql +{service="backend-services"} | json | level="error" +{service="backend-services"} | json | sessionId="" +{service="backend-services"} | json | statusCode >= 500 +``` + +## Log Retention + +Retention is enforced by the Loki compactor, which runs on a 10-minute interval and deletes chunks older than the configured retention period. The default retention period is 30 days (720 hours). + +To change the retention period, override `loki.retentionDays`: + +```bash +helm upgrade plg ./deployments/openshift/helm/plg --set loki.retentionDays=14 +``` + +## Deployment + +### OpenShift + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + -f ./deployments/openshift/helm/plg/values-openshift.yaml \ + -n +``` + +### Local Development + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + -f ./deployments/openshift/helm/plg/values-local.yaml +``` + +### Custom Overrides + +Any value can be overridden via `--set` flags: + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + --set loki.pvcSize=20Gi \ + --set loki.retentionDays=60 \ + --set loki.resources.limits.memory=512Mi +``` + +## Architecture Notes + +- Loki runs as a single-replica StatefulSet in monolithic mode (`-target=all`). +- Data is persisted to a PVC using the TSDB store with filesystem object storage. +- The existing Kustomize deployment for the application is not modified. PLG is a separate Helm release. +- Loki is not exposed via an OpenShift Route; access is via in-cluster services or port-forwarding. +- Log collection is handled by Promtail (configured separately) which sends logs to Loki's push API. diff --git a/docs-md/PLG_DEPLOYMENT_INTEGRATION.md b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md new file mode 100644 index 000000000..66011b389 --- /dev/null +++ b/docs-md/PLG_DEPLOYMENT_INTEGRATION.md @@ -0,0 +1,94 @@ +# PLG Deployment Integration + +## Overview + +The PLG (Prometheus, Loki, Grafana) monitoring stack is deployed as a separate Helm release alongside the application. It does not modify or interfere with the existing Kustomize-based application deployment. PLG deployment is integrated into both the GitHub Actions CI/CD pipeline and the local deployment scripts. + +## Deployment Methods + +### GitHub Actions (CI/CD) + +The `build-apps.yml` workflow includes a `deploy-plg` job that runs after application images are built. This job: + +1. Checks out the repository to access the Helm chart at `deployments/openshift/helm/plg/` +2. Installs the Helm and `oc` CLIs +3. Authenticates to OpenShift using environment secrets +4. Runs `helm upgrade --install` with the OpenShift values file + +The job runs regardless of whether application images were built (it depends on `build-apps` succeeding or being skipped), ensuring the PLG stack stays up to date even when no application code changed. + +#### Required GitHub Environment Secrets + +| Secret | Description | +|--------|-------------| +| `OPENSHIFT_SERVER` | OpenShift API server URL | +| `OPENSHIFT_TOKEN` | Service account token for the target namespace | +| `OPENSHIFT_NAMESPACE` | Target namespace (e.g., `fd34fb-dev`) | +| `GRAFANA_ADMIN_PASSWORD` | Grafana admin password (falls back to `admin` if unset) | + +### Local Deployment (`oc-deploy.sh`) + +The `scripts/oc-deploy.sh` script deploys the PLG stack as **Step 7**, between applying the Kustomize overlay (Step 6) and creating instance secrets (Step 8). This step: + +1. Reads PLG-specific configuration from the environment profile (`dev.env` or `prod.env`) +2. Derives instance-specific Prometheus scrape targets from the Kustomize instance name +3. Runs `helm upgrade --install` with environment-specific values passed via `--set` flags + +If the `helm` CLI is not installed, the PLG step is skipped with a warning. The application deployment continues normally. + +#### Instance-Specific Helm Release + +Each application instance gets its own PLG Helm release named `-plg`. Prometheus scrape targets are configured to point at the instance-specific Kubernetes service names (e.g., `my-instance-backend-services`, `my-instance-temporal`). + +### Teardown (`oc-teardown.sh`) + +The `scripts/oc-teardown.sh` script includes a step (3b) that uninstalls the PLG Helm release for the instance being torn down. If Helm is not installed or no PLG release exists, the step is skipped gracefully. + +## Environment Configuration + +PLG-specific variables are configured in the same environment profile files used by the application (`deployments/openshift/config/.env`). They follow the existing config merge pattern: profile defaults can be overridden by instance-specific files. + +| Variable | Default | Description | +|----------|---------|-------------| +| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin login password | +| `LOKI_RETENTION_DAYS` | `30` | Log retention period in days | +| `LOKI_PVC_SIZE` | `10Gi` | Persistent volume size for Loki data | +| `PROMETHEUS_PVC_SIZE` | `10Gi` | Persistent volume size for Prometheus TSDB | +| `METRICS_SCRAPE_INTERVAL` | `15s` | How often Prometheus scrapes targets | + +These variables are read by `oc-deploy.sh` via the `config-loader.sh` library and passed to Helm as `--set` overrides on top of the `values-openshift.yaml` base. + +## Separation from Kustomize + +The PLG deployment is completely independent of the Kustomize-based application deployment: + +- PLG resources are managed by Helm, not Kustomize +- PLG uses its own labels (`app.kubernetes.io/managed-by: Helm`, `app.kubernetes.io/part-of: plg`) +- No Kustomize base or overlay files are modified for PLG +- If PLG deployment fails, the application deployment is unaffected + +## Accessing Grafana + +Grafana is not exposed via an OpenShift Route. Access it via port-forwarding: + +```bash +# For instance-specific deployments (via oc-deploy.sh) +oc port-forward svc/-plg-grafana 3001:3001 -n + +# For CI-deployed PLG (single release per namespace) +oc port-forward svc/plg-grafana 3001:3001 -n +``` + +Then open `http://localhost:3001` and log in with `admin` / ``. + +## Files + +| File | Purpose | +|------|---------| +| `deployments/openshift/helm/plg/` | PLG Helm chart (templates, values) | +| `deployments/openshift/helm/plg/values-openshift.yaml` | OpenShift-specific value overrides | +| `deployments/openshift/config/dev.env.example` | Dev environment config template (includes PLG variables) | +| `deployments/openshift/config/prod.env.example` | Prod environment config template (includes PLG variables) | +| `.github/workflows/build-apps.yml` | CI workflow with `deploy-plg` job | +| `scripts/oc-deploy.sh` | Local deployment script (Step 7: PLG) | +| `scripts/oc-teardown.sh` | Teardown script (Step 3b: PLG uninstall) | diff --git a/docs-md/PROMETHEUS_HELM_CHART.md b/docs-md/PROMETHEUS_HELM_CHART.md new file mode 100644 index 000000000..49f90a8c3 --- /dev/null +++ b/docs-md/PROMETHEUS_HELM_CHART.md @@ -0,0 +1,102 @@ +# Prometheus Helm Chart + +Prometheus is deployed as part of the PLG (Prometheus, Loki, Grafana) observability stack via a standalone Helm chart located at `deployments/openshift/helm/plg/`. + +## Chart Structure + +``` +deployments/openshift/helm/plg/ + Chart.yaml # Chart metadata + values.yaml # Default values + values-local.yaml # Local Docker environment overrides + values-openshift.yaml # OpenShift environment overrides + templates/ + _helpers.tpl # Template helper functions + prometheus-configmap.yaml # Prometheus server configuration with scrape targets + prometheus-statefulset.yaml # Prometheus StatefulSet deployment with PVC + prometheus-service.yaml # ClusterIP Service for Prometheus + grafana-configmap.yaml # Grafana server configuration + grafana-datasources-configmap.yaml # Pre-provisioned data sources + grafana-deployment.yaml # Grafana Deployment + grafana-service.yaml # ClusterIP Service for Grafana +``` + +## Configurable Values + +| Value | Description | Default | +|-------|-------------|---------| +| `prometheus.image.repository` | Prometheus container image | `prom/prometheus` | +| `prometheus.image.tag` | Prometheus image tag | `v3.2.1` | +| `prometheus.retentionDays` | TSDB data retention period in days | `15` | +| `prometheus.pvcSize` | PVC storage size | `10Gi` | +| `prometheus.storageClassName` | Storage class (empty = cluster default) | `""` | +| `prometheus.scrapeInterval` | Scrape interval for all targets | `15s` | +| `prometheus.resources.requests.memory` | Memory request | `512Mi` | +| `prometheus.resources.requests.cpu` | CPU request | `500m` | +| `prometheus.resources.limits.memory` | Memory limit | `512Mi` | +| `prometheus.resources.limits.cpu` | CPU limit | `500m` | +| `prometheus.httpPort` | HTTP listen port | `9090` | +| `prometheus.scrapeTargets.backendServices.host` | Backend-services service hostname | `backend-services` | +| `prometheus.scrapeTargets.backendServices.port` | Backend-services metrics port | `3002` | +| `prometheus.scrapeTargets.temporalServer.host` | Temporal server service hostname | `temporal` | +| `prometheus.scrapeTargets.temporalServer.port` | Temporal server metrics port | `9090` | + +## Scrape Targets + +Prometheus is pre-configured with two scrape targets: + +### Backend-Services + +Scrapes the `/metrics` endpoint exposed by the NestJS backend-services application. This endpoint provides RED (Rate, Errors, Duration) metrics and Node.js runtime metrics via `prom-client`. See `docs-md/PROMETHEUS_METRICS.md` for details on the metrics exposed. + +### Temporal Server + +Scrapes the Temporal server's built-in `/metrics` endpoint, which exposes workflow execution, task queue, and schedule metrics in Prometheus format. No custom instrumentation is required. + +Both targets reference service names within the same Kubernetes namespace. + +## Scrape Interval + +The scrape interval defaults to `15s` and applies to all scrape targets. Override it via Helm values: + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + --set prometheus.scrapeInterval=30s +``` + +## Deployment + +### OpenShift + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + -f ./deployments/openshift/helm/plg/values-openshift.yaml \ + -n +``` + +### Local Development + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + -f ./deployments/openshift/helm/plg/values-local.yaml +``` + +### Custom Overrides + +Any value can be overridden via `--set` flags: + +```bash +helm upgrade --install plg ./deployments/openshift/helm/plg \ + --set prometheus.pvcSize=20Gi \ + --set prometheus.scrapeInterval=30s \ + --set prometheus.resources.limits.memory=1Gi +``` + +## Architecture Notes + +- Prometheus runs as a single-replica StatefulSet. +- Metrics data is persisted to a PVC using the Prometheus TSDB storage engine. +- The existing Kustomize deployment for the application is not modified. PLG is a separate Helm release. +- Prometheus is not exposed via an OpenShift Route; access is via in-cluster services or port-forwarding. +- No Alertmanager configuration is included. Prometheus is used for metrics collection and querying only. +- Config changes trigger automatic pod restarts via the `checksum/config` annotation on the StatefulSet pod template. diff --git a/docs-md/PROMETHEUS_METRICS.md b/docs-md/PROMETHEUS_METRICS.md new file mode 100644 index 000000000..f3717056d --- /dev/null +++ b/docs-md/PROMETHEUS_METRICS.md @@ -0,0 +1,54 @@ +# Prometheus RED Metrics + +## Overview + +The backend-services application exposes a `/metrics` endpoint for Prometheus scraping. This endpoint provides RED (Rate, Errors, Duration) metrics for HTTP requests and Node.js runtime metrics via the `prom-client` library. + +## Metrics Exposed + +### RED Metrics + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `http_requests_total` | Counter | `method`, `path`, `status_code` | Total number of HTTP requests processed | +| `http_request_errors_total` | Counter | `method`, `path`, `status_code` | Total HTTP requests with 4xx or 5xx status codes | +| `http_request_duration_seconds` | Histogram | `method`, `path` | Request duration in seconds with configurable buckets | + +### Node.js Runtime Metrics + +Default `prom-client` metrics are collected, including: +- Event loop lag +- Heap usage (used, total, external) +- Active handles and requests +- GC pause durations + +## Architecture + +The metrics implementation consists of four files in `apps/backend-services/src/metrics/`: + +- **`metrics.service.ts`** -- Registers the Prometheus registry, RED metric instruments, and default Node.js metrics collection. +- **`metrics.middleware.ts`** -- NestJS middleware applied to all routes. Instruments each HTTP request by incrementing counters and recording duration on response finish. The `/metrics` path itself is excluded to avoid self-referential metric inflation. +- **`metrics.controller.ts`** -- Exposes `GET /metrics` with the `@Public()` decorator (no JWT required). Blocks external access by checking for `X-Forwarded-Host` header (injected by the OpenShift router for external requests). +- **`metrics.module.ts`** -- Wires the service, middleware, and controller together. + +## Access Control + +The `/metrics` endpoint is only accessible from within the cluster: + +1. **Application level**: The controller rejects requests with an `X-Forwarded-Host` header (present when requests arrive via the OpenShift Route) with a 403 Forbidden response. +2. **Route level**: The OpenShift Route for backend-services includes a `haproxy.router.openshift.io/deny-list` annotation to block `/metrics` at the HAProxy router layer. + +Prometheus scrapes `/metrics` directly via the in-cluster Kubernetes Service, bypassing the Route entirely. + +## Authentication + +The `/metrics` endpoint is marked with `@Public()` and excluded from JWT/API-key authentication guards. Prometheus scrapes without credentials. + +## Histogram Buckets + +Duration histogram uses the following bucket boundaries (in seconds): +`0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10` + +## Path Label Strategy + +The middleware uses `req.route?.path` (the Express route pattern, e.g., `/api/documents/:id`) when available, falling back to `req.path` (the literal URL path). This prevents high-cardinality label values from dynamic URL segments. diff --git a/docs-md/PROMTAIL_SIDECARS.md b/docs-md/PROMTAIL_SIDECARS.md new file mode 100644 index 000000000..fa04a311d --- /dev/null +++ b/docs-md/PROMTAIL_SIDECARS.md @@ -0,0 +1,108 @@ +# Promtail Sidecar Containers + +Promtail sidecar containers are added to all application pods on OpenShift to collect logs and forward them to Loki. This sidecar pattern works within tenant-level namespace permissions without requiring cluster-admin access or DaemonSet deployments. + +## Architecture + +Each application pod includes a Promtail sidecar container that: + +1. Tails log files from a shared volume within the pod +2. Forwards log entries to the in-namespace Loki service at `http://loki:3100/loki/api/v1/push` +3. Adds a `service` label to all log entries for filtering in Grafana + +## Services with Promtail Sidecars + +| Service | Log Source | Service Label | Log Path | +|---------|-----------|---------------|----------| +| backend-services | Application logs via `tee` to shared PVC | `service=backend-services` | `/var/log/app/*.log` | +| temporal-worker | Worker logs via `tee` to shared PVC | `service=temporal-worker` | `/var/log/app/*.log` | +| temporal-server | Server logs via `tee` to emptyDir | `service=temporal-server` | `/var/log/app/*.log` | +| frontend | Nginx access/error logs via symlinks to emptyDir | `service=frontend` | `/var/log/app/*.log` | +| postgresql | PostgreSQL logging_collector output on pgdata volume | `service=postgresql` | `/pgdata/pg16/log/*.log` | + +## Resource Limits + +All Promtail sidecars use minimal resource allocations: + +| Resource | Request | Limit | +|----------|---------|-------| +| Memory | 32Mi | 64Mi | +| CPU | 50m | 100m | + +These defaults are sized for low-traffic environments. To adjust resource limits, modify the Promtail container resource specifications in the relevant deployment manifests under `deployments/openshift/kustomize/base/`. + +## Configuration + +Each service has its own Promtail ConfigMap containing the Promtail configuration: + +- `backend-services-promtail` - backend-services Promtail config +- `temporal-worker-promtail` - temporal-worker Promtail config +- `temporal-server-promtail` - temporal-server Promtail config +- `frontend-promtail` - frontend Promtail config +- `postgresql-promtail` - PostgreSQL Promtail config + +### Promtail Configuration Structure + +Each ConfigMap contains a `promtail.yaml` with: + +- **server**: HTTP and gRPC listen ports disabled (set to 0) since only log forwarding is needed +- **positions**: Tracks read positions in `/tmp/positions.yaml` within the container +- **clients**: Points to the Loki push API endpoint using in-namespace service DNS +- **scrape_configs**: Defines the job name, service label, and log file path glob pattern + +### Loki Endpoint + +All Promtail sidecars are configured to push logs to `http://loki:3100/loki/api/v1/push`. This uses the in-namespace Kubernetes service DNS name for the Loki instance deployed via the PLG Helm chart. + +## Shared Volume Patterns + +### PVC-backed (backend-services, temporal-worker) + +These services already had a logrotate sidecar writing to `/var/log/app/` on a PersistentVolumeClaim. The Promtail sidecar mounts the same PVC volume in read-only mode. + +### emptyDir (temporal-server, frontend) + +These services use ephemeral `emptyDir` volumes for log sharing. Logs are not persisted across pod restarts, but Promtail forwards them to Loki in near real-time. + +### pgdata volume (PostgreSQL) + +The Crunchy PostgreSQL operator manages the data volume. PostgreSQL's `logging_collector` writes logs to the `log/` subdirectory within the pgdata path. The Promtail sidecar reads from `/pgdata/pg16/log/*.log`. + +## Log Collection Flow + +``` +Application Container + | + | writes logs to shared volume + v +Shared Volume (/var/log/app/ or /pgdata/pg16/log/) + | + | Promtail tails log files + v +Promtail Sidecar + | + | HTTP POST to Loki push API + v +Loki (http://loki:3100) +``` + +## Promtail Image + +All sidecars use `grafana/promtail:3.4.2`. To update the Promtail version, change the image tag in each deployment manifest. + +## Files Modified + +- `deployments/openshift/kustomize/base/backend-services/deployment.yml` - Added Promtail sidecar container and config volume +- `deployments/openshift/kustomize/base/backend-services/promtail-configmap.yml` - New Promtail ConfigMap +- `deployments/openshift/kustomize/base/backend-services/kustomization.yml` - Added promtail-configmap.yml resource +- `deployments/openshift/kustomize/base/temporal/temporal-worker-deployment.yml` - Added Promtail sidecar container and config volume +- `deployments/openshift/kustomize/base/temporal/promtail-configmap-worker.yml` - New Promtail ConfigMap for worker +- `deployments/openshift/kustomize/base/temporal/temporal-server-deployment.yml` - Added log output redirection, logs volume, and Promtail sidecar +- `deployments/openshift/kustomize/base/temporal/promtail-configmap-server.yml` - New Promtail ConfigMap for server +- `deployments/openshift/kustomize/base/temporal/kustomization.yml` - Added promtail configmap resources +- `deployments/openshift/kustomize/base/frontend/deployment.yml` - Added nginx log redirection, logs volume, and Promtail sidecar +- `deployments/openshift/kustomize/base/frontend/promtail-configmap.yml` - New Promtail ConfigMap +- `deployments/openshift/kustomize/base/frontend/kustomization.yml` - Added promtail-configmap.yml resource +- `deployments/openshift/kustomize/base/crunchydb/postgrescluster.yml` - Added Promtail sidecar container, config volume, and PostgreSQL logging parameters +- `deployments/openshift/kustomize/base/crunchydb/promtail-configmap.yml` - New Promtail ConfigMap +- `deployments/openshift/kustomize/base/crunchydb/kustomization.yml` - Added promtail-configmap.yml resource diff --git a/docs-md/openshift-deployment/ENVIRONMENT_CONFIGURATION.md b/docs-md/openshift-deployment/ENVIRONMENT_CONFIGURATION.md index fafb132f7..9e06dbfb5 100644 --- a/docs-md/openshift-deployment/ENVIRONMENT_CONFIGURATION.md +++ b/docs-md/openshift-deployment/ENVIRONMENT_CONFIGURATION.md @@ -170,6 +170,16 @@ These values are derived automatically by the deploy script — do not set them | `THROTTLE_AUTH_REFRESH_TTL_MS` | Token refresh rate limit window | | `THROTTLE_AUTH_REFRESH_LIMIT` | Max refresh requests per IP (stricter in prod) | +### PLG Monitoring Stack + +| Variable | Default | Description | +|----------|---------|-------------| +| `GRAFANA_ADMIN_PASSWORD` | `admin` | Grafana admin login password | +| `LOKI_RETENTION_DAYS` | `30` | Log retention period in days | +| `LOKI_PVC_SIZE` | `10Gi` | Persistent volume size for Loki data | +| `PROMETHEUS_PVC_SIZE` | `10Gi` | Persistent volume size for Prometheus TSDB | +| `METRICS_SCRAPE_INTERVAL` | `15s` | How often Prometheus scrapes targets | + ## How Secrets Reach the Pods The deploy script creates per-instance OpenShift Secrets from values in the env file. Each instance gets its own copy. diff --git a/feature-docs/20260315052727-plg-monitoring-stack/REQUIREMENTS.md b/feature-docs/20260315052727-plg-monitoring-stack/REQUIREMENTS.md new file mode 100644 index 000000000..0b9d3b459 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/REQUIREMENTS.md @@ -0,0 +1,242 @@ +# PLG Monitoring Stack (Prometheus, Loki, Grafana) + +## Overview + +Add a Prometheus, Loki, and Grafana (PLG) observability stack to the platform. The stack must run both locally in Docker and on OpenShift, deployed via Helm charts. It integrates with the existing NDJSON structured logging and JWT-based session tracking to provide centralized log aggregation, metrics collection, and dashboarding. + +--- + +## 1. Deployment & Infrastructure + +### 1.1 Helm Charts for PLG + +- Deploy Prometheus, Loki, and Grafana using community Helm charts. +- The existing application deployment (Kustomize) remains unchanged — Helm is used only for PLG components. +- The Helm chart values must be configurable per environment (local Docker vs. OpenShift). + +### 1.2 OpenShift Deployment + +- PLG runs in the **same namespace** as the application, integrated with the existing Kustomize-based deployment architecture. +- Must work with the existing **GitHub Actions workflow** that builds and deploys the application. +- Must work with the existing **local deployment scripts** in `/scripts`. +- Loki stores logs in a PVC with configurable size. +- Prometheus uses a PVC for metrics storage. + +### 1.3 Local Docker Deployment + +- PLG is provided via a **separate `docker-compose.monitoring.yml`** file. +- Developers opt-in by running both compose files together (e.g., `docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up`). +- Must be compatible with the existing **startup scripts in `package.json`** and the **VS Code `Dev:All` task**. +- This keeps the core dev stack lightweight; PLG is not required for day-to-day development. +- Log and metric data persists via Docker volumes. + +--- + +## 2. Grafana UI Access + +### 2.1 Authentication + +- Grafana uses **username/password authentication** (configurable admin credentials). +- Both locally and on OpenShift, the same login/password approach is used. + +### 2.2 Network Exposure + +- On OpenShift, Grafana is **not exposed via a Route**. Developers access it via **port-forwarding/tunneling** (same pattern used for the Temporal UI on OpenShift). +- Locally, Grafana is exposed on a configurable port (default `localhost:3001`). Prometheus on `localhost:9090`, Loki on `localhost:3100` (community-standard defaults). + +--- + +## 3. Log Aggregation (Loki) + +### 3.1 Collection Method + +- **Loki scrapes container stdout** (Option A) — no changes to the application logging code for collection. +- On OpenShift, use **Promtail sidecar containers** added to each application pod to tail shared log volumes. This works within tenant-level namespace permissions (no DaemonSet or cluster-admin access required). The backend-services deployment already uses a logrotate sidecar writing to `/var/log/app/`, establishing the sidecar pattern. +- Locally in Docker, a **Promtail container** mounts the Docker socket (`/var/run/docker.sock`) to auto-discover and tail all running container logs. + +### 3.2 Services Collected + +Logs are collected from **all services**: +- `backend-services` (NestJS API) +- `temporal-worker` (Temporal workflow worker) +- `temporal-server` (Temporal server) +- `frontend` (nginx access logs) +- `PostgreSQL` (database logs) + +Each service is labeled in Loki (e.g., `service=backend-services`, `service=temporal-worker`) for filtering. + +### 3.3 Log Retention + +- **30-day retention** period, configured in Loki's `retention_period` setting. +- Retention is configurable via Helm values to allow adjustment per environment. + +### 3.4 Existing Log Format Compatibility + +The existing `@ai-di/shared-logging` package outputs NDJSON with the following fields already available for querying in Loki: +- `timestamp` (ISO 8601) +- `level` (debug, info, warn, error) +- `service` (e.g., "backend-services", "temporal-worker") +- `requestId` (UUID, injected by LoggingMiddleware) +- `userId` (from resolved identity) +- `method`, `path`, `statusCode` (from RequestLoggingInterceptor) +- `durationMs` (request duration) +- `workflowExecutionId`, `documentId` (contextual fields) + +No changes are needed to the log format for Loki to parse and index these fields. + +--- + +## 4. Session Tracking & User Activity Browsing + +### 4.1 Session Identification + +- Extract `session_state` from the existing `req.user` object (already available after Keycloak JWT validation via the `KeycloakJwtStrategy`). +- Add `sessionId` (value of `session_state`) to the request context stored in `AsyncLocalStorage`, alongside the existing `requestId` and `userId`. +- The `AppLoggerService` automatically includes `sessionId` in all NDJSON log output. +- **No manual JWT decoding** — reuse the existing Passport/IdentityGuard infrastructure that already parses and validates the JWT. + +### 4.2 API Key Requests + +- For API key-authenticated requests (no JWT/session), log the **API key prefix or key ID** from the database as an identifier. +- API key requests are not treated as "sessions" — the identifier is for audit filtering only. + +### 4.3 Grafana Session Browsing + +- Users can filter logs in Grafana by `sessionId` to see all API activity within a single Keycloak session. +- Users can filter by `userId` to see all activity for a specific user across sessions. +- This is surfaced via the Logs Explorer dashboard (see Section 7). + +--- + +## 5. Client IP Logging + +### 5.1 IP Extraction + +- Add `clientIp` to the NDJSON log context for every request. +- Extraction logic (in the `LoggingMiddleware` or `RequestLoggingInterceptor`): + ``` + clientIp = req.headers['x-forwarded-for']?.split(',')[0].trim() + || req.headers['x-real-ip'] + || req.socket.remoteAddress + ``` +- On OpenShift, the client IP comes from `X-Forwarded-For` (first entry) due to reverse proxy/ingress. +- Locally, `req.socket.remoteAddress` is used as fallback. + +All logged fields (including `clientIp`) are stored in Loki and retained per the configured retention period (see Section 3.3). + +--- + +## 6. Metrics (Prometheus) + +### 6.1 Application Metrics + +Expose a `/metrics` endpoint on the backend-services application using `prom-client`. The `/metrics` path must **not be publicly accessible** — it is excluded from the OpenShift Route so only in-cluster Prometheus can scrape it: + +- **RED Metrics** (Request, Error, Duration): + - `http_requests_total` — counter by method, path, status code + - `http_request_duration_seconds` — histogram by method, path + - `http_request_errors_total` — counter of 4xx/5xx responses + +- **Node.js Runtime Metrics** (via `prom-client` default metrics): + - Event loop lag + - Heap usage (used, total, external) + - Active handles and requests + - GC pause durations + +### 6.2 Temporal Metrics + +- Scrape the **Temporal server's built-in `/metrics` endpoint** — no custom instrumentation needed. +- Temporal already exposes workflow execution, task queue, and schedule metrics in Prometheus format. + +### 6.3 Scrape Configuration + +- Prometheus scrape configs are defined in the Helm chart values. +- Targets: backend-services `/metrics`, Temporal server `/metrics`. +- Scrape interval: 15s (configurable). + +--- + +## 7. Pre-Built Grafana Dashboards + +Ship the following dashboards as ConfigMaps in the Helm chart (dashboards-as-code): + +### 7.1 Application Overview Dashboard +- Request rate (requests/sec) +- Error rate (4xx/5xx per second) +- Latency percentiles (p50, p95, p99) +- Active sessions (unique sessionIds in last 5 minutes) + +### 7.2 Logs Explorer Dashboard +- Pre-configured Loki data source +- Label filters for: `service`, `userId`, `sessionId`, `level` +- Quick filters for error-level logs + +### 7.3 Node.js Runtime Dashboard +- Heap usage over time +- Event loop lag +- GC pause durations +- Active handles + +--- + +## 8. Code Changes Summary + +The following changes to existing application code are required: + +| Area | Change | Files Affected | +|------|--------|----------------| +| Session tracking | Add `sessionId` (from `req.user.session_state`) to request context and log output | `request-context.ts`, `request-logging.interceptor.ts`, `logging.middleware.ts` | +| Client IP logging | Add `clientIp` extraction and include in log context | `logging.middleware.ts` or `request-logging.interceptor.ts` | +| API key identifier | Log API key prefix/ID for non-JWT requests | `request-logging.interceptor.ts` | +| Prometheus metrics | Add `prom-client`, expose `/metrics` endpoint, instrument HTTP layer | New metrics module + middleware in `backend-services` | +| Dependencies | Add `prom-client` to `backend-services` | `package.json` | + +No changes to `@ai-di/shared-logging` package for log collection (Loki scrapes stdout). +The `LogContext` interface in the shared logging package needs `sessionId` and `clientIp` fields added. + +--- + +## 9. Configuration & Environment Variables + +New environment variables (configurable per environment): + +| Variable | Description | Example | +|----------|-------------|---------| +| `GRAFANA_ADMIN_PASSWORD` | Grafana admin password | (secret) | +| `LOKI_RETENTION_DAYS` | Log retention period in days | `30` | +| `LOKI_PVC_SIZE` | Loki storage PVC size | `10Gi` | +| `PROMETHEUS_PVC_SIZE` | Prometheus storage PVC size | `10Gi` | +| `METRICS_SCRAPE_INTERVAL` | Prometheus scrape interval | `15s` | + +--- + +## 10. Resilience + +- PLG is **fire-and-forget** — purely observational. If Loki, Prometheus, or Grafana is down, the application continues operating normally. Logs still go to container stdout regardless of Loki's health. +- No alerting rules are included in this scope. Dashboards are for manual inspection. Alerting (via Alertmanager) can be added as a follow-up once meaningful thresholds are established from real usage. + +--- + +## 11. Resource Limits + +PLG container resource limits are **configurable via Helm values** with minimal defaults: + +| Component | Memory Request/Limit | CPU Request/Limit | +|-----------|---------------------|-------------------| +| Loki | 256Mi | 500m | +| Prometheus | 512Mi | 500m | +| Grafana | 256Mi | 250m | +| Promtail (sidecar) | 64Mi | 100m | + +Defaults are sized for low-traffic environments. Override via Helm values per environment as needed. + +--- + +## 12. Constraints & Assumptions + +- The existing Kustomize deployment for the application is **not modified** — PLG is a separate Helm release. +- On OpenShift, Promtail runs as **sidecar containers** (not DaemonSets) to work within tenant-level namespace permissions. +- The `prom-client` library is added only to `backend-services`; other services (frontend, temporal) are not instrumented with custom metrics. +- No IP geo-location or IP-based analytics dashboards are included in this scope. +- No alerting rules or Alertmanager configuration is included in this scope. +- `session_state` from Keycloak JWTs is treated as non-sensitive (opaque UUID, meaningless outside the Keycloak instance). diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/README.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/README.md new file mode 100644 index 000000000..944421bc5 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/README.md @@ -0,0 +1,69 @@ +NOTE: The requirements document for this feature is available here: `feature-docs/20260315052727-plg-monitoring-stack/REQUIREMENTS.md`. + +All user stories files are located in `feature-docs/20260315052727-plg-monitoring-stack/user_stories/`. + +Read both requirements document and individual user story files for implementation details. + +After implementing the user story check it off at the bottom of this file. + +## Log Enrichment (US-001 to US-003) -- HIGH priority +| File | Title | +|---|---| +| `US-001-session-id-logging.md` | Add Session ID to Request Context and Log Output | +| `US-002-client-ip-logging.md` | Add Client IP to Log Output | +| `US-003-api-key-identifier-logging.md` | Add API Key Identifier to Log Output | + +## Prometheus Metrics (US-004) -- HIGH priority +| File | Title | +|---|---| +| `US-004-prometheus-red-metrics.md` | Expose Prometheus RED Metrics Endpoint | + +## PLG Helm Charts (US-005 to US-007) -- HIGH priority +| File | Title | +|---|---| +| `US-005-helm-chart-loki.md` | Create Helm Chart with Loki for Log Aggregation | +| `US-006-helm-chart-prometheus.md` | Add Prometheus to Helm Chart with Scrape Configuration | +| `US-007-helm-chart-grafana.md` | Add Grafana to Helm Chart with Auth and Data Sources | + +## Local & OpenShift Deployment (US-008 to US-010) -- HIGH priority +| File | Title | +|---|---| +| `US-008-docker-compose-monitoring.md` | Create Docker Compose for Local PLG Stack | +| `US-009-promtail-sidecar-openshift.md` | Add Promtail Sidecar Containers to OpenShift Deployments | +| `US-010-openshift-deployment-integration.md` | Integrate PLG Deployment with GitHub Actions and Scripts | + +## Grafana Dashboards (US-011 to US-013) -- MEDIUM priority +| File | Title | +|---|---| +| `US-011-application-overview-dashboard.md` | Create Application Overview Grafana Dashboard | +| `US-012-logs-explorer-dashboard.md` | Create Logs Explorer Grafana Dashboard | +| `US-013-nodejs-runtime-dashboard.md` | Create Node.js Runtime Grafana Dashboard | + +## Suggested Implementation Order (by dependency chain) + +### Phase 1 — Log Enrichment (application code changes) +- [x] **US-001** (Add sessionId from Keycloak session_state to request context and log output) +- [x] **US-002** (Add clientIp extraction from X-Forwarded-For/X-Real-IP/socket to log output) +- [x] **US-003** (Add API key prefix/ID logging for API key-authenticated requests) + +### Phase 2 — Prometheus Metrics (application code changes) +- [x] **US-004** (Add prom-client, expose /metrics endpoint with RED + Node.js runtime metrics) + +### Phase 3 — PLG Helm Charts (infrastructure) +- [x] **US-005** (Create Helm chart with Loki, NDJSON parsing, 30-day retention, PVC storage) +- [x] **US-006** (Add Prometheus to Helm chart with scrape configs for backend-services and Temporal) +- [x] **US-007** (Add Grafana to Helm chart with username/password auth and pre-configured data sources) + +### Phase 4 — Deployment Integration (local + OpenShift) +- [x] **US-008** (Create docker-compose.monitoring.yml with Promtail, Loki, Prometheus, Grafana) +- [x] **US-009** (Add Promtail sidecar containers to all OpenShift application pods) +- [x] **US-010** (Integrate PLG Helm deployment into GitHub Actions workflow and /scripts) + +### Phase 5 — Grafana Dashboards +- [ ] **US-011** (Application Overview dashboard — request rate, error rate, latency, active sessions) +- [ ] **US-012** (Logs Explorer dashboard — filter by service, userId, sessionId, level) +- [ ] **US-013** (Node.js Runtime dashboard — heap, event loop lag, GC, active handles) + +> Stories are ordered by dependency chain for automated implementation. +> Each story should be implementable after all stories in previous phases are complete. +> Do not start a phase until all stories in prior phases are checked off. diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-001-session-id-logging.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-001-session-id-logging.md new file mode 100644 index 000000000..8533fa25e --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-001-session-id-logging.md @@ -0,0 +1,38 @@ +# US-001: Add Session ID to Request Context and Log Output + +**As a** platform operator, +**I want to** see the Keycloak session ID in every log line for authenticated requests, +**So that** I can browse all activity within a single user session for debugging and audit purposes. + +## Acceptance Criteria + +- [x] **Scenario 1**: SessionId added to request context + - **Given** a request authenticated via Keycloak JWT (containing `session_state` claim) + - **When** the request passes through the logging middleware and interceptor + - **Then** the `session_state` value from `req.user` is stored as `sessionId` in the `AsyncLocalStorage` request context alongside the existing `requestId` and `userId` + +- [x] **Scenario 2**: SessionId appears in NDJSON log output + - **Given** a request with a resolved `sessionId` in the request context + - **When** any log statement is emitted via `AppLoggerService` during request processing + - **Then** the NDJSON log line includes a `sessionId` field with the Keycloak `session_state` value + +- [x] **Scenario 3**: LogContext interface updated in shared logging package + - **Given** the `@ai-di/shared-logging` package defines a `LogContext` interface + - **When** the `sessionId` field is added to the interface + - **Then** the `LogContext` interface includes an optional `sessionId: string` field and the logger accepts and outputs it + +- [x] **Scenario 4**: Unauthenticated requests omit sessionId + - **Given** a request to a public endpoint (no JWT present) + - **When** the request is logged + - **Then** the `sessionId` field is omitted from the log output (not set to null or empty string) + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Reuse existing Passport/IdentityGuard infrastructure — no manual JWT decoding +- `session_state` is already available on `req.user` after `KeycloakJwtStrategy` validation +- `session_state` is treated as non-sensitive (opaque Keycloak UUID) +- Files affected: `request-context.ts`, `request-logging.interceptor.ts`, `logging.middleware.ts`, `LogContext` interface in `packages/logging` diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-002-client-ip-logging.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-002-client-ip-logging.md new file mode 100644 index 000000000..11bfb86dd --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-002-client-ip-logging.md @@ -0,0 +1,42 @@ +# US-002: Add Client IP to Log Output + +**As a** platform operator, +**I want to** see the client's IP address in every request log line, +**So that** I can identify the source of requests for security auditing and incident investigation. + +## Acceptance Criteria + +- [x] **Scenario 1**: Client IP extracted from X-Forwarded-For header + - **Given** a request with an `X-Forwarded-For` header containing one or more comma-separated IPs (e.g., `"203.0.113.50, 70.41.3.18, 150.172.238.178"`) + - **When** the request is processed by the logging middleware + - **Then** the first IP in the list is extracted, trimmed, and stored as `clientIp` in the log context + +- [x] **Scenario 2**: Fallback to X-Real-IP header + - **Given** a request without an `X-Forwarded-For` header but with an `X-Real-IP` header + - **When** the request is processed by the logging middleware + - **Then** the `X-Real-IP` header value is used as `clientIp` + +- [x] **Scenario 3**: Fallback to socket remote address + - **Given** a request without `X-Forwarded-For` or `X-Real-IP` headers (e.g., local development) + - **When** the request is processed by the logging middleware + - **Then** `req.socket.remoteAddress` is used as `clientIp` + +- [x] **Scenario 4**: ClientIp appears in NDJSON log output + - **Given** a request with a resolved `clientIp` + - **When** any log statement is emitted via `AppLoggerService` during request processing + - **Then** the NDJSON log line includes a `clientIp` field + +- [x] **Scenario 5**: LogContext interface updated for clientIp + - **Given** the `@ai-di/shared-logging` package defines a `LogContext` interface + - **When** the `clientIp` field is added to the interface + - **Then** the `LogContext` interface includes an optional `clientIp: string` field and the logger accepts and outputs it + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Extraction priority: `X-Forwarded-For` (first entry) > `X-Real-IP` > `req.socket.remoteAddress` +- On OpenShift, the client IP arrives via `X-Forwarded-For` due to reverse proxy/ingress +- Files affected: `logging.middleware.ts` or `request-logging.interceptor.ts`, `LogContext` interface in `packages/logging` diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-003-api-key-identifier-logging.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-003-api-key-identifier-logging.md new file mode 100644 index 000000000..c150f8538 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-003-api-key-identifier-logging.md @@ -0,0 +1,37 @@ +# US-003: Add API Key Identifier to Log Output + +**As a** platform operator, +**I want to** see an API key identifier in log lines for API key-authenticated requests, +**So that** I can filter and audit activity by API consumer without exposing the full key. + +## Acceptance Criteria + +- [x] **Scenario 1**: API key prefix logged for API key requests + - **Given** a request authenticated via API key (x-api-key header, validated by `ApiKeyAuthGuard`) + - **When** the request is processed by the logging interceptor + - **Then** the API key prefix or key ID from the database is included as `apiKeyId` in the NDJSON log output + +- [x] **Scenario 2**: No sessionId for API key requests + - **Given** a request authenticated via API key (no JWT present) + - **When** the request is logged + - **Then** the `sessionId` field is omitted and `apiKeyId` is present instead + +- [x] **Scenario 3**: JWT-authenticated requests omit apiKeyId + - **Given** a request authenticated via Keycloak JWT + - **When** the request is logged + - **Then** the `apiKeyId` field is omitted and `sessionId` is present instead + +- [x] **Scenario 4**: Full API key value is never logged + - **Given** any API key-authenticated request + - **When** the request is logged + - **Then** only the key prefix or database ID appears in logs — the full API key value is never written to log output + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- The `ApiKeyAuthGuard` already sets `request.apiKeyGroupId` on successful validation +- Use the key prefix (already stored in DB) or the key's database ID as the identifier +- Files affected: `request-logging.interceptor.ts` diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-004-prometheus-red-metrics.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-004-prometheus-red-metrics.md new file mode 100644 index 000000000..eea212c34 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-004-prometheus-red-metrics.md @@ -0,0 +1,38 @@ +# US-004: Expose Prometheus RED Metrics Endpoint + +**As a** platform operator, +**I want to** collect HTTP request rate, error rate, and duration metrics from backend-services, +**So that** I can monitor application health and performance via Prometheus and Grafana. + +## Acceptance Criteria + +- [x] **Scenario 1**: /metrics endpoint exposes RED metrics + - **Given** the `prom-client` library is installed in backend-services + - **When** Prometheus scrapes `GET /metrics` + - **Then** the response includes `http_requests_total` (counter by method, path, status code), `http_request_duration_seconds` (histogram by method, path), and `http_request_errors_total` (counter of 4xx/5xx responses) + +- [x] **Scenario 2**: Metrics are collected for every HTTP request + - **Given** the metrics middleware is active + - **When** any HTTP request is processed by the backend-services application + - **Then** the request increments `http_requests_total`, records duration in `http_request_duration_seconds`, and increments `http_request_errors_total` if the status code is 4xx or 5xx + +- [x] **Scenario 3**: Node.js runtime default metrics are exposed + - **Given** `prom-client` default metrics collection is enabled + - **When** Prometheus scrapes `GET /metrics` + - **Then** the response includes Node.js runtime metrics: event loop lag, heap usage (used, total, external), active handles/requests, and GC pause durations + +- [x] **Scenario 4**: /metrics endpoint is not publicly accessible on OpenShift + - **Given** the backend-services application is deployed on OpenShift with a Route + - **When** an external client attempts to access `/metrics` via the Route URL + - **Then** the request is blocked — `/metrics` is excluded from the Route and only accessible within the cluster + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Add `prom-client` as a dependency to `apps/backend-services/package.json` +- Create a new NestJS metrics module with middleware to instrument HTTP requests +- The `/metrics` endpoint should be excluded from authentication guards (Prometheus scrapes without a JWT) +- Exclude the `/metrics` path itself from being counted in RED metrics to avoid self-referential inflation diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-005-helm-chart-loki.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-005-helm-chart-loki.md new file mode 100644 index 000000000..52d7d6290 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-005-helm-chart-loki.md @@ -0,0 +1,37 @@ +# US-005: Create Helm Chart with Loki for Log Aggregation + +**As a** platform operator, +**I want to** deploy Loki via a Helm chart with configurable retention and storage, +**So that** application logs are aggregated and queryable from a central location. + +## Acceptance Criteria + +- [x] **Scenario 1**: Helm chart structure created + - **Given** the project has no existing Helm charts for PLG + - **When** the Helm chart is created + - **Then** a Helm chart directory exists with configurable values for Loki, including PVC size (`LOKI_PVC_SIZE` default `10Gi`), retention period (`LOKI_RETENTION_DAYS` default `30`), and resource limits (memory `256Mi`, CPU `500m`) + +- [x] **Scenario 2**: Loki configured for NDJSON log parsing + - **Given** application services output NDJSON structured logs to stdout + - **When** Loki ingests logs via Promtail + - **Then** Loki can parse and index NDJSON fields (timestamp, level, service, requestId, userId, sessionId, clientIp, method, path, statusCode, durationMs) + +- [x] **Scenario 3**: 30-day log retention enforced + - **Given** Loki is configured with a 30-day retention period + - **When** logs older than 30 days exist in storage + - **Then** Loki automatically purges expired logs according to the `retention_period` configuration + +- [x] **Scenario 4**: Helm values configurable per environment + - **Given** the Helm chart has a `values.yaml` with defaults + - **When** deploying to different environments (local Docker vs. OpenShift) + - **Then** environment-specific values can override defaults (PVC size, retention, resource limits) via values files or `--set` flags + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Use community Loki Helm chart as a dependency or base +- The existing Kustomize deployment for the application is not modified +- Loki stores data in a PVC on OpenShift diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-006-helm-chart-prometheus.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-006-helm-chart-prometheus.md new file mode 100644 index 000000000..785a27f92 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-006-helm-chart-prometheus.md @@ -0,0 +1,37 @@ +# US-006: Add Prometheus to Helm Chart with Scrape Configuration + +**As a** platform operator, +**I want to** deploy Prometheus via the Helm chart with pre-configured scrape targets, +**So that** application and Temporal metrics are automatically collected without manual configuration. + +## Acceptance Criteria + +- [x] **Scenario 1**: Prometheus deployed via Helm chart + - **Given** the PLG Helm chart includes Prometheus configuration + - **When** the chart is deployed + - **Then** Prometheus is running with a PVC for metrics storage (configurable via `PROMETHEUS_PVC_SIZE`, default `10Gi`) and resource limits (memory `512Mi`, CPU `500m`) + +- [x] **Scenario 2**: Backend-services scrape target configured + - **Given** Prometheus scrape configs are defined in the Helm chart values + - **When** Prometheus starts + - **Then** it scrapes the backend-services `/metrics` endpoint at the configured interval (default `15s`, configurable via `METRICS_SCRAPE_INTERVAL`) + +- [x] **Scenario 3**: Temporal server scrape target configured + - **Given** Temporal server exposes a built-in `/metrics` endpoint + - **When** Prometheus starts + - **Then** it scrapes the Temporal server's `/metrics` endpoint at the configured interval + +- [x] **Scenario 4**: Scrape interval configurable + - **Given** a deployment with custom scrape interval requirements + - **When** `METRICS_SCRAPE_INTERVAL` is set to a different value (e.g., `30s`) + - **Then** Prometheus uses the specified interval for all scrape targets + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Use community Prometheus Helm chart as a dependency or base +- No Alertmanager configuration is included in this scope +- Scrape targets reference service names within the same namespace diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-007-helm-chart-grafana.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-007-helm-chart-grafana.md new file mode 100644 index 000000000..2118588d3 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-007-helm-chart-grafana.md @@ -0,0 +1,37 @@ +# US-007: Add Grafana to Helm Chart with Auth and Data Sources + +**As a** developer, +**I want to** deploy Grafana via the Helm chart with pre-configured data sources for Prometheus and Loki, +**So that** I can immediately query metrics and logs after deployment without manual setup. + +## Acceptance Criteria + +- [x] **Scenario 1**: Grafana deployed with username/password auth + - **Given** the PLG Helm chart includes Grafana configuration + - **When** the chart is deployed + - **Then** Grafana is running with configurable admin credentials (`GRAFANA_ADMIN_PASSWORD`) and resource limits (memory `256Mi`, CPU `250m`) + +- [x] **Scenario 2**: Prometheus data source pre-configured + - **Given** Grafana is deployed alongside Prometheus + - **When** a user logs into Grafana + - **Then** a Prometheus data source is already configured and available for querying without manual setup + +- [x] **Scenario 3**: Loki data source pre-configured + - **Given** Grafana is deployed alongside Loki + - **When** a user logs into Grafana + - **Then** a Loki data source is already configured and available for log querying without manual setup + +- [x] **Scenario 4**: Grafana not exposed via OpenShift Route + - **Given** the Helm chart is deployed on OpenShift + - **When** the deployment completes + - **Then** no OpenShift Route is created for Grafana — developers access it via port-forwarding/tunneling (same pattern as Temporal UI) + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Use community Grafana Helm chart as a dependency or base +- Data sources are provisioned via Grafana's provisioning mechanism (ConfigMaps or Helm values) +- Default local port: `localhost:3001` diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-008-docker-compose-monitoring.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-008-docker-compose-monitoring.md new file mode 100644 index 000000000..7c620c01b --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-008-docker-compose-monitoring.md @@ -0,0 +1,42 @@ +# US-008: Create Docker Compose for Local PLG Stack + +**As a** developer, +**I want to** run PLG locally via an opt-in Docker Compose file, +**So that** I can test logging, metrics, and dashboards in my local development environment without affecting the core dev stack. + +## Acceptance Criteria + +- [x] **Scenario 1**: Separate docker-compose.monitoring.yml created + - **Given** the project has an existing `docker-compose.yml` for core services (PostgreSQL, MinIO) + - **When** a developer wants to run PLG locally + - **Then** they can run `docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up` to start both the core stack and PLG together + +- [x] **Scenario 2**: Promtail auto-discovers container logs via Docker socket + - **Given** the `docker-compose.monitoring.yml` includes a Promtail container + - **When** the monitoring stack starts + - **Then** Promtail mounts `/var/run/docker.sock`, auto-discovers all running containers, and forwards their stdout logs to Loki with service labels + +- [x] **Scenario 3**: Community-standard ports exposed locally + - **Given** the monitoring compose file defines port mappings + - **When** the stack is running + - **Then** Grafana is available at `localhost:3001`, Prometheus at `localhost:9090`, and Loki at `localhost:3100` + +- [x] **Scenario 4**: Data persists via Docker volumes + - **Given** the monitoring stack stores logs and metrics + - **When** the stack is stopped and restarted + - **Then** previously collected logs (Loki) and metrics (Prometheus) are retained via named Docker volumes + +- [x] **Scenario 5**: Compatible with existing startup scripts and VS Code task + - **Given** the project has startup scripts in `package.json` and a VS Code `Dev:All` task + - **When** the monitoring stack is integrated + - **Then** existing startup workflows continue to function and the monitoring stack can be optionally started alongside them + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- PLG is opt-in — the core dev stack works without it +- Promtail needs Docker socket access for container log discovery +- Grafana should have Prometheus and Loki data sources pre-configured in the compose setup diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-009-promtail-sidecar-openshift.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-009-promtail-sidecar-openshift.md new file mode 100644 index 000000000..2702ef9c7 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-009-promtail-sidecar-openshift.md @@ -0,0 +1,42 @@ +# US-009: Add Promtail Sidecar Containers to OpenShift Deployments + +**As a** platform operator, +**I want to** collect logs from all application pods on OpenShift via Promtail sidecar containers, +**So that** logs are forwarded to Loki without requiring cluster-level DaemonSet permissions. + +## Acceptance Criteria + +- [x] **Scenario 1**: Promtail sidecar added to backend-services pod + - **Given** the backend-services deployment already has a logrotate sidecar writing to `/var/log/app/` + - **When** the Promtail sidecar is added to the pod spec + - **Then** Promtail tails log files from the shared log volume and forwards them to Loki with `service=backend-services` label + +- [x] **Scenario 2**: Promtail sidecar added to temporal-worker pod + - **Given** the temporal-worker deployment runs in the same namespace + - **When** the Promtail sidecar is added to the pod spec + - **Then** Promtail collects and forwards temporal-worker logs to Loki with `service=temporal-worker` label + +- [x] **Scenario 3**: Promtail sidecar added to temporal-server pod + - **Given** the temporal-server deployment runs in the same namespace + - **When** the Promtail sidecar is added to the pod spec + - **Then** Promtail collects and forwards temporal-server logs to Loki with `service=temporal-server` label + +- [x] **Scenario 4**: Promtail sidecar resource limits configured + - **Given** the Promtail sidecar has configurable resource limits + - **When** deployed on OpenShift + - **Then** the sidecar uses minimal resources (default: memory `64Mi`, CPU `100m`) configurable via Helm values + +- [x] **Scenario 5**: Logs from frontend and PostgreSQL collected + - **Given** frontend (nginx) and PostgreSQL pods run in the same namespace + - **When** Promtail sidecars are added to these pods + - **Then** logs are forwarded to Loki with appropriate service labels (`service=frontend`, `service=postgresql`) + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Sidecar pattern works within tenant-level namespace permissions (no cluster-admin needed) +- Backend-services already uses a logrotate sidecar establishing the shared volume pattern +- Promtail sidecar config references the Loki endpoint within the namespace diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-010-openshift-deployment-integration.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-010-openshift-deployment-integration.md new file mode 100644 index 000000000..1daa17dd0 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-010-openshift-deployment-integration.md @@ -0,0 +1,38 @@ +# US-010: Integrate PLG Deployment with GitHub Actions and Scripts + +**As a** platform operator, +**I want to** deploy the PLG stack using the existing GitHub Actions workflow and local deployment scripts, +**So that** PLG is deployed consistently alongside the application without a separate deployment process. + +## Acceptance Criteria + +- [x] **Scenario 1**: GitHub Actions workflow deploys PLG Helm chart + - **Given** the existing GitHub Actions workflow builds and deploys the application + - **When** the workflow runs + - **Then** it also deploys the PLG Helm chart to the same namespace as the application + +- [x] **Scenario 2**: Local deployment scripts deploy PLG + - **Given** the existing deployment scripts in `/scripts` handle application deployment + - **When** a developer runs the deployment scripts locally + - **Then** the PLG Helm chart is also deployed to the target namespace + +- [x] **Scenario 3**: PLG environment variables configurable per overlay + - **Given** environment-specific configuration exists for dev, test, and prod + - **When** deploying to a specific environment + - **Then** PLG-specific variables (`GRAFANA_ADMIN_PASSWORD`, `LOKI_RETENTION_DAYS`, `LOKI_PVC_SIZE`, `PROMETHEUS_PVC_SIZE`, `METRICS_SCRAPE_INTERVAL`) are sourced from the environment's configuration + +- [x] **Scenario 4**: PLG deployment does not affect existing Kustomize deployment + - **Given** the application is deployed via Kustomize + - **When** the PLG Helm chart is deployed + - **Then** the existing Kustomize resources are not modified or disrupted + +## Priority +- [x] High (Must Have) +- [ ] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- PLG is a separate Helm release, not integrated into Kustomize +- The GitHub Actions workflow needs a Helm install/upgrade step added +- Deployment scripts need a Helm install/upgrade command added +- Environment variables are managed via config files or secrets per environment diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-011-application-overview-dashboard.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-011-application-overview-dashboard.md new file mode 100644 index 000000000..9aba3d885 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-011-application-overview-dashboard.md @@ -0,0 +1,42 @@ +# US-011: Create Application Overview Grafana Dashboard + +**As a** developer, +**I want to** see an at-a-glance application health dashboard in Grafana, +**So that** I can quickly assess request rates, error rates, latency, and active sessions. + +## Acceptance Criteria + +- [ ] **Scenario 1**: Request rate panel + - **Given** Prometheus is scraping backend-services `/metrics` + - **When** a user opens the Application Overview dashboard in Grafana + - **Then** a panel displays the current request rate (requests/sec) derived from `http_requests_total` + +- [ ] **Scenario 2**: Error rate panel + - **Given** Prometheus is collecting error metrics + - **When** a user views the dashboard + - **Then** a panel displays the error rate (4xx/5xx per second) derived from `http_request_errors_total` + +- [ ] **Scenario 3**: Latency percentiles panel + - **Given** Prometheus is collecting duration histograms + - **When** a user views the dashboard + - **Then** a panel displays p50, p95, and p99 latency percentiles derived from `http_request_duration_seconds` + +- [ ] **Scenario 4**: Active sessions panel + - **Given** Loki is ingesting logs with `sessionId` fields + - **When** a user views the dashboard + - **Then** a panel displays the count of unique `sessionId` values seen in the last 5 minutes + +- [ ] **Scenario 5**: Dashboard shipped as ConfigMap + - **Given** the dashboard JSON definition exists + - **When** the Helm chart is deployed + - **Then** the dashboard is automatically provisioned in Grafana via a ConfigMap (dashboards-as-code) + +## Priority +- [ ] High (Must Have) +- [x] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Dashboard is defined as a JSON file and mounted via Grafana provisioning +- Requires Prometheus and Loki data sources to be pre-configured (US-007) +- Requires RED metrics endpoint (US-004) and sessionId logging (US-001) diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-012-logs-explorer-dashboard.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-012-logs-explorer-dashboard.md new file mode 100644 index 000000000..6766d4a28 --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-012-logs-explorer-dashboard.md @@ -0,0 +1,42 @@ +# US-012: Create Logs Explorer Grafana Dashboard + +**As a** developer, +**I want to** browse and filter application logs in Grafana with pre-configured label filters, +**So that** I can quickly find logs for a specific user session, service, or error level. + +## Acceptance Criteria + +- [ ] **Scenario 1**: Loki data source pre-configured in dashboard + - **Given** Grafana has a Loki data source configured + - **When** a user opens the Logs Explorer dashboard + - **Then** the dashboard uses the Loki data source by default with a log query panel ready + +- [ ] **Scenario 2**: Filter by service label + - **Given** logs are labeled with `service` (e.g., `backend-services`, `temporal-worker`) + - **When** a user selects a service from the filter dropdown + - **Then** only logs from that service are displayed + +- [ ] **Scenario 3**: Filter by userId and sessionId + - **Given** logs contain `userId` and `sessionId` fields + - **When** a user enters a `userId` or `sessionId` value in the filter + - **Then** only logs matching that user or session are displayed, showing all API activity within the session + +- [ ] **Scenario 4**: Filter by log level with error quick-filter + - **Given** logs contain a `level` field (debug, info, warn, error) + - **When** a user selects the error-level quick filter + - **Then** only error-level logs are displayed + +- [ ] **Scenario 5**: Dashboard shipped as ConfigMap + - **Given** the dashboard JSON definition exists + - **When** the Helm chart is deployed + - **Then** the dashboard is automatically provisioned in Grafana via a ConfigMap + +## Priority +- [ ] High (Must Have) +- [x] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- This dashboard enables the session browsing use case described in requirements Section 4.3 +- Requires Loki data source (US-007) and sessionId/userId logging (US-001) +- Label filters are implemented as Grafana template variables diff --git a/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-013-nodejs-runtime-dashboard.md b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-013-nodejs-runtime-dashboard.md new file mode 100644 index 000000000..622e4b27f --- /dev/null +++ b/feature-docs/20260315052727-plg-monitoring-stack/user_stories/US-013-nodejs-runtime-dashboard.md @@ -0,0 +1,42 @@ +# US-013: Create Node.js Runtime Grafana Dashboard + +**As a** developer, +**I want to** monitor Node.js runtime health in Grafana, +**So that** I can identify memory leaks, event loop bottlenecks, and GC pressure in backend-services. + +## Acceptance Criteria + +- [ ] **Scenario 1**: Heap usage panel + - **Given** Prometheus is scraping Node.js runtime metrics from backend-services + - **When** a user opens the Node.js Runtime dashboard in Grafana + - **Then** a panel displays heap usage over time (used, total, external memory) + +- [ ] **Scenario 2**: Event loop lag panel + - **Given** `prom-client` default metrics include event loop lag + - **When** a user views the dashboard + - **Then** a panel displays event loop lag over time + +- [ ] **Scenario 3**: GC pause durations panel + - **Given** `prom-client` default metrics include GC pause durations + - **When** a user views the dashboard + - **Then** a panel displays garbage collection pause durations over time + +- [ ] **Scenario 4**: Active handles panel + - **Given** `prom-client` default metrics include active handles count + - **When** a user views the dashboard + - **Then** a panel displays the number of active handles over time + +- [ ] **Scenario 5**: Dashboard shipped as ConfigMap + - **Given** the dashboard JSON definition exists + - **When** the Helm chart is deployed + - **Then** the dashboard is automatically provisioned in Grafana via a ConfigMap + +## Priority +- [ ] High (Must Have) +- [x] Medium (Should Have) +- [ ] Low (Nice to Have) + +## Technical Notes / Assumptions +- Requires Prometheus data source (US-007) and runtime metrics endpoint (US-004) +- All metrics come from `prom-client` default metrics collection +- Dashboard is defined as JSON and provisioned via Grafana's ConfigMap mechanism diff --git a/package-lock.json b/package-lock.json index 753b46ba5..a42ebda51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,6 +60,7 @@ "passport-jwt": "^4.0.1", "pg": "^8.16.3", "prisma": "7.2.0", + "prom-client": "^15.1.3", "rxjs": "^7.8.2", "uuid": "13.0.0" }, @@ -5582,6 +5583,15 @@ "npm": ">=5.10.0" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@paralleldrive/cuid2": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz", @@ -10386,6 +10396,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -17940,6 +17956,19 @@ "dev": true, "license": "MIT" }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -19913,6 +19942,15 @@ "streamx": "^2.15.0" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/terser": { "version": "5.44.1", "resolved": "https://registry.npmjs.org/terser/-/terser-5.44.1.tgz", diff --git a/package.json b/package.json index 160bb74b5..bf7aee10f 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,9 @@ "test:dir": "playwright test", "test:db:reset": "cd apps/backend-services && PRISMA_USER_CONSENT_FOR_DANGEROUS_AI_ACTION=\"yes\" npx prisma migrate reset --force && npm run db:seed", "generate:openapi": "cd apps/backend-services && SSO_AUTH_SERVER_URL=http://localhost:8080/realms/test SSO_REALM=test SSO_CLIENT_ID=test-client SSO_CLIENT_SECRET=test-secret FRONTEND_URL=http://localhost:3000 KEYCLOAK_PUBLIC_KEY=dummy-key DATABASE_URL=postgresql://dummy:dummy@localhost:5432/dummy BLOB_STORAGE_TYPE=filesystem BLOB_STORAGE_ROOT=/tmp/uploads TEMPORAL_ADDRESS=localhost:7233 TEMPORAL_NAMESPACE=default TS_NODE_TRANSPILE_ONLY=true node -r ts-node/register -r tsconfig-paths/register ../../docs/generate-openapi.ts", + "dev:monitoring": "docker compose -f deployments/local/docker-compose.monitoring.yml up -d", + "dev:monitoring:down": "docker compose -f deployments/local/docker-compose.monitoring.yml down", + "dev:monitoring:logs": "docker compose -f deployments/local/docker-compose.monitoring.yml logs -f", "docs:build": "npm run generate:openapi && cd docs && bash build.sh" }, "devDependencies": { diff --git a/packages/logging/src/logger.test.ts b/packages/logging/src/logger.test.ts index f3b335cd5..4d32e0da7 100644 --- a/packages/logging/src/logger.test.ts +++ b/packages/logging/src/logger.test.ts @@ -109,6 +109,31 @@ describe("createLogger", () => { out.restore(); } }); + + it("includes sessionId in the emitted object when provided", () => { + const out = captureStdout(); + try { + const log = createLogger(SERVICE); + log.info("session log", { sessionId: "sess-abc-123" }); + const entry = parseLastLine(out.lines); + expect(entry.sessionId).toBe("sess-abc-123"); + } finally { + out.restore(); + } + }); + + it("omits sessionId from the emitted object when not provided", () => { + const out = captureStdout(); + try { + const log = createLogger(SERVICE); + log.info("no session", { requestId: "req-1" }); + const entry = parseLastLine(out.lines); + expect(entry.sessionId).toBeUndefined(); + expect(entry.requestId).toBe("req-1"); + } finally { + out.restore(); + } + }); }); describe("LOG_LEVEL filtering", () => { diff --git a/packages/logging/src/types.ts b/packages/logging/src/types.ts index 928acf005..f2921fd75 100644 --- a/packages/logging/src/types.ts +++ b/packages/logging/src/types.ts @@ -15,6 +15,8 @@ export const LOG_LEVELS: readonly LogLevel[] = [ /** Optional context fields with consistent camelCase naming (per REQUIREMENTS Section 5). */ export interface LogContext { requestId?: string; + sessionId?: string; + clientIp?: string; workflowExecutionId?: string; documentId?: string; userId?: string; diff --git a/scripts/oc-deploy.sh b/scripts/oc-deploy.sh index 33f3bb990..af14bd160 100644 --- a/scripts/oc-deploy.sh +++ b/scripts/oc-deploy.sh @@ -474,9 +474,56 @@ oc apply -k "${OVERLAY_DIR}" -n "${NAMESPACE}" || { log_info "Resources applied successfully." # ============================================================ -# Step 7: Create/update instance secrets +# Step 7: Deploy PLG monitoring stack (Helm) # ============================================================ -log_step "Step 7: Creating instance secrets" +log_step "Step 7: Deploying PLG monitoring stack" + +if ! command -v helm &>/dev/null; then + log_error "'helm' CLI is not installed. Install Helm to deploy the PLG monitoring stack." + log_error "Skipping PLG deployment — the application will still work without it." +else + PLG_CHART_DIR="${PROJECT_ROOT}/deployments/openshift/helm/plg" + PLG_RELEASE_NAME="${INSTANCE_NAME}-plg" + + # Read PLG-specific configuration with defaults + GRAFANA_ADMIN_PASSWORD=$(get_config "GRAFANA_ADMIN_PASSWORD" 2>/dev/null || echo "admin") + LOKI_RETENTION_DAYS=$(get_config "LOKI_RETENTION_DAYS" 2>/dev/null || echo "30") + LOKI_PVC_SIZE=$(get_config "LOKI_PVC_SIZE" 2>/dev/null || echo "10Gi") + PROMETHEUS_PVC_SIZE=$(get_config "PROMETHEUS_PVC_SIZE" 2>/dev/null || echo "10Gi") + METRICS_SCRAPE_INTERVAL=$(get_config "METRICS_SCRAPE_INTERVAL" 2>/dev/null || echo "15s") + + # Scrape targets use instance-prefixed service names (Kustomize namePrefix adds -) + BACKEND_SERVICES_HOST=$(get_resource_name "${INSTANCE_NAME}" "backend-services") + TEMPORAL_HOST=$(get_resource_name "${INSTANCE_NAME}" "temporal") + + log_info "PLG release name: ${PLG_RELEASE_NAME}" + log_info "Helm chart: ${PLG_CHART_DIR}" + log_info "Loki retention: ${LOKI_RETENTION_DAYS} days, PVC: ${LOKI_PVC_SIZE}" + log_info "Prometheus PVC: ${PROMETHEUS_PVC_SIZE}, scrape interval: ${METRICS_SCRAPE_INTERVAL}" + + helm upgrade --install "${PLG_RELEASE_NAME}" "${PLG_CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + -f "${PLG_CHART_DIR}/values-openshift.yaml" \ + --set "grafana.adminPassword=${GRAFANA_ADMIN_PASSWORD}" \ + --set "loki.retentionDays=${LOKI_RETENTION_DAYS}" \ + --set "loki.pvcSize=${LOKI_PVC_SIZE}" \ + --set "prometheus.pvcSize=${PROMETHEUS_PVC_SIZE}" \ + --set "prometheus.scrapeInterval=${METRICS_SCRAPE_INTERVAL}" \ + --set "prometheus.scrapeTargets.backendServices.host=${BACKEND_SERVICES_HOST}" \ + --set "prometheus.scrapeTargets.temporalServer.host=${TEMPORAL_HOST}" \ + --wait --timeout 120s || { + log_error "Failed to deploy PLG monitoring stack." + log_error "The application deployment is unaffected. PLG can be deployed manually later." + log_error " helm upgrade --install ${PLG_RELEASE_NAME} ${PLG_CHART_DIR} -n ${NAMESPACE} -f ${PLG_CHART_DIR}/values-openshift.yaml" + } + + log_info "PLG monitoring stack deployed successfully." +fi + +# ============================================================ +# Step 8: Create/update instance secrets +# ============================================================ +log_step "Step 8: Creating instance secrets" # Secrets are read from the same env file loaded in Step 3 (via get_config). # No separate secrets file is needed. @@ -524,9 +571,9 @@ oc label secret "${WORKER_SECRET_NAME}" \ log_info "Instance secrets created successfully." # ============================================================ -# Step 8: Wait for rollout completion +# Step 9: Wait for rollout completion # ============================================================ -log_step "Step 8: Waiting for rollout completion" +log_step "Step 9: Waiting for rollout completion" DEPLOYMENT_SERVICES=("backend-services" "frontend" "temporal" "temporal-ui" "temporal-worker") @@ -547,9 +594,9 @@ done log_info "All deployments rolled out successfully." # ============================================================ -# Step 9: Print access URLs +# Step 10: Print access URLs # ============================================================ -log_step "Step 9: Deployment Complete" +log_step "Step 10: Deployment Complete" FRONTEND_ROUTE="https://${INSTANCE_NAME}-frontend-${NAMESPACE}.${CLUSTER_DOMAIN}" BACKEND_ROUTE="https://${INSTANCE_NAME}-backend-${NAMESPACE}.${CLUSTER_DOMAIN}" @@ -571,6 +618,10 @@ To access Temporal UI (not publicly exposed): oc port-forward deployment/${INSTANCE_NAME}-temporal-ui 8080:8080 -n ${NAMESPACE} Then open http://localhost:8080 +To access Grafana (not publicly exposed): + oc port-forward svc/${INSTANCE_NAME}-plg-grafana 3001:3001 -n ${NAMESPACE} + Then open http://localhost:3001 (admin / ) + To tear down this instance: ./scripts/oc-teardown.sh --instance ${INSTANCE_NAME} diff --git a/scripts/oc-teardown.sh b/scripts/oc-teardown.sh index 98dbee197..377483a4a 100644 --- a/scripts/oc-teardown.sh +++ b/scripts/oc-teardown.sh @@ -174,6 +174,24 @@ done log_info "All instance resources deleted." +# ============================================================ +# Step 3b: Uninstall PLG Helm release +# ============================================================ +PLG_RELEASE_NAME="${INSTANCE_NAME}-plg" + +if command -v helm &>/dev/null; then + if helm status "${PLG_RELEASE_NAME}" -n "${NAMESPACE}" &>/dev/null; then + log_info "Uninstalling PLG Helm release: ${PLG_RELEASE_NAME}" + helm uninstall "${PLG_RELEASE_NAME}" -n "${NAMESPACE}" || { + log_error "Failed to uninstall PLG Helm release '${PLG_RELEASE_NAME}'. Continuing with teardown." + } + else + log_info "No PLG Helm release '${PLG_RELEASE_NAME}' found — skipping." + fi +else + log_info "Helm CLI not installed — skipping PLG release cleanup." +fi + # ============================================================ # Step 4: Verify deletion # ============================================================