Merge pull request Talenttrust#136 from njohnchi/feature/backend-26-metrics-and-observability

feat: implement metrics and observability with tests and docs

Author: mikewheeleer

docs/backend/observability.md

@@ -0,0 +1,107 @@
+# Backend Observability
+
+This document explains service-level health signaling and Prometheus metrics exposure in `Talenttrust-Backend`.
+
+## Endpoints
+
+### `GET /health/live`
+
+Returns process liveness only.
+
+```json
+{
+  "status": "up",
+  "service": "talenttrust-backend"
+}
+```
+
+### `GET /health` and `GET /health/ready`
+
+Returns service-level health with runtime and dependency signals.
+
+```json
+{
+  "service": "talenttrust-backend",
+  "status": "up",
+  "timestamp": "2026-03-24T00:00:00.000Z",
+  "uptimeSeconds": 102.34,
+  "signals": {
+    "eventLoopLagMs": 12,
+    "heapUsedBytes": 23893648,
+    "heapTotalBytes": 30523392,
+    "heapUsedRatio": 0.78
+  },
+  "dependencies": []
+}
+```
+
+Status behavior:
+
+- `up`: all local and dependency checks are healthy.
+- `degraded`: one or more checks are elevated but still serving.
+- `down`: one or more checks are critical. HTTP status is `503`.
+
+### `GET /metrics`
+
+Exposes metrics in Prometheus text format.
+
+If `METRICS_AUTH_TOKEN` is set, requests must include:
+
+```text
+Authorization: Bearer <token>
+```
+
+If auth is missing/invalid, route returns `401`.
+
+## Configuration
+
+| Variable | Default | Notes |
+|---|---|---|
+| `PORT` | `3001` | API listener port |
+| `SERVICE_NAME` | `talenttrust-backend` | Name used in health payload and metrics labels |
+| `METRICS_ENABLED` | `true` | Set to `false` to return `404` on `/metrics` |
+| `METRICS_AUTH_TOKEN` | _unset_ | Enables bearer-token protection for `/metrics` |
+
+## Exported Prometheus Metrics
+
+- `http_requests_total{method,route,status_code}`
+- `http_request_duration_seconds{method,route,status_code}`
+- `service_health_status{service}` (`up=2`, `degraded=1`, `down=0`)
+- Node/process default metrics from `prom-client` (prefixed by `<service>_`)
+
+## Security and Threat Notes
+
+### Threat: unauthorized scraping of operational details
+
+Mitigation:
+
+- Token-gate `/metrics` with `METRICS_AUTH_TOKEN`.
+- Restrict route at network boundary (ingress, WAF, service mesh) to trusted scrapers only.
+
+### Threat: high cardinality metrics causing memory growth
+
+Mitigation:
+
+- Route labels use bounded path values from Express route templates.
+- No request payload, IDs, or user-provided fields are added as labels.
+
+### Threat: health endpoint leaking secrets
+
+Mitigation:
+
+- Health responses contain only runtime capacity indicators and dependency status.
+- Avoid including credentials or raw stack traces in dependency details.
+
+## Prometheus scrape example
+
+```yaml
+scrape_configs:
+  - job_name: talenttrust-backend
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['talenttrust-backend:3001']
+    authorization:
+      type: Bearer
+      credentials: ${METRICS_AUTH_TOKEN}
+```
+

src/observability/health-service.test.ts

@@ -0,0 +1,93 @@
+import {
+  defaultThresholds,
+  HealthService,
+  RuntimeSignalProviders,
+} from './health-service';
+import { DependencyChecker } from './types';
+
+function createProviders(overrides: Partial<RuntimeSignalProviders> = {}): RuntimeSignalProviders {
+  return {
+    now: () => new Date('2026-03-24T00:00:00.000Z'),
+    uptimeSeconds: () => 42,
+    eventLoopLagMs: () => 25,
+    memoryUsage: () => ({
+      rss: 10,
+      heapTotal: 100,
+      heapUsed: 45,
+      external: 1,
+      arrayBuffers: 1,
+    }),
+    ...overrides,
+  };
+}
+
+describe('HealthService', () => {
+  it('returns degraded when event loop lag crosses degraded threshold', async () => {
+    const service = new HealthService(
+      'talenttrust-backend',
+      [],
+      createProviders({
+        eventLoopLagMs: () => defaultThresholds.degradedEventLoopLagMs,
+      }),
+    );
+
+    const report = await service.getReport();
+
+    expect(report.status).toBe('degraded');
+    expect(report.signals.eventLoopLagMs).toBe(defaultThresholds.degradedEventLoopLagMs);
+  });
+
+  it('returns down when memory usage crosses down threshold', async () => {
+    const service = new HealthService(
+      'talenttrust-backend',
+      [],
+      createProviders({
+        memoryUsage: () => ({
+          rss: 10,
+          heapTotal: 100,
+          heapUsed: 95,
+          external: 1,
+          arrayBuffers: 1,
+        }),
+      }),
+    );
+
+    const report = await service.getReport();
+
+    expect(report.status).toBe('down');
+    expect(report.signals.heapUsedRatio).toBe(0.95);
+  });
+
+  it('marks dependency as down when checker throws and keeps error detail', async () => {
+    const failingDependency: DependencyChecker = {
+      name: 'database',
+      check: async () => {
+        throw new Error('dial timeout');
+      },
+    };
+
+    const service = new HealthService(
+      'talenttrust-backend',
+      [failingDependency],
+      createProviders(),
+    );
+
+    const report = await service.getReport();
+
+    expect(report.status).toBe('down');
+    expect(report.dependencies).toHaveLength(1);
+    expect(report.dependencies[0].name).toBe('database');
+    expect(report.dependencies[0].status).toBe('down');
+    expect(report.dependencies[0].details).toContain('dial timeout');
+  });
+
+  it('closes provider resources on close()', () => {
+    const close = jest.fn();
+    const service = new HealthService('talenttrust-backend', [], createProviders({ close }));
+
+    service.close();
+
+    expect(close).toHaveBeenCalledTimes(1);
+  });
+});
+

src/observability/health-service.ts

@@ -0,0 +1,179 @@
+import { monitorEventLoopDelay } from 'perf_hooks';
+
+import {
+  DependencyChecker,
+  DependencyHealth,
+  HealthReport,
+  ServiceStatus,
+} from './types';
+
+const STATUS_ORDER: Record<ServiceStatus, number> = {
+  up: 0,
+  degraded: 1,
+  down: 2,
+};
+
+export interface RuntimeSignalProviders {
+  now: () => Date;
+  uptimeSeconds: () => number;
+  eventLoopLagMs: () => number;
+  memoryUsage: () => NodeJS.MemoryUsage;
+  close?: () => void;
+}
+
+export interface HealthServiceLike {
+  getReport: () => Promise<HealthReport>;
+  close?: () => void;
+}
+
+export interface HealthThresholds {
+  degradedEventLoopLagMs: number;
+  downEventLoopLagMs: number;
+  degradedHeapUsedRatio: number;
+  downHeapUsedRatio: number;
+}
+
+export const defaultThresholds: HealthThresholds = {
+  degradedEventLoopLagMs: 250,
+  downEventLoopLagMs: 1000,
+  degradedHeapUsedRatio: 0.85,
+  downHeapUsedRatio: 0.95,
+};
+
+/**
+ * Computes service-level health from local runtime signals and dependency checks.
+ */
+export class HealthService implements HealthServiceLike {
+  constructor(
+    private readonly serviceName: string,
+    private readonly dependencyCheckers: DependencyChecker[] = [],
+    private readonly providers: RuntimeSignalProviders = createDefaultProviders(),
+    private readonly thresholds: HealthThresholds = defaultThresholds,
+  ) {}
+
+  async getReport(): Promise<HealthReport> {
+    const now = this.providers.now();
+    const memory = this.providers.memoryUsage();
+    const eventLoopLagMs = this.providers.eventLoopLagMs();
+    const heapUsedRatio = memory.heapTotal > 0 ? memory.heapUsed / memory.heapTotal : 0;
+    const dependencyResults = await Promise.all(
+      this.dependencyCheckers.map((checker) => evaluateDependency(checker, now.toISOString())),
+    );
+
+    const signalStatuses: ServiceStatus[] = [
+      evaluateEventLoopStatus(eventLoopLagMs, this.thresholds),
+      evaluateHeapStatus(heapUsedRatio, this.thresholds),
+    ];
+
+    const status = mergeStatuses(
+      signalStatuses.concat(dependencyResults.map((dependency) => dependency.status)),
+    );
+
+    return {
+      service: this.serviceName,
+      status,
+      timestamp: now.toISOString(),
+      uptimeSeconds: this.providers.uptimeSeconds(),
+      signals: {
+        eventLoopLagMs,
+        heapUsedBytes: memory.heapUsed,
+        heapTotalBytes: memory.heapTotal,
+        heapUsedRatio,
+      },
+      dependencies: dependencyResults,
+    };
+  }
+
+  close(): void {
+    this.providers.close?.();
+  }
+}
+
+function createDefaultProviders(): RuntimeSignalProviders {
+  const loopLagMonitor = monitorEventLoopDelay({ resolution: 20 });
+  loopLagMonitor.enable();
+
+  return {
+    now: () => new Date(),
+    uptimeSeconds: () => process.uptime(),
+    eventLoopLagMs: () => {
+      const lagMs = Number(loopLagMonitor.mean) / 1_000_000;
+      return Number.isFinite(lagMs) ? lagMs : 0;
+    },
+    memoryUsage: () => process.memoryUsage(),
+    close: () => loopLagMonitor.disable(),
+  };
+}
+
+async function evaluateDependency(
+  checker: DependencyChecker,
+  observedAt: string,
+): Promise<DependencyHealth> {
+  try {
+    const result = await checker.check();
+    return {
+      name: checker.name,
+      status: result.status,
+      details: result.details,
+      observedAt,
+    };
+  } catch (error) {
+    return {
+      name: checker.name,
+      status: 'down',
+      details:
+        error instanceof Error
+          ? `Dependency check failed: ${error.message}`
+          : 'Dependency check failed',
+      observedAt,
+    };
+  }
+}
+
+function evaluateEventLoopStatus(
+  eventLoopLagMs: number,
+  thresholds: HealthThresholds,
+): ServiceStatus {
+  if (eventLoopLagMs >= thresholds.downEventLoopLagMs) {
+    return 'down';
+  }
+
+  if (eventLoopLagMs >= thresholds.degradedEventLoopLagMs) {
+    return 'degraded';
+  }
+
+  return 'up';
+}
+
+function evaluateHeapStatus(
+  heapUsedRatio: number,
+  thresholds: HealthThresholds,
+): ServiceStatus {
+  if (heapUsedRatio >= thresholds.downHeapUsedRatio) {
+    return 'down';
+  }
+
+  if (heapUsedRatio >= thresholds.degradedHeapUsedRatio) {
+    return 'degraded';
+  }
+
+  return 'up';
+}
+
+function mergeStatuses(statuses: ServiceStatus[]): ServiceStatus {
+  if (statuses.length === 0) {
+    return 'up';
+  }
+
+  return statuses.reduce((current, next) =>
+    STATUS_ORDER[next] > STATUS_ORDER[current] ? next : current,
+  );
+}
+
+export function healthReportToHttpStatus(status: ServiceStatus): number {
+  return status === 'down' ? 503 : 200;
+}
+
+
+
+