feat(db): add read-replica routing with lag monitoring and Terraform (#644)

* feat(db): add read-replica routing with lag monitoring and Terraform

Route SELECT/WITH queries to read replicas via ReadWritePool, with automatic
failback to primary on high lag or replica failure, PgBouncer-backed local dev,
Prometheus replication metrics, and RDS read replica Terraform provisioning.

* chore: automated code formatting fixes via CI pipeline

* chore: automated code formatting fixes via CI pipeline

Author: od-hunter

backend/config/__tests__/database.test.ts

@@ -0,0 +1,74 @@
+import {
+  DEFAULT_DATABASE_CONFIG,
+  loadDatabaseConfig,
+  replicaPoolConfig,
+} from '../database';
+
+describe('database config', () => {
+  it('loads primary defaults when env vars are unset', () => {
+    const config = loadDatabaseConfig({});
+    expect(config.primary.host).toBe('localhost');
+    expect(config.primary.port).toBe(5432);
+    expect(config.primary.database).toBe('subtrackr');
+    expect(config.primary.user).toBe('postgres');
+    expect(config.primary.max).toBe(20);
+    expect(config.replicas).toEqual([]);
+    expect(config.replicaPoolSize).toBe(DEFAULT_DATABASE_CONFIG.replicaPoolSize);
+    expect(config.replicationLagP99AlarmMs).toBe(1_000);
+    expect(config.replicationLagFailoverMs).toBe(5_000);
+    expect(config.staleReadDefaultSeconds).toBe(30);
+  });
+
+  it('parses comma-separated read replica endpoints', () => {
+    const config = loadDatabaseConfig({
+      DB_READ_REPLICAS: 'replica-a.internal:6432,replica-b.internal:6433',
+    });
+    expect(config.replicas).toEqual([
+      { name: 'replica-1', host: 'replica-a.internal', port: 6432 },
+      { name: 'replica-2', host: 'replica-b.internal', port: 6433 },
+    ]);
+  });
+
+  it('parses replica host without explicit port', () => {
+    const config = loadDatabaseConfig({
+      DB_READ_REPLICAS: 'replica-only.internal',
+    });
+    expect(config.replicas).toEqual([
+      { name: 'replica-1', host: 'replica-only.internal', port: 5432 },
+    ]);
+  });
+
+  it('reads custom lag and pool thresholds', () => {
+    const config = loadDatabaseConfig({
+      DB_REPLICA_POOL_SIZE: '50',
+      DB_REPLICATION_LAG_P99_ALARM_MS: '800',
+      DB_REPLICATION_LAG_FAILOVER_MS: '4000',
+      DB_STALE_READ_DEFAULT_SECONDS: '60',
+      DB_LAG_POLL_INTERVAL_MS: '10000',
+    });
+    expect(config.replicaPoolSize).toBe(50);
+    expect(config.replicationLagP99AlarmMs).toBe(800);
+    expect(config.replicationLagFailoverMs).toBe(4_000);
+    expect(config.staleReadDefaultSeconds).toBe(60);
+    expect(config.lagPollIntervalMs).toBe(10_000);
+  });
+
+  it('falls back for invalid numeric env values', () => {
+    const config = loadDatabaseConfig({
+      DB_PORT: 'not-a-number',
+      DB_REPLICA_POOL_SIZE: '-1',
+    });
+    expect(config.primary.port).toBe(5432);
+    expect(config.replicaPoolSize).toBe(DEFAULT_DATABASE_CONFIG.replicaPoolSize);
+  });
+
+  it('builds replica pool config with PgBouncer pool size', () => {
+    const base = loadDatabaseConfig({}).primary;
+    const replica = { name: 'replica-1', host: 'pgbouncer-1', port: 6433 };
+    const poolConfig = replicaPoolConfig(replica, base, 25);
+    expect(poolConfig.host).toBe('pgbouncer-1');
+    expect(poolConfig.port).toBe(6433);
+    expect(poolConfig.max).toBe(25);
+    expect(poolConfig.database).toBe(base.database);
+  });
+});

backend/config/database.ts

@@ -0,0 +1,135 @@
+/**
+ * PostgreSQL connection configuration with read-replica endpoints.
+ *
+ * Environment variables (primary):
+ *   DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD, DB_SSL
+ *
+ * Read replicas (optional — comma-separated host:port pairs):
+ *   DB_READ_REPLICAS – e.g. "replica-1.internal:6432,replica-2.internal:6433"
+ *   DB_REPLICA_POOL_SIZE – PgBouncer pool size per replica (default: 25)
+ *
+ * Replication lag thresholds (milliseconds):
+ *   DB_REPLICATION_LAG_P99_ALARM_MS – P99 alarm threshold (default: 1000)
+ *   DB_REPLICATION_LAG_FAILOVER_MS – route reads to primary above this (default: 5000)
+ *
+ * Stale reads:
+ *   DB_STALE_READ_DEFAULT_SECONDS – default X-Stale-Accept for analytics (default: 30)
+ */
+
+import type { PoolConfig } from '../shared/db/connectionPool';
+
+export interface ReplicaEndpoint {
+  /** Logical name used in metrics labels (replica-1, replica-2, …). */
+  name: string;
+  host: string;
+  port: number;
+}
+
+export interface DatabaseConfig {
+  primary: Required<PoolConfig>;
+  replicas: ReplicaEndpoint[];
+  /** PgBouncer pool size per replica. Default: 25 */
+  replicaPoolSize: number;
+  /** P99 replication lag alarm threshold in ms. Default: 1000 */
+  replicationLagP99AlarmMs: number;
+  /** Lag above which reads fail back to primary. Default: 5000 */
+  replicationLagFailoverMs: number;
+  /** Default stale-read tolerance for analytics endpoints (seconds). Default: 30 */
+  staleReadDefaultSeconds: number;
+  /** How often to poll replication lag (ms). Default: 5000 */
+  lagPollIntervalMs: number;
+}
+
+export const DEFAULT_DATABASE_CONFIG: Readonly<{
+  replicaPoolSize: number;
+  replicationLagP99AlarmMs: number;
+  replicationLagFailoverMs: number;
+  staleReadDefaultSeconds: number;
+  lagPollIntervalMs: number;
+}> = {
+  replicaPoolSize: 25,
+  replicationLagP99AlarmMs: 1_000,
+  replicationLagFailoverMs: 5_000,
+  staleReadDefaultSeconds: 30,
+  lagPollIntervalMs: 5_000,
+};
+
+function parsePositiveInt(value: string | undefined, fallback: number): number {
+  if (value === undefined || value === '') return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+}
+
+function parseReplicaEndpoints(raw: string | undefined): ReplicaEndpoint[] {
+  if (!raw?.trim()) return [];
+
+  return raw
+    .split(',')
+    .map((entry) => entry.trim())
+    .filter(Boolean)
+    .map((entry, index) => {
+      const [host, portStr] = entry.includes(':') ? entry.split(':') : [entry, undefined];
+      return {
+        name: `replica-${index + 1}`,
+        host: host.trim(),
+        port: parsePositiveInt(portStr, 5432),
+      };
+    });
+}
+
+function buildPrimaryConfig(env: NodeJS.ProcessEnv): Required<PoolConfig> {
+  return {
+    host: env.DB_HOST?.trim() || 'localhost',
+    port: parsePositiveInt(env.DB_PORT, 5432),
+    database: env.DB_NAME?.trim() || 'subtrackr',
+    user: env.DB_USER?.trim() || 'postgres',
+    password: env.DB_PASSWORD ?? '',
+    max: parsePositiveInt(env.DB_POOL_MAX, 20),
+    idleTimeoutMillis: parsePositiveInt(env.DB_IDLE_TIMEOUT_MS, 10_000),
+    connectionTimeoutMillis: parsePositiveInt(env.DB_CONNECTION_TIMEOUT_MS, 30_000),
+    statementTimeout: parsePositiveInt(env.DB_STATEMENT_TIMEOUT_MS, 30_000),
+    ssl: env.DB_SSL === 'true' ? { rejectUnauthorized: true } : false,
+  };
+}
+
+/** Load database configuration from environment variables. */
+export function loadDatabaseConfig(env: NodeJS.ProcessEnv = process.env): DatabaseConfig {
+  return {
+    primary: buildPrimaryConfig(env),
+    replicas: parseReplicaEndpoints(env.DB_READ_REPLICAS),
+    replicaPoolSize: parsePositiveInt(
+      env.DB_REPLICA_POOL_SIZE,
+      DEFAULT_DATABASE_CONFIG.replicaPoolSize,
+    ),
+    replicationLagP99AlarmMs: parsePositiveInt(
+      env.DB_REPLICATION_LAG_P99_ALARM_MS,
+      DEFAULT_DATABASE_CONFIG.replicationLagP99AlarmMs,
+    ),
+    replicationLagFailoverMs: parsePositiveInt(
+      env.DB_REPLICATION_LAG_FAILOVER_MS,
+      DEFAULT_DATABASE_CONFIG.replicationLagFailoverMs,
+    ),
+    staleReadDefaultSeconds: parsePositiveInt(
+      env.DB_STALE_READ_DEFAULT_SECONDS,
+      DEFAULT_DATABASE_CONFIG.staleReadDefaultSeconds,
+    ),
+    lagPollIntervalMs: parsePositiveInt(
+      env.DB_LAG_POLL_INTERVAL_MS,
+      DEFAULT_DATABASE_CONFIG.lagPollIntervalMs,
+    ),
+  };
+}
+
+/** Build a pg PoolConfig for a read replica (via PgBouncer). */
+export function replicaPoolConfig(
+  replica: ReplicaEndpoint,
+  base: Required<PoolConfig>,
+  poolSize: number,
+): Required<PoolConfig> {
+  return {
+    ...base,
+    host: replica.host,
+    port: replica.port,
+    max: poolSize,
+  };
+}

backend/monitoring/__tests__/replicationLagExporter.test.ts

@@ -0,0 +1,92 @@
+import type { DatabaseConfig } from '../../config/database';
+import type { Pool } from '../shared/db/connectionPool';
+import type { ReplicaLagState, ReplicaQueryStats } from '../shared/db/readWriteRouter';
+import { formatReplicationPrometheus } from '../replicationLagExporter';
+
+function makePoolStats(total: number, idle: number, waiting: number): Pool {
+  return {
+    query: jest.fn(),
+    connect: jest.fn(),
+    end: jest.fn(),
+    on: jest.fn(),
+    totalCount: total,
+    idleCount: idle,
+    waitingCount: waiting,
+  } as unknown as Pool;
+}
+
+describe('replicationLagExporter', () => {
+  it('formats lag, pool, and query latency metrics', () => {
+    const config: DatabaseConfig = {
+      primary: {
+        host: 'primary',
+        port: 5432,
+        database: 'subtrackr',
+        user: 'postgres',
+        password: '',
+        max: 20,
+        idleTimeoutMillis: 10_000,
+        connectionTimeoutMillis: 30_000,
+        statementTimeout: 30_000,
+        ssl: false,
+      },
+      replicas: [{ name: 'replica-1', host: 'r1', port: 6433 }],
+      replicaPoolSize: 25,
+      replicationLagP99AlarmMs: 1_000,
+      replicationLagFailoverMs: 5_000,
+      staleReadDefaultSeconds: 30,
+      lagPollIntervalMs: 5_000,
+    };
+
+    const lagStates: ReplicaLagState[] = [
+      { name: 'replica-1', lagMs: 250, lagP99Ms: 400, available: true, lastCheckedAt: Date.now() },
+    ];
+    const queryStats: ReplicaQueryStats[] = [
+      {
+        name: 'replica-1',
+        queryCount: 42,
+        totalLatencyMs: 840,
+        lastLatencyMs: 20,
+        errors: 1,
+      },
+    ];
+
+    const replicaPools = new Map([['replica-1', makePoolStats(25, 10, 2)]]);
+
+    const mockPool = {
+      getReplicaPools: () => replicaPools,
+    };
+
+    const output = formatReplicationPrometheus(
+      { lagStates, queryStats, config },
+      mockPool as never,
+    );
+
+    expect(output).toContain('subtrackr_replication_lag_ms{replica="replica-1"} 250');
+    expect(output).toContain('subtrackr_replication_lag_p99_ms{replica="replica-1"} 400');
+    expect(output).toContain('subtrackr_replication_lag_failover_ms 5000');
+    expect(output).toContain('subtrackr_replica_available{replica="replica-1"} 1');
+    expect(output).toContain('subtrackr_replica_pool_idle{replica="replica-1"} 10');
+    expect(output).toContain('subtrackr_replica_query_latency_ms{replica="replica-1"} 20');
+    expect(output).toContain('subtrackr_replica_query_total{replica="replica-1"} 42');
+    expect(output).toContain('subtrackr_replica_query_errors_total{replica="replica-1"} 1');
+  });
+
+  it('handles unavailable replica with -1 lag', () => {
+    const config = {
+      replicationLagP99AlarmMs: 1_000,
+      replicationLagFailoverMs: 5_000,
+    } as DatabaseConfig;
+
+    const lagStates: ReplicaLagState[] = [
+      { name: 'replica-2', lagMs: Infinity, lagP99Ms: 0, available: false, lastCheckedAt: 0 },
+    ];
+
+    const output = formatReplicationPrometheus(
+      { lagStates, queryStats: [], config },
+      { getReplicaPools: () => new Map() } as never,
+    );
+
+    expect(output).toContain('subtrackr_replica_available{replica="replica-2"} 0');
+  });
+});