Finished phase09

.gitignore

@@ -12,8 +12,8 @@ node_modules
 !.yarn/versions
 
 # .claude and CLAUDE.md
-# CLAUDE.md
-#.claude
+CLAUDE.md
+.claude
 
 # testing
 coverage/

apps/backend/package.json

@@ -16,7 +16,10 @@
     "lint": "tsc --noEmit"
   },
   "dependencies": {
+    "@fastify/cors": "^11.2.0",
+    "@fastify/helmet": "^13.0.2",
     "@fastify/multipart": "^8.3.0",
+    "@fastify/rate-limit": "^10.3.0",
     "@langchain/core": "^0.3.0",
     "@prisma/client": "^5.22.0",
     "bullmq": "^5.12.0",
@@ -27,6 +30,7 @@
     "langchain": "^0.3.0",
     "pino": "^9.0.0",
     "pino-pretty": "^11.0.0",
+    "prom-client": "^15.1.3",
     "zod": "^3.23.0"
   },
   "devDependencies": {

apps/backend/src/alerting/webhook.ts

@@ -0,0 +1,59 @@
+import { logger } from '../logging/logger.js';
+
+interface AlertPayload {
+  level: 'error' | 'critical';
+  message: string;
+  context: Record<string, unknown>;
+  timestamp: string;
+}
+
+const ALERT_WEBHOOK_URL = process.env.ALERT_WEBHOOK_URL;
+const ALERT_THRESHOLD = parseInt(process.env.ALERT_THRESHOLD || '5', 10);
+
+// Track error counts for threshold alerting
+const errorCounts = new Map<string, { count: number; lastReset: number }>();
+
+export async function sendAlert(payload: AlertPayload): Promise<void> {
+  if (!ALERT_WEBHOOK_URL) {
+    logger.warn({ payload }, 'Alert webhook not configured');
+    return;
+  }
+
+  try {
+    await fetch(ALERT_WEBHOOK_URL, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        text: `[${payload.level.toUpperCase()}] ${payload.message}`,
+        ...payload,
+      }),
+    });
+  } catch (error) {
+    logger.error({ error, payload }, 'Failed to send alert');
+  }
+}
+
+export function trackError(errorType: string, context: Record<string, unknown>): void {
+  const now = Date.now();
+  const hourAgo = now - 3600000;
+
+  let entry = errorCounts.get(errorType);
+
+  // Reset if older than 1 hour
+  if (!entry || entry.lastReset < hourAgo) {
+    entry = { count: 0, lastReset: now };
+    errorCounts.set(errorType, entry);
+  }
+
+  entry.count++;
+
+  // Alert if threshold exceeded
+  if (entry.count === ALERT_THRESHOLD) {
+    sendAlert({
+      level: 'error',
+      message: `Error threshold exceeded: ${errorType} (${entry.count} in last hour)`,
+      context,
+      timestamp: new Date().toISOString(),
+    });
+  }
+}

apps/backend/src/app.ts

@@ -1,31 +1,51 @@
 import Fastify, { FastifyInstance } from 'fastify';
+import { logger } from './logging/logger.js';
+import { metricsHook, metricsRoute } from './metrics/prometheus.js';
 import { authMiddleware } from './middleware/auth-middleware.js';
+import { configureRateLimit } from './middleware/rate-limit.js';
+import { configureSecurity, securityHooks } from './middleware/security.js';
 import { listRoute } from './routes/documents/list-route.js';
 import { statusRoute } from './routes/documents/status-route.js';
 import { uploadRoute } from './routes/documents/upload-route.js';
+import { healthRoute } from './routes/health-route.js';
 import { callbackRoute } from './routes/internal/callback-route.js';
 import { searchRoute } from './routes/query/search-route.js';
 import { disconnectPrisma } from './services/database.js';
 
+const isProduction = process.env.NODE_ENV === 'production';
+
 export async function createApp(): Promise<FastifyInstance> {
   const app = Fastify({
-    logger: {
+    logger: isProduction ? logger : {
       transport: process.env.NODE_ENV !== 'test' ? {
         target: 'pino-pretty',
       } : undefined,
     },
   });
 
-  // Health check (no auth)
-  app.get('/health', async () => ({ status: 'ok' }));
+  // Security middleware (helmet, CORS)
+  await configureSecurity(app);
+  securityHooks(app);
+
+  // Rate limiting
+  await configureRateLimit(app);
+
+  // Metrics collection
+  metricsHook(app);
+
+  // Metrics endpoint (no auth required)
+  await metricsRoute(app);
+
+  // Health check endpoints (no auth required)
+  await healthRoute(app);
 
   // Internal routes (no auth)
   await callbackRoute(app);
 
-  // Auth middleware
+  // Auth middleware for protected routes
   app.addHook('onRequest', authMiddleware);
 
-  // Register routes
+  // Register protected routes
   await uploadRoute(app);
   await statusRoute(app);
   await listRoute(app);

apps/backend/src/index.ts

@@ -1,28 +1,26 @@
-import Fastify from 'fastify';
 import { config } from 'dotenv';
+import { createApp } from './app.js';
+import { logger } from './logging/logger.js';
+import { configureGracefulShutdown } from './shutdown.js';
 
 config();
 
-const app = Fastify({
-  logger: {
-    transport: {
-      target: 'pino-pretty',
-      options: { colorize: true },
-    },
-  },
-});
-
-app.get('/health', async () => ({ status: 'ok' }));
-
 const start = async () => {
   try {
+    const app = await createApp();
+
+    // Configure graceful shutdown
+    configureGracefulShutdown(app);
+
     const port = parseInt(process.env.PORT || '3000', 10);
     await app.listen({ port, host: '0.0.0.0' });
-    console.log(`Server running on port ${port}`);
+
+    logger.info(`Server running on port ${port}`);
   } catch (err) {
-    app.log.error(err);
+    logger.error({ err }, 'Failed to start server');
     process.exit(1);
   }
 };
 
 start();
+

apps/backend/src/logging/logger.ts

@@ -0,0 +1,49 @@
+import pino from 'pino';
+
+const isProduction = process.env.NODE_ENV === 'production';
+
+export const logger = pino({
+  level: process.env.LOG_LEVEL || 'info',
+  ...(isProduction
+    ? {
+        // Production: JSON format
+        formatters: {
+          level: (label) => ({ level: label }),
+        },
+        timestamp: pino.stdTimeFunctions.isoTime,
+      }
+    : {
+        // Development: pretty print
+        transport: {
+          target: 'pino-pretty',
+          options: {
+            colorize: true,
+          },
+        },
+      }),
+});
+
+// Request logger middleware
+export function createRequestLogger() {
+  return pino({
+    level: process.env.LOG_LEVEL || 'info',
+    serializers: {
+      req: (req) => ({
+        method: req.method,
+        url: req.url,
+        headers: {
+          'user-agent': req.headers['user-agent'],
+          'content-type': req.headers['content-type'],
+        },
+      }),
+      res: (res) => ({
+        statusCode: res.statusCode,
+      }),
+    },
+  });
+}
+
+// Create child logger with context
+export function createContextLogger(context: Record<string, unknown>) {
+  return logger.child(context);
+}

apps/backend/src/metrics/prometheus.ts

@@ -0,0 +1,81 @@
+import { FastifyInstance } from 'fastify';
+import {
+    Counter,
+    Gauge,
+    Histogram,
+    Registry,
+    collectDefaultMetrics,
+} from 'prom-client';
+
+const register = new Registry();
+
+// Collect default Node.js metrics
+collectDefaultMetrics({ register });
+
+// Custom metrics
+export const httpRequestsTotal = new Counter({
+  name: 'http_requests_total',
+  help: 'Total HTTP requests',
+  labelNames: ['method', 'path', 'status'],
+  registers: [register],
+});
+
+export const httpRequestDuration = new Histogram({
+  name: 'http_request_duration_seconds',
+  help: 'HTTP request duration in seconds',
+  labelNames: ['method', 'path', 'status'],
+  buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5],
+  registers: [register],
+});
+
+export const documentsProcessed = new Counter({
+  name: 'documents_processed_total',
+  help: 'Total documents processed',
+  labelNames: ['status', 'format', 'lane'],
+  registers: [register],
+});
+
+export const queueSize = new Gauge({
+  name: 'processing_queue_size',
+  help: 'Current size of processing queue',
+  labelNames: ['status'],
+  registers: [register],
+});
+
+export const embeddingDuration = new Histogram({
+  name: 'embedding_generation_duration_seconds',
+  help: 'Embedding generation duration',
+  buckets: [0.1, 0.5, 1, 2, 5, 10],
+  registers: [register],
+});
+
+// Metrics route
+export async function metricsRoute(fastify: FastifyInstance): Promise<void> {
+  fastify.get('/metrics', async (request, reply) => {
+    reply.header('Content-Type', register.contentType);
+    return register.metrics();
+  });
+}
+
+// Request metrics hook
+export function metricsHook(fastify: FastifyInstance): void {
+  fastify.addHook('onRequest', async (request) => {
+    (request as any).startTime = Date.now();
+  });
+
+  fastify.addHook('onResponse', async (request, reply) => {
+    const duration = (Date.now() - (request as any).startTime) / 1000;
+    const path = request.routeOptions?.url || request.url;
+
+    httpRequestsTotal.inc({
+      method: request.method,
+      path,
+      status: reply.statusCode,
+    });
+
+    httpRequestDuration.observe(
+      { method: request.method, path, status: reply.statusCode },
+      duration
+    );
+  });
+}

apps/backend/src/middleware/rate-limit.ts

@@ -0,0 +1,32 @@
+import rateLimit from '@fastify/rate-limit';
+import { FastifyInstance } from 'fastify';
+
+export async function configureRateLimit(fastify: FastifyInstance): Promise<void> {
+  await fastify.register(rateLimit, {
+    max: parseInt(process.env.RATE_LIMIT_MAX || '100', 10),
+    timeWindow: process.env.RATE_LIMIT_WINDOW || '1 minute',
+    skipOnError: true,
+
+    // Custom key generator (IP-based)
+    keyGenerator: (request) => {
+      return (
+        request.headers['x-forwarded-for']?.toString().split(',')[0] ||
+        request.ip
+      );
+    },
+
+    // Custom error response
+    errorResponseBuilder: (request, context) => ({
+      error: 'RATE_LIMITED',
+      message: 'Too many requests, please try again later',
+      retryAfter: context.after,
+    }),
+
+    // Skip rate limiting for internal routes
+    allowList: (request) => {
+      return request.url.startsWith('/internal/') ||
+             request.url === '/health' ||
+             request.url === '/metrics';
+    },
+  });
+}

apps/backend/src/middleware/security.ts

@@ -0,0 +1,46 @@
+import cors from '@fastify/cors';
+import helmet from '@fastify/helmet';
+import { FastifyInstance } from 'fastify';
+
+export async function configureSecurity(fastify: FastifyInstance): Promise<void> {
+  // Security headers via helmet
+  await fastify.register(helmet, {
+    contentSecurityPolicy: {
+      directives: {
+        defaultSrc: ["'self'"],
+        scriptSrc: ["'self'"],
+        styleSrc: ["'self'", "'unsafe-inline'"],
+        imgSrc: ["'self'", 'data:'],
+      },
+    },
+    crossOriginEmbedderPolicy: false,
+  });
+
+  // CORS configuration
+  await fastify.register(cors, {
+    origin: process.env.CORS_ORIGIN?.split(',') || true,
+    methods: ['GET', 'POST', 'PUT', 'DELETE'],
+    allowedHeaders: ['Content-Type', 'X-API-Key'],
+    credentials: true,
+  });
+}
+
+// Additional security middleware
+export function securityHooks(fastify: FastifyInstance): void {
+  // Remove server header
+  fastify.addHook('onSend', async (request, reply) => {
+    reply.removeHeader('x-powered-by');
+  });
+
+  // Add request ID for tracing
+  fastify.addHook('onRequest', async (request) => {
+    const requestId =
+      request.headers['x-request-id'] ||
+      `req-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
+    (request as any).requestId = requestId;
+  });
+
+  fastify.addHook('onSend', async (request, reply) => {
+    reply.header('x-request-id', (request as any).requestId);
+  });
+}