Merge pull request #2 from co-cddo/f-fix

F fix

Author: edwardlonsdale

CLAUDE.md

@@ -30,6 +30,9 @@ The daily cron (2 AM London) enqueues one job per service. Jobs are processed se
 - `src/db/migrate.ts` — lightweight migration runner using `schema_migrations` table
 - `src/db/seed.ts` — upserts services from `services.json` into the `services` table
 - `src/db/queries.ts` — all SQL queries (parameterised, no string interpolation)
+- `src/insights/sqlValidator.ts` — AST-based SQL validation (allowlist functions, cap LIMIT, reject non-SELECT)
+- `src/server/validateUrl.ts` — HTTPS + gov.uk domain validation for URL overrides
+- `src/insights/sanitiseHtml.ts` — sanitise-html wrapper for LLM output
 
 ## Running locally
 
@@ -62,6 +65,8 @@ APP_URL=http://localhost:3000
 
 OAuth2 authorization code flow via Internal Access (`sso.service.security.gov.uk`). Sessions stored in Postgres via `connect-pg-simple` (auto-creates `session` table). 2-hour session expiry matching token lifetime. No refresh tokens.
 
+Session is regenerated on login (`session.regenerate()`) to prevent session fixation. The OIDC `sub` claim is stored as the stable user identifier (email may change).
+
 Public routes: `/`, `/health`, `/auth/*`, `/public/*`, `/assets/*`
 Protected routes: `/accessibility`, `/cookies`, `/privacy`, `/insights`, `/services/*`
 
@@ -75,6 +80,18 @@ Protected routes: `/accessibility`, `/cookies`, `/privacy`, `/insights`, `/servi
 - **Migrations** in `src/db/migrations/` — numbered SQL files, applied by `src/db/migrate.ts`
 - **Sessions** — `session` table auto-created by `connect-pg-simple`
 
+## Security
+
+- **CSRF** — synchroniser token pattern via `csrf-sync`. Tokens delivered in a `<meta>` tag and submitted as `_csrf` body field or `x-csrf-token` header. All POST endpoints are protected.
+- **SQL validation** — LLM-generated SQL is parsed with `pgsql-ast-parser` (AST-based). Only single SELECT/WITH statements allowed; function calls checked against an allowlist; LIMIT capped at 100. The read-only pool also enforces `default_transaction_read_only` and `statement_timeout` at DB level.
+- **URL validation** — manual URL overrides restricted to HTTPS + `*.gov.uk` domains (`src/server/validateUrl.ts`).
+- **Prompt injection** — scraper extracts `innerText` (not `innerHTML`) before sending to Bedrock, so hidden elements, scripts, and HTML comments never reach the LLM.
+- **HTML sanitisation** — LLM prose responses are sanitised server-side with `sanitize-html` (strict tag allowlist) before rendering.
+- **TLS** — database connections use `ssl: { rejectUnauthorized: true }` in production.
+- **Headers** — `helmet` adds security headers (CSP, HSTS, X-Frame-Options, etc.).
+- **Rate limiting** — Bedrock-spending endpoints are rate-limited per session.
+- **Session** — `SESSION_SECRET` must be ≥32 chars in production; `httpOnly`, `sameSite: lax`, `secure` (in prod) cookies.
+
 ## Package manager
 
 **pnpm** (v10.33.0) — `.nvmrc` specifies Node 20. CI workflows use pnpm.

README.md

@@ -55,23 +55,13 @@ npm run db:seed
 
 ## Running locally
 
-### UI only (no worker)
-
 ```bash
 npm run dev
-```
-
-Opens at [http://localhost:3000](http://localhost:3000). This starts only the Express server — no scraping will happen.
-
-### Full app (server + worker)
-
-```bash
+npm run dev:watch
 npm run build && npm start
 ```
 
-Starts the Express server and the pg-boss job worker in the same process. The worker runs 3 concurrent scrape loops and a daily cron at 2am London time.
-
-Trigger checks from the Workers page (all services) or from individual service pages (single service).
+Opens at [http://localhost:3000](http://localhost:3000)
 
 ### Docker Compose
 

docker-compose.yml

@@ -40,13 +40,25 @@ services:
           }]
         }
 
+  oidc-mock-ready:
+    image: curlimages/curl:latest
+    depends_on:
+      oidc-mock:
+        condition: service_started
+    entrypoint: ["sleep", "infinity"]
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://oidc-mock:8090/isalive"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+
   compliance-scraper:
     build: .
     depends_on:
       postgres:
         condition: service_healthy
-      oidc-mock:
-        condition: service_started
+      oidc-mock-ready:
+        condition: service_healthy
     env_file:
       - path: .env
         required: false
@@ -93,6 +105,7 @@ services:
       - .:/app
       - e2e-node-modules:/app/node_modules
       - ./playwright-report:/app/playwright-report
+      - ./test-results:/app/test-results
     command: >
       bash -c "npm install -g $$(npm pkg get packageManager | tr -d '\"')
       && pnpm install --frozen-lockfile

package.json

@@ -25,14 +25,19 @@
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.1037.0",
     "connect-pg-simple": "^10.0.0",
+    "csrf-sync": "4.2.1",
     "dotenv": "^17.4.2",
     "express": "^4.22.1",
+    "express-rate-limit": "8.4.1",
     "express-session": "^1.19.0",
     "govuk-frontend": "^5.14.0",
+    "helmet": "8.1.0",
     "nunjucks": "^3.2.4",
     "pg": "^8.20.0",
     "pg-boss": "^10.4.2",
-    "playwright": "^1.59.1"
+    "pgsql-ast-parser": "12.0.2",
+    "playwright": "^1.59.1",
+    "sanitize-html": "2.17.3"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
@@ -43,11 +48,14 @@
     "@types/node": "^20.19.39",
     "@types/nunjucks": "^3.2.6",
     "@types/pg": "^8.20.0",
+    "@types/sanitize-html": "2.16.1",
+    "@types/supertest": "7.2.0",
     "eslint": "^10.2.1",
     "husky": "^9.1.7",
     "lint-staged": "^16.4.0",
     "nodemon": "^3.1.14",
     "prettier": "^3.8.3",
+    "supertest": "7.2.2",
     "ts-node": "^10.9.2",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.59.0"