Merge branch 'main' of github.com:KostasCherv/cortex

Author: KostasCherv

.github/workflows/ci.yml

@@ -34,6 +34,16 @@ jobs:
           SUPABASE_SECRET_KEY: dummy-secret-key-for-ci
           SUPABASE_JWKS_URL: https://example.com/auth/v1/.well-known/jwks.json
         run: uv run python -m pytest tests/ -q
+      - name: AI regression gate
+        if: always()
+        run: uv run python -m src.evals.regression_gate --output reports/ai-regression-${{ github.sha }}.json
+      - name: Upload AI regression score
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ai-regression-${{ github.sha }}
+          path: reports/ai-regression-${{ github.sha }}.json
+          if-no-files-found: error
 
   ui:
     runs-on: ubuntu-latest
@@ -56,6 +66,20 @@ jobs:
         run: npx tsc --noEmit -p tsconfig.app.json
       - name: Test
         run: npx vitest run
+      - name: Install Playwright browser
+        run: npx playwright install --with-deps chromium
+      - name: Browser smoke test
+        run: npx playwright test
+      - name: Upload Playwright failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-failure-artifacts
+          path: |
+            ui/playwright-report/
+            ui/test-results/
+          if-no-files-found: ignore
+          retention-days: 14
 
   docker:
     runs-on: ubuntu-latest

.gitignore

@@ -23,6 +23,8 @@ ui/node_modules
 ui/dist
 ui/.vercel
 .DS_Store
+ui/playwright-report/
+ui/test-results/
 supabase/.temp
 ui/.impeccable-live.json
 ui/index.html
@@ -40,4 +42,4 @@ scripts/finetune/router-gguf/
 # dev-session artifacts
 .deepeval/
 .archon/
-test_results_artifact.txt
+test_results_artifact.txt

Dockerfile

@@ -20,6 +20,6 @@ ENV PYTHONUNBUFFERED=1 \
 EXPOSE 8080
 
 HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
-  CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/ready')"
 
 CMD ["uvicorn", "src.api.endpoints:app", "--host", "0.0.0.0", "--port", "8080"]

README.md

@@ -247,6 +247,16 @@ Key production settings in `service.yaml`:
 - `CORS_ORIGINS` must be a JSON array string: `'["https://your-app.vercel.app"]'`
 - `LANGSMITH_TRACING=true` with `LANGSMITH_API_KEY` secret
 
+Production health checks use separate endpoints:
+
+- `GET /health` is a process-only liveness check and never contacts dependencies.
+- `GET /ready` checks LLM configuration plus Supabase, Neo4j, and optional Redis with
+  per-dependency timeouts. It returns `503` when a critical dependency is not ready and `200`
+  with `status: degraded` when only an optional dependency such as Redis is unavailable.
+- Cloud Run startup and readiness probes use `/ready`; its liveness probe uses `/health`.
+- `READINESS_REQUIRE_SUPABASE=true` and `READINESS_REQUIRE_NEO4J=true` make those production
+  dependencies critical. Both default to `false` so local development can run with them disabled.
+
 ### Frontend — Vercel
 
 ```bash
@@ -406,8 +416,20 @@ Used for generation-level observability, user scoring, and evaluation datasets.
 uv run pytest -v
 uv run ruff check src
 uv run mypy src
+
+cd ui
+npm run lint
+npm test
+npx playwright install chromium  # first run only
+npm run test:e2e
 ```
 
+The Playwright smoke journey uses local fixtures—no Google, Supabase, or model
+credentials are required. It restores an authenticated browser session, creates a
+research session, consumes paced SSE progress/report events, and verifies the
+completed report. CI retains the HTML report, trace, screenshot, and video when
+the journey fails.
+
 ## Local benchmarking
 
 Cortex includes a local-first k6 harness in [`load-tests/`](load-tests/README.md) for bottleneck discovery before production-like validation.
@@ -451,6 +473,25 @@ Local benchmark numbers are not production-capacity claims. Use them to find bot
 
 ## Model evaluation
 
+### Fast AI regression gate
+
+Every pull request runs a credential-free AI contract gate over 20 versioned cases in
+`src/evals/ai_regression_set.json`. The cases protect deterministic production boundaries:
+validated chat-router decisions, RAG/tool citation provenance, and progressive finance-tool
+selection and call planning. CI requires 100% overall and in every category, then uploads a
+commit-keyed JSON score artifact for comparison and audit.
+
+Run the same gate locally:
+
+```bash
+uv run python -m src.evals.regression_gate
+```
+
+Add a uniquely named case to the JSON dataset whenever one of these boundaries gains behavior
+or a production failure needs a permanent regression case. Keep PR cases deterministic and free
+of provider credentials. The model-backed DeepEval comparison below remains the appropriate
+manual evaluation for semantic generation quality; it is intentionally not duplicated in CI.
+
 Requires the `evals` optional extra (`uv sync --extra evals`) for `pandas` and `deepeval`.
 
 The repo includes a standalone summarize-only comparison script at `src/evals/model_comparison.py`.

cloudrun/service.yaml

@@ -21,6 +21,25 @@ spec:
             limits:
               memory: 1Gi
               cpu: "1"
+          startupProbe:
+            httpGet:
+              path: /ready
+              port: 8080
+            periodSeconds: 2
+            timeoutSeconds: 5
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            periodSeconds: 30
+            timeoutSeconds: 2
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8080
+            periodSeconds: 10
+            timeoutSeconds: 5
           env:
             # --- LLM ---
             - name: LLM_PROVIDER
@@ -67,6 +86,10 @@ spec:
                   key: latest
             - name: SUPABASE_JWKS_URL
               value: "https://oukmugleriangmcdkocn.supabase.co/auth/v1/.well-known/jwks.json"
+            - name: READINESS_REQUIRE_SUPABASE
+              value: "true"
+            - name: READINESS_REQUIRE_NEO4J
+              value: "true"
             # --- Inngest ---
             - name: INNGEST_EVENT_KEY
               valueFrom:

docs/env-vars-production.md

@@ -28,6 +28,9 @@ All secrets should be stored in **Google Secret Manager** and referenced via `va
 | `SUPABASE_SECRET_KEY` | yes | `sb_secret_...` from Dashboard → Settings → API Keys (`SUPABASE_SERVICE_ROLE_KEY` still accepted) |
 | `SUPABASE_JWKS_URL` | no | `https://<project>.supabase.co/auth/v1/.well-known/jwks.json` |
 | `SUPABASE_JWT_SECRET` | yes | |
+| `READINESS_REQUIRE_SUPABASE` | no | Set to `true` in production; `/ready` returns `503` if Supabase is missing or unavailable |
+| `READINESS_REQUIRE_NEO4J` | no | Set to `true` in production; `/ready` returns `503` if Neo4j is missing or unavailable |
+| `READINESS_TIMEOUT_SECONDS` | no | Per-dependency probe timeout; defaults to `2.0` seconds |
 | `INNGEST_EVENT_KEY` | yes | From Inngest dashboard → Keys |
 | `INNGEST_SIGNING_KEY` | yes | From Inngest dashboard → Keys. **Must not be empty in prod.** |
 | `REDIS_URL` | yes | Upstash `rediss://` URL |

src/api/endpoints.py

@@ -38,6 +38,7 @@
     inngest_client,
 )
 from src.storage import ensure_rag_storage_ready
+from src.readiness import run_readiness_checks
 from src.api.routers.billing import router as billing_router
 from src.api.routers.internal import router as internal_router
 from src.api.routers.memory import router as memory_router
@@ -203,6 +204,17 @@ class HealthResponse(BaseModel):
     version: str
 
 
+class ReadinessCheckResponse(BaseModel):
+    status: str
+    critical: bool
+
+
+class ReadinessResponse(BaseModel):
+    status: str
+    version: str
+    checks: dict[str, ReadinessCheckResponse]
+
+
 # ---------------------------------------------------------------------------
 # Exception handlers
 # ---------------------------------------------------------------------------
@@ -224,3 +236,24 @@ async def health():
     return HealthResponse(status="ok", version="0.1.0")
 
 
+@app.get(
+    "/ready",
+    response_model=ReadinessResponse,
+    responses={503: {"model": ReadinessResponse}},
+    tags=["Meta"],
+)
+async def readiness() -> JSONResponse:
+    """Report configuration and bounded live dependency readiness checks."""
+    checks = await run_readiness_checks()
+    unready = any(check.critical and check.status != "ok" for check in checks.values())
+    degraded = any(check.status == "unavailable" for check in checks.values())
+    status = "unready" if unready else "degraded" if degraded else "ready"
+    payload = ReadinessResponse(
+        status=status,
+        version=app.version,
+        checks={
+            name: ReadinessCheckResponse(status=check.status, critical=check.critical)
+            for name, check in checks.items()
+        },
+    )
+    return JSONResponse(status_code=503 if unready else 200, content=payload.model_dump())

src/config.py

@@ -218,6 +218,19 @@ def parse_cors_origins(cls, v: object) -> object:
     sentry_dsn: str = Field(
         default="", description="Sentry DSN for error tracking; empty disables it"
     )
+    readiness_timeout_seconds: float = Field(
+        default=2.0,
+        gt=0,
+        description="Maximum duration of each live dependency readiness check.",
+    )
+    readiness_require_supabase: bool = Field(
+        default=False,
+        description="Return unready when Supabase is missing or unreachable.",
+    )
+    readiness_require_neo4j: bool = Field(
+        default=False,
+        description="Return unready when Neo4j is missing or unreachable.",
+    )
 
     # Supabase
     supabase_url: str = Field(default="", description="Supabase project URL")

src/evals/ai_regression_set.json

@@ -0,0 +1,27 @@
+{
+  "version": 1,
+  "cases": [
+    {"id":"router-direct","category":"router","response":{"action":"answer_direct","reason":"Stable knowledge is sufficient"},"expected":{"valid":true,"action":"answer_direct","query":"","symbols":[]}},
+    {"id":"router-rag","category":"router","response":{"action":"answer_from_rag","reason":"The attached handbook contains the answer"},"expected":{"valid":true,"action":"answer_from_rag","query":"","symbols":[]}},
+    {"id":"router-web","category":"router","response":{"action":"web_search","reason":"Current information is required","query":"latest Swiss inflation rate"},"expected":{"valid":true,"action":"web_search","query":"latest Swiss inflation rate","symbols":[]}},
+    {"id":"router-price","category":"router","response":{"action":"asset_price","reason":"The user requested live prices","symbols":["BTC","ETH"],"currency":"USD"},"expected":{"valid":true,"action":"asset_price","query":"","symbols":["BTC","ETH"]}},
+    {"id":"router-finance-discovery","category":"router","response":{"action":"search_finance_tools","reason":"A specialist finance operation is needed","query":"company fundamentals lookup"},"expected":{"valid":true,"action":"search_finance_tools","query":"company fundamentals lookup","symbols":[]}},
+    {"id":"router-clarify","category":"router","response":{"action":"ask_clarifying","reason":"The requested company is ambiguous"},"expected":{"valid":true,"action":"ask_clarifying","query":"","symbols":[]}},
+    {"id":"router-web-missing-query","category":"router","response":{"action":"web_search","reason":"Fresh data is needed"},"expected":{"valid":false}},
+    {"id":"router-price-normalizes-symbols","category":"router","response":{"action":"asset_price","reason":"A quote is requested","symbols":" AAPL, MSFT "},"expected":{"valid":true,"action":"asset_price","query":"","symbols":["AAPL","MSFT"]}},
+
+    {"id":"citation-rag","category":"citation","input":{"rag_chunks":[{"source_title":"handbook.pdf","source_url":"","chunk_id":"rag-1","text":"Refunds are available for 30 days."}],"loop_citations":[],"web_used":false,"rag_context_text":"[source:handbook.pdf chunk:rag-1]"},"expected":{"chunk_ids":["rag-1"],"source_types":["rag"]}},
+    {"id":"citation-web-wins","category":"citation","input":{"rag_chunks":[{"source_title":"old.pdf","chunk_id":"rag-old","text":"Old figure"}],"loop_citations":[{"source_title":"Statistics office","source_url":"https://example.com/current","chunk_id":"web-1","text":"Current figure","source_type":"web"}],"web_used":true,"rag_context_text":"old"},"expected":{"chunk_ids":["web-1"],"source_types":["web"]}},
+    {"id":"citation-tool-evidence-wins","category":"citation","input":{"rag_chunks":[{"source_title":"notes.pdf","chunk_id":"rag-2","text":"Notes"}],"loop_citations":[{"source_title":"Wikipedia","source_url":"https://en.wikipedia.org/wiki/AI","chunk_id":"wiki-1","text":"AI","source_type":"wikipedia"}],"web_used":false,"rag_context_text":"notes"},"expected":{"chunk_ids":["wiki-1"],"source_types":["wikipedia"]}},
+    {"id":"citation-nonweb-tool-merges-rag","category":"citation","input":{"rag_chunks":[{"source_title":"brief.pdf","chunk_id":"rag-3","text":"Brief"}],"loop_citations":[{"source_title":"Internal tool","source_url":null,"chunk_id":"tool-1","text":"Result","source_type":"tool"}],"web_used":false,"rag_context_text":"brief"},"expected":{"chunk_ids":["tool-1","rag-3"],"source_types":["tool","rag"]}},
+    {"id":"citation-context-fallback","category":"citation","input":{"rag_chunks":[],"loop_citations":[],"web_used":false,"rag_context_text":"Workspace context without structured chunks"},"expected":{"chunk_ids":["workspace-context-fallback"],"source_types":["rag_fallback"]}},
+    {"id":"citation-no-evidence","category":"citation","input":{"rag_chunks":[],"loop_citations":[],"web_used":false,"rag_context_text":""},"expected":{"chunk_ids":[],"source_types":[]}},
+    {"id":"citation-no-false-rag-fallback-after-web","category":"citation","input":{"rag_chunks":[{"source_title":"local.pdf","chunk_id":"rag-4","text":"Local"}],"loop_citations":[],"web_used":true,"rag_context_text":"Local"},"expected":{"chunk_ids":[],"source_types":[]}},
+
+    {"id":"tool-selection-valid","category":"tool_selection","stage":"selection","response":{"tool_name":"get_company_profile","reason":"The tool returns company fundamentals"},"expected":{"valid":true,"tool_name":"get_company_profile"}},
+    {"id":"tool-selection-blank-name","category":"tool_selection","stage":"selection","response":{"tool_name":"   ","reason":"No usable tool"},"expected":{"valid":false}},
+    {"id":"tool-plan-call","category":"tool_selection","stage":"plan","response":{"should_call":true,"reason":"All required arguments are present","arguments":{"symbol":"NVDA"}},"expected":{"valid":true,"should_call":true,"arguments":{"symbol":"NVDA"}}},
+    {"id":"tool-plan-clarify","category":"tool_selection","stage":"plan","response":{"should_call":false,"reason":"Ticker is missing","clarifying_question":"Which company or ticker do you mean?"},"expected":{"valid":true,"should_call":false,"arguments":{}}},
+    {"id":"tool-plan-invalid-no-question","category":"tool_selection","stage":"plan","response":{"should_call":false,"reason":"Ticker is missing"},"expected":{"valid":false}}
+  ]
+}