Preserve embedding-model release-asset wiring (no history rewrite)

.github/workflows/build-embedding-image.yml

@@ -0,0 +1,54 @@
+name: Build embedding image
+
+on:
+  push:
+    branches: [ main, Phins-branch ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/lexcam-embedding-service
+      MODEL_NAME: intfloat/multilingual-e5-small
+      MODEL_URL: https://github.com/${{ github.repository }}/releases/download/${{ vars.EMBEDDING_MODEL_RELEASE_TAG }}/model.onnx
+      MODEL_SHA256: ${{ vars.EMBEDDING_MODEL_SHA256 }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build Docker image
+        run: |
+          docker build \
+            --tag $IMAGE_NAME:${{ github.sha }} \
+            --build-arg MODEL_NAME=$MODEL_NAME \
+            --build-arg MODEL_URL=$MODEL_URL \
+            --build-arg MODEL_SHA256=$MODEL_SHA256 \
+            services/embedding-service
+
+      - name: Smoke test image
+        run: |
+          id=$(docker create --name tmp_test -p 8001:8000 $IMAGE_NAME:${{ github.sha }})
+          docker start $id
+          sleep 5
+          docker ps --filter "name=tmp_test" --format "{{.Names}} {{.Status}}"
+          docker exec tmp_test /bin/sh -c "curl -sS -f http://localhost:8000/api/v1/health || exit 1"
+          docker stop $id
+          docker rm $id
+
+      - name: Publish to GHCR
+        if: github.event_name == 'push' && secrets.GHCR_PAT
+        env:
+          CR_PAT: ${{ secrets.GHCR_PAT }}
+        run: |
+          echo $CR_PAT | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+          docker tag $IMAGE_NAME:${{ github.sha }} $IMAGE_NAME:latest
+          docker push $IMAGE_NAME:${{ github.sha }}
+          docker push $IMAGE_NAME:latest

.gitignore

@@ -53,6 +53,9 @@ coverage.xml
 # Docker
 *.tar
 
+# Downloaded or generated embedding artifacts
+services/embedding-service/model/*.onnx
+
 # Helm
 charts/*.tgz
 

README.md

@@ -1 +1,15 @@
 # LexCamAI
+
+## Embedding model storage
+
+The embedding service now expects the ONNX model to live outside git, with a
+GitHub Releases asset as the preferred source.
+
+Use a release tag such as `embedding-model-v1`, upload the model as
+`model.onnx`, and set these repository variables for the build workflow:
+
+- `EMBEDDING_MODEL_RELEASE_TAG`
+- `EMBEDDING_MODEL_SHA256`
+
+The image build downloads the asset from:
+`https://github.com/<owner>/<repo>/releases/download/<tag>/model.onnx`

docker-compose.dev.yml

@@ -62,6 +62,8 @@ services:
   embedding-service:
     build:
       context: ./services/embedding-service
+      args:
+        - MODEL_NAME=intfloat/multilingual-e5-small
     container_name: lexcam-embedding-service
     environment:
       SERVICE_NAME: embedding-service
@@ -80,6 +82,28 @@ services:
     volumes:
       - embedding_cache:/cache
 
+  knowledge-base-service:
+    build:
+      context: ./services/knowledge-base-service
+    container_name: lexcam-knowledge-base-service
+    environment:
+      SERVICE_NAME: knowledge-base-service
+      API_PREFIX: /api/v1
+      DATABASE_URL: postgresql+psycopg://lexcam:lexcam_dev@postgres:5432/lexcam_knowledge
+      QDRANT_URL: http://qdrant:6333
+      EMBEDDING_SERVICE_URL: http://embedding-service:8000
+      REDIS_URL: redis://redis:6379/0
+      LOG_LEVEL: INFO
+      PLAIN_SUMMARY_CACHE_TTL_SECONDS: 604800
+      MAX_SEARCH_RESULTS: 10
+    ports:
+      - "8003:8000"
+    depends_on:
+      - postgres
+      - qdrant
+      - redis
+      - embedding-service
+
 volumes:
   postgres_data:
   qdrant_data:

scripts/init-databases.sql

@@ -1,7 +1,7 @@
 -- LexCam PostgreSQL initialization
 -- Runs automatically on first container start.
 -- Design choice: database-per-service on a single Postgres instance
--- (separate logical databases like lexcam_users, lexcam_rag_sessions, etc.).
+-- (separate logical databases like lexcam_users, lexcam_rag_sessions).
 
 CREATE DATABASE lexcam_users;
 CREATE DATABASE lexcam_lawyers;

scripts/populate_embedding_cache.ps1

@@ -0,0 +1,23 @@
+Param(
+    [string]$Image = 'lexcam/embedding-service:from-test'
+)
+
+Write-Output "Using image: $Image"
+
+$tmp = New-TemporaryFile
+$tmpDir = Split-Path $tmp -Parent
+
+try {
+    $cid = (docker create $Image).Trim()
+    Write-Output "Created container $cid"
+    docker cp "$cid`:/cache/model.onnx" "$tmpDir\model.onnx"
+    docker rm $cid | Out-Null
+
+    # create temporary container with volume mounted
+    $vcontainer = (docker create --name tmp_embedding_volume -v embedding_cache:/cache busybox).Trim()
+    docker cp "$tmpDir\model.onnx" "$vcontainer`:/cache/model.onnx"
+    docker rm $vcontainer | Out-Null
+    Write-Output "Model copied into volume 'embedding_cache'"
+} finally {
+    Remove-Item "$tmpDir\model.onnx" -ErrorAction SilentlyContinue
+}

scripts/populate_embedding_cache.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Populate the named Docker volume `embedding_cache` with model files
+# Usage: ./scripts/populate_embedding_cache.sh [image]
+# Default image: lexcam/embedding-service:from-test
+
+IMAGE=${1:-lexcam/embedding-service:from-test}
+
+echo "Using image: $IMAGE"
+
+TMPDIR=$(mktemp -d)
+cleanup() {
+  rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+echo "Creating temporary container from image to extract model..."
+CID=$(docker create "$IMAGE")
+echo "Copying /cache/model.onnx from container $CID to host temp"
+docker cp "$CID":/cache/model.onnx "$TMPDIR"/model.onnx
+docker rm "$CID" >/dev/null
+
+echo "Creating temporary container with embedding_cache volume mounted..."
+VCONTAINER=$(docker create --name tmp_embedding_volume -v embedding_cache:/cache busybox)
+echo "Copying model into volume"
+docker cp "$TMPDIR"/model.onnx tmp_embedding_volume:/cache/model.onnx
+docker rm tmp_embedding_volume >/dev/null
+
+echo "Model copied into volume 'embedding_cache'"
+
+echo "Done"

services/embedding-service/Dockerfile

@@ -1,10 +1,15 @@
-FROM python:3.11-slim
+FROM mcr.microsoft.com/devcontainers/python:1-3.11-bookworm
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     TRANSFORMERS_CACHE=/cache \
     HF_HOME=/cache
 
+ARG MODEL_NAME=intfloat/multilingual-e5-small
+ENV MODEL_NAME=${MODEL_NAME}
+ARG MODEL_URL
+ARG MODEL_SHA256=
+
 WORKDIR /app
 
 RUN useradd --create-home appuser \
@@ -14,6 +19,41 @@ RUN useradd --create-home appuser \
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
+RUN python3 - <<'PY'
+import hashlib
+import os
+import pathlib
+import urllib.request
+
+model_url = os.environ.get("MODEL_URL", "").strip()
+if not model_url:
+    raise SystemExit(
+        "MODEL_URL build arg is required. Point it at the externally stored model artifact."
+    )
+
+expected_sha256 = os.environ.get("MODEL_SHA256", "").strip().lower()
+target = pathlib.Path("/cache/model.onnx")
+target.parent.mkdir(parents=True, exist_ok=True)
+
+hasher = hashlib.sha256() if expected_sha256 else None
+with urllib.request.urlopen(model_url) as response, target.open("wb") as output_file:
+    while True:
+        chunk = response.read(1024 * 1024)
+        if not chunk:
+            break
+        output_file.write(chunk)
+        if hasher:
+            hasher.update(chunk)
+
+if hasher and hasher.hexdigest().lower() != expected_sha256:
+    target.unlink(missing_ok=True)
+    raise SystemExit(
+        f"MODEL_SHA256 mismatch for {target}: expected {expected_sha256}, got {hasher.hexdigest().lower()}"
+    )
+
+print(f"Downloaded external model artifact to {target}")
+PY
+
 COPY app ./app
 
 EXPOSE 8000

services/embedding-service/app/api/v1/routes.py

@@ -1,7 +1,5 @@
-from __future__ import annotations
-
 import time
-from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi import APIRouter, Body, Depends, HTTPException, Request
 
 from app.config import settings
 from app.limiting import rate_limit
@@ -29,7 +27,7 @@ async def health(request: Request) -> HealthResponse:
     dependencies=[Depends(verify_api_key)],
 )
 @rate_limit()
-async def embed(request: Request, payload: EmbeddingRequest) -> EmbeddingResponse:
+async def embed(request: Request, payload: EmbeddingRequest = Body(...)) -> EmbeddingResponse:
     if len(payload.texts) > settings.max_batch_size:
         raise HTTPException(
             status_code=413,

services/embedding-service/app/schemas.py

@@ -2,11 +2,11 @@
 
 from typing import Literal
 
-from pydantic import BaseModel, Field, conlist
+from pydantic import BaseModel, Field
 
 
 class EmbeddingRequest(BaseModel):
-    texts: conlist(str, min_items=1) = Field(..., description="Texts to embed")
+    texts: list[str] = Field(..., min_items=1, description="Texts to embed")
     input_type: Literal["query", "passage"] | None = Field(
         default=None,
         description="Optional E5 prefix to apply",