Services & Components Completed

.github/workflows/build-embedding-image.yml

@@ -0,0 +1,68 @@
+name: Build embedding image
+
+on:
+  push:
+    branches: [ main, Phins-branch ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/lexcam-embedding-service
+      MODEL_NAME: intfloat/multilingual-e5-small
+      MODEL_URL: https://github.com/${{ github.repository }}/releases/download/${{ vars.EMBEDDING_MODEL_RELEASE_TAG }}/model.onnx
+      MODEL_SHA256: ${{ vars.EMBEDDING_MODEL_SHA256 }}
+
+    steps:
+      - name: Debug workflow environment
+        run: |
+          echo "GITHUB_REPOSITORY=$GITHUB_REPOSITORY"
+          echo "GITHUB_REPOSITORY_OWNER=$GITHUB_REPOSITORY_OWNER"
+          echo "IMAGE_NAME=$IMAGE_NAME"
+          echo "MODEL_NAME=$MODEL_NAME"
+          echo "MODEL_URL=$MODEL_URL"
+          echo "MODEL_SHA256=$MODEL_SHA256"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build Docker image
+        run: |
+          docker build \
+            --tag $IMAGE_NAME:${{ github.sha }} \
+            --build-arg MODEL_NAME=$MODEL_NAME \
+            --build-arg MODEL_URL=$MODEL_URL \
+            --build-arg MODEL_SHA256=$MODEL_SHA256 \
+            services/embedding-service
+
+      - name: Smoke test image
+        run: |
+          id=$(docker create --name tmp_test -p 8001:8000 $IMAGE_NAME:${{ github.sha }})
+          docker start $id
+          sleep 5
+          docker ps --filter "name=tmp_test" --format "{{.Names}} {{.Status}}"
+          docker exec tmp_test /bin/sh -c "curl -sS -f http://localhost:8000/api/v1/health || exit 1"
+          docker stop $id
+          docker rm $id
+
+      - name: Publish to GHCR
+        if: github.event_name == 'push'
+        env:
+          CR_PAT: ${{ secrets.GHCR_PAT }}
+        run: |
+          if [ -z "$CR_PAT" ]; then
+            echo "GHCR_PAT not set; skipping publish"
+            exit 0
+          fi
+          echo "$CR_PAT" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+          docker tag $IMAGE_NAME:${{ github.sha }} $IMAGE_NAME:latest
+          docker push $IMAGE_NAME:${{ github.sha }}
+          docker push $IMAGE_NAME:latest

.github/workflows/ci.yml

@@ -0,0 +1,76 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, Phins-branch ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test-and-build:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17-alpine
+        ports: ['5432:5432']
+        env:
+          POSTGRES_USER: lexcam
+          POSTGRES_PASSWORD: lexcam_dev
+          POSTGRES_DB: postgres
+        options: >-
+          --health-cmd="pg_isready -U lexcam" --health-interval=10s --health-timeout=5s --health-retries=5
+      redis:
+        image: redis:7-alpine
+        ports: ['6379:6379']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Create service databases
+        env:
+          PGPASSWORD: lexcam_dev
+        run: |
+          psql -h localhost -U lexcam -d postgres -c "CREATE DATABASE lexcam_feedback;" || true
+          psql -h localhost -U lexcam -d postgres -c "CREATE DATABASE lexcam_notif;" || true
+          psql -h localhost -U lexcam -d postgres -c "CREATE DATABASE lexcam_scraping;" || true
+
+      - name: Install dependencies, run migrations and tests (feedback)
+        working-directory: services/feedback-service
+        env:
+          DATABASE_URL: postgresql://lexcam:lexcam_dev@localhost:5432/lexcam_feedback
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          alembic -c alembic.ini upgrade head
+          pytest -q tests
+
+      - name: Install dependencies, run migrations and tests (notification)
+        working-directory: services/notification-service
+        env:
+          DATABASE_URL: postgresql://lexcam:lexcam_dev@localhost:5432/lexcam_notif
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          alembic -c alembic.ini upgrade head
+          pytest -q tests
+
+      - name: Install dependencies, run migrations and tests (scraper)
+        working-directory: services/scraper-service
+        env:
+          DATABASE_URL: postgresql://lexcam:lexcam_dev@localhost:5432/lexcam_scraping
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          alembic -c alembic.ini upgrade head
+          pytest -q tests
+
+      - name: Build Docker images
+        run: |
+          docker build -t lexcam-feedback-service:ci services/feedback-service
+          docker build -t lexcam-notification-service:ci services/notification-service
+          docker build -t lexcam-scraper-service:ci services/scraper-service

.gitignore

@@ -53,5 +53,17 @@ coverage.xml
 # Docker
 *.tar
 
+# Downloaded or generated embedding artifacts
+services/embedding-service/model/*.onnx
+
 # Helm
 charts/*.tgz
+
+# md files
+build-plan.md
+description.md
+services-desccription.md
+
+# Local safe backups (onnx model artifacts)
+_safe_backups/
+

README.md

@@ -1 +1,15 @@
 # LexCamAI
+
+## Embedding model storage
+
+The embedding service now expects the ONNX model to live outside git, with a
+GitHub Releases asset as the preferred source.
+
+Use a release tag such as `embedding-model-v1`, upload the model as
+`model.onnx`, and set these repository variables for the build workflow:
+
+- `EMBEDDING_MODEL_RELEASE_TAG`
+- `EMBEDDING_MODEL_SHA256`
+
+The image build downloads the asset from:
+`https://github.com/<owner>/<repo>/releases/download/<tag>/model.onnx`

app/__init__.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+# Shim package so pytest can import `app.main` from the embedding service
+# when running from the repository root.
+root = Path(__file__).resolve().parent
+service_app = root.joinpath("..", "services", "embedding-service", "app").resolve()
+if str(service_app) not in __path__:
+    __path__.insert(0, str(service_app))

docker-compose.dev.yml

@@ -59,9 +59,170 @@ services:
     volumes:
       - minio_data:/data
 
+  embedding-service:
+    build:
+      context: ./services/embedding-service
+      args:
+        - MODEL_NAME=intfloat/multilingual-e5-small
+    container_name: lexcam-embedding-service
+    environment:
+      SERVICE_NAME: embedding-service
+      API_PREFIX: /api/v1
+      MODEL_NAME: intfloat/multilingual-e5-small
+      DEVICE: cpu
+      MAX_BATCH_SIZE: 64
+      MAX_SEQ_LENGTH: 512
+      NORMALIZE_EMBEDDINGS: "true"
+      LOG_LEVEL: INFO
+      REQUIRE_API_KEY: "false"
+      RATE_LIMIT_ENABLED: "true"
+      RATE_LIMIT: "120/minute"
+    ports:
+      - "8001:8000"
+    volumes:
+      - embedding_cache:/cache
+
+  knowledge-base-service:
+    build:
+      context: ./services/knowledge-base-service
+    container_name: lexcam-knowledge-base-service
+    environment:
+      SERVICE_NAME: knowledge-base-service
+      API_PREFIX: /api/v1
+      DATABASE_URL: postgresql+psycopg://lexcam:lexcam_dev@postgres:5432/lexcam_knowledge
+      QDRANT_URL: http://qdrant:6333
+      EMBEDDING_SERVICE_URL: http://embedding-service:8000
+      REDIS_URL: redis://redis:6379/0
+      LOG_LEVEL: INFO
+      PLAIN_SUMMARY_CACHE_TTL_SECONDS: 604800
+      MAX_SEARCH_RESULTS: 10
+    ports:
+      - "8003:8000"
+    depends_on:
+      - postgres
+      - qdrant
+      - redis
+      - embedding-service
+
+  rag-service:
+    build:
+      context: ./services/rag-service
+    container_name: lexcam-rag-service
+    environment:
+      SERVICE_NAME: rag-service
+      API_PREFIX: /api/v1
+      KNOWLEDGE_BASE_URL: http://knowledge-base-service:8000
+      EMBEDDING_SERVICE_URL: http://embedding-service:8000
+      HF_API_KEY: ${HF_API_KEY}
+      HF_MODEL: ${HF_MODEL:-google/flan-t5-small}
+      LOG_LEVEL: INFO
+    ports:
+      - "8004:8000"
+    depends_on:
+      - knowledge-base-service
+      - embedding-service
+
+  feedback-service:
+    build:
+      context: ./services/feedback-service
+    image: lexcam-feedback-service
+    container_name: lexcam-feedback-service
+    environment:
+      SERVICE_NAME: feedback-service
+      API_PREFIX: /v1
+      DATABASE_URL: postgresql://lexcam:lexcam_dev@postgres:5432/lexcam_feedback
+      REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: amqp://lexcam:lexcam_dev@rabbitmq:5672/%2F
+      CELERY_RESULT_BACKEND: rpc://
+      LOG_LEVEL: INFO
+    ports:
+      - "8010:8000"
+    depends_on:
+      - postgres
+      - redis
+
+  notification-service:
+    build:
+      context: ./services/notification-service
+    image: lexcam-notification-service
+    container_name: lexcam-notification-service
+    environment:
+      SERVICE_NAME: notification-service
+      API_PREFIX: /v1
+      DATABASE_URL: postgresql://lexcam:lexcam_dev@postgres:5432/lexcam_notif
+      REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: amqp://lexcam:lexcam_dev@rabbitmq:5672/%2F
+      CELERY_RESULT_BACKEND: rpc://
+      SMTP_HOST: smtp.example.com
+      SMTP_PORT: 587
+      SMTP_USER: smtp_user
+      SMTP_PASS: smtp_pass
+      LOG_LEVEL: INFO
+    ports:
+      - "8011:8000"
+    depends_on:
+      - postgres
+      - redis
+
+  scraper-service:
+    build:
+      context: ./services/scraper-service
+    image: lexcam-scraper-service
+    container_name: lexcam-scraper-service
+    environment:
+      SERVICE_NAME: scraper-service
+      API_PREFIX: /v1
+      DATABASE_URL: postgresql://lexcam:lexcam_dev@postgres:5432/lexcam_scraping
+      REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: amqp://lexcam:lexcam_dev@rabbitmq:5672/%2F
+      CELERY_RESULT_BACKEND: rpc://
+      MINIO_URL: http://minio:9000
+      MINIO_ACCESS_KEY: lexcam
+      MINIO_SECRET_KEY: lexcam_dev123
+      LOG_LEVEL: INFO
+    ports:
+      - "8012:8000"
+    depends_on:
+      - postgres
+      - redis
+
+  feedback-worker:
+    image: lexcam-feedback-service
+    container_name: lexcam-feedback-worker
+    depends_on:
+      - feedback-service
+      - redis
+    command: celery -A app.tasks.celery_app worker -l info -Q default
+
+  notification-worker:
+    image: lexcam-notification-service
+    container_name: lexcam-notification-worker
+    depends_on:
+      - notification-service
+      - redis
+    command: celery -A app.tasks.celery_app worker -l info -Q default
+
+  notification-consumer:
+    build:
+      context: ./services/notification-service
+    container_name: lexcam-notification-consumer
+    command: python -u -m app.events_consumer
+    depends_on:
+      - rabbitmq
+      - notification-service
+
+  scraper-worker:
+    image: lexcam-scraper-service
+    container_name: lexcam-scraper-worker
+    depends_on:
+      - scraper-service
+      - redis
+    command: celery -A app.tasks.celery_app worker -l info -Q default
+
 volumes:
   postgres_data:
   qdrant_data:
   redis_data:
   rabbitmq_data:
   minio_data:
+  embedding_cache:

infrastructure/helm/embedding-service/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: embedding-service
+description: LexCam Embedding Service
+type: application
+version: 0.1.0
+appVersion: "0.1.0"