Switch from Poetry to uv; split server/ into api/, common/, conserver/

.dockerignore

@@ -4,6 +4,8 @@
 .docker
 .git
 .gitignore
+.github
+.claude
 .pytest_cache
 .vscode
 __pycache__
@@ -17,8 +19,6 @@ build
 *.log
 *.swp
 *.swo
-*.pyc
-*.pyo
 *.DS_Store
 *.idea
 *.coverage
@@ -27,6 +27,8 @@ docs
 examples
 htmlcov
 test-reports
+tests
+redis_data
 venv
 config.yml
 docker-compose.dev.yml

.github/workflows/docker-publish.yml

@@ -1,24 +1,28 @@
-name: Build and Push Docker image
+name: Build and Push Docker images
 
 # CalVer Release Workflow
 #
 # Automatically creates a CalVer release on every push to main.
 # Version format: YYYY.MM.DD (e.g., 2026.01.16)
 # If multiple releases happen on the same day, adds sequence: YYYY.MM.DD.2, YYYY.MM.DD.3, etc.
 #
+# Two separate images are built and pushed for both linux/amd64 and linux/arm64:
+#   vcon-server-api        — lightweight FastAPI/uvicorn image (main deps only)
+#   vcon-server-conserver  — full processing image (main + links + storage deps)
+#
 # Build strategy: native runners (no QEMU) for maximum speed
 #   amd64 builds on ubuntu-latest
 #   arm64 builds on ubuntu-24.04-arm
 #
-# Docker tags created (linux/amd64 + linux/arm64 multi-arch manifest):
+# Docker tags created for each image:
 #   - CalVer tag (e.g., 2026.01.16)       [main only]
 #   - latest                               [main only]
 #   - Branch name (e.g., main)
 #   - Branch + short sha (e.g., main-a1b2c3d)
 
 on:
   push:
-    branches: [main, release-test]
+    branches: [main, release-test, optimize-docker-images]
     tags: ['v*']
 
 jobs:
@@ -35,7 +39,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Fetch all history for tags
+          fetch-depth: 0
 
       - name: Generate CalVer version
         id: calver
@@ -89,10 +93,28 @@ jobs:
     strategy:
       matrix:
         include:
-          - platform: linux/amd64
+          - service: api
+            dockerfile: ./docker/Dockerfile.api
+            image: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-api
+            platform: linux/amd64
+            platform_tag: amd64
+            runner: ubuntu-latest
+          - service: api
+            dockerfile: ./docker/Dockerfile.api
+            image: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-api
+            platform: linux/arm64
+            platform_tag: arm64
+            runner: ubuntu-24.04-arm
+          - service: conserver
+            dockerfile: ./docker/Dockerfile.conserver
+            image: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-conserver
+            platform: linux/amd64
             platform_tag: amd64
             runner: ubuntu-latest
-          - platform: linux/arm64
+          - service: conserver
+            dockerfile: ./docker/Dockerfile.conserver
+            image: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-conserver
+            platform: linux/arm64
             platform_tag: arm64
             runner: ubuntu-24.04-arm
 
@@ -110,16 +132,16 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY }}
           password: ${{ secrets.AWS_SECRET }}
 
-      - name: Build and push (${{ matrix.platform_tag }})
+      - name: Build and push ${{ matrix.service }} (${{ matrix.platform_tag }})
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/Dockerfile
+          file: ${{ matrix.dockerfile }}
           platforms: ${{ matrix.platform }}
           push: true
-          cache-from: type=gha,scope=vcon-server-${{ matrix.platform_tag }}
-          cache-to: type=gha,mode=max,scope=vcon-server-${{ matrix.platform_tag }}
-          tags: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server:${{ github.ref_name }}-${{ matrix.platform_tag }}
+          cache-from: type=gha,scope=${{ matrix.service }}-${{ matrix.platform_tag }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service }}-${{ matrix.platform_tag }}
+          tags: ${{ matrix.image }}:${{ github.ref_name }}-${{ matrix.platform_tag }}
           build-args: |
             VCON_SERVER_VERSION=${{ needs.prepare.outputs.version }}
             VCON_SERVER_GIT_COMMIT=${{ needs.prepare.outputs.short_sha }}
@@ -131,6 +153,13 @@ jobs:
     permissions:
       contents: read
       packages: write
+    strategy:
+      matrix:
+        include:
+          - service: api
+            image: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-api
+          - service: conserver
+            image: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-conserver
 
     steps:
       - name: Set up Docker Buildx
@@ -143,9 +172,9 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY }}
           password: ${{ secrets.AWS_SECRET }}
 
-      - name: Create multi-arch manifests
+      - name: Create multi-arch manifest for ${{ matrix.service }}
         env:
-          IMAGE: public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server
+          IMAGE: ${{ matrix.image }}
           VERSION: ${{ needs.prepare.outputs.version }}
           SHORT_SHA: ${{ needs.prepare.outputs.short_sha }}
           BRANCH: ${{ github.ref_name }}
@@ -186,21 +215,18 @@ jobs:
 
             ### Docker Images
 
-            Both `linux/amd64` and `linux/arm64` are supported.
-
-            Pull using CalVer:
-            ```bash
-            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server:${{ needs.prepare.outputs.version }}
-            ```
+            Both images support `linux/amd64` and `linux/arm64`.
 
-            Pull using git hash:
+            **API service** (FastAPI/uvicorn, lightweight):
             ```bash
-            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server:main-${{ needs.prepare.outputs.short_sha }}
+            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-api:${{ needs.prepare.outputs.version }}
+            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-api:latest
             ```
 
-            Pull latest:
+            **Conserver service** (processing pipeline, full dependencies):
             ```bash
-            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server:latest
+            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-conserver:${{ needs.prepare.outputs.version }}
+            docker pull public.ecr.aws/r4g1k2s3/vcon-dev/vcon-server-conserver:latest
             ```
           draft: false
           prerelease: false
@@ -218,8 +244,6 @@ jobs:
           echo "| **Build Time** | ${{ needs.prepare.outputs.build_time }} |" >> $GITHUB_STEP_SUMMARY
           echo "| **Branch** | ${{ github.ref_name }} |" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### Docker Tags (linux/amd64 + linux/arm64)" >> $GITHUB_STEP_SUMMARY
-          echo "- \`${{ needs.prepare.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`main\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`main-${{ needs.prepare.outputs.short_sha }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "### Docker Images Built (linux/amd64 + linux/arm64)" >> $GITHUB_STEP_SUMMARY
+          echo "- \`vcon-server-api\` — API service (main deps only)" >> $GITHUB_STEP_SUMMARY
+          echo "- \`vcon-server-conserver\` — Conserver service (main + links + storage)" >> $GITHUB_STEP_SUMMARY

.github/workflows/run-tests.yml

@@ -34,9 +34,6 @@ jobs:
     - name: Run tests inside Docker container
       run: |
         docker-compose run --rm conserver bash -c "
-          # Install tests dependencies
-          poetry install &&
-          # Run the tests
           pytest --maxfail=5 --disable-warnings
         "
 

.gitignore

@@ -21,4 +21,6 @@ tmp
 .qodo
 
 traefik/
-redis_data/
+redis_data/
+litellm_config.yaml
+litellm_config.yml

server/api.py → api/api.py

@@ -21,6 +21,7 @@
 """
 
 import os
+import re
 import traceback
 from typing import Dict, List, Optional
 from uuid import UUID
@@ -38,7 +39,7 @@
     PostgresqlExtDatabase,
     UUIDField,
 )
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, field_validator, model_validator
 from starlette.status import HTTP_403_FORBIDDEN
 
 from config import Configuration
@@ -250,7 +251,7 @@ async def version_endpoint() -> JSONResponse:
 )
 async def health_check() -> JSONResponse:
     """Health check endpoint.
-    
+
     Returns:
         JSONResponse with status and version info
     """
@@ -278,6 +279,79 @@ async def get_queue_depth(
         raise HTTPException(status_code=500, detail="Failed to get queue depth")
 
 
+# --- vCon field validation constants & helpers ---
+
+_VALID_ALG = frozenset({
+    "SHA-256", "SHA-384", "SHA-512",
+    "HS256", "HS384", "HS512",
+    "RS256", "RS384", "RS512",
+    "ES256", "ES384", "ES512",
+    "PS256", "PS384", "PS512",
+})
+_MIME_RE = re.compile(
+    r'^[a-zA-Z0-9][a-zA-Z0-9!\#$&\-^_]*/[a-zA-Z0-9][a-zA-Z0-9!\#$&\-^_.+]*$'
+)
+_URL_RE = re.compile(r'^[a-zA-Z][a-zA-Z0-9+\-.]*://.+')
+_TEL_RE = re.compile(r'^[+\d(][\d\s\-().+xX#*]{4,}$')
+
+
+class DialogEntry(BaseModel):
+    """A single dialog entry within a vCon."""
+    model_config = ConfigDict(extra='allow')
+
+    duration: Optional[float] = None
+    start:    Optional[str]   = None
+    parties:  Optional[List[int]] = None
+    url:      Optional[str]   = None
+    mimetype: Optional[str]   = None
+    alg:      Optional[str]   = None
+
+    @field_validator("duration")
+    @classmethod
+    def duration_non_negative(cls, v):
+        if v is not None and v < 0:
+            raise ValueError("duration must be >= 0")
+        return v
+
+    @field_validator("url")
+    @classmethod
+    def url_valid(cls, v):
+        if v is not None and not _URL_RE.match(v):
+            raise ValueError(f"url does not look like a valid URL: {v!r}")
+        return v
+
+    @field_validator("mimetype")
+    @classmethod
+    def mimetype_valid(cls, v):
+        if v is not None and not _MIME_RE.match(v):
+            raise ValueError(f"mimetype has invalid format: {v!r}")
+        return v
+
+    @field_validator("alg")
+    @classmethod
+    def alg_known(cls, v):
+        if v is not None and v not in _VALID_ALG:
+            raise ValueError(f"alg {v!r} is not a recognised algorithm")
+        return v
+
+
+class PartyEntry(BaseModel):
+    """A single party entry within a vCon."""
+    model_config = ConfigDict(extra='allow')
+
+    tel: Optional[str] = None
+
+    @field_validator("tel")
+    @classmethod
+    def tel_valid(cls, v):
+        if v is not None and not _TEL_RE.match(v):
+            raise ValueError(f"tel has invalid format: {v!r}")
+        return v
+
+
+# --- end vCon field validation ---
+
+
 class Vcon(BaseModel):
     """Pydantic model representing a vCon (Voice Conversation) record.
 
@@ -302,11 +376,25 @@ class Vcon(BaseModel):
     redacted: dict = {}
     appended: Optional[dict] = None
     group: List[Dict] = []
-    parties: List[Dict] = []
-    dialog: List[Dict] = []
+    parties: List[PartyEntry] = []
+    dialog: List[DialogEntry] = []
     analysis: List[Dict] = []
     attachments: List[Dict] = []
 
+    @model_validator(mode='after')
+    def check_party_refs(self) -> 'Vcon':
+        """Ensure every dialog.parties index references an existing party."""
+        n = len(self.parties)
+        for i, d in enumerate(self.dialog):
+            if d.parties:
+                for ref in d.parties:
+                    if ref < 0 or ref >= n:
+                        raise ValueError(
+                            f"dialog[{i}].parties contains index {ref} "
+                            f"which is out of range (parties has {n} entries)"
+                        )
+        return self
+
 
 if VCON_STORAGE:
     class VConPeeWee(Model):

server/config.py → common/config.py

@@ -8,7 +8,7 @@ def get_config() -> dict:
     """This is to keep logic of accessing config in one place"""
     global _config
     with open(settings.CONSERVER_CONFIG_FILE) as file:
-        _config = yaml.safe_load(file)
+        _config = yaml.safe_load(file) or {}
     return _config
 
 

server/storage/dataverse/__init__.py → common/storage/dataverse/__init__.py

@@ -20,7 +20,7 @@
 from datetime import datetime
 
 from lib.logging_utils import init_logger
-from server.lib.vcon_redis import VconRedis
+from lib.vcon_redis import VconRedis
 
 logger = init_logger(__name__)