feat: add local mock storage service for development

* Uses a mounted folder to store uploads in the `.data` directory
* This removes the dependency on S3 locally

Author: aydin-utting

backend/main.py

@@ -9,6 +9,7 @@
 
 from backend.api.routes import router as api_router
 from common.database.postgres_database import init_cleanup_scheduler
+from common.services.storage_services.local import mock_storage_app
 from common.settings import get_settings
 
 settings = get_settings()
@@ -58,5 +59,8 @@ async def lifespan(app_: FastAPI):  # noqa: ARG001
 
 app.include_router(api_router)
 
+if settings.STORAGE_SERVICE_NAME == "local":
+    app.mount("/mock_storage", mock_storage_app)
+
 if __name__ == "__main__":
     uvicorn.run("main:app", host="0.0.0.0", port=8080)  # noqa: S104

common/services/queue_services/sqs.py

@@ -15,6 +15,9 @@ def get_sqs_client():
     if settings.USE_LOCALSTACK and settings.ENVIRONMENT == "local":
         return boto3.client(
             "sqs",
+            aws_access_key_id="YOUR_ACCESS_KEY_ID",
+            aws_secret_access_key="YOUR_SECRET_ACCESS_KEY",  # noqa: S106
+            region_name="eu-west-2",
             endpoint_url=settings.LOCALSTACK_URL,
         )
 

common/services/storage_services/__init__.py

@@ -1,8 +1,13 @@
 from .azure_blob import AzureBlobStorageService
 from .base import StorageService
+from .local import LocalStorageService
 from .s3 import S3StorageService
 
-storage_services = {S3StorageService.name: S3StorageService, AzureBlobStorageService.name: AzureBlobStorageService}
+storage_services = {
+    S3StorageService.name: S3StorageService,
+    AzureBlobStorageService.name: AzureBlobStorageService,
+    LocalStorageService.name: LocalStorageService,
+}
 
 
 def get_storage_service(storage_service_name: str) -> StorageService:

common/services/storage_services/local.py

@@ -0,0 +1,56 @@
+import shutil
+from contextlib import suppress
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.staticfiles import StaticFiles
+
+from common.services.storage_services.base import StorageService
+from common.settings import get_settings
+
+settings = get_settings()
+
+
+class LocalStorageService(StorageService):
+    name = "local"
+
+    @classmethod
+    async def upload(cls, key: str, path: Path) -> None:
+        storage_path = Path(settings.LOCAL_STORAGE_PATH) / key
+        shutil.copy2(path, storage_path)
+
+    @classmethod
+    async def download(cls, key: str, path: Path) -> None:
+        storage_path = Path(settings.LOCAL_STORAGE_PATH) / key
+        shutil.copy2(storage_path, path)
+
+    @classmethod
+    async def generate_presigned_url_put_object(cls, key: str, expiry_seconds: int) -> str:  # noqa: ARG003
+        return f"/api/proxy/mock_storage/uploadfile/{key}"
+
+    @classmethod
+    async def generate_presigned_url_get_object(cls, key: str, filename: str, expiry_seconds: int) -> str:  # noqa: ARG003
+        return f"/api/proxy/mock_storage/static/{key}"
+
+    @classmethod
+    async def check_object_exists(cls, key: str) -> bool:
+        storage_path = Path(settings.LOCAL_STORAGE_PATH) / key
+        return storage_path.exists()
+
+    @classmethod
+    async def delete(cls, key: str) -> None:
+        with suppress(FileNotFoundError):
+            storage_path = Path(settings.LOCAL_STORAGE_PATH) / key
+            storage_path.unlink()
+
+
+mock_storage_app = FastAPI(title="Mock storage service")
+
+mock_storage_app.mount("/static", StaticFiles(directory=settings.LOCAL_STORAGE_PATH), name="static")
+
+
+@mock_storage_app.put("/uploadfile/{file_path:path}")
+async def upload_file_to_mock_storage(file_path: str, request: Request):
+    storage_path = Path(settings.LOCAL_STORAGE_PATH) / file_path
+    storage_path.parent.mkdir(parents=True, exist_ok=True)
+    storage_path.write_bytes(await request.body())

common/settings.py

@@ -147,6 +147,10 @@ class Settings(BaseSettings):
         ),
     )
 
+    LOCAL_STORAGE_PATH: str = Field(
+        default="", description="The folder where the data directory is mounted for the local storage service."
+    )
+
     # use a dotenv file for local development
     if dotenv_detected:
         model_config = SettingsConfigDict(env_file=DOT_ENV_PATH, extra="ignore")

docker-compose.yaml

@@ -8,17 +8,19 @@ services:
     env_file:
       - .env
     environment:
-      - AWS_ACCESS_KEY_ID
-      - AWS_SECRET_ACCESS_KEY
-      - AWS_SESSION_TOKEN
-      - AWS_DEFAULT_REGION=eu-west-2
       - POSTGRES_HOST=db
       - LOCALSTACK_URL=http://localstack:4566
+      - STORAGE_SERVICE_NAME=local
+      - LOCAL_STORAGE_PATH=/static
     depends_on:
       db:
         condition: service_healthy
       localstack:
         condition: service_started
+    volumes:
+      - type: bind
+        source: ./.data
+        target: /static
     develop:
       watch:
         - path: ./backend
@@ -45,14 +47,15 @@ services:
       - "8265:8265"
     volumes:
       - /tmp:/tmp
+      - type: bind
+        source: ./.data
+        target: /static
     environment:
-      - AWS_ACCESS_KEY_ID
-      - AWS_SECRET_ACCESS_KEY
-      - AWS_SESSION_TOKEN
-      - AWS_DEFAULT_REGION=eu-west-2
       - POSTGRES_HOST=db
       - RAY_DASHBOARD_HOST=0.0.0.0
       - LOCALSTACK_URL=http://localstack:4566
+      - STORAGE_SERVICE_NAME=local
+      - LOCAL_STORAGE_PATH=/static
     stop_grace_period: 10s
     depends_on:
       db:

poetry.lock

@@ -54,13 +54,14 @@ mistune = "^3.1.3"
 httpx = "^0.28.1"
 psycopg2-binary = "^2.9.9"
 boto3 = "^1.35.3"
+fastapi = ">=0.112.0"
 sentry-sdk = {extras = ["fastapi"], version = "^2.18.0"}
 sqlalchemy = {extras = ["asyncio"], version = "^2.0.42"}
 asyncpg = "^0.30.0"
 markdownify = "^1.2.0"
+python-multipart = "^0.0.20"
 
 [tool.poetry.group.backend.dependencies]
-fastapi = ">=0.112.0"
 uvicorn = "^0.31.0"
 pyjwt = "^2.9.0"
 cryptography = "^44.0.2"