Merge pull request #91 from GenerateNU/71-storage-research

71 storage research

Author: GordonBie123

.env.example

@@ -34,6 +34,10 @@ DB_PASSWORD=postgres
 SUPABASE_URL=
 SUPABASE_SERVICE_ROLE_KEY=
 
-# ── Webhooks (optional) ─────────────────────
-WEBHOOK_BASE_URL=http://localhost:8000
-WEBHOOK_SECRET=
+ENABLE_BACKEND_ACCESS_CONTROL=false
+
+# Cloudfare
+CLOUDFLARE_R2_ENDPOINT=
+`CLOUDFLARE_R2_ACCESS_KEY_ID=
+CLOUDFLARE_R2_SECRET_KEY=
+CLOUDFLARE_R2_BUCKET_NAME=

backend/app/main.py

@@ -27,7 +27,7 @@
 from app.services.supabase_check import wait_for_supabase  # noqa: E402
 
 from app.api import api_router  # noqa: E402
-from app.cognee_config import setup_cognee
+from app.cognee_config import setup_cognee  # noqa: E402
 
 
 @asynccontextmanager

backend/app/routes/documents.py

@@ -2,12 +2,17 @@
 Document routes for Cognee-powered document upload and search.
 """
 
+import os
 import shutil
 import uuid
 from pathlib import Path
-from typing import Optional
 
-from fastapi import APIRouter, HTTPException, UploadFile, File, Query
+from backend.app.services.ingest import ingest_document, search_knowledge_graph
+from backend.app.services.storage import (
+    download_file_cloudflare,
+    upload_file_cloudflare,
+)
+from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, UploadFile
 from pydantic import BaseModel
 
 #from app.services.ingest import ingest_document, search_knowledge_graph
@@ -83,6 +88,8 @@ async def upload_document(
     suffix = Path(file.filename).suffix if file.filename else ".bin"
     temp_path = UPLOAD_DIR / f"{document_id}{suffix}"
 
+    upload_file_cloudflare(temp_path, bucket=os.getenv("CLOUDFLARE_R2_BUCKET_NAME"), key=f"{dataset_name}/{document_id}{suffix}")
+
     try:
         # Save uploaded file to disk
         with temp_path.open("wb") as f:
@@ -102,7 +109,7 @@ async def upload_document(
         )
 
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Upload failed: {e}")
+        raise HTTPException(status_code=500, detail=f"Upload failed: {e}") from e
 
     finally:
         # Clean up temp file — never leave orphans
@@ -135,4 +142,13 @@ async def search_documents(
         )
 
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Search failed: {e}")
+        raise HTTPException(status_code=500, detail=f"Search failed: {e}") from e
+
+@router.get("/{document_id}", response_model=bytes)
+async def get_document(document_id: str, dataset: str):
+    """
+    Download a document by ID.
+    """
+    key = f"{dataset}/{document_id}"
+    file_bytes = await download_file_cloudflare(bucket=os.getenv("CLOUDFLARE_R2_BUCKET_NAME"), key=key)
+    return file_bytes

backend/app/services/storage.py

@@ -0,0 +1,41 @@
+"""
+Storage service for handling file uploads and downloads.
+"""
+import os
+
+import boto3
+
+s3 = boto3.client(
+    "s3",
+    endpoint_url=os.getenv("CLOUDFLARE_R2_ENDPOINT"),
+    aws_access_key_id=os.getenv("R2_ACCESS_KEY"),
+    aws_secret_access_key=os.getenv("R2_SECRET_KEY"),
+    region_name="auto",
+)
+
+async def upload_file_cloudflare(file_path, bucket, key) -> str:
+    s3.upload_file(file_path, bucket, key)
+    return f"s3://{bucket}/{key}"
+
+async def download_file_cloudflare(bucket, key) -> bytes:
+    response = s3.get_object(Bucket=bucket, Key=key)
+    return response["Body"].read()
+
+from supabase import create_client  # noqa: E402
+
+supabase = create_client(
+    os.getenv("SUPABASE_URL"),
+    os.getenv("SUPABASE_KEY"),
+)
+
+async def upload_file_supabase(file_path, bucket, key) -> str:
+    with open(file_path, "rb") as f:
+        supabase.storage.from_(bucket).upload(
+            path=key,
+            file=f,
+            file_options={"content-type": "application/octet-stream"},
+        )
+    return f"{bucket}/{key}"
+
+async def download_file_supabase(bucket, key) -> bytes:
+    return supabase.storage.from_(bucket).download(key)

backend/tests/conftest.py

@@ -0,0 +1,13 @@
+"""
+Pytest configuration and shared fixtures.
+
+Sets fake env vars at module level so that storage.py's module-level
+boto3.client() and supabase.create_client() don't blow up at import time.
+"""
+import os
+
+os.environ.setdefault("CLOUDFLARE_R2_ENDPOINT", "https://fake.r2.cloudflarestorage.com")
+os.environ.setdefault("R2_ACCESS_KEY", "fake-access-key")
+os.environ.setdefault("R2_SECRET_KEY", "fake-secret-key")
+os.environ.setdefault("SUPABASE_URL", "https://fake.supabase.co")
+os.environ.setdefault("SUPABASE_KEY", "fake-supabase-key")

backend/tests/test_cognee.py

@@ -2,10 +2,10 @@
 
 load_dotenv(override=True)
 
-import asyncio
+import asyncio  # noqa: E402
 
-import cognee
-from cognee.api.v1.search import SearchType
+import cognee  # noqa: E402
+from cognee.api.v1.search import SearchType  # noqa: E402
 
 
 async def setup_cognee():

backend/tests/test_storage.py

@@ -0,0 +1,143 @@
+"""
+Tests for storage service.
+"""
+from unittest.mock import ANY, MagicMock, mock_open, patch
+
+import pytest
+
+from app.services.storage import (
+    download_file_cloudflare,
+    download_file_supabase,
+    upload_file_cloudflare,
+    upload_file_supabase,
+)
+
+# ── Cloudflare R2 Tests ────────────────────────────────────────────────────────
+
+class TestUploadFileCloudflare:
+    @pytest.mark.asyncio
+    @patch("app.services.storage.s3")
+    async def test_upload_returns_s3_uri(self, mock_s3):
+        mock_s3.upload_file.return_value = None
+        result = await upload_file_cloudflare("local/file.txt", "my-bucket", "folder/file.txt")
+
+        assert result == "s3://my-bucket/folder/file.txt"
+
+    @pytest.mark.asyncio
+    @patch("app.services.storage.s3")
+    async def test_upload_calls_s3_with_correct_args(self, mock_s3):
+        mock_s3.upload_file.return_value = None
+
+        await upload_file_cloudflare("local/file.txt", "my-bucket", "folder/file.txt")
+
+        mock_s3.upload_file.assert_called_once_with("local/file.txt", "my-bucket", "folder/file.txt")
+
+    @pytest.mark.asyncio
+    @patch("app.services.storage.s3")
+    async def test_upload_propagates_s3_exception(self, mock_s3):
+        mock_s3.upload_file.side_effect = Exception("S3 upload failed")
+
+        with pytest.raises(Exception, match="S3 upload failed"):
+            await upload_file_cloudflare("local/file.txt", "my-bucket", "folder/file.txt")
+
+
+class TestDownloadFileCloudflare:
+    @pytest.mark.asyncio
+    @patch("app.services.storage.s3")
+    async def test_download_returns_bytes(self, mock_s3):
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"file content"
+        mock_s3.get_object.return_value = {"Body": mock_body}
+
+        result = await download_file_cloudflare("my-bucket", "folder/file.txt")
+
+        assert result == b"file content"
+
+    @pytest.mark.asyncio
+    @patch("app.services.storage.s3")
+    async def test_download_calls_get_object_with_correct_args(self, mock_s3):
+        mock_body = MagicMock()
+        mock_body.read.return_value = b""
+        mock_s3.get_object.return_value = {"Body": mock_body}
+
+        await download_file_cloudflare("my-bucket", "folder/file.txt")
+
+        mock_s3.get_object.assert_called_once_with(Bucket="my-bucket", Key="folder/file.txt")
+
+    @pytest.mark.asyncio
+    @patch("app.services.storage.s3")
+    async def test_download_propagates_s3_exception(self, mock_s3):
+        mock_s3.get_object.side_effect = Exception("Key not found")
+
+        with pytest.raises(Exception, match="Key not found"):
+            await download_file_cloudflare("my-bucket", "folder/file.txt")
+
+
+# ── Supabase Tests ─────────────────────────────────────────────────────────────
+
+class TestUploadFileSupabase:
+    @pytest.mark.asyncio
+    @patch("builtins.open", mock_open(read_data=b"file content"))
+    @patch("app.services.storage.supabase")
+    async def test_upload_returns_bucket_key_path(self, mock_supabase):
+        mock_supabase.storage.from_().upload.return_value = None
+
+        result = await upload_file_supabase("local/file.txt", "my-bucket", "folder/file.txt")
+
+        assert result == "my-bucket/folder/file.txt"
+
+    @pytest.mark.asyncio
+    @patch("builtins.open", mock_open(read_data=b"file content"))
+    @patch("app.services.storage.supabase")
+    async def test_upload_calls_storage_with_correct_args(self, mock_supabase):
+        mock_storage = MagicMock()
+        mock_supabase.storage.from_.return_value = mock_storage
+
+        await upload_file_supabase("local/file.txt", "my-bucket", "folder/file.txt")
+
+        mock_supabase.storage.from_.assert_called_once_with("my-bucket")
+        mock_storage.upload.assert_called_once_with(
+            path="folder/file.txt",
+            file=ANY,
+            file_options={"content-type": "application/octet-stream"},
+        )
+
+    @pytest.mark.asyncio
+    @patch("builtins.open", mock_open(read_data=b"file content"))
+    @patch("app.services.storage.supabase")
+    async def test_upload_propagates_storage_exception(self, mock_supabase):
+        mock_supabase.storage.from_().upload.side_effect = Exception("Upload failed")
+
+        with pytest.raises(Exception, match="Upload failed"):
+            await upload_file_supabase("local/file.txt", "my-bucket", "folder/file.txt")
+
+
+class TestDownloadFileSupabase:
+    @pytest.mark.asyncio
+    @patch("app.services.storage.supabase")
+    async def test_download_returns_bytes(self, mock_supabase):
+        mock_supabase.storage.from_().download.return_value = b"file content"
+
+        result = await download_file_supabase("my-bucket", "folder/file.txt")
+
+        assert result == b"file content"
+
+    @pytest.mark.asyncio
+    @patch("app.services.storage.supabase")
+    async def test_download_calls_storage_with_correct_args(self, mock_supabase):
+        mock_storage = MagicMock()
+        mock_storage.download.return_value = b""
+        mock_supabase.storage.from_.return_value = mock_storage
+
+        await download_file_supabase("my-bucket", "folder/file.txt")
+
+        mock_supabase.storage.from_.assert_called_once_with("my-bucket")
+        mock_storage.download.assert_called_once_with("folder/file.txt")
+
+    @pytest.mark.asyncio
+    @patch("app.services.storage.supabase")
+    async def test_download_propagates_storage_exception(self, mock_supabase):
+        mock_supabase.storage.from_().download.side_effect = Exception("File not found")
+
+        with pytest.raises(Exception, match="File not found"):
+            await download_file_supabase("my-bucket", "folder/file.txt")

docs/object-storage-research.md

@@ -0,0 +1,22 @@
+
+Supabase Storage:
+- Limited free tier 
+- Can work with s3 but requires a translation layer
+- Probably easiest to set up since its aready in the stack
+
+Cloudfare R2:
+- 10 GB free
+- Can use boto3 -> comptable with s3
+- better for lots of downloads + if you want predictable costs if we were to pay for it
+
+Blackblaze B2:
+- 10 GB free
+- Can use boto3 -> comptable with s3
+- Better for cheap large storage with infrequent downloads
+
+MiniIo:
+- Can store as much data as hardware lets
+- Can use boto3 -> comptable with s3
+- Probably the most difficult to set up
+
+Supabase Storage is the best choice for simplest setup but has limited storage in the free tier. Cloudflare R2 is the better option for more free storage and easier S3 compatibility. 

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.pytest.ini_options]
+pythonpath = ["backend"]
+testpaths = ["backend/tests"]