Merge pull request #22 from ClipABit/env-vars-and-api-url-refactor

refactor: env vars and api url logic

Author: adithayyil

backend/README.md

@@ -11,10 +11,23 @@ uv sync
 # 2. Authenticate with Modal (first time only - opens browser)
 uv run modal token new
 
-# 3. Get Environment variables and populate .env (check docs or reach out to TPMs)
-
-# 4. Start dev server (hot-reloads on file changes)
-uv run modal serve main.py
+# 3. Configure Modal secrets (dev and prod)
+modal secret create dev \
+  ENVIRONMENT=dev \
+  PINECONE_API_KEY=your_pinecone_api_key \
+  R2_ACCOUNT_ID=your_r2_account_id \
+  R2_ACCESS_KEY_ID=your_r2_access_key_id \
+  R2_SECRET_ACCESS_KEY=your_r2_secret_access_key
+
+modal secret create prod \
+  ENVIRONMENT=prod \
+  PINECONE_API_KEY=your_pinecone_api_key \
+  R2_ACCOUNT_ID=your_r2_account_id \
+  R2_ACCESS_KEY_ID=your_r2_access_key_id \
+  R2_SECRET_ACCESS_KEY=your_r2_secret_access_key
+
+# 4. Start dev server (hot-reloads on file changes, uses "dev" secret)
+uv run dev
 ```
 
 Note: `uv run` automatically uses the virtual environment - no need to activate it manually.
@@ -23,8 +36,9 @@ Note: `uv run` automatically uses the virtual environment - no need to activate
 
 - `main.py` defines a Modal App with a `Server` class
 - `/upload` endpoint accepts video files and spawns background processing jobs
-
-[...]
+- Environment variables stored in Modal secrets (no .env files needed)
+- `uv run dev` automatically uses "dev" secret for development
+- Production deployment handled via CI/CD or direct Modal CLI
 
 ## Managing Dependencies
 
@@ -34,3 +48,11 @@ uv add --dev package-name        # Add dev dependency
 uv remove package-name           # Remove dependency
 uv sync --upgrade                # Update all packages
 ```
+
+## Running Tests
+
+```bash
+uv run pytest                    # Run all tests
+uv run pytest -v                 # Verbose output
+uv run pytest --cov              # With coverage
+```

backend/cli.py

@@ -1,10 +1,4 @@
-import os
 import subprocess
 
 def serve():
     subprocess.run(["modal", "serve", "main.py"])
-
-def deploy():
-    env = os.environ.copy()
-    env["ENV"] = "prod"
-    subprocess.run(["modal", "deploy", "main.py"], env=env)

backend/database/r2_connector.py

@@ -21,16 +21,16 @@ def __init__(
         account_id: str,
         access_key_id: str,
         secret_access_key: str,
-        environment: str = "dev"  # dev, test, or prod
+        environment: str = "dev"  # dev or prod
     ):
         """
         Initialize R2 connector with bucket credentials.
-        
+
         Args:
             account_id: Cloudflare account ID
             access_key_id: R2 access key ID
             secret_access_key: R2 secret access key
-            environment: Environment name (dev/test/prod) which maps directly to bucket name
+            environment: Environment name (dev/prod) which maps directly to bucket name
         """
         self.bucket_name = environment
         self.endpoint_url = f"https://{account_id}.r2.cloudflarestorage.com"

backend/main.py

@@ -1,10 +1,13 @@
 import os
 import logging
-from fastapi import UploadFile, HTTPException
+from fastapi import UploadFile, HTTPException, Form
 import modal
 
-# Constants
-PINECONE_CHUNKS_INDEX = "chunks-index"
+# Pinecone index names per environment
+PINECONE_INDEX_MAP = {
+    "dev": "chunks-index",
+    "prod": "prod-chunks"
+}
 
 # Configure logging
 logging.basicConfig(
@@ -27,8 +30,8 @@
             )
         )
 
-# Environment: "dev" (default) or "prod" (set via ENV variable)
-env = os.environ.get("ENV", "dev")
+# Environment: "dev" (default) or "prod" (set via ENVIRONMENT variable)
+env = os.environ.get("ENVIRONMENT", "dev")
 
 # Create Modal app
 app = modal.App(
@@ -82,14 +85,18 @@ def startup(self):
             raise ValueError("R2_SECRET_ACCESS_KEY not found in environment variables")
 
         ENVIRONMENT = os.getenv("ENVIRONMENT", "dev")
-        if ENVIRONMENT not in ["dev", "test", "prod"]:
-            raise ValueError(f"Invalid ENVIRONMENT value: {ENVIRONMENT}. Must be one of: dev, test, prod")
+        if ENVIRONMENT not in ["dev", "prod"]:
+            raise ValueError(f"Invalid ENVIRONMENT value: {ENVIRONMENT}. Must be one of: dev, prod")
         logger.info(f"Running in environment: {ENVIRONMENT}")
 
+        # Select Pinecone index based on environment
+        pinecone_index = PINECONE_INDEX_MAP[ENVIRONMENT]
+        logger.info(f"Using Pinecone index: {pinecone_index}")
+
         # Instantiate classes
         self.preprocessor = Preprocessor(min_chunk_duration=1.0, max_chunk_duration=10.0, scene_threshold=13.0)
         self.video_embedder = VideoEmbedder()
-        self.pinecone_connector = PineconeConnector(api_key=PINECONE_API_KEY, index_name=PINECONE_CHUNKS_INDEX)
+        self.pinecone_connector = PineconeConnector(api_key=PINECONE_API_KEY, index_name=pinecone_index)
         self.job_store = JobStoreConnector(dict_name="clipabit-jobs")
 
         self.r2_connector = R2Connector(
@@ -101,7 +108,7 @@ def startup(self):
 
         self.searcher = Searcher(
             api_key=PINECONE_API_KEY,
-            index_name=PINECONE_CHUNKS_INDEX,
+            index_name=pinecone_index,
             r2_connector=self.r2_connector
         )
 
@@ -110,16 +117,16 @@ def startup(self):
         print(f"[Container] Started at {self.start_time.isoformat()}")
 
     @modal.method()
-    async def process_video(self, video_bytes: bytes, filename: str, job_id: str):
-        logger.info(f"[Job {job_id}] Processing started: {filename} ({len(video_bytes)} bytes)")
+    async def process_video(self, video_bytes: bytes, filename: str, job_id: str, namespace: str = ""):
+        logger.info(f"[Job {job_id}] Processing started: {filename} ({len(video_bytes)} bytes) | namespace='{namespace}'")
 
         try:
             # Upload original video to R2 bucket
             # TODO: do this in parallel with processing and provide url once done
             success, hashed_identifier = self.r2_connector.upload_video(
                 video_data=video_bytes,
                 filename=filename,
-                # user_id="user1" # Specify user ID once we have user management
+                namespace=namespace
             )
             if not success:
                 raise Exception(f"Failed to upload video to R2 storage: {hashed_identifier}")
@@ -175,7 +182,7 @@ async def process_video(self, video_bytes: bytes, filename: str, job_id: str):
                 self.pinecone_connector.upsert_chunk(
                     chunk_id=chunk['chunk_id'],
                     chunk_embedding=embedding.numpy(),
-                    namespace="",
+                    namespace=namespace,
                     metadata=chunk['metadata']
                 )            
 
@@ -244,12 +251,13 @@ async def status(self, job_id: str):
         return job_data
 
     @modal.fastapi_endpoint(method="POST")
-    async def upload(self, file: UploadFile = None):
+    async def upload(self, file: UploadFile = None, namespace: str = Form("")):
         """
         Handle video file upload and start background processing.
 
         Args:
             file (UploadFile): The uploaded video file.
+            namespace (str, optional): Namespace for Pinecone and R2 storage (default: "")
 
         Returns:
             dict: Contains job_id, filename, content_type, size_bytes, status, and message.
@@ -264,11 +272,12 @@ async def upload(self, file: UploadFile = None):
             "filename": file.filename,
             "status": "processing",
             "size_bytes": file_size,
-            "content_type": file.content_type
+            "content_type": file.content_type,
+            "namespace": namespace
         })
 
         # Spawn background processing (non-blocking - returns immediately)
-        self.process_video.spawn(contents, file.filename, job_id)
+        self.process_video.spawn(contents, file.filename, job_id, namespace)
 
         return {
             "job_id": job_id,
@@ -281,12 +290,13 @@ async def upload(self, file: UploadFile = None):
 
 
     @modal.fastapi_endpoint(method="GET")
-    async def search(self, query: str):
+    async def search(self, query: str, namespace: str = ""):
         """
         Search endpoint - accepts a text query and returns semantic search results.
 
         Args:
         - query (str): The search query string (required)
+        - namespace (str, optional): Namespace for Pinecone search (default: "")
         - top_k (int, optional): Number of top results to return (default: 10)
 
         Returns: dict with 'query', 'results', and 'timing'.
@@ -300,16 +310,17 @@ async def search(self, query: str):
                 raise HTTPException(status_code=400, detail="Missing 'query' parameter")
 
             top_k = 10
-            logger.info(f"[Search] Query: '{query}' | top_k={top_k}")
+            logger.info(f"[Search] Query: '{query}' | namespace='{namespace}' | top_k={top_k}")
 
             # Execute semantic search
             results = self.searcher.search(
                 query=query,
-                top_k=top_k
+                top_k=top_k,
+                namespace=namespace
             )
-            
+
             t_done = time.perf_counter()
-            
+
             # Log chunk-level results only
             logger.info(f"[Search] Found {len(results)} chunk-level results in {t_done - t_start:.3f}s")
 

backend/pyproject.toml

@@ -20,7 +20,6 @@ dependencies = [
 
 [project.scripts]
 dev = "cli:serve"
-deploy = "cli:deploy"
 
 [build-system]
 requires = ["hatchling"]

backend/search/searcher.py

@@ -61,14 +61,16 @@ def device(self) -> str:
     def search(
         self,
         query: str,
-        top_k: int = 5
+        top_k: int = 5,
+        namespace: Optional[str] = None
     ) -> List[Dict[str, Any]]:
         """
         Search for semantically similar content.
 
         Args:
             query: Natural language search query
             top_k: Number of results to return (default: 5)
+            namespace: Optional namespace to override default (default: None uses self.namespace)
 
         Returns:
             List of matches with scores and metadata, sorted by similarity
@@ -79,15 +81,17 @@ def search(
                 print(f"Score: {result['score']}")
                 print(f"Metadata: {result['metadata']}")
         """
-        logger.info(f"Searching for: '{query}' (top_k={top_k})")
-        
+        # Use provided namespace or fall back to default
+        search_namespace = namespace if namespace is not None else self.namespace
+        logger.info(f"Searching for: '{query}' (top_k={top_k}, namespace='{search_namespace}')")
+
         # Generate query embedding
         query_embedding = self.embedder.embed_text(query)
-        
+
         # Search Pinecone with optional filters
         matches = self.connector.query_chunks(
             query_embedding=query_embedding,
-            namespace=self.namespace,
+            namespace=search_namespace,
             top_k=top_k
         )
 

frontend/streamlit/app.py

@@ -2,6 +2,7 @@
 import time
 import requests
 import streamlit as st
+from config import Config
 
 
 # Page configuration
@@ -15,17 +16,18 @@
 if 'search_results' not in st.session_state:
     st.session_state.search_results = None
 
-# API endpoints
-SEARCH_API_URL = "https://clipabit01--clipabit-server-search-dev.modal.run"
-UPLOAD_API_URL = "https://clipabit01--clipabit-server-upload-dev.modal.run"
-STATUS_API_URL = "https://clipabit01--clipabit-server-status-dev.modal.run"
-LIST_VIDEOS_API_URL = "https://clipabit01--clipabit-server-list-videos-dev.modal.run"
+# API endpoints from config
+SEARCH_API_URL = Config.SEARCH_API_URL
+UPLOAD_API_URL = Config.UPLOAD_API_URL
+STATUS_API_URL = Config.STATUS_API_URL
+LIST_VIDEOS_API_URL = Config.LIST_VIDEOS_API_URL
+NAMESPACE = Config.NAMESPACE
 
 
 def search_videos(query: str):
     """Send search query to backend."""
     try:
-        resp = requests.get(SEARCH_API_URL, params={"query": query}, timeout=30)
+        resp = requests.get(SEARCH_API_URL, params={"query": query, "namespace": NAMESPACE}, timeout=30)
         if resp.status_code == 200:
             return resp.json()
         else:
@@ -38,7 +40,7 @@ def search_videos(query: str):
 def fetch_all_videos():
     """Fetch all videos from the backend."""
     try:
-        resp = requests.get(LIST_VIDEOS_API_URL, timeout=30)
+        resp = requests.get(LIST_VIDEOS_API_URL, params={"namespace": NAMESPACE}, timeout=30)
         if resp.status_code == 200:
             data = resp.json()
             return data.get("videos", [])
@@ -50,7 +52,8 @@ def fetch_all_videos():
 def upload_file_to_backend(file_bytes: bytes, filename: str, content_type: str | None = None):
     """Upload file to backend via multipart form-data."""
     files = {"file": (filename, io.BytesIO(file_bytes), content_type or "application/octet-stream")}
-    resp = requests.post(UPLOAD_API_URL, files=files, timeout=300)
+    data = {"namespace": NAMESPACE}
+    resp = requests.post(UPLOAD_API_URL, files=files, data=data, timeout=300)
     return resp
 
 

frontend/streamlit/config.py

@@ -0,0 +1,48 @@
+"""Configuration module for ClipABit Streamlit frontend."""
+
+import os
+
+
+class Config:
+    """Configuration class for environment-based settings."""
+
+    # Environment (defaults to "dev")
+    ENVIRONMENT = os.getenv("ENVIRONMENT", "dev")
+
+    # Validate environment
+    if ENVIRONMENT not in ["dev", "prod"]:
+        raise ValueError(f"Invalid ENVIRONMENT value: {ENVIRONMENT}. Must be one of: dev, prod")
+
+    # Modal app name (matches backend app name)
+    APP_NAME = ENVIRONMENT
+
+    # API Endpoints - dynamically constructed based on environment
+    # Pattern: https://clipabit01--{env}-server-{endpoint}-{env}.modal.run
+    SEARCH_API_URL = f"https://clipabit01--{APP_NAME}-server-search-{APP_NAME}.modal.run"
+    UPLOAD_API_URL = f"https://clipabit01--{APP_NAME}-server-upload-{APP_NAME}.modal.run"
+    STATUS_API_URL = f"https://clipabit01--{APP_NAME}-server-status-{APP_NAME}.modal.run"
+    LIST_VIDEOS_API_URL = f"https://clipabit01--{APP_NAME}-server-list-videos-{APP_NAME}.modal.run"
+
+    # Namespace for Pinecone and R2 (web-demo for public demo)
+    NAMESPACE = "web-demo"
+
+    @classmethod
+    def get_config(cls):
+        """Get configuration as a dictionary."""
+        return {
+            "environment": cls.ENVIRONMENT,
+            "app_name": cls.APP_NAME,
+            "search_api_url": cls.SEARCH_API_URL,
+            "upload_api_url": cls.UPLOAD_API_URL,
+            "status_api_url": cls.STATUS_API_URL,
+            "list_videos_api_url": cls.LIST_VIDEOS_API_URL,
+            "namespace": cls.NAMESPACE,
+        }
+
+    @classmethod
+    def print_config(cls):
+        """Print current configuration for debugging."""
+        config = cls.get_config()
+        print("Current Configuration:")
+        for key, value in config.items():
+            print(f"  {key}: {value}")