Merge pull request #65 from ClipABit/staging-issues

Staging issues

Author: eshaan-mehta

backend/api/fastapi_router.py

@@ -38,7 +38,7 @@ def __init__(
         self.router = APIRouter()
 
         # Initialize UploadHandler with process_video spawn function
-        from services.upload import UploadHandler
+        from services.upload_handler import UploadHandler
         self.upload_handler = UploadHandler(
             job_store=server_instance.job_store,
             process_video_spawn_fn=self._get_process_video_spawn_fn()
@@ -88,6 +88,7 @@ def _register_routes(self):
         self.router.add_api_route("/search", self.search, methods=["GET"])
         self.router.add_api_route("/videos", self.list_videos, methods=["GET"])
         self.router.add_api_route("/videos/{hashed_identifier}", self.delete_video, methods=["DELETE"])
+        self.router.add_api_route("/cache/clear", self.clear_cache, methods=["POST"])
 
     async def health(self):
         """
@@ -276,3 +277,33 @@ async def delete_video(self, hashed_identifier: str, filename: str, namespace: s
             "message": "Video deletion started, processing in background"
         }
 
+    async def clear_cache(self, namespace: str = "__default__"):
+        """
+        Clear the URL cache for a given namespace.
+        
+        Args:
+            namespace (str, optional): Namespace to clear cache for (default: "__default__")
+        
+        Returns:
+            dict: Contains status and number of entries cleared
+        
+        Raises:
+            HTTPException: If cache clearing is not allowed (403 Forbidden)
+        """
+        logger.info(f"[Clear Cache] Request to clear cache for namespace: {namespace}")
+        if not self.is_internal_env:
+            raise HTTPException(status_code=403, detail="Cache clearing is not allowed in the current environment.")
+        
+        try:
+            cleared_count = self.server_instance.r2_connector.clear_cache(namespace)
+            logger.info(f"[Clear Cache] Cleared {cleared_count} cache entries for namespace: {namespace}")
+            return {
+                "status": "success",
+                "namespace": namespace,
+                "cleared_entries": cleared_count,
+                "message": f"Successfully cleared {cleared_count} cache entries"
+            }
+        except Exception as e:
+            logger.error(f"[Clear Cache] Error clearing cache: {e}")
+            raise HTTPException(status_code=500, detail=str(e))
+

backend/apps/dev_combined.py

@@ -20,8 +20,8 @@
 
 from shared.config import get_environment, get_secrets, is_internal_env
 from shared.images import get_dev_image
-from services.search import SearchService
-from services.processing import ProcessingService
+from services.search_service import SearchService
+from services.processing_service import ProcessingService
 from services.http_server import ServerService
 
 logger = logging.getLogger(__name__)

backend/apps/processing_app.py

@@ -12,7 +12,7 @@
 
 from shared.config import get_environment, get_secrets
 from shared.images import get_processing_image
-from services.processing import ProcessingService
+from services.processing_service import ProcessingService
 
 logger = logging.getLogger(__name__)
 

backend/apps/search_app.py

@@ -12,7 +12,7 @@
 
 from shared.config import get_environment, get_secrets
 from shared.images import get_search_image
-from services.search import SearchService
+from services.search_service import SearchService
 
 logger = logging.getLogger(__name__)
 

backend/apps/server.py

@@ -9,9 +9,9 @@
 
 # Individual apps for staging/prod deployment
 APPS = {
-    "server": ("apps/server.py", "\033[36m"),      # Cyan
-    "search": ("apps/search_app.py", "\033[33m"),  # Yellow
-    "processing": ("apps/processing_app.py", "\033[35m"),  # Magenta
+    "server": ("services/http_server.py", "\033[36m"),      # Cyan
+    "search": ("services/search_service.py", "\033[33m"),  # Yellow
+    "processing": ("services/processing_service.py", "\033[35m"),  # Magenta
 }
 RESET = "\033[0m"
 

backend/cli.py

@@ -364,15 +364,12 @@ def fetch_video_page(
             params = {
                 "Bucket": self.bucket_name,
                 "Prefix": prefix,
-                "MaxKeys": min(page_size + 1, 1000),
+                "MaxKeys": page_size,  # Request exactly page_size items
             }
 
             if continuation_token:
-                cursor_key = self._decode_cursor_token(continuation_token)
-                if cursor_key:
-                    params["StartAfter"] = cursor_key
-                else:
-                    params["ContinuationToken"] = continuation_token
+                # Use S3's native ContinuationToken
+                params["ContinuationToken"] = continuation_token
 
             response = self.s3_client.list_objects_v2(**params)
 
@@ -385,15 +382,19 @@ def fetch_video_page(
                     continue
                 filtered.append(obj)
 
-            has_more_flag = response.get("IsTruncated", False)
-            items = filtered[:page_size]
-            has_more = has_more_flag or len(filtered) > page_size
+            # Use S3's IsTruncated flag to determine if there are more pages
+            has_more = response.get("IsTruncated", False)
+            items = filtered  # Use all filtered items since we requested exactly page_size
 
             videos: List[dict] = []
             for obj in items:
                 object_key = obj.get("Key")
                 try:
-                    filename = object_key.split('/', 1)[1]
+                    parts = object_key.split('/', 1)
+                    if len(parts) != 2:
+                        logger.warning(f"Skipping object with unexpected key format: {object_key}")
+                        continue
+                    filename = parts[1]
                     identifier = self._encode_path(self.bucket_name, namespace, filename)
                     url = self.s3_client.generate_presigned_url(
                         'get_object',
@@ -406,12 +407,12 @@ def fetch_video_page(
                             "hashed_identifier": identifier,
                             "presigned_url": url,
                         })
-                except ClientError as e:
-                    logger.error(f"Error processing video {object_key}: {e}")
+                    else:
+                        logger.warning(f"Generated empty presigned URL for {object_key}")
+                except Exception as e:
+                    logger.error(f"Error processing video {object_key}: {e}", exc_info=True)
 
-            next_token = response.get("NextContinuationToken") if has_more_flag else None
-            if not next_token and has_more and videos:
-                next_token = self._encode_cursor_token(items[-1].get("Key"))
+            next_token = response.get("NextContinuationToken") if has_more else None
 
             logger.info(
                 "Fetched %s video objects for namespace %s (has_more=%s)",

backend/database/r2_connector.py

@@ -1,3 +0,0 @@
-from .upload import UploadHandler
-
-__all__ = ["UploadHandler"]