Merge pull request #61 from ClipABit/redelete

added lost pagination

Author: eshaan-mehta

backend/api/fastapi_router.py

@@ -133,7 +133,12 @@ async def search(self, query: str, namespace: str = "", top_k: int = 10):
             logger.error(f"[Search] Error: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def list_videos(self, namespace: str = "__default__"):
+    async def list_videos(
+        self,
+        namespace: str = "__default__",
+        page_size: int = 20,
+        page_token: str | None = None,
+    ):
         """
         List all videos stored in R2 for the given namespace.
         
@@ -146,13 +151,30 @@ async def list_videos(self, namespace: str = "__default__"):
         Raises:
             HTTPException: If fetching videos fails (500 Internal Server Error)
         """
-        logger.info(f"[List Videos] Fetching videos for namespace: {namespace}")
+        if page_size <= 0:
+            raise HTTPException(status_code=400, detail="page_size must be positive")
+
+        logger.info(
+            "[List Videos] Fetching videos for namespace: %s (page_size=%s, page_token=%s)",
+            namespace,
+            page_size,
+            page_token,
+        )
         try:
-            video_data = self.server_instance.r2_connector.fetch_all_video_data(namespace)
+            videos, next_token, total_videos, total_pages = (
+                self.server_instance.r2_connector.list_videos_page(
+                    namespace=namespace,
+                    page_size=page_size,
+                    continuation_token=page_token,
+                )
+            )
             return {
                 "status": "success",
                 "namespace": namespace,
-                "videos": video_data
+                "videos": videos,
+                "next_page_token": next_token,
+                "total_videos": total_videos,
+                "total_pages": total_pages,
             }
         except Exception as e:
             logger.error(f"[List Videos] Error fetching videos: {e}")

backend/database/r2_connector.py

@@ -52,7 +52,7 @@ def __init__(
 
         logger.info(f"Initialized R2Connector for bucket: {self.bucket_name}")
 
-        self.url_cache_connnector = UrlCacheConnector(environment=environment)
+        self._url_cache = UrlCacheConnector(environment=environment)
 
     def _sanitize_filename(self, filename: str) -> str:
         """
@@ -442,14 +442,14 @@ def list_videos_page(
         total_videos: Optional[int] = None
         cache_hit = False
 
-        if self.url_cache_connnector:
-            cached = self.url_cache_connnector.get_page(namespace, normalized_token, page_size)
+        if self._url_cache:
+            cached = self._url_cache.get_page(namespace, normalized_token, page_size)
             if cached:
                 videos = cached.get("videos", [])
                 next_token = cached.get("next_token")
                 cache_hit = True
 
-            metadata = self.url_cache_connnector.get_namespace_metadata(namespace)
+            metadata = self._url_cache.get_namespace_metadata(namespace)
             if metadata is not None:
                 total_videos = metadata.get("total_videos")
 
@@ -459,13 +459,13 @@ def list_videos_page(
                 page_size=page_size,
                 continuation_token=normalized_token,
             )
-            if self.url_cache_connnector:
-                self.url_cache_connnector.set_page(namespace, normalized_token, page_size, videos, next_token)
+            if self._url_cache:
+                self._url_cache.set_page(namespace, normalized_token, page_size, videos, next_token)
 
         if total_videos is None:
             total_videos = self.count_videos(namespace=namespace)
-            if self.url_cache_connnector:
-                self.url_cache_connnector.set_namespace_metadata(
+            if self._url_cache:
+                self._url_cache.set_namespace_metadata(
                     namespace,
                     {
                         "total_videos": int(total_videos),
@@ -557,4 +557,4 @@ def fetch_all_video_data(
         return video_data_list
 
     def clear_cache(self, namespace: str) -> int:
-        return self.url_cache_connnector.clear_namespace(namespace or "__default__")
+        return self._url_cache.clear_namespace(namespace or "__default__")

backend/tests/conftest.py

@@ -391,3 +391,12 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers", "slow: marks tests as slow (deselect with '-m \"not slow\"')"
     )
+
+
+def pytest_collection_modifyitems(session, config, items):
+    """Ensure API endpoint tests execute after the rest for isolation."""
+    api_items = [item for item in items if "tests/integration/test_api_endpoints.py" in item.nodeid]
+    if not api_items:
+        return
+    remaining = [item for item in items if item not in api_items]
+    items[:] = remaining + api_items

backend/tests/integration/test_api_endpoints.py

@@ -52,6 +52,17 @@ def fetch_all_video_data(self, namespace: str) -> List[Dict[str, Any]]:
         self.last_namespace = namespace
         return self.videos
 
+    def list_videos_page(
+        self,
+        namespace: str,
+        page_size: int,
+        continuation_token: str | None,
+    ) -> Tuple[List[Dict[str, Any]], str | None, int, int]:
+        self.last_namespace = namespace
+        total_videos = len(self.videos)
+        total_pages = 1 if total_videos else 0
+        return self.videos[:page_size], None, total_videos, total_pages
+
 
 class ServerStub:
     """
@@ -126,6 +137,9 @@ def test_list_videos_returns_data(test_client_internal: Tuple[TestClient, Server
     assert data["namespace"] == "ns1"
     assert isinstance(data["videos"], list)
     assert data["videos"][0]["file_name"] == "sample.mp4"
+    assert data["total_videos"] == 1
+    assert data["total_pages"] == 1
+    assert data["next_page_token"] is None
     assert server.r2_connector.last_namespace == "ns1"