Merge pull request #28 from ClipABit/feature/av1

eshaan-mehta · web-flow · commit 8248cb41ecbc · 2025-12-20T13:30:20.000-05:00
Av1 codec support
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,6 +31,9 @@ jobs:
           enable-cache: true
           cache-suffix: "backend"
 
+      - name: install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
+
       - name: install dependencies
         run: uv sync --frozen
 
diff --git a/backend/main.py b/backend/main.py
@@ -3,12 +3,6 @@
 from fastapi import UploadFile, HTTPException, Form
 import modal
 
-# Pinecone index names per environment
-PINECONE_INDEX_MAP = {
-    "dev": "chunks-index",
-    "prod": "prod-chunks"
-}
-
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,
@@ -20,6 +14,7 @@
 # dependencies found in pyproject.toml
 image = (
             modal.Image.debian_slim(python_version="3.12")
+            .apt_install("ffmpeg", "libsm6", "libxext6") # for video processing
             .uv_sync(extra_options="--no-dev")  # exclude dev dependencies to avoid package conflicts
             .add_local_python_source(  # add all local modules here
                 "preprocessing",
@@ -91,7 +86,7 @@ def startup(self):
         logger.info(f"Running in environment: {ENVIRONMENT}")
 
         # Select Pinecone index based on environment
-        pinecone_index = PINECONE_INDEX_MAP[ENVIRONMENT]
+        pinecone_index = f"{ENVIRONMENT}-chunks"
         logger.info(f"Using Pinecone index: {pinecone_index}")
 
         # Instantiate classes
diff --git a/backend/preprocessing/preprocessor.py b/backend/preprocessing/preprocessor.py
@@ -1,5 +1,7 @@
 import logging
 import tempfile
+import subprocess
+import os
 from typing import List, Dict, Any, Optional
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import cv2
@@ -97,6 +99,81 @@ def _get_video_metadata(self, video_path: str) -> Dict[str, Any]:
         logger.debug("Cached metadata: fps=%.2f, frames=%d, duration=%.2fs", fps, frame_count, duration)
         return metadata
 
+    def _get_video_codec(self, video_path: str) -> str:
+        """Get the video codec using ffprobe."""
+        try:
+            cmd = [
+                "ffprobe", 
+                "-v", "error", 
+                "-select_streams", "v:0", 
+                "-show_entries", "stream=codec_name", 
+                "-of", "default=noprint_wrappers=1:nokey=1", 
+                video_path
+            ]
+            codec = subprocess.check_output(cmd).decode().strip()
+            return codec
+        except Exception as e:
+            logger.warning(f"Failed to detect codec for {video_path}: {e}")
+            return "unknown"
+        
+    def _transcode_to_h264(self, input_path: str, codec: str) -> str:
+        """
+            Transcode video to H.264 codec using ffmpeg.
+
+            Arguments:
+                input_path: Path to the input video file.
+                codec: Detected codec of the input video.
+
+            Returns:
+                Path to the transcoded video file.
+        """
+        if codec == "h264":
+            return input_path  # No transcoding needed
+
+        transcoded_temp = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False)
+        transcoded_path = transcoded_temp.name
+        transcoded_temp.close()  # Close so ffmpeg can write to it
+
+        logger.info(f"Transcoding video ({codec} -> h264) to ensure compatibility: {input_path} -> {transcoded_path}")
+        
+        # FFmpeg command: -y (overwrite), -i (input), -c:v libx264 (video codec), -c:a aac (audio), -preset fast
+        cmd = [
+            "ffmpeg", "-y", "-v", "error",
+            "-i", input_path,
+            "-c:v", "libx264",
+            "-preset", "fast",
+            "-c:a", "aac",
+            transcoded_path
+        ]
+        
+        try:
+            completed_process = subprocess.run(
+                cmd,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            # Log any non-fatal stderr output for debugging at a lower level
+            if completed_process.stderr:
+                logger.debug(
+                    "ffmpeg stderr output during successful transcoding for %s: %s",
+                    input_path,
+                    completed_process.stderr,
+                )
+            return transcoded_path
+        except subprocess.CalledProcessError as e:
+            logger.error(
+                "ffmpeg transcoding failed for %s with return code %s. stderr: %s",
+                input_path,
+                e.returncode,
+                e.stderr,
+            )
+            # Cleanup partial file if transcoding fails
+            if os.path.exists(transcoded_path):
+                os.unlink(transcoded_path)
+            raise
+
+
     def process_video_from_bytes(
         self,
         video_bytes: bytes,
@@ -107,20 +184,59 @@ def process_video_from_bytes(
         """Process video from uploaded bytes with automatic temp file cleanup."""
         logger.info("Starting preprocessing: video_id=%s, filename=%s", video_id, filename)
 
-        with tempfile.NamedTemporaryFile(suffix='.mp4', delete=True) as temp_file:
-            temp_file.write(video_bytes)
-            temp_file.flush()
+        input_path = None
+        processing_path = None
 
-            try:
-                return self.process_video(
-                    video_path=temp_file.name,
-                    video_id=video_id,
-                    filename=filename,
-                    hashed_identifier=hashed_identifier
-                )
-            except Exception as e:
-                logger.error("Processing failed for video_id=%s: %s", video_id, e)
-                raise
+        try:
+            # Create a temp file for the original upload
+            with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as input_temp:
+                input_path = input_temp.name
+                input_temp.write(video_bytes)
+                input_temp.flush()
+
+            processing_path = input_path
+
+            # Detect codec
+            codec = self._get_video_codec(input_path)
+            logger.info(f"Detected video codec: {codec}")
+
+            if codec != "h264":
+                processing_path = self._transcode_to_h264(input_path, codec)
+            else:
+                logger.info("Video is already H.264, skipping transcoding")
+            
+            return self.process_video(
+                video_path=processing_path,
+                video_id=video_id,
+                filename=filename,
+                hashed_identifier=hashed_identifier
+            )
+
+        except subprocess.CalledProcessError as e:
+            stderr_output = ""
+            if getattr(e, "stderr", None):
+                if isinstance(e.stderr, bytes):
+                    stderr_output = e.stderr.decode("utf-8", errors="replace")
+                else:
+                    stderr_output = str(e.stderr)
+            if stderr_output:
+                logger.error("FFmpeg transcoding failed. Stderr: %s", stderr_output)
+                error_message = f"Failed to process video codec. FFmpeg stderr: {stderr_output}"
+            else:
+                logger.error("FFmpeg transcoding failed: %s", e)
+                error_message = "Failed to process video codec"
+            raise RuntimeError(error_message) from e
+        except Exception as e:
+            logger.error("Processing failed for video_id=%s: %s", video_id, e)
+            raise
+        finally:
+            # Cleanup all temp files
+            if input_path and os.path.exists(input_path):
+                os.unlink(input_path)
+            
+            # If processing_path is different (transcoded), clean it up too
+            if processing_path and processing_path != input_path and os.path.exists(processing_path):
+                os.unlink(processing_path)
 
     def process_video(
         self,
@@ -198,7 +314,7 @@ def _process_single_chunk(
         Thread-safe for parallel execution.
         Returns None if processing fails.
         """
-        #TODO: Specifcy explicit return type and not just a dict in docstring
+        #TODO: Specify explicit return type and not just a dict in docstring
         try:
             # Extract frames with complexity analysis
             frames, sampling_fps, complexity_score = self.extractor.extract_frames(video_path, chunk)
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
@@ -37,7 +37,7 @@ dev = [
 ]
 
 [tool.pytest.ini_options]
-testpaths = ["tests"]
+testpaths = ["tests/unit", "tests/integration"]
 pythonpath = ["."]
 python_files = ["test_*.py"]
 markers = [
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
@@ -17,6 +17,7 @@
 from pathlib import Path
 import tempfile
 import shutil
+import subprocess
 
 from preprocessing.chunker import Chunker
 from preprocessing.frame_extractor import FrameExtractor
@@ -49,7 +50,8 @@ def sample_video_5s(tmp_path_factory) -> Path:
 
     video_dir = tmp_path_factory.mktemp("test_videos")
     video_path = video_dir / "sample_5s.mp4"
-
+    
+    # Create a simple video using OpenCV
     fps, duration = 30, 5
     width, height = 640, 480
 
@@ -71,6 +73,82 @@ def sample_video_5s(tmp_path_factory) -> Path:
     return video_path
 
 
+@pytest.fixture(scope="session")
+def sample_video_h264(tmp_path_factory) -> Path:
+    """1-second H.264 test video generated with ffmpeg."""
+    video_dir = tmp_path_factory.mktemp("test_videos_h264")
+    video_path = video_dir / "sample_h264.mp4"
+    
+    cmd = [
+        "ffmpeg", "-y",
+        "-f", "lavfi", "-i", "testsrc=duration=1:size=320x240:rate=30",
+        "-c:v", "libx264",
+        "-preset", "fast",
+        str(video_path)
+    ]
+    
+    try:
+        subprocess.run(cmd, check=True, capture_output=True)
+    except subprocess.CalledProcessError as e:
+        pytest.skip(f"Failed to generate H.264 video: {e.stderr.decode()}")
+    except FileNotFoundError:
+        pytest.skip("ffmpeg not found, skipping H.264 test")
+        
+    return video_path
+
+
+@pytest.fixture(scope="session")
+def sample_video_vp9(tmp_path_factory) -> Path:
+    """1-second VP9 test video generated with ffmpeg."""
+    video_dir = tmp_path_factory.mktemp("test_videos_vp9")
+    video_path = video_dir / "sample_vp9.mp4"
+    
+    cmd = [
+        "ffmpeg", "-y",
+        "-f", "lavfi", "-i", "testsrc=duration=1:size=320x240:rate=30",
+        "-c:v", "libvpx-vp9",
+        "-b:v", "0", "-crf", "30",
+        str(video_path)
+    ]
+    
+    try:
+        subprocess.run(cmd, check=True, capture_output=True)
+    except subprocess.CalledProcessError as e:
+        pytest.skip(f"Failed to generate VP9 video: {e.stderr.decode()}")
+    except FileNotFoundError:
+        pytest.skip("ffmpeg not found, skipping VP9 test")
+        
+    return video_path
+
+
+@pytest.fixture(scope="session")
+def sample_video_av1(tmp_path_factory) -> Path:
+    """1-second AV1 test video generated with ffmpeg."""
+    video_dir = tmp_path_factory.mktemp("test_videos_av1")
+    video_path = video_dir / "sample_av1.mp4"
+    
+    # Generate AV1 video using ffmpeg
+    # -f lavfi -i testsrc=duration=1:size=320x240:rate=30
+    # -c:v libsvtav1 -preset 8 -crf 50
+    cmd = [
+        "ffmpeg", "-y",
+        "-f", "lavfi", "-i", "testsrc=duration=1:size=320x240:rate=30",
+        "-c:v", "libsvtav1",
+        "-preset", "8",
+        "-crf", "50",
+        str(video_path)
+    ]
+    
+    try:
+        subprocess.run(cmd, check=True, capture_output=True)
+    except subprocess.CalledProcessError as e:
+        pytest.skip(f"Failed to generate AV1 video (ffmpeg might not support libsvtav1): {e.stderr.decode()}")
+    except FileNotFoundError:
+        pytest.skip("ffmpeg not found, skipping AV1 test")
+        
+    return video_path
+
+
 @pytest.fixture(scope="session")
 def sample_video_static(tmp_path_factory) -> Path:
     """10-second static video with minimal motion."""
diff --git a/backend/tests/integration/test_preprocessor.py b/backend/tests/integration/test_preprocessor.py
diff --git a/utils/pinecone_migration.py b/utils/pinecone_migration.py

Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ dev = [`
`37`	`37`	`]`
`38`	`38`
`39`	`39`	`[tool.pytest.ini_options]`
`40`		`-testpaths = ["tests"]`
	`40`	`+testpaths = ["tests/unit", "tests/integration"]`
`41`	`41`	`pythonpath = ["."]`
`42`	`42`	`python_files = ["test_*.py"]`
`43`	`43`	`markers = [`