diff --git a/.github/workflows/docker-integration-tests.yml b/.github/workflows/docker-integration-tests.yml new file mode 100644 index 000000000..59c3e9714 --- /dev/null +++ b/.github/workflows/docker-integration-tests.yml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +name: "Docker Integration Tests" + +on: + pull_request: + push: + branches: + - main + +jobs: + docker-integration-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build runtime image + uses: docker/build-push-action@v5 + with: + context: . + target: runtime + tags: aiperf:test + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build env-builder stage (for uv) + uses: docker/build-push-action@v5 + with: + context: . + target: env-builder + tags: aiperf:env-builder + load: true + cache-from: type=gha + + - name: Copy uv from env-builder to host + run: | + docker create --name uv-container aiperf:env-builder + docker cp uv-container:/bin/uv ./uv + docker rm uv-container + chmod +x ./uv + + - name: Fix workspace permissions for container user + run: | + # Make workspace writable by container user (UID 1000) + sudo chown -R 1000:1000 ${{ github.workspace }} + + - name: Run integration tests in container + run: | + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -v ${{ github.workspace }}/uv:/usr/local/bin/uv:ro \ + -w /workspace \ + aiperf:test \ + "uv pip install pytest pytest-xdist pytest-asyncio pytest-cov && \ + uv pip install -e 'tests/aiperf_mock_server[dev]' && \ + /opt/aiperf/venv/bin/pytest tests/integration/ -m 'integration and not performance and not stress' -n auto -v --tb=long" diff --git a/ATTRIBUTIONS-container.md b/ATTRIBUTIONS-container.md index 594681677..c4c04f9f9 100644 --- a/ATTRIBUTIONS-container.md +++ b/ATTRIBUTIONS-container.md @@ -17,7 +17,7 @@ This document provides attribution information for third-party software componen - **Website**: https://ffmpeg.org/ - **License**: LGPL v2.1+ - **Usage**: Video and audio processing library (included in runtime container) -- **Build Configuration**: Built without GPL components (`--disable-gpl --disable-nonfree`) +- **Build Configuration**: Built without GPL components (`--disable-gpl --disable-nonfree --enable-libvpx`) **License Text:** @@ -48,6 +48,65 @@ The FFmpeg source code used to build this container is available at: - Build configuration excludes GPL-licensed components - Apache 2.0 licensed code in this project remains separate from LGPL components +### libvpx + +**Component Information:** +- **Software**: libvpx (VP8/VP9 Codec SDK) +- **Version**: 1.12.0 (from Debian Bookworm) +- **Source**: Debian Bookworm +- **Website**: https://www.webmproject.org/ +- **License**: BSD 3-Clause +- **Usage**: VP9 video codec library (included in runtime container, used by FFmpeg) + +**License Text:** + +> Copyright (c) 2010, The WebM Project authors. All rights reserved. +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions are +> met: +> +> * Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> +> * Redistributions in binary form must reproduce the above copyright +> notice, this list of conditions and the following disclaimer in +> the documentation and/or other materials provided with the +> distribution. +> +> * Neither the name of Google, nor the WebM Project, nor the names +> of its contributors may be used to endorse or promote products +> derived from this software without specific prior written +> permission. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +> A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +> HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +> SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +> LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +> DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +> THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +> (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +> OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +> +> Full license text: https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/LICENSE + +**Source Code Availability:** + +The libvpx source code is available at: +- Debian package: https://packages.debian.org/bookworm/libvpx-dev +- Debian source package: `apt-get source libvpx` from Debian Bookworm repositories +- Upstream WebM Project source: https://chromium.googlesource.com/webm/libvpx/ + +**Compliance Notes:** + +- libvpx binary is copied from Debian Bookworm base image +- No modifications were made to libvpx source code +- libvpx is dynamically linked with FFmpeg +- BSD license is compatible with Apache 2.0 + ### Bash **Component Information:** @@ -97,6 +156,12 @@ LGPL is compatible with Apache 2.0 when: - No modifications were made to FFmpeg source code - Proper attribution is provided (as above) +### libvpx (BSD 3-Clause) +BSD 3-Clause is compatible with Apache 2.0: +- BSD is a permissive license that allows redistribution with minimal restrictions +- Attribution requirements are satisfied through this document +- No conflict with Apache 2.0 terms + ### Bash (GPL v3+) GPL is compatible with Apache 2.0 when: - Bash runs as a separate executable and is not linked with Apache 2.0 code diff --git a/Dockerfile b/Dockerfile index 6558d90a5..704616a2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ FROM base AS env-builder WORKDIR /workspace -# Build ffmpeg from source +# Build ffmpeg from source with libvpx RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ @@ -90,9 +90,10 @@ RUN apt-get update -y && \ pkg-config \ wget \ yasm \ + libvpx-dev \ && rm -rf /var/lib/apt/lists/* -# Download and build ffmpeg +# Download and build ffmpeg with libvpx (VP9 codec) RUN wget https://ffmpeg.org/releases/ffmpeg-7.1.tar.xz \ && tar -xf ffmpeg-7.1.tar.xz \ && cd ffmpeg-7.1 \ @@ -102,6 +103,7 @@ RUN wget https://ffmpeg.org/releases/ffmpeg-7.1.tar.xz \ --disable-nonfree \ --enable-shared \ --disable-static \ + --enable-libvpx \ --disable-doc \ --disable-htmlpages \ --disable-manpages \ @@ -137,6 +139,9 @@ COPY LICENSE ATTRIBUTIONS*.md /legal/ # Copy bash with executable permissions preserved using --chmod COPY --from=env-builder --chown=1000:1000 --chmod=755 /bin/bash /bin/bash +# Copy libvpx libraries +COPY --from=env-builder --chown=1000:1000 /usr/lib/x86_64-linux-gnu/libvpx.so* /usr/lib/x86_64-linux-gnu/ + # Copy ffmpeg binaries and libraries COPY --from=env-builder --chown=1000:1000 /opt/ffmpeg /opt/ffmpeg ENV PATH="/opt/ffmpeg/bin:${PATH}" \ @@ -154,4 +159,4 @@ ENV VIRTUAL_ENV=/opt/aiperf/venv \ PATH="/opt/aiperf/venv/bin:${PATH}" # Set bash as entrypoint -ENTRYPOINT ["/bin/bash", "-c"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "-c"] diff --git a/docs/tutorials/synthetic-video.md b/docs/tutorials/synthetic-video.md index 70819a4e9..e642f4457 100644 --- a/docs/tutorials/synthetic-video.md +++ b/docs/tutorials/synthetic-video.md @@ -40,7 +40,7 @@ The synthetic video feature provides: - Configurable resolution, frame rate, and duration - Hardware-accelerated encoding options (CPU and GPU codecs) - Base64-encoded video output for API requests -- MP4 format support +- MP4 and WebM format support ## Basic Usage @@ -160,13 +160,16 @@ aiperf profile \ --url localhost:8000 \ --video-width 640 \ --video-height 480 \ - --video-codec libx264 \ + --video-codec libvpx-vp9 \ + --video-format webm \ --request-count 20 ``` **Available CPU Codecs:** -- `libx264`: H.264 encoding, widely compatible (default) -- `libx265`: H.265 encoding, smaller file sizes, slower encoding +- `libvpx-vp9`: VP9 encoding, BSD-licensed (default, WebM format) +- `libopenh264`: H.264 encoding, BSD-licensed (MP4 format) +- `libx264`: H.264 encoding, GPL-licensed, widely compatible (MP4 format) +- `libx265`: H.265 encoding, GPL-licensed, smaller file sizes, slower encoding (MP4 format) #### GPU Encoding (NVIDIA) @@ -290,8 +293,22 @@ aiperf profile \ ### Video Format -Currently, AIPerf supports **MP4** format: +AIPerf supports both **WebM** (default) and **MP4** formats: + +**WebM format (default):** +```bash +aiperf profile \ + --model your-model-name \ + --endpoint-type chat \ + --url localhost:8000 \ + --video-width 640 \ + --video-height 480 \ + --video-format webm \ + --video-codec libvpx-vp9 \ + --request-count 20 +``` +**MP4 format:** ```bash aiperf profile \ --model your-model-name \ @@ -300,6 +317,7 @@ aiperf profile \ --video-width 640 \ --video-height 480 \ --video-format mp4 \ + --video-codec libx264 \ --request-count 20 ``` @@ -316,7 +334,7 @@ This allows seamless integration with vision-language model APIs that accept bas ### Encoding Performance -- **CPU codecs** (`libx264`, `libx265`): Slower but universally available +- **CPU codecs** (`libvpx-vp9`, `libopenh264`, `libx264`, `libx265`): Slower but universally available - **GPU codecs** (`h264_nvenc`, `hevc_nvenc`): Much faster, requires NVIDIA GPU - Higher resolution and frame rates increase encoding time @@ -359,7 +377,7 @@ Error: Encoder 'h264_nvenc' not found Solutions: 1. Verify NVIDIA GPU is available: `nvidia-smi` 2. Check FFmpeg was compiled with NVENC support: `ffmpeg -encoders | grep nvenc` -3. Fall back to CPU codec: `--video-codec libx264` +3. Fall back to CPU codec: `--video-codec libvpx-vp9 --video-format webm` or `--video-codec libx264 --video-format mp4` ### Out of Memory diff --git a/src/aiperf/common/config/config_defaults.py b/src/aiperf/common/config/config_defaults.py index fbe91f260..828551139 100644 --- a/src/aiperf/common/config/config_defaults.py +++ b/src/aiperf/common/config/config_defaults.py @@ -87,8 +87,8 @@ class VideoDefaults: WIDTH = None HEIGHT = None SYNTH_TYPE = VideoSynthType.MOVING_SHAPES - FORMAT = VideoFormat.MP4 - CODEC = "libx264" + FORMAT = VideoFormat.WEBM + CODEC = "libvpx-vp9" @dataclass(frozen=True) diff --git a/src/aiperf/common/config/video_config.py b/src/aiperf/common/config/video_config.py index 2b333d9af..f97e903f0 100644 --- a/src/aiperf/common/config/video_config.py +++ b/src/aiperf/common/config/video_config.py @@ -121,7 +121,9 @@ def validate_width_and_height(self) -> Self: Field( description=( "The video codec to use for encoding. Common options: " - "libx264 (CPU, widely compatible), libx265 (CPU, smaller files), " + "libvpx-vp9 (CPU, BSD-licensed, default for WebM), " + "libopenh264 (CPU, BSD-licensed), libx264 (CPU, GPL-licensed, widely compatible), " + "libx265 (CPU, GPL-licensed, smaller files), " "h264_nvenc (NVIDIA GPU), hevc_nvenc (NVIDIA GPU, smaller files). " "Any FFmpeg-supported codec can be used." ), diff --git a/src/aiperf/common/enums/dataset_enums.py b/src/aiperf/common/enums/dataset_enums.py index d8c954789..de37ff840 100644 --- a/src/aiperf/common/enums/dataset_enums.py +++ b/src/aiperf/common/enums/dataset_enums.py @@ -34,6 +34,7 @@ class AudioFormat(CaseInsensitiveStrEnum): class VideoFormat(CaseInsensitiveStrEnum): MP4 = "mp4" + WEBM = "webm" class VideoSynthType(CaseInsensitiveStrEnum): diff --git a/src/aiperf/dataset/generator/video.py b/src/aiperf/dataset/generator/video.py index c3a5a9c5b..24b08dc2f 100644 --- a/src/aiperf/dataset/generator/video.py +++ b/src/aiperf/dataset/generator/video.py @@ -20,8 +20,8 @@ class VideoGenerator(BaseGenerator): """A class that generates synthetic videos. This class provides methods to create synthetic videos with different patterns - like moving shapes or grid clocks. The videos are generated in MP4 format and - returned as base64 encoded strings. Currently only MP4 format is supported. + like moving shapes or grid clocks. The videos are generated in MP4 or WebM format + and returned as base64 encoded strings. """ def __init__(self, config: VideoConfig, **kwargs): @@ -246,7 +246,7 @@ def _generate_grid_clock_frames(self, total_frames: int) -> list[Image.Image]: def _encode_frames_to_base64(self, frames: list[Image.Image]) -> str: """Convert frames to video data and encode as base64 string. - Creates video data using the format specified in config. Currently only MP4 is supported. + Creates video data using the format specified in config. Supports MP4 and WebM formats. """ if not frames: return "" @@ -254,9 +254,9 @@ def _encode_frames_to_base64(self, frames: list[Image.Image]) -> str: # Validate format from aiperf.common.enums import VideoFormat - if self.config.format != VideoFormat.MP4: + if self.config.format not in [VideoFormat.MP4, VideoFormat.WEBM]: raise ValueError( - f"Unsupported video format: {self.config.format}. Only MP4 is supported." + f"Unsupported video format: {self.config.format}. Only MP4 and WebM are supported." ) # Check if FFmpeg is available before proceeding @@ -270,9 +270,11 @@ def _encode_frames_to_base64(self, frames: list[Image.Image]) -> str: ) try: - return self._create_mp4_with_ffmpeg(frames) + return self._create_video_with_ffmpeg(frames) except Exception as e: - self.logger.error(f"Failed to create MP4 with ffmpeg: {e}") + self.logger.error( + f"Failed to create {self.config.format.value.upper()} with ffmpeg: {e}" + ) # Provide specific error messages based on the error type if "No such file or directory" in str(e) or "not found" in str(e): @@ -282,7 +284,7 @@ def _encode_frames_to_base64(self, frames: list[Image.Image]) -> str: elif "Codec" in str(e) or "codec" in str(e): raise RuntimeError( f"Video codec '{self.config.codec}' is not supported. " - f"Please use a valid FFmpeg codec (e.g., libx264, libx265, h264_nvenc)." + f"Please use a valid FFmpeg codec (e.g., libvpx-vp9, libopenh264, libx264, libx265, h264_nvenc)." ) from e else: raise RuntimeError( @@ -290,23 +292,24 @@ def _encode_frames_to_base64(self, frames: list[Image.Image]) -> str: f"Codec: {self.config.codec}, Size: {self.config.width}x{self.config.height}, FPS: {self.config.fps}" ) from e - def _create_mp4_with_ffmpeg(self, frames: list[Image.Image]) -> str: - """Create MP4 data using ffmpeg-python with improved error handling.""" + def _create_video_with_ffmpeg(self, frames: list[Image.Image]) -> str: + """Create video data using ffmpeg-python with improved error handling.""" try: # First try the in-memory approach - return self._create_mp4_with_pipes(frames) + return self._create_video_with_pipes(frames) except (BrokenPipeError, OSError, RuntimeError) as e: self.logger.warning( f"Pipe method failed ({e}), falling back to temporary file method" ) # Fall back to temporary file approach if pipes fail - return self._create_mp4_with_temp_files(frames) + return self._create_video_with_temp_files(frames) - def _create_mp4_with_pipes(self, frames: list[Image.Image]) -> str: - """Create MP4 using pipes via temporary file (preferred method).""" - # MP4 muxer doesn't support non-seekable output (pipes), so we use a temp file - with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp_file: + def _create_video_with_pipes(self, frames: list[Image.Image]) -> str: + """Create video using pipes via temporary file (preferred method).""" + # Use temp file since some muxers don't support non-seekable output (pipes) + file_ext = f".{self.config.format.value}" + with tempfile.NamedTemporaryFile(suffix=file_ext, delete=False) as tmp_file: temp_path = tmp_file.name try: @@ -328,6 +331,17 @@ def _create_mp4_with_pipes(self, frames: list[Image.Image]) -> str: # Combine all frame data all_data = b"".join(frame_data) + # Build output options based on format + output_options = { + "format": self.config.format.value, + "vcodec": self.config.codec, + "pix_fmt": "yuv420p", + } + + # Add format-specific options + if self.config.format.value == "mp4": + output_options["movflags"] = "faststart" + # Use ffmpeg.run with input data, output to temporary file _ = ( ffmpeg.input( @@ -337,26 +351,20 @@ def _create_mp4_with_pipes(self, frames: list[Image.Image]) -> str: s=f"{self.config.width}x{self.config.height}", r=self.config.fps, ) - .output( - temp_path, - format="mp4", - vcodec=self.config.codec, - pix_fmt="yuv420p", - movflags="faststart", - ) + .output(temp_path, **output_options) .overwrite_output() # Allow overwriting the temp file .run(input=all_data, capture_stdout=True, capture_stderr=True) ) - # Read the MP4 file back + # Read the video file back with open(temp_path, "rb") as f: - mp4_data = f.read() + video_data = f.read() - if not mp4_data: + if not video_data: raise RuntimeError("FFmpeg produced no output") # Encode as base64 - base64_data = base64.b64encode(mp4_data).decode("utf-8") + base64_data = base64.b64encode(video_data).decode("utf-8") return f"data:video/{self.config.format.value};base64,{base64_data}" except ffmpeg.Error as e: @@ -368,8 +376,8 @@ def _create_mp4_with_pipes(self, frames: list[Image.Image]) -> str: if os.path.exists(temp_path): os.unlink(temp_path) - def _create_mp4_with_temp_files(self, frames: list[Image.Image]) -> str: - """Create MP4 using temporary files (fallback method).""" + def _create_video_with_temp_files(self, frames: list[Image.Image]) -> str: + """Create video using temporary files (fallback method).""" # Create temporary directory for frames temp_dir = tempfile.mkdtemp(prefix="aiperf_frames_") @@ -387,32 +395,38 @@ def _create_mp4_with_temp_files(self, frames: list[Image.Image]) -> str: frame.save(frame_path, "PNG", compress_level=6, optimize=False) # Create output file in the same temp directory - output_path = os.path.join(temp_dir, "output.mp4") + file_ext = self.config.format.value + output_path = os.path.join(temp_dir, f"output.{file_ext}") frame_pattern = os.path.join(temp_dir, "frame_%06d.png") - # Use ffmpeg to create MP4 from frames + # Build output options based on format + output_options = { + "format": self.config.format.value, + "vcodec": self.config.codec, + "pix_fmt": "yuv420p", + } + + # Add format-specific options + if self.config.format.value == "mp4": + output_options["movflags"] = "faststart" + + # Use ffmpeg to create video from frames _ = ( ffmpeg.input(frame_pattern, r=self.config.fps) - .output( - output_path, - format="mp4", - vcodec=self.config.codec, - pix_fmt="yuv420p", - movflags="faststart", - ) + .output(output_path, **output_options) .overwrite_output() .run(capture_stdout=True, capture_stderr=True) ) # Read the output file with open(output_path, "rb") as f: - mp4_data = f.read() + video_data = f.read() - if not mp4_data: + if not video_data: raise RuntimeError("FFmpeg produced no output") # Encode as base64 - base64_data = base64.b64encode(mp4_data).decode("utf-8") + base64_data = base64.b64encode(video_data).decode("utf-8") return f"data:video/{self.config.format.value};base64,{base64_data}" except ffmpeg.Error as e: diff --git a/tests/generators/test_video_generator.py b/tests/generators/test_video_generator.py index f24d6e4d3..b9dcdadc9 100644 --- a/tests/generators/test_video_generator.py +++ b/tests/generators/test_video_generator.py @@ -21,8 +21,8 @@ def base_config(): height=64, duration=0.5, fps=2, - format=VideoFormat.MP4, - codec="libx264", + format=VideoFormat.WEBM, + codec="libvpx-vp9", synth_type=VideoSynthType.MOVING_SHAPES, ) @@ -107,8 +107,8 @@ def test_generate_with_disabled_video(self): height=None, duration=1.0, fps=4, - format=VideoFormat.MP4, - codec="libx264", + format=VideoFormat.WEBM, + codec="libvpx-vp9", synth_type=VideoSynthType.MOVING_SHAPES, ) generator = VideoGenerator(config) @@ -129,8 +129,8 @@ def test_generate_frames(self, synth_type, width, height, duration, fps): height=height, duration=duration, fps=fps, - format=VideoFormat.MP4, - codec="libx264", + format=VideoFormat.WEBM, + codec="libvpx-vp9", synth_type=synth_type, ) generator = VideoGenerator(config) @@ -188,31 +188,31 @@ def test_encode_frames_codec_error(self, base_config): patch.object(generator, "_check_ffmpeg_availability", return_value=True), patch.object( generator, - "_create_mp4_with_pipes", + "_create_video_with_pipes", side_effect=Exception("Codec not supported"), ), patch.object( generator, - "_create_mp4_with_temp_files", + "_create_video_with_temp_files", side_effect=Exception("Codec not supported"), ), pytest.raises(RuntimeError, match="[Cc]odec"), ): generator._encode_frames_to_base64(frames) - def test_create_mp4_with_pipes_fallback(self, base_config): + def test_create_video_with_pipes_fallback(self, base_config): """Test fallback to temp files when pipes fail.""" generator = VideoGenerator(base_config) frames = [Image.new("RGB", (64, 64), (255, 0, 0))] - mock_result = "data:video/mp4;base64,FAKE_BASE64" + mock_result = "data:video/webm;base64,FAKE_BASE64" with ( patch.object( - generator, "_create_mp4_with_pipes", side_effect=BrokenPipeError + generator, "_create_video_with_pipes", side_effect=BrokenPipeError ), patch.object( - generator, "_create_mp4_with_temp_files", return_value=mock_result + generator, "_create_video_with_temp_files", return_value=mock_result ), ): - result = generator._create_mp4_with_ffmpeg(frames) + result = generator._create_video_with_ffmpeg(frames) assert result == mock_result