Skip to content

Commit 93195e5

Browse files
NicolasHugNicolasHug
authored
Make image decoders available when FFmpeg is missing (#1544)
1 parent f561b0e commit 93195e5

12 files changed

Lines changed: 981 additions & 610 deletions

‎.github/workflows/linux_cuda_wheel.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,65 @@ jobs:
196196
run: |
197197
pytest --override-ini="addopts=-v" test --tb=short
198198
199+
install-and-test-no-ffmpeg:
200+
# Asserts that the CUDA wheel works WITHOUT FFmpeg installed: it must import,
201+
# the FFmpeg-free image decoders must work, and the FFmpeg-backed entry
202+
# points must fail with a clear error.
203+
runs-on: ubuntu-latest
204+
needs: build
205+
env:
206+
PYTHON_VERSION: '3.10'
207+
CUDA_VERSION: '12.6'
208+
steps:
209+
- name: Setup env vars
210+
run: |
211+
cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
212+
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
213+
python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
214+
echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
215+
216+
- name: Check out repo
217+
uses: actions/checkout@v6
218+
219+
- name: Remove src/ folder
220+
run: bash packaging/remove_src.sh
221+
222+
- name: Setup conda env
223+
uses: conda-incubator/setup-miniconda@v3
224+
with:
225+
auto-update-conda: true
226+
miniforge-version: latest
227+
activate-environment: test
228+
python-version: ${{ env.PYTHON_VERSION }}
229+
230+
- name: Update pip
231+
run: python -m pip install --upgrade pip
232+
233+
- name: Install PyTorch
234+
run: bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
235+
236+
- uses: actions/download-artifact@v4
237+
with:
238+
name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
239+
path: dist/
240+
241+
- name: Install torchcodec from the wheel
242+
run: bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"
243+
244+
- name: Install test dependencies
245+
run: bash packaging/install_test_dependencies.sh
246+
247+
- name: Assert FFmpeg is absent and torchcodec still imports
248+
run: |
249+
source packaging/helpers.sh
250+
assert_ffmpeg_not_installed
251+
python -c "import torchcodec; from torchcodec._core import ops; assert not ops._FFMPEG_AVAILABLE, 'FFmpeg was unexpectedly loaded'; print('OK: torchcodec imported without FFmpeg')"
252+
253+
- name: Run FFmpeg-free Python tests
254+
run: |
255+
FAIL_WITHOUT_JPEG=1 FAIL_WITHOUT_PNG=1 FAIL_WITHOUT_WEBP=1 FAIL_WITHOUT_AVIF=1 \
256+
pytest --override-ini="addopts=-v" test/test_ffmpeg_optional.py test/test_decoders.py::TestImageDecoder
257+
199258
build-docs:
200259
runs-on: linux.g5.4xlarge.nvidia.gpu
201260
env:

‎.github/workflows/linux_wheel.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,60 @@ jobs:
109109
run: |
110110
FAIL_WITHOUT_JPEG=1 FAIL_WITHOUT_PNG=1 FAIL_WITHOUT_WEBP=1 FAIL_WITHOUT_AVIF=1 pytest --override-ini="addopts=-v" test
111111
112+
install-and-test-no-ffmpeg:
113+
# Asserts that torchcodec works WITHOUT FFmpeg installed: the package must
114+
# import, the FFmpeg-free image decoders must work, and the FFmpeg-backed
115+
# entry points must fail with a clear error.
116+
runs-on: ubuntu-latest
117+
strategy:
118+
fail-fast: false
119+
matrix:
120+
python-version: ['3.10']
121+
needs: build
122+
steps:
123+
- name: Check out repo
124+
uses: actions/checkout@v6
125+
126+
- name: Remove src/ folder
127+
run: bash packaging/remove_src.sh
128+
129+
- name: Setup conda env
130+
uses: conda-incubator/setup-miniconda@v3
131+
with:
132+
auto-update-conda: true
133+
miniforge-version: latest
134+
activate-environment: test
135+
python-version: ${{ matrix.python-version }}
136+
137+
- name: Update pip
138+
run: python -m pip install --upgrade pip
139+
140+
- name: Install PyTorch
141+
run: bash packaging/install_pytorch.sh cpu "torch torchvision"
142+
143+
- uses: actions/download-artifact@v4
144+
with:
145+
name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
146+
path: dist/
147+
148+
- name: Install torchcodec from the wheel
149+
run: bash packaging/install_torchcodec_wheel.sh
150+
151+
- name: Install test dependencies
152+
run: bash packaging/install_test_dependencies.sh
153+
154+
# Deliberately NO "Install ffmpeg" step here.
155+
- name: Assert FFmpeg is absent and torchcodec still imports
156+
run: |
157+
source packaging/helpers.sh
158+
assert_ffmpeg_not_installed
159+
python -c "import torchcodec; from torchcodec._core import ops; assert not ops._FFMPEG_AVAILABLE, 'FFmpeg was unexpectedly loaded'; print('OK: torchcodec imported without FFmpeg')"
160+
161+
- name: Run FFmpeg-free Python tests
162+
run: |
163+
FAIL_WITHOUT_JPEG=1 FAIL_WITHOUT_PNG=1 FAIL_WITHOUT_WEBP=1 FAIL_WITHOUT_AVIF=1 \
164+
pytest --override-ini="addopts=-v" test/test_ffmpeg_optional.py test/test_decoders.py::TestImageDecoder
165+
112166
install-and-test-third-party-interface:
113167
runs-on: ubuntu-latest
114168
strategy:

‎README.md‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ videos and audio, or run inference, TorchCodec is how you turn these into data.
1111
We achieve these capabilities through:
1212

1313
* Pythonic APIs that mirror Python and PyTorch conventions.
14-
* Relying on [FFmpeg](https://www.ffmpeg.org/) to do the decoding and encoding.
15-
TorchCodec uses the version of FFmpeg you already have installed. FFmpeg is a
16-
mature library with broad coverage available on most systems. It is, however,
17-
not easy to use. TorchCodec abstracts FFmpeg's complexity to ensure it is used
18-
correctly and efficiently.
14+
* Relying on [FFmpeg](https://www.ffmpeg.org/) to do the video and audio
15+
decoding and encoding. TorchCodec uses the version of FFmpeg you already have
16+
installed. FFmpeg is a mature library with broad coverage available on most
17+
systems. It is, however, not easy to use. TorchCodec abstracts FFmpeg's
18+
complexity to ensure it is used correctly and efficiently. (FFmpeg is
19+
optional, and the image decoders don't need it: see [Installing
20+
TorchCodec](#installing-torchcodec).)
1921
* Returning data as PyTorch tensors, ready to be fed into PyTorch transforms
2022
or used directly to train models.
2123

@@ -105,6 +107,13 @@ with encoder.open_file("output.mp4"):
105107
conda install "ffmpeg" -c conda-forge
106108
```
107109

110+
> **Note:** FFmpeg is an *optional* dependency. It is needed for video
111+
> and audio decoding and encoding (`VideoDecoder`, `AudioDecoder`,
112+
> `VideoEncoder`, `AudioEncoder`, etc.). The image decoders
113+
> (`decode_image`, `decode_jpeg`, `decode_png`, etc.)
114+
> do **not** require FFmpeg, so if you only need image decoding you can skip
115+
> this step.
116+
108117
2. Install PyTorch and TorchCodec:
109118

110119
```bash

‎src/torchcodec/_core/_decoder_utils.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
create_wav_decoder_from_file_like,
2828
create_wav_decoder_from_tensor,
2929
)
30+
from torchcodec._internally_replaced_utils import load_core_libraries
3031
from torchcodec.transforms import DecoderTransform
3132
from torchcodec.transforms._decoder_transforms import _make_transform_specs
3233

@@ -42,6 +43,7 @@ def create_decoder(
4243
source: str | Path | io.RawIOBase | io.BufferedReader | bytes | Tensor,
4344
seek_mode: str,
4445
) -> Tensor:
46+
load_core_libraries()
4547
if isinstance(source, str):
4648
return create_from_file(source, seek_mode)
4749
elif isinstance(source, Path):
@@ -108,6 +110,9 @@ def create_audio_decoder(
108110
def create_wav_decoder(
109111
source: str | Path | io.RawIOBase | io.BufferedReader | bytes | Tensor,
110112
) -> Tensor:
113+
# WavDecoder currently lives in the FFmpeg-linked core library (it uses the
114+
# FFmpeg-based AVIO I/O layer), so it needs FFmpeg too, for now.
115+
load_core_libraries()
111116
if isinstance(source, str):
112117
return create_wav_decoder_from_file(source)
113118
elif isinstance(source, Path):

‎src/torchcodec/_core/_ffmpeg_op_names.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""The names of all FFmpeg-dependent ops and helpers exposed by ``ops.py``.
8+
9+
Kept in its own tiny module so both ``ops.py`` and ``_ffmpeg_ops.py`` can import
10+
it without importing each other.
11+
"""
12+
13+
FFMPEG_OP_NAMES = frozenset(
14+
{
15+
"create_from_file",
16+
"create_from_tensor",
17+
"_create_from_file_like",
18+
"_add_video_stream_raw",
19+
"_add_video_stream",
20+
"add_video_stream",
21+
"add_audio_stream",
22+
"seek_to_pts",
23+
"get_next_frame",
24+
"get_frame_at_pts",
25+
"get_frame_at_index",
26+
"_get_frames_at_indices_tensor_input",
27+
"_get_frames_by_pts_tensor_input",
28+
"get_frames_in_range",
29+
"get_frames_by_pts_in_range",
30+
"get_frames_by_pts_in_range_audio",
31+
"get_json_metadata",
32+
"_blocks_create_demuxer",
33+
"_blocks_demuxer_next_packet",
34+
"_blocks_create_packet_decoder",
35+
"_blocks_packet_decoder_send_packet",
36+
"_blocks_packet_decoder_send_eof",
37+
"_blocks_packet_decoder_receive_frame",
38+
"_blocks_create_color_converter",
39+
"_blocks_convert_frame",
40+
"_test_frame_pts_equality",
41+
"_get_container_json_metadata",
42+
"_get_key_frame_indices",
43+
"scan_all_streams_to_update_metadata",
44+
"_get_stream_json_metadata",
45+
"_get_json_ffmpeg_library_versions",
46+
"_get_backend_details",
47+
"create_streaming_encoder",
48+
"streaming_encoder_close",
49+
"streaming_encoder_add_video_stream",
50+
"streaming_encoder_add_audio_stream",
51+
"streaming_encoder_open_file",
52+
"_streaming_encoder_open_file_like",
53+
"streaming_encoder_add_frames",
54+
"streaming_encoder_add_samples",
55+
"set_nvdec_cache_capacity",
56+
"get_nvdec_cache_capacity",
57+
"_get_nvdec_cache_size",
58+
"_set_cpp_log_level",
59+
"_get_log_level",
60+
"create_wav_decoder_from_file",
61+
"create_wav_decoder_from_tensor",
62+
"_create_wav_decoder_from_file_like",
63+
"get_wav_samples_in_range",
64+
"get_wav_metadata_from_decoder",
65+
"create_from_bytes",
66+
"create_from_file_like",
67+
"create_wav_decoder_from_bytes",
68+
"create_wav_decoder_from_file_like",
69+
"streaming_encoder_open_file_like",
70+
"get_frames_at_indices",
71+
"get_frames_by_pts",
72+
"get_ffmpeg_library_versions",
73+
}
74+
)

0 commit comments

Comments
 (0)