linux_cuda_wheel.yaml

name: Linux CUDA x86

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
        - v[0-9]+.[0-9]+.[0-9]+
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: write

defaults:
  run:
    shell: bash -l -eo pipefail {0}

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: linux
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      with-cpu: disable
      with-xpu: disable
      with-rocm: disable
      with-cuda: enable
      build-python-only: "disable"
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
    name: Build and Upload wheel
    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
    with:
      repository: meta-pytorch/torchcodec
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: packaging/pre_build_script.sh
      post-script: packaging/post_build_script.sh
      smoke-test-script: packaging/fake_smoke_test.py
      package-name: torchcodec
      trigger-event: ${{ github.event_name }}
      build-platform: "python-build-package"
      build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation"

  install-and-test:
    runs-on: linux.g5.4xlarge.nvidia.gpu
    strategy:
      fail-fast: false
      matrix:
          # 3.10 corresponds to the minimum python version for which we build
          # the wheel unless the label cliflow/binaries/all is present in the
          # PR.
          # For the actual release we should add that label and change this to
          # include more python versions.
        python-version: ['3.10']
        # We test against 12.6 and 13.0 to avoid having too big of a CI matrix,
        # but for releases we should add 12.8.
        cuda-version: ['12.6', '13.0']
        # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
        ffmpeg-version-for-tests: ['4', '6', '7', '8']

    container:
      image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
      options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
    needs: build
    steps:
      - name: Setup env vars
        run: |
          cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')
          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
          python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV

      - name: Check out repo
        uses: actions/checkout@v6

      - name: Remove src/ folder
        run: bash packaging/remove_src.sh

      - name: Setup miniconda using test-infra
        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
        with:
          python-version: ${{ matrix.python-version }}
          # We install conda packages at the start because otherwise conda may have conflicts with dependencies.
          # Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it.
          default-packages: "nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau"
      - name: Check env, set LD_LIBRARY_PATH
        run: |
          ${CONDA_RUN} env
          ${CONDA_RUN} conda info
          ${CONDA_RUN} nvidia-smi
          ${CONDA_RUN} conda list
          echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
      - name: Assert ffmpeg exists
        run: |
          ${CONDA_RUN} ffmpeg -buildconf
      - name: Update pip
        run: ${CONDA_RUN} python -m pip install --upgrade pip
      - name: Install PyTorch
        run: |
          ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
          ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'

      - uses: actions/download-artifact@v4
        with:
          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
          path: dist/

      - name: Install torchcodec from the wheel
        run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"

      # We skip installing libheif in cases where conda refuses to solve:
      # - FFmpeg 4 and 5 pin old aom/svt-av1 that conda's libheif can't satisfy.
      # - CUDA 12.6, for whatever confusing and annoying reason.
      # HEIC is CPU-only anyway *and* it's still covered on the remaining CUDA 13
      # jobs (FFmpeg 6/7/8), so this loses no real coverage.
      - name: Install test dependencies
        run: ${CONDA_RUN} bash packaging/install_test_dependencies.sh ${{ matrix.ffmpeg-version-for-tests != '4' && matrix.ffmpeg-version-for-tests != '5' && matrix.cuda-version != '12.6' }}
      - name: Run Python tests
        run: |
          if [ "${{ matrix.ffmpeg-version-for-tests }}" = "4" ] || [ "${{ matrix.ffmpeg-version-for-tests }}" = "5" ] || [ "${{ matrix.cuda-version }}" = "12.6" ]; then export FAIL_WITHOUT_HEIC=0; fi
          ${CONDA_RUN} python packaging/assert_ffmpeg_version.py ${{ matrix.ffmpeg-version-for-tests }}
          ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 FAIL_WITHOUT_IMAGE_CODECS=1 pytest --override-ini="addopts=-v" test --tb=short
      - name: Run Python benchmark
        run: |
          ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none

  install-and-test-on-cpu-only-machine:
    # This job tests that CUDA wheels work fine on CPU-only machines.  Note that
    # we still install a CUDA-enabled version of torch, and that's by design.
    # Essentially, what we want to make sure is that
    # `pip install torch torchcodec` works on CPU-only machines, and this
    # command should install CUDA-enabled versions of both torch and torchcodec.
    # It's critical that this job runs on a CPU-only machine.
    runs-on: ubuntu-latest
    needs: build
    env:
      PYTHON_VERSION: '3.10'
      CUDA_VERSION: '12.6'
      FFMPEG_VERSION: '7'
    steps:
      - name: Setup env vars
        run: |
          cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
          python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV

      - name: Check out repo
        uses: actions/checkout@v6

      - name: Remove src/ folder
        run: bash packaging/remove_src.sh

      - name: Setup conda env
        uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          miniforge-version: latest
          activate-environment: test
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Update pip
        run: python -m pip install --upgrade pip

      - name: Install PyTorch
        run: bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"

      - uses: actions/download-artifact@v4
        with:
          name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
          path: dist/

      - name: Install torchcodec from the wheel
        run: bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"

      - name: Install ffmpeg
        run: bash packaging/install_ffmpeg.sh ${{ env.FFMPEG_VERSION }}

      - name: Install test dependencies
        run: bash packaging/install_test_dependencies.sh ${{ env.FFMPEG_VERSION != '4' }}

      - name: Assert CUDA is not available
        run: |
          python -c "import torch; assert not torch.cuda.is_available()"

      - name: Run Python tests
        run: |
          if [ "${{ env.FFMPEG_VERSION }}" = "4" ]; then export FAIL_WITHOUT_HEIC=0; fi
          python packaging/assert_ffmpeg_version.py ${{ env.FFMPEG_VERSION }}
          FAIL_WITHOUT_IMAGE_CODECS=1 pytest --override-ini="addopts=-v" test --tb=short

  install-and-test-no-ffmpeg:
    # Asserts that the CUDA wheel works WITHOUT FFmpeg installed: it must import,
    # the FFmpeg-free image decoders must work, and the FFmpeg-backed entry
    # points must fail with a clear error.
    runs-on: ubuntu-latest
    needs: build
    env:
      PYTHON_VERSION: '3.10'
      CUDA_VERSION: '12.6'
    steps:
      - name: Setup env vars
        run: |
          cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
          python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV

      - name: Check out repo
        uses: actions/checkout@v6

      - name: Remove src/ folder
        run: bash packaging/remove_src.sh

      - name: Setup conda env
        uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          miniforge-version: latest
          activate-environment: test
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Update pip
        run: python -m pip install --upgrade pip

      - name: Install PyTorch
        run: bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"

      - uses: actions/download-artifact@v4
        with:
          name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
          path: dist/

      - name: Install torchcodec from the wheel
        run: bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"

      - name: Install test dependencies
        run: bash packaging/install_test_dependencies.sh

      - name: Assert FFmpeg is absent and torchcodec still imports
        run: |
          source packaging/helpers.sh
          assert_ffmpeg_not_installed
          python -c "import torchcodec; from torchcodec._core import ops; assert not ops._FFMPEG_AVAILABLE, 'FFmpeg was unexpectedly loaded'; print('OK: torchcodec imported without FFmpeg')"

      - name: Run FFmpeg-free Python tests
        run: |
          FAIL_WITHOUT_IMAGE_CODECS=1 FAIL_WITHOUT_HEIC=0 \
            pytest --override-ini="addopts=-v" test/test_ffmpeg_optional.py test/test_decoders.py::TestImageDecoder

  build-docs:
    runs-on: linux.g5.4xlarge.nvidia.gpu
    env:
      PYTHON_VERSION: '3.10'
      CUDA_VERSION: '12.6'
      FFMPEG_VERSION: '7'
    container:
      image: "pytorch/manylinux2_28-builder:cuda12.6"  # must be same as env!!
      options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
    needs: build
    steps:
      - name: Setup env vars
        run: |
          cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
          python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV

      - name: Check out repo
        uses: actions/checkout@v6

      - name: Remove src/ folder
        run: bash packaging/remove_src.sh

      - name: Setup miniconda using test-infra
        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          # We install conda packages at the start because otherwise conda may have conflicts with dependencies.
          default-packages: "nvidia::cuda-nvrtc=${{ env.CUDA_VERSION }} nvidia::cuda-toolkit=${{ env.CUDA_VERSION }} nvidia::cuda-cudart=${{ env.CUDA_VERSION }} nvidia::cuda-driver-dev=${{ env.CUDA_VERSION }} conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}"
      - name: Check env, set LD_LIBRARY_PATH
        run: |
          ${CONDA_RUN} env
          ${CONDA_RUN} conda info
          ${CONDA_RUN} nvidia-smi
          ${CONDA_RUN} conda list
          echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
      - name: Assert ffmpeg exists
        run: |
          ${CONDA_RUN} ffmpeg -buildconf
      - name: Update pip
        run: ${CONDA_RUN} python -m pip install --upgrade pip
      - name: Install PyTorch
        run: |
          ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
          ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'

      - uses: actions/download-artifact@v4
        with:
          name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
          path: dist/

      - name: Install torchcodec from the wheel
        run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"

      - name: Install doc dependencies
        run: |
          cd docs
          ${CONDA_RUN} python -m pip install -r requirements.txt
      - name: Build docs
        run: |
          cd docs
          ${CONDA_RUN} make html
      - uses: actions/upload-artifact@v4
        with:
          name: Built-Docs
          path: docs/build/html/

  doc-preview:
    runs-on: [self-hosted, linux.2xlarge]
    needs: build-docs
    if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'pull_request'
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: Built-Docs
          path: docs

      # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs
      - name: Add no-index tag
        run: |
          find docs -name "*.html" -print0 | xargs -0 sed -i '/<head>/a \ \ <meta name="robots" content="noindex">';

      - name: Upload docs preview
        uses: seemethere/upload-artifact-s3@v5
        with:
          retention-days: 14
          s3-bucket: doc-previews
          if-no-files-found: error
          path: docs
          s3-prefix: meta-pytorch/torchcodec/${{ github.event.pull_request.number }}

  upload-docs:
    # This job uploads built docs:
    # - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch
    # - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10).

    needs: build-docs
    if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' &&
        ((github.ref_type == 'branch' && github.ref_name == 'main') ||
         (github.ref_type == 'tag' && !contains(github.ref_name, '-rc')))
    runs-on: ubuntu-latest
    steps:
      - name: Check out gh-pages branch
        uses: actions/checkout@v6
        with:
          ref: gh-pages

      - uses: actions/download-artifact@v4
        with:
          name: Built-Docs
          path: docs-artifact/

      - name: Update docs and push
        run: |
          set -euo pipefail

          REF_TYPE=${{ github.ref_type }}
          REF_NAME=${{ github.ref_name }}

          if [[ "${REF_TYPE}" == branch ]]; then
            TARGET_FOLDER="${REF_NAME}"
          elif [[ "${REF_TYPE}" == tag ]]; then
            # Strip the leading "v" as well as the trailing patch version. For example:
            # 'v0.10.2' -> '0.10'
            TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/')
          fi
          echo "Target Folder: ${TARGET_FOLDER}"

          mkdir -p "${TARGET_FOLDER}"
          rm -rf "${TARGET_FOLDER}"/*
          cp -r docs-artifact/* "${TARGET_FOLDER}/"
          git add "${TARGET_FOLDER}"

          git config user.name 'pytorchbot'
          git config user.email 'soumith+bot@pytorch.org'
          git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit"
          git push