Refactor CI jobs

.github/workflows/cpp_tests.yaml

@@ -37,17 +37,7 @@ jobs:
       - name: Update pip
         run: python -m pip install --upgrade pip
       - name: Install torch dependencies
-        run: |
-          # If we're in a release branch or in a PR against a release branch,
-          # we install the PyTorch RCs from the test channel. Otherwise, e.g. in
-          # `main` or in PRs against `main`, we install the nightly builds.
-          # Note that the `test` RCs are
-          if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
-            CHANNEL=test
-          else
-            CHANNEL=nightly
-          fi
-          python -m pip install --pre torch --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu
+        run: bash packaging/install_pytorch.sh cpu "torch"
       - name: Install ffmpeg, pkg-config and pybind11
         run: |
           conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge

.github/workflows/lint.yaml

@@ -62,16 +62,7 @@ jobs:
         run: python -m pip install --upgrade pip
       - name: Install dependencies and FFmpeg
         run: |
-          # If we're in a release branch or in a PR against a release branch,
-          # we install the PyTorch RCs from the test channel. Otherwise, e.g. in
-          # `main` or in PRs against `main`, we install the nightly builds.
-          # Note that the `test` RCs are
-          if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
-            CHANNEL=test
-          else
-            CHANNEL=nightly
-          fi
-          python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu
+          bash packaging/install_pytorch.sh cpu "torch torchvision"
           conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge
           ffmpeg -version
       - name: Build and install torchcodec

.github/workflows/linux_cuda_aarch64_wheel.yaml

@@ -70,10 +70,12 @@ jobs:
         ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0']
     needs: build
     steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_aarch64
-          path: pytorch/torchcodec/dist/
+      - name: Check out repo
+        uses: actions/checkout@v3
+
+      - name: Remove src/ folder
+        run: bash packaging/remove_src.sh
+
       - name: Setup conda env
         uses: conda-incubator/setup-miniconda@v3
         with:
@@ -88,57 +90,23 @@ jobs:
       - name: Update pip
         run: python -m pip install --upgrade pip
       - name: Install PyTorch
-        run: |
-          # If we're in a release branch or in a PR against a release branch,
-          # we install the PyTorch RCs from the test channel. Otherwise, e.g. in
-          # `main` or in PRs against `main`, we install the nightly builds.
-          # Note that the `test` RCs are
-          if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
-            CHANNEL=test
-          else
-            CHANNEL=nightly
-          fi
-          ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cu${{ env.cuda_version_without_periods }}
+        run: ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_aarch64
+          path: dist/
+
       - name: Install torchcodec from the wheel
-        run: |
-          wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
-          echo Installing $wheel_path
-          python -m pip install $wheel_path -vvv
+        run: bash packaging/install_torchcodec_wheel.sh
 
-      - name: Check out repo
-        uses: actions/checkout@v3
       - name: Install ffmpeg, post build
         run: |
-          # Ideally we would have checked for that before installing the wheel,
-          # but we need to checkout the repo to access this file, and we don't
-          # want to checkout the repo before installing the wheel to avoid any
-          # side-effect. It's OK.
-          source packaging/helpers.sh
-          assert_ffmpeg_not_installed
-
-          conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
-          ffmpeg -version
+          bash packaging/install_ffmpeg.sh ${{ matrix.ffmpeg-version-for-tests }}
           echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
 
       - name: Install test dependencies
-        run: |
-          # Ideally we would find a way to get those dependencies from pyproject.toml
-          python -m pip install numpy pytest pillow
-
-      - name: Delete the src/ folder just for fun
-        run: |
-          # The only reason we checked-out the repo is to get access to the
-          # tests. We don't care about the rest. Out of precaution, we delete
-          # the src/ folder to be extra sure that we're running the code from
-          # the installed wheel rather than from the source.
-          # This is just to be extra cautious and very overkill because a)
-          # there's no way the `torchcodec` package from src/ can be found from
-          # the PythonPath: the main point of `src/` is precisely to protect
-          # against that and b) if we ever were to execute code from
-          # `src/torchcodec`, it would fail loudly because the built .so files
-          # aren't present there.
-          rm -r src/
-          ls
+        run: bash packaging/install_test_dependencies.sh
       - name: Run Python tests
         run: |
           pytest --override-ini="addopts=-v" test

.github/workflows/linux_cuda_wheel.yaml

@@ -1,4 +1,4 @@
-name: Build and test Linux CUDA wheels
+name: Build and test Linux CUDA wheels and docs
 
 on:
   pull_request:
@@ -84,10 +84,13 @@ jobs:
           echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
           python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
           echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
-      - uses: actions/download-artifact@v4
-        with:
-          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
-          path: pytorch/torchcodec/dist/
+
+      - name: Check out repo
+        uses: actions/checkout@v3
+
+      - name: Remove src/ folder
+        run: bash packaging/remove_src.sh
+
       - name: Setup miniconda using test-infra
         uses: pytorch/test-infra/.github/actions/setup-miniconda@main
         with:
@@ -109,49 +112,90 @@ jobs:
         run: ${CONDA_RUN} python -m pip install --upgrade pip
       - name: Install PyTorch
         run: |
-          # If we're in a release branch or in a PR against a release branch,
-          # we install the PyTorch RCs from the test channel. Otherwise, e.g. in
-          # `main` or in PRs against `main`, we install the nightly builds.
-          # Note that the `test` RCs are
-          if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
-            CHANNEL=test
-          else
-            CHANNEL=nightly
-          fi
-
-          ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cu${{ env.cuda_version_without_periods }}
+          ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
           ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
+          path: dist/
+
       - name: Install torchcodec from the wheel
+        run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"
+
+      - name: Install test dependencies
+        run: ${CONDA_RUN} bash packaging/install_test_dependencies.sh
+      - name: Run Python tests
+        run: |
+          ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short
+      - name: Run Python benchmark
         run: |
-          wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"`
-          echo Installing $wheel_path
-          ${CONDA_RUN} python -m pip install $wheel_path -vvv
+          ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none
+
+  build-docs:
+    runs-on: linux.4xlarge.nvidia.gpu
+    env:
+      PYTHON_VERSION: '3.10'
+      CUDA_VERSION: '12.6'
+      FFMPEG_VERSION: '7'
+    container:
+      image: "pytorch/manylinux2_28-builder:cuda12.6"  # must be same as env!!
+      options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
+    needs: build
+    steps:
+      - name: Setup env vars
+        run: |
+          cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
+          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
+          python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
+          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
 
       - name: Check out repo
         uses: actions/checkout@v3
 
-      - name: Install test dependencies
-        run: |
-          # Ideally we would find a way to get those dependencies from pyproject.toml
-          ${CONDA_RUN} python -m pip install numpy pytest pillow
+      - name: Remove src/ folder
+        run: bash packaging/remove_src.sh
 
-      - name: Delete the src/ folder just for fun
+      - name: Setup miniconda using test-infra
+        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          # We install conda packages at the start because otherwise conda may have conflicts with dependencies.
+          default-packages: "nvidia/label/cuda-${{ env.CUDA_VERSION }}.0::libnpp nvidia::cuda-nvrtc=${{ env.CUDA_VERSION }} nvidia::cuda-toolkit=${{ env.CUDA_VERSION }} nvidia::cuda-cudart=${{ env.CUDA_VERSION }} nvidia::cuda-driver-dev=${{ env.CUDA_VERSION }} conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}"
+      - name: Check env, set LD_LIBRARY_PATH
         run: |
-          # The only reason we checked-out the repo is to get access to the
-          # tests. We don't care about the rest. Out of precaution, we delete
-          # the src/ folder to be extra sure that we're running the code from
-          # the installed wheel rather than from the source.
-          # This is just to be extra cautious and very overkill because a)
-          # there's no way the `torchcodec` package from src/ can be found from
-          # the PythonPath: the main point of `src/` is precisely to protect
-          # against that and b) if we ever were to execute code from
-          # `src/torchcodec`, it would fail loudly because the built .so files
-          # aren't present there.
-          rm -r src/
-          ls
-      - name: Run Python tests
+          ${CONDA_RUN} env
+          ${CONDA_RUN} conda info
+          ${CONDA_RUN} nvidia-smi
+          ${CONDA_RUN} conda list
+          echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
+      - name: Assert ffmpeg exists
         run: |
-          ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short
-      - name: Run Python benchmark
+          ${CONDA_RUN} ffmpeg -buildconf
+      - name: Update pip
+        run: ${CONDA_RUN} python -m pip install --upgrade pip
+      - name: Install PyTorch
         run: |
-          ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none
+          ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
+          ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
+          path: dist/
+
+      - name: Install torchcodec from the wheel
+        run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"
+
+      - name: Install doc dependencies
+        run: |
+          cd docs
+          ${CONDA_RUN} python -m pip install -r requirements.txt
+      - name: Build docs
+        run: |
+          cd docs
+          ${CONDA_RUN} make html
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Built-Docs
+          path: docs/build/html/