diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index 159f834df..654fa6b8e 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -37,17 +37,7 @@ jobs: - name: Update pip run: python -m pip install --upgrade pip - name: Install torch dependencies - run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu + run: bash packaging/install_pytorch.sh cpu "torch" - name: Install ffmpeg, pkg-config and pybind11 run: | conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml deleted file mode 100644 index 8dccc7ce2..000000000 --- a/.github/workflows/docs.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: Docs - -on: - push: - branches: [ main ] - pull_request: - -permissions: - id-token: write - contents: write - -defaults: - run: - shell: bash -l -eo pipefail {0} - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-cpu: disable - with-xpu: disable - with-rocm: disable - with-cuda: enable - build-python-only: "disable" - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: meta-pytorch/torchcodec - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" - - build-docs: - runs-on: linux.4xlarge.nvidia.gpu - strategy: - fail-fast: false - matrix: - # 3.10 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - python-version: ['3.10'] - cuda-version: ['12.6'] - ffmpeg-version-for-tests: ['7'] - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - - name: Check env - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv - - - name: Check out repo - uses: actions/checkout@v3 - - - name: Install doc dependencies - run: | - cd docs - ${CONDA_RUN} python -m pip install -r requirements.txt - - name: Build docs - run: | - cd docs - ${CONDA_RUN} make html - - uses: actions/upload-artifact@v4 - with: - name: Built-Docs - path: docs/build/html/ diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f1717464e..47e262dbb 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -62,16 +62,7 @@ jobs: run: python -m pip install --upgrade pip - name: Install dependencies and FFmpeg run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu + bash packaging/install_pytorch.sh cpu "torch torchvision" conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge ffmpeg -version - name: Build and install torchcodec diff --git a/.github/workflows/linux_cuda_aarch64_wheel.yaml b/.github/workflows/linux_cuda_aarch64_wheel.yaml index 492c13d6e..0a2a0935d 100644 --- a/.github/workflows/linux_cuda_aarch64_wheel.yaml +++ b/.github/workflows/linux_cuda_aarch64_wheel.yaml @@ -70,10 +70,12 @@ jobs: ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0'] needs: build steps: - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_aarch64 - path: pytorch/torchcodec/dist/ + - name: Check out repo + uses: actions/checkout@v3 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh + - name: Setup conda env uses: conda-incubator/setup-miniconda@v3 with: @@ -88,57 +90,23 @@ jobs: - name: Update pip run: python -m pip install --upgrade pip - name: Install PyTorch - run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cu${{ env.cuda_version_without_periods }} + run: ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_aarch64 + path: dist/ + - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + run: bash packaging/install_torchcodec_wheel.sh - - name: Check out repo - uses: actions/checkout@v3 - name: Install ffmpeg, post build run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed - - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + bash packaging/install_ffmpeg.sh ${{ matrix.ffmpeg-version-for-tests }} echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls + run: bash packaging/install_test_dependencies.sh - name: Run Python tests run: | pytest --override-ini="addopts=-v" test diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 32a70cd3d..2ccf217bc 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test Linux CUDA wheels +name: Build and test Linux CUDA wheels and docs on: pull_request: @@ -84,10 +84,13 @@ jobs: echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ + + - name: Check out repo + uses: actions/checkout@v3 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh + - name: Setup miniconda using test-infra uses: pytorch/test-infra/.github/actions/setup-miniconda@main with: @@ -109,49 +112,90 @@ jobs: run: ${CONDA_RUN} python -m pip install --upgrade pip - name: Install PyTorch run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cu${{ env.cuda_version_without_periods }} + ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 + path: dist/ + - name: Install torchcodec from the wheel + run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl" + + - name: Install test dependencies + run: ${CONDA_RUN} bash packaging/install_test_dependencies.sh + - name: Run Python tests + run: | + ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short + - name: Run Python benchmark run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv + ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none + + build-docs: + runs-on: linux.4xlarge.nvidia.gpu + env: + PYTHON_VERSION: '3.10' + CUDA_VERSION: '12.6' + FFMPEG_VERSION: '7' + container: + image: "pytorch/manylinux2_28-builder:cuda12.6" # must be same as env!! + options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + needs: build + steps: + - name: Setup env vars + run: | + cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g') + echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g') + echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - name: Check out repo uses: actions/checkout@v3 - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - ${CONDA_RUN} python -m pip install numpy pytest pillow + - name: Remove src/ folder + run: bash packaging/remove_src.sh - - name: Delete the src/ folder just for fun + - name: Setup miniconda using test-infra + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: ${{ env.PYTHON_VERSION }} + # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + default-packages: "nvidia/label/cuda-${{ env.CUDA_VERSION }}.0::libnpp nvidia::cuda-nvrtc=${{ env.CUDA_VERSION }} nvidia::cuda-toolkit=${{ env.CUDA_VERSION }} nvidia::cuda-cudart=${{ env.CUDA_VERSION }} nvidia::cuda-driver-dev=${{ env.CUDA_VERSION }} conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}" + - name: Check env, set LD_LIBRARY_PATH run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests + ${CONDA_RUN} env + ${CONDA_RUN} conda info + ${CONDA_RUN} nvidia-smi + ${CONDA_RUN} conda list + echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV + - name: Assert ffmpeg exists run: | - ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short - - name: Run Python benchmark + ${CONDA_RUN} ffmpeg -buildconf + - name: Update pip + run: ${CONDA_RUN} python -m pip install --upgrade pip + - name: Install PyTorch run: | - ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none + ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" + ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64 + path: dist/ + + - name: Install torchcodec from the wheel + run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl" + + - name: Install doc dependencies + run: | + cd docs + ${CONDA_RUN} python -m pip install -r requirements.txt + - name: Build docs + run: | + cd docs + ${CONDA_RUN} make html + - uses: actions/upload-artifact@v4 + with: + name: Built-Docs + path: docs/build/html/ diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 1bdf80623..6ba84f3ee 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -66,10 +66,12 @@ jobs: ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0'] needs: build steps: - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 - path: pytorch/torchcodec/dist/ + - name: Check out repo + uses: actions/checkout@v3 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh + - name: Setup conda env uses: conda-incubator/setup-miniconda@v3 with: @@ -81,60 +83,27 @@ jobs: miniforge-version: latest activate-environment: test python-version: ${{ matrix.python-version }} + - name: Update pip run: python -m pip install --upgrade pip + - name: Install PyTorch - run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - - python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu + run: bash packaging/install_pytorch.sh cpu "torch torchvision" + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + path: dist/ + - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + run: bash packaging/install_torchcodec_wheel.sh - - name: Check out repo - uses: actions/checkout@v3 - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed - - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + run: bash packaging/install_ffmpeg.sh ${{ matrix.ffmpeg-version-for-tests }} - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow + run: bash packaging/install_test_dependencies.sh - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - name: Run Python tests run: | pytest --override-ini="addopts=-v" test diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index 3ab21a2d8..3faf24ca1 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -68,11 +68,11 @@ jobs: ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0'] needs: build steps: - - name: Download wheel - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_ - path: pytorch/torchcodec/dist/ + - name: Check out torchcodec repo + uses: actions/checkout@v3 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh - name: Setup conda env uses: conda-incubator/setup-miniconda@v3 @@ -85,50 +85,22 @@ jobs: run: python -m pip install --upgrade pip - name: Install PyTorch - run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu + run: bash packaging/install_pytorch.sh cpu "torch torchvision" - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_ + path: dist/ - - name: Check out torchcodec repo - uses: actions/checkout@v3 + - name: Install torchcodec from the wheel + run: bash packaging/install_torchcodec_wheel.sh - name: Install ffmpeg - run: | - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + run: bash packaging/install_ffmpeg.sh ${{ matrix.ffmpeg-version-for-tests }} - name: Install test dependencies - run: | - python -m pip install numpy pytest pillow - - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls -lh + run: bash packaging/install_test_dependencies.sh - name: Run Python tests run: | diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 47e91c2cd..905596ca0 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -74,12 +74,14 @@ jobs: ffmpeg-version-for-tests: ['4.4.2', '6.1.1', '7.0.1', '8.0'] needs: build steps: - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 - path: pytorch/torchcodec/dist/ + - name: Check out repo + uses: actions/checkout@v3 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh + - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true # Using miniforge instead of miniconda ensures that the default @@ -91,57 +93,25 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip run: python -m pip install --upgrade pip + - name: Install PyTorch - run: | - # If we're in a release branch or in a PR against a release branch, - # we install the PyTorch RCs from the test channel. Otherwise, e.g. in - # `main` or in PRs against `main`, we install the nightly builds. - # Note that the `test` RCs are - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - CHANNEL=test - else - CHANNEL=nightly - fi - python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/${CHANNEL}/cpu + run: bash packaging/install_pytorch.sh cpu "torch torchvision" + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 + path: dist/ + - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 + run: bash packaging/install_torchcodec_wheel.sh + - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - - name: Test torchcodec import after FFmpeg installation - run: | - echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." - python -c "import torchcodec; print('TorchCodec import successful!')" + # need -l for conda to be exposed + run: bash -l packaging/install_ffmpeg.sh ${{ matrix.ffmpeg-version-for-tests }} + - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls + run: bash packaging/install_test_dependencies.sh + - name: Run Python tests run: | pytest test -vvv diff --git a/packaging/install_ffmpeg.sh b/packaging/install_ffmpeg.sh new file mode 100755 index 000000000..32907596a --- /dev/null +++ b/packaging/install_ffmpeg.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This script installs FFmpeg from conda-forge after asserting that FFmpeg is +# not already installed. +# +# Usage: +# install_ffmpeg.sh FFMPEG_VERSION +# install_ffmpeg.sh 7.0.1 +# install_ffmpeg.sh 8.0 + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "Error: Missing required FFmpeg version" + echo "Usage: install_ffmpeg.sh FFMPEG_VERSION" + echo "Example: install_ffmpeg.sh 7.0.1" + exit 1 +fi + +FFMPEG_VERSION="$1" + +# Ideally we would have checked for that before installing the wheel, +# but we need to checkout the repo to access this file, and we don't +# want to checkout the repo before installing the wheel to avoid any +# side-effect. It's OK. +source packaging/helpers.sh +assert_ffmpeg_not_installed + +echo "Installing FFmpeg version $FFMPEG_VERSION from conda-forge..." +conda install "ffmpeg=$FFMPEG_VERSION" -c conda-forge +ffmpeg -version diff --git a/packaging/install_pytorch.sh b/packaging/install_pytorch.sh new file mode 100755 index 000000000..7e301c9ea --- /dev/null +++ b/packaging/install_pytorch.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This script installs PyTorch and other optional torch packages like +# torchvision from either the nightly or test channel based on the branch: test +# for release branches (and PRs against a release branch), nightly otherwise +# +# Example usage: +# install_pytorch.sh cpu "torch torchvision" +# install_pytorch.sh cpu "torch" +# install_pytorch.sh cu126 "torch torchvision" + +set -euo pipefail + +if [ $# -lt 2 ]; then + echo "Error: Missing required arguments" + echo "Usage: install_pytorch.sh COMPUTE_PLATFORM PACKAGES" + echo "Example: install_pytorch.sh cpu \"torch torchvision\"" + exit 1 +fi + +COMPUTE_PLATFORM="$1" +PACKAGES="$2" + +if [[ (${GITHUB_EVENT_NAME:-} = 'pull_request' && (${GITHUB_BASE_REF:-} = 'release'*)) || (${GITHUB_REF:-} = 'refs/heads/release'*) ]]; then + CHANNEL=test +else + CHANNEL=nightly +fi + +echo "Installing PyTorch packages: $PACKAGES" +echo "Compute platform: $COMPUTE_PLATFORM" +echo "Channel: $CHANNEL" + +python -m pip install --pre $PACKAGES --index-url https://download.pytorch.org/whl/${CHANNEL}/${COMPUTE_PLATFORM} diff --git a/packaging/install_test_dependencies.sh b/packaging/install_test_dependencies.sh new file mode 100755 index 000000000..69c2d6dcb --- /dev/null +++ b/packaging/install_test_dependencies.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This script installs the test dependencies needed to run the test suite. +# +# Example usage: +# install_test_dependencies.sh + +set -euo pipefail + +echo "Installing test dependencies..." +# Ideally we would find a way to get those dependencies from pyproject.toml +python -m pip install numpy pytest pillow + +echo "Test dependencies installed successfully!" diff --git a/packaging/install_torchcodec_wheel.sh b/packaging/install_torchcodec_wheel.sh new file mode 100755 index 000000000..77b7b1383 --- /dev/null +++ b/packaging/install_torchcodec_wheel.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This script finds and installs a torchcodec wheel from the dist directory. The +# wheel is expected to have been built and downloaded from a separate job. +# +# Usage: +# install_torchcodec_wheel.sh [WHEEL_PATTERN] +# +# Example usage: +# install_torchcodec_wheel.sh +# install_torchcodec_wheel.sh "*.whl" +# install_torchcodec_wheel.sh "*cu126-cp310*.whl" + +set -euo pipefail + +WHEEL_PATTERN="${1:-*.whl}" + +wheel_path=$(find dist -type f -name "$WHEEL_PATTERN") + +if [ -z "$wheel_path" ]; then + echo "Error: No wheel found matching pattern '$WHEEL_PATTERN' in dist/" + exit 1 +fi + +echo "Installing $wheel_path" +python -m pip install "$wheel_path" -vvv diff --git a/packaging/remove_src.sh b/packaging/remove_src.sh new file mode 100755 index 000000000..e3b2f43b6 --- /dev/null +++ b/packaging/remove_src.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This script removes the src/ directory to ensure tests run against the +# installed wheel rather than local source code. +# +# Usage: +# remove_src.sh + +set -euo pipefail + +echo "Deleting src/ folder to ensure tests use installed wheel..." +# The only reason we checked-out the repo is to get access to the +# tests and to the helper scripts for the CI. We don't care about the rest. +# Out of precaution, we delete +# the src/ folder to be extra sure that we're running the code from +# the installed wheel rather than from the source. +# This is just to be extra cautious and very overkill because a) +# there's no way the `torchcodec` package from src/ can be found from +# the PythonPath: the main point of `src/` is precisely to protect +# against that and b) if we ever were to execute code from +# `src/torchcodec`, it would fail loudly because the built .so files +# aren't present there. +rm -r src/ +ls + +echo "src/ folder removed successfully!"