Build wheels and deploy #305
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build wheels and deploy | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| deploy: | |
| description: 'Where to deploy the artifacts? Only build (build), deploy to test PyPI (test)' | |
| required: true | |
| type: choice | |
| default: 'build' | |
| options: | |
| - build | |
| - test | |
| - prod | |
| schedule: | |
| # Weekly canary, Monday 16:00 UTC = 8 AM PST / 9 AM PDT. | |
| # | |
| # We cannot explicitly set `deploy: build` here -- GitHub Actions does not | |
| # allow `inputs:` on `schedule:` triggers (only on `workflow_dispatch:`). | |
| # On a schedule trigger, github.event.inputs.deploy is null, so the | |
| # publish-to-pypi job's steps (gated on inputs.deploy == 'test' or 'prod') | |
| # are skipped automatically. Net effect: the schedule run builds wheels | |
| # and stops -- it cannot publish to PyPI. | |
| - cron: '0 16 * * MON' | |
| jobs: | |
| build-stubs: | |
| name: Build Python stubs | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Upgrade pip and python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install cibuildwheel==2.21.0 pybind11-stubgen==1.1 | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| # Update & upgrade package lists | |
| sudo apt-get update -y | |
| sudo apt-get upgrade | |
| # Deal with Github CI limitation | |
| # https://github.com/actions/runner-images/issues/6399#issuecomment-1285011525 | |
| sudo apt install -y libunwind-dev | |
| # Generic dependencies | |
| sudo apt-get install cmake libopenblas-dev | |
| # Install VRS dependencies | |
| sudo apt-get install -o Acquire::Retries=5 \ | |
| libgtest-dev libgmock-dev \ | |
| libfmt-dev \ | |
| libjpeg-dev libturbojpeg-dev libpng-dev \ | |
| liblz4-dev libzstd-dev libxxhash-dev \ | |
| libboost-date-time-dev \ | |
| libboost-filesystem-dev \ | |
| libboost-iostreams-dev \ | |
| libboost-system-dev \ | |
| libopus-dev | |
| # Clean APT cache | |
| sudo apt-get clean | |
| # Install ffmpeg dependency lib | |
| ./build_third_party_libs/build_ffmpeg_linuxunix.sh | |
| - name: Install projectaria_tools | |
| run: | | |
| # Build and install Python bindings (required to run the stubs on the pybind artifacts) | |
| CMAKE_BUILD_PARALLEL_LEVEL=4 pip3 install . | |
| rm -rf build | |
| - name: Generate stubs & Include stubs in python package | |
| run: | | |
| python generate_stubs.py | |
| cp -r projectaria_tools-stubs/projectaria_tools . | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: projectaria_tools_stubs | |
| path: projectaria_tools | |
| build-wheels: | |
| name: Build wheels ${{ matrix.os }} | ${{ matrix.python-version }} | |
| needs: | |
| - build-stubs | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-14, ubuntu-latest, ubuntu-24.04-arm] # macos-14 = macSilicon; ubuntu-24.04-arm = native Linux aarch64 | |
| python-version: ["3.10", "3.11", "3.12"] | |
| include: # Customize cibuildwheel python version to build | |
| - python-version: "3.10" | |
| cibw_build: "cp310-*" | |
| - python-version: "3.11" | |
| cibw_build: "cp311-*" | |
| - python-version: "3.12" | |
| cibw_build: "cp312-*" | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Download the pre-computed python stubs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: projectaria_tools_stubs | |
| path: projectaria_tools | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Upgrade pip and python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install cibuildwheel==2.21.0 | |
| - name: Install ffmpeg dependency lib | |
| if: ${{ (matrix.os == 'macos-14') || (matrix.os == 'ubuntu-latest') || (matrix.os == 'ubuntu-24.04-arm') }} | |
| shell: bash | |
| run: | | |
| ./build_third_party_libs/build_ffmpeg_linuxunix.sh | |
| - name: Build wheels for CPython | |
| if: matrix.os == 'macos-14' | |
| shell: bash | |
| run: | | |
| CMAKE_BUILD_PARALLEL_LEVEL=4 python -m cibuildwheel --output-dir dist | |
| env: | |
| CIBW_BUILD_VERBOSITY: 3 | |
| CIBW_BUILD: ${{ matrix.cibw_build }} | |
| CIBW_SKIP: "*-manylinux_i686 *-musllinux_* *x86_64*" | |
| CIBW_ARCHS: "arm64" | |
| CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=14.0" | |
| - name: Build wheels for CPython | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| shell: bash | |
| run: | | |
| CMAKE_BUILD_PARALLEL_LEVEL=4 python -m cibuildwheel --output-dir dist | |
| env: | |
| CIBW_BUILD_VERBOSITY: 3 | |
| CIBW_BUILD: ${{ matrix.cibw_build }} | |
| CIBW_BEFORE_BUILD_LINUX: bash build_third_party_libs/install_manylinux_deps.sh | |
| CIBW_ENVIRONMENT_LINUX: "PROJECTARIA_ENABLE_CUDA=1 PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig" | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude libcuda.so.1 --exclude libnvcuvid.so.1 -w {dest_dir} {wheel}" | |
| CIBW_SKIP: "*-manylinux_i686 *-musllinux_*" | |
| CIBW_ARCHS: "x86_64" | |
| - name: Build wheels for CPython | |
| if: ${{ matrix.os == 'ubuntu-24.04-arm' }} | |
| shell: bash | |
| run: | | |
| CMAKE_BUILD_PARALLEL_LEVEL=4 python -m cibuildwheel --output-dir dist | |
| env: | |
| CIBW_BUILD_VERBOSITY: 3 | |
| CIBW_BUILD: ${{ matrix.cibw_build }} | |
| CIBW_BEFORE_BUILD_LINUX: bash build_third_party_libs/install_manylinux_deps.sh | |
| # Unlike the x86_64 leg above, NVDEC is not enabled here: NVIDIA ships | |
| # libnvcuvid for x86_64 only, so there is nothing for the CUDA env vars | |
| # or the auditwheel exclusions to act on. | |
| CIBW_SKIP: "*-manylinux_i686 *-musllinux_*" | |
| CIBW_ARCHS: "aarch64" | |
| - name: Build wheels for CPython | |
| if: matrix.os == 'windows-2022' | |
| shell: bash | |
| run: | | |
| python -m cibuildwheel --output-dir dist | |
| env: | |
| CIBW_BUILD_VERBOSITY: 3 | |
| CIBW_BUILD: ${{ matrix.cibw_build }} | |
| CIBW_SKIP: "*-manylinux_i686 *-musllinux_* *-win32" | |
| CIBW_ARCHS: "AMD64" | |
| CIBW_PLATFORM: "windows" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: python-package-distributions-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: dist | |
| - name: Test wheel | |
| shell: bash | |
| run: | | |
| # Install wheel | |
| python -m pip install dist/*.whl | |
| # Run Python unit tests | |
| export TEST_FOLDER="./data/gen1/" | |
| export TEST_FOLDER_GEN2="./data/gen2/" | |
| python -m unittest core/python/test/corePyBindTest.py | |
| python -m unittest core/python/test/mpsPyBindTest.py | |
| # CUDA-agnostic checks. Runners have no GPU, so has_cuda_support() is | |
| # expected to return False here; the point is that it runs without a | |
| # CUDA install, and the Linux wheel links no CUDA libraries (they are | |
| # dlopen'd from the driver at runtime, never bundled). | |
| python -c "import projectaria_tools as t; print('has_cuda_support:', t.has_cuda_support())" | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| CORE_SO=$(python -c "import _core_pybinds; print(_core_pybinds.__file__)") | |
| echo "Checking $CORE_SO for CUDA linkage" | |
| if readelf -d "$CORE_SO" | grep NEEDED | grep -iE 'libcuda|libnvcuvid'; then | |
| echo "ERROR: wheel links CUDA libraries; it must be CUDA-agnostic" && exit 1 | |
| fi | |
| echo "OK: no CUDA libraries in NEEDED" | |
| fi | |
| publish-to-pypi: | |
| name: Publish to Pypi | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-wheels | |
| steps: | |
| - name: Download wheels from previous jobs | |
| # by default this will download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: python-package-distributions-* | |
| merge-multiple: true | |
| # PyPI publish action uploads everything under dist/* by default | |
| path: dist | |
| - name: Display the list of artifacts | |
| run: ls -R dist | |
| - name: Publish to Test PyPI | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test' | |
| uses: pypa/gh-action-pypi-publish@v1.13.0 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.TEST_PYPI_TOKEN_PROJECTARIA_TOOLS }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| skip_existing: true | |
| verbose: true | |
| - name: Publish to PyPI | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'prod' | |
| uses: pypa/gh-action-pypi-publish@v1.13.0 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN_PROJECTARIA_TOOLS_DEV }} | |
| skip_existing: true | |
| verbose: true |