build-wheels-and-deploy.yml

name: Build wheels and deploy

on:
  workflow_dispatch:
    inputs:
      deploy:
        description: 'Where to deploy the artifacts? Only build (build), deploy to test PyPI (test)'
        required: true
        type: choice
        default: 'build'
        options:
          - build
          - test
          - prod
  schedule:
    # Weekly canary, Monday 16:00 UTC = 8 AM PST / 9 AM PDT.
    #
    # We cannot explicitly set `deploy: build` here -- GitHub Actions does not
    # allow `inputs:` on `schedule:` triggers (only on `workflow_dispatch:`).
    # On a schedule trigger, github.event.inputs.deploy is null, so the
    # publish-to-pypi job's steps (gated on inputs.deploy == 'test' or 'prod')
    # are skipped automatically. Net effect: the schedule run builds wheels
    # and stops -- it cannot publish to PyPI.
    - cron: '0 16 * * MON'

jobs:
  build-stubs:
    name: Build Python stubs
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Upgrade pip and python packages
        run: |
          python -m pip install --upgrade pip
          python -m pip install cibuildwheel==2.21.0 pybind11-stubgen==1.1
      - name: Install dependencies
        shell: bash
        run: |
            # Update & upgrade package lists
            sudo apt-get update -y
            sudo apt-get upgrade
            # Deal with Github CI limitation
            # https://github.com/actions/runner-images/issues/6399#issuecomment-1285011525
            sudo apt install -y libunwind-dev
            # Generic dependencies
            sudo apt-get install cmake libopenblas-dev
            # Install VRS dependencies
            sudo apt-get install -o Acquire::Retries=5 \
              libgtest-dev libgmock-dev \
              libfmt-dev \
              libjpeg-dev libturbojpeg-dev libpng-dev \
              liblz4-dev libzstd-dev libxxhash-dev \
              libboost-date-time-dev \
              libboost-filesystem-dev \
              libboost-iostreams-dev \
              libboost-system-dev \
              libopus-dev
            # Clean APT cache
            sudo apt-get clean

            # Install ffmpeg dependency lib
            ./build_third_party_libs/build_ffmpeg_linuxunix.sh
      - name: Install projectaria_tools
        run: |
          # Build and install Python bindings (required to run the stubs on the pybind artifacts)
          CMAKE_BUILD_PARALLEL_LEVEL=4 pip3 install .
          rm -rf build
      - name: Generate stubs & Include stubs in python package
        run: |
          python generate_stubs.py
          cp -r projectaria_tools-stubs/projectaria_tools .
      - uses: actions/upload-artifact@v4
        with:
          name: projectaria_tools_stubs
          path: projectaria_tools

  build-wheels:
    name: Build wheels ${{ matrix.os }} | ${{ matrix.python-version }}
    needs:
      - build-stubs
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [macos-14, ubuntu-latest, ubuntu-24.04-arm]  # macos-14 = macSilicon; ubuntu-24.04-arm = native Linux aarch64
        python-version: ["3.10", "3.11", "3.12"]
        include: # Customize cibuildwheel python version to build
          - python-version: "3.10"
            cibw_build: "cp310-*"
          - python-version: "3.11"
            cibw_build: "cp311-*"
          - python-version: "3.12"
            cibw_build: "cp312-*"


    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download the pre-computed python stubs
        uses: actions/download-artifact@v4
        with:
          name: projectaria_tools_stubs
          path: projectaria_tools
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Upgrade pip and python packages
        run: |
          python -m pip install --upgrade pip
          python -m pip install cibuildwheel==2.21.0
      - name: Install ffmpeg dependency lib
        if: ${{ (matrix.os == 'macos-14') || (matrix.os == 'ubuntu-latest') || (matrix.os == 'ubuntu-24.04-arm') }}
        shell: bash
        run: |
          ./build_third_party_libs/build_ffmpeg_linuxunix.sh
      - name: Build wheels for CPython
        if: matrix.os == 'macos-14'
        shell: bash
        run: |
          CMAKE_BUILD_PARALLEL_LEVEL=4 python -m cibuildwheel --output-dir dist
        env:
          CIBW_BUILD_VERBOSITY: 3
          CIBW_BUILD: ${{ matrix.cibw_build }}
          CIBW_SKIP: "*-manylinux_i686 *-musllinux_* *x86_64*"
          CIBW_ARCHS: "arm64"
          CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=14.0"
      - name: Build wheels for CPython
        if: ${{ matrix.os == 'ubuntu-latest' }}
        shell: bash
        run: |
          CMAKE_BUILD_PARALLEL_LEVEL=4 python -m cibuildwheel --output-dir dist
        env:
          CIBW_BUILD_VERBOSITY: 3
          CIBW_BUILD: ${{ matrix.cibw_build }}
          CIBW_BEFORE_BUILD_LINUX: bash build_third_party_libs/install_manylinux_deps.sh
          CIBW_ENVIRONMENT_LINUX: "PROJECTARIA_ENABLE_CUDA=1 PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig"
          CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude libcuda.so.1 --exclude libnvcuvid.so.1 -w {dest_dir} {wheel}"
          CIBW_SKIP: "*-manylinux_i686 *-musllinux_*"
          CIBW_ARCHS: "x86_64"
      - name: Build wheels for CPython
        if: ${{ matrix.os == 'ubuntu-24.04-arm' }}
        shell: bash
        run: |
          CMAKE_BUILD_PARALLEL_LEVEL=4 python -m cibuildwheel --output-dir dist
        env:
          CIBW_BUILD_VERBOSITY: 3
          CIBW_BUILD: ${{ matrix.cibw_build }}
          CIBW_BEFORE_BUILD_LINUX: bash build_third_party_libs/install_manylinux_deps.sh
          # Unlike the x86_64 leg above, NVDEC is not enabled here: NVIDIA ships
          # libnvcuvid for x86_64 only, so there is nothing for the CUDA env vars
          # or the auditwheel exclusions to act on.
          CIBW_SKIP: "*-manylinux_i686 *-musllinux_*"
          CIBW_ARCHS: "aarch64"
      - name: Build wheels for CPython
        if: matrix.os == 'windows-2022'
        shell: bash
        run: |
          python -m cibuildwheel --output-dir dist
        env:
          CIBW_BUILD_VERBOSITY: 3
          CIBW_BUILD: ${{ matrix.cibw_build }}
          CIBW_SKIP: "*-manylinux_i686 *-musllinux_* *-win32"
          CIBW_ARCHS: "AMD64"
          CIBW_PLATFORM: "windows"
      - uses: actions/upload-artifact@v4
        with:
          name: python-package-distributions-${{ matrix.os }}-${{ matrix.python-version }}
          path: dist
      - name: Test wheel
        shell: bash
        run: |
          # Install wheel
          python -m pip install dist/*.whl
          # Run Python unit tests
          export TEST_FOLDER="./data/gen1/"
          export TEST_FOLDER_GEN2="./data/gen2/"
          python -m unittest core/python/test/corePyBindTest.py
          python -m unittest core/python/test/mpsPyBindTest.py
          # CUDA-agnostic checks. Runners have no GPU, so has_cuda_support() is
          # expected to return False here; the point is that it runs without a
          # CUDA install, and the Linux wheel links no CUDA libraries (they are
          # dlopen'd from the driver at runtime, never bundled).
          python -c "import projectaria_tools as t; print('has_cuda_support:', t.has_cuda_support())"
          if [ "$RUNNER_OS" == "Linux" ]; then
            CORE_SO=$(python -c "import _core_pybinds; print(_core_pybinds.__file__)")
            echo "Checking $CORE_SO for CUDA linkage"
            if readelf -d "$CORE_SO" | grep NEEDED | grep -iE 'libcuda|libnvcuvid'; then
              echo "ERROR: wheel links CUDA libraries; it must be CUDA-agnostic" && exit 1
            fi
            echo "OK: no CUDA libraries in NEEDED"
          fi

  publish-to-pypi:
    name: Publish to Pypi
    runs-on: ubuntu-latest
    needs:
      - build-wheels
    steps:
    - name: Download wheels from previous jobs
      # by default this will download all artifacts
      uses: actions/download-artifact@v4
      with:
        pattern: python-package-distributions-*
        merge-multiple: true
        # PyPI publish action uploads everything under dist/* by default
        path: dist

    - name: Display the list of artifacts
      run: ls -R dist

    - name: Publish to Test PyPI
      if: github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test'
      uses: pypa/gh-action-pypi-publish@v1.13.0
      with:
        user: __token__
        password: ${{ secrets.TEST_PYPI_TOKEN_PROJECTARIA_TOOLS }}
        repository_url: https://test.pypi.org/legacy/
        skip_existing: true
        verbose: true
    - name: Publish to PyPI
      if: github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'prod'
      uses: pypa/gh-action-pypi-publish@v1.13.0
      with:
        user: __token__
        password: ${{ secrets.PYPI_TOKEN_PROJECTARIA_TOOLS_DEV }}
        skip_existing: true
        verbose: true