Wheels #36
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Wheels | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - .github/workflows/wheels.yml | |
| schedule: | |
| - cron: "0 8 * * *" | |
| permissions: read-all | |
| jobs: | |
| Build-Wheels: | |
| timeout-minutes: 180 | |
| runs-on: ${{ matrix.config.runs_on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - {runs_on: ['self-hosted', 'CPU'], arch: 'x86_64'} | |
| - {runs_on: 'ubuntu-22.04-arm', arch: 'aarch64'} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| # The self-hosted x86_64 runner ships a docker CLI (1.52) newer than the | |
| # daemon's max API version (1.43), so unpinned docker commands fail with | |
| # "client version 1.52 is too new". Pin at the job level so every step | |
| # (including cibuildwheel's internal docker invocations) inherits it. | |
| env: | |
| DOCKER_API_VERSION: "1.43" | |
| steps: | |
| - name: Prune stale docker containers | |
| run: | | |
| # If cibuildwheel crashes (or, say, is OOM-killed), it leaves behind a | |
| # docker container. Eventually these consume all the disk space on | |
| # this machine. | |
| docker container prune -f | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| # The LATEST_DATE here should be kept in sync with the one in Patch setup.py | |
| - id: check-version | |
| name: Check latest version | |
| run: | | |
| export PACKAGE_DATE=$(python3 -m pip install --user --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ --dry-run triton-nightly== |& grep -oP '(?<=, )[0-9\.]+dev[0-9]+(?=\))' | grep -oP '(?<=dev)[0-9]+') | |
| export LATEST_DATE=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd") | |
| if cmp -s <(echo $PACKAGE_DATE) <(echo $LATEST_DATE); then | |
| echo "new_commit=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "new_commit=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.11' | |
| - name: Patch setup.py | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' }} | |
| run: | | |
| echo "" >> python/setup.cfg | |
| echo "[build_ext]" >> python/setup.cfg | |
| echo "base-dir=/project" >> python/setup.cfg | |
| - name: Build wheels | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' }} | |
| run: | | |
| python --version | |
| # Make sure cibuildwheel is updated to latest, this will enable latest python builds | |
| python3 -m pip install cibuildwheel --upgrade --user | |
| # Pass MAX_JOBS=4 because, at time of writing, the VM "only" has 32GB | |
| # of RAM and OOMs while building if we give it the default number of | |
| # workers (2 * NUM_CPUs). | |
| export CIBW_ENVIRONMENT="MAX_JOBS=4 \ | |
| TRITON_EXT_ENABLED=ON \ | |
| TRITON_BUILD_WITH_CLANG_LLD=1" | |
| # required to build Python 3.14 with cibuildwheel 2.23.3 | |
| # todo: Need to update system Python to 3.11 and update cibuildwheel to latest | |
| # many_linux_2_28 image comes with GCC 12.2.1, but not clang. | |
| # With this install, it gets clang 16.0.6. | |
| export CIBW_BEFORE_ALL="dnf install clang lld -y" | |
| # Skip auditwheel's dependency-bundling pass on libtriton.so. Without | |
| # this, every shared library libtriton.so links against is copied into | |
| # triton.libs/ and the wheel grows by ~130MB. PyTorch's build script | |
| # uses the same flag for the same reason. | |
| export CIBW_REPAIR_WHEEL_COMMAND_LINUX="auditwheel repair --exclude libtriton.so -w {dest_dir} {wheel}" | |
| if [[ ${{ matrix.config.arch }} == 'x86_64' ]]; then | |
| export CIBW_MANYLINUX_X86_64_IMAGE="quay.io/pypa/manylinux_2_28_${{ matrix.config.arch }}:latest" | |
| else | |
| export CIBW_MANYLINUX_AARCH64_IMAGE="quay.io/pypa/manylinux_2_28_${{ matrix.config.arch }}:latest" | |
| fi | |
| export CIBW_BUILD="cp3{10,11,12,13,13t,14,14t}-manylinux_${{ matrix.config.arch }}" | |
| export CIBW_SKIP="cp{35,36,37,38,39}-*" | |
| export CIBW_ENABLE=cpython-freethreading | |
| python3 -m cibuildwheel . --output-dir wheelhouse | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: cibw-wheels-manylinux_2_28_${{ matrix.config.arch }}-wheels-upload | |
| path: ./wheelhouse/*.whl | |
| - name: Install Azure CLI | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} | |
| run: | | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| - name: Azure login | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - id: generate-token | |
| name: Generate token | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} | |
| run: | | |
| AZ_TOKEN=$(az account get-access-token --query accessToken) | |
| echo "::add-mask::$AZ_TOKEN" | |
| echo "access_token=$AZ_TOKEN" >> "$GITHUB_OUTPUT" | |
| - name: Publish wheels to Azure DevOps | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} | |
| run: | | |
| python3 -m pip install twine | |
| python3 -m twine upload -r Triton-Nightly -u TritonArtifactsSP -p ${{ steps.generate-token.outputs.access_token }} --config-file utils/nightly.pypirc --non-interactive --verbose wheelhouse/* | |
| - name: Azure Logout | |
| if: ${{ steps.check-version.outputs.new_commit == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (success() || failure()) }} | |
| run: | | |
| az logout | |
| az cache purge | |
| az account clear |