MLIR Distro #5735
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MLIR Distro | |
| # When dispatched with DISPATCH_NONCE set, embed the nonce in the run title so | |
| # callers (e.g. update-llvm.yml) can identify *their* dispatched run via | |
| # `gh run list --json displayTitle` instead of guessing by timestamp. | |
| run-name: ${{ inputs.DISPATCH_NONCE && format('MLIR Distro [{0}]', inputs.DISPATCH_NONCE) || 'MLIR Distro' }} | |
| on: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/mlirDistro.yml' | |
| workflow_dispatch: | |
| inputs: | |
| DEBUG_ENABLED: | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| type: boolean | |
| required: false | |
| default: false | |
| DEBUG_OS: | |
| description: 'which runner os to run the tmate action in (if the tmate action is run)' | |
| type: string | |
| default: 'windows-2022' | |
| required: false | |
| DEBUG_ARCH: | |
| description: 'which runner arch to run the tmate action in (if the tmate action is run)' | |
| type: string | |
| default: 'x86_64' | |
| required: false | |
| DEBUG_DETACHED: | |
| description: 'whether to launch tmate in detached mode (if the tmate action is run)' | |
| type: boolean | |
| required: false | |
| default: true | |
| LLVM_COMMIT: | |
| description: 'llvm commit to build' | |
| type: string | |
| required: false | |
| default: '' | |
| APPLY_PATCHES: | |
| description: 'whether to apply patches to source' | |
| type: string | |
| required: false | |
| default: 'true' | |
| DISPATCH_NONCE: | |
| description: 'Optional unique identifier echoed into the run title so callers can locate this dispatched run.' | |
| type: string | |
| required: false | |
| default: '' | |
| schedule: | |
| # At minute 0 past every 4th hour. (see https://crontab.guru) | |
| - cron: '0 */4 * * *' | |
| # this is only for debugging this same yamls | |
| # comment it out when you're not working on these yamls | |
| # pull_request: | |
| concurrency: | |
| # PR runs from the same PR cancel earlier in-flight runs (standard supersede). | |
| # workflow_dispatch and schedule serialize per-event so the second publish | |
| # waits for the first — preventing the upload race that PR #2789 worked | |
| # around with `replacesArtifacts: false`. | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.event_name }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| get_llvm_project_commit: | |
| name: Get latest LLVM commit | |
| runs-on: ubuntu-latest | |
| outputs: | |
| LLVM_PROJECT_COMMIT: ${{ steps.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} | |
| DATETIME: ${{ steps.get_llvm_project_commit.outputs.DATETIME }} | |
| steps: | |
| - name: Get llvm-project commit | |
| id: get_llvm_project_commit | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ x"${{ inputs.LLVM_COMMIT }}" == x"" ]; then | |
| sudo apt install jq | |
| # Authenticated request to avoid the unauthenticated 60/hr per-IP | |
| # rate limit, which silently produced LLVM_PROJECT_COMMIT=null and | |
| # broke the wheel build. --fail-with-body keeps the response body | |
| # on HTTP error so future failures surface the actual cause in | |
| # the CI log instead of hiding behind a silent curl. | |
| RESPONSE=$(curl --fail-with-body -sSL \ | |
| -H "Authorization: Bearer $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| https://api.github.com/repos/llvm/llvm-project/commits/main) || { | |
| echo "::error::GitHub API request failed." | |
| echo "Response body:" | |
| echo "$RESPONSE" | |
| exit 1 | |
| } | |
| LLVM_PROJECT_COMMIT=$(echo "$RESPONSE" | jq -r '.sha') | |
| else | |
| LLVM_PROJECT_COMMIT="${{ inputs.llvm_commit }}" | |
| fi | |
| if [ -z "$LLVM_PROJECT_COMMIT" ] || [ "$LLVM_PROJECT_COMMIT" = "null" ]; then | |
| echo "::error::LLVM_PROJECT_COMMIT is empty or null; refusing to produce broken wheels." | |
| exit 1 | |
| fi | |
| echo "LLVM_PROJECT_COMMIT=${LLVM_PROJECT_COMMIT}" | |
| echo "LLVM_PROJECT_COMMIT=${LLVM_PROJECT_COMMIT:0:8}" | tee -a $GITHUB_OUTPUT | |
| DATETIME=$(date +"%Y%m%d%H") | |
| echo "DATETIME=${DATETIME}" | tee -a $GITHUB_OUTPUT | |
| build: | |
| needs: get_llvm_project_commit | |
| name: ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.ENABLE_RTTI }} | |
| continue-on-error: true | |
| runs-on: ${{ matrix.OS }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - OS: ubuntu-22.04 | |
| ARCH: x86_64 | |
| ENABLE_RTTI: ON | |
| - OS: ubuntu-22.04 | |
| ARCH: aarch64 | |
| ENABLE_RTTI: ON | |
| - OS: windows-2022 | |
| ARCH: AMD64 | |
| ENABLE_RTTI: ON | |
| - OS: ubuntu-22.04 | |
| ARCH: x86_64 | |
| ENABLE_RTTI: OFF | |
| - OS: ubuntu-22.04 | |
| ARCH: aarch64 | |
| ENABLE_RTTI: OFF | |
| - OS: windows-2022 | |
| ARCH: AMD64 | |
| ENABLE_RTTI: OFF | |
| steps: | |
| - name: set ENV | |
| shell: bash | |
| run: | | |
| PIP_FIND_LINKS_URL="https://github.com/Xilinx/mlir-aie/releases/expanded_assets/mlir-distro" | |
| if [ x"${{ github.event_name }}" == x"pull_request" ]; then | |
| PIP_FIND_LINKS_URL="$PIP_FIND_LINKS_URL https://github.com/Xilinx/mlir-aie/releases/expanded_assets/dev-wheels" | |
| fi | |
| echo "PIP_FIND_LINKS=$PIP_FIND_LINKS_URL" | tee -a $GITHUB_ENV | |
| echo "ENABLE_RTTI=${{ matrix.ENABLE_RTTI }}" | tee -a $GITHUB_ENV | |
| - name: Checkout actions | |
| uses: actions/checkout@v6 | |
| with: | |
| # checkout just the actions in order to pick and choose | |
| # where the actual repo is checked out manually (see actions/setup_base) | |
| sparse-checkout: .github/actions | |
| - uses: ./.github/actions/setup_base | |
| id: setup_base | |
| with: | |
| # optional | |
| DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }} | |
| DEBUG_OS: ${{ inputs.DEBUG_OS }} | |
| DEBUG_ARCH: ${{ inputs.DEBUG_ARCH }} | |
| DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }} | |
| # required | |
| MATRIX_OS: ${{ matrix.OS }} | |
| MATRIX_ARCH: ${{ matrix.ARCH }} | |
| - uses: ./.github/actions/setup_ccache | |
| id: setup_ccache | |
| with: | |
| MATRIX_OS: ${{ matrix.OS }} | |
| MATRIX_ARCH: ${{ matrix.ARCH }} | |
| EXTRA_KEY: mlir-distro-enable-rtti-${{ matrix.ENABLE_RTTI }} | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| # default workspace is repo root but we're not build mlir-aie here, we're building llvm+mlir | |
| # and thus llvm-project needs to be adjacent to utils/mlir_wheels/setup.py | |
| - name: Shift workspace root | |
| id: workspace_root | |
| shell: bash | |
| working-directory: ${{ env.TEMP }} | |
| run: | | |
| ls "${{ steps.setup_base.outputs.WORKSPACE_ROOT }}" | |
| if [ x"${{ matrix.OS }}" == x"windows-2022" ]; then | |
| WORKSPACE_ROOT="${{ steps.setup_base.outputs.WORKSPACE_ROOT }}\utils\mlir_wheels" | |
| else | |
| WORKSPACE_ROOT="${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/utils/mlir_wheels" | |
| fi | |
| echo "WORKSPACE_ROOT=$WORKSPACE_ROOT" | tee -a $GITHUB_OUTPUT | |
| # setup llvm | |
| - name: Get LLVM | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| shell: bash | |
| run: | | |
| curl -s https://codeload.github.com/llvm/llvm-project/zip/${{ needs.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} -o llvm.zip | |
| unzip -q llvm.zip | |
| rm -rf llvm.zip | |
| LLVM_PROJECT_COMMIT=${{ needs.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} | |
| mv llvm-project-$LLVM_PROJECT_COMMIT llvm-project | |
| # build | |
| - name: cibuildwheel | |
| if: ${{ matrix.OS != 'ubuntu-22.04' || matrix.ARCH != 'aarch64' }} | |
| shell: bash | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} \ | |
| CIBW_ARCHS=${{ matrix.ARCH }} \ | |
| CMAKE_GENERATOR=Ninja \ | |
| DATETIME=${{ needs.get_llvm_project_commit.outputs.DATETIME }} \ | |
| HOST_CCACHE_DIR=${{ steps.setup_ccache.outputs.HOST_CCACHE_DIR }} \ | |
| LLVM_PROJECT_COMMIT=${{ needs.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} \ | |
| MATRIX_OS=${{ matrix.OS }} \ | |
| PARALLEL_LEVEL=2 \ | |
| cibuildwheel --output-dir wheelhouse | |
| - name: build aarch ubuntu wheel | |
| if: ${{ matrix.OS == 'ubuntu-22.04' && matrix.ARCH == 'aarch64' }} | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| shell: bash | |
| run: | | |
| export APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} | |
| ./scripts/apply_patches.sh | |
| pip install --upgrade pip | |
| CIBW_ARCHS=${{ matrix.ARCH }} \ | |
| CMAKE_GENERATOR=Ninja \ | |
| DATETIME=${{ needs.get_llvm_project_commit.outputs.DATETIME }} \ | |
| LLVM_PROJECT_COMMIT=${{ needs.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} \ | |
| MATRIX_OS=${{ matrix.OS }} \ | |
| PARALLEL_LEVEL=2 \ | |
| pip wheel . -v -w wheelhouse | |
| - name: Clean llvm-project | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| shell: bash | |
| run: | | |
| rm -rf llvm-project | |
| rm -rf build | |
| - name: Docker prune | |
| if: contains(inputs.MATRIX_OS, 'ubuntu') | |
| shell: bash | |
| run: | | |
| docker system prune -a -f | |
| - name: Get wheel version | |
| id: get_wheel_version | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pkginfo | |
| WHL=$(ls wheelhouse/mlir*whl) | |
| echo "MLIR_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT | |
| - name: Download cache from container ubuntu | |
| if: (matrix.OS == 'ubuntu-22.04' && matrix.ARCH == 'x86_64') && (success() || failure()) | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| shell: bash | |
| run: | | |
| ccache -s | |
| HOST_CCACHE_DIR="$(ccache --get-config cache_dir)" | |
| rm -rf $HOST_CCACHE_DIR | |
| mv ./wheelhouse/.ccache $HOST_CCACHE_DIR | |
| ls -la $HOST_CCACHE_DIR | |
| ccache -s | |
| # For whatever reason, the timestamps on all of the files that come out of the docker container | |
| # are some amount ahead in time (maybe 12 hours?). that means if you try to rebuild at any point | |
| # within that window ccache will not hit because the timestamps of the cached objects are ahead of | |
| # your build time. I'm not 100% sure about this explanation/mechanism but this fixed ccache misses for me. | |
| - name: Reset datetime ccache | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| shell: bash | |
| run: | | |
| ccache --print-stats | |
| HOST_CCACHE_DIR="$(ccache --get-config cache_dir)" | |
| # Set the timestamp to the beginning of the current hour. | |
| find $HOST_CCACHE_DIR -exec touch -a -m -t 201108231405.14 {} \; | |
| # The "native tools" MLIR utilities that are necessary for cross-compiling MLIR - basically just tblgen. | |
| # Now if you build a whole distro you naturally do get those utilities but it's easier to just bundle them | |
| # together here and now and make them also available as a separate, much lighter wheel, | |
| # instead of asking/expecting someone to download the entire distro just for tblgen. | |
| - name: Build native_tools wheel | |
| working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} | |
| if: matrix.ENABLE_RTTI == 'ON' | |
| shell: bash | |
| id: build_native_tools_wheel | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools | |
| for TOOL in "llvm-tblgen" "mlir-tblgen" "mlir-linalg-ods-yaml-gen" "mlir-pdll" "llvm-config" "FileCheck"; do | |
| if [ x"${{ matrix.OS }}" == x"windows-2022" ]; then | |
| TOOL="$TOOL.exe" | |
| fi | |
| unzip -j wheelhouse/mlir*whl "mlir/bin/$TOOL" -d native_tools/ | |
| done | |
| if [ x"${{ matrix.OS }}" == x"ubuntu-22.04" ]; then | |
| PLAT="linux" | |
| elif [ x"${{ matrix.OS }}" == x"windows-2022" ]; then | |
| PLAT="win" | |
| fi | |
| PLAT=${PLAT}_$(echo ${{ matrix.ARCH }} | tr '[:upper:]' '[:lower:]') | |
| pushd native_tools | |
| MLIR_WHEEL_VERSION=${{ steps.get_wheel_version.outputs.MLIR_WHEEL_VERSION }} \ | |
| python setup.py bdist_wheel --dist-dir ../wheelhouse --plat $PLAT | |
| popd | |
| # done | |
| - name: Upload wheels | |
| if: github.event_name != 'schedule' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl | |
| name: build_artifact_${{ matrix.OS }}_${{ matrix.ARCH }}_rtti_${{ matrix.ENABLE_RTTI }} | |
| smoke_test_wheels: | |
| if: github.event_name != 'schedule' | |
| name: test ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.ENABLE_RTTI }} | |
| needs: [build] | |
| runs-on: ${{ matrix.OS }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - OS: ubuntu-22.04 | |
| ARCH: x86_64 | |
| ENABLE_RTTI: ON | |
| - OS: windows-2022 | |
| ARCH: AMD64 | |
| ENABLE_RTTI: ON | |
| - OS: ubuntu-22.04 | |
| ARCH: x86_64 | |
| ENABLE_RTTI: OFF | |
| - OS: windows-2022 | |
| ARCH: AMD64 | |
| ENABLE_RTTI: OFF | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build_artifact_${{ matrix.OS }}_${{ matrix.ARCH }}_rtti_${{ matrix.ENABLE_RTTI }} | |
| path: dist | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: test | |
| shell: bash | |
| run: | | |
| # Validate wheel format | |
| for w in dist/mlir*.whl; do | |
| python -m zipfile -t "$w" | |
| done | |
| # Unpack zip file and add contents to python path | |
| for w in dist/mlir*.whl; do | |
| unzip -o -q "$w" | |
| done | |
| PYTHONPATH=$(find . -name mlir_core) python -c 'import mlir.ir' | |
| upload_distro_wheels: | |
| if: github.event_name != 'schedule' | |
| name: upload ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.ENABLE_RTTI }} | |
| needs: smoke_test_wheels | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - OS: ubuntu-22.04 | |
| ARCH: x86_64 | |
| ENABLE_RTTI: ON | |
| - OS: ubuntu-22.04 | |
| ARCH: aarch64 | |
| ENABLE_RTTI: ON | |
| - OS: windows-2022 | |
| ARCH: AMD64 | |
| ENABLE_RTTI: ON | |
| - OS: ubuntu-22.04 | |
| ARCH: x86_64 | |
| ENABLE_RTTI: OFF | |
| - OS: ubuntu-22.04 | |
| ARCH: aarch64 | |
| ENABLE_RTTI: OFF | |
| - OS: windows-2022 | |
| ARCH: AMD64 | |
| ENABLE_RTTI: OFF | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| # if `name: artifact` is omitted, the action will create extra parent dir | |
| name: build_artifact_${{ matrix.OS }}_${{ matrix.ARCH }}_rtti_${{ matrix.ENABLE_RTTI }} | |
| path: dist | |
| # Inject a PEP 427 build tag so concurrent CI runs (PR re-pushes, sibling | |
| # PRs, cron) cannot collide on a release-asset name. The build tag does | |
| # not change the wheel's PEP 440 version, so PIP_FIND_LINKS consumers | |
| # asking for `mlir==<version>` keep matching; pip uses the build tag only | |
| # to break ties among wheels of the same version (higher tag wins). | |
| # Skipped on workflow_dispatch so the canonical `mlir-distro` release | |
| # keeps deterministic filenames that `find_wheel_in_distro` looks up. | |
| - name: Inject build tag for non-canonical wheel uploads | |
| if: github.event_name != 'workflow_dispatch' | |
| shell: bash | |
| working-directory: dist | |
| run: | | |
| shopt -s nullglob | |
| wheels=( *.whl ) | |
| if [ ${#wheels[@]} -eq 0 ]; then | |
| echo "::error::No wheels found in dist/; cannot inject build tag." | |
| exit 1 | |
| fi | |
| BUILD_TAG="${{ github.run_id }}_${{ github.run_attempt }}" | |
| for w in "${wheels[@]}"; do | |
| newname=$(echo "$w" | sed -E "s/-(py3-none-[^-]+)\.whl$/-${BUILD_TAG}-\1.whl/") | |
| if [ "$w" != "$newname" ]; then mv "$w" "$newname"; fi | |
| done | |
| ls -la | |
| - name: Release current commit | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| uses: ncipollo/release-action@v1.21.0 | |
| with: | |
| artifacts: "dist/*.whl" | |
| token: "${{ secrets.GITHUB_TOKEN }}" | |
| tag: ${{ github.event_name == 'workflow_dispatch' && 'mlir-distro' || 'dev-wheels' }} | |
| name: ${{ github.event_name == 'workflow_dispatch' && 'mlir-distro' || 'dev-wheels' }} | |
| removeArtifacts: false | |
| allowUpdates: true | |
| replacesArtifacts: false | |
| omitBodyDuringUpdate: true | |
| makeLatest: false |