build_llvm.yml

#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
#  See https://llvm.org/LICENSE.txt for license information.
#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#  Copyright (c) 2024.

name: "Build LLVM and MLIR"

on:
  workflow_dispatch:
    inputs:
      force_debug_with_tmate:
        type: boolean
        description: 'Run the build with tmate session'
        required: false
        default: false
      debug_with_tmate:
        type: boolean
        description: 'Run the build with a tmate session ONLY in case of failure'
        required: false
        default: false
  pull_request:
    paths:
      - ".github/actions/setup_base/action.yml"
      - ".github/workflows/build_llvm.yml"
      - "scripts/cmake/*"
      - "scripts/llvm_wasm/*"
      - "scripts/build_llvm.sh"
      - "scripts/build_mlir_native_tools.sh"
      - "third_party/llvm-project"
  push:
    branches:
      - main
    paths:
      - ".github/actions/setup_base/action.yml"
      - ".github/workflows/build_llvm.yml"
      - "scripts/cmake/*"
      - "scripts/llvm_wasm/*"
      - "scripts/build_llvm.sh"
      - "scripts/build_mlir_native_tools.sh"
      - "third_party/llvm-project"

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit). The workflow name is prepended to avoid conflicts between
  # different workflows.
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}-build_llvm
  cancel-in-progress: true

jobs:

  build:

    strategy:
      fail-fast: false
      matrix:
        include:
          - name: "manylinux_x86_64"
            runs-on: "ubuntu-22.04"
            container: "quay.io/pypa/manylinux_2_28_x86_64"
            os: "almalinux"
            host-arch: x86_64
            target-arch: x86_64

          - name: "manylinux_aarch64"
            runs-on: "ubuntu-22.04-arm"
            container: "quay.io/pypa/manylinux_2_34_aarch64"
            os: "almalinux"
            host-arch: "aarch64"
            target-arch: "aarch64"

          - name: "windows_amd64"
            runs-on: "windows-2022"
            os: "windows"
            host-arch: amd64
            target-arch: amd64

          - name: "macos_arm64"
            runs-on: "macos-14"
            os: "macos"
            host-arch: arm64
            target-arch: arm64

    runs-on: ${{ matrix.runs-on }}

    name: build-llvm-mlir ${{ matrix.name }}

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: write

    env:
      # either the PR number or `branch-N` where N always increments
      cache-key: "mlir_${{ matrix.name }}_clang_${{ format('{0}-{1}', github.ref_name, github.run_number) }}"

    container:
      image: ${{ matrix.container }}
      volumes:
        - /:/ghhost
      options: --privileged --pid=host --ipc=host
      env:
        AGENT_TOOLSDIRECTORY: ${{ env.AGENT_TOOLSDIRECTORY }}
        GH_HOST_DIR: "/ghhost"

    outputs:
      WHEEL_VERSION: ${{ steps.build.outputs.WHEEL_VERSION }}

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2
        with:
          submodules: true

      - name: "Setup base"
        uses: ./.github/actions/setup_base
        id: setup_base
        with:
          cache-key: ${{ env.cache-key }}
          restore-key: "mlir_${{ matrix.name }}_clang"
          os: ${{ matrix.os }}
          host-arch: ${{ matrix.host-arch }}
          target-arch: ${{ matrix.target-arch }}
          container: ${{ matrix.container }}
          runs-on: ${{ matrix.runs-on }}

      - name: "Build LLVM and MLIR"
        id: build
        run: |
          
          export LLVM_SOURCE_DIR="$PWD/third_party/llvm-project"
          export LLVM_BUILD_DIR="$PWD/llvm-build"
          # double nested so that upload artifacts uploads a folder 
          export LLVM_INSTALL_DIR="$PWD/llvm-install/llvm-install"
          pushd $LLVM_SOURCE_DIR && LLVM_SHA_SHORT="$(git rev-parse --short HEAD)" && popd
          export WHEEL_VERSION="$(date +'%Y%m%d')+$LLVM_SHA_SHORT"
          echo "WHEEL_VERSION=$WHEEL_VERSION" >> $GITHUB_OUTPUT
          
          ccache -z
          $python3_command -m pip install -r third_party/llvm-project/mlir/python/requirements.txt
          $python3_command -m pip install setuptools wheel
          
          ###############################
          # Build MLIR/LLVM distro
          ###############################
          
          scripts/build_llvm.sh
          echo "*********************** SMOKE TEST *********************************"
          "$LLVM_INSTALL_DIR/bin/mlir-tblgen" --version
          echo "*********************** SMOKE TEST *********************************"
          ccache -s
          
          ###############################
          # Build mlir-native-tools wheel
          ###############################
          
          for TOOL in "llvm-tblgen" "mlir-tblgen" "mlir-linalg-ods-yaml-gen" "mlir-pdll" "llvm-config" "FileCheck" "mlir-opt" "mlir-translate"; do
            if [ x"${{ matrix.os }}" == x"windows" ]; then
             TOOL="$TOOL.exe"
            fi
            cp "$LLVM_INSTALL_DIR/bin/$TOOL" projects/mlir-native-tools
          done
          
          if [ x"${{ matrix.os }}" == x"almalinux" ] || [ x"${{ matrix.os }}" == x"ubuntu" ]; then
            PLAT="linux"
          elif [ x"${{ matrix.os }}" == x"macos" ]; then
            PLAT="macosx_12_0"
          elif [ x"${{ matrix.os }}" == x"windows" ]; then
            PLAT="win"
          fi
          PLAT="${PLAT}_${{ matrix.host-arch }}"
          
          pushd projects/mlir-native-tools
          $python3_command setup.py bdist_wheel --dist-dir ../../wheelhouse --plat $PLAT
          popd
          
          ###############################
          # Tar up MLIR/LLVM distro
          ###############################
          
          # if timezone causes datetime to be ahead of local datetime then ninja will go into a loop... 
          find $LLVM_INSTALL_DIR -exec touch -a -m -t 197001010000 {} \;
          
          tar -czf "mlir_${{ matrix.name }}_$WHEEL_VERSION.tar.gz" -C "$LLVM_INSTALL_DIR/.." llvm-install
          rm -rf "$LLVM_BUILD_DIR" "$LLVM_SOURCE_DIR"
          
          ###############################
          # Build mlir_wheel
          ###############################
          
          pushd projects/mlir-wheel
          cp -R $LLVM_INSTALL_DIR/* mlir_wheel
          # remove so no one is confused (the distro wheel is for arbitrary python version)
          rm -rf mlir_wheel/python_packages
          $python3_command setup.py bdist_wheel --dist-dir ../../wheelhouse --plat $PLAT
          popd

      - name: Upload LLVM distro artifacts
        uses: actions/upload-artifact@v4
        with:
          name: mlir_${{ matrix.name }}_artifact
          path: llvm-install
          if-no-files-found: warn

      - name: Upload mlir-native-tools artifact
        uses: actions/upload-artifact@v4
        with:
          name: mlir_native_tools_${{ matrix.name }}_artifact
          path: wheelhouse/mlir_native_tools*
          if-no-files-found: warn

      - name: Upload mlir-wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: mlir_wheel_${{ matrix.name }}_artifact
          path: wheelhouse/mlir_wheel*
          if-no-files-found: warn

      - name: Upload ccache log
        uses: actions/upload-artifact@v4
        with:
          name: llvm_ccache_log_${{ matrix.name }}
          path: ${{ startsWith(matrix.os, 'windows') && 'D:\a\ccache.log' || '/tmp/ccache.log' }}

      - name: Release current commit
        if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'
        uses: ncipollo/release-action@v1.12.0
        with:
          artifacts: "*.tar.gz,wheelhouse/*.whl"
          token: "${{ secrets.GITHUB_TOKEN }}"
          tag: "llvm"
          name: "llvm"
          removeArtifacts: false
          allowUpdates: true
          replacesArtifacts: true
          makeLatest: true

      - name: "Save cache"
        uses: actions/cache/save@v3
        if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main')
        with:
          path: ${{ steps.setup_base.outputs.cache-dir }}
          key: ${{ env.cache-key }}

      - name: "Setup tmate session"
        if: (failure() && inputs.debug_with_tmate) || inputs.force_debug_with_tmate
        uses: mxschmitt/action-tmate@v3.18
        with:
          limit-access-to-actor: true
          install-dependencies: ${{ startsWith(matrix.runs-on, 'macos') || startsWith(matrix.runs-on, 'windows') }}

  build-wasm:

    runs-on: ubuntu-22.04

    name: "Build llvm-mlir wasm"

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: write

    env:
      cache-key: mlir_wasm_emscripten_${{ format('{0}-{1}', github.ref_name, github.run_number) }}

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2
        with:
          submodules: true

      - name: "Setup base"
        uses: ./.github/actions/setup_base
        id: setup_base
        with:
          cache-key: ${{ env.cache-key }}
          restore-key: "mlir_wasm_emscripten"
          os: "ubuntu"
          host-arch: "x86_64"
          target-arch: "wasm32"

      - name: Build MLIR and LLVM
        run: |
          
          ###############################
          # Build host mlir-native-tools wheel
          ###############################
          
          scripts/build_mlir_native_tools.sh
          export LLVM_NATIVE_TOOL_DIR="$PWD/mlir-native-tools-install/bin"
          export LLVM_NATIVE_TOOL_DIR="$LLVM_NATIVE_TOOL_DIR"
          export LLVM_TABLEGEN="$LLVM_NATIVE_TOOL_DIR/llvm-tblgen"
          export MLIR_TABLEGEN="$LLVM_NATIVE_TOOL_DIR/mlir-tblgen"
          export MLIR_LINALG_ODS_YAML_GEN="$LLVM_NATIVE_TOOL_DIR/mlir-linalg-ods-yaml-gen"
          
          source "$EMSDK/emsdk_env.sh"
          
          export PATH="$EMSDK/ccache/git-emscripten_64bit/bin:$PATH"
          export CCACHE_COMPILERCHECK="string:$($EMSDK/upstream/bin/clang --version | head -n 1)"
          export CCACHE="$EMSDK/ccache/git-emscripten_64bit/bin/ccache"
          export "CMAKE_C_COMPILER_LAUNCHER=$CCACHE"
          export "CMAKE_CXX_COMPILER_LAUNCHER=$CCACHE"
          
          export LLVM_SOURCE_DIR="$PWD/third_party/llvm-project"
          pushd $LLVM_SOURCE_DIR && LLVM_SHA_SHORT="$(git rev-parse --short HEAD)" && popd
          export WHEEL_VERSION="$(date +'%Y%m%d')+$LLVM_SHA_SHORT"
          echo "WHEEL_VERSION=$WHEEL_VERSION" >> $GITHUB_OUTPUT
          
          ###############################
          # Build MLIR/LLVM distro
          ###############################
          
          $CCACHE -z
          pyodide build scripts/llvm_wasm -o wheelhouse --compression-level 10
          $CCACHE -s

      - name: Upload ccache log
        uses: actions/upload-artifact@v4
        with:
          name: llvm_ccache_log_mlir_wasm32
          path: '/tmp/ccache.log'

      - name: "Save cache"
        uses: actions/cache/save@v3
        if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main')
        with:
          path: ${{ steps.setup_base.outputs.cache-dir }}
          key: ${{ env.cache-key }}

      - name: Upload LLVM distro artifacts
        uses: actions/upload-artifact@v4
        with:
          name: mlir_ubuntu_wasm32_artifact
          path: llvm-install
          if-no-files-found: warn

      - name: Upload mlir-wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: mlir_wheel_ubuntu_wasm32_artifact
          path: wheelhouse/mlir_wheel*
          if-no-files-found: warn

      - name: Release current commit
        if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'
        uses: ncipollo/release-action@v1.12.0
        with:
          artifacts: "*.tar.gz,wheelhouse/*.whl"
          token: "${{ secrets.GITHUB_TOKEN }}"
          tag: "llvm"
          name: "llvm"
          removeArtifacts: false
          allowUpdates: true
          replacesArtifacts: true
          makeLatest: true

  call-build-python-bindings:

    needs: [build, build-wasm]

    permissions:
      contents: write
      id-token: write
      pages: write

    uses: ./.github/workflows/build_mlir_python_bindings_wheel.yml
    secrets: inherit # pass all secrets
    with:
      wheel_version: ${{ needs.build.outputs.WHEEL_VERSION }}
      workflow_call: true
      workflow_caller_run_id: ${{ github.run_id }}

  call-build-eudsl:

    needs: [build]

    permissions:
      contents: write
      id-token: write
      pages: write

    uses: ./.github/workflows/build_test_release_eudsl.yml
    secrets: inherit # pass all secrets
    with:
      workflow_call: true
      workflow_caller_run_id: ${{ github.run_id }}

  call-deploy-pip-page:

    if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'

    needs: [build]

    permissions:
      contents: read
      id-token: write
      pages: write

    uses: ./.github/workflows/deploy_pip_page.yml
    secrets: inherit # pass all secrets