build_test_release_eudsl_python_extras.yml

# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
# Copyright (c) 2024.

name: "Build, test, release eudsl-python-extras"

on:
  workflow_dispatch:
  workflow_call:
    inputs:
      workflow_call:
        description: 'To distinguish workflow_call from regular push'
        type: boolean
        required: false
        default: true
      workflow_caller_run_id:
        description: ''
        type: string
        required: false
        default: ''
  pull_request:
    branches:
      - main
    paths:
      - ".github/workflows/build_test_release_eudsl_python_extras.yml"
      - "projects/eudsl-python-extras/**"
  push:
    branches:
      - main
    paths:
      - ".github/workflows/build_test_release_eudsl_python_extras.yml"
      - "projects/eudsl-python-extras/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}-build_test_release_eudsl_python_extras
  cancel-in-progress: true

jobs:

  build-eudsl-python-extras:

    runs-on: ubuntu-22.04

    name: "Build eudsl-python-extras sdist"

    defaults:
      run:
        shell: bash

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2
        with:
          submodules: false

      - name: "Install Python"
        uses: actions/setup-python@v6.0.0
        with:
          python-version: "3.12"

      - name: "Install prereqs"
        run: python -m pip install setuptools build

      - name: "Build eudsl-python-extras sdist"
        run: |
          
          SHA_SHORT="$(git rev-parse --short HEAD)"
          WHEEL_VERSION="$(date +'%Y%m%d.%H%M')+$SHA_SHORT"
          
          pushd projects/eudsl-python-extras 
          sed -i.bak "s/XXXWHEEL_VERSIONXXX/$WHEEL_VERSION/g" setup.py
          python -m build . --sdist
          popd

      - name: Upload eudsl-python-extras sdist
        uses: actions/upload-artifact@v4
        with:
          path: projects/eudsl-python-extras/dist/*.tar.gz
          name: eudsl_python_extras_sdist_artifact

  test-eudsl-python-extras:

    if: github.event_name == 'pull_request'

    needs: [build-eudsl-python-extras]

    strategy:
      fail-fast: false
      matrix:
        runs-on: [
          "ubuntu-22.04",
          "ubuntu-22.04-arm",
          "macos-14",
          "windows-2022"
        ]
        python-version: [
          "3.10", "3.11", "3.12",
          "3.13", "3.14", "3.14t"
        ]
        include: [
          {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
          {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
          {runs-on: "windows-2022", name: "windows_amd64", os: "windows"},
          {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
        ]
        exclude:
          # <frozen importlib._bootstrap>:491: Warning: Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for testing. You are advised not to use it for production.
          - runs-on: windows-2022
            python-version: "3.14"

          - runs-on: windows-2022
            python-version: "3.14t"

          - runs-on: macos-14
            python-version: "3.10"

          - runs-on: macos-14
            python-version: "3.11"

          - runs-on: macos-14
            python-version: "3.12"

          - runs-on: macos-14
            python-version: "3.13"

          - runs-on: macos-14
            python-version: "3.14"

    runs-on: ${{ matrix.runs-on }}

    name: "Test eudsl-python-extras ${{ matrix.name }} ${{ matrix.python-version }}"

    defaults:
      run:
        shell: bash

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2
        with:
          submodules: false

      - name: "Install Python"
        uses: actions/setup-python@v6.0.0
        with:
          python-version: "${{ matrix.python-version }}"

      - name: "Download mlir-python-bindings artifacts"
        uses: dawidd6/action-download-artifact@v11
        if: ${{ inputs.workflow_call }}
        with:
          name: mlir_python_bindings_${{ matrix.name }}_artifact
          path: wheelhouse
          run_id: ${{ inputs.workflow_caller_run_id }}

      - name: "Install mlir-python-bindings"
        run: python -m pip install mlir-python-bindings -f ${{ inputs.workflow_call && 'wheelhouse' || 'https://llvm.github.io/eudsl' }}

      - name: "Install development dependencies"
        run: python -m pip install pytest mlir-native-tools -f https://llvm.github.io/eudsl

      - uses: actions/download-artifact@v4
        with:
          name: eudsl_python_extras_sdist_artifact
          path: dist

      - name: "Install eudsl-python-extras"
        run: python -m pip install "eudsl-python-extras[test]" -f dist


      - name: "Test eudsl-python-extras"
        run: |
          
          IGNORE=""
          if [[ $(python -c "print(__import__('sys').version_info < (3, 13))") == "True" ]]; then
            IGNORE="--ignore projects/eudsl-python-extras/tests/test_generics.py"
          fi
          
          python -m pytest projects/eudsl-python-extras/tests $IGNORE

      - name: "Test examples"
        run: |
          
          python projects/eudsl-python-extras/examples/flash_attention.py
          python projects/eudsl-python-extras/examples/mwe.py
          python projects/eudsl-python-extras/examples/rdna_matmul_opt.py
          
          if [[ $(python -c "print(__import__('sys').version_info >= (3, 13))") == "True" ]]; then
            python projects/eudsl-python-extras/examples/cuda_matmul_opt.py
          fi

      - name: Test jupyter notebooks
        # sed: can't read C:\hostedtoolcache\windows\Python\3.12.10\x64/jupyter_client/runapp.py: No such file or directory
        if: matrix.os != 'windows'
        shell: bash
        env:
          BRANCH: ${{ github.head_ref || github.ref_name }}
        run: |
          
          pip install -q jupyter
          
          sed -i.bak 's/OUTPUT_TIMEOUT = 10/OUTPUT_TIMEOUT = 1000/g' \
            $(python -c 'import site; print(site.getsitepackages()[0])')/jupyter_client/runapp.py
          
          jupyter execute projects/eudsl-python-extras/examples/mlir_python_extras.ipynb --output=mlir_python_extras_output
          cat projects/eudsl-python-extras/examples/mlir_python_extras_output.ipynb | jq '.cells[].outputs | select(length > 0) | .[0] | .text'
          jupyter execute projects/eudsl-python-extras/examples/vectorization_e2e.ipynb --output=vectorization_e2e_output
          cat projects/eudsl-python-extras/examples/vectorization_e2e_output.ipynb | jq '.cells[].outputs | select(length > 0) | .[0] | .text'
          
          # TODO(max): build wheels with nv targets
          # if [ ${{ matrix.os }} == 'ubuntu' ]; then
          #   jupyter execute projects/eudsl-python-extras/examples/cuda_e2e.ipynb --output=cuda_e2e_output
          #   cat projects/eudsl-python-extras/examples/cuda_e2e_output.ipynb | jq '.cells[].outputs | select(length > 0) | .[0] | .text'
          # fi

      - name: "Test other hosts"
        # ImportError: DLL load failed while importing _mlir: The specified module could not be found.
        if: matrix.os != 'windows'
        run: |
          
          pip uninstall -y eudsl-python-extras
          pip install -r projects/eudsl-python-extras/tests/other_host_reqs.txt -f dist --no-cache-dir
          
          pytest -v --capture=tee-sys projects/eudsl-python-extras/tests/test_other_hosts.py

  release-eudsl-python-extras:

    if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'

    needs: [build-eudsl-python-extras]

    runs-on: "ubuntu-22.04"

    permissions:
      id-token: write
      contents: write

    name: "Release eudsl-python-extras"

    steps:

      - uses: actions/download-artifact@v4
        with:
          name: eudsl_python_extras_sdist_artifact
          path: dist

      - name: Release current commit
        uses: ncipollo/release-action@v1.12.0
        with:
          artifacts: "dist/eudsl_python_extras*.tar.gz"
          token: "${{ secrets.GITHUB_TOKEN }}"
          tag: eudsl-python-extras
          name: eudsl-python-extras
          removeArtifacts: false
          allowUpdates: true
          replacesArtifacts: true
          makeLatest: true
          omitBody: true

  call-deploy-pip-page:

    if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'

    needs: [release-eudsl-python-extras]

    permissions:
      contents: read
      id-token: write
      pages: write

    uses: ./.github/workflows/deploy_pip_page.yml
    secrets: inherit # pass all secrets