publish_release.yml

name: Upload PyPI And Release

on:
  push:
    tags:
      - "v*"

  workflow_dispatch:

env:
  PYTHON_VERSION: "3.10.9"
  PYPI_PUBLISH_URL: ${{ vars.PYPI_PUBLISH_URL || 'https://upload.pypi.org/legacy/' }}
  PYPI_DOWNLOAD_URL: ${{ vars.PYPI_DOWNLOAD_URL || 'https://pypi.org/simple/' }}

defaults:
  run:
    shell: bash

jobs:
  check-exists:
    name: Check Pypi version exists
    if: github.repository == 'LazyAGI/LazyLLM'
    runs-on: ubuntu-latest
    outputs:
      TAG: ${{ steps.set_tag_var.outputs.tag }}
      pypi_version_exists: ${{ steps.pypi_check.outputs.pypi_version_exists }}
    steps:
      - name: Extract tag from ref
        id: set_tag_var
        run: |
          FULL_TAG_NAME="${{ github.ref_name }}"
          echo "tag=${FULL_TAG_NAME#v}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v4
        with:
          submodules: false
          fetch-depth: 0

      - name: Set up python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Check PyPI for existing version
        id: pypi_check
        run: |
          TAG=${{ steps.set_tag_var.outputs.tag }}
          echo "TAG: ${TAG}"
          PACKAGE_NAME=$(grep -m 1 'name = "' pyproject.toml | cut -d'"' -f2)
          RESPONSE=$(curl -s https://test.pypi.org/pypi/$PACKAGE_NAME/json)
          if echo "$RESPONSE" | jq -e ".releases | has(\"$TAG\")"; then
            echo "Version $TAG already exists on PyPI. Skipping publish."
            echo "pypi_version_exists=true" >> $GITHUB_OUTPUT
          else
            echo "Version $TAG does not exist on PyPI. Proceeding with publish."
            echo "pypi_version_exists=false" >> $GITHUB_OUTPUT
          fi

  build-doc:
    name: Build docs
    needs: check-exists
    if: needs.check-exists.outputs.pypi_version_exists == 'false'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: false
          fetch-depth: 0

      - name: Set up python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Update version in pyproject.toml and __init__.py
        run: |
          TAG=${{ needs.check-exists.outputs.TAG }}
          echo "TAG: ${TAG}"
          sed -i "s#^version = \".*\"#version = \"$TAG\"#g" pyproject.toml
          sed -i "s#^__version__ = '.*'#__version__ = '$TAG'#" lazyllm/__init__.py

      - name: Install deps
        run: pip install -e .

      - name: Test import
        working-directory: /tmp
        run: python -c "import lazyllm"

      - name: Check requirements
        run: |
          pip install toml
          python scripts/check_requirements.py

      - name: Copy files
        run: cp pyproject.toml lazyllm/

      - name: Build doc
        run: |
          set -ex
          export PYTHONPATH=$PWD:$PYTHONPATH
          pip install -r requirements.txt
          pip install -r docs/requirements.txt
          LAZYLLM_INIT_DOC=True python docs/add_docstrings.py

      - name: Clean cache
        shell: bash
        run: |
          pip cache purge
          pip freeze | grep -v "^-e" | xargs pip uninstall -y

      - name: Tar repo with docs
        run: |
          touch repo-with-docs.tar.gz
          tar --exclude=.git --exclude=repo-with-docs.tar.gz \
            -czf repo-with-docs.tar.gz .
          ls -lh repo-with-docs.tar.gz

      - name: Upload repo artifact (with docs inserted)
        uses: actions/upload-artifact@v4
        with:
          name: repo-with-docs
          path: repo-with-docs.tar.gz
          retention-days: 7

      - name: Save wheels to cache
        uses: actions/cache/save@v3
        with:
          path: repo-with-docs.tar.gz
          key: repo-with-docs

  build-wheel-rollback-version:
    needs: build-doc
    runs-on: ubuntu-latest
    steps:
      - name: Download repo-with-docs
        uses: actions/download-artifact@v4
        with:
          name: repo-with-docs
          path: ./repo_artifact

      - name: Extract repo-with-docs
        run: |
          set -ex
          mkdir -p repo
          tar -xzf repo_artifact/repo-with-docs.tar.gz -C repo
          # copy extracted content to workspace root (overwrite)
          cp -a repo/. ./
          # show a bit for debug
          ls -la | sed -n '1,200p'

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Build default wheel
        run: |
          pip install build tomlkit
          cp pyproject.toml pyproject.toml.backup
          python scripts/generate_toml_optional_deps.py --split
          python -m build --config-setting=skbuild.wheel.cmake=false
          ls -la dist/
          mv pyproject.toml.backup pyproject.toml

      - name: Upload source_code
        uses: actions/upload-artifact@v4
        with:
          name: source_code
          path: dist/*.tar.gz

      - name: Upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: wheels-pure-python
          path: dist/*.whl
          retention-days: 7

  build-wheel-with-cpp-extension:
    name: Build wheels with native extensions (${{ matrix.os }})
    needs: build-doc
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]

    steps:
      - name: Download repo-with-docs
        uses: actions/download-artifact@v4
        with:
          name: repo-with-docs
          path: ./repo_artifact

      - name: Extract repo-with-docs
        run: |
          set -ex
          mkdir -p repo
          tar -xzf repo_artifact/repo-with-docs.tar.gz -C repo
          # copy extracted content to workspace root (overwrite)
          cp -a repo/. ./
          # show a bit for debug
          ls -la | sed -n '1,200p'

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Build wheel
        run: |
          pip install cibuildwheel tomlkit
          cp pyproject.toml pyproject.toml.backup
          python scripts/generate_toml_optional_deps.py --split
          cibuildwheel --output-dir repaired_wheel
          ls -la repaired_wheel/
          mv pyproject.toml.backup pyproject.toml
        env:
          CIBW_BUILD_VERBOSITY: 1

      - name: Upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.os }}
          path: repaired_wheel/*.whl
          retention-days: 7

  publish:
    name: publish to PyPI
    if: github.repository == 'LazyAGI/LazyLLM'
    needs: [build-wheel-with-cpp-extension, build-wheel-rollback-version]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read

    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: wheels-*
          path: ./dist
          merge-multiple: true

      - name: List downloaded files
        run: |
          echo "Files in dist directory:"
          ls -la ./dist/

      - name: Publish all wheels to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: ${{ env.PYPI_PUBLISH_URL }}
          packages_dir: dist/

  check-pypi-installation:
    name: Check pypi installation
    needs: publish
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    steps:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install LazyLLM from PyPI
        run: |
          pip install -U pip setuptools wheel
          sleep 180
          FULL_TAG_NAME="${{ github.ref_name }}"
          if [ -n "${{ vars.PYPI_DOWNLOAD_URL }}" ]; then
            pip install lazyllm==${FULL_TAG_NAME#v} \
              --index-url ${{ vars.PYPI_DOWNLOAD_URL }} \
              --no-deps
            pip install lazyllm==${FULL_TAG_NAME#v} \
              --extra-index-url https://pypi.org/simple \
              --no-build-isolation
          else
            pip install lazyllm==${FULL_TAG_NAME#v}
          fi

      - name: Verify installation
        run: |
          python -c "import lazyllm; print(lazyllm.__version__)"
          python -c "from lazyllm import lazyllm_cpp"

  check-release:
    name: check and create GitHub Release
    needs: publish
    runs-on: ubuntu-latest
    steps:
      - name: Check if GitHub Release exists
        id: release_check
        run: |
          RELEASE_TAG=${GITHUB_REF#refs/tags/}
          RELEASES=$(curl -H "Authorization: token ${{ secrets.PERSONAL_GITHUB_TOKEN }}" -s https://api.github.com/repos/${GITHUB_REPOSITORY}/releases)
          if echo "$RELEASES" | jq -e ".[] | select(.tag_name == \"$RELEASE_TAG\")" > /dev/null; then
            echo "Release $RELEASE_TAG already exists. Skipping release creation."
            echo "release_exists=true" >> $GITHUB_ENV
            UPLOAD_URL=$(echo "$RELEASES" | jq -r ".[] | select(.tag_name == \"$RELEASE_TAG\") | .upload_url")
            echo "upload_url=$UPLOAD_URL" >> $GITHUB_ENV
          else
            echo "Release $RELEASE_TAG does not exist. Proceeding with release creation."
            echo "release_exists=false" >> $GITHUB_ENV
          fi

      - name: Create GitHub Release
        if: env.release_exists == 'false'
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
        with:
          tag_name: ${{ github.ref }}
          release_name: ${{ github.ref }}
          draft: false
          prerelease: false

      - name: Set upload_url for new release
        if: env.release_exists == 'false'
        run: echo "upload_url=${{ steps.create_release.outputs.upload_url }}" >> $GITHUB_ENV

      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: source_code
          path: ./dist

      - name: Upload Release Asset (sdist)
        run: |
          FILE_PATH=$(ls dist/*)
          gh release upload "${GITHUB_REF#refs/tags/}" "$FILE_PATH" --repo "$GITHUB_REPOSITORY" --clobber
        env:
          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}