[Refactor] Publish workflow check

.github/workflows/publish_release.yml

@@ -8,28 +8,51 @@ on:
   workflow_dispatch:
 
 env:
-  POETRY_VERSION: "1.8.3"
   PYTHON_VERSION: "3.10.9"
+  PYPI_PUBLISH_URL: ${{ vars.PYPI_PUBLISH_URL || 'https://pypi.org/legacy/' }}
+  PYPI_DOWNLOAD_URL: ${{ vars.PYPI_DOWNLOAD_URL || 'https://pypi.org/simple/' }}
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   check-exists:
     name: Check Pypi version exists
-    if: github.repository == 'LazyAGI/LazyLLM'
+    if: endsWith(github.repository, '/LazyLLM')
     runs-on: ubuntu-latest
     outputs:
+      TAG: ${{ steps.set_tag_var.outputs.tag }}
       pypi_version_exists: ${{ steps.pypi_check.outputs.pypi_version_exists }}
     steps:
+      - name: Extract tag from ref
+        id: set_tag_var
+        run: |
+          FULL_TAG_NAME="${{ github.ref_name }}"
+          echo "tag=${FULL_TAG_NAME#v}" >> $GITHUB_OUTPUT
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: false
+          fetch-depth: 0
+
+      - name: Set up python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Check PyPI for existing version
         id: pypi_check
         run: |
-          VERSION=$(poetry version -s)
-          PACKAGE_NAME=$(poetry version | awk '{print $1}')
-          RESPONSE=$(curl -s https://pypi.org/pypi/$PACKAGE_NAME/json)
-          if echo "$RESPONSE" | jq -e ".releases | has(\"$VERSION\")"; then
-            echo "Version $VERSION already exists on PyPI. Skipping publish."
+          TAG=${{ steps.set_tag_var.outputs.tag }}
+          echo "TAG: ${TAG}"
+          PACKAGE_NAME=$(grep -m 1 'name = "' pyproject.toml | cut -d'"' -f2)
+          RESPONSE=$(curl -s https://test.pypi.org/pypi/$PACKAGE_NAME/json)
+          if echo "$RESPONSE" | jq -e ".releases | has(\"$TAG\")"; then
+            echo "Version $TAG already exists on PyPI. Skipping publish."
             echo "pypi_version_exists=true" >> $GITHUB_OUTPUT
           else
-            echo "Version $VERSION does not exist on PyPI. Proceeding with publish."
+            echo "Version $TAG does not exist on PyPI. Proceeding with publish."
             echo "pypi_version_exists=false" >> $GITHUB_OUTPUT
           fi
 
@@ -46,50 +69,34 @@ jobs:
 
       - name: Init specific submodule
         run: |
-          git submodule init LazyLLM-Env
-          git submodule update LazyLLM-Env
+          git submodule update --init LazyLLM-Env
 
       - name: Set up python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          version: ${{ env.POETRY_VERSION }}
-
-      - name: Extract tag from ref
-        id: extract_tag
-        run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
-
       - name: Update version in pyproject.toml and __init__.py
         run: |
-          sed -i "s/^version = \".*\"/version = \"$TAG\"/" pyproject.toml
-          sed -i "s/version='[^']*',/version='$TAG',/" setup.py
-          sed -i "s/^__version__ = '.*'/__version__ = '$TAG'/" lazyllm/__init__.py
-          echo $TAG
-        env:
-          TAG: ${{ env.TAG }}
+          TAG=${{ needs.check-exists.outputs.TAG }}
+          echo "TAG: ${TAG}"
+          sed -i "s#^version = \".*\"#version = \"$TAG\"#" pyproject.toml
+          sed -i "s#^__version__ = '.*'#__version__ = '$TAG'#" lazyllm/__init__.py
 
       - name: Install deps
-        shell: bash
         run: pip install -e .
 
       - name: Test import
         working-directory: /tmp
-        shell: bash
         run: python -c "import lazyllm"
 
       - name: Check requirements
-        shell: bash
         run: |
           pip install toml
           python scripts/check_requirements.py
 
       - name: Copy files
-        shell: bash
-        run: cp LazyLLM-Env/poetry.lock . && cp pyproject.toml lazyllm/
+        run: cp pyproject.toml lazyllm/
 
       - name: Build doc
         run: |
@@ -107,7 +114,9 @@ jobs:
 
       - name: Tar repo with docs
         run: |
-          tar --exclude='.git' -czf repo-with-docs.tar.gz .
+          touch repo-with-docs.tar.gz
+          tar --exclude=.git --exclude=repo-with-docs.tar.gz \
+            -czf repo-with-docs.tar.gz .
           ls -lh repo-with-docs.tar.gz
 
       - name: Upload repo artifact (with docs inserted)
@@ -117,14 +126,15 @@ jobs:
           path: repo-with-docs.tar.gz
           retention-days: 7
 
-  build-wheels:
-    name: Build wheels with native extensions (${{ matrix.os }}-py${{ matrix.python-version }})
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
-
+      - name: Save wheels to cache
+        uses: actions/cache/save@v3
+        with:
+          path: repo-with-docs.tar.gz
+          key: repo-with-docs
+
+  build-wheel-rollback-version:
+    needs: build-doc
+    runs-on: ubuntu-latest
     steps:
       - name: Download repo-with-docs
         uses: actions/download-artifact@v4
@@ -138,63 +148,149 @@ jobs:
           mkdir -p repo
           tar -xzf repo_artifact/repo-with-docs.tar.gz -C repo
           # copy extracted content to workspace root (overwrite)
-          rsync -a repo/ ./
+          cp -a repo/. ./
           # show a bit for debug
           ls -la | sed -n '1,200p'
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install build deps
-        run: |
-          python -m pip install -U pip build setuptools wheel
+          python-version: "3.10"
 
       - name: Build default wheel
-        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
         run: |
-          python -m build --wheel --outdir dist/ --config-setting="--universal"
+          pip install build
+          python -m build --config-setting=skbuild.wheel.cmake=false
           ls -la dist/
 
-      - name: Build C++ extension
+      - name: Upload source_code
+        uses: actions/upload-artifact@v4
+        with:
+          name: source_code
+          path: dist/*.tar.gz
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-pure-python
+          path: dist/*.whl
+          retention-days: 7
+
+  build-wheel-with-cpp-extension:
+    name: Build wheels with native extensions (${{ matrix.os }})
+    needs: build-doc
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      # - uses: actions/checkout@v4
+      #   with:
+      #     submodules: false
+
+      - name: Download repo-with-docs
+        uses: actions/download-artifact@v4
+        with:
+          name: repo-with-docs
+          path: ./repo_artifact
+
+      - name: Extract repo-with-docs
         run: |
           set -ex
-          python setup.py build_ext --inplace
+          mkdir -p repo
+          tar -xzf repo_artifact/repo-with-docs.tar.gz -C repo
+          # copy extracted content to workspace root (overwrite)
+          cp -a repo/. ./
+          # show a bit for debug
+          ls -la | sed -n '1,200p'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Build wheel
         run: |
-          python -m build --wheel --outdir dist/
-          ls -la dist/
+          pip install cibuildwheel
+          cibuildwheel --output-dir repaired_wheel
+          ls -la repaired_wheel/
+        env:
+          CIBW_BUILD_VERBOSITY: 1
 
       - name: Upload wheel artifact
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-${{ matrix.os }}-py${{ matrix.python-version }}
-          path: dist/*.whl
+          name: wheels-${{ matrix.os }}
+          path: repaired_wheel/*.whl
           retention-days: 7
 
   publish:
     name: publish to PyPI
-    if: github.repository == 'LazyAGI/LazyLLM'
-    needs: build-wheels
+    if: endsWith(github.repository, '/LazyLLM')
+    needs: [build-wheel-with-cpp-extension, build-wheel-rollback-version]
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
 
     steps:
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
+          pattern: wheels-*
           path: ./dist
-
+          merge-multiple: true
+
+      - name: List downloaded files
+        run: |
+          echo "Files in dist directory:"
+          ls -la ./dist/
+
       - name: Publish all wheels to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.6.7
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          password: ${{ secrets.LazyLLM_INDEX_PYPI_TOKEN }}
+          repository-url: ${{ env.PYPI_PUBLISH_URL }}
           packages_dir: dist/
 
+  check-pypi-installation:
+    name: Check pypi installation
+    needs: publish
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install LazyLLM from PyPI
+        run: |
+          pip install -U pip setuptools wheel
+          sleep 180
+          FULL_TAG_NAME="${{ github.ref_name }}"
+          if [ -n "${{ vars.PYPI_DOWNLOAD_URL }}" ]; then
+            pip install lazyllm==${FULL_TAG_NAME#v} \
+              --index-url ${{ vars.PYPI_DOWNLOAD_URL }} \
+              --no-deps
+            pip install lazyllm==${FULL_TAG_NAME#v} \
+              --extra-index-url https://pypi.org/simple \
+              --no-build-isolation
+          else
+            pip install lazyllm==${FULL_TAG_NAME#v}
+          fi
+
+      - name: Verify installation
+        run: |
+          python -c "import lazyllm; print(lazyllm.__version__)"
+          python -c "from lazyllm import lazyllm_cpp"
+
   check-release:
     name: check and create GitHub Release
-    if: github.repository == 'LazyAGI/LazyLLM'
     needs: publish
     runs-on: ubuntu-latest
     steps:
@@ -224,21 +320,20 @@ jobs:
           release_name: ${{ github.ref }}
           draft: false
           prerelease: false
+
       - name: Set upload_url for new release
         if: env.release_exists == 'false'
         run: echo "upload_url=${{ steps.create_release.outputs.upload_url }}" >> $GITHUB_ENV
-      - name: Get Asset name
-        run: |
-          export PKG=$(ls dist/ | grep tar)
-          set -- $PKG
-          echo "name=$1" >> $GITHUB_ENV
-      - name: Upload Release Asset (sdist) to GitHub
-        id: upload-release-asset
-        uses: actions/upload-release-asset@v1
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: source_code
+          path: ./dist
+
+      - name: Upload Release Asset (sdist)
+        run: |
+          FILE_PATH=$(ls dist/*)
+          gh release upload "${GITHUB_REF#refs/tags/}" "$FILE_PATH" --repo "$GITHUB_REPOSITORY" --clobber
         env:
           GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ env.upload_url }}
-          asset_path: dist/${{ env.name }}
-          asset_name: ${{ env.name }}
-          asset_content_type: application/zip

.gitignore

@@ -57,4 +57,5 @@ docs/zh/assets
 *.app
 
 # build directories
-build*
+build*
+lazyllm_cpp.egg-info/