main.yml

name: CI
on:
  workflow_dispatch:
  push:
    branches:
      - main
    tags:
      - "v*.*.*a*"
      - "v*.*.*b*"
      - "v*.*.*rc*"
    paths-ignore:
      - "**.md"
      - ".git*"
      - "docs/**"
  pull_request_target:
    branches:
      - main
    paths-ignore:
      - "**.md"
      - ".git*"
      - "docs/**"

env:
  CI_PATH: '/home/mnt/platform_ci/GitHub/${{ github.repository }}/${{ github.run_number }}'
  PYTHON_VERSION: "3.10.9"
  LAZYLLM_DEFAULT_RECENT_K: 20

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{
    github.event_name == 'pull_request_target' && github.event.pull_request.title || github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}  # Push event: Roll back to checkout target's HEAD.
          fetch-depth: 2

      - name: Get changed files
        id: changed_files
        uses: tj-actions/changed-files@v47
        with:
          files_yaml: |
            python:
              - ./**.py

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10.9'

      - name: Install dependencies
        run: |
          pip install flake8
          pip install flake8-bugbear
          pip install flake8-quotes

      - name: Run flake8
        run: |
          CHANGED_FILES="${{ steps.changed_files.outputs.python_all_changed_files }}"
          if [ -n "$CHANGED_FILES" ]; then
            echo "Changed Python files: $CHANGED_FILES"
            make lint-only-diff CHANGED_FILES="$CHANGED_FILES"
          else
            echo "No Python files changed, skipping lint"
          fi

  doc_check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Run DocCheck
        run: |
          set -e
          pip install -r requirements.txt
          pip install -r docs/requirements.txt
          pip install pytest
          pytest -v tests/doc_check
          git checkout .
          LAZYLLM_LANGUAGE=CHINESE pytest -v tests/doc_check
          git checkout .

  wait_approve:
    if: github.ref_type != 'tag'
    needs: [lint, doc_check]
    runs-on: tps_sco_nv
    environment: protected
    steps:
      - run: echo "run on branch"

  skip_approve:
    if: github.ref_type == 'tag'
    needs: [lint, doc_check]
    runs-on: tps_sco_nv
    steps:
      - run: echo "run on tag"

  clone:
    if: |
      always()
      && !contains(github.event.head_commit.message, '[skip ci]')
      && !contains(github.event.pull_request.title, '[skip ci]')
      && (needs.wait_approve.result == 'success' || needs.skip_approve.result == 'success')
    needs: [ wait_approve, skip_approve ]
    runs-on: tps_sco_nv
    outputs:
      changed_files: ${{ steps.changed_files_yaml.outputs.all_changed_files }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 2

      - name: Get changed files
        id: changed_files_yaml
        uses: tj-actions/changed-files@v47

      - name: List all changed files
        run: |
          echo "ALL_CHANGED_FILES:"
          for file in ${{ steps.changed_files_yaml.outputs.all_changed_files }}; do
            echo "$file"
          done
          echo ""

      - name: Fetch base branch for PR
        if: github.event_name == 'pull_request'
        run: |
          git fetch origin ${{ github.event.pull_request.base.ref }} --depth=1

      - name: Build doc
        run: |
          set -e
          pip install -r requirements.txt
          pip install -r docs/requirements.txt
          LAZYLLM_INIT_DOC=True python docs/add_docstrings.py

      - name: Setup CI directory
        run: |
          set -ex
          echo ${{ env.CI_PATH }}
          mkdir -p ${{ env.CI_PATH }}
          rm -rf ${{ env.CI_PATH }}/* ${{ env.CI_PATH }}/.[!.]*
          mv $GITHUB_WORKSPACE/* $GITHUB_WORKSPACE/.[!.]* ${{ env.CI_PATH }}/
          cd ${{ env.CI_PATH }}
          pip install -r tests/requirements.txt
          pip install -r tests/requirements_linux.txt

  basic_tests:
    runs-on: tps_sco_nv
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: RunTests
        uses: ./.github/actions/run_tests
        with:
          working_directory: ${{ env.CI_PATH }}
          changed_files: ${{ needs.clone.outputs.changed_files }}
          markers: "not skip_on_linux"
          tests_dir: "tests/basic_tests"
          launcher: "empty"
          nproc: 4

  advanced_tests:
    runs-on: tps_sco_nv
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    env:
      LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }}
      LAZYLLM_AIPING_API_KEY: ${{ secrets.LAZYLLM_AIPING_API_KEY }}
      LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }}
      LAZYLLM_GLM_MODEL_NAME: ${{ secrets.LAZYLLM_GLM_MODEL_NAME }}
      LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }}
      LAZYLLM_QWEN_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_MODEL_NAME }}
      LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME }}
      LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }}
      LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }}
      LAZYLLM_DOUBAO_API_KEY: ${{ secrets.LAZYLLM_DOUBAO_API_KEY }}
      LAZYLLM_DOUBAO_MODEL_NAME: ${{ secrets.LAZYLLM_DOUBAO_MODEL_NAME }}
      LAZYLLM_SILICONFLOW_API_KEY: ${{ secrets.LAZYLLM_SILICONFLOW_API_KEY }}
      LAZYLLM_SILICONFLOW_MODEL_NAME: ${{ secrets.LAZYLLM_SILICONFLOW_MODEL_NAME }}
      LAZYLLM_MINIMAX_API_KEY: ${{ secrets.LAZYLLM_MINIMAX_API_KEY }}
      LAZYLLM_MINIMAX_MODEL_NAME: ${{ secrets.LAZYLLM_MINIMAX_MODEL_NAME }}
      LAZYLLM_PPOP_API_KEY: ${{ secrets.LAZYLLM_PPOP_API_KEY }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: RunTests
        uses: ./.github/actions/run_tests
        with:
          working_directory: ${{ env.CI_PATH }}
          changed_files: ${{ needs.clone.outputs.changed_files }}
          markers: "not skip_on_linux"
          tests_dir: "tests/advanced_tests"

  engine_tests:
    runs-on: tps_sco_nv
    needs: [ clone ]
    if: |
      always()
      && needs.clone.result == 'success'
      && (contains(needs.clone.outputs.changed_files, 'lazyllm/engine/') || github.ref_type == 'tag')
    env:
      LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }}
      LAZYLLM_AIPING_API_KEY: ${{ secrets.LAZYLLM_AIPING_API_KEY }}
      LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }}
      LAZYLLM_GLM_MODEL_NAME: ${{ secrets.LAZYLLM_GLM_MODEL_NAME }}
      LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }}
      LAZYLLM_QWEN_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_MODEL_NAME }}
      LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME }}
      LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }}
      LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }}
      LAZYLLM_DOUBAO_API_KEY: ${{ secrets.LAZYLLM_DOUBAO_API_KEY }}
      LAZYLLM_DOUBAO_MODEL_NAME: ${{ secrets.LAZYLLM_DOUBAO_MODEL_NAME }}
      LAZYLLM_SILICONFLOW_API_KEY: ${{ secrets.LAZYLLM_SILICONFLOW_API_KEY }}
      LAZYLLM_SILICONFLOW_MODEL_NAME: ${{ secrets.LAZYLLM_SILICONFLOW_MODEL_NAME }}
      LAZYLLM_MINIMAX_API_KEY: ${{ secrets.LAZYLLM_MINIMAX_API_KEY }}
      LAZYLLM_MINIMAX_MODEL_NAME: ${{ secrets.LAZYLLM_MINIMAX_MODEL_NAME }}
      LAZYLLM_PPOP_API_KEY: ${{ secrets.LAZYLLM_PPOP_API_KEY }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: RunTests
        uses: ./.github/actions/run_tests
        with:
          working_directory: ${{ env.CI_PATH }}
          changed_files: ${{ needs.clone.outputs.changed_files }}
          markers: "not skip_on_linux"
          tests_dir: "tests/engine_tests"

  charge_tests:
    runs-on: tps_sco_nv
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    env:
      LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }}
      LAZYLLM_AIPING_API_KEY: ${{ secrets.LAZYLLM_AIPING_API_KEY }}
      LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }}
      LAZYLLM_GLM_MODEL_NAME: ${{ secrets.LAZYLLM_GLM_MODEL_NAME }}
      LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }}
      LAZYLLM_QWEN_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_MODEL_NAME }}
      LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME }}
      LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }}
      LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }}
      LAZYLLM_DOUBAO_API_KEY: ${{ secrets.LAZYLLM_DOUBAO_API_KEY }}
      LAZYLLM_DOUBAO_MODEL_NAME: ${{ secrets.LAZYLLM_DOUBAO_MODEL_NAME }}
      LAZYLLM_SILICONFLOW_API_KEY: ${{ secrets.LAZYLLM_SILICONFLOW_API_KEY }}
      LAZYLLM_SILICONFLOW_MODEL_NAME: ${{ secrets.LAZYLLM_SILICONFLOW_MODEL_NAME }}
      LAZYLLM_MINIMAX_API_KEY: ${{ secrets.LAZYLLM_MINIMAX_API_KEY }}
      LAZYLLM_MINIMAX_MODEL_NAME: ${{ secrets.LAZYLLM_MINIMAX_MODEL_NAME }}
      LAZYLLM_PPOP_API_KEY: ${{ secrets.LAZYLLM_PPOP_API_KEY }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4 # For run_tests afterwards.
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Load cache
        uses: ./.github/actions/load_cache
        with:
          working_directory: ${{ env.CI_PATH }}

      - name: Clone LazyLLM-Cache for Output and Cache
        run: |
          cd ${{ env.CI_PATH }}
          echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV

      - name: Set Cache Mode Based on Run Context
        run: |
          echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT"
          echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV
          if [ "$GITHUB_RUN_ATTEMPT" = "1" ] && [ "$GITHUB_EVENT_NAME" = "push" ]; then
            echo "This is the first run, and push to main branch."
            echo "LAZYLLM_CACHE_MODE=RW" >> $GITHUB_ENV
          elif [ "$GITHUB_RUN_ATTEMPT" != "1" ] && [ "$GITHUB_EVENT_NAME" = "push" ]; then
            echo "This is a re-run! (GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT), and push to main branch."
            echo "LAZYLLM_CACHE_MODE=WO" >> $GITHUB_ENV
          elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
            echo "This is a pull request from other branch to main."
            echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV
          fi
          echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR"
          echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR"
          echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE"
          echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY"
          echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE"

      - name: RunTests
        id: RunTests
        uses: ./.github/actions/run_tests
        with:
          working_directory: ${{ env.CI_PATH }}
          changed_files: ${{ needs.clone.outputs.changed_files }}
          markers: "not skip_on_linux"
          tests_dir: "tests/charge_tests"

      - name: Update LazyLLM-Cache
        if: ${{ success() && steps.RunTests.conclusion == 'success' && github.event_name == 'push' }}
        run: |
          cd ${{ env.CI_PATH }}
          cd github_cache
          ls -alh .
          if [ -n "$(git status --porcelain)" ]; then
            echo "Detected file changes, committing to main branch..."
            git add .
            git commit -m "ci: auto-commit workflow changes [skip ci]"
            git push origin main
            echo "Changes successfully committed to main branch"
          else
            echo "No file changes detected, no commit needed"
          fi

  coverage_linux:
    runs-on: ubuntu-latest
    needs: [ basic_tests, advanced_tests, charge_tests ]
    if: |
      always()
      && needs.basic_tests.result == 'success'
      && needs.advanced_tests.result == 'success'
      && needs.charge_tests.result == 'success'
    steps:
      - name: Add Coverage Report Link to Summary
        run: |
          echo "## 代码覆盖率报告" >> $GITHUB_STEP_SUMMARY
          echo "分支： ${GITHUB_REF}" >> $GITHUB_STEP_SUMMARY
          echo "提交 SHA： ${GITHUB_SHA}" >> $GITHUB_STEP_SUMMARY
          echo "您可以通过以下链接查看详细报告：" >> $GITHUB_STEP_SUMMARY
          echo "[http://10.210.0.49:8088/${GITHUB_RUN_NUMBER}/htmlcov](http://10.210.0.49:8088/${GITHUB_RUN_NUMBER}/htmlcov)" >> $GITHUB_STEP_SUMMARY

  build_and_test_package:
    runs-on: ubuntu-latest
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: false
          fetch-depth: 0

      - name: Set up python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Build
        run: |
          pip install build
          python -m build --config-setting=skbuild.wheel.cmake=false

      - name: Test installing built package
        run: pip install dist/lazyllm*.whl

      - name: Test import
        working-directory: /tmp
        run: python -c "import lazyllm"

      - name: Check requirements
        run: |
          pip install toml
          python scripts/check_requirements.py

  deploy_doc:
    runs-on: ubuntu-latest
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: false
          fetch-depth: 0

      - name: Init specific submodule
        run: |
          git submodule init Tutorial
          git submodule update Tutorial

      - name: Set up python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        run: |
          set -e
          pip install -r requirements.txt
          pip install -r docs/requirements.txt
          cp -r docs/assets docs/zh
          cp -r docs/assets docs/en
          LAZYLLM_INIT_DOC=True python docs/add_docstrings.py
          python docs/link_assets.py
          python docs/ipynb2md.py
          python docs/gen_mkdocs_yaml.py

      - name: Start MkDocs server
        run: mkdocs serve -a localhost:1369 > mkdocs.log 2>&1 &

      - name: Wait for server to start
        run: sleep 100

      - name: Check if MkDocs server is running
        run: |
          if ! curl --silent --fail http://localhost:1369; then
            echo "MkDocs server failed to start. Here's the log:"
            cat mkdocs.log
            exit 1
          fi

  mac_basic_tests:
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    runs-on: macos-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: false

      - name: Setup
        uses: ./.github/actions/setup
        with:
          tests_dir: "tests/basic_tests"

      - name: Download model
        run: |
          pip install modelscope
          mkdir -p /Users/runner/.lazyllm
          mkdir -p /Users/runner/.cache/modelscope/hub
          ln -s /Users/runner/.cache/modelscope/hub /Users/runner/.lazyllm/model
          modelscope download qwen/qwen2-1.5b &

      - name: RunTests
        run: |
          git clone https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git /tmp/lazyllm/data
          brew install ffmpeg
          pip install -r tests/requirements.txt
          pip install -r tests/requirements_mac.txt
          export LAZYLLM_DATA_PATH=/tmp/lazyllm/data
          pytest --lf --last-failed-no-failures=all -m "not skip_on_mac" -v \
            --override-ini=cache_dir=tests/basic_tests/.pytest_cache --reruns=2 tests/basic_tests/
        timeout-minutes: 30
        env:
          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}

      - name: Save pytest cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: tests/basic_tests/.pytest_cache
          key: ${{ github.job }}-${{ github.event_name == 'pull_request_target' &&
            github.event.pull_request.head.sha || github.sha }}

  mac_charge_tests:
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    runs-on: macos-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: false

      - name: Setup
        uses: ./.github/actions/setup
        with:
          tests_dir: "tests/charge_tests"

      - name: Load cache
        uses: ./.github/actions/load_cache

      - name: Create Output and Cache Directories
        run: |
          echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV
      
      - name: Set Cache Mode Based on Run Context
        run: |
          echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT"
          echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV
          echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR"
          echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR"
          echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE"
          echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY"
          echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE"

      - name: Run tests
        run: |
          git clone https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git /tmp/lazyllm/data
          pip install -r tests/requirements.txt
          export LAZYLLM_DATA_PATH=/tmp/lazyllm/data
          if [ -f tests/charge_tests/.pytest_cache/v/cache/lastfailed ]; then
            pytest --lf --last-failed-no-failures=none \
              -m "not skip_on_mac" --order-scope=class -v --reruns=2 \
              --override-ini=cache_dir=tests/charge_tests/.pytest_cache \
              tests/charge_tests
          else
            pytest \
              -m "not skip_on_mac" --order-scope=class -v --reruns=2 \
              --override-ini=cache_dir=tests/charge_tests/.pytest_cache \
              tests/charge_tests
          fi
        env:
          CHANGED_FILES: ${{ needs.clone.outputs.changed_files }}
          LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }}
          LAZYLLM_AIPING_API_KEY: ${{ secrets.LAZYLLM_AIPING_API_KEY }}
          LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }}
          LAZYLLM_GLM_MODEL_NAME: ${{ secrets.LAZYLLM_GLM_MODEL_NAME }}
          LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }}
          LAZYLLM_QWEN_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_MODEL_NAME }}
          LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME }}
          LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }}
          LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }}
          LAZYLLM_DOUBAO_API_KEY: ${{ secrets.LAZYLLM_DOUBAO_API_KEY }}
          LAZYLLM_DOUBAO_MODEL_NAME: ${{ secrets.LAZYLLM_DOUBAO_MODEL_NAME }}
          LAZYLLM_SILICONFLOW_API_KEY: ${{ secrets.LAZYLLM_SILICONFLOW_API_KEY }}
          LAZYLLM_SILICONFLOW_MODEL_NAME: ${{ secrets.LAZYLLM_SILICONFLOW_MODEL_NAME }}
          LAZYLLM_MINIMAX_API_KEY: ${{ secrets.LAZYLLM_MINIMAX_API_KEY }}
          LAZYLLM_MINIMAX_MODEL_NAME: ${{ secrets.LAZYLLM_MINIMAX_MODEL_NAME }}
          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
        timeout-minutes: 25

      - name: Save pytest cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: tests/charge_tests/.pytest_cache
          key: ${{ github.job }}-${{ github.event_name == 'pull_request_target' &&
            github.event.pull_request.head.sha || github.sha }}

  win_basic_tests:
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    runs-on: windows-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: false

      - name: Setup
        uses: ./.github/actions/setup
        with:
          tests_dir: "tests/basic_tests"

      - name: Download model
        shell: bash
        run: |
          pip install modelscope pyyaml
          mkdir -p C:/Users/runneradmin/.lazyllm
          ls C:/Users/runneradmin/.lazyllm
          mkdir -p C:/Users/runneradmin/.cache/modelscope/hub
          ln -s  C:/Users/runneradmin/.cache/modelscope/hub C:/Users/runneradmin/.lazyllm/model
          modelscope download qwen/qwen2-1.5b &

      - name: Run tests
        shell: bash
        run: |
          git clone https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git D:/a/LazyLLM/data
          powershell -Command "Set-ExecutionPolicy RemoteSigned -Scope CurrentUser -Force; irm get.scoop.sh | iex"
          export PATH="$HOME/scoop/shims:$PATH"
          scoop install ffmpeg
          pip install -r tests/requirements.txt
          export LAZYLLM_DATA_PATH=D:/a/LazyLLM/data
          pytest --lf --last-failed-no-failures=all -m "not skip_on_win" -v --reruns=2 \
            --override-ini=cache_dir=tests/basic_tests/.pytest_cache tests/basic_tests
        timeout-minutes: 30
        env:
          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}

      - name: Save pytest cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: tests/basic_tests/.pytest_cache
          key: ${{ github.job }}-${{ github.event_name == 'pull_request_target' &&
            github.event.pull_request.head.sha || github.sha }}

  win_charge_tests:
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    runs-on: windows-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          submodules: false

      - name: Setup
        uses: ./.github/actions/setup
        with:
          tests_dir: "tests/charge_tests"

      - name: Load cache
        uses: ./.github/actions/load_cache

      - name: Create Output and Cache Directories
        shell: bash
        run: |
          echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV
      
      - name: Set Cache Mode Based on Run Context
        shell: bash
        run: |
          echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT"
          echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV
          echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV
          echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR"
          echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR"
          echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE"
          echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY"
          echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE"

      - name: charge_tests
        shell: bash
        run: |
         git clone https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git D:/a/LazyLLM/data
         pip install -r tests/requirements.txt
         export LAZYLLM_DATA_PATH=D:/a/LazyLLM/data
         if [ -f tests/charge_tests/.pytest_cache/v/cache/lastfailed ]; then
            pytest --lf --last-failed-no-failures=none -m "not skip_on_win" --order-scope=class -v \
              --reruns=2 --override-ini=cache_dir=tests/charge_tests/.pytest_cache tests/charge_tests
         else
            pytest -m "not skip_on_win" --order-scope=class -v --reruns=2 \
              --override-ini=cache_dir=tests/charge_tests/.pytest_cache tests/charge_tests
         fi
        env:
          CHANGED_FILES: ${{ needs.clone.outputs.changed_files }}
          LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }}
          LAZYLLM_AIPING_API_KEY: ${{ secrets.LAZYLLM_AIPING_API_KEY }}
          LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }}
          LAZYLLM_GLM_MODEL_NAME: ${{ secrets.LAZYLLM_GLM_MODEL_NAME }}
          LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }}
          LAZYLLM_QWEN_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_MODEL_NAME }}
          LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME }}
          LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }}
          LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }}
          LAZYLLM_DOUBAO_API_KEY: ${{ secrets.LAZYLLM_DOUBAO_API_KEY }}
          LAZYLLM_DOUBAO_MODEL_NAME: ${{ secrets.LAZYLLM_DOUBAO_MODEL_NAME }}
          LAZYLLM_SILICONFLOW_API_KEY: ${{ secrets.LAZYLLM_SILICONFLOW_API_KEY }}
          LAZYLLM_SILICONFLOW_MODEL_NAME: ${{ secrets.LAZYLLM_SILICONFLOW_MODEL_NAME }}
          LAZYLLM_MINIMAX_API_KEY: ${{ secrets.LAZYLLM_MINIMAX_API_KEY }}
          LAZYLLM_MINIMAX_MODEL_NAME: ${{ secrets.LAZYLLM_MINIMAX_MODEL_NAME }}
          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
        timeout-minutes: 30

      - name: Save pytest cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: tests/charge_tests/.pytest_cache
          key: ${{ github.job }}-${{ github.event_name == 'pull_request_target' &&
            github.event.pull_request.head.sha || github.sha }}

  k8s_tests:
    runs-on: tps_k8s
    needs: [ clone ]
    if: always() && needs.clone.result == 'success'
    env:
      K8S_CI_PATH: '/mnt/nfs_share/GitHub/${{ github.repository }}/${GITHUB_RUN_NUMBER}'
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Setup CI directory
        run: |
          set -ex
          echo ${{ env.K8S_CI_PATH }}
          mkdir -p ${{ env.K8S_CI_PATH }}
          if [ -d "${{ env.K8S_CI_PATH }}" ]; then
            rm -rf ${{ env.K8S_CI_PATH }}/*
          fi
          mv $GITHUB_WORKSPACE/* ${{ env.K8S_CI_PATH }}/

      - name: k8s_test
        run: |
          cd ${{ env.K8S_CI_PATH }}
          pip install -r tests/requirements.txt --user
          export PYTHONPATH=$PWD:$PYTHONPATH
          export LAZYLLM_DEFAULT_LAUNCHER="k8s"
          export LAZYLLM_DATA_PATH="/mnt/lustre/share_data/lazyllm/data"
          export LAZYLLM_K8S_ENV_NAME="lazyllm"
          export LAZYLLM_K8S_CONFIG_PATH="/mnt/nfs_share/k8s_config.yaml"
          export LAZYLLM_HOME="${{ env.K8S_CI_PATH }}/${{ github.run_id }}-${{ github.job }}"
          mkdir -p $LAZYLLM_HOME
          source /mnt/nfs_share/env.sh 
          pytest --lf --last-failed-no-failures=all --durations=0 --reruns=2 -v tests/k8s_tests

  cpp_ext_test:
    name: C++ Extension Test (${{ matrix.os }})
    needs: [ clone ]
    if:  always()
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
            submodules: false

      - name: Set up python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Test
        run: |
          pip install pybind11
          bash csrc/scripts/build_test.sh