distribute.yml

name: PaddleNLP Distribute CI (V100)

on:
  push:
    branches: [develop, release/*]
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

env:
  PR_ID: ${{ github.event.pull_request.number }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
  work_dir: /workspace/PaddleNLP
  PADDLENLP_ROOT: /workspace/PaddleNLP
  TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-Distribut-V100
  ci_scripts: /workspace/PaddleNLP/scripts/distribute
  BRANCH: ${{ github.event.pull_request.base.ref }}
  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
  CI_name: distribute-ci
  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"

defaults:
  run:
    shell: bash

jobs:
  distribute-ci:
    name: distribute-test
    runs-on: [self-hosted, Auto-Parallel]
    steps:
      - name: Download PaddleNLP
        env:
          work_dir: ${{ github.workspace }}
        run: |
          sudo rm -rf * .[^.]*
          echo "Downloading PaddleNLP.tar.gz"
          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
          echo "Extracting PaddleNLP.tar.gz"
          tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
          # wget -q --no-proxy https://paddle-qa.bj.bcebos.com/paddlenlp/Bos.zip --no-check-certificate
          # unzip -P "$(cat ${{ github.workspace }}/../../../bos_key)" Bos.zip
          # mkdir paddlenlp && mv Bos/* ./paddlenlp/
          source ${{ github.workspace }}/../../../proxy
          cd PaddleNLP
          git config --global user.name "PaddleCI"
          git config --global user.email "paddle_ci@example.com"
          git pull
          git submodule update --init --recursive --force
          git fetch origin pull/${PR_ID}/head
          git checkout -b PR_${PR_ID} FETCH_HEAD
          git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
          git fetch upstream ${BRANCH}
          git merge ${BRANCH}
          git diff --numstat ${BRANCH} |awk '{print $NF}'
          git log --pretty=oneline -10

      - name: Check docker image and run container
        env:
          CACHE_DIR: /home/data/cfs/.cache
          FLAGS_dynamic_static_unified_comm: "True"
          python_version: "3.10"
          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
        run: |
          container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker_image=registry.baidubce.com/paddlepaddle/paddle:latest-dev-cuda11.8-cudnn8.6-trt8.5-gcc82
          nvidia-docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}:/workspace \
            -v /home/FleetX_CI:/gpt_data \
            -v /home/Llm_gpt_CI:/llm_gpt_data \
            -v /home/Llama_CI:/llama_data \
            -v /home/.cache/pip:/home/.cache/pip \
            -e BRANCH \
            -e AGILE_COMPILE_BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e work_dir \
            -e PADDLENLP_ROOT \
            -e ci_scripts \
            -e no_proxy \
            -e CI_name \
            -e paddle_whl \
            -e FLAGS_dynamic_static_unified_comm \
            -e python_version \
            -w /workspace --runtime=nvidia ${docker_image}
      
      - name: Test
        env:
          work_dir: ${{ github.workspace }}
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          ldconfig
          pip config set global.cache-dir "/home/.cache/pip"
          ln -sf $(which python${python_version}) /usr/bin/python
          python -m pip install bce-python-sdk==0.8.74
          '
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          ldconfig
          set -e
          cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
          timeout 80m bash scripts/distribute/run_ci.sh ${paddle_whl}
          '

      - name: Upload logs
        if: always()
        env:
          home_path: ${{ github.workspace }}/..
          bos_file: ${{ github.workspace }}/../bos/BosClient.py
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -c '
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos
            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
          fi
          cd /workspace/case_logs
          for FILE in /workspace/case_logs/*; do
            file=$(basename "$FILE")
            python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/distribute/${PR_ID}/${COMMIT_ID}/logs
            echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/distribute/${PR_ID}/${COMMIT_ID}/logs/$file"
          done
          tar -czf products.tar.gz ./
          python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/distribute/${PR_ID}/${COMMIT_ID}/logs
          echo "products: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/distribute/${PR_ID}/${COMMIT_ID}/logs/products.tar.gz"
          '

      - name: Terminate and delete the container
        if: always()
        run: |
          docker rm -f ${{ env.container_name }}

  upload-products:
    name: upload-products
    needs: [distribute-ci]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: wget files
        if: always()
        run: |
          wget -q --no-proxy  https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/distribute/${PR_ID}/${COMMIT_ID}/logs/products.tar.gz --no-check-certificate
          mkdir -p case_logs
          tar -xzf products.tar.gz -C case_logs

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: distribute-log
          path: |
            case_logs
          if-no-files-found: ignore