Build Primus Docker (Private) #943
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Primus-CI-TAS | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*" | |
| pull_request: | |
| env: | |
| PRIMUS_TURBO_COMMIT: a97e986f01893fec5789243400c7046d18591ba9 # chore: update aiter to fix pybind11 issue (#132) | |
| jobs: | |
| code-lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| steps: | |
| - run: echo "🎉 Begin Primus Python Lint." | |
| - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" | |
| - uses: actions/checkout@v4 | |
| - run: git config --global --add safe.directory /github/workspace | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Install Autoflake | |
| run: pip install autoflake==2.3.1 | |
| - name: Run Autoflake | |
| if: always() | |
| run: | | |
| output=$(autoflake . --remove-all-unused-imports --remove-unused-variables --expand-star-imports --ignore-init-module-imports --recursive) | |
| if [[ -n "$output" ]]; then | |
| echo "Autoflake check failed: $output" | |
| exit 1 | |
| else | |
| echo "Autoflake check success." | |
| fi | |
| - uses: isort/isort-action@v1 | |
| if: always() | |
| with: | |
| isort-version: "5.13.2" | |
| configuration: "--profile black --check-only --diff" | |
| - uses: psf/black@stable | |
| if: always() | |
| with: | |
| options: "--check --diff --color --verbose --line-length=110" | |
| docker-build: | |
| needs: [code-lint] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| steps: | |
| - run: echo "🎉 Begin Build Primus Docker Image." | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Show Environment Info | |
| run: | | |
| echo "Hostname: $(hostname)" | |
| echo "PWD: $(pwd)" | |
| echo "HOME: $HOME" | |
| echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE" | |
| echo "Runner Temp Dir: $RUNNER_TEMP" | |
| echo "Runner Tool Cache: $RUNNER_TOOL_CACHE" | |
| - name: Build Primus Docker Image | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "IMAGE_TAG=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "IMAGE_TAG=latest" >> $GITHUB_ENV | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| TAG_NAME="${{ github.ref }}" | |
| TAG="${TAG_NAME#refs/tags/}" | |
| echo "IMAGE_TAG=$TAG" >> $GITHUB_ENV | |
| else | |
| echo "IMAGE_TAG=others" >> $GITHUB_ENV | |
| fi | |
| echo "> Login to Docker Hub" | |
| docker login -u tasimage -p ${{secrets.PRIMUS_DOCKER_HUB_TOKEN}} | |
| echo "> Building Docker Image with tag: ${{ env.IMAGE_TAG }}" | |
| docker build -f $GITHUB_WORKSPACE/.github/workflows/docker/Dockerfile \ | |
| -t tasimage/primus:${{env.IMAGE_TAG}} \ | |
| --build-arg PRIMUS_TURBO_COMMIT=${PRIMUS_TURBO_COMMIT} . | |
| echo "> Docker tag image for Docker Hub" | |
| docker tag tasimage/primus:${{env.IMAGE_TAG}} docker.io/tasimage/primus:${{env.IMAGE_TAG}} | |
| echo "> Docker push to Docker Hub" | |
| docker push docker.io/tasimage/primus:${{env.IMAGE_TAG}} | |
| run-unittest: | |
| env: | |
| PRIMUS_WORKDIR: /apps/tas/0_public/primus_k8s_ci | |
| needs: [code-lint] | |
| runs-on: [tas-k8s] | |
| steps: | |
| - run: echo "🎉 Begin Primus-Turbo Checkout." | |
| - name: Checkout Repo Primus-Turbo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: AMD-AIG-AIMA/Primus-Turbo | |
| submodules: "recursive" | |
| path: Primus-Turbo | |
| ref: ${PRIMUS_TURBO_COMMIT} | |
| # ref: a97e986f01893fec5789243400c7046d18591ba9 # chore: update aiter to fix pybind11 issue (#132) | |
| - run: echo "Begin Primus-Turbo Install." | |
| - name: Install Primus-Turbo | |
| run: | | |
| mv Primus-Turbo /tmp/ | |
| echo "Primus-Turbo dir: /tmp/Primus-Turbo" | |
| git config --global --add safe.directory /tmp/Primus-Turbo | |
| cd /tmp/Primus-Turbo | |
| start_time=$(date +%s) | |
| echo "✅ [Pip install requirements] started at: $(date)" | |
| mkdir -p ./pip_cache | |
| MAX_JOBS=128 pip install --cache-dir=${PRIMUS_WORKDIR}/primus-cache --no-build-isolation --no-clean -r requirements.txt | |
| end_time=$(date +%s) | |
| elapsed=$((end_time - start_time)) | |
| echo "✅ [Pip install requirements] ended at: $(date)" | |
| echo "⏱️ [Pip install requirements] Total elapsed time: ${elapsed} seconds" | |
| start_time=$(date +%s) | |
| echo "✅ [build primus-turbo] started at: $(date)" | |
| pip3 install --no-build-isolation -e . -v | |
| end_time=$(date +%s) | |
| elapsed=$((end_time - start_time)) | |
| echo "✅ [build primus-turbo] ended at: $(date)" | |
| echo "⏱️ [build primus-turbo] Total elapsed time: ${elapsed} seconds" | |
| - run: echo "🎉 Begin Primus Unit Test." | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Show Environment Info | |
| run: | | |
| echo "Hostname: $(hostname)" | |
| echo "PWD: $(pwd)" | |
| echo "HOME: $HOME" | |
| echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE" | |
| echo "Runner Temp Dir: $RUNNER_TEMP" | |
| echo "Runner Tool Cache: $RUNNER_TOOL_CACHE" | |
| - name: Install Primus | |
| run: | | |
| pip install -r requirements.txt | |
| - name: Set UT_LOG_PATH | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "UT_LOG_PATH=${PRIMUS_WORKDIR}/ut_out/pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "UT_LOG_PATH=${PRIMUS_WORKDIR}/ut_out/latest" >> $GITHUB_ENV | |
| elif [[ "${{ github.event_name }}" == "release" ]]; then | |
| TAG_NAME="${{ github.ref }}" | |
| TAG="${TAG_NAME#refs/tags/}" | |
| echo "UT_LOG_PATH=${PRIMUS_WORKDIR}/ut_out/$TAG" >> $GITHUB_ENV | |
| else | |
| echo "UT_LOG_PATH=${PRIMUS_WORKDIR}/ut_out/others" >> $GITHUB_ENV | |
| fi | |
| - name: Run Unit Tests | |
| env: | |
| HF_TOKEN: ${{secrets.HF_TOKEN}} | |
| run: | | |
| echo "Set UT_LOG_PATH: ${{ env.UT_LOG_PATH }}" | |
| rm -rf "${{ env.UT_LOG_PATH }}" | |
| mkdir -p "${{ env.UT_LOG_PATH }}" | |
| MASTER_PORT=10009 DATA_PATH=/apps/tas/0_public/data \ | |
| python ./tests/run_unit_tests.py | |
| - name: Clean | |
| run: | | |
| rm -rf ${PRIMUS_WORKDIR}/Primus-Turbo | |
| rm -rf ${PRIMUS_WORKDIR}/Primus |