[CICD] Add auto build and push CUDA Docker images to Harbor pipeline #13
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Docker Images - CUDA | |
| on: | |
| # Manual trigger with configurable options | |
| workflow_dispatch: | |
| inputs: | |
| task: | |
| description: 'Task to build' | |
| required: true | |
| type: choice | |
| options: | |
| - train | |
| - inference | |
| - all | |
| default: 'train' | |
| target: | |
| description: 'Build target stage (dev includes dev tools, release is production)' | |
| required: true | |
| type: choice | |
| options: | |
| - dev | |
| - release | |
| default: 'dev' | |
| push: | |
| description: 'Push image to registry' | |
| type: boolean | |
| default: true | |
| no_cache: | |
| description: 'Build without Docker cache' | |
| type: boolean | |
| default: false | |
| pkg_mgr: | |
| description: 'Package manager to use' | |
| required: true | |
| type: choice | |
| options: | |
| - conda | |
| - uv | |
| default: 'conda' | |
| build_all_tasks: | |
| description: 'Build all tasks (train, inference, all) - overrides task selection' | |
| type: boolean | |
| default: false | |
| # Trigger on PRs that modify docker-related files | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'docker/cuda/**' | |
| - 'docker/build.sh' | |
| - 'tools/install/**' | |
| - 'requirements/**' | |
| - '.github/workflows/build_image_cuda.yml' | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Local registry used by CI runners (see .github/configs/cuda.yml) | |
| REGISTRY: localhost:5000 | |
| # Default build versions (keep in sync with docker/build.sh) | |
| CUDA_VERSION: '12.8.1' | |
| UBUNTU_VERSION: '22.04' | |
| PYTHON_VERSION: '3.12' | |
| UV_VERSION: '0.7.2' | |
| PKG_MGR: ${{ inputs.pkg_mgr || 'conda' }} | |
| jobs: | |
| # --------------------------------------------------------------------------- | |
| # Prepare: compute build matrix and parameters based on trigger type | |
| # --------------------------------------------------------------------------- | |
| prepare: | |
| name: Prepare build matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| target: ${{ steps.params.outputs.target }} | |
| push: ${{ steps.params.outputs.push }} | |
| no_cache: ${{ steps.params.outputs.no_cache }} | |
| steps: | |
| - name: Determine build matrix | |
| id: set-matrix | |
| run: | | |
| EVENT="${{ github.event_name }}" | |
| if [ "$EVENT" = "workflow_dispatch" ] && [ "${{ inputs.build_all_tasks }}" != "true" ]; then | |
| # Manual trigger: build selected task only | |
| echo 'matrix={"task":["${{ inputs.task }}"]}' >> $GITHUB_OUTPUT | |
| else | |
| # PR or build_all_tasks=true: build all tasks | |
| echo 'matrix={"task":["train","inference","all"]}' >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set build parameters | |
| id: params | |
| run: | | |
| EVENT="${{ github.event_name }}" | |
| if [ "$EVENT" = "pull_request" ]; then | |
| # PR: always build dev images, push to local registry, use cache | |
| echo "target=dev" >> $GITHUB_OUTPUT | |
| echo "push=true" >> $GITHUB_OUTPUT | |
| echo "no_cache=false" >> $GITHUB_OUTPUT | |
| else | |
| # workflow_dispatch: use user-provided inputs | |
| echo "target=${{ inputs.target || 'dev' }}" >> $GITHUB_OUTPUT | |
| echo "push=${{ inputs.push }}" >> $GITHUB_OUTPUT | |
| echo "no_cache=${{ inputs.no_cache }}" >> $GITHUB_OUTPUT | |
| fi | |
| # --------------------------------------------------------------------------- | |
| # Build: build and push Docker images (matrix across tasks) | |
| # --------------------------------------------------------------------------- | |
| build: | |
| name: Build ${{ matrix.task }} | |
| needs: prepare | |
| runs-on: [self-hosted, Linux, X64, nvidia-0, gpus-8] | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.prepare.outputs.matrix) }} | |
| outputs: | |
| train_tag: ${{ steps.export.outputs.train_tag }} | |
| inference_tag: ${{ steps.export.outputs.inference_tag }} | |
| all_tag: ${{ steps.export.outputs.all_tag }} | |
| steps: | |
| - name: Clean workspace | |
| run: sudo rm -rf "$GITHUB_WORKSPACE"/* "$GITHUB_WORKSPACE"/.[!.]* 2>/dev/null || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| # Use docker driver to avoid pulling moby/buildkit from Docker Hub | |
| driver: docker | |
| - name: Compute build metadata | |
| id: meta | |
| run: | | |
| set -euo pipefail | |
| TASK="${{ matrix.task }}" | |
| TARGET="${{ needs.prepare.outputs.target }}" | |
| CUDA_VERSION="${{ env.CUDA_VERSION }}" | |
| PYTHON_VERSION="${{ env.PYTHON_VERSION }}" | |
| UBUNTU_VERSION="${{ env.UBUNTU_VERSION }}" | |
| CUDA_MAJOR=$(echo "$CUDA_VERSION" | cut -d. -f1) | |
| CUDA_MINOR=$(echo "$CUDA_VERSION" | cut -d. -f2) | |
| TIMESTAMP=$(date +%Y%m%d%H%M%S) | |
| # Image naming follows docker/build.sh convention: | |
| # flagscale-<task>:<target>-cu<major><minor>-py<version>[-<timestamp>] | |
| IMAGE_NAME="flagscale-${TASK}" | |
| TAG="${TARGET}-cu${CUDA_MAJOR}${CUDA_MINOR}-py${PYTHON_VERSION}" | |
| TAG_TS="${TAG}-${TIMESTAMP}" | |
| # Build tag (with timestamp, used during docker build) | |
| BUILD_TAG="${{ env.REGISTRY }}/${IMAGE_NAME}:${TAG_TS}" | |
| # Registry tag (without timestamp, used when pushing to registry) | |
| LOCAL_TAG="${{ env.REGISTRY }}/${IMAGE_NAME}:${TAG}" | |
| # Derived build arguments | |
| BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}" | |
| PYTORCH_INDEX="https://download.pytorch.org/whl/cu${CUDA_MAJOR}${CUDA_MINOR}" | |
| echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "tag=${TAG}" >> $GITHUB_OUTPUT | |
| echo "build_tag=${BUILD_TAG}" >> $GITHUB_OUTPUT | |
| echo "local_tag=${LOCAL_TAG}" >> $GITHUB_OUTPUT | |
| echo "base_image=${BASE_IMAGE}" >> $GITHUB_OUTPUT | |
| echo "pytorch_index=${PYTORCH_INDEX}" >> $GITHUB_OUTPUT | |
| # Job summary | |
| { | |
| echo "### Build: ${IMAGE_NAME}" | |
| echo "" | |
| echo "| Parameter | Value |" | |
| echo "|---|---|" | |
| echo "| Task | \`${TASK}\` |" | |
| echo "| Target | \`${TARGET}\` |" | |
| echo "| CUDA | \`${CUDA_VERSION}\` |" | |
| echo "| Python | \`${PYTHON_VERSION}\` |" | |
| echo "| Dockerfile | \`docker/cuda/Dockerfile.${TASK}\` |" | |
| echo "| Build Tag | \`${BUILD_TAG}\` |" | |
| echo "| Registry Tag | \`${LOCAL_TAG}\` |" | |
| } >> $GITHUB_STEP_SUMMARY | |
| # - name: Build Docker image | |
| # uses: docker/build-push-action@v6 | |
| # with: | |
| # context: . | |
| # file: docker/cuda/Dockerfile.${{ matrix.task }} | |
| # target: ${{ needs.prepare.outputs.target }} | |
| # load: true | |
| # tags: ${{ steps.meta.outputs.build_tag }} | |
| # build-args: | | |
| # BASE_IMAGE=${{ steps.meta.outputs.base_image }} | |
| # CUDA_VERSION=${{ env.CUDA_VERSION }} | |
| # UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} | |
| # PYTHON_VERSION=${{ env.PYTHON_VERSION }} | |
| # UV_VERSION=${{ env.UV_VERSION }} | |
| # PYTORCH_INDEX=${{ steps.meta.outputs.pytorch_index }} | |
| # PKG_MGR=${{ env.PKG_MGR }} | |
| # no-cache: ${{ needs.prepare.outputs.no_cache == 'true' }} | |
| # - name: Push Docker image | |
| # if: needs.prepare.outputs.push == 'true' | |
| # run: | | |
| # docker tag ${{ steps.meta.outputs.build_tag }} ${{ steps.meta.outputs.local_tag }} | |
| # docker push ${{ steps.meta.outputs.local_tag }} | |
| - name: Export image tag for config update | |
| id: export | |
| if: success() && needs.prepare.outputs.push == 'true' | |
| run: | | |
| TASK="${{ matrix.task }}" | |
| echo "${TASK}_tag=${{ steps.meta.outputs.local_tag }}" >> $GITHUB_OUTPUT | |
| - name: Print build result | |
| if: success() | |
| run: | | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Result:** Built successfully" >> $GITHUB_STEP_SUMMARY | |
| echo "**Pushed:** ${{ needs.prepare.outputs.push }}" >> $GITHUB_STEP_SUMMARY | |
| # --------------------------------------------------------------------------- | |
| # Summary: verify all builds completed | |
| # --------------------------------------------------------------------------- | |
| summary: | |
| name: Build summary | |
| needs: build | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Verify build results | |
| run: | | |
| if [ "${{ needs.build.result }}" != "success" ]; then | |
| echo "::error::One or more image builds failed" | |
| exit 1 | |
| fi | |
| echo "All Docker images built successfully!" | |
| # # --------------------------------------------------------------------------- | |
| # # Run CUDA tests after build succeeds | |
| # # --------------------------------------------------------------------------- | |
| # run_cuda_tests: | |
| # name: Run CUDA tests | |
| # needs: summary | |
| # if: needs.summary.result == 'success' | |
| # uses: ./.github/workflows/all_tests_cuda.yml | |
| # secrets: inherit | |
| # --------------------------------------------------------------------------- | |
| # Push validated images to Harbor after tests pass | |
| # --------------------------------------------------------------------------- | |
| push_images_to_harbor: | |
| name: Push images to Harbor | |
| # needs: run_cuda_tests | |
| needs: summary | |
| # if: needs.run_cuda_tests.result == 'success' | |
| if: needs.summary.result == 'success' | |
| uses: ./.github/workflows/push_image_harbor.yml | |
| with: | |
| platform: cuda | |
| secrets: inherit |