backend.ai-kernels/.github/workflows/build-ngc-pytorch.yml at f8de34bac4cee01a9f9052b47a98a15c6892ad79 · lablup/backend.ai-kernels · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
name: Build NGC PyTorch Docker Images

on:
  push:
    branches:
      - '**'  # Trigger on all branches
    paths:
      - 'vendor/ngc-pytorch/Dockerfile.*'

jobs:
  detect-changes:
    runs-on: ubuntu-latest
    outputs:
      dockerfiles: ${{ steps.changes.outputs.dockerfiles }}
      has_changes: ${{ steps.changes.outputs.has_changes }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Detect changed Dockerfiles
        id: changes
        run: |
          # Get changed files in vendor/ngc-pytorch
          CHANGED_FILES=$(git diff --name-only HEAD^ HEAD | grep '^vendor/ngc-pytorch/Dockerfile\.' || true)

          if [ -z "$CHANGED_FILES" ]; then
            echo "has_changes=false" >> $GITHUB_OUTPUT
            echo "dockerfiles=[]" >> $GITHUB_OUTPUT
            echo "No Dockerfile changes detected"
            exit 0
          fi

          echo "has_changes=true" >> $GITHUB_OUTPUT

          # Convert to JSON array
          DOCKERFILES_JSON=$(echo "$CHANGED_FILES" | jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "dockerfiles=$DOCKERFILES_JSON" >> $GITHUB_OUTPUT

          echo "Changed Dockerfiles:"
          echo "$CHANGED_FILES"

  build-arm64:
    needs: detect-changes
    if: needs.detect-changes.outputs.has_changes == 'true'
    runs-on: arm64-dgx-spark
    strategy:
      matrix:
        dockerfile: ${{ fromJson(needs.detect-changes.outputs.dockerfiles) }}
      fail-fast: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Extract version from Dockerfile name
        id: version
        run: |
          DOCKERFILE="${{ matrix.dockerfile }}"
          # Extract just the filename (e.g., Dockerfile.26.01-pytorch2.10-py312-cuda13.1)
          FILENAME=$(basename "$DOCKERFILE")
          # Extract version part (everything after "Dockerfile.")
          VERSION=${FILENAME#Dockerfile.}
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "filename=$FILENAME" >> $GITHUB_OUTPUT
          echo "Building version: $VERSION"

      - name: Build ARM64 Docker image
        working-directory: vendor/ngc-pytorch
        run: |
          docker buildx build \
            --platform linux/arm64 \
            --progress=plain \
            -t cr.backend.ai/testing/ngc-pytorch:${{ steps.version.outputs.version }}-arm64 \
            -f ${{ steps.version.outputs.filename }} \
            --load \
            .

      - name: Push ARM64 image (optional)
        if: success()
        working-directory: vendor/ngc-pytorch
        run: |
          # Uncomment when ready to push to registry
          # docker push cr.backend.ai/testing/ngc-pytorch:${{ steps.version.outputs.version }}-arm64
          echo "Image built: cr.backend.ai/testing/ngc-pytorch:${{ steps.version.outputs.version }}-arm64"

  build-amd64:
    needs: detect-changes
    if: needs.detect-changes.outputs.has_changes == 'true'
    runs-on: self-hosted-imagebuild-x86-rtx5060
    strategy:
      matrix:
        dockerfile: ${{ fromJson(needs.detect-changes.outputs.dockerfiles) }}
      fail-fast: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Extract version from Dockerfile name
        id: version
        run: |
          DOCKERFILE="${{ matrix.dockerfile }}"
          # Extract just the filename (e.g., Dockerfile.26.01-pytorch2.10-py312-cuda13.1)
          FILENAME=$(basename "$DOCKERFILE")
          # Extract version part (everything after "Dockerfile.")
          VERSION=${FILENAME#Dockerfile.}
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "filename=$FILENAME" >> $GITHUB_OUTPUT
          echo "Building version: $VERSION"

      - name: Build AMD64 Docker image
        working-directory: vendor/ngc-pytorch
        run: |
          docker buildx build \
            --platform linux/amd64 \
            --progress=plain \
            -t cr.backend.ai/testing/ngc-pytorch:${{ steps.version.outputs.version }}-amd64 \
            -f ${{ steps.version.outputs.filename }} \
            --load \
            .

      - name: Push AMD64 image (optional)
        if: success()
        working-directory: vendor/ngc-pytorch
        run: |
          # Uncomment when ready to push to registry
          # docker push cr.backend.ai/testing/ngc-pytorch:${{ steps.version.outputs.version }}-amd64
          echo "Image built: cr.backend.ai/testing/ngc-pytorch:${{ steps.version.outputs.version }}-amd64"