trainer/.github/workflows/build-and-push-images.yaml at 1d6ac04576678e0dbf1189618a7e27ea29e0e265 · kubeflow/trainer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
name: Build and Publish Images

on:
  push:
    branches:
      - master
      - "release-*"
    tags:
      - "v*"
  pull_request:

jobs:
  build-and-publish:
    name: Build and Publish Images
    if: github.repository == 'kubeflow/trainer'
    runs-on: oracle-vm-16cpu-64gb-x86-64

    env:
      SHOULD_PUBLISH: ${{ github.event_name == 'push' }}

    strategy:
      fail-fast: false
      matrix:
        include:
          - component-name: trainer-controller-manager
            dockerfile: cmd/trainer-controller-manager/Dockerfile
            platforms: linux/amd64,linux/arm64,linux/ppc64le
          - component-name: model-initializer
            dockerfile: cmd/initializers/model/Dockerfile
            platforms: linux/amd64,linux/arm64
          - component-name: dataset-initializer
            dockerfile: cmd/initializers/dataset/Dockerfile
            platforms: linux/amd64,linux/arm64
          - component-name: deepspeed-runtime
            dockerfile: cmd/runtimes/deepspeed/Dockerfile
            platforms: linux/amd64,linux/arm64
          - component-name: xgboost-runtime
            dockerfile: cmd/runtimes/xgboost/Dockerfile
            platforms: linux/amd64,linux/arm64
          # TODO (andreyvelich): mlx[cuda] doesn't support arm at the moment: https://github.com/ml-explore/mlx/issues/2469
          - component-name: mlx-runtime
            dockerfile: cmd/runtimes/mlx/Dockerfile
            platforms: linux/amd64
          - component-name: torchtune-trainer
            dockerfile: cmd/trainers/torchtune/Dockerfile
            platforms: linux/amd64,linux/arm64
          - component-name: data-cache
            dockerfile: cmd/data_cache/Dockerfile
            platforms: linux/amd64,linux/arm64
    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: GHCR Login
        if: env.SHOULD_PUBLISH == 'true'
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Docker Hub Login
        if: env.SHOULD_PUBLISH == 'true'
        uses: docker/login-action@v4
        with:
          registry: docker.io
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Publish Component ${{ matrix.component-name }}
        if: env.SHOULD_PUBLISH == 'true'
        id: publish
        uses: ./.github/workflows/template-publish-image
        with:
          image: |
            ghcr.io/kubeflow/trainer/${{ matrix.component-name }}
            docker.io/kubeflow/${{ matrix.component-name }}
          dockerfile: ${{ matrix.dockerfile }}
          platforms: ${{ matrix.platforms }}
          context: ${{ matrix.context }}
          push: true

      - name: Test Build For Component ${{ matrix.component-name }}
        if: env.SHOULD_PUBLISH != 'true'
        uses: ./.github/workflows/template-publish-image
        with:
          image: |
            ghcr.io/kubeflow/trainer/${{ matrix.component-name }}
            docker.io/kubeflow/${{ matrix.component-name }}
          dockerfile: ${{ matrix.dockerfile }}
          platforms: ${{ matrix.platforms }}
          context: ${{ matrix.context }}
          push: false