mscclpp/.azure-pipelines/templates/deploy.yml at 2a44522cf6f8ddee404ef102a83226698bd1c0f1 · microsoft/mscclpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
parameters:
- name: subscription
  type: string
- name: vmssName
  type: string
- name: resourceGroup
  type: string
  default: mscclpp
# Build parameters
- name: platform
  type: string
  default: 'cuda'
- name: gpuArch
  type: string
  default: ''
- name: buildType
  type: string
  default: 'Release'
- name: buildTests
  type: string
  default: 'true'
- name: cmakeArgs
  type: string
  default: ''
- name: buildName
  type: string
  default: 'Build'
- name: buildDisplayName
  type: string
  default: 'Build'
# Deploy parameters
- name: deployArgs
  type: string
  default: ''
- name: containerName
  type: string
  default: 'mscclpp-test'

steps:
# 0. Ensure Azure CLI exists before running AzureCLI@2 tasks.
- task: Bash@3
  name: EnsureAzureCLI
  displayName: Ensure Azure CLI Installed
  inputs:
    targetType: inline
    script: |
      set -e
      if command -v az >/dev/null 2>&1; then
        az version >/dev/null
        exit 0
      fi
      curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash

# 1. Build
- task: Bash@3
  name: ${{ parameters.buildName }}
  displayName: ${{ parameters.buildDisplayName }}
  inputs:
    targetType: 'inline'
    script: |
      set -e
      sudo apt-get update -y
      sudo apt-get install cmake -y
      rm -rf build
      mkdir -p build && cd build
      BUILD_TESTS_ARG=""
      if [ "${{ parameters.buildTests }}" = "true" ]; then
        BUILD_TESTS_ARG="-DMSCCLPP_BUILD_TESTS=ON"
      fi

      GPU_ARCH_ARG=""
      if [ -n "${{ parameters.gpuArch }}" ]; then
        GPU_ARCH_ARG="-DMSCCLPP_GPU_ARCHS=${{ parameters.gpuArch }}"
      fi

      CMAKE_EXTRA_ARGS='${{ parameters.cmakeArgs }}'
      if [ "${{ parameters.platform }}" = "rocm" ]; then
        eval CXX=/opt/rocm/bin/hipcc cmake \
          -DCMAKE_BUILD_TYPE=${{ parameters.buildType }} \
          -DMSCCLPP_BYPASS_GPU_CHECK=ON \
          -DMSCCLPP_USE_ROCM=ON \
          ${BUILD_TESTS_ARG} \
          ${GPU_ARCH_ARG} \
          ${CMAKE_EXTRA_ARGS} ..
      else
        eval cmake \
          -DCMAKE_BUILD_TYPE=${{ parameters.buildType }} \
          -DMSCCLPP_BYPASS_GPU_CHECK=ON \
          -DMSCCLPP_USE_CUDA=ON \
          ${BUILD_TESTS_ARG} \
          ${GPU_ARCH_ARG} \
          ${CMAKE_EXTRA_ARGS} ..
      fi
      make -j
      cd ..
      pwd > build/BUILD_PREFIX
      echo "=== Build artifacts ==="
      ls -la build/bin/ || echo "ERROR: build/bin/ missing after build"
      du -sh build/bin/* 2>/dev/null || true
    workingDirectory: '$(System.DefaultWorkingDirectory)'

# 2. Write CMake args for pip install on remote VMs
- task: Bash@3
  name: WritePipCmakeArgs
  displayName: Write pip CMake args
  inputs:
    targetType: 'inline'
    script: |
      set -e
      PIP_CMAKE_ARGS=""
      if [ -n "${{ parameters.gpuArch }}" ]; then
        PIP_CMAKE_ARGS="-DMSCCLPP_GPU_ARCHS=${{ parameters.gpuArch }}"
      fi
      CMAKE_EXTRA_ARGS='${{ parameters.cmakeArgs }}'
      if [ -n "${CMAKE_EXTRA_ARGS}" ]; then
        PIP_CMAKE_ARGS="${PIP_CMAKE_ARGS} ${CMAKE_EXTRA_ARGS}"
      fi
      echo "${PIP_CMAKE_ARGS}" > pip_cmake_args.txt
      echo "pip CMake args: $(cat pip_cmake_args.txt)"
    workingDirectory: '$(System.DefaultWorkingDirectory)'

# 3. Download SSH key + install packages + start VMSS
- task: DownloadSecureFile@1
  name: SshKeyFile
  displayName: Download key file
  inputs:
    secureFile: mscclpp.pem

- task: Bash@3
  name: InstallPackages
  displayName: Install Packages
  inputs:
    targetType: 'inline'
    script: |
      sudo apt-get update -y
      sudo apt-get install pssh -y

- task: AzureCLI@2
  name: StartVMSS
  displayName: Start VMSS
  inputs:
    azureSubscription: ${{ parameters.subscription }}
    scriptType: bash
    scriptLocation: inlineScript
    inlineScript: |
      az vmss start --name ${{ parameters.vmssName }} --resource-group ${{ parameters.resourceGroup }}

# 4. Deploy test environment
- task: Bash@3
  name: DeployTestEnv
  displayName: Deploy Test Env
  inputs:
    targetType: filePath
    filePath: test/deploy/deploy.sh
    arguments: ${{ parameters.deployArgs }} ${{ parameters.containerName }}
    workingDirectory: '$(System.DefaultWorkingDirectory)'