Skip to content

Commit ddb1a42

Browse files
chtruong814chtruong814
authored
ci: Enable AWS runners (#557)
Signed-off-by: Charlie Truong <chtruong@nvidia.com>
1 parent 6c8f548 commit ddb1a42

7 files changed

Lines changed: 81 additions & 191 deletions

File tree

‎.github/actions/test-template/action.yml‎

Lines changed: 13 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ inputs:
5353
inference-framework:
5454
description: "Inference Framework"
5555
required: true
56+
test-data-path:
57+
description: "Test data path"
58+
required: true
59+
runner:
60+
description: "The GHA runner to use"
61+
required: true
5662

5763
runs:
5864
using: "composite"
@@ -73,73 +79,19 @@ runs:
7379
tenant-id: ${{ inputs.azure-tenant-id }}
7480
subscription-id: ${{ inputs.azure-subscription-id }}
7581

76-
- name: Azure ACR Login
77-
if: ${{ inputs.has-azure-credentials == 'true' }}
78-
shell: bash
82+
- name: Install uuidgen
83+
shell: bash -x -e -u -o pipefail {0}
84+
if: ${{ contains(inputs.runner, 'aws') }}
7985
run: |
80-
echo ::group::Azure ACR Login
81-
az acr login --name nemoci
82-
echo ::endgroup::
83-
84-
# - name: Azure Fileshare
85-
# if: ${{ inputs.has-azure-credentials == 'true' && inputs.is_unit_test == 'false' }}
86-
# shell: bash
87-
# id: azure-fileshare
88-
# run: |
89-
# sudo apt update
90-
# sudo apt install -y cifs-utils
91-
92-
# RESOURCE_GROUP_NAME="azure-gpu-vm-runner_group"
93-
# STORAGE_ACCOUNT_NAME="nemocistorageaccount2"
94-
# FILE_SHARE_NAME="fileshare"
95-
96-
# MNT_ROOT="/media"
97-
# MNT_PATH="$MNT_ROOT/$STORAGE_ACCOUNT_NAME/$FILE_SHARE_NAME"
98-
99-
# echo "MNT_PATH=$MNT_PATH" | tee -a "$GITHUB_OUTPUT"
100-
101-
# sudo mkdir -p $MNT_PATH
102-
103-
# # Create a folder to store the credentials for this storage account and
104-
# # any other that you might set up.
105-
# CREDENTIAL_ROOT="/etc/smbcredentials"
106-
# sudo mkdir -p "/etc/smbcredentials"
107-
108-
# # Get the storage account key for the indicated storage account.
109-
# # You must be logged in with az login and your user identity must have
110-
# # permissions to list the storage account keys for this command to work.
111-
# STORAGE_ACCOUNT_KEY=$(az storage account keys list \
112-
# --resource-group $RESOURCE_GROUP_NAME \
113-
# --account-name $STORAGE_ACCOUNT_NAME \
114-
# --query "[0].value" --output tsv | tr -d '"')
115-
116-
# # Create the credential file for this individual storage account
117-
# SMB_CREDENTIAL_FILE="$CREDENTIAL_ROOT/$STORAGE_ACCOUNT_NAME.cred"
118-
# if [ ! -f $SMB_CREDENTIAL_FILE ]; then
119-
# echo "username=$STORAGE_ACCOUNT_NAME" | sudo tee $SMB_CREDENTIAL_FILE > /dev/null
120-
# echo "password=$STORAGE_ACCOUNT_KEY" | sudo tee -a $SMB_CREDENTIAL_FILE > /dev/null
121-
# else
122-
# echo "The credential file $SMB_CREDENTIAL_FILE already exists, and was not modified."
123-
# fi
124-
125-
# # Change permissions on the credential file so only root can read or modify the password file.
126-
# sudo chmod 600 $SMB_CREDENTIAL_FILE
127-
128-
# # This command assumes you have logged in with az login
129-
# HTTP_ENDPOINT=$(az storage account show --resource-group $RESOURCE_GROUP_NAME --name $STORAGE_ACCOUNT_NAME --query "primaryEndpoints.file" --output tsv | tr -d '"')
130-
# SMB_PATH=$(echo $HTTP_ENDPOINT | cut -c7-${#HTTP_ENDPOINT})$FILE_SHARE_NAME
131-
132-
# STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group $RESOURCE_GROUP_NAME --account-name $STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv | tr -d '"')
133-
134-
# sudo mount -t cifs $SMB_PATH $MNT_PATH -o credentials=$SMB_CREDENTIAL_FILE,serverino,nosharesock,actimeo=30,mfsymlinks
135-
136-
# ls -al $MNT_PATH/TestData
86+
apt-get update
87+
apt-get install -y uuid-runtime
13788
13889
- name: Checkout repository
13990
uses: actions/checkout@v2
14091
with:
14192
path: Export-Deploy
14293

94+
14395
- name: Cache uv
14496
uses: actions/cache@v4
14597
id: cache
@@ -167,7 +119,6 @@ runs:
167119
shell: bash
168120
run: |
169121
echo ::group::Start test container
170-
MNT_PATH=/mnt/datadrive # ${{ steps.azure-fileshare.outputs.mnt_path }}
171122
172123
ARG=("")
173124
if [[ "${{ inputs.cpu-only }}" == "false" ]]; then
@@ -188,7 +139,7 @@ runs:
188139
--env RUN_ID=${{ github.run_id }} \
189140
--workdir /workspace \
190141
--volume $(pwd)/Export-Deploy:/workspace \
191-
--volume $MNT_PATH/TestData:/home/TestData \
142+
--volume ${{ inputs.test-data-path }}:/home/TestData \
192143
export-deploy \
193144
bash -c "sleep $(( ${{ inputs.timeout }} * 60 + 60 ))"
194145
RUN_TEST_EOF

‎.github/workflows/cicd-main.yml‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,24 @@ permissions:
3131

3232
jobs:
3333
pre-flight:
34-
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.64.2
34+
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.69.1
35+
with:
36+
default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }}
37+
non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }}
38+
default_test_data_path: ${{ vars.DEFAULT_TEST_DATA_PATH }}
39+
non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_TEST_DATA_PATH }}
40+
secrets:
41+
NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }}
3542

3643
linting:
3744
runs-on: ubuntu-latest
3845
needs: [pre-flight]
3946
if: |
4047
(
41-
needs.pre-flight.outputs.is_deployment_workflow == 'false'
48+
needs.pre-flight.outputs.is_deployment_workflow == 'false'
4249
&& needs.pre-flight.outputs.is_ci_workload == 'true'
4350
) || (
44-
needs.pre-flight.outputs.is_deployment_workflow == 'false'
51+
needs.pre-flight.outputs.is_deployment_workflow == 'false'
4552
&& needs.pre-flight.outputs.is_ci_workload == 'false'
4653
&& needs.pre-flight.outputs.docs_only == 'false'
4754
)
@@ -80,14 +87,14 @@ jobs:
8087
matrix:
8188
include:
8289
- script: L0_Unit_Tests_GPU
83-
runner: self-hosted-nemo
90+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
8491
- script: L0_Unit_Tests_CPU
8592
runner: linux-amd64-cpu16
8693
cpu-only: true
8794
needs: [pre-flight, cicd-wait-in-queue]
8895
if: |
8996
(
90-
success()
97+
success()
9198
|| needs.pre-flight.outputs.is_ci_workload == 'true'
9299
|| needs.pre-flight.outputs.force_run_all == 'true'
93100
)
@@ -111,21 +118,23 @@ jobs:
111118
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
112119
PAT: ${{ secrets.PAT }}
113120
inference-framework: trtllm
121+
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
122+
runner: ${{ matrix.runner }}
114123

115124
cicd-unit-tests-vllm:
116125
strategy:
117126
fail-fast: false
118127
matrix:
119128
include:
120129
- script: L0_Unit_Tests_GPU
121-
runner: self-hosted-nemo
130+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
122131
- script: L0_Unit_Tests_CPU
123132
runner: linux-amd64-cpu16
124133
cpu-only: true
125134
needs: [pre-flight, cicd-wait-in-queue]
126135
if: |
127136
(
128-
success()
137+
success()
129138
|| needs.pre-flight.outputs.is_ci_workload == 'true'
130139
|| needs.pre-flight.outputs.force_run_all == 'true'
131140
)
@@ -149,22 +158,24 @@ jobs:
149158
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
150159
PAT: ${{ secrets.PAT }}
151160
inference-framework: vllm
161+
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
162+
runner: ${{ matrix.runner }}
152163

153164
cicd-e2e-tests-trtllm:
154165
strategy:
155166
fail-fast: false
156167
matrix:
157168
include:
158169
- script: L2_Launch_TRTLLM
159-
runner: self-hosted-nemo
170+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
160171
- script: L2_TRTLLM_API_Deploy_Query
161-
runner: self-hosted-nemo
162-
needs: [cicd-unit-tests-trtllm]
172+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
173+
needs: [pre-flight, cicd-unit-tests-trtllm]
163174
runs-on: ${{ matrix.runner }}
164175
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
165176
if: |
166177
(
167-
success()
178+
success()
168179
|| needs.pre-flight.outputs.is_ci_workload == 'true'
169180
|| needs.pre-flight.outputs.force_run_all == 'true'
170181
)
@@ -186,15 +197,17 @@ jobs:
186197
timeout: 60
187198
PAT: ${{ secrets.PAT }}
188199
inference-framework: trtllm
200+
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
201+
runner: ${{ matrix.runner }}
189202

190203
cicd-e2e-tests-trt-onnx:
191-
needs: [cicd-unit-tests-trtllm]
192-
runs-on: self-hosted-nemo
204+
needs: [cicd-unit-tests-trtllm, pre-flight]
205+
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
193206
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
194207
environment: nemo-ci
195208
if: |
196209
(
197-
success()
210+
success()
198211
|| (
199212
needs.cicd-wait-in-queue.result == 'skipped'
200213
&& needs.pre-flight.outputs.is_ci_workload == 'true'
@@ -217,15 +230,17 @@ jobs:
217230
timeout: 60
218231
PAT: ${{ secrets.PAT }}
219232
inference-framework: trt-onnx
233+
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
234+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
220235

221236
cicd-e2e-tests-vllm:
222-
needs: [cicd-unit-tests-vllm]
223-
runs-on: self-hosted-nemo
237+
needs: [cicd-unit-tests-vllm, pre-flight]
238+
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
224239
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
225240
environment: nemo-ci
226241
if: |
227242
(
228-
success()
243+
success()
229244
|| needs.pre-flight.outputs.is_ci_workload == 'true'
230245
|| needs.pre-flight.outputs.force_run_all == 'true'
231246
)
@@ -246,15 +261,17 @@ jobs:
246261
timeout: 60
247262
PAT: ${{ secrets.PAT }}
248263
inference-framework: vllm
264+
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
265+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
249266

250267
cicd-e2e-tests-inframework:
251-
needs: [cicd-unit-tests-trtllm, cicd-unit-tests-vllm]
252-
runs-on: self-hosted-nemo
268+
needs: [pre-flight, cicd-unit-tests-trtllm, cicd-unit-tests-vllm]
269+
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
253270
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
254271
environment: nemo-ci
255272
if: |
256273
(
257-
success()
274+
success()
258275
|| needs.pre-flight.outputs.is_ci_workload == 'true'
259276
|| needs.pre-flight.outputs.force_run_all == 'true'
260277
)
@@ -275,6 +292,8 @@ jobs:
275292
timeout: 60
276293
PAT: ${{ secrets.PAT }}
277294
inference-framework: inframework
295+
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
296+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
278297

279298
Nemo_CICD_Test:
280299
needs:
@@ -283,6 +302,7 @@ jobs:
283302
- cicd-unit-tests-vllm
284303
- cicd-e2e-tests-trtllm
285304
- cicd-e2e-tests-vllm
305+
- cicd-e2e-tests-inframework
286306
if: |
287307
(
288308
needs.pre-flight.outputs.docs_only == 'true'
@@ -336,7 +356,7 @@ jobs:
336356
script: |
337357
await github.rest.repos.createCommitStatus({
338358
owner: context.repo.owner,
339-
repo: context.repo.repo,
359+
repo: context.repo.repo,
340360
sha: context.sha,
341361
state: 'success',
342362
description: 'No code changes - coverage check skipped',

‎tests/functional_tests/L2_TRTLLM_API_Deploy_Query.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,13 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
#!/bin/bash
16+
set -xeuo pipefail # Exit immediately if a command exits with a non-zero status
17+
18+
export CUDA_VISIBLE_DEVICES="0,1"
19+
export HF_MODULES_CACHE=/tmp/hf_modules_cache
20+
1521
coverage run -a --data-file=/workspace/.coverage --source=/workspace tests/functional_tests/utils/run_trtllm_api_deploy_query.py \
1622
--hf_model_path /home/TestData/hf/Llama-3.2-1B \
1723
--tensor_parallel_size 2
18-
coverage combine -q
24+
coverage combine -q

0 commit comments

Comments
 (0)