feat(ci): migrate from Azure to AWS ephemeral runners

.github/actions/check-nvidia-sso-membership/action.yml

@@ -0,0 +1,139 @@
+name: 'Check NVIDIA SSO Membership'
+description: 'Check if a GitHub username exists in the NVIDIA SSO users list from github-audits'
+author: 'NVIDIA'
+
+inputs:
+  username:
+    description: 'GitHub username to check'
+    required: true
+  github_audits_repo:
+    description: 'Repository containing SSO users file'
+    required: false
+    default: 'NVIDIA-GitHub-Management/github-audits'
+  github_audits_version:
+    description: 'Release version tag'
+    required: false
+    default: 'v0.1.0'
+  sso_users_filename:
+    description: 'Filename of SSO users JSON'
+    required: false
+    default: 'users_sso.json'
+  github_token:
+    description: 'GitHub token with access to github-audits repo'
+    required: true
+
+outputs:
+  is_member:
+    description: 'Boolean - true if user is in NVIDIA SSO list, false otherwise'
+    value: ${{ steps.check-membership.outputs.is_member }}
+  is_org_member:
+    description: 'Boolean - true if user has NVIDIA or NVIDIA-NeMo in org_roles'
+    value: ${{ steps.check-membership.outputs.is_org_member }}
+  user_orgs:
+    description: 'Comma-separated list of orgs user is member of'
+    value: ${{ steps.check-membership.outputs.user_orgs }}
+  sso_file_available:
+    description: 'Boolean - true if SSO file was successfully downloaded'
+    value: ${{ steps.download-sso.outputs.sso_file_available }}
+  user_count:
+    description: 'Number of users in the SSO file (0 if download failed)'
+    value: ${{ steps.download-sso.outputs.user_count }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Download NVIDIA SSO users from github-audits
+      id: download-sso
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.github_token }}
+      run: |
+        echo "Downloading ${{ inputs.sso_users_filename }} from ${{ inputs.github_audits_repo }} ${{ inputs.github_audits_version }} release..."
+
+        # Download the release asset using gh CLI
+        gh release download ${{ inputs.github_audits_version }} \
+          --repo ${{ inputs.github_audits_repo }} \
+          --pattern ${{ inputs.sso_users_filename }} \
+          --clobber 2>&1 || {
+            echo "ERROR: Failed to download ${{ inputs.sso_users_filename }} from github-audits release"
+            echo "sso_file_available=false" >> $GITHUB_OUTPUT
+            echo "user_count=0" >> $GITHUB_OUTPUT
+            exit 0
+          }
+
+        # Verify file was downloaded and is valid JSON
+        if [ ! -f ${{ inputs.sso_users_filename }} ]; then
+          echo "ERROR: ${{ inputs.sso_users_filename }} file not found after download"
+          echo "sso_file_available=false" >> $GITHUB_OUTPUT
+          echo "user_count=0" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+
+        # Validate JSON structure
+        if ! jq -e 'type == "object"' ${{ inputs.sso_users_filename }} > /dev/null 2>&1; then
+          echo "ERROR: ${{ inputs.sso_users_filename }} is not a valid JSON object"
+          echo "sso_file_available=false" >> $GITHUB_OUTPUT
+          echo "user_count=0" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+
+        USER_COUNT=$(jq 'length' ${{ inputs.sso_users_filename }})
+        echo "Successfully downloaded ${{ inputs.sso_users_filename }} with $USER_COUNT NVIDIA SSO users"
+        echo "sso_file_available=true" >> $GITHUB_OUTPUT
+        echo "user_count=$USER_COUNT" >> $GITHUB_OUTPUT
+
+    - name: Check if user is in SSO list
+      id: check-membership
+      shell: bash
+      run: |
+        USERNAME="${{ inputs.username }}"
+        SSO_FILE="${{ inputs.sso_users_filename }}"
+
+        echo "Checking if $USERNAME is in NVIDIA SSO users list..."
+
+        # Check if SSO file is available
+        if [ "${{ steps.download-sso.outputs.sso_file_available }}" != "true" ] || [ ! -f "$SSO_FILE" ]; then
+          echo "ERROR: $SSO_FILE not available - cannot check membership"
+          echo "is_member=false" >> $GITHUB_OUTPUT
+          echo "is_org_member=false" >> $GITHUB_OUTPUT
+          echo "user_orgs=" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+
+        # Check if username exists as a key in the JSON object
+        if jq -e --arg user "$USERNAME" 'has($user)' "$SSO_FILE" > /dev/null 2>&1; then
+          echo "$USERNAME found in NVIDIA SSO users"
+          echo "is_member=true" >> $GITHUB_OUTPUT
+
+          # Extract and check org membership
+          IS_ORG_MEMBER=$(jq -r --arg user "$USERNAME" '
+            .[$user].org_roles // [] |
+            map(select(test("^(NVIDIA|NVIDIA-NeMo):Member$"))) |
+            length > 0
+          ' "$SSO_FILE")
+
+          USER_ORGS=$(jq -r --arg user "$USERNAME" '
+            .[$user].org_roles // [] |
+            map(split(":")[0]) |
+            unique |
+            join(",")
+          ' "$SSO_FILE")
+
+          echo "is_org_member=$IS_ORG_MEMBER" >> $GITHUB_OUTPUT
+          echo "user_orgs=$USER_ORGS" >> $GITHUB_OUTPUT
+
+          if [ "$IS_ORG_MEMBER" == "true" ]; then
+            echo "$USERNAME is a member of NVIDIA or NVIDIA-NeMo org"
+          else
+            echo "$USERNAME has @nvidia.com email but is not in NVIDIA or NVIDIA-NeMo org (orgs: $USER_ORGS)"
+          fi
+        else
+          echo "$USERNAME NOT found in NVIDIA SSO users"
+          echo "is_member=false" >> $GITHUB_OUTPUT
+          echo "is_org_member=false" >> $GITHUB_OUTPUT
+          echo "user_orgs=" >> $GITHUB_OUTPUT
+        fi
+
+branding:
+  icon: 'shield'
+  color: 'green'

.github/actions/test-template/action.yml

@@ -42,9 +42,12 @@ inputs:
     required: false
     default: '["all"]'
   image:
-    description: "Image to use for test"
+    description: "Full image URL with tag (e.g., 766267172432.dkr.ecr.us-east-1.amazonaws.com/nemo-speech:nemo_container-12345)"
+    required: true
+  test-data-path:
+    description: "Host path to mount as /home/TestData inside the container"
     required: false
-    default: "nemo_container"
+    default: /mnt/datadrive/TestData
   cpu-only:
     description: "Run tests on CPU only"
     required: false
@@ -70,7 +73,7 @@ runs:
     - name: Docker pull image
       shell: bash
       run: |
-        docker pull nemoci.azurecr.io/${{ inputs.image }}:${{ github.run_id }}
+        docker pull ${{ inputs.image }}
 
     - name: Clean repos
       shell: bash
@@ -122,7 +125,7 @@ runs:
           --env HF_HOME=/home/TestData/HF_HOME \
           --env RUN_ID=${{ github.run_id }} \
           --volume $(pwd)/${{ github.run_id }}/${{steps.uuid.outputs.id }}/NeMo:/workspace \
-          --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/${{ inputs.image }}:${{ github.run_id }} \
+          --volume ${{ inputs.test-data-path }}:/home/TestData ${{ inputs.image }} \
           bash -c "sleep $(( ${{ inputs.timeout }} * 60 + 60 ))"
         RUN_TEST_EOF
         )
@@ -190,7 +193,7 @@ runs:
         docker cp nemo_container_${{ github.run_id }}_${{ inputs.runner }}:/workspace/.coverage $DIR/.coverage
         docker cp nemo_container_${{ github.run_id }}_${{ inputs.runner }}:/workspace/coverage.xml $DIR/coverage.xml
 
-        coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
+        coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(python3 -c "import uuid; print(uuid.uuid4())")
         echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
 
         IS_SUCCESS=$(tail -n 1 $DIR/err.log | grep -q "Finished successfully." && echo "true" || echo "false")

.github/workflows/_build_container.yml

@@ -11,9 +11,22 @@ on:
         type: string
       runner:
         required: false
-        default: self-hosted-azure-builder
+        default: nemo-ci-aws-gpu-x2
         type: string
         description: "The runner to use for the build"
+      registry:
+        required: false
+        default: 766267172432.dkr.ecr.us-east-1.amazonaws.com
+        type: string
+        description: "Container registry"
+    outputs:
+      image:
+        description: "Full image URL (registry/nemo-speech:image-name-run_id)"
+        value: ${{ jobs.build.outputs.image }}
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   pre-flight:
@@ -60,30 +73,60 @@ jobs:
                 }
               }
             }' | jq -r '.data.repository.pullRequests.nodes[].number' | while read -r number; do
-              echo "nemoci.azurecr.io/${{ inputs.image-name }}-buildcache:$number"
+              echo "type=registry,ref=${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-buildcache-$number,mode=max"
             done)
 
-          echo "LAST_PRS<<EOF" >> $GITHUB_OUTPUT
-          echo "$LAST_PRS" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
+          echo "LAST_PRS<<EOF" | tee -a $GITHUB_OUTPUT
+          echo "$LAST_PRS" | tee -a $GITHUB_OUTPUT
+          echo "EOF" | tee -a $GITHUB_OUTPUT
 
   build:
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_container.yml@v0.27.0
+    runs-on: ${{ inputs.runner }}
     needs: [pre-flight]
-    with:
-      image-name: ${{ inputs.image-name }}
-      dockerfile: ${{ inputs.dockerfile }}
-      image-label: nemo-core
-      build-args: |
-        IMAGE_LABEL=nemo-core
-        NEMO_TAG=${{ github.sha }}
-        NEMO_REPO=https://github.com/NVIDIA/NeMo
-        PR_NUMBER=${{ github.event.pull_request.number || 0 }}
-        ${{ needs.pre-flight.outputs.build_args }}
-      prune-filter-timerange: 24h
-      use-inline-cache: false
-      cache-from: |
-        nemoci.azurecr.io/${{ inputs.image-name }}-buildcache:main
-        nemoci.azurecr.io/${{ inputs.image-name }}-buildcache:${{ github.event.pull_request.number || 0 }}
-        ${{ needs.pre-flight.outputs.cache-from }}
-      runner: ${{ inputs.runner }}
+    outputs:
+      image: ${{ steps.image-tag.outputs.image }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Compute image tag
+        id: image-tag
+        run: |
+          echo "image=${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-${{ github.run_id }}" | tee -a $GITHUB_OUTPUT
+
+      - name: Compute cache keys
+        id: cache-keys
+        run: |
+          PR_NUMBER="${{ github.event.pull_request.number }}"
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            KEY="main"
+          elif [[ -n "$PR_NUMBER" ]]; then
+            KEY="$PR_NUMBER"
+          else
+            KEY=$(echo "${{ github.ref_name }}" | tr '/' '-' | tr -cd '[:alnum:]._-')
+          fi
+          echo "cache-to=type=registry,ref=${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-buildcache-${KEY},mode=max" | tee -a $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          file: ${{ inputs.dockerfile }}
+          push: true
+          context: .
+          build-args: |
+            IMAGE_LABEL=nemo-core
+            NEMO_TAG=${{ github.sha }}
+            NEMO_REPO=https://github.com/NVIDIA/NeMo
+            PR_NUMBER=${{ github.event.pull_request.number || 0 }}
+            ${{ needs.pre-flight.outputs.build_args }}
+          cache-from: |
+            type=registry,ref=${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-buildcache-main,mode=max
+            type=registry,ref=${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-buildcache-${{ github.event.pull_request.number || 0 }},mode=max
+            ${{ needs.pre-flight.outputs.cache-from }}
+          cache-to: ${{ steps.cache-keys.outputs.cache-to }}
+          tags: |
+            ${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-${{ github.run_id }}
+            ${{ inputs.registry }}/nemo-speech:${{ inputs.image-name }}-${{ github.sha }}