precompiled.yaml

# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Run this workflow on a schedule
name: Precompiled images

on:
  schedule:
    - cron: '00 09 * * *'
  workflow_dispatch:

concurrency:
  group: precompiled-driver-build-${{ github.ref }}
  cancel-in-progress: false

jobs:
  set-driver-version-matrix:
    runs-on: linux-amd64-cpu4
    outputs:
      driver_branch: ${{ steps.extract_driver_branch.outputs.driver_branch }}
      exclude_build_matrix_pairs: ${{ steps.extract_driver_branch.outputs.exclude_build_matrix_pairs }}
      kernel_flavors: ${{ steps.extract_driver_branch.outputs.kernel_flavors }}
      dist: ${{ steps.extract_driver_branch.outputs.dist }}
      lts_kernel: ${{ steps.extract_driver_branch.outputs.lts_kernel }}
      platforms: ${{ steps.extract_driver_branch.outputs.platforms }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
      - name: Read driver versions
        id: extract_driver_branch
        run: |
          # get driver_branch
          DRIVER_BRANCH=("535" "570" "580")
          driver_branch_json=$(printf '%s\n' "${DRIVER_BRANCH[@]}" | jq -R . | jq -cs .)
          echo "driver_branch=$driver_branch_json" >> $GITHUB_OUTPUT

          # get kernel flavors
          KERNEL_FLAVORS=("aws" "azure" "azure-fde" "generic" "nvidia" "oracle")
          kernel_flavors_json=$(printf '%s\n' "${KERNEL_FLAVORS[@]}" | jq -R . | jq -cs .)
          echo "kernel_flavors=$kernel_flavors_json" >> $GITHUB_OUTPUT

          # get ubuntu distributions
          DIST=("ubuntu22.04" "ubuntu24.04")
          dist_json=$(printf '%s\n' "${DIST[@]}" | jq -R . | jq -cs .)
          echo "dist=$dist_json" >> $GITHUB_OUTPUT

          # LTS_KERNEL setup
          LTS_KERNEL=("5.15" "6.8")
          lts_kernel_json=$(printf '%s\n' "${LTS_KERNEL[@]}" | jq -R . | jq -cs .)
          echo "lts_kernel=$lts_kernel_json" >> $GITHUB_OUTPUT

          # platforms for precompiled build (amd64 always; arm64 for ubuntu24.04)
          PLATFORMS=("amd64" "arm64")
          platforms_json=$(printf '%s\n' "${PLATFORMS[@]}" | jq -R . | jq -cs .)
          echo "platforms=$platforms_json" >> $GITHUB_OUTPUT

          EXCLUDE_BUILD_MATRIX_PAIRS=("ubuntu24.04 535" "ubuntu24.04 570")
          echo "exclude_build_matrix_pairs=$(printf '%s\n' "${EXCLUDE_BUILD_MATRIX_PAIRS[@]}" | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT

  precompiled-build-image:
    needs: set-driver-version-matrix
    runs-on: linux-amd64-cpu4
    strategy:
      matrix:
        driver_branch: ${{ fromJson(needs.set-driver-version-matrix.outputs.driver_branch) }}
        flavor: ${{ fromJson(needs.set-driver-version-matrix.outputs.kernel_flavors) }}
        dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }}
        lts_kernel: ${{ fromJson(needs.set-driver-version-matrix.outputs.lts_kernel) }}
        exclude:
          - dist: ubuntu24.04
            driver_branch: 535
          - dist: ubuntu24.04
            driver_branch: 570
          - lts_kernel: 5.15
            dist: ubuntu24.04
          - flavor: azure-fde
            dist: ubuntu22.04
    steps:
      - uses: actions/checkout@v6
        name: Check out code
      - name: Calculate build vars
        id: vars
        run: |
          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV
          REPO_FULL_NAME="${{ github.repository }}"
          echo "LABEL_IMAGE_SOURCE=https://github.com/${REPO_FULL_NAME}" >> $GITHUB_ENV

          GENERATE_ARTIFACTS="false"
          echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
          echo "BUILD_MULTI_ARCH_IMAGES=true" >> $GITHUB_ENV

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build base image and get kernel version
        env:
          IMAGE_NAME: ghcr.io/nvidia/driver
          LTS_KERNEL: ${{ matrix.lts_kernel }}
        run: |
          if [[ "${{ matrix.dist }}" == "ubuntu22.04" ]]; then
            BASE_TARGET="jammy"
          elif [[ "${{ matrix.dist }}" == "ubuntu24.04" ]]; then
            BASE_TARGET="noble"
          fi
          export DOCKER_BUILD_PLATFORM_OPTIONS="--platform=linux/amd64"
          make DRIVER_BRANCH=${{ matrix.driver_branch }} KERNEL_FLAVOR=${{ matrix.flavor }} LTS_KERNEL=${LTS_KERNEL} build-base-${BASE_TARGET}

          trap "docker rm -f base-${BASE_TARGET}-${{ matrix.flavor }}" EXIT
          docker run -d --name base-${BASE_TARGET}-${{ matrix.flavor }} ghcr.io/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${{ matrix.flavor }}-${{ matrix.driver_branch }}
          # try 3 times every 10 seconds to get the file, if success exit the loop
          for i in {1..3}; do
              docker cp base-${BASE_TARGET}-${{ matrix.flavor }}:/var/kernel_version.txt kernel_version.txt && break
              sleep 10
          done
      - name: Build image
        env:
          IMAGE_NAME: ghcr.io/nvidia/driver
          PRECOMPILED: "true"
          DIST: signed_${{ matrix.dist }}
        run: |
          source kernel_version.txt
          export DRIVER_MULTI_ARCH_TAR="driver-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${{ matrix.dist }}.tar"
          export DOCKER_BUILD_OPTIONS="--output=type=oci,dest=./${DRIVER_MULTI_ARCH_TAR}"
          # build multi-arch images for ubuntu24.04 except azure-fde , arm64 does not support azure-fde
          if [[ "${{ matrix.dist }}" == "ubuntu24.04" ]] && [[ "${{ matrix.flavor }}" != "azure-fde" ]]; then
            export DOCKER_BUILD_PLATFORM_OPTIONS="--platform=linux/amd64,linux/arm64"
          else
            export DOCKER_BUILD_PLATFORM_OPTIONS="--platform=linux/amd64"
          fi
          make DRIVER_VERSIONS=${DRIVER_VERSIONS} DRIVER_BRANCH=${{ matrix.driver_branch }} KERNEL_VERSION=${KERNEL_VERSION} build-${DIST}-${DRIVER_VERSION}
      - name: Save base image and kernel version file
        env:
          DIST: ${{ matrix.dist }}
          PRIVATE_REGISTRY: "ghcr.io"
          LTS_KERNEL: ${{ matrix.lts_kernel }}
        run: |
          source kernel_version.txt
          if [[ "${{ matrix.dist }}" == "ubuntu22.04" ]]; then
            BASE_TARGET="jammy"
          elif [[ "${{ matrix.dist }}" == "ubuntu24.04" ]]; then
            BASE_TARGET="noble"
          fi
          tar -cvf kernel-version-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar kernel_version.txt
          docker save "${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${{ matrix.flavor }}-${{ matrix.driver_branch }}" \
            -o  ./base-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar
          # set env for artifacts upload
          echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV
          echo "DIST=$DIST" >> $GITHUB_ENV

      - name: Upload base image as an artifact
        uses: actions/upload-artifact@v7
        with:
           name: base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}
           path: ./base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar
           retention-days: 1

      - name: Upload build image as an artifact
        uses: actions/upload-artifact@v7
        with:
           name: driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}
           path: ./driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar
           retention-days: 1
      
      - name: Upload kernel version as an artifact
        uses: actions/upload-artifact@v7
        with:
           name: kernel-version-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}
           path: ./kernel-version-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar
           retention-days: 1
  
  determine-e2e-test-matrix:
    runs-on: linux-amd64-cpu4
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }}
        lts_kernel: ${{ fromJson(needs.set-driver-version-matrix.outputs.lts_kernel) }}
        exclude:
          - lts_kernel: 5.15
            dist: ubuntu24.04
    needs:
      - precompiled-build-image
      - set-driver-version-matrix
    steps:
      - name: Check out code
        uses: actions/checkout@v6
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all kernel-version artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: kernel-version*${{ matrix.lts_kernel }}*${{ matrix.dist }}
          path: ./kernel-version-artifacts
          merge-multiple: true
  
      - name: Set kernel version
        env:
          DIST: ${{ matrix.dist }}
          GH_TOKEN:  ${{ secrets.GITHUB_TOKEN }}
          LTS_KERNEL: ${{ matrix.lts_kernel }}
        run: |
          kernel_flavors_json='${{ needs.set-driver-version-matrix.outputs.kernel_flavors }}'
          KERNEL_FLAVORS=($(echo "$kernel_flavors_json" | jq -r '.[]'))
          driver_branch_json='${{ needs.set-driver-version-matrix.outputs.driver_branch }}'
          exclude_pairs_json='${{ needs.set-driver-version-matrix.outputs.exclude_build_matrix_pairs }}'
          DRIVER_BRANCHES=()
          for b in $(echo "$driver_branch_json" | jq -r '.[]'); do
            pair="$DIST $b"
            if ! echo "$exclude_pairs_json" | jq -r '.[]' | grep -qx "$pair"; then
              DRIVER_BRANCHES+=("$b")
            fi
          done
          source ./tests/scripts/ci-precompiled-helpers.sh
          platforms_json='${{ needs.set-driver-version-matrix.outputs.platforms }}'
          platform=$(echo "$platforms_json" | jq -r '.[]')
          for PLATFORM in $platform; do
            if [[ "$PLATFORM" == "arm64" ]] && [[ "$DIST" == "ubuntu22.04" ]]; then
              continue
            fi
            if [[ "$PLATFORM" == "arm64" ]]; then
              PLATFORM_SUFFIX="-arm64"
              FLAVORS_FOR_PLATFORM=()
              for f in "${KERNEL_FLAVORS[@]}"; do
                if [[ "$f" != "azure-fde" ]]; then
                  FLAVORS_FOR_PLATFORM+=("$f")
                fi
              done
            else
              PLATFORM_SUFFIX=""
              FLAVORS_FOR_PLATFORM=("${KERNEL_FLAVORS[@]}")
            fi
            KERNEL_VERSIONS=($(get_kernel_versions_to_test FLAVORS_FOR_PLATFORM[@] DRIVER_BRANCHES[@] $DIST $LTS_KERNEL $PLATFORM_SUFFIX))
            if [ -n "${KERNEL_VERSIONS[*]}" ]; then
              printf '%s\n' "${KERNEL_VERSIONS[@]}" | jq -R . | jq -s . > ./matrix_values_${{ matrix.dist }}_${{ matrix.lts_kernel }}$PLATFORM_SUFFIX.json
            fi
          done

      - name: Upload kernel matrix values as artifacts
        uses: actions/upload-artifact@v7
        with:
          name: matrix-values-${{ matrix.dist }}-${{ matrix.lts_kernel }}
          path: ./matrix_values_${{ matrix.dist }}_${{ matrix.lts_kernel }}*.json
          retention-days: 1

  collect-e2e-test-matrix:
    runs-on: linux-amd64-cpu4
    needs:
      - determine-e2e-test-matrix
      - set-driver-version-matrix
    outputs:
      matrix_values_not_empty: ${{ steps.set_kernel_version.outputs.matrix_values_not_empty }}
      matrix_values: ${{ steps.set_kernel_version.outputs.matrix_values }}
      exclude_matrix_values: ${{ steps.set_kernel_version.outputs.exclude_matrix_values }}
    steps:
      - name: Check out code
        uses: actions/checkout@v6
      - name: Download all matrix artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: matrix-values-*
          path: ./matrix-values-artifacts
          merge-multiple: false
      - name: Set and append matrix values for ubuntu
        id: set_kernel_version
        env:
          GH_TOKEN:  ${{ secrets.GITHUB_TOKEN }}
        run: |
          echo "matrix_values_not_empty=0" >> $GITHUB_OUTPUT
          kernel_versions=()

          # Read and merge kernel_version values from all platform artifacts (amd64 and arm64)
          for f in $(find ./matrix-values-artifacts -name "matrix_values_*.json" -type f 2>/dev/null); do
            value=$(jq -r '.[]' "$f")
            kernel_versions+=($value)
            echo "matrix_values_not_empty=1" >> $GITHUB_OUTPUT
          done
          kernel_versions=($(printf '%s\n' "${kernel_versions[@]}" | sort -u))
          echo "Collected Kernel Versions: ${kernel_versions[@]}"
          combined_values=$(printf '%s\n' "${kernel_versions[@]}" | jq -R . | jq -s -c .  | tr -d ' \n')
          echo "Combined Kernel Versions JSON: $combined_values"
          # FIXME -- remove once azure kernel upgrade starts working
          exclude_combined_values=$(printf '%s\n' "${kernel_versions[@]}" | jq -R . | jq -s -c 'map(select(test("azure")))')
          exclude_combined_values=$(echo "$exclude_combined_values" | jq -c '[.[] | {kernel_version: .}]')
          echo "exclude_combined_values Kernel Versions : $exclude_combined_values"
          echo "matrix_values=$combined_values" >> $GITHUB_OUTPUT
          echo "exclude_matrix_values=$exclude_combined_values" >> $GITHUB_OUTPUT
          published_kernels=$(printf " %s " "${kernel_versions[@]}")
          echo "published_kernels=${published_kernels}" >> $GITHUB_OUTPUT

      # slack notification for new kernel release before e2e tests starts
      # as e2e tests may fail for new kernel release
      - name: Slack notification
        if: ${{ steps.set_kernel_version.outputs.matrix_values_not_empty == '1' && github.ref == 'refs/heads/main' }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          method: chat.postMessage
          payload: |
            {
              "channel": "${{ secrets.SLACK_CHANNEL_ID }}",
              "text": "${{ secrets.DETECTED_PRECOMPILED_KERNEL_MESSAGE }} ${{ steps.set_kernel_version.outputs.published_kernels }} \n
              Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
            }

  e2e-tests-nvidiadriver:
    runs-on: linux-amd64-cpu4
    needs:
      - collect-e2e-test-matrix
      - set-driver-version-matrix
    if: ${{ needs.collect-e2e-test-matrix.outputs.matrix_values_not_empty == '1' }}
    strategy:
      matrix:
        kernel_version: ${{ fromJson(needs.collect-e2e-test-matrix.outputs.matrix_values) }}
        exclude: ${{ fromJson(needs.collect-e2e-test-matrix.outputs.exclude_matrix_values) }}
      max-parallel: 5
    steps:
      - name: Check out code
        uses: actions/checkout@v6
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name:  Set and Calculate test vars
        run: |
          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
          echo "${{ secrets.AWS_SSH_KEY }}" > ${{ github.workspace }}/key.pem && chmod 400 ${{ github.workspace }}/key.pem
          echo "PRIVATE_REGISTRY=ghcr.io" >> $GITHUB_ENV
          KERNEL_VERSION="${{ matrix.kernel_version }}"
          if [[ "$KERNEL_VERSION" == *-arm64 ]]; then
            echo "PLATFORM=arm64" >> $GITHUB_ENV
            KERNEL_VERSION="${KERNEL_VERSION%-arm64}"
          else
            echo "PLATFORM=amd64" >> $GITHUB_ENV
          fi
          # Extract the last segment after the last dash
          DIST=${KERNEL_VERSION##*-}
          echo "DIST=$DIST" >> $GITHUB_ENV
          KERNEL_VERSION=${KERNEL_VERSION%-*}
          echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV
          echo "DRIVER_BRANCHES=${{ needs.set-driver-version-matrix.outputs.driver_branch }}" >> $GITHUB_ENV
          echo "exclude_pairs=${{ needs.set-driver-version-matrix.outputs.exclude_build_matrix_pairs }}" >> $GITHUB_ENV
      - name: Download driver image artifacts
        uses: actions/download-artifact@v7
        with:
          pattern: driver-images-*-${{ env.KERNEL_VERSION }}-${{ env.DIST }}*
          path: ./tests/
          merge-multiple: true
      - name: Install skopeo
        run: |
          sudo apt-get update && sudo apt-get install -y skopeo
      - name: Configure Holodeck e2e test config (kernel, OS, instance)
        run: |
          yq eval '.spec += {"kernel": {"version": strenv(KERNEL_VERSION)}}' -i tests/holodeck_ubuntu.yaml
          if [[ "$DIST" == "ubuntu24.04" ]]; then
            yq eval '.spec.instance.os = "ubuntu-24.04"' -i tests/holodeck_ubuntu.yaml
          fi
          if [[ "$PLATFORM" == "arm64" ]]; then
            yq eval '.spec.instance.image.architecture = strenv(PLATFORM)' -i tests/holodeck_ubuntu.yaml
            yq eval '.spec.instance.type = "g5g.xlarge"' -i tests/holodeck_ubuntu.yaml
            yq eval '.spec.instance.region = "us-west-2"' -i tests/holodeck_ubuntu.yaml
          fi

      - name: Set up Holodeck
        uses: NVIDIA/holodeck@main
        env:
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
        with:
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
          holodeck_config: "tests/holodeck_ubuntu.yaml"
      - name: Get public dns name
        id: get_public_dns_name
        uses: mikefarah/yq@master
        with:
          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
      - name:  Calculate holodeck instance hostname
        run: |
          echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV

      - name: Install GitHub CLI
        run: |
          sudo apt-get update
          sudo apt-get install -y gh

      - name: Precompiled e2e test gpu driver validation
        env:
          TEST_CASE: "./tests/cases/nvidia-driver.sh"
          GPU_OPERATOR_OPTIONS: "--set driver.repository=${{ env.PRIVATE_REGISTRY }}/nvidia --set driver.usePrecompiled=true \
            --set driver.imagePullPolicy=Never"
          GH_TOKEN:  ${{ secrets.GITHUB_TOKEN }}
        run: |
          rc=0
          DRIVER_BRANCHES=($(echo '${{ env.DRIVER_BRANCHES }}' | jq -r '.[]'))
          exclude_pairs='${{ env.exclude_pairs }}'
          for DRIVER_VERSION in "${DRIVER_BRANCHES[@]}"; do
            pair="$DIST $DRIVER_VERSION"
            if echo "$exclude_pairs" | jq -r '.[]' | grep -qx "$pair"; then
              continue
            fi
            echo "Running e2e for DRIVER_VERSION=$DRIVER_VERSION"
            status=0
            TEST_CASE_ARGS="${GPU_OPERATOR_OPTIONS} --set driver.version=${DRIVER_VERSION}"
            # add escape character for space
            TEST_CASE_ARGS=$(printf '%q ' "$TEST_CASE_ARGS")
            IMAGE_PATH="./tests/driver-images-${DRIVER_VERSION}-${KERNEL_VERSION}-${DIST}.tar"
            skopeo copy --override-os linux --override-arch "${PLATFORM}" "oci-archive:${IMAGE_PATH}" "docker-archive:./tests/tmp.tar:${PRIVATE_REGISTRY}/nvidia/driver:${DRIVER_VERSION}-${KERNEL_VERSION}-${DIST}"
            mv "./tests/tmp.tar" "${IMAGE_PATH}"
            ./tests/ci-run-e2e.sh "${TEST_CASE}" "${TEST_CASE_ARGS}" ${IMAGE_PATH} || status=$?
            if [ $status -eq 1 ]; then
              echo "e2e validation failed for driver version $DRIVER_VERSION with status $status"
              rc=$status
            fi
            rm -f $IMAGE_PATH
          done
          ./tests/scripts/pull.sh /tmp/logs logs
          exit $rc

      - name: Archive test logs
        if: ${{ failure() }}
        uses: actions/upload-artifact@v7
        with:
          name: nvidiadriver-Precompiled-e2e-test-logs
          path: ./logs/
          retention-days: 15

  publish-precompiled-image:
    runs-on: linux-amd64-cpu4
    needs:
      - set-driver-version-matrix
      - collect-e2e-test-matrix
      - e2e-tests-nvidiadriver
    env:
      REGISTRY_AUTH_FILE: ${{ github.workspace }}/config.json
    strategy:
      max-parallel: 5
      matrix:
        driver_branch: ${{ fromJson(needs.set-driver-version-matrix.outputs.driver_branch) }}
        kernel_version: ${{ fromJson(needs.collect-e2e-test-matrix.outputs.matrix_values) }}
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name:  Set image vars
        id: set_image_vars
        run: |
          echo "PRIVATE_REGISTRY=ghcr.io" >> $GITHUB_ENV
          KERNEL_VERSION="${{ matrix.kernel_version }}"
          if [[ "$KERNEL_VERSION" == *-arm64 ]]; then
            KERNEL_VERSION="${KERNEL_VERSION%-arm64}"
          fi
          echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV
          DIST="${KERNEL_VERSION##*-}"
          pair="$DIST ${{ matrix.driver_branch }}"
          echo "run_publish=true" >> $GITHUB_OUTPUT
          if echo '${{ needs.set-driver-version-matrix.outputs.exclude_build_matrix_pairs }}' | jq -r '.[]' | grep -qx "$pair"; then
            echo "run_publish=false" >> $GITHUB_OUTPUT
          fi

      - name: Install skopeo and login to GitHub Container Registry
        run: |
          sudo apt-get update && sudo apt-get install -y skopeo
          mkdir -p "$(dirname "${REGISTRY_AUTH_FILE}")"
          echo "${{ secrets.GITHUB_TOKEN }}" | skopeo login ${PRIVATE_REGISTRY} -u ${{ github.actor }} --password-stdin --authfile "${REGISTRY_AUTH_FILE}"

      - name: Download base image artifact
        if: steps.set_image_vars.outputs.run_publish == 'true'
        uses: actions/download-artifact@v8
        with:
          name: base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}
          path: ./

      - name: Publish base image
        if: steps.set_image_vars.outputs.run_publish == 'true'
        run: |
          LTS_KERNEL=$(echo "${{ env.KERNEL_VERSION }}" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
          KERNEL_FLAVOR=$(echo "${{ env.KERNEL_VERSION }}" | sed -E 's/^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-(.*)-ubuntu[0-9]+\.[0-9]+$/\1/')
          DIST=$(echo "${{ env.KERNEL_VERSION }}" | sed -E 's/^.*-(ubuntu[0-9]+\.[0-9]+)$/\1/')
          if [[ "${DIST}" == "ubuntu22.04" ]]; then
            BASE_TARGET="jammy"
          elif [[ "${DIST}" == "ubuntu24.04" ]]; then
            BASE_TARGET="noble"
          fi
          image_path="./base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}.tar"
          echo "uploading  $image_path"
          if [[ "${{ github.ref == 'refs/heads/main' }}" == "true" ]]; then
            skopeo copy --authfile "${REGISTRY_AUTH_FILE}" "docker-archive:${image_path}" docker://${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${KERNEL_FLAVOR}-${{ matrix.driver_branch }}
          else
            echo "Skipping base image push for non-main branch ${{ github.ref }}"
          fi

      - name: Download built image artifact
        if: steps.set_image_vars.outputs.run_publish == 'true'
        uses: actions/download-artifact@v8
        with:
           name: driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}
           path: ./

      - name: Publish image
        if: steps.set_image_vars.outputs.run_publish == 'true'
        run: |
          image_path="./driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}.tar"
          echo "uploading  $image_path"
          if [[ "${{ github.ref == 'refs/heads/main' }}" == "true" ]]; then
            skopeo copy --authfile "${REGISTRY_AUTH_FILE}" "oci-archive:${image_path}" docker://${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}
          else
            echo "Skipping image push for non-main branch ${{ github.ref }}"
          fi

      - name: Slack notification
        if: ${{ steps.set_image_vars.outputs.run_publish == 'true' && github.ref == 'refs/heads/main' }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          method: chat.postMessage
          payload: |
            {
              "channel": "${{ secrets.SLACK_CHANNEL_ID }}",
              "text": "${{ secrets.PUBLISHED_PRECOMPILED_IMAGE_MESSAGE }} ${{ env.PRIVATE_REGISTRY }}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}"
            }