e2e-tests.yaml

# Copyright NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: E2E Tests

on:
  workflow_call:
    inputs:
      operator_image:
        required: true
        type: string
      operator_version:
        required: true
        type: string
      use_values_override:
        required: false
        type: boolean
        default: false
        description: 'Use values-overrides artifact for component image configuration'
    secrets:
      AWS_ACCESS_KEY_ID:
        required: true
      AWS_SECRET_ACCESS_KEY:
        required: true
      AWS_SSH_KEY:
        required: true
      SLACK_BOT_TOKEN:
        required: false
      SLACK_CHANNEL_ID:
        required: false
  workflow_dispatch:
    inputs:
      operator_image:
        description: 'Operator image to test (override)'
        required: false
        type: string
      operator_version:
        description: 'Operator version to test (override)'
        required: false
        type: string
      use_values_override:
        description: 'Use values-overrides artifact for component image configuration'
        required: false
        type: boolean
        default: false

env:
  # Pinned versions of the tooling the test scripts drive from the runner.
  # helm used to be installed on the test node by tests/scripts/prerequisites.sh
  # straight from the helm get-helm-3 master script, i.e. whatever release was
  # current at the time the job ran. Pinning it here is a deliberate change.
  HELM_VERSION: v3.19.0
  # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
  KUBECTL_VERSION: v1.35.4
  JQ_VERSION: "1.7.1"
  # From https://github.com/jqlang/jq/releases/download/jq-1.7.1/sha256sum.txt
  JQ_SHA256: "5942c9b0934e510ee61eb3e30273f1b3fe2590df93933a93d7c58b81d19c8ff5"

jobs:
  variables:
    uses: ./.github/workflows/variables.yaml
    with:
      operator_image: ${{ inputs.operator_image }}
      operator_version: ${{ inputs.operator_version }}

  publish-helm-oci-chart:
    needs: [variables]
    permissions:
      contents: read
      packages: write
    uses: ./.github/workflows/publish-helm-oci-chart.yaml
    with:
      operator_image: ${{ needs.variables.outputs.operator_image }}
      operator_version: ${{ needs.variables.outputs.operator_version }}

  e2e-tests-containerd:
    needs: [variables, publish-helm-oci-chart]
    runs-on: linux-amd64-cpu4
    timeout-minutes: 90
    permissions:
      contents: read
      id-token: write
    env:
      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
      # kubeconfig pointing at the node's public API server endpoint here.
      KUBECONFIG: ${{ github.workspace }}/kubeconfig
      LOG_DIR: ${{ github.workspace }}/logs
    steps:
      - uses: actions/checkout@v7
        name: Check out code
        with:
          persist-credentials: false
      - name: Download values override file
        if: ${{ inputs.use_values_override }}
        uses: actions/download-artifact@v8
        with:
          name: values-overrides
          path: ${{ github.workspace }}
      - name: Install helm
        uses: azure/setup-helm@v4
        with:
          version: ${{ env.HELM_VERSION }}
      - name: Install kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: ${{ env.KUBECTL_VERSION }}
      - name: Install jq and verify runner tooling
        run: |
          set -euo pipefail
          mkdir -p "${RUNNER_TEMP}/bin"
          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
          chmod +x "${RUNNER_TEMP}/bin/jq"
          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
          helm version
          kubectl version --client
          "${RUNNER_TEMP}/bin/jq" --version
      - name: Set up Holodeck
        uses: NVIDIA/holodeck@v0.3.7
        with:
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
          holodeck_config: "tests/holodeck.yaml"
      - name: Get public dns name
        id: get_public_dns_name
        uses: mikefarah/yq@v4
        with:
          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
      - name: Configure node SSH access
        env:
          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
          PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
        run: |
          set -euo pipefail
          test -n "${PUBLIC_DNS_NAME}"
          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
          mkdir -p "${SSH_DIR}"
          chmod 700 "${SSH_DIR}"
          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
      - name: Verify cluster access
        run: |
          set -euo pipefail
          test -r "${KUBECONFIG}"
          kubectl cluster-info
          kubectl get nodes -o wide
      - name: Select values override file
        if: ${{ inputs.use_values_override }}
        run: |
          set -euo pipefail
          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
      - name: Load kernel modules on the node
        run: |
          set -euo pipefail
          # node-exec.sh falls back to running the operation locally when
          # NODE_SSH_HOST is empty, which is the developer path. In CI that
          # would run modprobe against the shared runner, so fail loudly
          # instead.
          test -n "${NODE_SSH_HOST}"
          ./tests/scripts/node-exec.sh load-modules
      - name: Run e2e tests
        env:
          OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
          OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
          GPU_PRODUCT_NAME: "Tesla-T4"
          CONTAINER_RUNTIME: "containerd"
        run: |
          set -euo pipefail
          # The restart-operator test shells out to node-exec.sh, which runs
          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
          # let crictl run against the shared runner.
          test -n "${NODE_SSH_HOST}"
          mkdir -p "${LOG_DIR}"
          ./tests/cases/defaults.sh
      - name: Collect cluster diagnostics
        if: always()
        run: |
          mkdir -p "${LOG_DIR}"
          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
      - name: Archive test logs
        # always() rather than failure(): a job stopped by timeout-minutes is
        # cancelled, not failed, so failure() would skip the upload on exactly
        # the runs whose logs we most need.
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: containerd-e2e-test-logs
          path: ./logs/
          retention-days: 15
      - name: Remove credentials from the runner
        if: always()
        run: |
          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
          rm -f "${KUBECONFIG}"

  e2e-tests-nvidiadriver:
    needs: [variables, publish-helm-oci-chart]
    runs-on: linux-amd64-cpu4
    timeout-minutes: 90
    permissions:
      contents: read
      id-token: write
    env:
      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
      # kubeconfig pointing at the node's public API server endpoint here.
      KUBECONFIG: ${{ github.workspace }}/kubeconfig
      LOG_DIR: ${{ github.workspace }}/logs
    steps:
      - uses: actions/checkout@v7
        name: Check out code
        with:
          persist-credentials: false
      - name: Download values override file
        if: ${{ inputs.use_values_override }}
        uses: actions/download-artifact@v8
        with:
          name: values-overrides
          path: ${{ github.workspace }}
      - name: Install helm
        uses: azure/setup-helm@v4
        with:
          version: ${{ env.HELM_VERSION }}
      - name: Install kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: ${{ env.KUBECTL_VERSION }}
      - name: Install jq and verify runner tooling
        run: |
          set -euo pipefail
          mkdir -p "${RUNNER_TEMP}/bin"
          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
          chmod +x "${RUNNER_TEMP}/bin/jq"
          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
          helm version
          kubectl version --client
          "${RUNNER_TEMP}/bin/jq" --version
      - name: Set up Holodeck
        uses: NVIDIA/holodeck@v0.3.7
        with:
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
          holodeck_config: "tests/holodeck.yaml"
      - name: Get public dns name
        id: get_public_dns_name
        uses: mikefarah/yq@v4
        with:
          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
      - name: Configure node SSH access
        env:
          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
          PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
        run: |
          set -euo pipefail
          test -n "${PUBLIC_DNS_NAME}"
          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
          mkdir -p "${SSH_DIR}"
          chmod 700 "${SSH_DIR}"
          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
      - name: Verify cluster access
        run: |
          set -euo pipefail
          test -r "${KUBECONFIG}"
          kubectl cluster-info
          kubectl get nodes -o wide
      - name: Select values override file
        if: ${{ inputs.use_values_override }}
        run: |
          set -euo pipefail
          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
      - name: Load kernel modules on the node
        run: |
          set -euo pipefail
          # node-exec.sh falls back to running the operation locally when
          # NODE_SSH_HOST is empty, which is the developer path. In CI that
          # would run modprobe against the shared runner, so fail loudly
          # instead.
          test -n "${NODE_SSH_HOST}"
          ./tests/scripts/node-exec.sh load-modules
      - name: Run e2e tests
        env:
          OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
          OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
          GPU_PRODUCT_NAME: "Tesla-T4"
          CONTAINER_RUNTIME: "containerd"
        run: |
          set -euo pipefail
          # The restart-operator test shells out to node-exec.sh, which runs
          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
          # let crictl run against the shared runner.
          test -n "${NODE_SSH_HOST}"
          mkdir -p "${LOG_DIR}"
          ./tests/cases/nvidia-driver.sh
      - name: Collect cluster diagnostics
        if: always()
        run: |
          mkdir -p "${LOG_DIR}"
          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
      - name: Archive test logs
        # always() rather than failure(): a job stopped by timeout-minutes is
        # cancelled, not failed, so failure() would skip the upload on exactly
        # the runs whose logs we most need.
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: nvidiadriver-e2e-test-logs
          path: ./logs/
          retention-days: 15
      - name: Remove credentials from the runner
        if: always()
        run: |
          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
          rm -f "${KUBECONFIG}"