vuln-scan-images.yaml

# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Daily Image Vulnerability Scan

on:
  schedule:
    - cron: '0 6 * * *'  # Daily at 6:00 AM UTC
  workflow_dispatch:
    inputs:
      notify_slack:
        description: 'Post results to Slack (scheduled runs always post; manual runs skip by default)'
        type: boolean
        default: false

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

env:
  VALIDATOR_IMAGE_PREFIX: ghcr.io/nvidia/aicr-validators
  VALIDATOR_PHASES: "deployment performance conformance aiperf-bench"
  GATE_IMAGE: ghcr.io/nvidia/aicr-gate
  SCAN_TAG: scan-${{ github.sha }}

jobs:

  # =============================================================================
  # Build Ko Images (aicr, aicrd)
  # =============================================================================

  build-ko:
    name: Build Ko Images
    if: github.repository == 'nvidia/aicr'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Load versions
        id: versions
        uses: ./.github/actions/load-versions

      - name: Setup Go
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e  # v7.0.0
        with:
          go-version: ${{ steps.versions.outputs.go }}
          cache: true

      - name: Setup build tools
        uses: ./.github/actions/setup-build-tools
        with:
          install_ko: 'true'
          ko_version: ${{ steps.versions.outputs.ko }}

      - name: Authenticate to registry
        uses: ./.github/actions/ghcr-login

      - name: Build and push aicr
        env:
          GOFLAGS: -mod=vendor
          KO_DOCKER_REPO: ghcr.io/nvidia/aicr
        run: |
          set -euo pipefail
          ko build ./cmd/aicr \
            --bare \
            --platform=linux/amd64,linux/arm64 \
            --image-label="org.opencontainers.image.revision=${{ github.sha }}" \
            --tags="${SCAN_TAG}"

      - name: Build and push aicrd
        env:
          GOFLAGS: -mod=vendor
          KO_DOCKER_REPO: ghcr.io/nvidia/aicrd
          # Base image inherited from .ko.yaml (nvcr.io/nvidia/distroless/static),
          # same as the aicr build above.
        run: |
          set -euo pipefail
          ko build ./cmd/aicrd \
            --bare \
            --platform=linux/amd64,linux/arm64 \
            --image-label="org.opencontainers.image.revision=${{ github.sha }}" \
            --tags="${SCAN_TAG}"

  # =============================================================================
  # Build Docker Validator Images
  # =============================================================================

  build-docker:
    name: Docker Validator (${{ matrix.phase }}/${{ matrix.arch }})
    if: github.repository == 'nvidia/aicr'
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 15
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        phase: [deployment, performance, conformance, aiperf-bench]
        arch: [amd64, arm64]
        include:
          - arch: amd64
            runner: ubuntu-latest
            platform: linux/amd64
          - arch: arm64
            runner: ubuntu-24.04-arm
            platform: linux/arm64
          - phase: aiperf-bench
            dockerfile: validators/performance/aiperf-bench.Dockerfile
            image_title: aicr-aiperf-bench
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c  # v4.2.0

      - name: Authenticate to registry
        uses: ./.github/actions/ghcr-login

      - name: Read Go version
        id: go
        shell: bash
        run: echo "version=$(cat .go-version)" >> "$GITHUB_OUTPUT"

      - name: Build and push
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
        with:
          context: .
          file: ${{ matrix.dockerfile || format('validators/{0}/Dockerfile', matrix.phase) }}
          platforms: ${{ matrix.platform }}
          push: true
          # CHAINSAW_* args were dropped in #1236 — the deployment validator
          # no longer ships /usr/local/bin/chainsaw; it executes Chainsaw
          # Test format in-process via validators/chainsaw/inprocess.go.
          build-args: |
            GO_VERSION=${{ steps.go.outputs.version }}
          tags: ${{ env.VALIDATOR_IMAGE_PREFIX }}/${{ matrix.phase }}:${{ env.SCAN_TAG }}-${{ matrix.arch }}
          provenance: false
          cache-from: type=gha
          cache-to: type=gha,mode=max
          labels: |
            org.opencontainers.image.title=${{ matrix.image_title || format('aicr-validator-{0}', matrix.phase) }}
            org.opencontainers.image.revision=${{ github.sha }}

  # =============================================================================
  # Build Gate Image (aicr-gate)
  # =============================================================================
  # The in-cluster readiness gate image (cmd/gate/Dockerfile) is published only
  # on release tags by on-tag.yaml, so without this it is never vuln-scanned and
  # never refreshed on main. Mirror on-tag's build-args exactly (GO_VERSION +
  # CHAINSAW_VERSION + VERSION) so the scanned image matches what ships.

  build-gate:
    name: Docker Gate (${{ matrix.arch }})
    if: github.repository == 'nvidia/aicr'
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 15
    permissions:
      contents: read
      # packages: write — pushes the per-arch aicr-gate image to GHCR so the
      # manifest + scan jobs can pull it (mirrors build-docker/build-ko).
      packages: write
    strategy:
      fail-fast: false
      matrix:
        arch: [amd64, arm64]
        include:
          - arch: amd64
            runner: ubuntu-latest
            platform: linux/amd64
          - arch: arm64
            runner: ubuntu-24.04-arm
            platform: linux/arm64
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c  # v4.2.0

      - name: Authenticate to registry
        uses: ./.github/actions/ghcr-login

      - name: Load versions
        id: versions
        uses: ./.github/actions/load-versions

      - name: Build and push
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
        with:
          context: .
          file: cmd/gate/Dockerfile
          platforms: ${{ matrix.platform }}
          push: true
          build-args: |
            GO_VERSION=${{ steps.versions.outputs.go }}
            CHAINSAW_VERSION=${{ steps.versions.outputs.chainsaw }}
            VERSION=${{ env.SCAN_TAG }}
          tags: ${{ env.GATE_IMAGE }}:${{ env.SCAN_TAG }}-${{ matrix.arch }}
          provenance: false
          cache-from: type=gha
          cache-to: type=gha,mode=max
          labels: |
            org.opencontainers.image.title=aicr-gate
            org.opencontainers.image.revision=${{ github.sha }}

  # =============================================================================
  # Docker Manifest: Combine per-arch images into multi-arch manifests
  # =============================================================================

  docker-manifest:
    name: Docker Manifest
    runs-on: ubuntu-latest
    needs: [build-docker, build-gate]
    timeout-minutes: 5
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Authenticate to registry
        uses: ./.github/actions/ghcr-login

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c  # v4.2.0

      - name: Create multi-arch manifests
        run: |
          set -euo pipefail
          for phase in ${VALIDATOR_PHASES}; do
            IMAGE="${VALIDATOR_IMAGE_PREFIX}/${phase}"
            docker buildx imagetools create \
              -t "${IMAGE}:${SCAN_TAG}" \
              "${IMAGE}:${SCAN_TAG}-amd64" \
              "${IMAGE}:${SCAN_TAG}-arm64"
          done
          docker buildx imagetools create \
            -t "${GATE_IMAGE}:${SCAN_TAG}" \
            "${GATE_IMAGE}:${SCAN_TAG}-amd64" \
            "${GATE_IMAGE}:${SCAN_TAG}-arm64"

  # =============================================================================
  # Scan: Vulnerability scan all images
  # =============================================================================

  scan:
    name: Scan (${{ matrix.image }})
    runs-on: ubuntu-latest
    needs: [build-ko, docker-manifest]
    timeout-minutes: 15
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      matrix:
        image:
          - ghcr.io/nvidia/aicr
          - ghcr.io/nvidia/aicrd
          - ghcr.io/nvidia/aicr-validators/deployment
          - ghcr.io/nvidia/aicr-validators/performance
          - ghcr.io/nvidia/aicr-validators/conformance
          - ghcr.io/nvidia/aicr-validators/aiperf-bench
          - ghcr.io/nvidia/aicr-gate
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Authenticate to registry
        uses: ./.github/actions/ghcr-login

      - name: Scan container image
        id: scan
        uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2  # v7.4.0
        with:
          image: '${{ matrix.image }}:${{ env.SCAN_TAG }}'
          severity-cutoff: 'high'
          output-format: json
          only-fixed: true
          fail-build: false
          config: .grype.yaml
          # OpenVEX statements with per-CVE reachability evidence for
          # aiperf-bench suppressions. Grype applies statements only to
          # products whose PURL matches, so this is a no-op for the other
          # images in the matrix.
          vex: .openvex.json

      - name: Extract severity counts and IDs
        shell: bash
        env:
          IMAGE: ${{ matrix.image }}
        run: |
          set -euo pipefail
          JSON_FILE="${{ steps.scan.outputs.json }}"
          SHORT_NAME="${IMAGE##*/}"
          mkdir -p scan-results
          OUT="scan-results/${SHORT_NAME}.txt"

          if [[ ! -f "${JSON_FILE}" ]]; then
            echo "${SHORT_NAME}: scan output missing" > "${OUT}"
            exit 0
          fi

          # Severity counts from non-suppressed matches.
          CRITICAL=$(jq '[.matches[] | select(.vulnerability.severity == "Critical")] | length' "${JSON_FILE}")
          HIGH=$(jq '[.matches[] | select(.vulnerability.severity == "High")] | length' "${JSON_FILE}")
          MEDIUM=$(jq '[.matches[] | select(.vulnerability.severity == "Medium")] | length' "${JSON_FILE}")
          LOW=$(jq '[.matches[] | select(.vulnerability.severity == "Low")] | length' "${JSON_FILE}")
          NEGLIGIBLE=$(jq '[.matches[] | select(.vulnerability.severity == "Negligible")] | length' "${JSON_FILE}")
          # ignoredMatches conflates grype's internal filters with VEX
          # suppressions, so report VEX hits separately to make it
          # obvious when a statement actually applied.
          VEX_APPLIED=$(jq '[.ignoredMatches[]? | select((.appliedIgnoreRules // []) | any(.namespace == "vex"))] | length' "${JSON_FILE}")

          # HIGH+ identifiers for triage. Emits primary ID (often a GHSA)
          # plus any related CVE aliases so reviewers can cross-reference
          # NVD without re-running the scan.
          HIGH_IDS=$(jq -r '
            [ .matches[]
              | select(.vulnerability.severity == "High" or .vulnerability.severity == "Critical")
              | (.artifact.name + " " + .vulnerability.id
                 + ( (.relatedVulnerabilities // [] | map(.id) | join(","))
                     | if . == "" then "" else " (" + . + ")" end )) ]
            | join(", ")' "${JSON_FILE}")

          {
            echo "${SHORT_NAME}: ${CRITICAL} critical, ${HIGH} high, ${MEDIUM} medium, ${LOW} low, ${NEGLIGIBLE} negligible (${VEX_APPLIED} VEX-suppressed)"
            if [[ "${CRITICAL}" -gt 0 || "${HIGH}" -gt 0 ]]; then
              echo "  HIGH+: ${HIGH_IDS}"
            fi
          } > "${OUT}"
          cat "${OUT}"

      - name: Upload scan result
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: scan-${{ strategy.job-index }}
          path: scan-results/
          retention-days: 1

  # =============================================================================
  # Notify: Post scan results to Slack (only on critical/high findings)
  # =============================================================================

  notify:
    name: Slack Notification
    runs-on: ubuntu-latest
    needs: [scan]
    if: always() && needs.scan.result != 'cancelled'
    timeout-minutes: 5
    permissions:
      contents: read
    steps:
      - name: Download all scan results
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          pattern: scan-*
          merge-multiple: true
          path: scan-results

      - name: Post to Slack
        env:
          SLACK_SERVICE: ${{ secrets.SLACK_SERVICE }}
          # Scheduled runs always post; manual runs post only when the
          # notify_slack input is set. The message is echoed to the logs
          # either way, so manual runs can validate findings without Slack.
          POST_SLACK: ${{ github.event_name == 'schedule' || inputs.notify_slack }}
          SHA: ${{ github.sha }}
          SERVER: ${{ github.server_url }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
        run: |
          set -euo pipefail
          SHORT_SHA="${SHA:0:7}"
          TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M')
          RUN_URL="${SERVER}/${REPO}/actions/runs/${RUN_ID}"

          # Aggregate critical and high counts across all images
          TOTAL_CRITICAL=0
          TOTAL_HIGH=0
          RESULTS=""
          for f in scan-results/*.txt; do
            [[ -f "$f" ]] || continue
            # First line: summary counts. Subsequent lines (when present):
            # HIGH+ CVE/GHSA IDs for triage. Both flow into the Slack message.
            HEADER=$(head -1 "$f")
            BODY=$(cat "$f")
            RESULTS="${RESULTS}\\n• ${BODY//$'\n'/\\n  }"
            # Parse counts from the header line only so trailing ID lines
            # never contaminate the totals.
            C=$(echo "${HEADER}" | grep -oP '\d+(?= critical)' || echo 0)
            H=$(echo "${HEADER}" | grep -oP '\d+(?= high)' || echo 0)
            TOTAL_CRITICAL=$((TOTAL_CRITICAL + C))
            TOTAL_HIGH=$((TOTAL_HIGH + H))
          done

          # Only notify when critical or high vulnerabilities are found
          if [[ "${TOTAL_CRITICAL}" -eq 0 && "${TOTAL_HIGH}" -eq 0 ]]; then
            echo "No critical or high vulnerabilities found, skipping Slack notification"
            echo -e "Vulnerability Scan: ${TIMESTAMP} (${SHORT_SHA})${RESULTS}"
            exit 0
          fi

          MESSAGE="Vulnerability Scan: ${TIMESTAMP} (${SHORT_SHA})${RESULTS}\\n<${RUN_URL}|View run>"

          # Always echo the message so the run logs carry the findings,
          # then post to Slack only when enabled for this trigger.
          echo -e "${MESSAGE}"

          if [[ "${POST_SLACK}" != "true" ]]; then
            echo "Slack notification disabled for this run (POST_SLACK=${POST_SLACK}); skipping post"
          elif [[ -n "${SLACK_SERVICE}" ]]; then
            curl -sSf -X POST \
              -H 'Content-type: application/json' \
              --data "{\"text\":\"${MESSAGE}\"}" \
              "https://hooks.slack.com/services/${SLACK_SERVICE}"
          else
            echo "SLACK_SERVICE not set, skipping notification"
          fi

  # =============================================================================
  # Cleanup: Delete all scan-* images from GHCR (always runs)
  # =============================================================================

  cleanup:
    name: Cleanup Scan Images
    runs-on: ubuntu-latest
    needs: [notify]
    if: always()
    timeout-minutes: 10
    permissions:
      contents: read
      packages: write
    steps:
      - name: Delete scan images
        env:
          GH_TOKEN: ${{ github.token }}
          SHA: ${{ github.sha }}
        run: |
          set -euo pipefail

          # All image paths to clean up (org-level packages)
          IMAGES=(
            "nvidia/aicr"
            "nvidia/aicrd"
            "nvidia/aicr-validators/deployment"
            "nvidia/aicr-validators/performance"
            "nvidia/aicr-validators/conformance"
            "nvidia/aicr-validators/aiperf-bench"
            "nvidia/aicr-gate"
          )

          for IMAGE in "${IMAGES[@]}"; do
            # URL-encode the package name (slashes become %2F)
            PKG_NAME=$(echo "${IMAGE}" | sed 's|/|%2F|g')

            echo "Cleaning ${IMAGE}..."

            # List versions matching the scan tag
            VERSIONS=$(gh api \
              "orgs/NVIDIA/packages/container/${PKG_NAME}/versions?per_page=100" \
              --jq ".[] | select(.metadata.container.tags | any(startswith(\"${SCAN_TAG}\"))) | .id" \
              2>/dev/null || true)

            for VID in ${VERSIONS}; do
              echo "  Deleting version ${VID}"
              gh api \
                --method DELETE \
                "orgs/NVIDIA/packages/container/${PKG_NAME}/versions/${VID}" \
                2>/dev/null || true
            done
          done