Skip to content
Daily Image Vulnerability Scan

Daily Image Vulnerability Scan #98

Sign in to view logs

Daily Image Vulnerability Scan

Daily Image Vulnerability Scan #98

Workflow file for this run

.github/workflows/vuln-scan-images.yaml at 6f9050b
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Daily Image Vulnerability Scan
on:
schedule:
- cron: '0 6 * * *' # Daily at 6:00 AM UTC
workflow_dispatch:
inputs:
notify_slack:
description: 'Post results to Slack (scheduled runs always post; manual runs skip by default)'
type: boolean
default: false
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: true
env:
VALIDATOR_IMAGE_PREFIX: ghcr.io/nvidia/aicr-validators
VALIDATOR_PHASES: "deployment performance conformance aiperf-bench"
GATE_IMAGE: ghcr.io/nvidia/aicr-gate
SCAN_TAG: scan-${{ github.sha }}
jobs:
# =============================================================================
# Build Ko Images (aicr, aicrd)
# =============================================================================
build-ko:
name: Build Ko Images
if: github.repository == 'nvidia/aicr'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
packages: write
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Load versions
id: versions
uses: ./.github/actions/load-versions
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: ${{ steps.versions.outputs.go }}
cache: true
- name: Setup build tools
uses: ./.github/actions/setup-build-tools
with:
install_ko: 'true'
ko_version: ${{ steps.versions.outputs.ko }}
- name: Authenticate to registry
uses: ./.github/actions/ghcr-login
- name: Build and push aicr
env:
GOFLAGS: -mod=vendor
KO_DOCKER_REPO: ghcr.io/nvidia/aicr
run: |
set -euo pipefail
ko build ./cmd/aicr \
--bare \
--platform=linux/amd64,linux/arm64 \
--image-label="org.opencontainers.image.revision=${{ github.sha }}" \
--tags="${SCAN_TAG}"
- name: Build and push aicrd
env:
GOFLAGS: -mod=vendor
KO_DOCKER_REPO: ghcr.io/nvidia/aicrd
# Base image inherited from .ko.yaml (nvcr.io/nvidia/distroless/static),
# same as the aicr build above.
run: |
set -euo pipefail
ko build ./cmd/aicrd \
--bare \
--platform=linux/amd64,linux/arm64 \
--image-label="org.opencontainers.image.revision=${{ github.sha }}" \
--tags="${SCAN_TAG}"
# =============================================================================
# Build Docker Validator Images
# =============================================================================
build-docker:
name: Docker Validator (${{ matrix.phase }}/${{ matrix.arch }})
if: github.repository == 'nvidia/aicr'
runs-on: ${{ matrix.runner }}
timeout-minutes: 15
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
phase: [deployment, performance, conformance, aiperf-bench]
arch: [amd64, arm64]
include:
- arch: amd64
runner: ubuntu-latest
platform: linux/amd64
- arch: arm64
runner: ubuntu-24.04-arm
platform: linux/arm64
- phase: aiperf-bench
dockerfile: validators/performance/aiperf-bench.Dockerfile
image_title: aicr-aiperf-bench
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Authenticate to registry
uses: ./.github/actions/ghcr-login
- name: Read Go version
id: go
shell: bash
run: echo "version=$(cat .go-version)" >> "$GITHUB_OUTPUT"
- name: Build and push
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: ${{ matrix.dockerfile || format('validators/{0}/Dockerfile', matrix.phase) }}
platforms: ${{ matrix.platform }}
push: true
# CHAINSAW_* args were dropped in #1236 — the deployment validator
# no longer ships /usr/local/bin/chainsaw; it executes Chainsaw
# Test format in-process via validators/chainsaw/inprocess.go.
build-args: |
GO_VERSION=${{ steps.go.outputs.version }}
tags: ${{ env.VALIDATOR_IMAGE_PREFIX }}/${{ matrix.phase }}:${{ env.SCAN_TAG }}-${{ matrix.arch }}
provenance: false
cache-from: type=gha
cache-to: type=gha,mode=max
labels: |
org.opencontainers.image.title=${{ matrix.image_title || format('aicr-validator-{0}', matrix.phase) }}
org.opencontainers.image.revision=${{ github.sha }}
# =============================================================================
# Build Gate Image (aicr-gate)
# =============================================================================
# The in-cluster readiness gate image (cmd/gate/Dockerfile) is published only
# on release tags by on-tag.yaml, so without this it is never vuln-scanned and
# never refreshed on main. Mirror on-tag's build-args exactly (GO_VERSION +
# CHAINSAW_VERSION + VERSION) so the scanned image matches what ships.
build-gate:
name: Docker Gate (${{ matrix.arch }})
if: github.repository == 'nvidia/aicr'
runs-on: ${{ matrix.runner }}
timeout-minutes: 15
permissions:
contents: read
# packages: write — pushes the per-arch aicr-gate image to GHCR so the
# manifest + scan jobs can pull it (mirrors build-docker/build-ko).
packages: write
strategy:
fail-fast: false
matrix:
arch: [amd64, arm64]
include:
- arch: amd64
runner: ubuntu-latest
platform: linux/amd64
- arch: arm64
runner: ubuntu-24.04-arm
platform: linux/arm64
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Authenticate to registry
uses: ./.github/actions/ghcr-login
- name: Load versions
id: versions
uses: ./.github/actions/load-versions
- name: Build and push
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: cmd/gate/Dockerfile
platforms: ${{ matrix.platform }}
push: true
build-args: |
GO_VERSION=${{ steps.versions.outputs.go }}
CHAINSAW_VERSION=${{ steps.versions.outputs.chainsaw }}
VERSION=${{ env.SCAN_TAG }}
tags: ${{ env.GATE_IMAGE }}:${{ env.SCAN_TAG }}-${{ matrix.arch }}
provenance: false
cache-from: type=gha
cache-to: type=gha,mode=max
labels: |
org.opencontainers.image.title=aicr-gate
org.opencontainers.image.revision=${{ github.sha }}
# =============================================================================
# Docker Manifest: Combine per-arch images into multi-arch manifests
# =============================================================================
docker-manifest:
name: Docker Manifest
runs-on: ubuntu-latest
needs: [build-docker, build-gate]
timeout-minutes: 5
permissions:
contents: read
packages: write
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Authenticate to registry
uses: ./.github/actions/ghcr-login
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Create multi-arch manifests
run: |
set -euo pipefail
for phase in ${VALIDATOR_PHASES}; do
IMAGE="${VALIDATOR_IMAGE_PREFIX}/${phase}"
docker buildx imagetools create \
-t "${IMAGE}:${SCAN_TAG}" \
"${IMAGE}:${SCAN_TAG}-amd64" \
"${IMAGE}:${SCAN_TAG}-arm64"
done
docker buildx imagetools create \
-t "${GATE_IMAGE}:${SCAN_TAG}" \
"${GATE_IMAGE}:${SCAN_TAG}-amd64" \
"${GATE_IMAGE}:${SCAN_TAG}-arm64"
# =============================================================================
# Scan: Vulnerability scan all images
# =============================================================================
scan:
name: Scan (${{ matrix.image }})
runs-on: ubuntu-latest
needs: [build-ko, docker-manifest]
timeout-minutes: 15
permissions:
contents: read
packages: read
strategy:
fail-fast: false
matrix:
image:
- ghcr.io/nvidia/aicr
- ghcr.io/nvidia/aicrd
- ghcr.io/nvidia/aicr-validators/deployment
- ghcr.io/nvidia/aicr-validators/performance
- ghcr.io/nvidia/aicr-validators/conformance
- ghcr.io/nvidia/aicr-validators/aiperf-bench
- ghcr.io/nvidia/aicr-gate
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Authenticate to registry
uses: ./.github/actions/ghcr-login
- name: Scan container image
id: scan
uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0
with:
image: '${{ matrix.image }}:${{ env.SCAN_TAG }}'
severity-cutoff: 'high'
output-format: json
only-fixed: true
fail-build: false
config: .grype.yaml
# OpenVEX statements with per-CVE reachability evidence for
# aiperf-bench suppressions. Grype applies statements only to
# products whose PURL matches, so this is a no-op for the other
# images in the matrix.
vex: .openvex.json
- name: Extract severity counts and IDs
shell: bash
env:
IMAGE: ${{ matrix.image }}
run: |
set -euo pipefail
JSON_FILE="${{ steps.scan.outputs.json }}"
SHORT_NAME="${IMAGE##*/}"
mkdir -p scan-results
OUT="scan-results/${SHORT_NAME}.txt"
if [[ ! -f "${JSON_FILE}" ]]; then
echo "${SHORT_NAME}: scan output missing" > "${OUT}"
exit 0
fi
# Severity counts from non-suppressed matches.
CRITICAL=$(jq '[.matches[] | select(.vulnerability.severity == "Critical")] | length' "${JSON_FILE}")
HIGH=$(jq '[.matches[] | select(.vulnerability.severity == "High")] | length' "${JSON_FILE}")
MEDIUM=$(jq '[.matches[] | select(.vulnerability.severity == "Medium")] | length' "${JSON_FILE}")
LOW=$(jq '[.matches[] | select(.vulnerability.severity == "Low")] | length' "${JSON_FILE}")
NEGLIGIBLE=$(jq '[.matches[] | select(.vulnerability.severity == "Negligible")] | length' "${JSON_FILE}")
# ignoredMatches conflates grype's internal filters with VEX
# suppressions, so report VEX hits separately to make it
# obvious when a statement actually applied.
VEX_APPLIED=$(jq '[.ignoredMatches[]? | select((.appliedIgnoreRules // []) | any(.namespace == "vex"))] | length' "${JSON_FILE}")
# HIGH+ identifiers for triage. Emits primary ID (often a GHSA)
# plus any related CVE aliases so reviewers can cross-reference
# NVD without re-running the scan.
HIGH_IDS=$(jq -r '
[ .matches[]
| select(.vulnerability.severity == "High" or .vulnerability.severity == "Critical")
| (.artifact.name + " " + .vulnerability.id
+ ( (.relatedVulnerabilities // [] | map(.id) | join(","))
| if . == "" then "" else " (" + . + ")" end )) ]
| join(", ")' "${JSON_FILE}")
{
echo "${SHORT_NAME}: ${CRITICAL} critical, ${HIGH} high, ${MEDIUM} medium, ${LOW} low, ${NEGLIGIBLE} negligible (${VEX_APPLIED} VEX-suppressed)"
if [[ "${CRITICAL}" -gt 0 || "${HIGH}" -gt 0 ]]; then
echo " HIGH+: ${HIGH_IDS}"
fi
} > "${OUT}"
cat "${OUT}"
- name: Upload scan result
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: scan-${{ strategy.job-index }}
path: scan-results/
retention-days: 1
# =============================================================================
# Notify: Post scan results to Slack (only on critical/high findings)
# =============================================================================
notify:
name: Slack Notification
runs-on: ubuntu-latest
needs: [scan]
if: always() && needs.scan.result != 'cancelled'
timeout-minutes: 5
permissions:
contents: read
steps:
- name: Download all scan results
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: scan-*
merge-multiple: true
path: scan-results
- name: Post to Slack
env:
SLACK_SERVICE: ${{ secrets.SLACK_SERVICE }}
# Scheduled runs always post; manual runs post only when the
# notify_slack input is set. The message is echoed to the logs
# either way, so manual runs can validate findings without Slack.
POST_SLACK: ${{ github.event_name == 'schedule' || inputs.notify_slack }}
SHA: ${{ github.sha }}
SERVER: ${{ github.server_url }}
REPO: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
run: |
set -euo pipefail
SHORT_SHA="${SHA:0:7}"
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M')
RUN_URL="${SERVER}/${REPO}/actions/runs/${RUN_ID}"
# Aggregate critical and high counts across all images
TOTAL_CRITICAL=0
TOTAL_HIGH=0
RESULTS=""
for f in scan-results/*.txt; do
[[ -f "$f" ]] || continue
# First line: summary counts. Subsequent lines (when present):
# HIGH+ CVE/GHSA IDs for triage. Both flow into the Slack message.
HEADER=$(head -1 "$f")
BODY=$(cat "$f")
RESULTS="${RESULTS}\\n• ${BODY//$'\n'/\\n }"
# Parse counts from the header line only so trailing ID lines
# never contaminate the totals.
C=$(echo "${HEADER}" | grep -oP '\d+(?= critical)' || echo 0)
H=$(echo "${HEADER}" | grep -oP '\d+(?= high)' || echo 0)
TOTAL_CRITICAL=$((TOTAL_CRITICAL + C))
TOTAL_HIGH=$((TOTAL_HIGH + H))
done
# Only notify when critical or high vulnerabilities are found
if [[ "${TOTAL_CRITICAL}" -eq 0 && "${TOTAL_HIGH}" -eq 0 ]]; then
echo "No critical or high vulnerabilities found, skipping Slack notification"
echo -e "Vulnerability Scan: ${TIMESTAMP} (${SHORT_SHA})${RESULTS}"
exit 0
fi
MESSAGE="Vulnerability Scan: ${TIMESTAMP} (${SHORT_SHA})${RESULTS}\\n<${RUN_URL}|View run>"
# Always echo the message so the run logs carry the findings,
# then post to Slack only when enabled for this trigger.
echo -e "${MESSAGE}"
if [[ "${POST_SLACK}" != "true" ]]; then
echo "Slack notification disabled for this run (POST_SLACK=${POST_SLACK}); skipping post"
elif [[ -n "${SLACK_SERVICE}" ]]; then
curl -sSf -X POST \
-H 'Content-type: application/json' \
--data "{\"text\":\"${MESSAGE}\"}" \
"https://hooks.slack.com/services/${SLACK_SERVICE}"
else
echo "SLACK_SERVICE not set, skipping notification"
fi
# =============================================================================
# Cleanup: Delete all scan-* images from GHCR (always runs)
# =============================================================================
cleanup:
name: Cleanup Scan Images
runs-on: ubuntu-latest
needs: [notify]
if: always()
timeout-minutes: 10
permissions:
contents: read
packages: write
steps:
- name: Delete scan images
env:
GH_TOKEN: ${{ github.token }}
SHA: ${{ github.sha }}
run: |
set -euo pipefail
# All image paths to clean up (org-level packages)
IMAGES=(
"nvidia/aicr"
"nvidia/aicrd"
"nvidia/aicr-validators/deployment"
"nvidia/aicr-validators/performance"
"nvidia/aicr-validators/conformance"
"nvidia/aicr-validators/aiperf-bench"
"nvidia/aicr-gate"
)
for IMAGE in "${IMAGES[@]}"; do
# URL-encode the package name (slashes become %2F)
PKG_NAME=$(echo "${IMAGE}" | sed 's|/|%2F|g')
echo "Cleaning ${IMAGE}..."
# List versions matching the scan tag
VERSIONS=$(gh api \
"orgs/NVIDIA/packages/container/${PKG_NAME}/versions?per_page=100" \
--jq ".[] | select(.metadata.container.tags | any(startswith(\"${SCAN_TAG}\"))) | .id" \
2>/dev/null || true)
for VID in ${VERSIONS}; do
echo " Deleting version ${VID}"
gh api \
--method DELETE \
"orgs/NVIDIA/packages/container/${PKG_NAME}/versions/${VID}" \
2>/dev/null || true
done
done