Security scan and Linear sync #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Security scan + Linear sync. Registry-only: one matrix job per *image repository* (full ref before the last ":tag"); tag variants of the same repo share a runner and are scanned serially per Trivy/Grype. Very many repos may still hit matrix or output-size limits. Profile JSON defines `registry.repos` and variant suffix rules. See https://docs.github.com/en/actions/using-workflows/reusing-workflow-configurations#yaml-anchors-and-aliases | |
| name: Security scan and Linear sync | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| docker_tag: | |
| description: "Image tag in the container registry to pull/scan. Default is `head` (DataHub’s published branch image tag; see `MAIN_BRANCH_TAG` in docker helpers). Examples: head, latest, or a release tag." | |
| required: true | |
| type: string | |
| default: head | |
| scan_profile: | |
| description: "Profile key (see `.github/security-scan-profiles.*.json`)." | |
| required: true | |
| type: choice | |
| default: allImages | |
| options: | |
| - allImages | |
| - javaImages | |
| - actionsImages | |
| - executorImages | |
| - integrationImages | |
| variant_scope: | |
| description: "Tag variants to scan: base only (`primary`), all profile suffixes (`all`), or `slim` / `locked`." | |
| required: true | |
| type: choice | |
| default: all | |
| options: | |
| - primary | |
| - all | |
| - slim | |
| - locked | |
| scan_severity_levels: | |
| description: "Scanner severity levels. Default is critical+high; optional includes medium." | |
| required: true | |
| type: choice | |
| default: CRITICAL,HIGH | |
| options: | |
| - CRITICAL,HIGH | |
| - CRITICAL,HIGH,MEDIUM | |
| scan_with_trivy: | |
| description: "Run Trivy" | |
| required: true | |
| type: boolean | |
| default: true | |
| scan_with_grype: | |
| description: "Run Grype" | |
| required: true | |
| type: boolean | |
| default: true | |
| upload_trivy_to_github: | |
| description: "Upload Trivy SARIF results to the GitHub Security tab" | |
| required: true | |
| type: boolean | |
| default: false | |
| scan_artifact_retention_days: | |
| description: "Retention (days) for raw scan result artifacts." | |
| required: true | |
| type: number | |
| default: 5 | |
| workflow_run: | |
| workflows: | |
| - Nightly Docker Test | |
| types: | |
| - completed | |
| permissions: | |
| actions: write | |
| contents: read | |
| security-events: write | |
| concurrency: | |
| group: security-scan-linear-${{ github.run_id }} | |
| cancel-in-progress: false | |
| # workflow_dispatch fills github.event.inputs; workflow_run does not — resolve defaults for automated runs. | |
| env: | |
| RESOLVED_DOCKER_TAG: ${{ github.event_name == 'workflow_run' && 'head' || github.event.inputs.docker_tag }} | |
| RESOLVED_SCAN_PROFILE: ${{ github.event_name == 'workflow_run' && 'allImages' || github.event.inputs.scan_profile }} | |
| RESOLVED_VARIANT_SCOPE: ${{ github.event_name == 'workflow_run' && 'all' || github.event.inputs.variant_scope }} | |
| RESOLVED_SCAN_SEVERITY_LEVELS: ${{ github.event_name == 'workflow_run' && 'CRITICAL,HIGH' || github.event.inputs.scan_severity_levels }} | |
| RESOLVED_SCAN_WITH_TRIVY: ${{ github.event_name == 'workflow_run' && 'true' || (inputs.scan_with_trivy && 'true' || 'false') }} | |
| RESOLVED_SCAN_WITH_GRYPE: ${{ github.event_name == 'workflow_run' && 'true' || (inputs.scan_with_grype && 'true' || 'false') }} | |
| RESOLVED_UPLOAD_TRIVY_TO_GITHUB: ${{ (github.event_name == 'workflow_run' && github.ref_name == 'master' && 'true') || (inputs.upload_trivy_to_github && 'true' || 'false') }} | |
| RESOLVED_SCAN_ARTIFACT_RETENTION_DAYS: ${{ github.event_name == 'workflow_run' && '5' || github.event.inputs.scan_artifact_retention_days }} | |
| jobs: | |
| select-runner: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| env: | |
| DOCKER_REGISTRY: ${{ vars.DOCKER_REGISTRY }} | |
| DEPOT_PROJECT_ID: ${{ vars.DEPOT_PROJECT_ID }} | |
| outputs: | |
| image_matrix_json: ${{ steps.build-image-matrix.outputs.matrix_json }} | |
| runner: ${{ steps.pick-runner.outputs.runner }} | |
| grype_runner: ${{ steps.pick-runner.outputs.grype_runner }} | |
| default_gh_runner: ${{ steps.runners.outputs['default-gh-runner'] }} | |
| ref_kind: ${{ steps.git-ref.outputs.kind }} | |
| ref_name: ${{ steps.git-ref.outputs.name }} | |
| ref_sha: ${{ steps.git-ref.outputs.sha }} | |
| github_ref: ${{ steps.git-ref.outputs.github_ref }} | |
| variant_suffix_labels: ${{ steps.scan-profile.outputs.variant_suffix_labels }} | |
| steps: | |
| - name: Validate inputs | |
| env: | |
| EVENT_INPUTS_JSON: ${{ toJSON(github.event.inputs) }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| INPUT_DOCKER_TAG: ${{ env.RESOLVED_DOCKER_TAG }} | |
| INPUT_SCAN_WITH_TRIVY: ${{ env.RESOLVED_SCAN_WITH_TRIVY }} | |
| INPUT_SCAN_WITH_GRYPE: ${{ env.RESOLVED_SCAN_WITH_GRYPE }} | |
| run: | | |
| set -euo pipefail | |
| echo "::group::event inputs (${EVENT_NAME})" | |
| echo "${EVENT_INPUTS_JSON}" | |
| echo "::endgroup::" | |
| echo "github.ref_name=${GITHUB_REF_NAME} DOCKER_REGISTRY=${DOCKER_REGISTRY:-<unset>}" | |
| if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then | |
| dt="${INPUT_DOCKER_TAG//[[:space:]]/}" | |
| if [[ -z "${dt}" ]]; then | |
| echo "::error::docker_tag is required: set a non-empty container image tag." | |
| exit 1 | |
| fi | |
| fi | |
| if [[ -z "${DOCKER_REGISTRY:-}" ]]; then | |
| echo "::error::Organization / repository var DOCKER_REGISTRY is required; this workflow only pulls images from the registry (no local Gradle build path)." | |
| exit 1 | |
| fi | |
| if [[ "${INPUT_SCAN_WITH_TRIVY}" != "true" && "${INPUT_SCAN_WITH_GRYPE}" != "true" ]]; then | |
| echo "::error::Enable at least one of scan_with_trivy or scan_with_grype" | |
| exit 1 | |
| fi | |
| - name: Checkout (minimal) | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| fetch-tags: true | |
| sparse-checkout: | | |
| .github/security-scan-profiles.core.json | |
| .github/security-scan-profiles.saas.json | |
| .github/actions/determine-runners | |
| .github/scripts/security_scan_linear_sync.py | |
| .github/scripts/utils/security_scan_utils.py | |
| .github/scripts/utils/linear_sync_utils.py | |
| trivy.yaml | |
| sparse-checkout-cone-mode: false | |
| - name: Determine runners | |
| id: runners | |
| uses: ./.github/actions/determine-runners | |
| with: | |
| is-community-pr: "false" | |
| has-depot-label: "false" | |
| - name: Resolve security scan profile config | |
| run: | | |
| set -euo pipefail | |
| mkdir -p .github | |
| if [[ -f .github/security-scan-profiles.saas.json ]]; then | |
| : | |
| elif git cat-file -e "HEAD:.github/security-scan-profiles.saas.json" 2>/dev/null; then | |
| git show "HEAD:.github/security-scan-profiles.saas.json" > .github/security-scan-profiles.saas.json | |
| else | |
| echo '{}' > .github/security-scan-profiles.saas.json | |
| fi | |
| out=".github/security-scan-profiles.resolved.json" | |
| if jq -e 'type == "object" and (length > 0)' .github/security-scan-profiles.saas.json >/dev/null 2>&1; then | |
| cp .github/security-scan-profiles.saas.json "${out}" | |
| else | |
| cp .github/security-scan-profiles.core.json "${out}" | |
| fi | |
| - name: Apply scan profile | |
| id: scan-profile | |
| env: | |
| SCAN_PROFILE: ${{ env.RESOLVED_SCAN_PROFILE }} | |
| run: | | |
| set -euo pipefail | |
| CFG=".github/security-scan-profiles.resolved.json" | |
| SCAN_PROFILE="${SCAN_PROFILE//[[:space:]]/}" | |
| if [[ -z "${SCAN_PROFILE}" ]]; then | |
| echo "::error::scan_profile is empty" | |
| exit 1 | |
| fi | |
| if [[ "${SCAN_PROFILE}" == _* ]]; then | |
| echo "::error::Invalid scan_profile: ${SCAN_PROFILE}" | |
| exit 1 | |
| fi | |
| if ! jq -e --arg p "${SCAN_PROFILE}" '.[$p] | type == "object"' "${CFG}" >/dev/null; then | |
| echo "::error::Unknown or invalid scan_profile: ${SCAN_PROFILE} (see ${CFG})" | |
| exit 1 | |
| fi | |
| { | |
| echo "registry_repos=$(jq -r --arg p "${SCAN_PROFILE}" ' | |
| if (.[$p].registry.repos | type) == "object" then (.[$p].registry.repos | keys | sort | join(" ")) | |
| elif (.[$p].registry.repos | type) == "array" then (.[$p].registry.repos | join(" ")) | |
| else (.[$p].registry_repos // []) | join(" ") | |
| end | |
| ' "${CFG}")" | |
| echo "variant_suffix_labels=$(jq -r --arg p "${SCAN_PROFILE}" ' | |
| if (.[$p].registry.repos | type) == "object" then | |
| [ .[$p].registry.repos | to_entries[] | .value.variant_suffixes[]? | |
| | select(type == "string" and startswith("-") and length > 1) | |
| | ltrimstr("-") | |
| ] | unique | join(",") | |
| else | |
| [ ((.[$p].registry.variant_tag_suffixes // .[$p].registry_variant_tag_suffixes // {}) | .[] | .[]?) | |
| | select(type == "string" and startswith("-") and length > 1) | |
| | ltrimstr("-") | |
| ] | unique | join(",") | |
| end | |
| ' "${CFG}")" | |
| } >> "${GITHUB_OUTPUT}" | |
| # Linear ref label: use the repo default branch when scanning image tags `head` or `latest` | |
| # (nightly / default tip); otherwise use the provided docker tag as a tag ref (e.g. release). | |
| # Container pulls always use RESOLVED_DOCKER_TAG. | |
| - name: Resolve git ref (Linear) | |
| id: git-ref | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| RESOLVED_DOCKER_TAG: ${{ env.RESOLVED_DOCKER_TAG }} | |
| run: | | |
| set -euo pipefail | |
| commit="$(git rev-parse HEAD)" | |
| default_branch="$(gh api "repos/${GITHUB_REPOSITORY}" --jq '.default_branch')" | |
| if [[ -z "${default_branch}" || "${default_branch}" == "null" ]]; then | |
| echo "::error::Could not resolve repository default branch via GitHub API" | |
| exit 1 | |
| fi | |
| scan_tag="${RESOLVED_DOCKER_TAG//[[:space:]]/}" | |
| if [[ -z "${scan_tag}" ]]; then | |
| echo "::error::RESOLVED_DOCKER_TAG is empty (cannot set Linear ref label)" | |
| exit 1 | |
| fi | |
| lt="$(printf '%s' "${scan_tag}" | tr '[:upper:]' '[:lower:]')" | |
| if [[ "${lt}" == "head" || "${lt}" == "latest" ]]; then | |
| { | |
| echo "kind=branch" | |
| echo "name=${default_branch}" | |
| echo "github_ref=refs/heads/${default_branch}" | |
| echo "sha=${commit}" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "::notice::Linear ref label: default branch ${default_branch} (image tag ${scan_tag})" | |
| else | |
| { | |
| echo "kind=tag" | |
| echo "name=${scan_tag}" | |
| echo "github_ref=refs/tags/${scan_tag}" | |
| echo "sha=${commit}" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "::notice::Linear ref label: image tag ${scan_tag} (not head/latest)" | |
| fi | |
| - &ecr_login | |
| name: Login to Amazon ECR | |
| if: ${{ contains(env.DOCKER_REGISTRY, 'amazonaws.com') }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }} | |
| run: | | |
| set -euo pipefail | |
| [[ -z "${AWS_SECRET_ACCESS_KEY:-}" ]] && { echo "::notice::ECR secrets unset; private pulls may fail."; exit 0; } | |
| reg="${DOCKER_REGISTRY}" | |
| [[ "${reg}" =~ \.dkr\.ecr\.([a-z0-9-]+)\.amazonaws\.com ]] || { echo "::error::Bad DOCKER_REGISTRY for ECR"; exit 1; } | |
| export AWS_DEFAULT_REGION="${BASH_REMATCH[1]}" | |
| aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${reg}" | |
| - name: Resolve images from registry | |
| id: resolve-images | |
| env: | |
| DOCKER_TAG_INPUT: ${{ env.RESOLVED_DOCKER_TAG }} | |
| SCAN_PROFILE: ${{ env.RESOLVED_SCAN_PROFILE }} | |
| IMAGE_REPOS: ${{ steps.scan-profile.outputs.registry_repos }} | |
| VARIANT_SCOPE: ${{ env.RESOLVED_VARIANT_SCOPE }} | |
| run: | | |
| set -euo pipefail | |
| CFG=".github/security-scan-profiles.resolved.json" | |
| SCAN_PROFILE="${SCAN_PROFILE//[[:space:]]/}" | |
| export IMAGE_REPOS | |
| CANARY_REPO="$(jq -re --arg p "${SCAN_PROFILE}" '.[$p].canary_repo' "${CFG}")" \ | |
| || { echo "::error::Profile ${SCAN_PROFILE} is missing required field 'canary_repo' in ${CFG}"; exit 1; } | |
| REGISTRY_PREFIX="${DOCKER_REGISTRY}" | |
| probe() { docker manifest inspect "$1" >/dev/null 2>&1; } | |
| set_reg_prefix() { | |
| if [[ "${REGISTRY_PREFIX}" == *"amazonaws.com"* ]]; then reg="ecr"; else reg="dockerhub"; fi | |
| prefix="${REGISTRY_PREFIX}/" | |
| } | |
| if [[ -z "${REGISTRY_PREFIX}" ]]; then | |
| echo "::error::vars.DOCKER_REGISTRY is not set. This workflow only resolves images from the container registry (no local build path)." | |
| exit 1 | |
| fi | |
| scan_tag="${DOCKER_TAG_INPUT//[[:space:]]/}" | |
| if [[ -z "${scan_tag}" ]]; then | |
| echo "::error::docker_tag is empty after trimming whitespace." | |
| exit 1 | |
| fi | |
| reg="" prefix="" | |
| set_reg_prefix | |
| if ! probe "${REGISTRY_PREFIX}/${CANARY_REPO}:${scan_tag}"; then | |
| echo "::error::No manifest ${REGISTRY_PREFIX}/${CANARY_REPO}:${scan_tag} (${CANARY_REPO} canary; check the tag, pull permissions, and DOCKER_REGISTRY)." | |
| exit 1 | |
| fi | |
| : > image-list.txt | |
| skip_count=0 | |
| for repo in ${IMAGE_REPOS}; do | |
| if [[ "${VARIANT_SCOPE}" == "primary" ]]; then | |
| _ref="${prefix}${repo}:${scan_tag}" | |
| if probe "${_ref}"; then | |
| echo "${_ref}" >> image-list.txt | |
| else | |
| echo "::warning::Skipping ${_ref}: image not found in registry." | |
| skip_count=$((skip_count + 1)) | |
| fi | |
| continue | |
| fi | |
| _use=() | |
| mapfile -t _use < <(jq -r --arg p "${SCAN_PROFILE}" --arg r "${repo}" --arg vs "${VARIANT_SCOPE}" ' | |
| . as $root | |
| | | |
| def repo_suffixes: | |
| ($root[$p].registry.repos) as $R | |
| | if ($R | type) == "object" then (($R[$r] // {}) | .variant_suffixes // []) | |
| else ((($root[$p].registry.variant_tag_suffixes // $root[$p].registry_variant_tag_suffixes // {})[$r]) // []) | |
| end; | |
| def scope_filter: | |
| (($root[$p].registry.variant_scope_suffixes // {})[$vs]); | |
| if $vs == "all" then | |
| repo_suffixes | .[]? | |
| elif $vs == "primary" then | |
| empty | |
| else | |
| (scope_filter) as $f | |
| | (if ($f | type) == "array" then [repo_suffixes[] as $s | select($f | index($s)) | $s] | |
| elif $f == "*" then repo_suffixes | |
| else [repo_suffixes[] as $s | select($s == ("-" + $vs)) | $s] | |
| end) | |
| | .[]? | |
| end | |
| ' "${CFG}" || true) | |
| if [[ ${#_use[@]} -eq 0 ]]; then | |
| if [[ "${VARIANT_SCOPE}" == "all" ]]; then | |
| _ref="${prefix}${repo}:${scan_tag}" | |
| if probe "${_ref}"; then | |
| echo "${_ref}" >> image-list.txt | |
| else | |
| echo "::warning::Skipping ${_ref}: image not found in registry." | |
| skip_count=$((skip_count + 1)) | |
| fi | |
| else | |
| echo "::warning::Skip ${repo}: no \"${VARIANT_SCOPE}\" match for registry.repos.${repo}.variant_suffixes (profile ${SCAN_PROFILE})." | |
| fi | |
| continue | |
| fi | |
| for _suf in "${_use[@]}"; do | |
| _ref="${prefix}${repo}:${scan_tag}${_suf}" | |
| if probe "${_ref}"; then | |
| echo "${_ref}" >> image-list.txt | |
| else | |
| echo "::warning::Skipping ${_ref}: variant image not found in registry." | |
| skip_count=$((skip_count + 1)) | |
| fi | |
| done | |
| done | |
| if [[ "${skip_count}" -gt 0 ]]; then | |
| echo "::warning::Skipped ${skip_count} image(s) not found in registry." | |
| fi | |
| if [[ ! -s image-list.txt ]]; then | |
| echo "::error::No images to pull: VARIANT_SCOPE=${VARIANT_SCOPE} did not match any repo (check registry.repos and variant_suffixes)." | |
| exit 1 | |
| fi | |
| echo "Canary ${CANARY_REPO}:${scan_tag} (${reg}); $(wc -l < image-list.txt) image ref(s)." | |
| head -30 image-list.txt | |
| { echo "scan_tag=${scan_tag}"; echo "registry_source=${reg}"; } >> "$GITHUB_OUTPUT" | |
| - name: Scan group matrix JSON (one matrix cell per image repository) | |
| id: build-image-matrix | |
| if: success() | |
| run: | | |
| set -euo pipefail | |
| # One workflow matrix leg per *repository* (the ref without the last ":tag" segment). Tag variants | |
| # (e.g. :head vs :head-slim) are scanned on the same runner, serially, per scanner. Different repos stay parallel. | |
| m="$( | |
| jq -Rs 'split("\n") | |
| | map(select(length > 0)) | |
| | group_by( | |
| if test(":") then sub(":([^:]+)$";"") else . end | |
| ) | |
| | map({ | |
| group: (.[0] | (if test(":") then sub(":([^:]+)$";"") else . end)), | |
| images: sort, | |
| label: ( | |
| .[0] | |
| | (if test(":") then sub(":([^:]+)$";"") else . end) | |
| | split("/") | |
| | .[-1] | |
| ) | |
| }) | |
| ' < image-list.txt | |
| )" | |
| echo "scan_group_count=$(jq 'length' <<< "${m}")" "image_count=$(jq '[.[].images | length] | add // 0' <<< "${m}")" | |
| { | |
| echo "matrix_json<<__MATRIX_JSON__" | |
| echo "${m}" | |
| echo "__MATRIX_JSON__" | |
| } >> "${GITHUB_OUTPUT}" | |
| - name: Upload image list | |
| if: success() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: security-scan-image-list | |
| path: image-list.txt | |
| if-no-files-found: error | |
| retention-days: ${{ env.RESOLVED_SCAN_ARTIFACT_RETENTION_DAYS }} | |
| - name: Select runner for main job | |
| id: pick-runner | |
| env: | |
| DEPOT_SET: ${{ steps.runners.outputs['depot-config-set'] }} | |
| DEFAULT_GH: ${{ steps.runners.outputs['default-gh-runner'] }} | |
| DEPOT_LARGE: ${{ steps.runners.outputs['depot-runner-large'] }} | |
| GRYPE_OVERR: ${{ vars.SECURITY_SCAN_GRYPE_RUNS_ON }} | |
| run: | | |
| set -euo pipefail | |
| echo "runner=${DEFAULT_GH}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Trivy on ${DEFAULT_GH} (from determine-runners). Grype on grype_runner (see next notice)." | |
| if [[ -n "${GRYPE_OVERR}" ]]; then | |
| echo "grype_runner=${GRYPE_OVERR}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Grype: ${GRYPE_OVERR} (vars.SECURITY_SCAN_GRYPE_RUNS_ON)." | |
| elif [[ "${DEPOT_SET}" == "true" ]]; then | |
| echo "grype_runner=${DEPOT_LARGE}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Grype: ${DEPOT_LARGE} (from determine-runners; override: SECURITY_SCAN_GRYPE_RUNS_ON)." | |
| else | |
| echo "grype_runner=${DEFAULT_GH}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Grype: ${DEFAULT_GH} (no Depot). For Depot, set vars.DEPOT_PROJECT_ID; or set SECURITY_SCAN_GRYPE_RUNS_ON to an explicit runner." | |
| fi | |
| scan-trivy: | |
| name: Trivy (${{ matrix.label }}) | |
| needs: [select-runner] | |
| # Boolean inputs are booleans in expressions, not the string 'true' — compare truthy. | |
| if: github.event_name == 'workflow_run' || inputs.scan_with_trivy | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 32 | |
| matrix: | |
| include: ${{ fromJson(needs.select-runner.outputs.image_matrix_json) }} | |
| runs-on: ${{ needs.select-runner.outputs.runner }} | |
| timeout-minutes: 300 | |
| env: | |
| DOCKER_REGISTRY: ${{ vars.DOCKER_REGISTRY }} | |
| steps: | |
| - name: Checkout (Trivy config) | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| fetch-tags: true | |
| sparse-checkout: | | |
| trivy.yaml | |
| sparse-checkout-cone-mode: false | |
| - *ecr_login | |
| - name: Install Trivy | |
| uses: aquasecurity/setup-trivy@3fb12ec12f41e471780db15c232d5dd185dcb514 # v0.2.6 | |
| with: | |
| version: ${{ vars.TRIVY_CLI_VERSION || 'v0.70.0' }} | |
| cache: true | |
| - name: Trivy vulnerability DB cache key | |
| id: trivy-db-cache | |
| run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" | |
| - name: Restore Trivy vulnerability DB cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: ~/.cache/trivy | |
| key: trivy-vuln-db-${{ runner.os }}-${{ steps.trivy-db-cache.outputs.week }} | |
| restore-keys: | | |
| trivy-vuln-db-${{ runner.os }}- | |
| - name: Artifact name slug | |
| id: trivy-art | |
| run: | | |
| s=$(printf '%s' "${REPO_GROUP}" | sha256sum | head -c16) | |
| echo "slug=${s}" >> "$GITHUB_OUTPUT" | |
| env: | |
| REPO_GROUP: ${{ matrix.group }} | |
| - name: Run Trivy | |
| id: trivy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }} | |
| TRIVY_DB_REPOSITORY_OVERRIDE: ${{ vars.TRIVY_DB_REPOSITORY }} | |
| TRIVY_JAVA_DB_REPOSITORY_OVERRIDE: ${{ vars.TRIVY_JAVA_DB_REPOSITORY }} | |
| TRIVY_ECR_PTR_PREFIX: ${{ vars.TRIVY_ECR_PTR_PREFIX }} | |
| TRIVY_GHCR_ECR_PTR_PREFIX: ${{ vars.TRIVY_GHCR_ECR_PTR_PREFIX }} | |
| SCAN_SEVERITY_LEVELS: ${{ env.RESOLVED_SCAN_SEVERITY_LEVELS }} | |
| IMAGES_JSON: ${{ toJson(matrix.images) }} | |
| REPO_GROUP: ${{ matrix.group }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p scan-reports/trivy | |
| PTR_PREFIX="${TRIVY_ECR_PTR_PREFIX:-public-ecr-aws}" | |
| GHCR_PT="${TRIVY_GHCR_ECR_PTR_PREFIX:-ghcr}" | |
| PUB_DB="public.ecr.aws/aquasecurity/trivy-db:2" | |
| PUB_DB_FALL="ghcr.io/aquasecurity/trivy-db:2" | |
| PUB_JDB="public.ecr.aws/aquasecurity/trivy-java-db:1" | |
| PUB_JDB_FALL="ghcr.io/aquasecurity/trivy-java-db:1" | |
| if [[ -n "${TRIVY_DB_REPOSITORY_OVERRIDE:-}" ]]; then | |
| export TRIVY_DB_REPOSITORY="${TRIVY_DB_REPOSITORY_OVERRIDE}" | |
| elif [[ -n "${DOCKER_REGISTRY:-}" ]] && [[ "${DOCKER_REGISTRY}" == *dkr.ecr.*.amazonaws.com* ]]; then | |
| PTR_DB="${DOCKER_REGISTRY}/${PTR_PREFIX}/aquasecurity/trivy-db:2" | |
| PTR_GHCR_DB="${DOCKER_REGISTRY}/${GHCR_PT}/aquasecurity/trivy-db:2" | |
| echo "::notice::Trivy DB: ECR pull-through (Public ECR then GHCR), then direct public mirrors." | |
| export TRIVY_DB_REPOSITORY="${PTR_DB},${PTR_GHCR_DB},${PUB_DB},${PUB_DB_FALL}" | |
| else | |
| export TRIVY_DB_REPOSITORY="${PUB_DB},${PUB_DB_FALL}" | |
| fi | |
| if [[ -n "${TRIVY_JAVA_DB_REPOSITORY_OVERRIDE:-}" ]]; then | |
| export TRIVY_JAVA_DB_REPOSITORY="${TRIVY_JAVA_DB_REPOSITORY_OVERRIDE}" | |
| elif [[ -n "${DOCKER_REGISTRY:-}" ]] && [[ "${DOCKER_REGISTRY}" == *dkr.ecr.*.amazonaws.com* ]]; then | |
| PTR_JDB="${DOCKER_REGISTRY}/${PTR_PREFIX}/aquasecurity/trivy-java-db:1" | |
| PTR_GHCR_JDB="${DOCKER_REGISTRY}/${GHCR_PT}/aquasecurity/trivy-java-db:1" | |
| export TRIVY_JAVA_DB_REPOSITORY="${PTR_JDB},${PTR_GHCR_JDB},${PUB_JDB},${PUB_JDB_FALL}" | |
| else | |
| export TRIVY_JAVA_DB_REPOSITORY="${PUB_JDB},${PUB_JDB_FALL}" | |
| fi | |
| echo "::group::Trivy: prefetch vulnerability DBs (shared for this image repository)" | |
| trivy image --download-db-only | |
| trivy image --download-java-db-only | |
| echo "::endgroup::" | |
| mapfile -t imgs < <(jq -r '.[]' <<< "${IMAGES_JSON}") | |
| fail_count=0 skip_count=0 | |
| for img in "${imgs[@]}"; do | |
| [[ -z "${img}" ]] && continue | |
| if ! docker manifest inspect "${img}" >/dev/null 2>&1; then | |
| echo "::warning::Skipping Trivy scan for ${img}: image not found in registry." | |
| skip_count=$((skip_count + 1)) | |
| continue | |
| fi | |
| if ! trivy image \ | |
| --config trivy.yaml \ | |
| --ignore-unfixed \ | |
| --scanners vuln \ | |
| --pkg-types os,library \ | |
| --severity "${SCAN_SEVERITY_LEVELS}" \ | |
| --skip-db-update \ | |
| --skip-java-db-update \ | |
| --format json \ | |
| --output "scan-reports/trivy/trivy-tmp.json" \ | |
| "${img}"; then | |
| fail_count=$((fail_count + 1)) | |
| continue | |
| fi | |
| img_key="$(echo "${img}" | sed -E 's#[^A-Za-z0-9._-]+#_#g')" | |
| out="scan-reports/trivy/trivy-${img_key}.json" | |
| cp "scan-reports/trivy/trivy-tmp.json" "${out}" | |
| done | |
| rm -f scan-reports/trivy/trivy-tmp.json | |
| ls -la scan-reports/trivy/ | |
| if [[ "${skip_count}" -gt 0 ]]; then | |
| echo "::warning::Trivy skipped ${skip_count} image(s) not found in registry (group ${REPO_GROUP})." | |
| fi | |
| if [[ "${fail_count}" -gt 0 ]]; then | |
| echo "::error::Trivy failed for ${fail_count} image(s) in group ${REPO_GROUP}" | |
| exit 1 | |
| fi | |
| - name: Upload Trivy reports | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: security-scan-trivy--${{ steps.trivy-art.outputs.slug }} | |
| path: scan-reports/trivy | |
| if-no-files-found: warn | |
| retention-days: ${{ env.RESOLVED_SCAN_ARTIFACT_RETENTION_DAYS }} | |
| - name: Generate Trivy SARIF reports | |
| if: success() && env.RESOLVED_UPLOAD_TRIVY_TO_GITHUB == 'true' | |
| env: | |
| SCAN_SEVERITY_LEVELS: ${{ env.RESOLVED_SCAN_SEVERITY_LEVELS }} | |
| IMAGES_JSON: ${{ toJson(matrix.images) }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p scan-reports/trivy-sarif | |
| mapfile -t imgs < <(jq -r '.[]' <<< "${IMAGES_JSON}") | |
| for img in "${imgs[@]}"; do | |
| [[ -z "${img}" ]] && continue | |
| if ! docker manifest inspect "${img}" >/dev/null 2>&1; then | |
| echo "::warning::Skipping SARIF generation for ${img}: image not found in registry." | |
| continue | |
| fi | |
| img_key="$(echo "${img}" | sed -E 's#[^A-Za-z0-9._-]+#_#g')" | |
| trivy image \ | |
| --config trivy.yaml \ | |
| --ignore-unfixed \ | |
| --scanners vuln \ | |
| --pkg-types os,library \ | |
| --severity "${SCAN_SEVERITY_LEVELS}" \ | |
| --skip-db-update \ | |
| --skip-java-db-update \ | |
| --format sarif \ | |
| --output "scan-reports/trivy-sarif/trivy-${img_key}.sarif" \ | |
| "${img}" || true | |
| done | |
| - name: Merge Trivy SARIF files into single run | |
| if: success() && env.RESOLVED_UPLOAD_TRIVY_TO_GITHUB == 'true' | |
| run: | | |
| set -euo pipefail | |
| shopt -s nullglob | |
| sarif_files=(scan-reports/trivy-sarif/*.sarif) | |
| if [[ ${#sarif_files[@]} -eq 0 ]]; then | |
| echo "::warning::No SARIF files to merge." | |
| exit 0 | |
| fi | |
| jq -s ' | |
| .[0].runs[0] as $base_run | | |
| { | |
| "$schema": .[0]["$schema"], | |
| version: .[0].version, | |
| runs: [{ | |
| tool: { | |
| driver: { | |
| fullName: $base_run.tool.driver.fullName, | |
| informationUri: $base_run.tool.driver.informationUri, | |
| name: $base_run.tool.driver.name, | |
| version: $base_run.tool.driver.version, | |
| rules: [.[].runs[0].tool.driver.rules // [] | .[]] | unique_by(.id) | |
| } | |
| }, | |
| results: [.[].runs[0].results // [] | .[]], | |
| columnKind: $base_run.columnKind | |
| } + if $base_run.originalUriBaseIds then | |
| {originalUriBaseIds: $base_run.originalUriBaseIds} | |
| else {} end | |
| ] | |
| } | |
| ' "${sarif_files[@]}" > scan-reports/trivy-merged.sarif | |
| echo "Merged ${#sarif_files[@]} SARIF file(s) into scan-reports/trivy-merged.sarif" | |
| - name: Upload Trivy SARIF to GitHub Security tab | |
| if: success() && env.RESOLVED_UPLOAD_TRIVY_TO_GITHUB == 'true' | |
| uses: github/codeql-action/upload-sarif@0e9f55954318745b37b7933c693bc093f7336125 # v4.35.1 | |
| with: | |
| sarif_file: scan-reports/trivy-merged.sarif | |
| category: trivy-${{ matrix.label }} | |
| ref: ${{ needs.select-runner.outputs.github_ref }} | |
| sha: ${{ needs.select-runner.outputs.ref_sha }} | |
| scan-grype: | |
| name: Grype (${{ matrix.label }}) | |
| needs: [select-runner] | |
| if: github.event_name == 'workflow_run' || inputs.scan_with_grype | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 32 | |
| matrix: | |
| include: ${{ fromJson(needs.select-runner.outputs.image_matrix_json) }} | |
| runs-on: ${{ needs.select-runner.outputs.grype_runner }} | |
| timeout-minutes: 300 | |
| env: | |
| DOCKER_REGISTRY: ${{ vars.DOCKER_REGISTRY }} | |
| steps: | |
| - *ecr_login | |
| - name: Install Grype | |
| id: grype-install | |
| uses: anchore/scan-action/download-grype@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 | |
| with: | |
| grype-version: ${{ vars.GRYPE_CLI_VERSION || 'v0.111.1' }} | |
| - name: Artifact name slug | |
| id: grype-art | |
| run: | | |
| s=$(printf '%s' "${REPO_GROUP}" | sha256sum | head -c16) | |
| echo "slug=${s}" >> "$GITHUB_OUTPUT" | |
| env: | |
| REPO_GROUP: ${{ matrix.group }} | |
| - name: Run Grype | |
| id: grype | |
| env: | |
| SCAN_SEVERITY_LEVELS: ${{ env.RESOLVED_SCAN_SEVERITY_LEVELS }} | |
| GRYPE_CMD: ${{ steps.grype-install.outputs.cmd }} | |
| IMAGES_JSON: ${{ toJson(matrix.images) }} | |
| REPO_GROUP: ${{ matrix.group }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p scan-reports/grype scan-reports-raw/grype | |
| G="${GRYPE_CMD}" | |
| if [[ -z "${G}" ]]; then | |
| echo "::error::Grype binary path empty (install step output)" | |
| exit 1 | |
| fi | |
| mapfile -t imgs < <(jq -r '.[]' <<< "${IMAGES_JSON}") | |
| fail_count=0 skip_count=0 | |
| for img in "${imgs[@]}"; do | |
| [[ -z "${img}" ]] && continue | |
| if ! docker manifest inspect "${img}" >/dev/null 2>&1; then | |
| echo "::warning::Skipping Grype scan for ${img}: image not found in registry." | |
| skip_count=$((skip_count + 1)) | |
| continue | |
| fi | |
| img_key="$(echo "${img}" | sed -E 's#[^A-Za-z0-9._-]+#_#g')" | |
| out_raw="scan-reports-raw/grype/grype-${img_key}.json" | |
| out="scan-reports/grype/grype-${img_key}.json" | |
| if ! "${G}" "${img}" -o json --only-fixed --by-cve > "${out_raw}"; then | |
| fail_count=$((fail_count + 1)) | |
| continue | |
| fi | |
| jq --arg sev "${SCAN_SEVERITY_LEVELS}" ' | |
| def allowed: ($sev | split(",") | map(ascii_upcase | gsub("^\\s+|\\s+$"; ""))); | |
| if (.matches | type) == "array" then | |
| .matches |= map( | |
| select( | |
| ((.vulnerability.severity // "") | ascii_upcase) as $s | |
| | (allowed | index($s)) != null | |
| ) | |
| ) | |
| else | |
| . | |
| end | |
| ' "${out_raw}" > "${out}" | |
| done | |
| ls -la scan-reports/grype/ | |
| if [[ "${skip_count}" -gt 0 ]]; then | |
| echo "::warning::Grype skipped ${skip_count} image(s) not found in registry (group ${REPO_GROUP})." | |
| fi | |
| if [[ "${fail_count}" -gt 0 ]]; then | |
| echo "::error::Grype failed for ${fail_count} image(s) in group ${REPO_GROUP}" | |
| exit 1 | |
| fi | |
| - name: Package Grype merged + raw (single artifact) | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| mkdir -p grype-artifact/scan-reports grype-artifact/scan-reports-raw | |
| if [[ -d scan-reports/grype ]]; then cp -a scan-reports/grype grype-artifact/scan-reports/; fi | |
| if [[ -d scan-reports-raw/grype ]]; then cp -a scan-reports-raw/grype grype-artifact/scan-reports-raw/; fi | |
| - name: Upload Grype reports | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: security-scan-grype--${{ steps.grype-art.outputs.slug }} | |
| path: grype-artifact | |
| if-no-files-found: warn | |
| retention-days: ${{ env.RESOLVED_SCAN_ARTIFACT_RETENTION_DAYS }} | |
| sync-to-linear: | |
| name: Linear sync | |
| needs: [select-runner, scan-trivy, scan-grype] | |
| if: | | |
| always() && !cancelled() && needs.select-runner.result == 'success' && ( | |
| needs['scan-trivy'].result == 'skipped' || | |
| needs['scan-trivy'].result == 'success' || | |
| needs['scan-trivy'].result == 'failure' | |
| ) && ( | |
| needs['scan-grype'].result == 'skipped' || | |
| needs['scan-grype'].result == 'success' || | |
| needs['scan-grype'].result == 'failure' | |
| ) | |
| runs-on: ${{ needs.select-runner.outputs.default_gh_runner }} | |
| timeout-minutes: 60 | |
| env: | |
| DOCKER_REGISTRY: ${{ vars.DOCKER_REGISTRY }} | |
| steps: | |
| # Cone checkout of `.github/scripts` for the sync entrypoint and `utils/` imports. | |
| - name: Checkout (sync script) | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| fetch-tags: true | |
| sparse-checkout-cone-mode: true | |
| sparse-checkout: | | |
| .github/scripts | |
| - name: Ensure report directories | |
| run: | | |
| set -euo pipefail | |
| mkdir -p scan-reports/trivy scan-reports/grype scan-reports-raw/grype | |
| - name: Download Trivy report artifacts (per-image matrix) | |
| if: env.RESOLVED_SCAN_WITH_TRIVY == 'true' | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 | |
| with: | |
| pattern: security-scan-trivy--* | |
| merge-multiple: true | |
| path: scan-reports/trivy | |
| - name: Download Grype report artifacts (per-image matrix) | |
| if: env.RESOLVED_SCAN_WITH_GRYPE == 'true' | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 | |
| with: | |
| pattern: security-scan-grype--* | |
| merge-multiple: true | |
| path: scan-reports/grype | |
| - name: Download Trivy raw report artifacts (per-image matrix) | |
| if: env.RESOLVED_SCAN_WITH_TRIVY == 'true' | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 | |
| with: | |
| pattern: security-scan-trivy-raw--* | |
| merge-multiple: true | |
| path: scan-reports-raw/trivy | |
| - name: Download Grype raw report artifacts (per-image matrix) | |
| if: env.RESOLVED_SCAN_WITH_GRYPE == 'true' | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 | |
| with: | |
| pattern: security-scan-grype-raw--* | |
| merge-multiple: true | |
| path: scan-reports-raw/grype | |
| - name: Sync to Linear | |
| env: | |
| LINEAR_SECURITY_SCAN_API_KEY: ${{ secrets.LINEAR_SECURITY_SCAN_API_KEY }} | |
| SCANNER: trivy_grype | |
| LINEAR_TEAM_ID: ${{ vars.LINEAR_TEAM_ID || '18ea98cd-cf5f-4296-a16d-b16426c70307' }} | |
| # Optional workflow-state UUID for new issues; if unset and triage is enabled for the team, | |
| # the script sets stateId to Team.triageIssueState (integrations/non-members triage inbox). | |
| LINEAR_ISSUE_STATE_ID: ${{ vars.LINEAR_ISSUE_STATE_ID }} | |
| LINEAR_LABEL_IDS: ${{ vars.LINEAR_LABEL_IDS || '623bed97-f0a8-47aa-b673-2f533251f096' }} | |
| SCAN_REF_KIND: ${{ needs.select-runner.outputs.ref_kind }} | |
| SCAN_REF_NAME: ${{ needs.select-runner.outputs.ref_name }} | |
| GITHUB_SHA: ${{ needs.select-runner.outputs.ref_sha }} | |
| GITHUB_SERVER_URL: ${{ github.server_url }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| SCAN_WITH_TRIVY: ${{ env.RESOLVED_SCAN_WITH_TRIVY }} | |
| SCAN_WITH_GRYPE: ${{ env.RESOLVED_SCAN_WITH_GRYPE }} | |
| DATAHUB_SCAN_SEVERITIES: ${{ env.RESOLVED_SCAN_SEVERITY_LEVELS }} | |
| DATAHUB_VARIANT_TAG_SUFFIXES: ${{ needs.select-runner.outputs.variant_suffix_labels }} | |
| run: bash .github/scripts/ci/run_security_scan_linear_sync.sh | |
| - name: Fail on scanner errors | |
| if: | | |
| (env.RESOLVED_SCAN_WITH_TRIVY == 'true' && needs['scan-trivy'].result == 'failure') || | |
| (env.RESOLVED_SCAN_WITH_GRYPE == 'true' && needs['scan-grype'].result == 'failure') | |
| run: | | |
| echo "::error::Matrix scan job(s) failed (Trivy: ${{ needs['scan-trivy'].result }}; Grype: ${{ needs['scan-grype'].result }})" | |
| exit 1 |