config.yaml

x-ref:
  default-validations: &default-validations
    max_f1_regression: 0.0
    max_new_false_negatives: 0
    max_unlabeled_percent: 10
    max_year: 2021
    candidate_tool_label: custom-db
yardstick:
  default_max_year: 2021

  tools:
    - name: syft
      # note: if there is ever a problem with the syft version, it can be pinned explicitly here (instead of "latest")
      version: latest
      produces: SBOM
      refresh: false

    - name: grype
      label: custom-db
      # allowed values:
      #  - "latest" to use the latest released grype
      #  - a released version name (e.g. "v0.52.1")
      #  - a branch name (e.g. "dev-fix-foo")
      #  - a repo reference and optional "@branch" (e.g. "github.com/my-user-fork/grype@dev-fix-foo")
      # Note:
      #  - ALWAYS leave the "import-db" annotation as-is
      #  - this version should ALWAYS match that of the other "grype" tool below
      version: main+import-db=build/vulnerability.db
      takes: SBOM

    - name: grype
      # allowed values:
      #  - "latest" to use the latest released grype
      #  - a released version name (e.g. "v0.52.1")
      #  - a branch name (e.g. "dev-fix-foo")
      #  - a repo reference and optional "@branch" (e.g. "github.com/my-user-fork/grype@dev-fix-foo")
      # Note:
      #  - this version should ALWAYS match that of the other "grype" tool above
      version: main+import-db=https://grype.anchore.io/databases/v6/vulnerability-db_v6.0.2_2025-07-10T01:31:11Z_1752120925.tar.zst
      takes: SBOM
      label: reference

grype_db:
  # values:
  #  - "latest" to use the latest released grype-db
  #  - a released version name (e.g. "v0.15.2")
  #  - a branch name (e.g. "dev-fix-foo")
  #  - a repo reference and optional "@branch" (e.g. "my-user-fork/grype-db@dev-fix-foo")
  #  - a local file path (e.g. "file://~/code/grype-db")
  version: main

tests:

  - provider: alpine
    additional_providers:
      - name: nvd
        use_cache: true
    images:
      - docker.io/alpine:3.2@sha256:ddac200f3ebc9902fb8cfcd599f41feb2151f1118929da21bcef57dc276975f9
      - docker.io/anchore/test_images:alpine-package-cpe-vuln-match-bd0aaef@sha256:0825acea611c7c5cc792bc7cc20de44d7413fd287dc5afc4aab9c1891d037b4f
    expected_namespaces:
      - alpine:distro:alpine:3.2
      - alpine:distro:alpine:3.3
      - alpine:distro:alpine:3.4
      - alpine:distro:alpine:3.5
      - alpine:distro:alpine:3.6
      - alpine:distro:alpine:3.7
      - alpine:distro:alpine:3.8
      - alpine:distro:alpine:3.9
      - alpine:distro:alpine:3.10
      - alpine:distro:alpine:3.11
      - alpine:distro:alpine:3.12
      - alpine:distro:alpine:3.13
      - alpine:distro:alpine:3.14
      - alpine:distro:alpine:3.15
      - alpine:distro:alpine:3.16
      - alpine:distro:alpine:3.17
      - alpine:distro:alpine:3.18
      - alpine:distro:alpine:3.19
      - alpine:distro:alpine:3.20
      - alpine:distro:alpine:3.21
      - alpine:distro:alpine:3.22
      - alpine:distro:alpine:edge
      - nvd:cpe # alpine lists fixes to NVD entries, so NVD entries are also expected
    validations:
      - *default-validations

  - provider: echo
    additional_providers:
      - name: nvd
        use_cache: true
    images:
      - ghcr.io/buildecho/scanner-test:latest@sha256:60557350ad6976dad3b88d891de8f090b20b3271c660272d30d44b5d07b23edc
    expected_namespaces:
      - echo:distro:echo:rolling
    validations:
      - *default-validations

  - provider: amazon
    validations:
      - <<: *default-validations
        # TODO: docker.io/amazonlinux:2@sha256:1301cc9f889f21dc45733df9e58034ac1c318202b4b0f0a08d88b3fdc03004de
        # has no matches before 2022. Label more things, move max_year to 2022, and then
        # change fail_on_empty_match_set back to true (the default).
        fail_on_empty_match_set: false
    images:
      - docker.io/amazonlinux:2@sha256:1301cc9f889f21dc45733df9e58034ac1c318202b4b0f0a08d88b3fdc03004de
      - docker.io/anchore/test_images:vulnerabilities-amazonlinux-2-5c26ce9@sha256:cf742eca189b02902a0a7926ac3fbb423e799937bf4358b0d2acc6cc36ab82aa
    expected_namespaces:
      - amazon:distro:amazonlinux:2
      - amazon:distro:amazonlinux:2022
      - amazon:distro:amazonlinux:2023

  - provider: chainguard
    additional_providers:
      - name: nvd
        use_cache: true
    additional_trigger_globs:
      # this provider imports and uses the wolfi provider code
      - src/vunnel/providers/wolfi/**
    images:
      - ghcr.io/chainguard-images/scanner-test:latest@sha256:59bddc101fba0c45d5c093575c6bc5bfee7f0e46ff127e6bb4e5acaaafb525f9
    expected_namespaces:
      - chainguard:distro:chainguard:rolling
    validations:
      - *default-validations

  - provider: chainguard-libraries
    additional_providers:
      - name: github
        use_cache: true
    images:
      - ghcr.io/chainguard-images/scanner-test:python-library-aiohttp-chainguard@sha256:046b2c1b2df1e496eb3abf9e21224ffd879d92abc2bd57401456764d9aa8ddf1
    validations:
      - <<: *default-validations
        max_year: 2025

  - provider: debian
    # ideally we would not use cache, however, the in order to test if we are properly keeping the processing
    # of legacy information that is in the debian data cache (for debian 7, 8, and 9) we must test with
    # cache enabled.
    use_cache: true
    images:
      - docker.io/debian:7@sha256:81e88820a7759038ffa61cff59dfcc12d3772c3a2e75b7cfe963c952da2ad264
      - ghcr.io/anchore/test-images/bitnami/spark:3.2.4-debian-11-r8@sha256:267d5a6345636710b4b57b7fe981c9760203e7e092c705416310ea30a9806d74
    expected_namespaces:
      - debian:distro:debian:7
      - debian:distro:debian:8
      - debian:distro:debian:9
      - debian:distro:debian:10
      - debian:distro:debian:11
      - debian:distro:debian:12
      - debian:distro:debian:13
      - debian:distro:debian:14
      - debian:distro:debian:unstable

  - provider: github
    additional_providers:
      # we need to convert GHSAs to CVEs so that we can filter based on date
      - name: nvd
        use_cache: true
      # note: the base images for most of the github test images are alpine and we are including the NVD namespace.
      - name: alpine
        use_cache: true
      - name: wolfi
        use_cache: true
    additional_trigger_globs:
      - src/vunnel/utils/fdb.py
    images:
      - docker.io/anchore/test_images:java-56d52bc@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da
      - docker.io/anchore/test_images:npm-56d52bc@sha256:ba42ded8613fc643d407a050faf5ab48cfb405ad3ef2015bf6feeb5dff44738d
      - docker.io/anchore/test_images:gems-56d52bc@sha256:5763c8a225f950961bf01ddec68e36f18e236130e182f2b9290a6e03b9777bfe
      - docker.io/anchore/test_images:golang-56d52bc@sha256:d1819e59e89e8ea90073460acb4ebb2ee18ccead9fa880dae91e8fc61b19ca1c
      - docker.io/anchore/test_images:grype-quality-node-d89207b@sha256:f56164678054e5eb59ab838367373a49df723b324617b1ba6de775749d7f91d4
      - docker.io/anchore/test_images:grype-quality-python-d89207b@sha256:b2b58a55c0b03c1626d2aaae2add9832208b02124dda7b7b41811e14f0fb272c
      - docker.io/anchore/test_images:grype-quality-java-d89207b@sha256:b3534fc2e37943136d5b54e3a58b55d4ccd4363d926cf7aa5bf55a524cf8275b
      - docker.io/anchore/test_images:grype-quality-golang-d89207b@sha256:7536ee345532f674ec9e448e3768db4e546c48220ba2b6ec9bc9cfbfb3b7b74a
      - docker.io/anchore/test_images:grype-quality-ruby-d89207b@sha256:1a5a5f870924e88a6f0f2b8089cf276ef0a79b5244a052cdfe4a47bb9e5a2c10
    expected_namespaces:
      - github:language:dart
      - github:language:dotnet
      - github:language:go
      - github:language:java
      - github:language:javascript
      - github:language:php
      - github:language:python
      - github:language:ruby
      - github:language:rust
      - github:language:swift
    validations:
      - *default-validations

  - provider: mariner
    images:
      - mcr.microsoft.com/cbl-mariner/base/core:2.0.20220731-amd64@sha256:3c0f7e103ff3c39e81e7c9c042d2b321d833fb6d26d8636567f7d88a6bdde74a
      - docker.io/anchore/test_images:azurelinux3-63671fe@sha256:2d761ba36575ddd4e07d446f4f2a05448298c20e5bdcd3dedfbbc00f9865240d
    expected_namespaces:
      - mariner:distro:mariner:1.0
      - mariner:distro:mariner:2.0
      - mariner:distro:azurelinux:3.0
    validations:
      - <<: *default-validations
        max_year: 2022 # important - Azure Linux 3 doesn't have much to match on going back to 2021
        candidate_tool_label: custom-db

  - provider: minimos
    additional_providers:
      - name: nvd
        use_cache: true
      - name: github
        use_cache: true
    images:
      - docker.io/dimastopelmini/forgrype@sha256:ebe0c6ca122deef072c29be2f915130e5c8b4c277ad5ef551385f6496dae4dfa
      - docker.io/dimastopelmini/forgrype@sha256:653c8980c63a9ac403a3b9f56a08f43f929432ece69894423c165b4d61d3dcdb
    expected_namespaces:
      - nvd:cpe
      - minimos:distro:minimos:rolling
    validations:
      - <<: *default-validations
        max_year: 2024 # important - minimOS is a new distro and older years don't find enough
        candidate_tool_label: custom-db

  - provider: nvd
    images:
      - docker.io/busybox:1.28.1@sha256:2107a35b58593c58ec5f4e8f2c4a70d195321078aebfadfbfb223a2ff4a4ed21
    expected_namespaces:
      - nvd:cpe
    validations:
      - *default-validations

  - provider: oracle
    additional_trigger_globs:
      - src/vunnel/utils/oval_parser.py
    images:
      - docker.io/oraclelinux:6@sha256:a06327c0f1d18d753f2a60bb17864c84a850bb6dcbcf5946dd1a8123f6e75495
      - docker.io/anchore/test_images:appstreams-oraclelinux-8-1a287dd@sha256:c8d664b0e728d52f57eeb98ed1899c16d3b265f02ddfb41303d7a16c31e0b0f1
    expected_namespaces:
      - oracle:distro:oraclelinux:5
      - oracle:distro:oraclelinux:6
      - oracle:distro:oraclelinux:7
      - oracle:distro:oraclelinux:8
      - oracle:distro:oraclelinux:9
      - oracle:distro:oraclelinux:10
    validations:
      - <<: *default-validations
        # TODO: docker.io/anchore/test_images:appstreams-oraclelinux-8-1a287dd@sha256:c8d664b0e728
        # has no matches before 2022. Label more things, move max_year to 2022, and then
        # change fail_on_empty_match_set back to true (the default).
        max_year: 2021
        fail_on_empty_match_set: false

  - provider: rhel
    # ideally we would not use cache, however, the ubuntu provider is currently very expensive to run.
    # This will still test incremental updates relative to the nightly cache that is populated.
    additional_trigger_globs:
      - src/vunnel/utils/oval_parser.py
    use_cache: true
    images:
      - registry.access.redhat.com/ubi8@sha256:68fecea0d255ee253acbf0c860eaebb7017ef5ef007c25bee9eeffd29ce85b29
      - docker.io/centos:6@sha256:3688aa867eb84332460e172b9250c9c198fdfd8d987605fd53f246f498c60bcf
      - docker.io/almalinux:8@sha256:cd49d7250ed7bb194d502d8a3e50bd775055ca275d1d9c2785aea72b890afe6a
      - docker.io/rockylinux:8@sha256:72afc2e1a20c9ddf56a81c51148ebcbe927c0a879849efe813bee77d69df1dd8
      - docker.io/anchore/test_images:appstreams-centos-stream-8-1a287dd@sha256:808f6cf3cf4473eb39ff9bb47ead639d2ed71255b75b9b140162b58c6102bcc9
      - docker.io/anchore/test_images:appstreams-rhel-8-1a287dd@sha256:524ff8a75f21fd886ec7ed82387766df386671e8b77e898d05786118d5b7880b
      - docker.io/anchore/test_images:vulnerabilities-centos@sha256:746d31247006cc06434ce91ccf3523b2c230ff6c378ffed7ca1c60bbb48ea86f
    validations:
      - *default-validations

    expected_namespaces:
      - redhat:distro:redhat:5
      - redhat:distro:redhat:6
      - redhat:distro:redhat:7
      - redhat:distro:redhat:8
      - redhat:distro:redhat:9
      - redhat:distro:redhat:10

  - provider: sles
    additional_trigger_globs:
      - src/vunnel/utils/oval_v2.py

    images:
      - registry.suse.com/suse/sles12sp4:26.380@sha256:94b537f5b312e7397b5d0bbb3d892f961acdd9454950fc233d77f771e25335fb
      - registry.suse.com/suse/sle15:15.1.6.2.461@sha256:6e613c994c3b33224e439ef8ee9003fb69416f77f7a6b1da0b18981d5aa3bb75

    expected_namespaces:
      - sles:distro:sles:11
      - sles:distro:sles:11.1
      - sles:distro:sles:11.2
      - sles:distro:sles:11.3
      - sles:distro:sles:11.4
      - sles:distro:sles:12
      - sles:distro:sles:12.1
      - sles:distro:sles:12.2
      - sles:distro:sles:12.3
      - sles:distro:sles:12.4
      - sles:distro:sles:12.5
      - sles:distro:sles:15
      - sles:distro:sles:15.1
      - sles:distro:sles:15.2
      - sles:distro:sles:15.3
      - sles:distro:sles:15.4
      - sles:distro:sles:15.5
      - sles:distro:sles:15.6
      - sles:distro:sles:15.7
    validations:
      - *default-validations

  - provider: ubuntu
    # ideally we would not use cache, however, the ubuntu provider is currently very expensive to run.
    # This will still test incremental updates relative to the nightly cache that is populated.
    use_cache: true
    images:
      - docker.io/ubuntu:16.10@sha256:8dc9652808dc091400d7d5983949043a9f9c7132b15c14814275d25f94bca18a
      - docker.io/ubuntu:19.04@sha256:3db17bfc30b41cc18552578f4a66d7010050eb9fdc42bf6c3d82bb0dcdf88d58
      - docker.io/ubuntu:22.04@sha256:aa6c2c047467afc828e77e306041b7fa4a65734fe3449a54aa9c280822b0d87d
      - docker.io/ubuntu:22.10@sha256:80fb4ea0c0a384a3072a6be1879c342bb636b0d105209535ba893ba75ab38ede
      - docker.io/ubuntu:23.04@sha256:09f035f46361d193ded647342903b413d57d05cc06acff8285f9dda9f2d269d5
    expected_namespaces:
      - ubuntu:distro:ubuntu:12.04
      - ubuntu:distro:ubuntu:12.10
      - ubuntu:distro:ubuntu:13.04
      - ubuntu:distro:ubuntu:14.04
      - ubuntu:distro:ubuntu:14.10
      - ubuntu:distro:ubuntu:15.04
      - ubuntu:distro:ubuntu:15.10
      - ubuntu:distro:ubuntu:16.04
      - ubuntu:distro:ubuntu:16.10
      - ubuntu:distro:ubuntu:17.04
      - ubuntu:distro:ubuntu:17.10
      - ubuntu:distro:ubuntu:18.04
      - ubuntu:distro:ubuntu:18.10
      - ubuntu:distro:ubuntu:19.04
      - ubuntu:distro:ubuntu:19.10
      - ubuntu:distro:ubuntu:20.04
      - ubuntu:distro:ubuntu:20.10
      - ubuntu:distro:ubuntu:21.04
      - ubuntu:distro:ubuntu:21.10
      - ubuntu:distro:ubuntu:22.04
      - ubuntu:distro:ubuntu:22.10
      - ubuntu:distro:ubuntu:23.04
      - ubuntu:distro:ubuntu:23.10
      - ubuntu:distro:ubuntu:24.04
      - ubuntu:distro:ubuntu:24.10
      - ubuntu:distro:ubuntu:25.04
    validations:
      - *default-validations

  - provider: wolfi
    additional_providers:
      - name: nvd
        use_cache: true
    images:
      - cgr.dev/chainguard/wolfi-base:latest-20221001@sha256:be3834598c3c4b76ace6a866edcbbe1fa18086f9ee238b57769e4d230cd7d507
    expected_namespaces:
      - wolfi:distro:wolfi:rolling
    validations:
      - *default-validations

  - provider: alma
    additional_providers:
      - name: rhel
        use_cache: true
    images:
      - docker.io/anchore/test_images:almalinux8-271722c@sha256:6485db654df0452bd15ea71ec43e808bc8eb05b91f1c2754669a5573479a6c19
      - docker.io/almalinux:8@sha256:cd49d7250ed7bb194d502d8a3e50bd775055ca275d1d9c2785aea72b890afe6a
    expected_namespaces:
      - almalinux:distro:almalinux:8
      - almalinux:distro:almalinux:9
      - almalinux:distro:almalinux:10
      - redhat:distro:redhat:8
      - redhat:distro:redhat:9
      - redhat:distro:redhat:10
    validations:
      - <<: *default-validations
        max_year: 2022
        candidate_tool_label: custom-db

  - provider: rootio
    # Root IO provides patched packages for multiple ecosystems
    # Test images contain Root IO patched versions (rootio- prefix, _rootio_ version suffix)
    additional_providers:
      - name: nvd
        use_cache: true
      - name: alpine
        use_cache: true
      - name: debian
        use_cache: true
      - name: ubuntu
        use_cache: true
      - name: github
        use_cache: true
    images:
      - docker.io/rootpublic/cassandra:latest@sha256:02272b14efbe14e70ee5512ce707c4e300d3c1813f0e5df9562512c1b96be835
      - docker.io/rootpublic/ubuntu:22.04@sha256:1390a26823a5a761dfbb7f591ae74a71afd8e23583a2f0c58dca6943b606f6d5
    expected_namespaces:
      # Root IO namespaces (per grype-db implementation)
      - rootio:distro:ubuntu:22.04
      # Upstream provider namespaces (for NAK pattern verification)
      - ubuntu:distro:ubuntu:22.04
      - github:language:java
      - github:language:go
      - nvd:cpe
    validations:
      - <<: *default-validations
        max_year: 2021  # Root IO is a new provider - limiting to older CVEs for initial validation
        max_new_false_negatives: 5  # Allow some FNs for initial provider PR
        max_unlabeled_percent: 90  # Relaxed for initial PR with limited test image
        max_f1_regression: 0.30  # Allow F1 score up to 0.70 (vs 1.00 reference)
        candidate_tool_label: custom-db