Merge pull request vyos#1198 from c-po/spdx-refactoring

T8542: refactor SBOM generation - run after ISO build

Author: c-po

Makefile

@@ -90,11 +90,6 @@ qemu-live: checkiso
 oci: checkiso
 	scripts/iso-to-oci $(ISO_PATH)
 
-.PHONY: make_sbom
-.ONESHELL:
-make_sbom: checkiso
-	@scripts/check-qemu-install --debug --iso $(ISO_PATH) --sbom --cpu 2 --memory 4 $(if $(SBOM_OUTPUT_DIR),--sbom-output-dir "$(SBOM_OUTPUT_DIR)")
-
 .PHONY: clean
 .ONESHELL:
 clean:

docker/Dockerfile

@@ -348,6 +348,11 @@ RUN apt-get update && apt-get install -y \
       debmake \
       python3-debian
 
+# Install syft binary required for SBOM generation
+RUN cd /tmp && curl -sSfL -o syft.tar.gz \
+      https://cdn.vyos.io/tools/syft_1.44.0_linux_$(dpkg-architecture -qDEB_HOST_ARCH).tar.gz; \
+      tar --extract --file=syft.tar.gz syft; mv syft /usr/local/bin/
+
 # Allow password-less 'sudo' for all users in group 'sudo'
 RUN sed "s/^%sudo.*/%sudo\tALL=(ALL) NOPASSWD:ALL/g" -i /etc/sudoers && \
     echo "vyos_bld\tALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \

scripts/check-qemu-install

@@ -125,10 +125,6 @@ parser.add_argument('--tpmtest', help='Execute TPM encrypted config tests',
                 action='store_true', default=False)
 parser.add_argument('--sbtest', help='Execute Secure Boot tests',
                 action='store_true', default=False)
-parser.add_argument('--sbom', help='Generate SBOM file using syft',
-                action='store_true', default=False)
-parser.add_argument('--sbom-output-dir', help='Host directory to mount and copy SBOM file into',
-                type=str, default=None)
 parser.add_argument('--cloud-init', help='Execute cloud-init tests',
                 action='store_true', default=False)
 parser.add_argument('--qemu-cmd', help='Only generate QEMU launch command',
@@ -149,12 +145,6 @@ args = parser.parse_args()
 if os.geteuid() != 0:
     exit('You need to have root privileges to run this script.')
 
-if args.sbom and not args.sbom_output_dir:
-    args.sbom_output_dir = os.getcwd()
-
-if args.sbom_output_dir:
-    os.makedirs(args.sbom_output_dir, exist_ok=True)
-
 if args.cloud_init:
     hostname = CI_CONFIG_ARGS['hostname']
     op_mode_prompt = rf'vyos@{hostname}:~\$'
@@ -200,7 +190,7 @@ class EarlyExit(Exception):
 def _kvm_exists():
     return os.path.exists("/dev/kvm")
 
-def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False, vnc_enabled=False, secure_boot=False, transfer_disk=None):
+def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False, vnc_enabled=False, secure_boot=False):
     uefi = ""
     uuid = "f48b60b2-e6ad-49ef-9d09-4245d0585e52"
     accel = ',accel=kvm' if _kvm_exists() else ''
@@ -286,10 +276,6 @@ def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False
                ' -tpmdev emulator,id=tpm0,chardev=chrtpm' \
                ' -device tpm-tis,tpmdev=tpm0'
 
-    if transfer_disk:
-        cmd += f' -drive format=raw,file={transfer_disk},if=none,media=disk,id=drive-transfer,readonly=off' \
-               f' -device scsi-hd,bus=scsi0.0,drive=drive-transfer,id=transfer-disk'
-
     return cmd
 
 def shutdownVM(c, log, message=''):
@@ -407,45 +393,10 @@ if args.raid:
 # must be called after the raid disk as args.disk name is altered in the RAID path
 gen_disk(args.disk)
 
-# Create 512MB transfer disk for SBOM generation if requested, download the syft util to use inside the VM for SBOM generation process.
-# It will be used to copy generated SBOM files from the VM to the host via the mounted transfer disk.
-transfer_disk_path = None
-if args.sbom:
-    _ts = datetime.now().strftime('%Y%m%d-%H%M%S')
-    _rand = '%04x' % random.randint(0, 0xFFFF)
-    transfer_disk_path = f'/tmp/vyos-sbom-transfer-{_ts}-{_rand}.img'
-    subprocess.check_output(['dd', 'if=/dev/zero', f'of={transfer_disk_path}', 'bs=1M', 'count=512'])
-    subprocess.check_output(['mkfs.vfat', '-F', '32', '-n', 'SBOMXFER', transfer_disk_path])
-    log.info(f'Created SBOM transfer disk: {transfer_disk_path}')
-    setup_mount = f'{transfer_disk_path}.setup'
-    os.makedirs(setup_mount, exist_ok=True)
-    subprocess.check_output(['mount', '-o', 'loop', transfer_disk_path, setup_mount])
-    try:
-        log.info('Downloading syft to transfer disk')
-        if platform.machine() in ['amd64', 'x86_64']:
-            syft_tar_url = 'https://cdn.vyos.io/tools/syft_1.44.0_linux_amd64.tar.gz'
-        elif platform.machine() in ['arm64', 'aarch64']:
-            syft_tar_url = 'https://cdn.vyos.io/tools/syft_1.44.0_linux_arm64.tar.gz'
-        else:
-            raise ValueError(f'Unsupported architecture for syft download: {platform.machine()}')
-        import tarfile, tempfile
-        with tempfile.TemporaryDirectory() as tar_tmp:
-            tar_path = os.path.join(tar_tmp, 'syft.tar.gz')
-            subprocess.check_call(['curl', '-sSfL', '-o', tar_path, syft_tar_url])
-            with tarfile.open(tar_path) as tf:
-                with tf.extractfile(tf.getmember('syft')) as src:
-                    dest_path = os.path.join(setup_mount, 'syft')
-                    with open(dest_path, 'wb') as dst:
-                        dst.write(src.read())
-            os.chmod(os.path.join(setup_mount, 'syft'), 0o755)
-        log.info('Syft downloaded to transfer disk')
-    finally:
-        subprocess.check_output(['umount', setup_mount])
-        os.rmdir(setup_mount)
-
 # Create software emulated TPM - clear existing TPM data first - might be a
 # leftover from a previous run
 clearTPM()
+
 def start_swtpm():
     if not os.path.exists(tpm_folder):
         os.mkdir(tpm_folder)
@@ -696,7 +647,7 @@ try:
     # Installing image to disk
     #################################################
     log.info('Installing system')
-    cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, tpm=args.tpmtest, iso_img=args.iso, vnc_enabled=args.vnc, secure_boot=args.sbtest, transfer_disk=transfer_disk_path)
+    cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, tpm=args.tpmtest, iso_img=args.iso, vnc_enabled=args.vnc, secure_boot=args.sbtest)
     log.debug(f'Executing command: {cmd}')
     c = pexpect.spawn(cmd, logfile=stl, timeout=60)
 
@@ -1297,68 +1248,6 @@ try:
             tmp = 'Configtest failed :/ - check debug output'
             log.error(tmp)
             raise Exception(tmp)
-    elif args.sbom:
-        # Determine SBOM output file names based on ISO name
-        iso_real = os.path.realpath(args.iso) if args.iso else None
-        iso_name = os.path.splitext(os.path.basename(iso_real))[0] if iso_real else 'vyos'
-        sbom_file_cdx = f'{iso_name}.iso.cdx.json'
-        sbom_file_spdx = f'{iso_name}.iso.spdx.json'
-
-        # Create a mount point for the SBOM transfer disk
-        log.info('Mounting transfer disk in VM')
-        c.sendline('sudo mkdir -p /mnt/sbom_transfer')
-        c.expect(op_mode_prompt)
-
-        c.sendline('sudo mount $(sudo blkid -L SBOMXFER) /mnt/sbom_transfer')
-        c.expect(op_mode_prompt)
-
-        c.sendline('mountpoint -q /mnt/sbom_transfer && echo MOUNT_OK || echo MOUNT_FAIL')
-        i = c.expect(['MOUNT_OK', 'MOUNT_FAIL'])
-        if i != 0:
-            raise Exception('Failed to mount SBOM transfer disk inside VM')
-        c.expect(op_mode_prompt)
-
-        # Copy the syft binary from the transfer disk and verify it is available before running the SBOM generation process
-        c.sendline('test -x /mnt/sbom_transfer/syft && echo SYFT_SRC_OK || echo SYFT_SRC_FAIL')
-        i = c.expect(['SYFT_SRC_OK', 'SYFT_SRC_FAIL'])
-        if i != 0:
-            raise Exception('Syft binary is missing or not executable on the SBOM transfer disk')
-        c.expect(op_mode_prompt)
-
-        c.sendline('sudo cp /mnt/sbom_transfer/syft /tmp/syft && sudo chmod +x /tmp/syft && test -x /tmp/syft && echo SYFT_COPY_OK || echo SYFT_COPY_FAIL')
-        i = c.expect(['SYFT_COPY_OK', 'SYFT_COPY_FAIL'])
-        if i != 0:
-            raise Exception('Failed to copy Syft binary from SBOM transfer disk to /tmp/syft')
-        c.expect(op_mode_prompt)
-        c.sendline('TERM=dumb /tmp/syft --version')
-        c.expect(op_mode_prompt)
-        lines = c.before.decode(errors='replace').strip().splitlines()
-        syft_version = next((l.strip() for l in reversed(lines) if l.strip()), 'unknown')
-        log.info(f'syft version: {syft_version}')
-
-        # Generate SBOMs using the syft util for the whole filesystem with all layers, output both CycloneDX and SPDX formats
-        syft_cmd = (
-            f'sudo TERM=dumb /tmp/syft / -q'
-            f' --exclude **/mnt/sbom_transfer/**'
-            f' --source-name {iso_name}.iso --source-version {iso_name}'
-            f' -o cyclonedx-json=/mnt/sbom_transfer/{sbom_file_cdx}'
-            f' -o spdx-json=/mnt/sbom_transfer/{sbom_file_spdx}'
-        )
-
-        log.info(f'Generating SBOMs: {sbom_file_cdx} and {sbom_file_spdx}')
-        c.sendline(syft_cmd)
-        c.expect(op_mode_prompt, timeout=1800)
-
-        c.sendline('echo EXITCODE:$\x16?')
-        i = c.expect(['EXITCODE:0', r'EXITCODE:\d+'])
-        if i != 0:
-            raise Exception('syft SBOM generation failed')
-
-        # Unmount transfer disk
-        c.sendline('sudo umount /mnt/sbom_transfer')
-        c.expect(op_mode_prompt)
-        log.info('SBOM files written to transfer disk. The transfer disk was unmounted.')
-
     elif args.sbtest:
         c.sendline('show secure-boot')
         c.expect('SecureBoot enabled')
@@ -1369,26 +1258,6 @@ try:
     shutdownVM(c, log, 'Powering off system')
     c.close()
 
-    # Extract SBOM files from transfer disk to host
-    if args.sbom and transfer_disk_path and args.sbom_output_dir:
-        log.info('Extracting SBOM files from transfer disk to host.')
-        mount_point = f'{transfer_disk_path}.mnt'
-        mounted = False
-        os.makedirs(mount_point, exist_ok=True)
-        try:
-            subprocess.check_output(['mount', '-o', 'loop', transfer_disk_path, mount_point])
-            mounted = True
-            for sbom_f in [sbom_file_cdx, sbom_file_spdx]:
-                src = os.path.join(mount_point, sbom_f)
-                if os.path.isfile(src):
-                    shutil.copy2(src, args.sbom_output_dir)
-                    log.info(f'SBOM file saved to: {args.sbom_output_dir}/{sbom_f}')
-        finally:
-            if mounted:
-                subprocess.check_output(['umount', mount_point])
-            if os.path.isdir(mount_point):
-                os.rmdir(mount_point)
-
 except EarlyExit:
     pass
 
@@ -1425,8 +1294,6 @@ if not args.keep:
             os.remove(diskname_raid)
         if args.sbtest:
             os.remove(OVMF_VARS_TMP)
-        if transfer_disk_path and os.path.isfile(transfer_disk_path):
-            os.remove(transfer_disk_path)
     except Exception:
         log.error('Exception while removing diskimage!')
         log.error(traceback.format_exc())
@@ -1437,4 +1304,4 @@ if EXCEPTION:
     log.error('The ISO image is not considered usable!')
     sys.exit(1)
 
-sys.exit(0)
+sys.exit(0)

scripts/image-build/build-vyos-image

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2022-2024 VyOS maintainers and contributors
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -17,7 +17,6 @@
 # File: build-vyos-image
 # Purpose: builds VyOS images using a fork of Debian's live-build tool
 
-# Import Python's standard library modules
 import re
 import os
 import sys
@@ -31,11 +30,11 @@ import argparse
 import datetime
 import functools
 import string
+import subprocess
 
 class ImageBuildError(Exception):
     pass
 
-
 # argparse converts hyphens to underscores,
 # so for lookups in the original options hash we have to convert them back
 def field_to_option(s):
@@ -722,11 +721,32 @@ Pin-Priority: 600
         cmd("lb build 2>&1")
 
         # Copy the image
-        shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file)
+        shutil.copy(f'live-image-{build_config["architecture"]}.hybrid.iso', iso_file)
 
         # Add the image to the manifest
         manifest['artifacts'].append(iso_file)
 
+        # Now create SBOM
+        syft_target_dir = 'chroot'
+        syft_base_path = os.getcwd() + f'/{syft_target_dir}'
+        cmd = [['syft', syft_target_dir,
+                '--source-name', 'VyOS', '--source-version', version,
+               '-o', f'cyclonedx-json=vyos-{version}.cdx.json',
+               '-o', f'spdx-json=vyos-{version}.spdx.json']]
+
+        # syft bug for CycloneDX https://github.com/anchore/syft/issues/4592#issuecomment-4567247328
+        cmd.append(['sed', '-i', '-e', f's@{syft_base_path}@@g', f'vyos-{version}.cdx.json'])
+        cmd.append(['sed', '-i', '-e', f's@{syft_base_path}@//@g', f'vyos-{version}.spdx.json'])
+
+        for c in cmd:
+            print(c)
+            with subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                  text=True, bufsize=1) as p:
+                for line in p.stdout:
+                    sys.stdout.write(line)
+                    sys.stdout.flush()
+                p.wait()
+
     # If the flavor has `image_format = "iso"`, then the work is done.
     # If not, build additional flavors from the ISO.
     if build_config["image_format"] != ["iso"]: