check-qemu-install

#!/usr/bin/env python3
#
# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# File: check-qemu-install
# Purpose:
#  This script installs a system on a emulated qemu host to verify
#  that the iso produced is installable and boots.
#  after the iso is booted from disk it also tries to execute the
#  vyos-smoketest script to verify checks there.
#
#  For now it will not fail on failed smoketest but will fail on
#  install and boot errors.
#  Arguments:
#    iso            iso image to install
#    [disk]         disk filename to use, if none is provided it
#                   is autogenerated
#    [--keep]       Keep the disk image after completion
#    [--logfile]    name of logfile to save, defaulting to stdout
#    [--silent]     only print on errors
#    [--debug]      print all communication with the device

import sys
import os
import time
import argparse
import subprocess
import random
import traceback
import logging
import re
import shutil
import json
import platform

from io import BytesIO
from datetime import datetime
from glob import glob

import tomli
import pexpect

EXCEPTION = 0
DISK_IMAGE_EXTENSION = '.raw'

# Nested installer-ISO QEMU test: mkisofs payload (see --nested-installer-iso-test)
NESTED_ISO_DATA_DIR = 'nested_iso_data'
NESTED_INSTALLER_PAYLOAD_ISO = 'nested_installer_payload.iso'
NESTED_INNER_ISO_NAME = 'vyos-installer.iso'

# Local-only HTTP nested-ISO upgrade test: default VRF (main RIB) then named VRF purple
NESTED_HTTP_SERV_PORT = 18088
NESTED_HTTP_IMAGE_NAME = 'test-image-update'
NESTED_HTTP_VRF_NAME = 'purple'
NESTED_HTTP_VRF_DUMMY = 'dum8000'
NESTED_HTTP_VRF_TABLE = '42042'
NESTED_HTTP_VRF_ADDR = '198.51.100.99'  # RFC 5737 TEST-NET-3: bind address for http.server inside VRF
NESTED_HTTP_MOUNT_POINT = '/mnt/nested_installer_iso'

# Cloud-Init data
CI_CONFIG_ARGS = {
    'hostname': 'vyos-CLOUD-INIT',
    'eth0_ipv4': '192.0.2.1/25',
    'eth0_ipv6': '2001:db8::1/64',
    'default_nexthop' : '192.0.2.126',
    'default_nexthop6' : '2001:db8::ffff',
    }

tpm_folder = '/tmp/vyos_tpm_test'
tpm_sock = f'{tpm_folder}/swtpm-sock'
qemu_name = 'VyOS-QEMU'

# RFC7042 section 2.1.2 MAC addresses used for documentation
macbase = '00:00:5E:00:53'

test_timeout = 5 *3600 # 5 hours (in seconds) to complete individual testcases

op_mode_prompt = r'vyos@vyos:~\$'
cfg_mode_prompt = r'vyos@vyos#'
default_user = 'vyos'
default_password = 'vyos'

# getch.py
KEY_F2 = chr(27) + chr(91) + chr(49) + chr(50) + chr(126)
KEY_F10 = chr(27) + chr(91) + chr(50) + chr(49) + chr(126)
KEY_UP = chr(27) + chr(91) + chr(65)
KEY_DOWN = chr(27) + chr(91) + chr(66)
KEY_SPACE = chr(32)
KEY_RETURN = chr(13)
KEY_ESC = chr(27)
KEY_Y = chr(121)

mok_password = '1234'

# Map QEMU arch
QEMU_CONFIG = {
    'amd64': {
        'bin' : 'qemu-system-x86_64',
        'machine' : 'pc',
        'bios' : '/usr/share/OVMF/OVMF_CODE.fd',
    },
    "arm64": {
        'bin': 'qemu-system-aarch64',
        'machine': 'virt,highmem=on',
        'bios': '/usr/share/qemu-efi-aarch64/QEMU_EFI.fd',
    },
}

parser = argparse.ArgumentParser(description='Install and start a test VyOS vm.')
parser.add_argument('--iso', help='ISO file to install')
parser.add_argument('--disk', help='name of disk image file', nargs='?')
parser.add_argument('--keep', help='Do not remove disk-image after installation',
                              action='store_true', default=False)
parser.add_argument('--silent', help='Do not show output on stdout unless an error has occurred',
                              action='store_true', default=False)
parser.add_argument('--debug', help='Send all debug output to stdout',
                               action='store_true', default=False)
parser.add_argument('--logfile', help='Log to file')
parser.add_argument('--match', help='Smoketests to run')
parser.add_argument('--uefi', help='Boot using UEFI', action='store_true', default=False)
parser.add_argument('--vnc', help='Enable VNC', action='store_true', default=False)
parser.add_argument('--raid', help='Perform a RAID-1 install', action='store_true', default=False)
parser.add_argument('--no-interfaces', help='Execute testsuite without interface tests to save time',
                action='store_true', default=False)
parser.add_argument('--smoketest', help='Execute script based CLI smoketests',
                action='store_true', default=False)
parser.add_argument('--configtest', help='Execute load/commit config tests',
				action='store_true', default=False)
parser.add_argument('--tpmtest', help='Execute TPM encrypted config tests',
                action='store_true', default=False)
parser.add_argument('--ifnametest', help='Execute interface naming/hw-id persistence tests',
                action='store_true', default=False)
parser.add_argument('--sbtest', help='Execute Secure Boot tests',
                action='store_true', default=False)
parser.add_argument('--cloud-init', help='Execute cloud-init tests',
                action='store_true', default=False)
parser.add_argument('--test-image-update', help='Test ISO image update in default VRF and custom VRF',
                action='store_true', default=False)
parser.add_argument('--qemu-cmd', help='Only generate QEMU launch command',
                action='store_true', default=False)
parser.add_argument('--cpu', help='Set QEMU CPU', type=int, default=2)
parser.add_argument('--cpu-type', help='Set QEMU CPU type', type=str, default='host')
parser.add_argument('--memory', help='Set QEMU memory', type=int, default=4)
parser.add_argument('--vyconf', help='Execute testsuite with vyconfd', action='store_true',
                    default=False)
parser.add_argument('--no-vpp', help='Execute testsuite without VPP tests',
                action='store_true', default=False)
parser.add_argument('--huge-page-size', help='Huge page size (e.g., 2M, 1G)', type=str)
parser.add_argument('--huge-page-count', help='Number of huge pages to allocate', type=int)
parser.add_argument('--isolate-cpus', help='CPU cores to isolate (e.g., 1,3-4', type=str)

args = parser.parse_args()

if os.geteuid() != 0:
    exit('You need to have root privileges to run this script.')

if args.cloud_init:
    hostname = CI_CONFIG_ARGS['hostname']
    op_mode_prompt = rf'vyos@{hostname}:~\$'
    cfg_mode_prompt = rf'vyos@{hostname}#'

# This is what we requested the build to contain
with open('data/defaults.toml', 'rb') as f:
    vyos_defaults = tomli.load(f)

# This is what we got from the build
manifest_file = 'build/manifest.json'
if os.path.isfile(manifest_file):
    with open('build/manifest.json', 'rb') as f:
        manifest = json.load(f)

    vyos_version = manifest['build_config']['version']
    vyos_codename = manifest['build_config']['release_train']

class StreamToLogger(object):
    """
    Fake file-like stream object that redirects writes to a logger instance.
    """
    def __init__(self, logger, log_level=logging.INFO):
        self.logger = logger
        self.log_level = log_level
        self.linebuf = b''
        self.ansi_escape = re.compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]')

    def write(self, buf):
        self.linebuf += buf
        while b'\n' in self.linebuf:
            f = self.linebuf.split(b'\n', 1)
            if len(f) == 2:
                self.logger.debug(self.ansi_escape.sub('', f[0].decode(errors="replace").rstrip()))
                self.linebuf = f[1]

    def flush(self):
        pass

class EarlyExit(Exception):
    pass

def _kvm_exists():
    return os.path.exists("/dev/kvm")

def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False,
                 vnc_enabled=False, secure_boot=False, nested_cdrom_iso=None):
    uefi = ""
    uuid = "f48b60b2-e6ad-49ef-9d09-4245d0585e52"
    accel = ',accel=kvm' if _kvm_exists() else ''

    if platform.machine() in ['amd64', 'x86_64']:
        architecture = 'amd64'
    elif platform.machine() in ['arm64', 'aarch64']:
        architecture = 'arm64'
    else:
        raise ValueError('Unsupported architecture!')

    qemu_arch_config = QEMU_CONFIG[architecture]
    qemu_bin = qemu_arch_config['bin']
    bios = qemu_arch_config['bios']

    cpu_type = args.cpu_type
    enable_kvm = '-enable-kvm' if _kvm_exists() else ''
    machine = qemu_arch_config['machine']
    vga = '-vga none'
    vnc = ''
    if vnc_enabled:
        vga = '-vga virtio'
        vnc = '-vnc :0'

    if enable_uefi:
        uefi = f'-bios {bios}'
        name = f'{name}-UEFI'

        if secure_boot:
            name = f'{name}-SECURE-BOOT'
            machine = 'q35,smm=on'

            uefi = f'-drive "if=pflash,unit=0,format=raw,readonly=on,file={OVMF_CODE}" ' \
                   f'-drive "if=pflash,unit=1,format=raw,file={OVMF_VARS_TMP}"'
            # Changing UEFI settings require a display
            vga = '-vga virtio'

    cdrom = ""
    nested_cdrom = ""
    if iso_img:
        bootindex = '10'
        cdrom = f' -drive file={iso_img},format=raw,if=none,media=cdrom,id=drive-cd1,readonly=on'
        if architecture == 'arm64':
            cdrom = f' {cdrom} -device scsi-cd,bus=scsi0.0,drive=drive-cd1,id=cd1,bootindex={bootindex}'
        else:
            cdrom = f' {cdrom} -device ahci,id=achi0 -device ide-cd,bus=achi0.0,drive=drive-cd1,id=cd1,bootindex={bootindex}'

    if nested_cdrom_iso:
        drive_settings = 'drive=drive-cd2,id=cd2,bootindex=11'
        nested_cdrom = f' -drive file={nested_cdrom_iso},format=raw,if=none,media=cdrom,id=drive-cd2,readonly=on'
        if architecture == 'arm64':
            nested_cdrom = f' {nested_cdrom} -device scsi-cd,bus=scsi0.0,{drive_settings}'
        else:
            if not iso_img:
                # Second IDE CD on its own AHCI controller (no installer CD in this command)
                nested_cdrom = f'{nested_cdrom} -device ahci,id=nestedahci' \
                               f' -device ide-cd,bus=nestedahci.0,{drive_settings}'
            else:
                nested_cdrom = f'{nested_cdrom} -device ide-cd,bus=achi0.1,{drive_settings}'

    # Set QEmu disk image format - this differs if VyOS was installed via smoketest
    # or we use an already ewxisting image
    disk_format = 'qcow2' if args.disk.endswith('.qcow2') else 'raw'
    cmd = f'{qemu_bin} \
        -name "{name}" \
        -smp {args.cpu},sockets=1,cores={args.cpu},threads=1 \
        -cpu {cpu_type} \
        -machine {machine}{accel} \
        {uefi} \
        -m {args.memory}G \
        -nographic \
        {vga} {vnc}\
        -uuid {uuid} \
        {enable_kvm} \
        -monitor unix:/tmp/qemu-monitor-socket-{disk_img},server,nowait \
        -netdev user,id=n0,net=192.0.2.0/24,dhcpstart=192.0.2.101,dns=192.0.2.10 -device virtio-net-pci,netdev=n0,mac={macbase}:00,romfile="",host_mtu=1500 \
        -netdev user,id=n1 -device virtio-net-pci,netdev=n1,mac={macbase}:01,romfile="",host_mtu=1500 \
        -netdev user,id=n2 -device virtio-net-pci,netdev=n2,mac={macbase}:02,romfile="",host_mtu=1500 \
        -netdev user,id=n3 -device virtio-net-pci,netdev=n3,mac={macbase}:03,romfile="",host_mtu=1500 \
        -netdev user,id=n4 -device e1000e,netdev=n4,mac={macbase}:04,romfile="" \
        -netdev user,id=n5 -device e1000e,netdev=n5,mac={macbase}:05,romfile="" \
        -netdev user,id=n6 -device vmxnet3,netdev=n6,mac={macbase}:06,romfile="" \
        -netdev user,id=n7 -device vmxnet3,netdev=n7,mac={macbase}:07,romfile="" \
        -device virtio-scsi-pci,id=scsi0 \
        {cdrom}{nested_cdrom} \
        -drive format={disk_format},file={disk_img},if=none,media=disk,id=drive-hd1,readonly=off \
        -device scsi-hd,bus=scsi0.0,drive=drive-hd1,id=hd1,bootindex=1'

    if raid:
        cmd += f' -drive format={disk_format},file={raid},if=none,media=disk,id=drive-hd2,readonly=off' \
               f' -device scsi-hd,bus=scsi0.0,drive=drive-hd2,id=hd2,bootindex=2'

    if tpm:
        cmd += f' -chardev socket,id=chrtpm,path={tpm_sock}' \
               ' -tpmdev emulator,id=tpm0,chardev=chrtpm' \
               ' -device tpm-tis,tpmdev=tpm0'

    return cmd

def shutdownVM(c, log, message=''):
    #################################################
    # Powering off system
    #################################################
    if message:
        log.info(message)

    c.sendline('poweroff now')
    log.info('Shutting down virtual machine')
    for i in range(30):
        log.info('Waiting for shutdown...')
        # Shutdown in qemu doesnt work first time
        # Use this workaround
        # https://vyos.dev/T5024
        c.sendline('poweroff now')
        if not c.isalive():
            log.info('VM is shut down!')
            break
        time.sleep(10)
    else:
        tmp = 'VM Did not shut down after 300sec'
        log.error(tmp)
        raise Exception(tmp)
    c.close()

def waitForLogin(child, log, timeout=20) -> None:
    try:
        child.expect('The highlighted entry will be executed automatically in',
                     timeout=timeout)
        child.sendline('')
    except pexpect.TIMEOUT:
        log.warning('Did not find GRUB countdown window, ignoring')

def loginVM(c, log):
    log.info('Waiting for login prompt')
    c.expect('[Ll]ogin:', timeout=600)
    c.sendline(default_user)
    c.expect('[Pp]assword:')
    c.sendline(default_password)
    c.expect(op_mode_prompt)
    log.info('Logged in!')
    c.sendline('set terminal width 160')
    c.expect(op_mode_prompt)
    c.sendline('set terminal length 60')
    c.expect(op_mode_prompt)

def clearTPM():
    for f in glob(f'{tpm_folder}/*'):
        os.remove(f)

# Setting up logger
log = logging.getLogger()
log.setLevel(logging.DEBUG)

stl = StreamToLogger(log)
formatter = logging.Formatter('%(levelname)5s - %(message)s')

handler = logging.StreamHandler(sys.stdout)
if args.silent:
    handler.setLevel(logging.ERROR)
elif args.debug:
    handler.setLevel(logging.DEBUG)
else:
    handler.setLevel(logging.INFO)

handler.setFormatter(formatter)
log.addHandler(handler)

# Mutually exclusive primary testcase selectors (--no-interfaces is a
# smoketest modifier).
_primary_modes = []
if args.cloud_init:
    _primary_modes.append('--cloud-init')
if args.test_image_update:
    _primary_modes.append('--test-image-update')
if args.tpmtest:
    _primary_modes.append('--tpmtest')
if args.ifnametest:
    _primary_modes.append('--ifnametest')
if args.raid:
    _primary_modes.append('--raid')
if args.smoketest:
    _primary_modes.append('--smoketest')
if args.configtest:
    _primary_modes.append('--configtest')
if args.sbtest:
    _primary_modes.append('--sbtest')
if len(_primary_modes) > 1:
    log.error('Incompatible combination of testcase flags (%s): only one of '
        '--cloud-init, --test-image-update, --tpmtest, --ifnametest, --raid, '
        '--smoketest, --configtest, --sbtest may be set.', ', '.join(_primary_modes))
    sys.exit(1)

if args.no_interfaces and not args.smoketest:
    log.error('--no-interfaces requires --smoketest')
    sys.exit(1)

if args.sbtest and not args.uefi:
    log.error('--sbtest requires --uefi')
    sys.exit(1)

if args.logfile:
    filehandler = logging.FileHandler(args.logfile)
    filehandler.setLevel(logging.DEBUG)
    filehandler.setFormatter(formatter)
    log.addHandler(filehandler)

if args.silent:
    output = BytesIO()
else:
    output = sys.stdout.buffer

if not os.path.exists('/dev/kvm'):
    log.error('KVM not enabled on host, proceeding with software emulation')

if not args.iso and not args.disk:
    log.error('Neither ISO not QCOW2 disk image supplied - error!')
    sys.exit(1)

if not args.disk:
    tmp_disk_time = datetime.now().strftime('%Y%m%d-%H%M%S')
    tmp_disk_random = "%04x" % random.randint(0,65535)
    args.disk = f'testinstall-{tmp_disk_time}-{tmp_disk_random}{DISK_IMAGE_EXTENSION}'

if args.iso and not os.path.isfile(args.iso):
    log.error('Unable to find VyOS ISO image needed by testcases!')
    sys.exit(1)

if args.test_image_update and not args.iso:
    log.error('--test-image-update requires --iso file!')
    sys.exit(1)

OVMF_CODE = '/usr/share/OVMF/OVMF_CODE_4M.secboot.fd'
OVMF_VARS_TMP = args.disk.replace(DISK_IMAGE_EXTENSION, '.efivars')
if args.sbtest:
    shutil.copy('/usr/share/OVMF/OVMF_VARS_4M.ms.fd', OVMF_VARS_TMP)

# Creating diskimage!!
diskname_raid = None
def gen_disk(name):
    if not os.path.isfile(name):
        log.info(f'Creating Disk image {name}')
        c = subprocess.check_output(['qemu-img', 'create', name, '5G'])
        log.debug(c.decode())
    else:
        log.info(f'Re-using already existing disk image "{name}".')

if args.raid:
    filename, ext = os.path.splitext(args.disk)
    diskname_raid = f'{filename}_disk1{ext}'
    # change primary diskname, too
    args.disk = f'{filename}_disk0{ext}'
    gen_disk(diskname_raid)

# must be called after the raid disk as args.disk name is altered in the RAID path
gen_disk(args.disk)

nested_payload_iso_path = None
if args.test_image_update:
    if os.path.isdir(NESTED_ISO_DATA_DIR):
        shutil.rmtree(NESTED_ISO_DATA_DIR)
    os.makedirs(NESTED_ISO_DATA_DIR)
    shutil.copy2(args.iso, os.path.join(NESTED_ISO_DATA_DIR, NESTED_INNER_ISO_NAME))
    if os.path.isfile(NESTED_INSTALLER_PAYLOAD_ISO):
        os.unlink(NESTED_INSTALLER_PAYLOAD_ISO)
    log.info('Assembling nested installer payload ISO (second CD-ROM, drive-cd2)')
    subprocess.check_call([
        'mkisofs', '-joliet', '-rock', '-volid', 'VYOSNESTED',
        '-output', NESTED_INSTALLER_PAYLOAD_ISO, NESTED_ISO_DATA_DIR,
    ])
    nested_payload_iso_path = os.path.abspath(NESTED_INSTALLER_PAYLOAD_ISO)
    log.info('Removing nested ISO staging directory %s', NESTED_ISO_DATA_DIR)
    shutil.rmtree(NESTED_ISO_DATA_DIR)

# Create software emulated TPM - clear existing TPM data first - might be a
# leftover from a previous run
clearTPM()

def start_swtpm():
    if not os.path.exists(tpm_folder):
        os.mkdir(tpm_folder)

    def swtpm_thread():
        subprocess.check_output([
            'swtpm', 'socket', '--tpmstate', f'dir={tpm_folder}',
            '--ctrl', f'type=unixio,path={tpm_sock}', '--tpm2', '--log', 'level=1'
        ])

    from multiprocessing import Process
    tpm_process = Process(target=swtpm_thread)
    tpm_process.start()
    return tpm_process

def toggleUEFISecureBoot(c):
    def UEFIKeyPress(c, key):
        UEFI_SLEEP = 1
        c.send(key)
        time.sleep(UEFI_SLEEP)

    # Enter UEFI
    for ii in range(1, 10):
        c.send(KEY_F2)
        time.sleep(0.250)

    time.sleep(10)

    # Device Manager
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_RETURN)

    # Secure Boot Configuration
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_RETURN)

    # Attempt Secure Boot Toggle
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_RETURN)
    UEFIKeyPress(c, KEY_RETURN)

    # Save Secure Boot
    UEFIKeyPress(c, KEY_F10)
    UEFIKeyPress(c, KEY_Y)

    # Go Back to Menu
    UEFIKeyPress(c, KEY_ESC)
    UEFIKeyPress(c, KEY_ESC)

    # Go Down for reset
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_DOWN)
    UEFIKeyPress(c, KEY_RETURN)

def BOOTLOADERchooseSerialConsole(child, live: bool) -> None:
    """ Select GRUB boot entry that uses the serial console. This differs
        between a LIVE ISO image and an already installed system. """
    BOOTLOADER_TMO = 40
    BOOTLOADER_SLEEP = 1.5
    BOOTLOADER_LOAD_TMO = 5 # let GRUB screen load

    UEFI_STRING = r'BdsDxe.*'
    GRUB_STRING = r'GNU GRUB.*'
    ISOLINUX_STRING = r'ISOLINUX.*'

    # XXX: UEFI systems directly load GRUB, BIOS systems fallback to ISOLINUX
    if live and args.uefi:
        # Let UEFI start before GRUB
        child.expect(UEFI_STRING, timeout=BOOTLOADER_TMO)
        # Wait for GRUB
        child.expect(GRUB_STRING, timeout=BOOTLOADER_TMO)
        time.sleep(BOOTLOADER_LOAD_TMO)

        # Select GRUB serial console
        # Key DOWN -> fail-safe mode
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        # Key DOWN -> Serial Console boot
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        # Boot
        child.send(KEY_RETURN)
    elif live and not args.uefi:
        # Wait for ISOLINUX
        child.expect(ISOLINUX_STRING, timeout=BOOTLOADER_TMO)
        time.sleep(BOOTLOADER_LOAD_TMO)

        # Boot Menu starts at Live system (vyos) - KVM console
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        # Live system (vyos fail-safe mode)
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        # Live system (vyos) - Serial console - boot it up
        child.send(KEY_RETURN)
    else:
        # Let UEFI start before GRUB
        if args.uefi:
            child.expect(UEFI_STRING, timeout=BOOTLOADER_TMO)

        # Wait for GRUB
        child.expect(GRUB_STRING, timeout=BOOTLOADER_TMO)
        time.sleep(BOOTLOADER_LOAD_TMO)

        # Select GRUB serial console
        # Boot options
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        child.send(KEY_RETURN)
        time.sleep(BOOTLOADER_SLEEP)
        # GRUB submenus never time out on their own, so confirm we actually
        # landed on this submenu before navigating further - otherwise a
        # dropped keypress leaves the VM stuck here until the login wait
        # elsewhere expires
        child.expect('Select console type', timeout=BOOTLOADER_TMO)

        # Select console type
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        child.send(KEY_RETURN)
        time.sleep(BOOTLOADER_SLEEP)
        child.expect(r'ttyS \(serial\)', timeout=BOOTLOADER_TMO)

        # *ttyS (serial)
        child.send(KEY_DOWN)
        time.sleep(BOOTLOADER_SLEEP)
        child.send(KEY_RETURN)
        time.sleep(BOOTLOADER_SLEEP)
        # Boot
        child.send(KEY_RETURN)

    return None

def basic_cli_tests(c):
    # Repeating / shared basic CLI test between installed images and the
    # cloud-init test-case
    c.sendline('configure')
    c.expect(cfg_mode_prompt)
    c.sendline('exit')
    c.expect(op_mode_prompt)
    c.sendline('show version')
    c.expect(op_mode_prompt)
    c.sendline('show version kernel')
    c.expect(f'{vyos_defaults["kernel_version"]}-{vyos_defaults["kernel_flavor"]}')
    c.expect(op_mode_prompt)
    c.sendline('show version frr')
    c.expect(op_mode_prompt)
    c.sendline('show interfaces')
    c.expect(op_mode_prompt)
    # c.sendline('systemd-detect-virt')
    # c.expect('kvm')
    # c.expect(op_mode_prompt)
    c.sendline('show system cpu')
    c.expect(op_mode_prompt)
    c.sendline('show system memory')
    c.expect(op_mode_prompt)
    c.sendline('show system memory detail | no-more')
    c.expect(op_mode_prompt)
    c.sendline('show configuration commands | match "option kernel"')
    c.expect(op_mode_prompt)
    c.sendline('cat /proc/cmdline')
    c.expect(op_mode_prompt)
    c.sendline('show version all | grep -e "vpp" -e "vyos-1x"')
    c.expect(op_mode_prompt)

    # Get serial console interface name from flavor definition
    flavor_json = '/usr/share/vyos/flavor.json'
    c.sendline(f'jq -r ".console_type" {flavor_json}')
    c.expect(op_mode_prompt)
    lines = [l.strip() for l in c.before.splitlines() if l.strip()]
    console_type = lines[-1].decode('utf-8').strip() # e.g. ttyS

    # We do only care for ttyS or ttyAMA console types, exit early
    # for VGA consoles
    if console_type == 'tty':
        return

    c.sendline(f'jq -r ".console_num" {flavor_json}')
    c.expect(op_mode_prompt)
    lines = [l.strip() for l in c.before.splitlines() if l.strip()]
    console_num = lines[-1].decode('utf-8').strip() # e.g. 0

    # Get serial console speed from flavor definition
    c.sendline(f'jq -r ".console_speed" {flavor_json}')
    c.expect(op_mode_prompt)
    lines = [l.strip() for l in c.before.splitlines() if l.strip()]
    console_speed = lines[-1].decode('utf-8').strip() # e.g. 115200

    # Ensure serial console with kernel option is set in CLI, done automatically by image installer
    c.sendline('show configuration commands | match "system console"')
    c.expect(rf'set system console device {console_type}{console_num} kernel\r\r\nset system console device {console_type}{console_num} speed \'{console_speed}\'')
    c.expect(op_mode_prompt)

    # Validate GRUB has the right console_type defined
    c.sendline('cat /boot/grub/grub.cfg.d/20-vyos-defaults-autoload.cfg | grep "set console_type"')
    c.expect(f'set console_type="{console_type}"')
    c.expect(op_mode_prompt)

def verify_eth_mac_mapping(c, log):
    """ NICs are attached with a mix of drivers (virtio/e1000e/vmxnet3, see
        get_qemu_cmd()) to cover different naming schemes. Regardless of
        driver, udev must always enumerate them in ascending order: eth0
        carries mac0, eth1 mac1, ... up to eth7 mac7 - never scrambled. """
    log.info('Verify eth0..eth7 are enumerated in ascending MAC order')
    c.sendline('ip -json link show | jq -r \'.[] | select(.ifname|test("^eth[0-9]+$")) | "\(.ifname) \(.address)"\'')
    c.expect(op_mode_prompt)
    lines = [l.strip() for l in c.before.decode(errors='replace').splitlines() if l.strip()]

    macs = {}
    for line in lines:
        parts = line.split()
        if len(parts) == 2 and re.fullmatch(r'eth\d+', parts[0]):
            macs[parts[0]] = parts[1].lower()

    for i in range(8):
        ifname = f'eth{i}'
        expected_mac = f'{macbase}:{i:02x}'.lower()
        if ifname not in macs:
            raise Exception(f'Interface {ifname} not found on installed system')
        if macs[ifname] != expected_mac:
            raise Exception(f'Interface {ifname} has MAC {macs[ifname]}, expected {expected_mac} - naming race?')
    log.info('eth0..eth7 MAC mapping verified')

def _image_update_cli_sequence(c, log, new_image_name, server_bind_host='127.0.0.1', use_vrf=False):
    """One add-system-image/delete cycle for nested ISO over HTTP (optional Linux VRF + VyOS vrf arg)."""
    url = f'http://{server_bind_host}:{NESTED_HTTP_SERV_PORT}/{NESTED_INNER_ISO_NAME}'

    vrf_exec = ''
    if use_vrf:
        log.info(f'Configure VRF ({NESTED_HTTP_VRF_NAME}) for HTTP server - used for image update ')
        vrf_exec = f'ip vrf exec {NESTED_HTTP_VRF_NAME} '
        c.sendline('configure')
        c.expect(cfg_mode_prompt)
        c.sendline(f'set vrf name {NESTED_HTTP_VRF_NAME} table {NESTED_HTTP_VRF_TABLE}')
        c.expect(cfg_mode_prompt)
        c.sendline(f'set interfaces dummy {NESTED_HTTP_VRF_DUMMY} address {server_bind_host}/32')
        c.expect(cfg_mode_prompt)
        c.sendline(f'set interfaces dummy {NESTED_HTTP_VRF_DUMMY} vrf {NESTED_HTTP_VRF_NAME}')
        c.expect(cfg_mode_prompt)
        c.sendline('commit')
        c.expect(cfg_mode_prompt)
        c.sendline('exit')
        c.expect(op_mode_prompt)

    c.sendline(
        f'sudo bash -c \'{vrf_exec}python3 -m http.server {NESTED_HTTP_SERV_PORT} --bind {server_bind_host} '
        f'--directory {NESTED_HTTP_MOUNT_POINT} </dev/null >/tmp/nested_http.log 2>&1 &\''
    )
    c.expect(op_mode_prompt)

    time.sleep(5) # Wait for HTTP server to start

    update_cmd_cli = f'TERM=dumb add system image {url}'
    if use_vrf:
        update_cmd_cli = f'{update_cmd_cli} vrf {NESTED_HTTP_VRF_NAME}'
    c.sendline(update_cmd_cli)

    timeout = 600
    deadline = time.time() + timeout
    while True:
        if time.time() > deadline:
            raise Exception(f'add system image timed out after {timeout}s')
        i = c.expect([
            'What would you like to name this image',
            'Would you like to set the new image as the default one for boot',
            'An active configuration was found. Would you like to copy it to the new image',
            'Would you like to copy SSH host keys',
            'Would you like to copy Bash history',
            'Would you like to save the SSH known hosts (fingerprints)',
            'Signature is not available. Do you want to continue with installation',
            'There are unsaved changes to the configuration',
            'Would you like to continue',
            'Unable to',
            'Error:',
            op_mode_prompt,
        ], timeout=300)
        if i == 0:
            c.sendline(new_image_name)
        elif i == 1:
            c.sendline('n')
        elif i == 2:
            c.sendline('y')
        elif i == 3:
            c.sendline('y')
        elif i == 4:
            c.sendline('y')
        elif i == 5:
            c.sendline('y')
        elif i == 6:
            c.sendline('y')
        elif i == 7:
            c.sendline('y')
        elif i == 8:
            c.sendline('y')
        elif i == 9 or i == 10:
            raise Exception('add system image reported an error')
        elif i == 11:
            log.info('add system image completed')
            break

    c.sendline('show system image')
    c.expect(new_image_name)
    c.expect(op_mode_prompt)

    c.sendline(f'TERM=dumb delete system image {new_image_name}')
    deadline = time.time() + timeout
    while time.time() < deadline:
        j = c.expect([
            'Are you sure you want to delete',
            'Do you really want to delete the image',
            'Cannot ',
            'Error:',
            'Unable to ',
            op_mode_prompt,
        ], timeout=300)
        if j == 0 or j == 1:
            c.sendline('y')
        elif j == 2 or j == 3 or j == 4:
            raise Exception('delete system image failed')
        elif j == 5:
            break
    else:
        raise Exception('delete system image timed out')

    c.sendline('show system image')
    c.expect(op_mode_prompt)
    if new_image_name in c.before.decode(errors='replace'):
        raise Exception(f'Image {new_image_name!r} still listed after delete')

    c.sendline(f'sudo fuser -k {NESTED_HTTP_SERV_PORT}/tcp 2>/dev/null || true')
    c.expect(op_mode_prompt)

if args.qemu_cmd:
    tmp = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid,
                       iso_img=args.iso, vnc_enabled=args.vnc, secure_boot=args.sbtest,
                       nested_cdrom_iso=nested_payload_iso_path)
    os.system(tmp)
    exit(0)

if args.cloud_init:
    # Build cloud-init seed iso
    CI_ISO = 'ci_seed.iso'
    CI_DATA_DIR = 'ci_data'
    # Insert cloud-init ISO into CD-ROM
    args.iso = CI_ISO

    if not os.path.exists(CI_DATA_DIR):
        os.makedirs(CI_DATA_DIR)

    # create "empty" meta-data file
    open(f'{CI_DATA_DIR}/meta-data', mode='w').close()

    network_config = """version: 2
ethernets:
  eth0:
    dhcp4: false
    dhcp6: false
"""

    user_data = """#cloud-config
vyos_config_commands:
  - set system host-name '{hostname}'
  - set service ntp server 1.pool.ntp.org
  - set service ntp server 2.pool.ntp.org
  - delete interfaces ethernet eth0 address 'dhcp'
  - set interfaces ethernet eth0 address '{eth0_ipv4}'
  - set interfaces ethernet eth0 address '{eth0_ipv6}'
  - set protocols static route 0.0.0.0/0 next-hop '{default_nexthop}'
  - set protocols static route6 ::/0 next-hop '{default_nexthop6}'
""".format(**CI_CONFIG_ARGS)

    with open(f'{CI_DATA_DIR}/network-config', mode='w') as f:
        f.writelines(network_config)

    with open(f'{CI_DATA_DIR}/user-data', mode='w') as f:
        f.writelines(user_data)

    # Assemble cloud-init ISO file
    if os.path.exists(CI_ISO):
        os.unlink(CI_ISO)
    os.system(f'mkisofs -joliet -rock -volid "cidata" -output {CI_ISO} {CI_DATA_DIR}')

tpm_process = None
try:
    # Start TPM emulator
    if args.tpmtest:
        tpm_process = start_swtpm()

    #################################################
    # Installing image to disk
    #################################################
    log.info('Installing system')
    cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid,
                       tpm=args.tpmtest, iso_img=args.iso, vnc_enabled=args.vnc,
                       secure_boot=args.sbtest, nested_cdrom_iso=nested_payload_iso_path)
    log.debug(f'Executing command: {cmd}')
    c = pexpect.spawn(cmd, logfile=stl, timeout=60)

    #################################################
    # Logging into VyOS system
    #################################################
    if args.sbtest:
        log.info('Disable UEFI Secure Boot for initial installation')
        toggleUEFISecureBoot(c)

    BOOTLOADERchooseSerialConsole(c, live=(not args.cloud_init))
    loginVM(c, log)

    #################################################
    # Cloud-Init comes with a pre-assembled ISO - just boot and test it
    #################################################
    if args.cloud_init:
        log.info('Perform basic CLI configuration mode tests from cloud-init...')
        basic_cli_tests(c)

        # Valida assigned IPv4 and IPv6 addresses
        c.sendline('ip --json addr show dev eth0 | jq -r \'.[0].addr_info[] | select(.family=="inet") | "\(.local)/\(.prefixlen)"\'')
        c.expect(CI_CONFIG_ARGS['eth0_ipv4'])
        c.expect(op_mode_prompt)

        c.sendline('ip --json addr show dev eth0 | jq -r \'.[0].addr_info[] | select(.family=="inet6" and .scope=="global" and (.temporary|not)) | "\(.local)/\(.prefixlen)"\'')
        c.expect(CI_CONFIG_ARGS['eth0_ipv6'])
        c.expect(op_mode_prompt)

        # Valida default route nexthops
        c.sendline('ip -4 --json route show default | jq -r ".[0].gateway"')
        c.expect(CI_CONFIG_ARGS['default_nexthop'])
        c.expect(op_mode_prompt)

        c.sendline('ip -6 --json route show default | jq -r ".[0].gateway"')
        c.expect(CI_CONFIG_ARGS['default_nexthop6'])
        c.expect(op_mode_prompt)

        shutdownVM(c, log, 'Powering off system')
        c.close()
        raise EarlyExit

    #################################################
    # Check for no private key contents within the image
    #################################################
    msg = 'Found private key - bailing out'
    c.sendline(f'if sudo grep -rq "BEGIN PRIVATE KEY" /var/lib/shim-signed/mok; then echo {msg}; exit 1; fi')
    tmp = c.expect([f'\n{msg}', op_mode_prompt])
    if tmp == 0:
        log.error(msg)
        exit(1)

    #################################################
    # Configure boot options if required
    #################################################
    if args.huge_page_size and args.huge_page_count:
        c.sendline('configure')
        c.expect(cfg_mode_prompt)
        c.sendline(f'set system option kernel memory hugepage-size {args.huge_page_size} hugepage-count {args.huge_page_count}')
        c.expect(cfg_mode_prompt)
        if args.isolate_cpus:
            c.sendline(f'set system option kernel cpu isolate-cpus {args.isolate_cpus}')
            c.expect(cfg_mode_prompt)
        c.sendline('set system option kernel disable-mitigations')
        c.expect(cfg_mode_prompt)
        c.sendline('commit')
        c.expect(cfg_mode_prompt)
        c.sendline('save')
        c.expect(cfg_mode_prompt)
        c.sendline('exit')
        c.expect(op_mode_prompt)

    #################################################
    # Installing into VyOS system
    #################################################
    log.info('Starting installer')
    c.sendline('install image')
    c.expect('\nWould you like to continue?.*')
    c.sendline('y')
    c.expect('\nWhat would you like to name this image?.*')
    c.sendline('')
    c.expect(f'\nPlease enter a password for the "{default_user}" user.*')
    c.sendline('vyos')
    c.expect(f'\nPlease confirm password for the "{default_user}" user.*')
    c.sendline('vyos')
    c.expect('\nWhat console should be used by default?.*')
    c.sendline('S')

    if args.raid:
        c.expect('\nWould you like to configure RAID-1 mirroring??.*')
        c.sendline('y')
        c.expect('\nWould you like to configure RAID-1 mirroring on them?.*')
        c.sendline('y')
        c.expect('\nInstallation will delete all data on both drives. Continue?.*')
        c.sendline('y')
        c.expect('\nWhich file would you like as boot config?.*')
        c.sendline('')
    else:
        c.expect('\nWhich one should be used for installation?.*')
        c.sendline('')
        c.expect('\nInstallation will delete all data on the drive. Continue?.*')
        c.sendline('y')
        c.expect('\nWould you like to use all the free space on the drive?.*')
        c.sendline('y')
        c.expect('\nWhich file would you like as boot config?.*')
        c.sendline('')

    c.expect(op_mode_prompt)

    if args.sbtest:
        c.sendline('install mok')
        c.expect('input password:.*')
        c.sendline(mok_password)
        c.expect('input password again:.*')
        c.sendline(mok_password)
        c.expect(op_mode_prompt)

    log.info('system installed, rebooting')
    c.sendline('reboot now')

    #################################################
    # SHIM Mok Manager
    #################################################
    if args.sbtest:
        log.info('Install Secure Boot Machine Owner Key')
        MOK_SLEEP = 0.5
        c.expect('BdsDxe: starting Boot00.*')
        time.sleep(3)
        # press any key
        c.send(KEY_RETURN)
        time.sleep(MOK_SLEEP)

        # Enroll MOK
        c.send(KEY_DOWN)
        time.sleep(MOK_SLEEP)
        c.send(KEY_RETURN)
        time.sleep(MOK_SLEEP)

        # Continue
        c.send(KEY_DOWN)
        time.sleep(MOK_SLEEP)
        c.send(KEY_RETURN)
        time.sleep(MOK_SLEEP)

        # Enroll Keys
        c.send(KEY_DOWN)
        time.sleep(MOK_SLEEP)
        c.send(KEY_RETURN)
        time.sleep(MOK_SLEEP)

        c.sendline(mok_password)
        c.send(KEY_RETURN)
        time.sleep(MOK_SLEEP)

        # Reboot
        c.send(KEY_RETURN)
        time.sleep(MOK_SLEEP)

    #################################################
    # Re-Enable Secure Boot
    #################################################
    if args.sbtest:
        log.info('Enable UEFI Secure Boot for initial installation')
        toggleUEFISecureBoot(c)

    #################################################
    # Removing CD installation media
    #################################################
    time.sleep(2)
    log.info('eject installation media')
    os.system(f'echo "eject -f drive-cd1" | socat - unix-connect:/tmp/qemu-monitor-socket-{args.disk}')

    #################################################
    # Booting installed system
    #################################################
    log.info('Booting installed system')
    BOOTLOADERchooseSerialConsole(c, live=False)

    #################################################
    # Logging into VyOS system
    #################################################
    waitForLogin(c, log)
    loginVM(c, log)

    #################################################
    # Boot options require a reboot
    #################################################
    if args.huge_page_size and args.huge_page_count:
        log.info('Rebooting to apply kernel boot options')
        c.sendline('reboot now')
        loginVM(c, log)

    ################################################
    # Always load the WiFi simulation module
    ################################################
    c.sendline('sudo modprobe mac80211_hwsim')
    c.expect(op_mode_prompt)

    # Inform smoketest about this environment
    c.sendline('touch /tmp/vyos.smoketests.hint')
    c.expect(op_mode_prompt)

    #################################################
    # Basic Configmode/Opmode switch
    #################################################
    log.info('Basic CLI configuration mode test')
    basic_cli_tests(c)

    #################################################
    # Verify NIC driver mix did not scramble interface naming
    #################################################
    verify_eth_mac_mapping(c, log)

    #################################################
    # Verify /etc/os-release via lsb_release
    #################################################
    c.sendline('lsb_release --short --id 2>/dev/null')
    c.expect('VyOS')
    if os.path.isfile(manifest_file):
        c.sendline('lsb_release --short --release 2>/dev/null')
        c.expect(vyos_version)
        c.sendline('lsb_release --short --codename 2>/dev/null')
        c.expect(vyos_codename)

    # Ensure ephemeral key is loaded
    vyos_kernel_key = 'VyOS build time autogenerated kernel key'
    c.sendline(f'show log kernel | match "{vyos_kernel_key}"')
    c.expect(f'.*{vyos_kernel_key}.*')
    c.expect(op_mode_prompt)

    #################################################
    # Executing test-suite
    #################################################
    if args.tpmtest:
        def verify_mount():
            # Verify encrypted and mounted
            c.sendline('mount | grep vyos_config')
            c.expect('/dev/mapper/vyos_config on /opt/vyatta/etc/config.*')
            c.expect(op_mode_prompt)

        def verify_config():
            # Verify encrypted config is loaded
            c.sendline('show config commands | cat')
            c.expect('set system option performance \'network-latency\'')
            c.expect('set system option reboot-on-panic')
            c.expect(op_mode_prompt)

        log.info('Running TPM encrypted config tests')
        tpm_timeout = 600 # Give it 10 mins to encrypt

        # Verify TPM is loaded
        c.sendline('find /dev -name tpm0')
        c.expect('/dev/tpm0')
        c.expect(op_mode_prompt)

        # Create recovery key
        import string
        from random import choices
        test_recovery_key = ''.join(choices(string.ascii_uppercase + string.digits, k=32))

        log.info('Encrypting config to TPM')
        c.sendline('encryption enable')
        c.expect('Automatically generate a recovery key\?.*')
        c.sendline('n')
        c.expect('Enter recovery key: ')
        c.sendline(test_recovery_key)
        c.expect('Enter size of encrypted config partition.*', timeout=30)
        c.sendline('32')
        c.expect('Encrypted config volume has been enabled', timeout=tpm_timeout)
        c.expect('Backup the recovery key in a safe place!')
        c.expect(f'Recovery key: {test_recovery_key}')
        c.expect(op_mode_prompt)

        verify_mount()

        # Add some non-default config nodes this encrypted config
        log.info('Adding nodes for encrypted config test')
        c.sendline('configure')
        c.expect(cfg_mode_prompt)
        c.sendline('set system option performance network-latency')
        c.expect(cfg_mode_prompt)
        c.sendline('set system option reboot-on-panic')
        c.expect(cfg_mode_prompt)
        c.sendline('commit')
        c.expect(cfg_mode_prompt)
        c.sendline('save')
        c.expect(cfg_mode_prompt)
        c.sendline('exit')
        c.expect(op_mode_prompt)

        log.info('system installed, rebooting')
        c.sendline('reboot now')

        waitForLogin(c, log)

        # Check for vyos-router message
        c.expect('.*Mounted encrypted config volume', timeout=120)

        loginVM(c, log)

        verify_mount()
        verify_config()

        # Shutdown VM
        shutdownVM(c, log, 'Shutdown VM for TPM fail test')

        # Clear swtpm
        clearTPM()

        # Shutdown kills swtpm
        tpm_process.join()
        tpm_process.close()

        # Start emulator again
        tpm_process = start_swtpm()

        # Booting back into VM
        log.info('Booting system with cleared TPM')
        cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, tpm=args.tpmtest, vnc_enabled=args.vnc)
        log.debug(f'Executing command: {cmd}')
        c = pexpect.spawn(cmd, logfile=stl)

        waitForLogin(c, log)

        c.expect('.*Encrypted config volume has not been mounted', timeout=120)

        loginVM(c, log)

        # Test loading config with recovery key
        c.sendline('encryption load')
        c.expect('Enter recovery key: ')
        c.sendline(test_recovery_key)
        c.expect('Encrypted config volume has been mounted', timeout=120)
        c.expect(op_mode_prompt)

        verify_mount()

        log.info('Loading encrypted config.boot')
        c.sendline('configure')
        c.expect(cfg_mode_prompt)
        c.sendline('load /opt/vyatta/etc/config/config.boot')
        c.expect(cfg_mode_prompt)
        c.sendline('commit')
        c.expect(cfg_mode_prompt)
        c.sendline('exit')
        c.expect(op_mode_prompt)

        verify_config()

        # Shutdown VM
        shutdownVM(c, log, 'Shutdown VM for non-TPM backed test')
        clearTPM()

        # Shutdown kills swtpm
        tpm_process.join()
        tpm_process.close()
        tpm_process = None

        # Booting back into VM
        log.info('Booting system without TPM')
        cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, tpm=False, vnc_enabled=args.vnc)
        log.debug(f'Executing command: {cmd}')
        c = pexpect.spawn(cmd, logfile=stl)

        waitForLogin(c, log)
        loginVM(c, log)

        # New recovery key
        test_recovery_key = ''.join(choices(string.ascii_uppercase + string.digits, k=32))

        log.info('Encrypting config')
        c.sendline('encryption enable')
        c.expect('Are you sure you want to proceed\?.*')
        c.sendline('y')
        c.expect('Enter key: ')
        c.sendline(test_recovery_key)
        c.expect('Confirm key:')
        c.sendline(test_recovery_key)
        c.expect('Enter size of encrypted config partition.*', timeout=30)
        c.sendline('32')
        c.expect('Encrypted config volume has been enabled', timeout=tpm_timeout)
        c.expect('Backup the key in a safe place!')
        c.expect(f'Key: {test_recovery_key}')
        c.expect(op_mode_prompt)

        verify_mount()

        shutdownVM(c, log, 'Shutdown VM for non-TPM config load')

        # Booting back into VM
        log.info('Booting system without TPM')
        cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, tpm=False, vnc_enabled=args.vnc)
        log.debug(f'Executing command: {cmd}')
        c = pexpect.spawn(cmd, logfile=stl)

        waitForLogin(c, log)

        c.expect('.*encrypted config volume will not be automatically mounted', timeout=120)

        loginVM(c, log)

        # Test loading config with recovery key
        c.sendline('encryption load')
        c.expect('Enter key: ')
        c.sendline(test_recovery_key)
        c.expect('Encrypted config volume has been mounted', timeout=120)
        c.expect(op_mode_prompt)

        verify_mount()

        log.info('Loading encrypted config.boot')
        c.sendline('configure')
        c.expect(cfg_mode_prompt)
        c.sendline('load /opt/vyatta/etc/config/config.boot')
        c.expect(cfg_mode_prompt)
        c.sendline('commit')
        c.expect(cfg_mode_prompt)
        c.sendline('exit')
        c.expect(op_mode_prompt)

    elif args.ifnametest:
        # A missing/deleted hw-id binding, or a fully deleted interface
        # config, must not change the eth0..eth7 <-> MAC mapping after
        # the next reboot (regression check for the boot-time naming race).
        log.info('Running interface naming/hw-id persistence tests')
        del_idx, hwid_idx = random.sample(range(8), 2)
        log.info(f'Deleting eth{del_idx} entirely, removing hw-id only on eth{hwid_idx}')

        c.sendline('configure')
        c.expect(cfg_mode_prompt)
        c.sendline(f'delete interfaces ethernet eth{del_idx}')
        c.expect(cfg_mode_prompt)
        c.sendline(f'delete interfaces ethernet eth{hwid_idx} hw-id')
        c.expect(cfg_mode_prompt)
        c.sendline('commit')
        c.expect(cfg_mode_prompt)
        c.sendline('save')
        c.expect(cfg_mode_prompt)
        c.sendline('exit')
        c.expect(op_mode_prompt)

        log.info('Rebooting to verify interface naming survives across reboot')
        c.sendline('reboot now')
        waitForLogin(c, log)
        loginVM(c, log)

        log.info('Collecting interface naming diagnostics')
        c.sendline('show configuration commands | match "hw-id"')
        c.expect(op_mode_prompt)
        c.sendline('show interfaces ethernet')
        c.expect(op_mode_prompt)
        c.sendline('ip link show')
        c.expect(op_mode_prompt)
        c.sendline('show log | match "hw-id"')
        c.expect(op_mode_prompt)
        c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true')
        c.expect(op_mode_prompt)
        c.sendline('show log kernel | match "eth"')
        c.expect(op_mode_prompt)

        verify_eth_mac_mapping(c, log)

    elif args.raid:
        # Verify RAID subsystem - by deleting a disk and re-create the array
        # from scratch
        c.sendline('cat /proc/mdstat')
        c.expect(op_mode_prompt)

        shutdownVM(c, log, f'Shutdown VM and start with empty RAID member "{args.disk}"')

        if os.path.exists(args.disk):
            os.unlink(args.disk)

        gen_disk(args.disk)

        #################################################
        # Booting RAID-1 system with one missing disk
        #################################################
        log.info('Booting RAID-1 system')
        cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, vnc_enabled=args.vnc)

        # We need to swap boot indexes to boot from second harddisk so we can
        # recreate the RAID on the first disk
        cmd = cmd.replace('bootindex=1', 'bootindex=X')
        cmd = cmd.replace('bootindex=2', 'bootindex=1')
        cmd = cmd.replace('bootindex=X', 'bootindex=2')

        log.debug(f'Executing command: {cmd}')
        c = pexpect.spawn(cmd, logfile=stl)

        #################################################
        # Logging into VyOS system
        #################################################
        waitForLogin(c, log)
        loginVM(c, log)

        c.sendline('cat /proc/mdstat')
        c.expect(op_mode_prompt)

        log.info('Re-format new RAID member')
        c.sendline('format by-id disk drive-hd1 like drive-hd2')
        c.sendline('yes')
        c.expect(op_mode_prompt)

        log.info('Add member to RAID1 (md0)')
        c.sendline('add raid md0 by-id member drive-hd1-part3')
        c.expect(op_mode_prompt)

        log.info('Now we need to wait for re-sync to complete')

        start_time = time.time()
        timeout = 60
        while True:
            if (start_time + timeout) < time.time():
                break
            c.sendline('cat /proc/mdstat')
            c.expect(op_mode_prompt)
            time.sleep(20)

        # Reboot system with new primary RAID1 disk
        shutdownVM(c, log, f'Shutdown VM and start from recovered RAID member "{args.disk}"')

        log.info('Booting RAID-1 system')
        cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, vnc_enabled=args.vnc)
        log.debug(f'Executing command: {cmd}')
        c = pexpect.spawn(cmd, logfile=stl)

        loginVM(c, log)

        c.sendline('cat /proc/mdstat')
        c.expect(op_mode_prompt)

    elif args.smoketest:
        # run default smoketest suite
        smoketests_path = '/usr/libexec/vyos/tests/smoke'
        if args.match:
            # Remove tests that we don't want to run
            match_str = '-o '.join([f'-name "test_*{name}*.py" ' for name in args.match.split("|")]).strip()
            c.sendline(f'sudo find {smoketests_path} -maxdepth 2 -type f -name test_* ! \( {match_str} \) -delete')
            c.expect(op_mode_prompt)
        if args.no_interfaces:
            # remove interface tests as they consume a lot of time
            c.sendline(f'sudo rm -f {smoketests_path}/cli/test_interfaces_*')
            c.expect(op_mode_prompt)
        if args.no_vpp:
            # remove VPP tests
            c.sendline(f'sudo rm -f {smoketests_path}/cli/test_vpp*')
            c.expect(op_mode_prompt)

        if args.vyconf:
            c.sendline('sudo /usr/libexec/vyos/set_vyconf_backend.py --no-prompt &> /dev/null')
            c.expect(op_mode_prompt)
            log.info('Smoketests will be run using vyconfd/vyos-commitd')

        log.info('Executing VyOS smoketests')
        c.sendline('ls /usr/bin/vyos-smoketest 2>/dev/null')
        c.expect('/usr/bin/vyos-smoketest')
        c.expect(op_mode_prompt)
        c.sendline('/usr/bin/vyos-smoketest')
        i = c.expect(['\n +Invalid command:', '\n +Set failed',
                      r'\n\S+@\S+[$#]'], timeout=test_timeout)

        if i == 0:
            raise Exception('Invalid command detected')
        if i == 1:
            raise Exception('Set syntax failed :/')

        c.sendline('echo EXITCODE:$\x16?')
        i = c.expect(['EXITCODE:0', 'EXITCODE:\d+'])
        if i == 0:
            log.info('Smoketest finished successfully!')
            pass
        elif i == 1:
            log.error('Smoketest failed :/')
            raise Exception("Smoketest-failed, please look into debug output")

    # else, run configtest suite
    elif args.configtest:
        # Remove config-tests that we don't want to run
        if args.match:
            configs_path = '/usr/libexec/vyos/tests/configs'
            if args.match.startswith("!"):
                # Exclude mode — delete only the matched names
                names = args.match[1:].split("|")
                match_str = '-o '.join([f'-name "{name}"' for name in names])
                cleanup_config_dir_cmd = f'sudo find {configs_path} -mindepth 1 -maxdepth 1 -type f \\( {match_str} \\) -exec rm -rf {{}} +'
                cleanup_config_tests_dir_cmd = f'sudo find {configs_path}/assert -mindepth 1 -maxdepth 1 -type f \\( {match_str} \\) -exec rm -rf {{}} +'
            else:
                # Include mode — keep only the matched names, delete the rest
                names = args.match.split("|")
                match_str = '-o '.join([f'-name "{name}"' for name in names])
                cleanup_config_dir_cmd = f'sudo find {configs_path} -mindepth 1 -maxdepth 1 -type f ! \\( {match_str} \\) -exec rm -rf {{}} +'
                cleanup_config_tests_dir_cmd = f'sudo find {configs_path}/assert -mindepth 1 -maxdepth 1 -type f ! \\( {match_str} \\) -exec rm -rf {{}} +'

            c.sendline(cleanup_config_dir_cmd)
            c.expect(op_mode_prompt)
            c.sendline(cleanup_config_tests_dir_cmd)
            c.expect(op_mode_prompt)


        log.info('Adding a legacy WireGuard default keypair for migrations')
        c.sendline('sudo mkdir -p /config/auth/wireguard/default')
        c.expect(op_mode_prompt)
        c.sendline('echo "aGx+fvW916Ej7QRnBbW3QMoldhNv1u95/WHz45zDmF0=" | sudo tee /config/auth/wireguard/default/private.key')
        c.expect(op_mode_prompt)
        c.sendline('echo "x39C77eavJNpvYbNzPSG3n1D68rHYei6q3AEBEyL1z8=" | sudo tee /config/auth/wireguard/default/public.key')
        c.expect(op_mode_prompt)

        log.info('Generating PKI objects')
        c.sendline('/usr/bin/vyos-configtest-pki')
        c.expect(op_mode_prompt, timeout=900)

        script_file = '/config/scripts/vyos-foo-update.script'
        c.sendline(f'echo "#!/bin/sh" > {script_file}; chmod 775 {script_file}')
        c.expect(op_mode_prompt)

        log.info('Executing load config tests')
        c.sendline('/usr/bin/vyos-configtest')
        i = c.expect(['\n +Invalid command:', 'No such file or directory',
                     r'\n\S+@\S+[$#]'], timeout=test_timeout)

        if i==0:
            raise Exception('Invalid command detected')
        elif i==1:
            tmp = 'VyOS smoketest not found!'
            log.error(tmp)
            raise Exception(tmp)

        c.sendline('echo EXITCODE:$\x16?')
        i = c.expect(['EXITCODE:0', 'EXITCODE:\d+'])
        if i == 0:
            log.info('Configtest finished successfully!')
            pass
        elif i == 1:
            tmp = 'Configtest failed :/ - check debug output'
            log.error(tmp)
            raise Exception(tmp)
    elif args.test_image_update:
        log.info(f'Nested installer ISO: mount second CD-ROM ({NESTED_INNER_ISO_NAME})')
        c.sendline(f'sudo mkdir -p {NESTED_HTTP_MOUNT_POINT}')
        c.expect(op_mode_prompt)

        # After drive-cd1 eject the first SCSI CD node is often an empty tray; real
        # media may appear on sr1+. Try every optical device until one mounts.
        c.sendline(
            f'for d in /dev/sr*; do sudo mount -o ro "$d" {NESTED_HTTP_MOUNT_POINT} 2>/dev/null '
            '&& echo NESTED_ISO_MOUNT_OK && break; done; '
            f'mountpoint -q {NESTED_HTTP_MOUNT_POINT} || echo NESTED_ISO_MOUNT_FAIL'
        )
        if c.expect(['NESTED_ISO_MOUNT_OK', 'NESTED_ISO_MOUNT_FAIL']) != 0:
            raise Exception('Failed to mount nested installer payload ISO (no usable /dev/sr* media)')
        c.expect(op_mode_prompt)

        _image_update_cli_sequence(c, log, NESTED_HTTP_IMAGE_NAME)
        _image_update_cli_sequence(c, log, f'{NESTED_HTTP_IMAGE_NAME}-{NESTED_HTTP_VRF_NAME}', NESTED_HTTP_VRF_ADDR, use_vrf=True)

        log.info('Unmount nested installer ISO')
        c.sendline(f'sudo umount {NESTED_HTTP_MOUNT_POINT}')
        c.expect(op_mode_prompt)
        log.info('Nested installer ISO testcase complete')
    elif args.sbtest:
        c.sendline('show secure-boot')
        c.expect('SecureBoot enabled')
        c.expect(op_mode_prompt)
    else:
        log.info('No testcase selected!')

    shutdownVM(c, log, 'Powering off system')
    c.close()

except EarlyExit:
    pass

except pexpect.exceptions.TIMEOUT:
    log.error('Timeout waiting for VyOS system')
    log.error(traceback.format_exc())
    EXCEPTION = 1

except pexpect.exceptions.ExceptionPexpect:
    log.error('Exception while executing QEMU')
    log.error('Is qemu working on this system?')
    log.error(traceback.format_exc())
    EXCEPTION = 1

except Exception:
    log.error('Unknown error occurred!')
    traceback.print_exc()
    EXCEPTION = 1

#################################################
# Cleaning up
#################################################
log.info("Cleaning up")

if tpm_process:
    tpm_process.terminate()
    tpm_process.join()

if args.test_image_update:
    try:
        if os.path.isdir(NESTED_ISO_DATA_DIR):
            shutil.rmtree(NESTED_ISO_DATA_DIR)
        if nested_payload_iso_path and os.path.isfile(nested_payload_iso_path) \
                and not args.keep:
            os.remove(nested_payload_iso_path)
    except OSError:
        log.warning('Could not remove nested installer ISO artefacts', exc_info=True)

if not args.keep:
    log.info(f'Removing disk file: {args.disk}')
    try:
        os.remove(args.disk)
        if diskname_raid:
            os.remove(diskname_raid)
        if args.sbtest:
            os.remove(OVMF_VARS_TMP)
    except Exception:
        log.error('Exception while removing diskimage!')
        log.error(traceback.format_exc())
        EXCEPTION = 1

if EXCEPTION:
    log.error('Hmm... system got an exception while processing.')
    log.error('The ISO image is not considered usable!')
    sys.exit(1)

sys.exit(0)