From 0edf02d37d1c376ec605558e09f23f6fa86548a0 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Fri, 17 Feb 2023 18:41:07 -0700 Subject: [PATCH 1/4] boot-qemu.py: Refactor getting values from configuration This will come in handy in trying to warn people when they are missing configurations needed for certain features. Signed-off-by: Nathan Chancellor --- boot-qemu.py | 52 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/boot-qemu.py b/boot-qemu.py index eef5cd0..2d72958 100755 --- a/boot-qemu.py +++ b/boot-qemu.py @@ -48,6 +48,7 @@ def __init__(self): self.gh_json_file = None self.interactive = False self.kernel = None + self.kernel_config = None self.kernel_dir = None self.memory = '512m' self.supports_efi = False @@ -91,33 +92,40 @@ def _find_dtb(self): ) def _get_default_smp_value(self): - if not self.kernel_dir: - raise RuntimeError('No kernel build folder specified?') - - # If kernel_dir is the kernel source, the configuration will be at - # /.config - # - # If kernel_dir is the direct parent to the full kernel image, the - # configuration could either be: - # * /.config (if the image is vmlinux) - # * /../../../.config (if the image is in arch/*/boot/) - # * /config (if the image is in a TuxMake folder) - possible_locations = ['.config', '../../../.config', 'config'] - configuration = utils.find_first_file(self.kernel_dir, - possible_locations, - required=False) - - config_nr_cpus = 8 # sensible default based on treewide defaults, - if configuration: - conf_txt = configuration.read_text(encoding='utf-8') - if (match := re.search(r'CONFIG_NR_CPUS=(\d+)', conf_txt)): - config_nr_cpus = int(match.groups()[0]) - # Use the minimum of the number of usable processers for the script or # CONFIG_NR_CPUS. + config_nr_cpus_default = 8 # sensible default based on treewide defaults, + config_nr_cpus = int( + self._get_kernel_config_val('CONFIG_NR_CPUS', + config_nr_cpus_default)) usable_cpus = os.cpu_count() return min(usable_cpus, config_nr_cpus) + def _get_kernel_config_val(self, config, default='n'): + if not self.kernel_config: + if not self.kernel_dir: + raise RuntimeError('No kernel build folder specified?') + + # If kernel_dir is the kernel source, the configuration will be at + # /.config + # + # If kernel_dir is the direct parent to the full kernel image, the + # configuration could either be: + # * /.config (if the image is vmlinux) + # * /../../../.config (if the image is in arch/*/boot/) + # * /config (if the image is in a TuxMake folder) + possible_locations = ['.config', '../../../.config', 'config'] + self.kernel_config = utils.find_first_file(self.kernel_dir, + possible_locations, + required=False) + + if self.kernel_config: + conf_txt = self.kernel_config.read_text(encoding='utf-8') + if (match := re.search(fr'^{config}=(.*)$', conf_txt, flags=re.M)): + return match.groups()[0] + + return default + def _get_kernel_ver_tuple(self, decomp_prog): if not self.kernel: raise RuntimeError('No kernel set?') From eff8d85cd0abba8ca229613da200838f65f6866b Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 13 Mar 2023 12:05:31 -0700 Subject: [PATCH 2/4] boot-qemu.py: Add support for mounting a folder into the guest via virtiofs virtiofs, available in QEMU 5.2 or newer and Linux guests 5.4 or newer, is a more modern way to pass local folders along to QEMU, as it takes advantage of the fact that the folders are on the same machine as the hypervisor. To use virtiofs, we first need to find and run virtiofsd, which has two different implementations: a C implementation included with QEMU up until 8.0 (available on most distros) and a standalone Rust implementation available on GitLab (not packaged on many distros but easy to build and install). Once we find it, we run it in the background and connect to it using some QEMU parameters, which were shamelessly taken from the official virtiofs website: https://virtio-fs.gitlab.io/howto-qemu.html To use it within the guest (you can use a different path than /mnt/shared but 'mount -t virtio shared' must be used): # mkdir /mnt/shared # mount -t virtiofs shared /mnt/shared # echo "$(uname -a)" >/mnt/shared/foo On the host: $ cat shared/foo Linux (none) 6.1.0-rc8-next-20221207 #2 SMP PREEMPT Wed Dec 7 14:56:03 MST 2022 aarch64 GNU/Linux This does require guest kernel support (CONFIG_VIRTIO_FS=y), otherwise it will not work inside the guest: / # mount -t virtiofs shared /mnt/shared mount: mounting shared on /mnt/shared failed: No such device Closes: https://github.com/ClangBuiltLinux/boot-utils/issues/81 Link: https://gitlab.com/virtio-fs/virtiofsd Signed-off-by: Nathan Chancellor --- .gitignore | 2 + boot-qemu.py | 132 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 122 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 2740e0d..9783300 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ images/ qemu-binaries/ *.pyc +shared/ +.vfsd.* diff --git a/boot-qemu.py b/boot-qemu.py index 2d72958..007dc43 100755 --- a/boot-qemu.py +++ b/boot-qemu.py @@ -3,6 +3,7 @@ from argparse import ArgumentParser import contextlib +import grp import os from pathlib import Path import platform @@ -14,6 +15,7 @@ import utils +SHARED_FOLDER = Path(utils.BOOT_UTILS, 'shared') SUPPORTED_ARCHES = [ 'arm', 'arm32_v5', @@ -40,7 +42,7 @@ class QEMURunner: def __init__(self): - # Properties that can be adjusted by the user or class + # Properties that can be adjusted by the user or class (keep alphabetized if possible) self.cmdline = [] self.efi = False self.gdb = False @@ -51,14 +53,15 @@ def __init__(self): self.kernel_config = None self.kernel_dir = None self.memory = '512m' + self.share_folder_with_guest = False + self.smp = 0 self.supports_efi = False + self.timeout = '' # It may be tempting to use self.use_kvm during initialization of # subclasses to set certain properties but the user can explicitly opt # out of KVM after instantiation, so any decisions based on it should # be confined to run(). self.use_kvm = False - self.smp = 0 - self.timeout = '' self._default_kernel_path = None self._dtbs = [] @@ -72,6 +75,14 @@ def __init__(self): '-nodefaults', ] # yapf: disable self._qemu_path = None + self._vfsd_conf = { + 'cmd': [], + 'files': { + 'log': Path(utils.BOOT_UTILS, '.vfsd.log'), + 'mem': Path(utils.BOOT_UTILS, '.vfsd.mem'), + 'sock': Path(utils.BOOT_UTILS, '.vfsd.sock'), + }, + } def _find_dtb(self): if not self._dtbs: @@ -173,6 +184,65 @@ def _get_qemu_ver_tuple(self): def _have_dev_kvm_access(self): return os.access('/dev/kvm', os.R_OK | os.W_OK) + def _prepare_for_shared_folder(self): + if self._get_kernel_config_val('CONFIG_VIRTIO_FS') != 'y': + utils.yellow( + 'CONFIG_VIRTIO_FS may not be enabled in your configuration, shared folder may not work...' + ) + + # Print information about using shared folder + utils.green('To mount shared folder in guest (e.g. to /mnt/shared):') + utils.green('\t/ # mkdir /mnt/shared') + utils.green('\t/ # mount -t virtiofs shared /mnt/shared') + + SHARED_FOLDER.mkdir(exist_ok=True, parents=True) + + # Make sure sudo is available and we have permission to use it + if not (sudo := shutil.which('sudo')): + raise FileNotFoundError( + 'sudo is required to use virtiofsd but it could not be found!') + utils.green( + 'Requesting sudo permission to run virtiofsd in the background...') + subprocess.run([sudo, 'true'], check=True) + + # There are two implementations of virtiofsd. The original C + # implementation was bundled and built with QEMU up until 8.0, where it + # was removed after being deprecated in 7.0: + # + # https://lore.kernel.org/20230216182628.126139-1-dgilbert@redhat.com/ + # + # The standalone Rust implementation is preferred now, which should be + # available in PATH. If it is not available, see if there is a C + # implementation available in QEMU's prefix. + if not (virtiofsd := shutil.which('virtiofsd')): + utils.yellow( + 'Could not find Rust implementation of virtiofsd (https://gitlab.com/virtio-fs/virtiofsd), searching for old C implementation...' + ) + + qemu_prefix = self._qemu_path.resolve().parents[1] + virtiofsd_locations = [ + Path('libexec/virtiofsd'), # Default QEMU installation, Fedora + Path('lib/qemu/virtiofsd'), # Arch Linux, Debian, Ubuntu + ] + virtiofsd = utils.find_first_file(qemu_prefix, virtiofsd_locations) + + # Prepare QEMU arguments + self._qemu_args += [ + '-chardev', f"socket,id=char0,path={self._vfsd_conf['files']['sock']}", + '-device', 'vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=shared', + '-object', f"memory-backend-file,id=shm,mem-path={self._vfsd_conf['files']['mem']},share=on,size={self.memory}", + '-numa', 'node,memdev=shm', + ] # yapf: disable + + self._vfsd_conf['cmd'] = [ + sudo, + virtiofsd, + f"--socket-group={grp.getgrgid(os.getgid()).gr_name}", + f"--socket-path={self._vfsd_conf['files']['sock']}", + '-o', f"source={SHARED_FOLDER}", + '-o', 'cache=always', + ] # yapf: disable + def _prepare_initrd(self): if not self._initrd_arch: raise RuntimeError('No initrd architecture specified?') @@ -180,6 +250,9 @@ def _prepare_initrd(self): gh_json_file=self.gh_json_file) def _run_fg(self): + if self.share_folder_with_guest: + self._prepare_for_shared_folder() + # Pretty print and run QEMU command qemu_cmd = [] @@ -192,15 +265,32 @@ def _run_fg(self): qemu_cmd += [self._qemu_path, *self._qemu_args] - print(f"$ {' '.join(shlex.quote(str(elem)) for elem in qemu_cmd)}") - try: - subprocess.run(qemu_cmd, check=True) - except subprocess.CalledProcessError as err: - if err.returncode == 124: - utils.red("ERROR: QEMU timed out!") - else: - utils.red("ERROR: QEMU did not exit cleanly!") - sys.exit(err.returncode) + print(f"\n$ {' '.join(shlex.quote(str(elem)) for elem in qemu_cmd)}") + null_cm = contextlib.nullcontext() + with self._vfsd_conf['files']['log'].open('w', encoding='utf-8') if self.share_folder_with_guest else null_cm as vfsd_log, \ + subprocess.Popen(self._vfsd_conf['cmd'], stderr=vfsd_log, stdout=vfsd_log) if self.share_folder_with_guest else null_cm as vfsd_proc: + try: + subprocess.run(qemu_cmd, check=True) + except subprocess.CalledProcessError as err: + if err.returncode == 124: + utils.red("ERROR: QEMU timed out!") + else: + utils.red("ERROR: QEMU did not exit cleanly!") + # If virtiofsd is dead, it is pretty likely that it was the + # cause of QEMU failing so add to the existing exception using + # 'from'. + if vfsd_proc and vfsd_proc.poll(): + # yapf: disable + vfsd_log_txt = self._vfsd_conf['files']['log'].read_text(encoding='utf-8') + raise RuntimeError(f"virtiofsd failed with: {vfsd_log_txt}") from err + # yapf: enable + sys.exit(err.returncode) + finally: + if vfsd_proc: + vfsd_proc.kill() + # Delete the memory to save space, it does not have to be + # persistent + self._vfsd_conf['files']['mem'].unlink(missing_ok=True) def _run_gdb(self): qemu_cmd = [self._qemu_path, *self._qemu_args] @@ -846,6 +936,12 @@ def parse_arguments(): help= 'Number of processors for virtual machine (default: only KVM machines will use multiple vCPUs.)', ) + parser.add_argument( + '--share-folder-with-guest', + action='store_true', + help= + f"Share {SHARED_FOLDER} with the guest using virtiofs (requires interactive, not supported with gdb).", + ) parser.add_argument('-t', '--timeout', default='3m', @@ -918,6 +1014,18 @@ def parse_arguments(): if args.no_kvm: runner.use_kvm = False + if args.share_folder_with_guest: + if args.gdb: + utils.yellow( + 'Shared folder requested during a debugging session, ignoring...' + ) + elif not args.interactive: + utils.yellow( + 'Shared folder requested without an interactive session, ignoring...' + ) + else: + runner.share_folder_with_guest = True + if args.smp: runner.smp = args.smp From 86e0a454e3979efc4cb2acc7d0e38ea208ba3ab7 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 3 Apr 2023 10:08:03 -0700 Subject: [PATCH 3/4] boot-qemu.py: Add another location for virtiofsd Signed-off-by: Nathan Chancellor --- boot-qemu.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/boot-qemu.py b/boot-qemu.py index 007dc43..c4753ae 100755 --- a/boot-qemu.py +++ b/boot-qemu.py @@ -212,9 +212,11 @@ def _prepare_for_shared_folder(self): # https://lore.kernel.org/20230216182628.126139-1-dgilbert@redhat.com/ # # The standalone Rust implementation is preferred now, which should be - # available in PATH. If it is not available, see if there is a C - # implementation available in QEMU's prefix. - if not (virtiofsd := shutil.which('virtiofsd')): + # available in PATH or at '/usr/lib/virtiofsd', in the case of Arch + # Linux. If it is not available, see if there is a C implementation + # available in QEMU's prefix. + if not ((virtiofsd := shutil.which('virtiofsd')) or + (virtiofsd := Path('/usr/lib/virtiofsd')).exists()): utils.yellow( 'Could not find Rust implementation of virtiofsd (https://gitlab.com/virtio-fs/virtiofsd), searching for old C implementation...' ) From 901a795068f6df89ec042be1d17d063cc74ea373 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 19 Jul 2023 12:31:06 -0700 Subject: [PATCH 4/4] boot-qemu.py: Handle difference in arguments between C and Rust implementations of virtiofsd In at least virtiofsd 1.6.1 (the Rust implementation), the '-o' options warn that they are deprecated and the '--help' text agrees: [2023-07-19T19:32:50Z WARN virtiofsd] Use of deprecated option format '-o': Please specify options without it (e.g., '--cache auto' instead of '-o cache=auto') -o ... Options in a format compatible with the legacy implementation [deprecated] To defend against a release removing the deprecated option and breaking the invocation, maintain two sets of arguments depending on what implementation is being used. This allows us to drop support for the C implementation once the Rust one is more widely available in distributions. Signed-off-by: Nathan Chancellor --- boot-qemu.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/boot-qemu.py b/boot-qemu.py index c4753ae..9e5ebcb 100755 --- a/boot-qemu.py +++ b/boot-qemu.py @@ -236,13 +236,42 @@ def _prepare_for_shared_folder(self): '-numa', 'node,memdev=shm', ] # yapf: disable + # recent versions of the rust implementation of virtiofsd have + # deprecated the '-o' syntax for options. Check for the '-o' syntax in + # the help text of virtiofsd and use that if present or the new syntax + # if not. + base_virtiofsd_cmd = [sudo, virtiofsd] + virtiofsd_version_text = subprocess.run( + [*base_virtiofsd_cmd, '--version'], + capture_output=True, + check=True, + text=True).stdout + # C / QEMU / Reference implementation (deprecated) + if 'virtiofsd version' in virtiofsd_version_text: + virtiofsd_args = [ + f"--socket-group={grp.getgrgid(os.getgid()).gr_name}", + f"--socket-path={self._vfsd_conf['files']['sock']}", + '-o', 'cache=always', + '-o', f"source={SHARED_FOLDER}", + ] # yapf: disable + # Rust implementation + # The some of the above options are parsed as legacy compatibility + # options and as of at least 1.7.1, they are documented as deprecated. + # To guard against a release where those options are no longer parsed + # properly or at all, use the new option format. Once the Rust + # implementation is more widely available in distributions, support for + # the deprecated C implementation can be dropped. + else: + virtiofsd_args = [ + '--cache', 'always', + '--shared-dir', SHARED_FOLDER, + '--socket-group', grp.getgrgid(os.getgid()).gr_name, + '--socket-path', self._vfsd_conf['files']['sock'], + ] # yapf: disable + self._vfsd_conf['cmd'] = [ - sudo, - virtiofsd, - f"--socket-group={grp.getgrgid(os.getgid()).gr_name}", - f"--socket-path={self._vfsd_conf['files']['sock']}", - '-o', f"source={SHARED_FOLDER}", - '-o', 'cache=always', + *base_virtiofsd_cmd, + *virtiofsd_args ] # yapf: disable def _prepare_initrd(self):