Allow attaching GDB to guest VMs launched with KMT

.gitlab/common/test_infra_version.yml

@@ -5,5 +5,5 @@
 
 ---
 variables:
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: ef809e329658
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: b58de42a23a2
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ''

tasks/kernel_matrix_testing/gdb.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import semver
+from invoke.context import Context
+
+from tasks.kernel_matrix_testing import stacks
+from tasks.kernel_matrix_testing.infra import LibvirtDomain, build_infrastructure
+from tasks.kernel_matrix_testing.platforms import get_platforms
+from tasks.kernel_matrix_testing.tool import Exit, info
+from tasks.kernel_matrix_testing.vars import KMTPaths
+from tasks.libs.common.utils import get_repo_root
+from tasks.libs.types.arch import Arch
+
+if TYPE_CHECKING:
+    from tasks.kernel_matrix_testing.types import (
+        Component,  # noqa: F401
+        KMTArchNameOrLocal,
+    )
+
+
+class GDBPaths:
+    def __init__(self, vm_tag: str, image_version: str, stack: str, arch: KMTArchNameOrLocal):
+        self.tag = vm_tag
+        self.image_version = image_version
+        self.kmt_paths = KMTPaths(stack, Arch.from_str(arch))
+
+    @property
+    def vmlinux(self):
+        return self.kmt_paths.gdb / self.tag / self.image_version / "vmlinux.dbg"
+
+    @property
+    def kernel_source(self):
+        return self.kmt_paths.gdb / self.tag / self.image_version / "kernel-source"
+
+
+class UbuntuGDBProvision:
+    def __init__(self, vm: LibvirtDomain, image_version: str, kernel: str):
+        self.target = vm
+        self.image_version = image_version
+        self.kernel = semver.VersionInfo.parse(kernel)
+
+    def run(self, ctx: Context, stack: str):
+        self.target.copy(
+            ctx, get_repo_root() / "tasks/kernel_matrix_testing/provision/ubuntu-dbg.sh", "/tmp/provision.sh"
+        )
+        self.target.run_cmd(ctx, "chmod +x /tmp/provision.sh && /tmp/provision.sh")
+
+        gdb_paths = GDBPaths(self.target.tag, self.image_version, stack, self.target.arch)
+        gdb_paths.vmlinux.parent.mkdir(exist_ok=True, parents=True)
+        self.target.download(ctx, "/usr/lib/debug/boot/vmlinux.dbg", f"{gdb_paths.vmlinux}")
+
+        ctx.run(f"rm -rf {gdb_paths.kernel_source}")
+        gdb_paths.kernel_source.mkdir(parents=True)
+        self.target.download(
+            ctx,
+            f"/usr/src/linux-source-{self.kernel.finalize_version()}/linux-source-{self.kernel.finalize_version()}.tar.bz2",
+            f"{gdb_paths.kernel_source.parent}",
+        )
+        ctx.run(
+            f"cd {gdb_paths.kernel_source.parent} && tar xvf linux-source-{self.kernel.finalize_version()}.tar.bz2 -C {gdb_paths.kernel_source} --strip-components=1",
+            hide="out",
+            echo=True,
+        )
+
+        if self.kernel > semver.VersionInfo.parse("4.4.0"):
+            ctx.run(f"cd {gdb_paths.kernel_source} && make defconfig && make scripts_gdb")
+
+        self.target.run_cmd(ctx, "shutdown -h now", verbose=True, allow_fail=True)
+
+
+gdb_provision = {
+    "ubuntu": {
+        "22.04": UbuntuGDBProvision,
+        "23.10": UbuntuGDBProvision,
+        "24.04": UbuntuGDBProvision,
+        "24.10": UbuntuGDBProvision,
+        "20.04": UbuntuGDBProvision,
+        # TODO: Add support for bionic/ubuntu_18.04. Currently failing to find debug kernel build.
+        "16.04": UbuntuGDBProvision,
+    }
+}
+
+
+def setup_gdb_debugging(ctx: Context, stack: str) -> None:
+    infra = build_infrastructure(stack)
+    platforms = get_platforms()
+
+    arch = Arch.local().kmt_arch
+    for kmt_arch, instance in infra.items():
+        if kmt_arch != "local":
+            # TODO: add support to attach gdb to remote VMs
+            raise Exit("stacks with remote VMs cannot be launched with GDB")
+
+        for vm in instance.microvms:
+            platinfo = platforms[arch][vm.tag]
+            os_id = platinfo['os_id']
+            os_version = platinfo['os_version']
+            image_version = platinfo['image_version']
+            kernel = platinfo['kernel']
+
+            if os_id not in gdb_provision:
+                raise Exit(f"{os_id} is currently not supported for kernel debugging")
+
+            if os_version not in gdb_provision[os_id]:
+                raise Exit(f"{os_id}_{os_version} is currently not supported for kernel debugging")
+
+            provisioner = gdb_provision[os_id][os_version](vm, image_version, kernel)
+            info(f"[+] Provisioning {vm.tag} for debugging.")
+            provisioner.run(ctx, stack)
+
+    stacks.pause_stack(stack)
+    stacks.resume_stack(stack)

tasks/kernel_matrix_testing/infra.py

@@ -103,6 +103,7 @@ def __init__(
         arch: KMTArchNameOrLocal | None,
         instance: HostInstance,
         user: str = "root",
+        gdb_port: int = 0,
     ):
         self.ip = ip
         self.name = domain_id
@@ -112,6 +113,7 @@ def __init__(
         self.instance = instance
         self.arch = arch
         self.user = user
+        self.gdb_port = gdb_port
 
     def run_cmd(self, ctx: Context, cmd: str, allow_fail=False, verbose=False, timeout_sec=None):
         if timeout_sec is not None:
@@ -214,7 +216,14 @@ def build_infrastructure(stack: str, ssh_key_obj: SSHKey | None = None):
             # location in the local machine.
             instance.add_microvm(
                 LibvirtDomain(
-                    vm["ip"], vm["id"], vm["tag"], vm["vmset-tags"], os.fspath(get_kmt_os().ddvm_rsa), arch, instance
+                    vm["ip"],
+                    vm["id"],
+                    vm["tag"],
+                    vm["vmset-tags"],
+                    os.fspath(get_kmt_os().ddvm_rsa),
+                    arch,
+                    instance,
+                    gdb_port=vm["gdb-port"],
                 )
             )
 

tasks/kernel_matrix_testing/provision/bata24.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+if [[ $UID = 0 ]] ; then
+  echo "Please dont run this script as root, since the gef scripts will get setup for the root user"
+  exit 1
+fi
+
+echo "[+] apt"
+sudo apt-get update
+sudo apt-get install -y gdb-multiarch binutils gcc file python3-pip ruby-dev git
+
+echo "[+] pip3"
+pip3 install crccheck unicorn capstone ropper keystone-engine tqdm
+
+echo "[+] install seccomp-tools, one_gadget"
+if [ "x$(which seccomp-tools)" = "x" ]; then
+    sudo gem install seccomp-tools
+fi
+
+if [ "x$(which one_gadget)" = "x" ]; then
+    sudo gem install one_gadget
+fi
+
+echo "[+] install rp++"
+if [ "x$(uname -m)" = "xx86_64" ]; then
+    if [ "x$(which rp-lin)" = "x" ] && [ ! -e /usr/local/bin/rp-lin ]; then
+        wget -q https://github.com/0vercl0k/rp/releases/download/v2.1.3/rp-lin-clang.zip -P /tmp
+        sudo unzip /tmp/rp-lin-clang.zip -d /usr/local/bin/
+        sudo chmod +x /usr/local/bin/rp-lin
+        rm /tmp/rp-lin-clang.zip
+    fi
+fi
+
+echo "[+] install vmlinux-to-elf"
+if [ "x$(which vmlinux-to-elf)" = "x" ] && [ ! -e /usr/local/bin/vmlinux-to-elf ]; then
+    pip3 install --upgrade lz4 zstandard git+https://github.com/clubby789/python-lzo@b4e39df
+    pip3 install --upgrade git+https://github.com/marin-m/vmlinux-to-elf
+fi
+
+echo "[+] download gef"
+if [ -e ~/.gdbinit-gef.py ]; then
+    echo "[-] ~/.gdbinit-gef.py already exists. Please delete or rename."
+    echo "[-] INSTALLATION FAILED"
+    exit 1
+else
+    wget -q https://raw.githubusercontent.com/bata24/gef/dev/gef.py -O ~/.gdbinit-gef.py
+fi
+
+echo "[+] setup gef"
+STARTUP_COMMAND="source ~/.gdbinit-gef.py"
+if [ ! -e ~/.gdbinit ] || [ "x$(grep "$STARTUP_COMMAND" ~/.gdbinit)" = "x" ]; then
+    echo "$STARTUP_COMMAND" >> ~/.gdbinit
+fi
+
+echo "[+] INSTALLATION SUCCESSFUL"
+exit 0

tasks/kernel_matrix_testing/provision/ubuntu-dbg.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+set -o xtrace
+
+# 1. disable kaslr
+[ -f $(grep -q "GRUB_CMDLINE_LINUX=\".*nokaslr" /etc/default/grub) ] && sed -i 's/^GRUB_CMDLINE_LINUX="/&nokaslr /' /etc/default/grub
+update-grub
+
+# 2. download kernel debug build and kernel sources
+codename=$(lsb_release -c | awk  '{print $2}')
+if [ "${codename}" == "xenial" ]; then
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C8CAB6595FDFF622
+    tee /etc/apt/sources.list.d/ddebs.list << EOF
+    deb http://ddebs.ubuntu.com/ ${codename}      main restricted universe multiverse
+    deb http://ddebs.ubuntu.com/ ${codename}-updates  main restricted universe multiverse
+    deb http://ddebs.ubuntu.com/ ${codename}-proposed main restricted universe multiverse
+EOF
+else
+    apt install -y ubuntu-dbgsym-keyring
+    echo "\
+Types: deb
+URIs: http://ddebs.ubuntu.com/
+Suites: $(lsb_release -cs) $(lsb_release -cs)-updates $(lsb_release -cs)-proposed
+Components: main restricted universe multiverse
+Signed-by: /usr/share/keyrings/ubuntu-dbgsym-keyring.gpg" | tee -a /etc/apt/sources.list.d/ddebs.sources
+fi
+
+apt update
+apt install -y linux-image-`uname -r`-dbgsym linux-source
+
+cp /usr/lib/debug/boot/vmlinux-`uname -r` /usr/lib/debug/boot/vmlinux.dbg

tasks/kernel_matrix_testing/stacks.py

@@ -184,7 +184,13 @@ def check_env(ctx: Context):
 
 
 def launch_stack(
-    ctx: Context, stack: str | None, ssh_key: str | None, x86_ami: str, arm_ami: str, provision_microvms: bool
+    ctx: Context,
+    stack: str | None,
+    ssh_key: str | None,
+    x86_ami: str,
+    arm_ami: str,
+    provision_microvms: bool,
+    with_gdb: bool,
 ):
     stack = check_and_get_stack(stack)
     if not stack_exists(stack):
@@ -231,6 +237,7 @@ def launch_stack(
         vmconfig=vm_config,
         stack_name=stack,
         local=local,
+        with_gdb=with_gdb,
     )
 
     prefix = ""
@@ -286,6 +293,7 @@ def start_microvms_cmd(
     provision_microvms=False,
     run_agent=False,
     agent_version=None,
+    with_gdb=False,
 ):
     args = [
         f"--instance-type-x86 {instance_type_x86}" if instance_type_x86 else "",
@@ -305,6 +313,7 @@ def start_microvms_cmd(
         f"--agent-version {agent_version}" if agent_version else "",
         "--provision-instance" if provision_instance else "",
         "--provision-microvms" if provision_microvms else "",
+        "--setup-gdb" if with_gdb else "",
     ]
     go_args = ' '.join(filter(lambda x: x != "", args))
     return f"./test/new-e2e/start-microvms {go_args}"

tasks/kernel_matrix_testing/vars.py

@@ -55,3 +55,7 @@ def test_results(self):
 
     def vm_test_results(self, vm_name: str):
         return self.test_results / vm_name
+
+    @property
+    def gdb(self):
+        return self.root / self.arch.kmt_arch / "gdb"

tasks/kmt.py

@@ -30,6 +30,7 @@
 from tasks.kernel_matrix_testing.compiler import CONTAINER_AGENT_PATH, get_compiler
 from tasks.kernel_matrix_testing.config import ConfigManager
 from tasks.kernel_matrix_testing.download import update_rootfs
+from tasks.kernel_matrix_testing.gdb import GDBPaths, setup_gdb_debugging
 from tasks.kernel_matrix_testing.infra import (
     SSH_OPTIONS,
     HostInstance,
@@ -297,6 +298,49 @@ def gen_config_from_ci_pipeline(
     print(f"dda inv -- kmt.test --packages=\"{','.join(failed_packages)}\" --run='^{'|'.join(failed_tests)}$'")
 
 
+@task
+def attach_gdb(ctx: Context, vm: str, stack: str | None = None, dry=True):
+    stack = check_and_get_stack(stack)
+    if not stacks.stack_exists(stack):
+        raise Exit(f"Stack {stack} does not exist. Please create with 'dda inv kmt.create-stack --stack=<name>'")
+
+    if not os.path.exists(f"{Path.home()}/.gdbinit-gef.py"):
+        resp = ask(
+            "It is recommended to use gdb with the bata24 extension (https://github.com/bata24/gef) which greatly enhances the kernel debugging experience. You can install the extension with `inv -e kmt.install_bata24_gef`. Continue without installing? (y/N)"
+        )
+        if resp.lower().strip() != "y":
+            raise Exit("Aborted by user")
+
+    domains = get_target_domains(ctx, stack, None, None, vm, None)
+    assert len(domains) > 0, f"no running VM discovered for the provided vm {vm}"
+    assert len(domains) == 1, "GDB can only be attached to one VM at a time"
+
+    domain = domains[0]
+    if domain.gdb_port == 0:
+        raise Exit(
+            "VM was not launched with GDB debugging support. To use this feature specify the `--gdb` flag when invoke the `kmt.launch-stack` task"
+        )
+
+    platforms = get_platforms()
+    kmt_arch = Arch.from_str(domain.arch).kmt_arch
+    platinfo = platforms[kmt_arch][domain.tag]
+    gdb_paths = GDBPaths(domain.tag, platinfo['image_version'], stack, domain.arch)
+
+    cmd = f"gdb \
+-ex \"add-auto-load-safe-path {gdb_paths.kernel_source}\" \
+-ex \"file {gdb_paths.vmlinux}\" \
+-ex \"set arch i386:x86-64:intel\" -ex \"target remote localhost:{domain.gdb_port}\" \
+-ex \"source {gdb_paths.kernel_source}/vmlinux-gdb.py\" \
+-ex \"set disassembly-flavor intel\" \
+-ex \"set pagination off\" \
+"
+
+    if dry:
+        info(f"[+] Run the following command: {cmd}")
+    else:
+        ctx.run(cmd)
+
+
 @task
 def launch_stack(
     ctx: Context,
@@ -306,15 +350,25 @@ def launch_stack(
     arm_ami: str = ARM_AMI_ID_SANDBOX,
     provision_microvms: bool = True,
     provision_script: str | None = None,
+    gdb: bool = False,
 ):
     stack = check_and_get_stack(stack)
     if not stacks.stack_exists(stack):
         raise Exit(f"Stack {stack} does not exist. Please create with 'dda inv kmt.create-stack --stack=<name>'")
 
-    stacks.launch_stack(ctx, stack, ssh_key, x86_ami, arm_ami, provision_microvms)
+    if gdb and get_kmt_os().name != "linux":
+        # TODO: add kernel debugging support for MacOS
+        raise Exit("GDB attached to guest VM is only supported for linux systems")
+
+    stacks.launch_stack(ctx, stack, ssh_key, x86_ami, arm_ami, provision_microvms, gdb)
+
     if provision_script is not None:
         provision_stack(ctx, provision_script, stack, ssh_key)
 
+    if gdb:
+        setup_gdb_debugging(ctx, stack)
+        info("[+] GDB setup complete")
+
 
 @task
 def provision_stack(
@@ -2544,3 +2598,8 @@ def start_microvms(
             agent_version=agent_version,
         )
     )
+
+
+@task
+def install_bata24_gef(ctx):
+    ctx.run("tasks/kernel_matrix_testing/provision/bata24.sh")

test/new-e2e/go.mod

@@ -21,7 +21,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.4-0.20250702174234-ef809e329658
+	github.com/DataDog/test-infra-definitions v0.0.4-0.20250707171134-b58de42a23a2
 	github.com/aws/aws-sdk-go-v2 v1.36.5
 	github.com/aws/aws-sdk-go-v2/config v1.29.17
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.226.0