Skip to content

Commit 663e81f

Browse files
authored
Allow attaching GDB to guest VMs launched with KMT (#38503)
1 parent c529751 commit 663e81f

9 files changed

Lines changed: 297 additions & 3 deletions

File tree

tasks/kernel_matrix_testing/gdb.py

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
from __future__ import annotations
2+
3+
from typing import TYPE_CHECKING
4+
5+
import semver
6+
from invoke.context import Context
7+
8+
from tasks.kernel_matrix_testing import stacks
9+
from tasks.kernel_matrix_testing.infra import LibvirtDomain, build_infrastructure
10+
from tasks.kernel_matrix_testing.platforms import get_platforms
11+
from tasks.kernel_matrix_testing.tool import Exit, info
12+
from tasks.kernel_matrix_testing.vars import KMTPaths
13+
from tasks.libs.common.utils import get_repo_root
14+
from tasks.libs.types.arch import Arch
15+
16+
if TYPE_CHECKING:
17+
from tasks.kernel_matrix_testing.types import (
18+
Component, # noqa: F401
19+
KMTArchNameOrLocal,
20+
)
21+
22+
23+
class GDBPaths:
24+
def __init__(self, vm_tag: str, image_version: str, stack: str, arch: KMTArchNameOrLocal):
25+
self.tag = vm_tag
26+
self.image_version = image_version
27+
self.kmt_paths = KMTPaths(stack, Arch.from_str(arch))
28+
29+
@property
30+
def vmlinux(self):
31+
return self.kmt_paths.gdb / self.tag / self.image_version / "vmlinux.dbg"
32+
33+
@property
34+
def kernel_source(self):
35+
return self.kmt_paths.gdb / self.tag / self.image_version / "kernel-source"
36+
37+
38+
class UbuntuGDBProvision:
39+
def __init__(self, vm: LibvirtDomain, image_version: str, kernel: str):
40+
self.target = vm
41+
self.image_version = image_version
42+
self.kernel = semver.VersionInfo.parse(kernel)
43+
44+
def run(self, ctx: Context, stack: str):
45+
self.target.copy(
46+
ctx, get_repo_root() / "tasks/kernel_matrix_testing/provision/ubuntu-dbg.sh", "/tmp/provision.sh"
47+
)
48+
self.target.run_cmd(ctx, "chmod +x /tmp/provision.sh && /tmp/provision.sh")
49+
50+
gdb_paths = GDBPaths(self.target.tag, self.image_version, stack, self.target.arch)
51+
gdb_paths.vmlinux.parent.mkdir(exist_ok=True, parents=True)
52+
self.target.download(ctx, "/usr/lib/debug/boot/vmlinux.dbg", f"{gdb_paths.vmlinux}")
53+
54+
# make sure we are deleteing from a sane path
55+
if gdb_paths.kernel_source.name == "kernel-source" and f"{gdb_paths.kernel_source.absolute()}".startswith(
56+
f"{get_repo_root().absolute()}"
57+
):
58+
ctx.run(f"rm -rf {gdb_paths.kernel_source}")
59+
60+
gdb_paths.kernel_source.mkdir(parents=True)
61+
self.target.download(
62+
ctx,
63+
f"/usr/src/linux-source-{self.kernel.finalize_version()}/linux-source-{self.kernel.finalize_version()}.tar.bz2",
64+
f"{gdb_paths.kernel_source.parent}",
65+
)
66+
ctx.run(
67+
f"cd {gdb_paths.kernel_source.parent} && tar xvf linux-source-{self.kernel.finalize_version()}.tar.bz2 -C {gdb_paths.kernel_source} --strip-components=1",
68+
hide="out",
69+
echo=True,
70+
)
71+
72+
if self.kernel > semver.VersionInfo.parse("4.4.0"):
73+
ctx.run(f"cd {gdb_paths.kernel_source} && make defconfig && make scripts_gdb")
74+
75+
self.target.run_cmd(ctx, "shutdown -h now", verbose=True, allow_fail=True)
76+
77+
78+
gdb_provision = {
79+
"ubuntu": {
80+
"22.04": UbuntuGDBProvision,
81+
"23.10": UbuntuGDBProvision,
82+
"24.04": UbuntuGDBProvision,
83+
"24.10": UbuntuGDBProvision,
84+
"20.04": UbuntuGDBProvision,
85+
# TODO: Add support for bionic/ubuntu_18.04. Currently failing to find debug kernel build.
86+
"16.04": UbuntuGDBProvision,
87+
}
88+
}
89+
90+
91+
def setup_gdb_debugging(ctx: Context, stack: str) -> None:
92+
infra = build_infrastructure(stack)
93+
platforms = get_platforms()
94+
95+
arch = Arch.local().kmt_arch
96+
for kmt_arch, instance in infra.items():
97+
if kmt_arch != "local":
98+
# TODO: add support to attach gdb to remote VMs
99+
raise Exit("stacks with remote VMs cannot be launched with GDB")
100+
101+
for vm in instance.microvms:
102+
platinfo = platforms[arch][vm.tag]
103+
os_id = platinfo['os_id']
104+
os_version = platinfo['os_version']
105+
image_version = platinfo['image_version']
106+
kernel = platinfo['kernel']
107+
108+
if os_id not in gdb_provision:
109+
raise Exit(f"{os_id} is currently not supported for kernel debugging")
110+
111+
if os_version not in gdb_provision[os_id]:
112+
raise Exit(f"{os_id}_{os_version} is currently not supported for kernel debugging")
113+
114+
provisioner = gdb_provision[os_id][os_version](vm, image_version, kernel)
115+
info(f"[+] Provisioning {vm.tag} for debugging.")
116+
provisioner.run(ctx, stack)
117+
118+
stacks.pause_stack(stack)
119+
stacks.resume_stack(stack)

tasks/kernel_matrix_testing/infra.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ def __init__(
103103
arch: KMTArchNameOrLocal | None,
104104
instance: HostInstance,
105105
user: str = "root",
106+
gdb_port: int = 0,
106107
):
107108
self.ip = ip
108109
self.name = domain_id
@@ -112,6 +113,7 @@ def __init__(
112113
self.instance = instance
113114
self.arch = arch
114115
self.user = user
116+
self.gdb_port = gdb_port
115117

116118
def run_cmd(self, ctx: Context, cmd: str, allow_fail=False, verbose=False, timeout_sec=None):
117119
if timeout_sec is not None:
@@ -214,7 +216,14 @@ def build_infrastructure(stack: str, ssh_key_obj: SSHKey | None = None):
214216
# location in the local machine.
215217
instance.add_microvm(
216218
LibvirtDomain(
217-
vm["ip"], vm["id"], vm["tag"], vm["vmset-tags"], os.fspath(get_kmt_os().ddvm_rsa), arch, instance
219+
vm["ip"],
220+
vm["id"],
221+
vm["tag"],
222+
vm["vmset-tags"],
223+
os.fspath(get_kmt_os().ddvm_rsa),
224+
arch,
225+
instance,
226+
gdb_port=vm["gdb-port"],
218227
)
219228
)
220229

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
if [[ $UID == 0 ]] ; then
6+
echo "Please dont run this script as root, since the gef scripts will get setup for the root user"
7+
exit 1
8+
fi
9+
10+
echo "[+] apt"
11+
sudo apt-get update
12+
sudo apt-get install -y gdb-multiarch binutils gcc file python3-pip ruby-dev git
13+
14+
echo "[+] pip3"
15+
pip3 install crccheck unicorn capstone ropper keystone-engine tqdm
16+
17+
echo "[+] install seccomp-tools, one_gadget"
18+
if [[ -z "$(which seccomp-tools)" ]]; then
19+
sudo gem install seccomp-tools
20+
fi
21+
22+
if [[ -z "$(which one_gadget)" ]]; then
23+
sudo gem install one_gadget
24+
fi
25+
26+
echo "[+] install rp++"
27+
if [[ "$(uname -m)" == "x86_64" ]]; then
28+
if [[ -z "$(which rp-lin)" ]] && [[ ! -e /usr/local/bin/rp-lin ]]; then
29+
wget -q https://github.com/0vercl0k/rp/releases/download/v2.1.3/rp-lin-clang.zip -P /tmp
30+
sudo unzip /tmp/rp-lin-clang.zip -d /usr/local/bin/
31+
sudo chmod +x /usr/local/bin/rp-lin
32+
rm /tmp/rp-lin-clang.zip
33+
fi
34+
fi
35+
36+
echo "[+] install vmlinux-to-elf"
37+
if [[ -z "$(which vmlinux-to-elf)" ]] && [[ ! -e /usr/local/bin/vmlinux-to-elf ]]; then
38+
pip3 install --upgrade lz4 zstandard git+https://github.com/clubby789/python-lzo@b4e39df
39+
pip3 install --upgrade git+https://github.com/marin-m/vmlinux-to-elf
40+
fi
41+
42+
echo "[+] download gef"
43+
if [[ -e ~/.gdbinit-gef.py ]]; then
44+
echo "[-] ~/.gdbinit-gef.py already exists. Please delete or rename."
45+
echo "[-] INSTALLATION FAILED"
46+
exit 1
47+
else
48+
wget -q https://raw.githubusercontent.com/bata24/gef/dev/gef.py -O ~/.gdbinit-gef.py
49+
fi
50+
51+
echo "[+] setup gef"
52+
STARTUP_COMMAND="source ~/.gdbinit-gef.py"
53+
if [[ ! -e ~/.gdbinit ]] || [[ -z "$(grep "$STARTUP_COMMAND" ~/.gdbinit)" ]]; then
54+
echo "$STARTUP_COMMAND" >> ~/.gdbinit
55+
fi
56+
57+
echo "[+] INSTALLATION SUCCESSFUL"
58+
exit 0
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
set -o errexit
3+
set -o pipefail
4+
set -o nounset
5+
6+
# 1. disable kaslr
7+
[[ ! $(grep -q "GRUB_CMDLINE_LINUX=\".*nokaslr" /etc/default/grub) ]] && sed -i 's/^GRUB_CMDLINE_LINUX="/&nokaslr /' /etc/default/grub
8+
update-grub
9+
10+
# 2. download kernel debug build and kernel sources
11+
codename=$(lsb_release -c | awk '{print $2}')
12+
if [[ "${codename}" == "xenial" ]]; then
13+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C8CAB6595FDFF622
14+
tee /etc/apt/sources.list.d/ddebs.list << EOF
15+
deb http://ddebs.ubuntu.com/ ${codename} main restricted universe multiverse
16+
deb http://ddebs.ubuntu.com/ ${codename}-updates main restricted universe multiverse
17+
deb http://ddebs.ubuntu.com/ ${codename}-proposed main restricted universe multiverse
18+
EOF
19+
else
20+
apt install -y ubuntu-dbgsym-keyring
21+
echo "\
22+
Types: deb
23+
URIs: http://ddebs.ubuntu.com/
24+
Suites: $(lsb_release -cs) $(lsb_release -cs)-updates $(lsb_release -cs)-proposed
25+
Components: main restricted universe multiverse
26+
Signed-by: /usr/share/keyrings/ubuntu-dbgsym-keyring.gpg" | tee -a /etc/apt/sources.list.d/ddebs.sources
27+
fi
28+
29+
apt update
30+
apt install -y linux-image-`uname -r`-dbgsym linux-source
31+
32+
cp /usr/lib/debug/boot/vmlinux-`uname -r` /usr/lib/debug/boot/vmlinux.dbg

tasks/kernel_matrix_testing/stacks.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,13 @@ def check_env(ctx: Context):
194194

195195

196196
def launch_stack(
197-
ctx: Context, stack: str | None, ssh_key: str | None, x86_ami: str, arm_ami: str, provision_microvms: bool
197+
ctx: Context,
198+
stack: str | None,
199+
ssh_key: str | None,
200+
x86_ami: str,
201+
arm_ami: str,
202+
provision_microvms: bool,
203+
with_gdb: bool,
198204
):
199205
stack = check_and_get_stack_or_exit(stack)
200206

@@ -239,6 +245,7 @@ def launch_stack(
239245
vmconfig=vm_config,
240246
stack_name=stack,
241247
local=local,
248+
with_gdb=with_gdb,
242249
)
243250

244251
prefix = ""
@@ -294,6 +301,7 @@ def start_microvms_cmd(
294301
provision_microvms=False,
295302
run_agent=False,
296303
agent_version=None,
304+
with_gdb=False,
297305
):
298306
args = [
299307
f"--instance-type-x86 {instance_type_x86}" if instance_type_x86 else "",
@@ -313,6 +321,7 @@ def start_microvms_cmd(
313321
f"--agent-version {agent_version}" if agent_version else "",
314322
"--provision-instance" if provision_instance else "",
315323
"--provision-microvms" if provision_microvms else "",
324+
"--setup-gdb" if with_gdb else "",
316325
]
317326
go_args = ' '.join(filter(lambda x: x != "", args))
318327
return f"./test/new-e2e/start-microvms {go_args}"

tasks/kernel_matrix_testing/vars.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,7 @@ def test_results(self):
5555

5656
def vm_test_results(self, vm_name: str):
5757
return self.test_results / vm_name
58+
59+
@property
60+
def gdb(self):
61+
return self.root / self.arch.kmt_arch / "gdb"

tasks/kmt.py

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from tasks.kernel_matrix_testing.compiler import CONTAINER_AGENT_PATH, get_compiler
3131
from tasks.kernel_matrix_testing.config import ConfigManager
3232
from tasks.kernel_matrix_testing.download import update_rootfs
33+
from tasks.kernel_matrix_testing.gdb import GDBPaths, setup_gdb_debugging
3334
from tasks.kernel_matrix_testing.infra import (
3435
SSH_OPTIONS,
3536
HostInstance,
@@ -298,6 +299,49 @@ def gen_config_from_ci_pipeline(
298299
print(f"dda inv -- kmt.test --packages=\"{','.join(failed_packages)}\" --run='^{'|'.join(failed_tests)}$'")
299300

300301

302+
@task
303+
def attach_gdb(ctx: Context, vm: str, stack: str | None = None, dry=True):
304+
stack = check_and_get_stack(stack)
305+
if not stacks.stack_exists(stack):
306+
raise Exit(f"Stack {stack} does not exist. Please create with 'dda inv kmt.create-stack --stack=<name>'")
307+
308+
if not os.path.exists(f"{Path.home()}/.gdbinit-gef.py"):
309+
resp = ask(
310+
"It is recommended to use gdb with the bata24 extension (https://github.com/bata24/gef) which greatly enhances the kernel debugging experience. You can install the extension with `inv -e kmt.install_bata24_gef`. Continue without installing? (y/N)"
311+
)
312+
if resp.lower().strip() != "y":
313+
raise Exit("Aborted by user")
314+
315+
domains = get_target_domains(ctx, stack, None, None, vm, None)
316+
assert len(domains) > 0, f"no running VM discovered for the provided vm {vm}"
317+
assert len(domains) == 1, "GDB can only be attached to one VM at a time"
318+
319+
domain = domains[0]
320+
if domain.gdb_port == 0:
321+
raise Exit(
322+
"VM was not launched with GDB debugging support. To use this feature specify the `--gdb` flag when invoke the `kmt.launch-stack` task"
323+
)
324+
325+
platforms = get_platforms()
326+
kmt_arch = Arch.from_str(domain.arch).kmt_arch
327+
platinfo = platforms[kmt_arch][domain.tag]
328+
gdb_paths = GDBPaths(domain.tag, platinfo['image_version'], stack, domain.arch)
329+
330+
cmd = f"gdb \
331+
-ex \"add-auto-load-safe-path {gdb_paths.kernel_source}\" \
332+
-ex \"file {gdb_paths.vmlinux}\" \
333+
-ex \"set arch i386:x86-64:intel\" -ex \"target remote localhost:{domain.gdb_port}\" \
334+
-ex \"source {gdb_paths.kernel_source}/vmlinux-gdb.py\" \
335+
-ex \"set disassembly-flavor intel\" \
336+
-ex \"set pagination off\" \
337+
"
338+
339+
if dry:
340+
info(f"[+] Run the following command: {cmd}")
341+
else:
342+
ctx.run(cmd)
343+
344+
301345
@task
302346
def launch_stack(
303347
ctx: Context,
@@ -307,13 +351,23 @@ def launch_stack(
307351
arm_ami: str = ARM_AMI_ID_SANDBOX,
308352
provision_microvms: bool = True,
309353
provision_script: str | None = None,
354+
gdb: bool = False,
310355
):
311356
stack = check_and_get_stack_or_exit(stack)
312357

313-
stacks.launch_stack(ctx, stack, ssh_key, x86_ami, arm_ami, provision_microvms)
358+
if gdb and get_kmt_os().name != "linux":
359+
# TODO: add kernel debugging support for MacOS
360+
raise Exit("GDB attached to guest VM is only supported for linux systems")
361+
362+
stacks.launch_stack(ctx, stack, ssh_key, x86_ami, arm_ami, provision_microvms, gdb)
363+
314364
if provision_script is not None:
315365
provision_stack(ctx, provision_script, stack, ssh_key)
316366

367+
if gdb:
368+
setup_gdb_debugging(ctx, stack)
369+
info("[+] GDB setup complete")
370+
317371

318372
@task
319373
def provision_stack(
@@ -2534,3 +2588,8 @@ def start_microvms(
25342588
agent_version=agent_version,
25352589
)
25362590
)
2591+
2592+
2593+
@task
2594+
def install_bata24_gef(ctx):
2595+
ctx.run("tasks/kernel_matrix_testing/provision/bata24.sh")

test/new-e2e/scenarios/system-probe/main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ func main() {
5454
agentVersionPtr := flag.String("agent-version", "", "Version of datadog-agent")
5555
provisionInstancePtr := flag.Bool("provision-instance", false, "run provision step for metal instance")
5656
provisionMicrovmsPtr := flag.Bool("provision-microvms", false, "run provision step for microvms")
57+
setupGdbPtr := flag.Bool("setup-gdb", false, "setup gdb server in QEMU attached to guest kernel")
5758

5859
flag.Parse()
5960

@@ -77,6 +78,7 @@ func main() {
7778
Local: *local,
7879
RunAgent: *runAgentPtr,
7980
AgentVersion: *agentVersionPtr,
81+
SetupGDB: *setupGdbPtr,
8082
}
8183

8284
err := run(*envNamePtr, *x86InstanceTypePtr, *armInstanceTypePtr, *destroyPtr, &opts)

0 commit comments

Comments
 (0)