fixes for glibc image with local KMT

Author: brycekahle

tasks/kernel_matrix_testing/compiler.py

@@ -23,10 +23,9 @@
 AMD64_DEBIAN_KERNEL_HEADERS_URL = "http://deb.debian.org/debian-security/pool/updates/main/l/linux-5.10/linux-headers-5.10.0-0.deb10.28-amd64_5.10.209-2~deb10u1_amd64.deb"
 ARM64_DEBIAN_KERNEL_HEADERS_URL = "http://deb.debian.org/debian-security/pool/updates/main/l/linux-5.10/linux-headers-5.10.0-0.deb10.28-arm64_5.10.209-2~deb10u1_arm64.deb"
 
-DOCKER_REGISTRY = "486234852809.dkr.ecr.us-east-1.amazonaws.com"
 DOCKER_BASE_IMAGES = {
-    "x64": f"{DOCKER_REGISTRY}/ci/datadog-agent-buildimages/linux-glibc-2.17-x64",
-    "arm64": f"{DOCKER_REGISTRY}/ci/datadog-agent-buildimages/linux-glibc-2.23-arm64",
+    "x64": f"registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64",
+    "arm64": f"registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64",
 }
 
 
@@ -49,21 +48,6 @@ def get_docker_image_name(ctx: Context, container: str) -> str:
     return data[0]["Config"]["Image"]
 
 
-def has_docker_auth_helpers() -> bool:
-    docker_config = Path("~/.docker/config.json").expanduser()
-    if not docker_config.exists():
-        return False
-
-    try:
-        with open(docker_config) as f:
-            config = json.load(f)
-    except json.JSONDecodeError:
-        # Invalid JSON (or empty file), we don't have the helper
-        return False
-
-    return DOCKER_REGISTRY in config.get("credHelpers", {})
-
-
 class CompilerImage:
     def __init__(self, ctx: Context, arch: Arch):
         self.ctx = ctx
@@ -115,15 +99,18 @@ def ensure_version(self):
             warn(f"[!] Running compiler image {image_used} is different from the expected {self.image}, will restart")
             self.start()
 
-    def exec(self, cmd: str, user="compiler", verbose=True, run_dir: PathOrStr | None = None, allow_fail=False):
+    def exec(self, cmd: str, user="compiler", verbose=True, run_dir: PathOrStr | None = None, allow_fail=False, force_color=True):
         if run_dir:
             cmd = f"cd {run_dir} && {cmd}"
 
         self.ensure_running()
+        color_env = "-e FORCE_COLOR=1"
+        if not force_color:
+            color_env = ""
 
         # Set FORCE_COLOR=1 so that termcolor works in the container
         return self.ctx.run(
-            f"docker exec -u {user} -i -e FORCE_COLOR=1 {self.name} bash -c \"{cmd}\"",
+            f"docker exec -u {user} -i {color_env} {self.name} bash -l -c \"{cmd}\"",
             hide=(not verbose),
             warn=allow_fail,
         )
@@ -139,23 +126,14 @@ def start(self) -> None:
         # Check if the image exists
         res = self.ctx.run(f"docker image inspect {self.image}", hide=True, warn=True)
         if res is None or not res.ok:
-            info(f"[!] Image {self.image} not found, logging in and pulling...")
-
-            if has_docker_auth_helpers():
-                # With ddtool helpers (installed with ddtool auth helpers install), docker automatically
-                # pulls credentials from ddtool, and we require the aws-vault context to pull
-                docker_pull_auth = "aws-vault exec sso-build-stable-developer -- "
-            else:
-                # Without the helpers, we need to get the password and login manually to docker
-                self.ctx.run(
-                    "aws-vault exec sso-build-stable-developer -- aws ecr --region us-east-1 get-login-password | docker login --username AWS --password-stdin 486234852809.dkr.ecr.us-east-1.amazonaws.com"
-                )
-                docker_pull_auth = ""
-
-            self.ctx.run(f"{docker_pull_auth}docker pull {self.image}")
+            info(f"[!] Image {self.image} not found, pulling...")
+            self.ctx.run(f"docker pull {self.image}")
 
+        platform = ""
+        if self.arch != Arch.local():
+            platform = f"--platform linux/{self.arch.go_arch}"
         res = self.ctx.run(
-            f"docker run -d --restart always --name {self.name} "
+            f"docker run {platform} -d --restart always --name {self.name} "
             f"--mount type=bind,source={os.getcwd()},target={CONTAINER_AGENT_PATH} "
             f"{self.image} sleep \"infinity\"",
             warn=True,
@@ -183,51 +161,18 @@ def start(self) -> None:
             )
 
         self.exec("chmod a+rx /root", user="root")  # Some binaries will be in /root and need to be readable
-        self.exec("apt install sudo", user="root")
+        self.exec("apt-get install -y --no-install-recommends sudo", user="root")
         self.exec("usermod -aG sudo compiler && echo 'compiler ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers", user="root")
-        self.exec("echo conda activate ddpy3 >> /home/compiler/.bashrc", user="compiler")
+        self.exec(f"cp /root/.bashrc /home/compiler/.bashrc && chown {uid}:{gid} /home/compiler/.bashrc", user="root")
+        self.exec("mkdir ~/.cargo && touch ~/.cargo/env", user="compiler")
+        self.exec("dda self telemetry disable", user="compiler", force_color=False)
         self.exec(f"install -d -m 0777 -o {uid} -g {uid} /go", user="root")
+        self.exec(f"echo export DD_CC={self.arch.gcc_arch}-unknown-linux-gnu-gcc >> /home/compiler/.bashrc", user="compiler")
+        self.exec(f"echo export DD_CXX={self.arch.gcc_arch}-unknown-linux-gnu-g++ >> /home/compiler/.bashrc", user="compiler")
 
-        self.prepare_for_cross_compile()
 
-    def ensure_ready_for_cross_compile(self):
-        res = self.exec("test -f /tmp/cross-compile-ready", user="root", allow_fail=True)
-        if res is None or not res.ok:
-            info("[*] Compiler image not ready for cross-compilation, preparing...")
-            self.prepare_for_cross_compile()
-
-    def prepare_for_cross_compile(self):
-        target = ARCH_AMD64 if self.arch == ARCH_ARM64 else ARCH_ARM64
-
-        # Hardcoded links to the header packages for each architecture. Why do this and not have something more automated?
-        # 1. While right now the URLs are similar and we'd only need a single link with variable replacement, this might
-        #    change as the repository layout is not under our control.
-        # 2. Automatic detection of these URLs is not direct (querying the package repo APIs is not trivial) and we'd need some
-        #    level of hard-coding some URLs or assumptions anyways.
-        # 3. Even if someone forgets to update these URLs, it's not a big deal, as we're building inside of a Docker image which will
-        #    likely have a different kernel than the target system where the built eBPF files are going to run anyways.
-        header_package_urls: dict[Arch, str] = {
-            ARCH_AMD64: AMD64_DEBIAN_KERNEL_HEADERS_URL,
-            ARCH_ARM64: ARM64_DEBIAN_KERNEL_HEADERS_URL,
-        }
-
-        header_package_path = "/tmp/headers.deb"
-        self.exec(f"wget -O {header_package_path} {header_package_urls[target]}")
-
-        # Uncompress the package in the root directory, so that we have access to the headers
-        # We cannot install because the architecture will not match
-        # Extract into a .tar file and then use tar to extract the contents to avoid issues
-        # with dpkg-deb not respecting symlinks.
-        self.exec(f"dpkg-deb --fsys-tarfile {header_package_path} > {header_package_path}.tar", user="root")
-        self.exec(f"tar -h -xf {header_package_path}.tar -C /", user="root")
-
-        # Install the corresponding arch compilers
-        self.exec(f"apt update && apt install -y gcc-{target.gcc_arch.replace('_', '-')}-linux-gnu", user="root")
-        self.exec("touch /tmp/cross-compile-ready")  # Signal that we're ready for cross-compilation
-
-
-def get_compiler(ctx: Context):
-    cc = CompilerImage(ctx, Arch.local())
+def get_compiler(ctx: Context, arch_obj: Arch):
+    cc = CompilerImage(ctx, arch_obj)
     cc.ensure_version()
     cc.ensure_running()
 

tasks/kmt.py

@@ -478,8 +478,8 @@ def update_resources(
 
 
 @task
-def start_compiler(ctx: Context):
-    cc = get_compiler(ctx)
+def start_compiler(ctx: Context, arch: Arch | str = "local",):
+    cc = get_compiler(ctx, Arch.from_str(arch))
     info(f"[+] Starting compiler {cc.name}")
     try:
         cc.start()
@@ -522,10 +522,12 @@ def download_gotestsum(ctx: Context, arch: Arch, fgotestsum: PathOrStr):
     paths = KMTPaths(None, arch)
     paths.tools.mkdir(parents=True, exist_ok=True)
 
-    cc = get_compiler(ctx)
+    cc = get_compiler(ctx, arch)
     target_path = CONTAINER_AGENT_PATH / paths.tools.relative_to(paths.repo_root) / "gotestsum"
+    env = {"GOARCH": arch.go_arch, "CC": "\\$DD_CC", "CXX": "\\$DD_CXX"}
+    env_str = " ".join(f"{key}={value}" for key, value in env.items())
     cc.exec(
-        f"cd {TOOLS_PATH} && GOARCH={arch.go_arch} go build -o {target_path} {GOTESTSUM}",
+        f"cd {TOOLS_PATH} && {env_str} go build -o {target_path} {GOTESTSUM}",
     )
 
     ctx.run(f"cp {paths.tools}/gotestsum {fgotestsum}")
@@ -766,10 +768,7 @@ def _prepare(
     domains: list[LibvirtDomain] | None = None,
 ):
     if not ci:
-        cc = get_compiler(ctx)
-
-        if arch_obj.is_cross_compiling():
-            cc.ensure_ready_for_cross_compile()
+        cc = get_compiler(ctx, arch_obj)
 
     pkgs = ""
     if packages:
@@ -896,7 +895,8 @@ def build_target_packages(filter_packages: list[str], build_tags: list[str]):
 
 
 def build_object_files(ctx, fp, arch: Arch):
-    info("[+] Generating eBPF object files...")
+    setup_runtime_clang(ctx)
+    info(f"[+] Generating eBPF object files... {fp}")
     ninja_generate(ctx, fp, arch=arch)
     ctx.run(f"ninja -d explain -f {fp}")
 
@@ -1312,14 +1312,11 @@ def build(
     paths = KMTPaths(stack, arch_obj)
     paths.arch_dir.mkdir(parents=True, exist_ok=True)
 
-    cc = get_compiler(ctx)
+    cc = get_compiler(ctx, arch_obj)
 
     inv_echo = "-e" if ctx.config.run["echo"] else ""
-    cc.exec(f"cd {CONTAINER_AGENT_PATH} && dda inv {inv_echo} system-probe.object-files")
-
-    build_task = "build-sysprobe-binary" if component == "system-probe" else "build"
     cc.exec(
-        f"cd {CONTAINER_AGENT_PATH} && git config --global --add safe.directory {CONTAINER_AGENT_PATH} && dda inv {inv_echo} {component}.{build_task} --arch={arch_obj.name}",
+        f"cd {CONTAINER_AGENT_PATH} && git config --global --add safe.directory {CONTAINER_AGENT_PATH} && dda inv {inv_echo} {component}.build --arch={arch_obj.name}",
     )
 
     cc.exec(f"tar cf {CONTAINER_AGENT_PATH}/kmt-deps/{stack}/build-embedded-dir.tar {EMBEDDED_SHARE_DIR}")
@@ -1331,7 +1328,7 @@ def build(
 
     domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms)
     if alien_vms is not None:
-        err_msg = f"no alient VMs discovered from provided profile {alien_vms}."
+        err_msg = f"no alien VMs discovered from provided profile {alien_vms}."
     else:
         err_msg = f"no vms found from list {vms}. Run `dda inv -e kmt.status` to see all VMs in current stack"
 

tasks/system_probe.py

@@ -656,6 +656,7 @@ def build_libpcap(ctx):
     lib_dir = os.path.join(dist_dir, f"libpcap-{LIBPCAP_VERSION}")
     ctx.run(f"rm -rf {lib_dir}")
     with ctx.cd(dist_dir):
+        # TODO check the checksum of the download before using
         ctx.run(f"curl -L https://www.tcpdump.org/release/libpcap-{LIBPCAP_VERSION}.tar.xz | tar xJ")
     with ctx.cd(lib_dir):
         env = {}