build(profiling): add native heap-gotter cdylib build (PROF-15750)

.gitignore

@@ -234,6 +234,10 @@ vgcore.*
 
 # Rust build artifacts
 src/native/target*
+src/native_heap_gotter/target*
+
+# Profiling collector local CMake build dir
+ddtrace/profiling/collector/build-test/
 
 # Fuzzing corpus, output and artifacts
 .fuzz/

.gitlab-ci.yml

@@ -678,6 +678,10 @@ profiling_native:
         - src/native/**/*.toml
         - src/native/**/*.txt
         - src/native/**/Cargo.lock
+        # Standalone heap-gotter cdylib crate (opt-in build via setup.py)
+        - src/native_heap_gotter/**/*.rs
+        - src/native_heap_gotter/**/*.toml
+        - src/native_heap_gotter/**/Cargo.lock
         # Top-level build config
         - setup.py
         - pyproject.toml

.gitlab/native.yml

@@ -21,6 +21,21 @@ include:
       echo -e "\e[0Ksection_start:`date +%s`:cargo_test[collapsed=true]\r\e[0Kcargo test"
       cargo test --no-fail-fast --locked
       echo -e "\e[0Ksection_end:`date +%s`:cargo_test\r\e[0K"
+    # The standalone heap-gotter cdylib crate lives outside src/native, so run
+    # the same fmt/clippy/test gate against it here to keep it under native CI.
+    - |
+      cd ../native_heap_gotter
+      echo -e "\e[0Ksection_start:`date +%s`:gotter_cargo_fmt[collapsed=true]\r\e[0Kheap-gotter cargo fmt"
+      cargo fmt --all -- --check
+      echo -e "\e[0Ksection_end:`date +%s`:gotter_cargo_fmt\r\e[0K"
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:gotter_cargo_clippy[collapsed=true]\r\e[0Kheap-gotter cargo clippy"
+      cargo clippy --locked --all-features -- -D warnings
+      echo -e "\e[0Ksection_end:`date +%s`:gotter_cargo_clippy\r\e[0K"
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:gotter_cargo_test[collapsed=true]\r\e[0Kheap-gotter cargo test"
+      cargo test --no-fail-fast --locked
+      echo -e "\e[0Ksection_end:`date +%s`:gotter_cargo_test\r\e[0K"
 
 "clang-tidy profiling":
   stage: tests

.gitlab/scripts/build-wheel-helpers.sh

@@ -99,9 +99,45 @@ build_wheel() {
 repair_wheel() {
   # Extract debug symbols
   section_start "extract_debug_symbols" "Extracting debug symbols"
-  uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}"
+  uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" \
+    --output-dir "${DEBUG_WHEEL_DIR}" \
+    --ignore-patterns "libddwaf*,libdd_heap_gotter*"
   section_end "extract_debug_symbols"
 
+  # Heap-gotter cdylib debug symbols are extracted in setup.py (build_heap_gotter);
+  # merge any staged .debug sidecars into the debug-symbols package.
+  section_start "merge_heap_gotter_debug_symbols" "Merging heap-gotter debug symbols"
+  uv run --no-project python - <<'PY'
+import glob
+import os
+import zipfile
+from pathlib import Path
+
+project_dir = os.environ["PROJECT_DIR"]
+debug_dir = os.environ["DEBUG_WHEEL_DIR"]
+sidecars = sorted(Path(project_dir, "build").rglob("libdd_heap_gotter*.debug"))
+if not sidecars:
+    print("No heap-gotter debug sidecars found")
+    raise SystemExit(0)
+packages = glob.glob(os.path.join(debug_dir, "*-debug-symbols.zip"))
+if not packages:
+    print("WARNING: no debug-symbols package to merge heap-gotter sidecars into")
+    raise SystemExit(0)
+pkg = packages[0]
+with zipfile.ZipFile(pkg, "a", zipfile.ZIP_DEFLATED) as zf:
+    existing = set(zf.namelist())
+    for sidecar in sidecars:
+        parts = sidecar.parts
+        try:
+            arc = str(Path(*parts[parts.index("ddtrace") :]))
+        except ValueError:
+            arc = sidecar.name
+        if arc not in existing:
+            zf.write(sidecar, arc)
+            print(f"Added heap-gotter debug symbols: {arc}")
+PY
+  section_end "merge_heap_gotter_debug_symbols"
+
   # Strip wheel
   section_start "strip_wheel" "Stripping unneeded files"
   uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md
@@ -115,7 +151,69 @@ repair_wheel() {
   # Repair wheel (ONLY PLATFORM-SPECIFIC CODE)
   section_start "repair_wheel" "Repairing wheel"
   if [[ "$(uname -s)" == "Linux" ]]; then
+    # The opt-in heap-gotter cdylib (DD_PROFILING_NATIVE_HEAP_BUILD=1) has
+    # non-standard ELF versioning sections that trip auditwheel's iter_versions
+    # parser. --exclude does not help: it only drops a SONAME from dependency
+    # grafting, while repair still parses every ELF listed in the wheel's RECORD.
+    # So the cdylib has to leave the wheel entirely and be reinserted after.
+    GOTTER_STASH_DIR="${WORK_DIR}/heap_gotter_stash"
+    GOTTER_PATTERN='*libdd_heap_gotter*.so'
+    if unzip -l "${BUILT_WHEEL_FILE}" | grep -q 'libdd_heap_gotter.*\.so$'; then
+      mkdir -p "${GOTTER_STASH_DIR}"
+      unzip -q "${BUILT_WHEEL_FILE}" "${GOTTER_PATTERN}" -d "${GOTTER_STASH_DIR}"
+      uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" "${GOTTER_PATTERN}"
+    fi
+
     auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}"
+
+    if [[ -d "${GOTTER_STASH_DIR}" ]]; then
+      REPAIRED_WHEEL_FILE=$(ls "${TMP_WHEEL_DIR}"/*.whl | head -n 1)
+      GOTTER_STASH_DIR="${GOTTER_STASH_DIR}" REPAIRED_WHEEL_FILE="${REPAIRED_WHEEL_FILE}" \
+        uv run --no-project python - <<'PY'
+import base64
+import csv
+import hashlib
+import io
+import os
+import zipfile
+from pathlib import Path
+
+wheel = Path(os.environ["REPAIRED_WHEEL_FILE"])
+stash = Path(os.environ["GOTTER_STASH_DIR"])
+
+additions = {str(p.relative_to(stash)): p for p in sorted(stash.rglob("*")) if p.is_file()}
+if not additions:
+    print("No stashed heap-gotter cdylib to reinsert")
+    raise SystemExit(0)
+
+tmp_wheel = Path(f"{wheel}.tmp")
+with (
+    zipfile.ZipFile(wheel, "r") as source_zip,
+    zipfile.ZipFile(tmp_wheel, "w", zipfile.ZIP_DEFLATED) as temp_zip,
+):
+    record = next((f for f in source_zip.infolist() if f.filename.endswith(".dist-info/RECORD")), None)
+    if record is None:
+        raise SystemExit(f"no RECORD found in {wheel}")
+    # DEV: Use ZipInfo objects to ensure original file attributes are preserved
+    for file in source_zip.infolist():
+        if file.filename == record.filename or file.filename in additions:
+            continue
+        temp_zip.writestr(file, source_zip.read(file.filename))
+    rows = [r for r in csv.reader(io.StringIO(source_zip.read(record.filename).decode("utf-8"))) if r]
+    rows = [r for r in rows if r[0] != record.filename and r[0] not in additions]
+    for arcname, path in additions.items():
+        data = path.read_bytes()
+        temp_zip.writestr(arcname, data)
+        digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode("ascii")
+        rows.append([arcname, f"sha256={digest}", str(len(data))])
+        print(f"Reinserted heap-gotter cdylib: {arcname}")
+    rows.append([record.filename, "", ""])
+    output = io.StringIO()
+    csv.writer(output, lineterminator="\n").writerows(rows)
+    temp_zip.writestr(record, output.getvalue())
+os.replace(tmp_wheel, wheel)
+PY
+    fi
   else
     # macOS
     MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel \

MANIFEST.in

@@ -7,3 +7,4 @@ prune .riot/
 prune benchmarks/
 prune releasenotes/
 prune src/native/target*
+prune src/native_heap_gotter/target*

scripts/check_profiling_native_coverage.py

@@ -27,8 +27,9 @@
 
 from pathlib import Path
 import re
-import subprocess
+import subprocess  # nosec B404
 import sys
+from typing import Any
 
 from ruamel.yaml import YAML
 
@@ -39,6 +40,7 @@
     "ddtrace/internal/datadog/profiling",
     "ddtrace/profiling",
     "src/native",
+    "src/native_heap_gotter",
 ]
 
 # File extensions that belong to the native build graph.
@@ -96,9 +98,9 @@ def extract_profiling_native_patterns(ci_path: Path) -> list[str]:
     """Extract rules:changes patterns for the profiling_native job."""
     yaml: YAML = YAML()
     yaml.allow_duplicate_keys = True
-    data: dict = yaml.load(ci_path)
+    data: dict[Any, Any] = yaml.load(ci_path)
 
-    rules: list[dict] = data["profiling_native"]["rules"]
+    rules: list[dict[Any, Any]] = data["profiling_native"]["rules"]
     for rule in rules:
         if "changes" in rule:
             return list(rule["changes"])
@@ -111,7 +113,7 @@ def tracked_files(dirs: list[str] | None = None) -> list[str]:
     if dirs is None:
         dirs = ["."]
 
-    result: subprocess.CompletedProcess[str] = subprocess.run(
+    result: subprocess.CompletedProcess[str] = subprocess.run(  # nosec B603
         ["git", "ls-files", "--"] + dirs,
         capture_output=True,
         text=True,