Merge remote-tracking branch 'origin/main' into codex/otlp-trace-metrics-libdatadog

# Conflicts:
#	src/native/Cargo.toml
#	tests/tracer/test_otel_thread_context.py

Author: mabdinur

.claude/skills/llmobs-integrations/references/implementation-guide.md

@@ -135,6 +135,56 @@ Do not dump raw `kwargs` into LLMObs metadata. Prefer shared helpers such as `ge
 
 Normalize `INPUT_TOKENS_METRIC_KEY` to the total input tokens sent to the model, including cached and non-cached tokens. Providers report this differently: Anthropic reports non-cached `input_tokens` separately from cache read/write input tokens, so add `input_tokens + cache_creation_input_tokens + cache_read_input_tokens`; OpenAI reports prompt/input tokens as the combined total and exposes cached tokens separately in details.
 
+## Agent Integrations: Stamp Kind and Name at Span Start
+
+Agent integrations have a critical ordering constraint. `_resolve_parent_agent()` in `ddtrace/llmobs/_utils.py` resolves agent attribution when a **child** span activates, not when the parent finishes. Under LIFO nesting the parent span is still open when the child starts — so any field written at span *finish* time is invisible to children that have already been attributed.
+
+**Contract: any integration that produces `kind="agent"` LLMObs spans must stamp kind at span creation, not at finish.**
+
+`BaseLLMIntegration.trace()` and `LlmTracingSubscriber.on_started` both call `_stamp_llmobs_span_kind_at_start()` automatically. Integrations should not call it directly; instead override the two hooks below.
+
+### Hooks to implement for agent spans
+
+**`_llmobs_span_kind(self, operation_id, span, **kwargs) -> Optional[str]`**
+
+Return `"agent"` when the kwargs signal an agent span. The base-class default returns `"agent"` when `kwargs.get("kind") == "agent"`. Override only when the integration uses a different signal (e.g. `operation="agent"` for CrewAI, `interface_type="agent"` for Bedrock). Must be determinable at `trace()` call time — do NOT rely on data available only at finish.
+
+**`_llmobs_agent_name_at_start(self, span, **kwargs) -> Optional[str]`**
+
+Return the agent's LLMObs display name when it can be determined at `trace()` call time. Return `None` to fall back to the span resource name. Override when the integration can resolve the name from kwargs (e.g. Google ADK passes `_dd_agent=<agent_instance>` and the integration reads `agent.name`).
+
+If the integration's `trace()` captures the instance reference before `**kwargs` (e.g. LangGraph), `_llmobs_agent_name_at_start` cannot reach it through `super().trace()`. In that case, stamp the name directly in the integration's `trace()` override after calling `super()`:
+
+```python
+def trace(self, operation_id, submit_to_llmobs=False, **kwargs):
+    span = super().trace(operation_id, submit_to_llmobs=submit_to_llmobs, **kwargs)
+    # instance captured before **kwargs; stamp name after super() has written the kind
+    if submit_to_llmobs and self.llmobs_enabled and kwargs.get("kind") == "agent":
+        agent_name = getattr(instance, "name", None) if instance else None
+        if agent_name:
+            _annotate_llmobs_span_data(span, name=agent_name)
+    return span
+```
+
+### Example: Google ADK
+
+Patch layer passes the agent object via a private kwarg:
+```python
+span = integration.trace(
+    "%s.%s" % (instance.__class__.__name__, wrapped.__name__),
+    kind="agent",
+    submit_to_llmobs=True,
+    _dd_agent=agent,   # <-- agent instance available at trace() call time
+)
+```
+
+Integration overrides the hook:
+```python
+def _llmobs_agent_name_at_start(self, span: Span, **kwargs: Any) -> Optional[str]:
+    agent = kwargs.get("_dd_agent")
+    return getattr(agent, "name", None) if agent else None
+```
+
 ## LLM-Specific Registration Checklist
 
 In addition to the full checklist in the apm-integrations [Implementation Guide](../../apm-integrations/references/implementation-guide.md):

.github/CODEOWNERS

@@ -319,6 +319,7 @@ tests/test_sampling.py                             @DataDog/apm-sdk-capabilities
 tests/test_tracemethods.py                         @DataDog/apm-sdk-capabilities-python
 tests/opentelemetry/                               @DataDog/apm-sdk-capabilities-python
 tests/tracer/                                      @DataDog/apm-sdk-capabilities-python
+tests/tracer/test_otel_thread_context.py            @DataDog/apm-sdk-capabilities-python @DataDog/apm-core-python @DataDog/asm-python
 # Override because order matters
 tests/tracer/test_ci.py                            @DataDog/ci-app-libraries
 tests/tracer/test_ci_utils.py                      @DataDog/ci-app-libraries

.gitignore

@@ -234,6 +234,10 @@ vgcore.*
 
 # Rust build artifacts
 src/native/target*
+src/native_heap_gotter/target*
+
+# Profiling collector local CMake build dir
+ddtrace/profiling/collector/build-test/
 
 # Fuzzing corpus, output and artifacts
 .fuzz/

.gitlab-ci.yml

@@ -25,7 +25,7 @@ variables:
   # Repository URL for CI Visibility
   DD_GIT_REPOSITORY_URL: "https://github.com/DataDog/dd-trace-py.git"
   # General purpose CI image (built from dd/images/dd-trace-py/ci-utils)
-  CI_UTILS_IMAGE: "registry.ddbuild.io/dd-trace-py:v129724447-ccf4260-ci-utils@sha256:3ff0b19510872d740ee4bd3bddde9bb47567ccac7b6e4684bafb6b2bdaf62247"
+  CI_UTILS_IMAGE: "registry.ddbuild.io/dd-trace-py:v130207166-153cbe2-ci-utils@sha256:816d1defebb9f049f3bc2e9987ce90d47f7a6c0316b7fc720c30f56376be63ed"
 
   # One pipeline injection package size ratchet
   OCI_PACKAGE_MAX_SIZE_BYTES: 80_000_000
@@ -340,7 +340,12 @@ codeowners:
   stage: tests
   needs: []
   tags: ["arch:amd64"]
+  variables:
+    GIT_DEPTH: "0"
   image: ${CI_UTILS_IMAGE}
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   script:
     # Validate that each file entry only exists once in CODEOWNERS to prevent confusion about ownership.
     # The file is evaluated from top to bottom where the last match wins, so duplicate entries can lead to unexpected owners being assigned.
@@ -368,20 +373,27 @@ codeowners:
             echo "Please remove duplicates from .github/CODEOWNERS."
             exit 1
         fi
-    - git config --global --add safe.directory "${CI_PROJECT_DIR}"
-    - git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA > /tmp/changed_all.txt
-    # Filter to only files that exist on disk (deleted files cause codeowners to crash)
-    - |
-      > /tmp/changed_files.txt
-      while IFS= read -r f; do
-        [ -f "$f" ] && echo "$f" >> /tmp/changed_files.txt
-      done < /tmp/changed_all.txt
     - |
-      if [ -s /tmp/changed_files.txt ]; then
-        echo '```' > codeowners.txt
-        xargs codeowners < /tmp/changed_files.txt | tee -a codeowners.txt
-        echo '```' >> codeowners.txt
+      if [ -z "${GH_TOKEN:-}" ]; then
+        export GH_TOKEN=$(dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read)
       fi
+      git config --global --add safe.directory "${CI_PROJECT_DIR}"
+      BASE_BRANCH=$(.gitlab/scripts/resolve-base-branch.sh)
+      git fetch origin "${BASE_BRANCH}"
+      git show "FETCH_HEAD:.github/CODEOWNERS" > /tmp/base-CODEOWNERS
+      git diff --name-only --diff-filter=ACMRTUXB -z "FETCH_HEAD...${CI_COMMIT_SHA}" > /tmp/changed_files.txt
+    - |
+      {
+        printf 'Resolved from the full PR diff against `%s` using the target branch CODEOWNERS file.\n' "${BASE_BRANCH}"
+        printf 'CODEOWNERS team requests not listed below are not required by the current file set.\n\n'
+        if [ -s /tmp/changed_files.txt ]; then
+          echo '```'
+          xargs -0 codeowners --file /tmp/base-CODEOWNERS < /tmp/changed_files.txt
+          echo '```'
+        else
+          echo 'No remaining files require a CODEOWNERS review.'
+        fi
+      } | tee codeowners.txt
     - .gitlab/scripts/post-pr-comment.sh "Codeowners resolved as" codeowners.txt
 
 detect_circular_imports:
@@ -410,9 +422,10 @@ detect_circular_imports:
       fi
       BASE_BRANCH=$(.gitlab/scripts/resolve-base-branch.sh)
       git fetch origin "${BASE_BRANCH}"
-      git checkout FETCH_HEAD
+      MERGE_BASE=$(git merge-base FETCH_HEAD HEAD)
+      git checkout "${MERGE_BASE}"
       cd ..
-    # Analyse the base branch ddtrace using the PR version of cycles.py (has uv metadata)
+    # Analyse the merge base ddtrace using the PR version of cycles.py (has uv metadata)
     - uv run --script cycles.py analyze --root dd-trace-py/ddtrace cycles-base.json
     - cat cycles-base.json
     - |
@@ -451,9 +464,10 @@ detect_layering_violations:
       fi
       BASE_BRANCH=$(.gitlab/scripts/resolve-base-branch.sh)
       git fetch origin "${BASE_BRANCH}"
-      git checkout FETCH_HEAD
+      MERGE_BASE=$(git merge-base FETCH_HEAD HEAD)
+      git checkout "${MERGE_BASE}"
       cd ..
-    # Analyse the base branch ddtrace using the PR version of layers.py/layers.json (has uv metadata)
+    # Analyse the merge base using the PR version of layers.py/layers.json (has uv metadata)
     - uv run --script layers.py analyze --root dd-trace-py/ddtrace layers-base.json
     - cat layers-base.json
     - |
@@ -469,6 +483,11 @@ check_added_file_size:
   stage: tests
   needs: []
   extends: .testrunner
+  variables:
+    # Needs full history to reliably compute a merge-base against the fetched
+    # base branch; a shallow clone can leave HEAD and FETCH_HEAD without a
+    # common ancestor.
+    GIT_DEPTH: "0"
   rules:
     - if: !reference [.is_main]
       when: never
@@ -666,6 +685,10 @@ profiling_native:
         - src/native/**/*.toml
         - src/native/**/*.txt
         - src/native/**/Cargo.lock
+        # Standalone heap-gotter cdylib crate (opt-in build via setup.py)
+        - src/native_heap_gotter/**/*.rs
+        - src/native_heap_gotter/**/*.toml
+        - src/native_heap_gotter/**/Cargo.lock
         # Top-level build config
         - setup.py
         - pyproject.toml

.gitlab/benchmarks/microbenchmarks.yml

@@ -138,24 +138,26 @@ baseline:build:
     # and clamps the build container, causing OOMKills during parallel compilation.
     DD_DISABLE_VPA: "true"
   script: |
-    CACHED_WHL=$(ls *.whl | head -n 1) 2>/dev/null || echo ""
+    mkdir -p baseline-wheel
+    CACHED_WHL=$(ls baseline-wheel/*.whl | head -n 1) 2>/dev/null || echo ""
     if [ ! -f "${CACHED_WHL}" ];
     then
       .gitlab/benchmarks/steps/build-baseline.sh
+      mv *.whl baseline-wheel/
     else
       echo "Using wheel from cache for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA}:${CACHED_WHL}"
     fi
 
-    echo "BASELINE_WHL=$(ls *.whl | head -n 1)" | tee -a baseline.env
+    echo "BASELINE_WHL=$(ls baseline-wheel/*.whl | head -n 1)" | tee -a baseline.env
   cache:
     - key: v0-microbenchmarks-baseline-build-${BASELINE_COMMIT_SHA}
       paths:
-        - "*.whl"
+        - "baseline-wheel/*.whl"
   artifacts:
     reports:
       dotenv: baseline.env
     paths:
-      - "*.whl"
+      - "baseline-wheel/*.whl"
 
 candidate:
   image: $PACKAGE_IMAGE
@@ -166,16 +168,17 @@ candidate:
       job: "build linux: [amd64, cp39-cp39, v113741238-d2b8243-manylinux2014_x86_64]"
       artifacts: true
   script: |
-    cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./
-    echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env
+    mkdir -p candidate-wheel
+    cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl candidate-wheel/
+    echo "CANDIDATE_WHL=$(ls candidate-wheel/*.whl | head -n 1)" | tee candidate.env
     echo "CANDIDATE_BRANCH=${CI_COMMIT_REF_NAME}" | tee -a candidate.env
     echo "CANDIDATE_COMMIT_SHA=${CI_COMMIT_SHA}" | tee -a candidate.env
     echo "CANDIDATE_COMMIT_DATE=$(git show -s --format=%ct $CI_COMMIT_SHA)" | tee -a candidate.env
   artifacts:
     reports:
       dotenv: candidate.env
     paths:
-      - "*.whl"
+      - "candidate-wheel/*.whl"
 
 benchmarks-pr-comment:
   image: $MICROBENCHMARKS_CI_IMAGE

.gitlab/native.yml

@@ -21,6 +21,21 @@ include:
       echo -e "\e[0Ksection_start:`date +%s`:cargo_test[collapsed=true]\r\e[0Kcargo test"
       cargo test --no-fail-fast --locked
       echo -e "\e[0Ksection_end:`date +%s`:cargo_test\r\e[0K"
+    # The standalone heap-gotter cdylib crate lives outside src/native, so run
+    # the same fmt/clippy/test gate against it here to keep it under native CI.
+    - |
+      cd ../native_heap_gotter
+      echo -e "\e[0Ksection_start:`date +%s`:gotter_cargo_fmt[collapsed=true]\r\e[0Kheap-gotter cargo fmt"
+      cargo fmt --all -- --check
+      echo -e "\e[0Ksection_end:`date +%s`:gotter_cargo_fmt\r\e[0K"
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:gotter_cargo_clippy[collapsed=true]\r\e[0Kheap-gotter cargo clippy"
+      cargo clippy --locked --all-features -- -D warnings
+      echo -e "\e[0Ksection_end:`date +%s`:gotter_cargo_clippy\r\e[0K"
+    - |
+      echo -e "\e[0Ksection_start:`date +%s`:gotter_cargo_test[collapsed=true]\r\e[0Kheap-gotter cargo test"
+      cargo test --no-fail-fast --locked
+      echo -e "\e[0Ksection_end:`date +%s`:gotter_cargo_test\r\e[0K"
 
 "clang-tidy profiling":
   stage: tests

.gitlab/one-pipeline.locked.yml

@@ -1,4 +1,4 @@
 # DO NOT EDIT THIS FILE MANUALLY
 # This file is auto-generated by automation.
 include:
-  - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.2.0/one-pipeline.yml
+  - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.3.0/one-pipeline.yml

.gitlab/scripts/build-wheel-helpers.sh

@@ -99,9 +99,45 @@ build_wheel() {
 repair_wheel() {
   # Extract debug symbols
   section_start "extract_debug_symbols" "Extracting debug symbols"
-  uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}"
+  uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" \
+    --output-dir "${DEBUG_WHEEL_DIR}" \
+    --ignore-patterns "libddwaf*,libdd_heap_gotter*"
   section_end "extract_debug_symbols"
 
+  # Heap-gotter cdylib debug symbols are extracted in setup.py (build_heap_gotter);
+  # merge any staged .debug sidecars into the debug-symbols package.
+  section_start "merge_heap_gotter_debug_symbols" "Merging heap-gotter debug symbols"
+  uv run --no-project python - <<'PY'
+import glob
+import os
+import zipfile
+from pathlib import Path
+
+project_dir = os.environ["PROJECT_DIR"]
+debug_dir = os.environ["DEBUG_WHEEL_DIR"]
+sidecars = sorted(Path(project_dir, "build").rglob("libdd_heap_gotter*.debug"))
+if not sidecars:
+    print("No heap-gotter debug sidecars found")
+    raise SystemExit(0)
+packages = glob.glob(os.path.join(debug_dir, "*-debug-symbols.zip"))
+if not packages:
+    print("WARNING: no debug-symbols package to merge heap-gotter sidecars into")
+    raise SystemExit(0)
+pkg = packages[0]
+with zipfile.ZipFile(pkg, "a", zipfile.ZIP_DEFLATED) as zf:
+    existing = set(zf.namelist())
+    for sidecar in sidecars:
+        parts = sidecar.parts
+        try:
+            arc = str(Path(*parts[parts.index("ddtrace") :]))
+        except ValueError:
+            arc = sidecar.name
+        if arc not in existing:
+            zf.write(sidecar, arc)
+            print(f"Added heap-gotter debug symbols: {arc}")
+PY
+  section_end "merge_heap_gotter_debug_symbols"
+
   # Strip wheel
   section_start "strip_wheel" "Stripping unneeded files"
   uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md
@@ -115,7 +151,69 @@ repair_wheel() {
   # Repair wheel (ONLY PLATFORM-SPECIFIC CODE)
   section_start "repair_wheel" "Repairing wheel"
   if [[ "$(uname -s)" == "Linux" ]]; then
+    # The opt-in heap-gotter cdylib (DD_PROFILING_NATIVE_HEAP_BUILD=1) has
+    # non-standard ELF versioning sections that trip auditwheel's iter_versions
+    # parser. --exclude does not help: it only drops a SONAME from dependency
+    # grafting, while repair still parses every ELF listed in the wheel's RECORD.
+    # So the cdylib has to leave the wheel entirely and be reinserted after.
+    GOTTER_STASH_DIR="${WORK_DIR}/heap_gotter_stash"
+    GOTTER_PATTERN='*libdd_heap_gotter*.so'
+    if unzip -l "${BUILT_WHEEL_FILE}" | grep -q 'libdd_heap_gotter.*\.so$'; then
+      mkdir -p "${GOTTER_STASH_DIR}"
+      unzip -q "${BUILT_WHEEL_FILE}" "${GOTTER_PATTERN}" -d "${GOTTER_STASH_DIR}"
+      uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" "${GOTTER_PATTERN}"
+    fi
+
     auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}"
+
+    if [[ -d "${GOTTER_STASH_DIR}" ]]; then
+      REPAIRED_WHEEL_FILE=$(ls "${TMP_WHEEL_DIR}"/*.whl | head -n 1)
+      GOTTER_STASH_DIR="${GOTTER_STASH_DIR}" REPAIRED_WHEEL_FILE="${REPAIRED_WHEEL_FILE}" \
+        uv run --no-project python - <<'PY'
+import base64
+import csv
+import hashlib
+import io
+import os
+import zipfile
+from pathlib import Path
+
+wheel = Path(os.environ["REPAIRED_WHEEL_FILE"])
+stash = Path(os.environ["GOTTER_STASH_DIR"])
+
+additions = {str(p.relative_to(stash)): p for p in sorted(stash.rglob("*")) if p.is_file()}
+if not additions:
+    print("No stashed heap-gotter cdylib to reinsert")
+    raise SystemExit(0)
+
+tmp_wheel = Path(f"{wheel}.tmp")
+with (
+    zipfile.ZipFile(wheel, "r") as source_zip,
+    zipfile.ZipFile(tmp_wheel, "w", zipfile.ZIP_DEFLATED) as temp_zip,
+):
+    record = next((f for f in source_zip.infolist() if f.filename.endswith(".dist-info/RECORD")), None)
+    if record is None:
+        raise SystemExit(f"no RECORD found in {wheel}")
+    # DEV: Use ZipInfo objects to ensure original file attributes are preserved
+    for file in source_zip.infolist():
+        if file.filename == record.filename or file.filename in additions:
+            continue
+        temp_zip.writestr(file, source_zip.read(file.filename))
+    rows = [r for r in csv.reader(io.StringIO(source_zip.read(record.filename).decode("utf-8"))) if r]
+    rows = [r for r in rows if r[0] != record.filename and r[0] not in additions]
+    for arcname, path in additions.items():
+        data = path.read_bytes()
+        temp_zip.writestr(arcname, data)
+        digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode("ascii")
+        rows.append([arcname, f"sha256={digest}", str(len(data))])
+        print(f"Reinserted heap-gotter cdylib: {arcname}")
+    rows.append([record.filename, "", ""])
+    output = io.StringIO()
+    csv.writer(output, lineterminator="\n").writerows(rows)
+    temp_zip.writestr(record, output.getvalue())
+os.replace(tmp_wheel, wheel)
+PY
+    fi
   else
     # macOS
     MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel \