feat(llmobs): capture inline base64 images for OpenAI chat and responses

Inline `data:image/...;base64,...` content is captured as typed `image_parts`
on LLM span messages for both Chat Completions and the Responses API.

Three paths previously wrote a raw base64 payload into span text instead:

- the Responses API concatenated the whole data URL into the message content,
  which also dropped an image-only message because no text remained;
- a reusable-prompt variable holding a data URL recorded it verbatim;
- image-generation results and computer-use screenshots fell through to a
  `str(item)` fallback that renders every field of the SDK model, base64
  included.

Each now degrades to a marker, or to the remote reference where one exists.
Capturing generated images as `image_parts` is follow-up work.

A per-image size guard mirrors the audio helper and shares its base64 length
calculation. An oversize inline image is left as a distinct marker so the
surrounding text and the model response survive. Remote URLs and file_ids are
not fetched. The guard is per image, not cumulative; a test pins that limit.

MLOB-6408

Author: joizddog

ddtrace/llmobs/_constants.py

@@ -177,6 +177,9 @@ class LLMObsSamplingDecision(str, Enum):
 IMAGE_FALLBACK_MARKER = "[image]"
 FILE_FALLBACK_MARKER = "[file]"
 AUDIO_FALLBACK_MARKER = "[audio]"
+# Distinct from IMAGE_FALLBACK_MARKER so a dropped inline image stays greppable instead of looking
+# like a remote reference we never fetch.
+IMAGE_TOO_LARGE_MARKER = "[image omitted: too large]"
 
 # OpenAI input types
 INPUT_TYPE_IMAGE = "input_image"

ddtrace/llmobs/_integrations/utils.py

@@ -16,6 +16,7 @@
 from ddtrace.llmobs._constants import DISPATCH_ON_TOOL_CALL_OUTPUT_USED
 from ddtrace.llmobs._constants import FILE_FALLBACK_MARKER
 from ddtrace.llmobs._constants import IMAGE_FALLBACK_MARKER
+from ddtrace.llmobs._constants import IMAGE_TOO_LARGE_MARKER
 from ddtrace.llmobs._constants import INPUT_COST_METRIC_KEY
 from ddtrace.llmobs._constants import INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import INPUT_TYPE_FILE
@@ -34,6 +35,7 @@
 # public names here so existing ``from ...utils import <helper>`` imports keep working.
 from ddtrace.llmobs._integrations.audio_utils import G711_SAMPLE_RATE  # noqa: F401
 from ddtrace.llmobs._integrations.audio_utils import LLMOBS_AUDIO_INLINE_MAX_BYTES  # noqa: F401
+from ddtrace.llmobs._integrations.audio_utils import _base64_encoded_len
 from ddtrace.llmobs._integrations.audio_utils import audio_mime_type_from_format  # noqa: F401
 from ddtrace.llmobs._integrations.audio_utils import concat_base64_audio  # noqa: F401
 from ddtrace.llmobs._integrations.audio_utils import format_audio_part  # noqa: F401
@@ -395,16 +397,81 @@ def format_image_part(data: Union[bytes, str], mime_type: str) -> ImagePart:
     return ImagePart(mime_type=mime_type, content=content)
 
 
-def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart]]:
-    """Extract readable text and audio segments from multimodal content parts (e.g., text + image + audio)."""
+# Budget for a single inline image, measured on its base64 form (what actually rides the span event).
+# Kept under the 5 MB per-event limit with headroom for the rest of the event: an oversize event has
+# its whole input *and* output replaced with a placeholder (see ``_writer._truncate_span_event``).
+# AIDEV-NOTE: per-image, not cumulative. N images that each fit can still bust the event limit
+# together; a shared per-request budget is a deliberate follow-up.
+LLMOBS_IMAGE_INLINE_MAX_BYTES = 4 * 1024 * 1024
+
+# Tolerates extra media-type parameters (e.g. ``;charset=utf-8``) and is case-insensitive because
+# the data-URL scheme is. The payload class admits the whitespace some encoders use to wrap base64
+# across lines, but nothing else -- a non-base64 payload must fail here rather than be captured.
+_BASE64_IMAGE_DATA_URL = re.compile(r"^data:(image/[-\w.+]+)(?:;[\w.=+-]+)*;base64,([A-Za-z0-9+/=\s]+)$", re.IGNORECASE)
+
+
+def format_image_part_with_guard(
+    data: Union[bytes, str], mime_type: str, max_bytes: int = LLMOBS_IMAGE_INLINE_MAX_BYTES
+) -> Optional[ImagePart]:
+    """Build an ``ImagePart`` only when its base64 payload fits the inline budget.
+
+    Returns ``None`` for oversize images; callers keep a text marker instead so the surrounding
+    message text and the model response survive.
+    """
+    if not data or not mime_type:
+        return None
+    encoded_len = _base64_encoded_len(len(data)) if isinstance(data, bytes) else len(data)
+    if encoded_len > max_bytes:
+        logger.debug(
+            "Image (%d encoded bytes) exceeds inline budget %d; omitting inline image content", encoded_len, max_bytes
+        )
+        return None
+    return format_image_part(data, mime_type)
+
+
+def _is_data_url(value: Any) -> bool:
+    """Whether ``value`` carries its payload inline as a ``data:`` URL rather than by reference."""
+    return isinstance(value, str) and value[:5].lower() == "data:"
+
+
+def _capture_inline_image(url: Any) -> tuple[Optional[ImagePart], Optional[str]]:
+    """Capture a ``data:image/...;base64,...`` URL as an ``ImagePart``.
+
+    Returns ``(part, None)`` on capture and ``(None, marker)`` when the payload must be dropped.
+    ``(None, None)`` means ``url`` is not a data URL at all, so the caller keeps its own reference
+    text: remote URLs and ``file_id``s are never fetched.
+    """
+    # Skip the regex for the remote-URL case, which is everything not starting with the scheme.
+    if not _is_data_url(url):
+        return None, None
+    match = _BASE64_IMAGE_DATA_URL.match(url)
+    # Any data URL we can't parse must still not reach the caller's reference text, or the whole
+    # payload would land in the message content.
+    payload = "".join(match.group(2).split()) if match else ""
+    if not payload:
+        return None, IMAGE_FALLBACK_MARKER
+    part = format_image_part_with_guard(payload, match.group(1).lower())
+    return (part, None) if part else (None, IMAGE_TOO_LARGE_MARKER)
+
+
+def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart], list[ImagePart]]:
+    """Extract readable text, audio and image segments from multimodal content parts."""
     extracted = []
     audio_parts: list[AudioPart] = []
+    image_parts: list[ImagePart] = []
     for part in parts:
         part_type = _get_attr(part, "type", "")
         if part_type == "text":
             extracted.append(str(_get_attr(part, "text", "")))
         elif part_type == "image_url":
-            extracted.append(IMAGE_FALLBACK_MARKER)
+            # Chat completions nest the URL as ``image_url.url``; some callers pass a bare string.
+            image_url = _get_attr(part, "image_url", None)
+            image_part, marker = _capture_inline_image(_get_attr(image_url, "url", None) or image_url)
+            if image_part:
+                # Captured as a structured part (rendered inline), so no text marker is needed.
+                image_parts.append(image_part)
+            else:
+                extracted.append(marker or IMAGE_FALLBACK_MARKER)
         elif part_type == "input_audio":
             input_audio = _get_attr(part, "input_audio", {}) or {}
             data = _get_attr(input_audio, "data", "")
@@ -418,7 +485,7 @@ def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart]]:
                 extracted.append(AUDIO_FALLBACK_MARKER)
         else:
             extracted.append(f"[{part_type}]")
-    return "\n".join(extracted), audio_parts
+    return "\n".join(extracted), audio_parts, image_parts
 
 
 def openai_set_meta_tags_from_chat(
@@ -429,8 +496,9 @@ def openai_set_meta_tags_from_chat(
     for m in kwargs.get("messages", []):
         raw_content = _get_attr(m, "content", "")
         audio_parts: list[AudioPart] = []
+        image_parts: list[ImagePart] = []
         if isinstance(raw_content, list):
-            content, audio_parts = _extract_content_parts(raw_content)
+            content, audio_parts, image_parts = _extract_content_parts(raw_content)
         elif raw_content is None:
             content = ""
         else:
@@ -439,6 +507,8 @@ def openai_set_meta_tags_from_chat(
         processed_message: Message = Message(content=content, role=role)
         if audio_parts:
             processed_message["audio_parts"] = audio_parts
+        if image_parts:
+            processed_message["image_parts"] = image_parts
         tool_call_id = _get_attr(m, "tool_call_id", None)
         if tool_call_id:
             core.dispatch(DISPATCH_ON_TOOL_CALL_OUTPUT_USED, (tool_call_id, span))
@@ -735,21 +805,32 @@ def _openai_parse_input_response_messages(
         # Handle regular message
         if role is not None and content is not None:
             processed_item_content = ""
+            image_parts: list[ImagePart] = []
             if isinstance(content, list):
                 for content_part in content:
                     processed_item_content += str(_get_attr(content_part, "text", "") or "")
                     processed_item_content += str(_get_attr(content_part, "refusal", "") or "")
 
                     content_part_type = _get_attr(content_part, "type", None)
                     if content_part_type == INPUT_TYPE_IMAGE:
-                        processed_item_content += _extract_image_reference(content_part)
+                        image_part, marker = _capture_inline_image(_get_attr(content_part, "image_url", None))
+                        if image_part:
+                            image_parts.append(image_part)
+                        else:
+                            # str(): _extract_image_reference returns whatever ``image_url`` holds,
+                            # so a dict there would raise and cost the span all of its tags.
+                            processed_item_content += str(marker or _extract_image_reference(content_part))
                     elif content_part_type == INPUT_TYPE_FILE:
                         processed_item_content += _extract_file_reference(content_part)
             else:
                 processed_item_content = content
-            if processed_item_content:
+            # An image-only message has no text left once the image is captured, but still must be
+            # emitted so the captured image_parts reach the span.
+            if processed_item_content or image_parts:
                 processed_item["content"] = str(processed_item_content)
                 processed_item["role"] = role
+            if image_parts:
+                processed_item["image_parts"] = image_parts
         elif item_type in ("function_call", "custom_tool_call"):
             # Process `ResponseFunctionToolCallParam` or ResponseCustomToolCallParam type from input messages
             arguments_str = arguments or input_ or OAI_HANDOFF_TOOL_ARG
@@ -913,6 +994,13 @@ def _openai_parse_output_response_messages(
         elif message_type == "mcp_list_tools":
             mcp_tool_definitions.extend(_openai_get_tool_definitions(_get_attr(item, "tools", [])))
             continue
+        elif message_type in ("image_generation_call", "computer_call_output"):
+            # Both carry an image the generic str(item) fallback below would dump whole into the
+            # text: a generated image's ``result`` is raw base64, and a computer-use screenshot's
+            # ``image_url`` may be an inline data URL. Capturing them as image_parts is follow-up
+            # work; for now only the reference survives.
+            screenshot = _get_attr(item, "output", None)
+            message.update({"content": _image_reference_text(screenshot or item), "role": "assistant"})
         else:
             message.update({"content": str(item), "role": "assistant"})
 
@@ -962,11 +1050,21 @@ def _extract_file_reference(obj: Any) -> str:
     )
 
 
+def _image_reference_text(obj: Any) -> str:
+    """Reference text for an image, never the inline payload itself.
+
+    Prompt variables and chat templates are plain strings with nowhere to put an ``ImagePart``, so
+    an inline data URL degrades to the marker rather than putting megabytes of base64 on the span.
+    """
+    reference = _extract_image_reference(obj)
+    return IMAGE_FALLBACK_MARKER if _is_data_url(reference) else reference
+
+
 def _extract_content_item_text(content_item: Any) -> str:
     """Extract text representation from a content item (text/image/file)."""
     item_type = _get_attr(content_item, "type", None)
     if item_type == INPUT_TYPE_IMAGE:
-        return _extract_image_reference(content_item)
+        return _image_reference_text(content_item)
     elif item_type == INPUT_TYPE_FILE:
         return _extract_file_reference(content_item)
     elif item_type == INPUT_TYPE_TEXT or item_type is None:

releasenotes/notes/llmobs-openai-image-parts-edf32dd7ad1d69db.yaml

@@ -0,0 +1,26 @@
+---
+features:
+  - |
+    LLM Observability: The OpenAI integration now captures inline base64 images from message content
+    as ``image_parts`` on LLM span messages, for both Chat Completions and the Responses API, so
+    image inputs render in LLM Observability. Images referenced by a remote URL or ``file_id`` are
+    not fetched and keep their existing text reference. A single inline image whose base64 payload
+    exceeds 4 MiB is left as an ``[image omitted: too large]`` marker instead. Note that this budget
+    is per image, not per request: several inline images (or an image alongside inline audio) can
+    still take a span event past the 5 MB per-event limit, at which point the event's input and
+    output are replaced with a placeholder.
+fixes:
+  - |
+    LLM Observability: Fixes an issue where an inline base64 image sent in message content to the
+    OpenAI Responses API (including through the OpenAI Agents SDK) was recorded as the entire base64
+    data URL inside the message text, producing unreadable multi-megabyte span content.
+  - |
+    LLM Observability: Fixes an issue where an inline base64 image passed as a reusable-prompt
+    variable to the OpenAI Responses API was recorded as the entire base64 data URL on the span.
+    Such a variable is now recorded as ``[image]``; remote URL and ``file_id`` references are
+    unchanged.
+  - |
+    LLM Observability: Fixes an issue where OpenAI Responses API image-generation results and
+    computer-use screenshots were recorded as a stringified dump of the whole response item, which
+    embedded the image's base64 payload in the output message. They are now recorded as an
+    ``[image]`` marker, or as the screenshot's URL when it is a remote reference.

tests/contrib/openai/test_openai_llmobs.py

@@ -6,6 +6,7 @@
 
 from ddtrace.internal.utils.version import parse_version
 from ddtrace.llmobs import LLMObs
+from ddtrace.llmobs._integrations.utils import LLMOBS_IMAGE_INLINE_MAX_BYTES
 from ddtrace.llmobs._integrations.utils import _est_tokens
 from ddtrace.llmobs._utils import _get_attr
 from ddtrace.llmobs._utils import _get_llmobs_data_metastruct
@@ -491,6 +492,72 @@ def test_chat_completion_multimodal_content(self, openai, openai_llmobs, test_sp
             tags={"ml_app": "<ml-app-name>", "service": "tests.contrib.openai", "integration": "openai"},
         )
 
+    def test_chat_completion_inline_image_captured(self, openai, openai_llmobs, test_spans):
+        """An inline base64 image reaches the span as an image_part, leaving only the text content."""
+        with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_image_input.yaml"):
+            client = openai.OpenAI()
+            resp = client.chat.completions.create(
+                model="gpt-4-vision-preview",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": "What’s in this image?"},
+                            {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAECAw=="}},
+                        ],
+                    }
+                ],
+            )
+        spans = [s for trace in test_spans.pop_traces() for s in trace]
+        assert len(spans) == 1
+        assert_llmobs_span_data(
+            _get_llmobs_data_metastruct(spans[0]),
+            span_kind="llm",
+            name="OpenAI.createChatCompletion",
+            model_name=resp.model,
+            model_provider="openai",
+            input_messages=[
+                {
+                    "role": "user",
+                    "content": "What’s in this image?",
+                    "image_parts": [{"mime_type": "image/png", "content": "AAECAw=="}],
+                }
+            ],
+            output_messages=[{"role": "assistant", "content": resp.choices[0].message.content}],
+            metadata={},
+            metrics={"input_tokens": 1118, "output_tokens": 16, "total_tokens": 1134},
+            tags={"ml_app": "<ml-app-name>", "service": "tests.contrib.openai", "integration": "openai"},
+        )
+
+    def test_chat_completion_oversize_inline_image_preserves_text_and_response(self, openai, openai_llmobs, test_spans):
+        """Regression: an oversize inline image degrades to a marker rather than costing the whole
+        span input+output, so the message text and the model response both survive.
+        """
+        oversize_data_url = "data:image/png;base64," + "A" * (LLMOBS_IMAGE_INLINE_MAX_BYTES + 4)
+        with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_image_input.yaml"):
+            client = openai.OpenAI()
+            resp = client.chat.completions.create(
+                model="gpt-4-vision-preview",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": "What’s in this image?"},
+                            {"type": "image_url", "image_url": {"url": oversize_data_url}},
+                        ],
+                    }
+                ],
+            )
+        spans = [s for trace in test_spans.pop_traces() for s in trace]
+        assert len(spans) == 1
+        span_event = _get_llmobs_data_metastruct(spans[0])
+        assert span_event["meta"]["input"]["messages"] == [
+            {"role": "user", "content": "What’s in this image?\n[image omitted: too large]"}
+        ]
+        assert span_event["meta"]["output"]["messages"] == [
+            {"role": "assistant", "content": resp.choices[0].message.content}
+        ]
+
     def test_chat_completion_multimodal_lazy_iterator(self, openai, openai_llmobs, test_spans):
         """Test that iterable message content is materialized to a list before the SDK
         consumes it, so post-call tag extraction still sees the content.