feat(llmobs): capture inline base64 images for OpenAI chat completions and responses

ddtrace/llmobs/_constants.py

@@ -177,6 +177,9 @@ class LLMObsSamplingDecision(str, Enum):
 IMAGE_FALLBACK_MARKER = "[image]"
 FILE_FALLBACK_MARKER = "[file]"
 AUDIO_FALLBACK_MARKER = "[audio]"
+# Distinct from IMAGE_FALLBACK_MARKER so a dropped inline image stays greppable instead of looking
+# like a remote reference we never fetch.
+IMAGE_TOO_LARGE_MARKER = "[image omitted: too large]"
 
 # OpenAI input types
 INPUT_TYPE_IMAGE = "input_image"

ddtrace/llmobs/_integrations/utils.py

@@ -7,6 +7,7 @@
 from typing import Optional
 from typing import Union
 
+from ddtrace import config
 from ddtrace._trace.span import Span
 from ddtrace.internal import core
 from ddtrace.internal.logger import get_logger
@@ -16,6 +17,7 @@
 from ddtrace.llmobs._constants import DISPATCH_ON_TOOL_CALL_OUTPUT_USED
 from ddtrace.llmobs._constants import FILE_FALLBACK_MARKER
 from ddtrace.llmobs._constants import IMAGE_FALLBACK_MARKER
+from ddtrace.llmobs._constants import IMAGE_TOO_LARGE_MARKER
 from ddtrace.llmobs._constants import INPUT_COST_METRIC_KEY
 from ddtrace.llmobs._constants import INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import INPUT_TYPE_FILE
@@ -395,16 +397,86 @@ def format_image_part(data: Union[bytes, str], mime_type: str) -> ImagePart:
     return ImagePart(mime_type=mime_type, content=content)
 
 
-def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart]]:
-    """Extract readable text and audio segments from multimodal content parts (e.g., text + image + audio)."""
+# AIDEV-NOTE: per-image, not cumulative. N images that each fit can still bust the event limit
+# together; a shared per-request budget is tracked under MLOB-6408.
+_IMAGE_INLINE_BUDGET_FRACTION = 0.8
+
+_UNKNOWN_OUTPUT_ITEM_MAX_CHARS = 4096
+
+# Case-insensitive per the data-URL scheme; tolerates extra media-type params (;charset=utf-8) and
+# the whitespace some encoders use to wrap base64. A non-base64 payload must fail here.
+_BASE64_IMAGE_DATA_URL = re.compile(r"^data:(image/[-\w.+]+)(?:;[\w.=+-]+)*;base64,([A-Za-z0-9+/=\s]+)$", re.IGNORECASE)
+
+
+def _inline_image_budget() -> int:
+    """Bytes of base64 one image may contribute, leaving room for the rest of the event.
+
+    Tracks the configured limit: admitting an image bigger than the active event allowance costs
+    the span its whole input and output (_writer._truncate_span_event).
+    """
+    return int(config._llmobs_event_size_limit * _IMAGE_INLINE_BUDGET_FRACTION)
+
+
+def _is_data_url(value: Any) -> bool:
+    """Whether value carries its payload inline as a data: URL rather than by reference."""
+    if not isinstance(value, str):
+        return False
+    # Scan past leading whitespace rather than slicing: a fixed prefix can cut into the scheme, and
+    # lstrip() on a multi-megabyte URL copies the whole payload.
+    i = 0
+    while i < len(value) and value[i].isspace():
+        i += 1
+    return value[i : i + 5].lower() == "data:"
+
+
+def _capture_inline_image(url: Any) -> tuple[Optional[ImagePart], Optional[str]]:
+    """Capture a data:image/...;base64,... URL as an ImagePart.
+
+    (part, None) on capture, (None, marker) when the payload must be dropped, and (None, None)
+    when url is not a data URL, leaving the caller's own reference text in place.
+    """
+    if not _is_data_url(url):
+        return None, None
+    match = _BASE64_IMAGE_DATA_URL.match(url)
+    # An unparseable data URL must not reach the caller's reference text, or the whole payload
+    # would land in the message content.
+    if not match:
+        return None, IMAGE_FALLBACK_MARKER
+    budget = _inline_image_budget()
+    # Reject absurd inputs on the raw span first so we don't copy megabytes we discard; anything
+    # plausible is normalized and measured exactly, so wrapped payloads that do fit are kept.
+    if match.end(2) - match.start(2) > 8 * budget:
+        return None, IMAGE_TOO_LARGE_MARKER
+    payload = "".join(match.group(2).split())
+    if not payload:
+        return None, IMAGE_FALLBACK_MARKER
+    # Size is decided here rather than read off the guard's None: the guard also rejects a bad mime
+    # or empty data, and mapping that to "too large" would report a wrong reason. Every non-size
+    # rejection happens above, so this marker is always accurate.
+    if len(payload) > budget:
+        logger.debug("Image (%d base64 bytes) exceeds inline budget %d; omitting inline content", len(payload), budget)
+        return None, IMAGE_TOO_LARGE_MARKER
+    return format_image_part(payload, match.group(1).lower()), None
+
+
+def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart], list[ImagePart]]:
+    """Extract readable text, audio and image segments from multimodal content parts."""
     extracted = []
     audio_parts: list[AudioPart] = []
+    image_parts: list[ImagePart] = []
     for part in parts:
         part_type = _get_attr(part, "type", "")
         if part_type == "text":
             extracted.append(str(_get_attr(part, "text", "")))
         elif part_type == "image_url":
-            extracted.append(IMAGE_FALLBACK_MARKER)
+            # Chat completions nest the URL as image_url.url; some callers pass a bare string.
+            image_url = _get_attr(part, "image_url", None)
+            image_part, marker = _capture_inline_image(_get_attr(image_url, "url", None) or image_url)
+            if image_part:
+                # Captured as a structured part (rendered inline), so no text marker is needed.
+                image_parts.append(image_part)
+            else:
+                extracted.append(marker or IMAGE_FALLBACK_MARKER)
         elif part_type == "input_audio":
             input_audio = _get_attr(part, "input_audio", {}) or {}
             data = _get_attr(input_audio, "data", "")
@@ -418,7 +490,7 @@ def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart]]:
                 extracted.append(AUDIO_FALLBACK_MARKER)
         else:
             extracted.append(f"[{part_type}]")
-    return "\n".join(extracted), audio_parts
+    return "\n".join(extracted), audio_parts, image_parts
 
 
 def openai_set_meta_tags_from_chat(
@@ -429,8 +501,9 @@ def openai_set_meta_tags_from_chat(
     for m in kwargs.get("messages", []):
         raw_content = _get_attr(m, "content", "")
         audio_parts: list[AudioPart] = []
+        image_parts: list[ImagePart] = []
         if isinstance(raw_content, list):
-            content, audio_parts = _extract_content_parts(raw_content)
+            content, audio_parts, image_parts = _extract_content_parts(raw_content)
         elif raw_content is None:
             content = ""
         else:
@@ -439,6 +512,8 @@ def openai_set_meta_tags_from_chat(
         processed_message: Message = Message(content=content, role=role)
         if audio_parts:
             processed_message["audio_parts"] = audio_parts
+        if image_parts:
+            processed_message["image_parts"] = image_parts
         tool_call_id = _get_attr(m, "tool_call_id", None)
         if tool_call_id:
             core.dispatch(DISPATCH_ON_TOOL_CALL_OUTPUT_USED, (tool_call_id, span))
@@ -735,21 +810,33 @@ def _openai_parse_input_response_messages(
         # Handle regular message
         if role is not None and content is not None:
             processed_item_content = ""
+            image_parts: list[ImagePart] = []
             if isinstance(content, list):
                 for content_part in content:
                     processed_item_content += str(_get_attr(content_part, "text", "") or "")
                     processed_item_content += str(_get_attr(content_part, "refusal", "") or "")
 
                     content_part_type = _get_attr(content_part, "type", None)
                     if content_part_type == INPUT_TYPE_IMAGE:
-                        processed_item_content += _extract_image_reference(content_part)
+                        raw_url = _get_attr(content_part, "image_url", None)
+                        # Unwrap the chat-shaped nested object too: the SDK does not enforce
+                        # the Responses shape at runtime and a dict here would leak inline.
+                        image_part, marker = _capture_inline_image(_get_attr(raw_url, "url", None) or raw_url)
+                        if image_part:
+                            image_parts.append(image_part)
+                        else:
+                            processed_item_content += marker or _extract_image_reference(content_part)
                     elif content_part_type == INPUT_TYPE_FILE:
                         processed_item_content += _extract_file_reference(content_part)
             else:
                 processed_item_content = content
-            if processed_item_content:
+            # An image-only message has no text left once the image is captured, but still must be
+            # emitted so the captured image_parts reach the span.
+            if processed_item_content or image_parts:
                 processed_item["content"] = str(processed_item_content)
                 processed_item["role"] = role
+            if image_parts:
+                processed_item["image_parts"] = image_parts
         elif item_type in ("function_call", "custom_tool_call"):
             # Process `ResponseFunctionToolCallParam` or ResponseCustomToolCallParam type from input messages
             arguments_str = arguments or input_ or OAI_HANDOFF_TOOL_ARG
@@ -913,8 +1000,17 @@ def _openai_parse_output_response_messages(
         elif message_type == "mcp_list_tools":
             mcp_tool_definitions.extend(_openai_get_tool_definitions(_get_attr(item, "tools", [])))
             continue
+        elif message_type == "image_generation_call":
+            # result holds the generated image as raw base64. Capturing it as an image_part is
+            # follow-up work (MLOB-6408); the fallback below would put the whole payload in the text.
+            message.update({"content": IMAGE_FALLBACK_MARKER, "role": "assistant"})
+        elif message_type == "computer_call_output":
+            # A screenshot's image_url may be an inline data URL, which must not reach the text.
+            message.update({"content": _extract_image_reference(_get_attr(item, "output", None)), "role": "user"})
         else:
-            message.update({"content": str(item), "role": "assistant"})
+            # Bound the catch-all: an unrecognized item may carry a binary payload on any field, and
+            # str() of an SDK model renders every value. Truncate rather than leak the next one.
+            message.update({"content": str(item)[:_UNKNOWN_OUTPUT_ITEM_MAX_CHARS], "role": "assistant"})
 
         processed.append(message)
 
@@ -948,8 +1044,16 @@ def openai_get_metadata_from_response(
 
 
 def _extract_image_reference(obj: Any) -> str:
-    """Extract image reference with fallback priority: image_url → file_id → [image]."""
-    return _get_attr(obj, "image_url", None) or _get_attr(obj, "file_id", None) or IMAGE_FALLBACK_MARKER
+    """Reference text for an image: its URL, else its file_id, else the marker.
+
+    Never inline content, and never a non-str: callers concatenate this straight into message
+    text. Accepts the nested image_url.url object as well as the bare string.
+    """
+    image_url = _get_attr(obj, "image_url", None)
+    reference = _get_attr(image_url, "url", None) or image_url or _get_attr(obj, "file_id", None)
+    if not isinstance(reference, str) or _is_data_url(reference):
+        return IMAGE_FALLBACK_MARKER
+    return reference
 
 
 def _extract_file_reference(obj: Any) -> str:

releasenotes/notes/llmobs-openai-image-parts-edf32dd7ad1d69db.yaml

@@ -0,0 +1,29 @@
+---
+features:
+  - |
+    LLM Observability: The OpenAI integration now captures inline base64 images from message content
+    as ``image_parts`` on LLM span messages, for both Chat Completions and the Responses API, so
+    image inputs render in LLM Observability. This also applies to the OpenAI Agents SDK and to the
+    LiteLLM integration, which share the same message extraction. Images referenced by a remote URL
+    or ``file_id`` are not fetched and keep their existing text reference. A single inline image
+    whose base64 payload exceeds 4 MiB is left as an ``[image omitted: too large]`` marker instead.
+    Note that this budget is per image, not per request: several inline images (or an image
+    alongside inline audio) can still take a span event past the 5 MB per-event limit, at which
+    point the event's input and output are replaced with a placeholder. To stop image bytes from
+    being recorded, register a span processor with ``LLMObs.register_processor`` and remove the
+    ``image_parts`` key from the messages on ``span.input``.
+fixes:
+  - |
+    LLM Observability: Fixes an issue where an inline base64 image sent in message content to the
+    OpenAI Responses API (including through the OpenAI Agents SDK) was recorded as the entire base64
+    data URL inside the message text, producing unreadable multi-megabyte span content.
+  - |
+    LLM Observability: Fixes an issue where an inline base64 image passed as a reusable-prompt
+    variable to the OpenAI Responses API was recorded as the entire base64 data URL on the span.
+    Such a variable is now recorded as ``[image]``; remote URL and ``file_id`` references are
+    unchanged.
+  - |
+    LLM Observability: Fixes an issue where OpenAI Responses API image-generation results and
+    computer-use screenshots were recorded as a stringified dump of the whole response item, which
+    embedded the image's base64 payload in the output message. They are now recorded as an
+    ``[image]`` marker, or as the screenshot's URL when it is a remote reference.

tests/contrib/openai/test_openai_llmobs.py

@@ -7,6 +7,7 @@
 from ddtrace.internal.utils.version import parse_version
 from ddtrace.llmobs import LLMObs
 from ddtrace.llmobs._integrations.utils import _est_tokens
+from ddtrace.llmobs._integrations.utils import _inline_image_budget
 from ddtrace.llmobs._utils import _get_attr
 from ddtrace.llmobs._utils import _get_llmobs_data_metastruct
 from ddtrace.llmobs._utils import get_llmobs_input_messages
@@ -491,6 +492,72 @@ def test_chat_completion_multimodal_content(self, openai, openai_llmobs, test_sp
             tags={"ml_app": "<ml-app-name>", "service": "tests.contrib.openai", "integration": "openai"},
         )
 
+    def test_chat_completion_inline_image_captured(self, openai, openai_llmobs, test_spans):
+        """An inline base64 image reaches the span as an image_part, leaving only the text content."""
+        with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_image_input.yaml"):
+            client = openai.OpenAI()
+            resp = client.chat.completions.create(
+                model="gpt-4-vision-preview",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": "What’s in this image?"},
+                            {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAECAw=="}},
+                        ],
+                    }
+                ],
+            )
+        spans = [s for trace in test_spans.pop_traces() for s in trace]
+        assert len(spans) == 1
+        assert_llmobs_span_data(
+            _get_llmobs_data_metastruct(spans[0]),
+            span_kind="llm",
+            name="OpenAI.createChatCompletion",
+            model_name=resp.model,
+            model_provider="openai",
+            input_messages=[
+                {
+                    "role": "user",
+                    "content": "What’s in this image?",
+                    "image_parts": [{"mime_type": "image/png", "content": "AAECAw=="}],
+                }
+            ],
+            output_messages=[{"role": "assistant", "content": resp.choices[0].message.content}],
+            metadata={},
+            metrics={"input_tokens": 1118, "output_tokens": 16, "total_tokens": 1134},
+            tags={"ml_app": "<ml-app-name>", "service": "tests.contrib.openai", "integration": "openai"},
+        )
+
+    def test_chat_completion_oversize_inline_image_preserves_text_and_response(self, openai, openai_llmobs, test_spans):
+        """Regression: an oversize inline image degrades to a marker rather than costing the whole
+        span input+output, so the message text and the model response both survive.
+        """
+        oversize_data_url = "data:image/png;base64," + "A" * (_inline_image_budget() + 4)
+        with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_image_input.yaml"):
+            client = openai.OpenAI()
+            resp = client.chat.completions.create(
+                model="gpt-4-vision-preview",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": "What’s in this image?"},
+                            {"type": "image_url", "image_url": {"url": oversize_data_url}},
+                        ],
+                    }
+                ],
+            )
+        spans = [s for trace in test_spans.pop_traces() for s in trace]
+        assert len(spans) == 1
+        span_event = _get_llmobs_data_metastruct(spans[0])
+        assert span_event["meta"]["input"]["messages"] == [
+            {"role": "user", "content": "What’s in this image?\n[image omitted: too large]"}
+        ]
+        assert span_event["meta"]["output"]["messages"] == [
+            {"role": "assistant", "content": resp.choices[0].message.content}
+        ]
+
     def test_chat_completion_multimodal_lazy_iterator(self, openai, openai_llmobs, test_spans):
         """Test that iterable message content is materialized to a list before the SDK
         consumes it, so post-call tag extraction still sees the content.