diff --git a/ddtrace/llmobs/_constants.py b/ddtrace/llmobs/_constants.py index b3f051fbf90..3c52b935d3d 100644 --- a/ddtrace/llmobs/_constants.py +++ b/ddtrace/llmobs/_constants.py @@ -177,6 +177,9 @@ class LLMObsSamplingDecision(str, Enum): IMAGE_FALLBACK_MARKER = "[image]" FILE_FALLBACK_MARKER = "[file]" AUDIO_FALLBACK_MARKER = "[audio]" +# Distinct from IMAGE_FALLBACK_MARKER so a dropped inline image stays greppable instead of looking +# like a remote reference we never fetch. +IMAGE_TOO_LARGE_MARKER = "[image omitted: too large]" # OpenAI input types INPUT_TYPE_IMAGE = "input_image" diff --git a/ddtrace/llmobs/_integrations/utils.py b/ddtrace/llmobs/_integrations/utils.py index ced5cdae6fe..864b77781e0 100644 --- a/ddtrace/llmobs/_integrations/utils.py +++ b/ddtrace/llmobs/_integrations/utils.py @@ -7,6 +7,7 @@ from typing import Optional from typing import Union +from ddtrace import config from ddtrace._trace.span import Span from ddtrace.internal import core from ddtrace.internal.logger import get_logger @@ -16,6 +17,7 @@ from ddtrace.llmobs._constants import DISPATCH_ON_TOOL_CALL_OUTPUT_USED from ddtrace.llmobs._constants import FILE_FALLBACK_MARKER from ddtrace.llmobs._constants import IMAGE_FALLBACK_MARKER +from ddtrace.llmobs._constants import IMAGE_TOO_LARGE_MARKER from ddtrace.llmobs._constants import INPUT_COST_METRIC_KEY from ddtrace.llmobs._constants import INPUT_TOKENS_METRIC_KEY from ddtrace.llmobs._constants import INPUT_TYPE_FILE @@ -395,16 +397,86 @@ def format_image_part(data: Union[bytes, str], mime_type: str) -> ImagePart: return ImagePart(mime_type=mime_type, content=content) -def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart]]: - """Extract readable text and audio segments from multimodal content parts (e.g., text + image + audio).""" +# AIDEV-NOTE: per-image, not cumulative. N images that each fit can still bust the event limit +# together; a shared per-request budget is tracked under MLOB-6408. +_IMAGE_INLINE_BUDGET_FRACTION = 0.8 + +_UNKNOWN_OUTPUT_ITEM_MAX_CHARS = 4096 + +# Case-insensitive per the data-URL scheme; tolerates extra media-type params (;charset=utf-8) and +# the whitespace some encoders use to wrap base64. A non-base64 payload must fail here. +_BASE64_IMAGE_DATA_URL = re.compile(r"^data:(image/[-\w.+]+)(?:;[\w.=+-]+)*;base64,([A-Za-z0-9+/=\s]+)$", re.IGNORECASE) + + +def _inline_image_budget() -> int: + """Bytes of base64 one image may contribute, leaving room for the rest of the event. + + Tracks the configured limit: admitting an image bigger than the active event allowance costs + the span its whole input and output (_writer._truncate_span_event). + """ + return int(config._llmobs_event_size_limit * _IMAGE_INLINE_BUDGET_FRACTION) + + +def _is_data_url(value: Any) -> bool: + """Whether value carries its payload inline as a data: URL rather than by reference.""" + if not isinstance(value, str): + return False + # Scan past leading whitespace rather than slicing: a fixed prefix can cut into the scheme, and + # lstrip() on a multi-megabyte URL copies the whole payload. + i = 0 + while i < len(value) and value[i].isspace(): + i += 1 + return value[i : i + 5].lower() == "data:" + + +def _capture_inline_image(url: Any) -> tuple[Optional[ImagePart], Optional[str]]: + """Capture a data:image/...;base64,... URL as an ImagePart. + + (part, None) on capture, (None, marker) when the payload must be dropped, and (None, None) + when url is not a data URL, leaving the caller's own reference text in place. + """ + if not _is_data_url(url): + return None, None + match = _BASE64_IMAGE_DATA_URL.match(url) + # An unparseable data URL must not reach the caller's reference text, or the whole payload + # would land in the message content. + if not match: + return None, IMAGE_FALLBACK_MARKER + budget = _inline_image_budget() + # Reject absurd inputs on the raw span first so we don't copy megabytes we discard; anything + # plausible is normalized and measured exactly, so wrapped payloads that do fit are kept. + if match.end(2) - match.start(2) > 8 * budget: + return None, IMAGE_TOO_LARGE_MARKER + payload = "".join(match.group(2).split()) + if not payload: + return None, IMAGE_FALLBACK_MARKER + # Size is decided here rather than read off the guard's None: the guard also rejects a bad mime + # or empty data, and mapping that to "too large" would report a wrong reason. Every non-size + # rejection happens above, so this marker is always accurate. + if len(payload) > budget: + logger.debug("Image (%d base64 bytes) exceeds inline budget %d; omitting inline content", len(payload), budget) + return None, IMAGE_TOO_LARGE_MARKER + return format_image_part(payload, match.group(1).lower()), None + + +def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart], list[ImagePart]]: + """Extract readable text, audio and image segments from multimodal content parts.""" extracted = [] audio_parts: list[AudioPart] = [] + image_parts: list[ImagePart] = [] for part in parts: part_type = _get_attr(part, "type", "") if part_type == "text": extracted.append(str(_get_attr(part, "text", ""))) elif part_type == "image_url": - extracted.append(IMAGE_FALLBACK_MARKER) + # Chat completions nest the URL as image_url.url; some callers pass a bare string. + image_url = _get_attr(part, "image_url", None) + image_part, marker = _capture_inline_image(_get_attr(image_url, "url", None) or image_url) + if image_part: + # Captured as a structured part (rendered inline), so no text marker is needed. + image_parts.append(image_part) + else: + extracted.append(marker or IMAGE_FALLBACK_MARKER) elif part_type == "input_audio": input_audio = _get_attr(part, "input_audio", {}) or {} data = _get_attr(input_audio, "data", "") @@ -418,7 +490,7 @@ def _extract_content_parts(parts: list) -> tuple[str, list[AudioPart]]: extracted.append(AUDIO_FALLBACK_MARKER) else: extracted.append(f"[{part_type}]") - return "\n".join(extracted), audio_parts + return "\n".join(extracted), audio_parts, image_parts def openai_set_meta_tags_from_chat( @@ -429,8 +501,9 @@ def openai_set_meta_tags_from_chat( for m in kwargs.get("messages", []): raw_content = _get_attr(m, "content", "") audio_parts: list[AudioPart] = [] + image_parts: list[ImagePart] = [] if isinstance(raw_content, list): - content, audio_parts = _extract_content_parts(raw_content) + content, audio_parts, image_parts = _extract_content_parts(raw_content) elif raw_content is None: content = "" else: @@ -439,6 +512,8 @@ def openai_set_meta_tags_from_chat( processed_message: Message = Message(content=content, role=role) if audio_parts: processed_message["audio_parts"] = audio_parts + if image_parts: + processed_message["image_parts"] = image_parts tool_call_id = _get_attr(m, "tool_call_id", None) if tool_call_id: core.dispatch(DISPATCH_ON_TOOL_CALL_OUTPUT_USED, (tool_call_id, span)) @@ -735,6 +810,7 @@ def _openai_parse_input_response_messages( # Handle regular message if role is not None and content is not None: processed_item_content = "" + image_parts: list[ImagePart] = [] if isinstance(content, list): for content_part in content: processed_item_content += str(_get_attr(content_part, "text", "") or "") @@ -742,14 +818,25 @@ def _openai_parse_input_response_messages( content_part_type = _get_attr(content_part, "type", None) if content_part_type == INPUT_TYPE_IMAGE: - processed_item_content += _extract_image_reference(content_part) + raw_url = _get_attr(content_part, "image_url", None) + # Unwrap the chat-shaped nested object too: the SDK does not enforce + # the Responses shape at runtime and a dict here would leak inline. + image_part, marker = _capture_inline_image(_get_attr(raw_url, "url", None) or raw_url) + if image_part: + image_parts.append(image_part) + else: + processed_item_content += marker or _extract_image_reference(content_part) elif content_part_type == INPUT_TYPE_FILE: processed_item_content += _extract_file_reference(content_part) else: processed_item_content = content - if processed_item_content: + # An image-only message has no text left once the image is captured, but still must be + # emitted so the captured image_parts reach the span. + if processed_item_content or image_parts: processed_item["content"] = str(processed_item_content) processed_item["role"] = role + if image_parts: + processed_item["image_parts"] = image_parts elif item_type in ("function_call", "custom_tool_call"): # Process `ResponseFunctionToolCallParam` or ResponseCustomToolCallParam type from input messages arguments_str = arguments or input_ or OAI_HANDOFF_TOOL_ARG @@ -913,8 +1000,17 @@ def _openai_parse_output_response_messages( elif message_type == "mcp_list_tools": mcp_tool_definitions.extend(_openai_get_tool_definitions(_get_attr(item, "tools", []))) continue + elif message_type == "image_generation_call": + # result holds the generated image as raw base64. Capturing it as an image_part is + # follow-up work (MLOB-6408); the fallback below would put the whole payload in the text. + message.update({"content": IMAGE_FALLBACK_MARKER, "role": "assistant"}) + elif message_type == "computer_call_output": + # A screenshot's image_url may be an inline data URL, which must not reach the text. + message.update({"content": _extract_image_reference(_get_attr(item, "output", None)), "role": "user"}) else: - message.update({"content": str(item), "role": "assistant"}) + # Bound the catch-all: an unrecognized item may carry a binary payload on any field, and + # str() of an SDK model renders every value. Truncate rather than leak the next one. + message.update({"content": str(item)[:_UNKNOWN_OUTPUT_ITEM_MAX_CHARS], "role": "assistant"}) processed.append(message) @@ -948,8 +1044,16 @@ def openai_get_metadata_from_response( def _extract_image_reference(obj: Any) -> str: - """Extract image reference with fallback priority: image_url → file_id → [image].""" - return _get_attr(obj, "image_url", None) or _get_attr(obj, "file_id", None) or IMAGE_FALLBACK_MARKER + """Reference text for an image: its URL, else its file_id, else the marker. + + Never inline content, and never a non-str: callers concatenate this straight into message + text. Accepts the nested image_url.url object as well as the bare string. + """ + image_url = _get_attr(obj, "image_url", None) + reference = _get_attr(image_url, "url", None) or image_url or _get_attr(obj, "file_id", None) + if not isinstance(reference, str) or _is_data_url(reference): + return IMAGE_FALLBACK_MARKER + return reference def _extract_file_reference(obj: Any) -> str: diff --git a/releasenotes/notes/llmobs-openai-image-parts-edf32dd7ad1d69db.yaml b/releasenotes/notes/llmobs-openai-image-parts-edf32dd7ad1d69db.yaml new file mode 100644 index 00000000000..734f8c427d6 --- /dev/null +++ b/releasenotes/notes/llmobs-openai-image-parts-edf32dd7ad1d69db.yaml @@ -0,0 +1,29 @@ +--- +features: + - | + LLM Observability: The OpenAI integration now captures inline base64 images from message content + as ``image_parts`` on LLM span messages, for both Chat Completions and the Responses API, so + image inputs render in LLM Observability. This also applies to the OpenAI Agents SDK and to the + LiteLLM integration, which share the same message extraction. Images referenced by a remote URL + or ``file_id`` are not fetched and keep their existing text reference. A single inline image + whose base64 payload exceeds 4 MiB is left as an ``[image omitted: too large]`` marker instead. + Note that this budget is per image, not per request: several inline images (or an image + alongside inline audio) can still take a span event past the 5 MB per-event limit, at which + point the event's input and output are replaced with a placeholder. To stop image bytes from + being recorded, register a span processor with ``LLMObs.register_processor`` and remove the + ``image_parts`` key from the messages on ``span.input``. +fixes: + - | + LLM Observability: Fixes an issue where an inline base64 image sent in message content to the + OpenAI Responses API (including through the OpenAI Agents SDK) was recorded as the entire base64 + data URL inside the message text, producing unreadable multi-megabyte span content. + - | + LLM Observability: Fixes an issue where an inline base64 image passed as a reusable-prompt + variable to the OpenAI Responses API was recorded as the entire base64 data URL on the span. + Such a variable is now recorded as ``[image]``; remote URL and ``file_id`` references are + unchanged. + - | + LLM Observability: Fixes an issue where OpenAI Responses API image-generation results and + computer-use screenshots were recorded as a stringified dump of the whole response item, which + embedded the image's base64 payload in the output message. They are now recorded as an + ``[image]`` marker, or as the screenshot's URL when it is a remote reference. diff --git a/tests/contrib/openai/test_openai_llmobs.py b/tests/contrib/openai/test_openai_llmobs.py index a3fb19f9f81..5e8dcdffd8f 100644 --- a/tests/contrib/openai/test_openai_llmobs.py +++ b/tests/contrib/openai/test_openai_llmobs.py @@ -7,6 +7,7 @@ from ddtrace.internal.utils.version import parse_version from ddtrace.llmobs import LLMObs from ddtrace.llmobs._integrations.utils import _est_tokens +from ddtrace.llmobs._integrations.utils import _inline_image_budget from ddtrace.llmobs._utils import _get_attr from ddtrace.llmobs._utils import _get_llmobs_data_metastruct from ddtrace.llmobs._utils import get_llmobs_input_messages @@ -491,6 +492,72 @@ def test_chat_completion_multimodal_content(self, openai, openai_llmobs, test_sp tags={"ml_app": "", "service": "tests.contrib.openai", "integration": "openai"}, ) + def test_chat_completion_inline_image_captured(self, openai, openai_llmobs, test_spans): + """An inline base64 image reaches the span as an image_part, leaving only the text content.""" + with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_image_input.yaml"): + client = openai.OpenAI() + resp = client.chat.completions.create( + model="gpt-4-vision-preview", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s in this image?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAECAw=="}}, + ], + } + ], + ) + spans = [s for trace in test_spans.pop_traces() for s in trace] + assert len(spans) == 1 + assert_llmobs_span_data( + _get_llmobs_data_metastruct(spans[0]), + span_kind="llm", + name="OpenAI.createChatCompletion", + model_name=resp.model, + model_provider="openai", + input_messages=[ + { + "role": "user", + "content": "What’s in this image?", + "image_parts": [{"mime_type": "image/png", "content": "AAECAw=="}], + } + ], + output_messages=[{"role": "assistant", "content": resp.choices[0].message.content}], + metadata={}, + metrics={"input_tokens": 1118, "output_tokens": 16, "total_tokens": 1134}, + tags={"ml_app": "", "service": "tests.contrib.openai", "integration": "openai"}, + ) + + def test_chat_completion_oversize_inline_image_preserves_text_and_response(self, openai, openai_llmobs, test_spans): + """Regression: an oversize inline image degrades to a marker rather than costing the whole + span input+output, so the message text and the model response both survive. + """ + oversize_data_url = "data:image/png;base64," + "A" * (_inline_image_budget() + 4) + with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_image_input.yaml"): + client = openai.OpenAI() + resp = client.chat.completions.create( + model="gpt-4-vision-preview", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s in this image?"}, + {"type": "image_url", "image_url": {"url": oversize_data_url}}, + ], + } + ], + ) + spans = [s for trace in test_spans.pop_traces() for s in trace] + assert len(spans) == 1 + span_event = _get_llmobs_data_metastruct(spans[0]) + assert span_event["meta"]["input"]["messages"] == [ + {"role": "user", "content": "What’s in this image?\n[image omitted: too large]"} + ] + assert span_event["meta"]["output"]["messages"] == [ + {"role": "assistant", "content": resp.choices[0].message.content} + ] + def test_chat_completion_multimodal_lazy_iterator(self, openai, openai_llmobs, test_spans): """Test that iterable message content is materialized to a list before the SDK consumes it, so post-call tag extraction still sees the content. diff --git a/tests/llmobs/test_integrations_utils.py b/tests/llmobs/test_integrations_utils.py index 393917fb5d2..0cc3acd8083 100644 --- a/tests/llmobs/test_integrations_utils.py +++ b/tests/llmobs/test_integrations_utils.py @@ -2,6 +2,9 @@ from types import SimpleNamespace from ddtrace.ext import SpanTypes +from ddtrace.internal.evp_proxy.constants import DEFAULT_EVP_EVENT_SIZE_LIMIT +from ddtrace.llmobs._constants import IMAGE_FALLBACK_MARKER +from ddtrace.llmobs._constants import IMAGE_TOO_LARGE_MARKER from ddtrace.llmobs._integrations.agent_manifest import MAX_WIRE_DEPTH from ddtrace.llmobs._integrations.agent_manifest import is_number from ddtrace.llmobs._integrations.agent_manifest import prune_empty @@ -16,10 +19,13 @@ from ddtrace.llmobs._integrations.audio_utils import is_renderable_audio_mime from ddtrace.llmobs._integrations.audio_utils import pcm16_to_wav from ddtrace.llmobs._integrations.audio_utils import realtime_audio_format_to_mime +from ddtrace.llmobs._integrations.utils import _capture_inline_image from ddtrace.llmobs._integrations.utils import _extract_chat_template_from_instructions from ddtrace.llmobs._integrations.utils import _extract_content_parts +from ddtrace.llmobs._integrations.utils import _inline_image_budget from ddtrace.llmobs._integrations.utils import _normalize_prompt_variables from ddtrace.llmobs._integrations.utils import _openai_parse_input_response_messages +from ddtrace.llmobs._integrations.utils import _openai_parse_output_response_messages from ddtrace.llmobs._integrations.utils import format_image_part from ddtrace.llmobs._integrations.utils import openai_construct_message_from_streamed_chunks from ddtrace.llmobs._integrations.utils import openai_construct_tool_call_from_streamed_chunk @@ -27,6 +33,7 @@ from ddtrace.llmobs._utils import _annotate_llmobs_span_data from ddtrace.llmobs._utils import get_llmobs_input_messages from ddtrace.llmobs._utils import safe_json +from tests.utils import override_global_config def test_format_audio_part_from_bytes(): @@ -55,6 +62,14 @@ def test_format_image_part_from_base64_string(): assert part == {"mime_type": "image/jpeg", "content": "AAECAw=="} +def _data_url(b64, mime_type="image/png"): + return "data:{};base64,{}".format(mime_type, b64) + + +# Just over the live budget, so the oversize tests exercise the configured value, not a literal. +_OVERSIZE_B64 = "A" * (_inline_image_budget() + 4) + + def test_audio_mime_type_from_format(): """OpenAI audio formats map to MIME types, falling back to audio/.""" assert audio_mime_type_from_format("wav") == "audio/wav" @@ -67,7 +82,7 @@ def test_audio_mime_type_from_format(): def test_extract_content_parts_collects_audio(): """Captured input_audio becomes an AudioPart and leaves no '[audio]' text marker behind.""" - text, audio_parts = _extract_content_parts( + text, audio_parts, _ = _extract_content_parts( [ {"type": "text", "text": "what is said here?"}, {"type": "input_audio", "input_audio": {"data": "AAECAw==", "format": "mp3"}}, @@ -79,7 +94,7 @@ def test_extract_content_parts_collects_audio(): def test_extract_content_parts_multiple_audio_only(): """A message with only input_audio parts captures each as an AudioPart and has empty text.""" - text, audio_parts = _extract_content_parts( + text, audio_parts, _ = _extract_content_parts( [ {"type": "input_audio", "input_audio": {"data": "AAA=", "format": "wav"}}, {"type": "input_audio", "input_audio": {"data": "BBB=", "format": "mp3"}}, @@ -94,7 +109,7 @@ def test_extract_content_parts_multiple_audio_only(): def test_extract_content_parts_audio_marker_fallback_when_no_data(): """When an input_audio part carries no data, fall back to the '[audio]' text marker.""" - text, audio_parts = _extract_content_parts( + text, audio_parts, _ = _extract_content_parts( [ {"type": "text", "text": "listen:"}, {"type": "input_audio", "input_audio": {"format": "wav"}}, @@ -106,7 +121,7 @@ def test_extract_content_parts_audio_marker_fallback_when_no_data(): def test_extract_content_parts_no_audio(): """Text/image-only content yields no audio parts.""" - text, audio_parts = _extract_content_parts( + text, audio_parts, _ = _extract_content_parts( [ {"type": "text", "text": "hello"}, {"type": "image_url", "image_url": "http://example.com/x.png"}, @@ -116,6 +131,180 @@ def test_extract_content_parts_no_audio(): assert audio_parts == [] +def test_extract_content_parts_captures_inline_image(): + """An inline base64 data URL becomes an ImagePart and leaves no '[image]' text marker.""" + text, _, image_parts = _extract_content_parts( + [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", "image_url": {"url": _data_url("AAECAw==")}}, + ] + ) + assert text == "what is in this image?" + assert image_parts == [{"mime_type": "image/png", "content": "AAECAw=="}] + + +def test_extract_content_parts_captures_inline_image_bare_string(): + """The URL may arrive as a bare string rather than the nested image_url.url object.""" + _, _, image_parts = _extract_content_parts( + [{"type": "image_url", "image_url": _data_url("BBBB", mime_type="image/webp")}] + ) + assert image_parts == [{"mime_type": "image/webp", "content": "BBBB"}] + + +def test_extract_content_parts_multiple_inline_images(): + """Each inline image is captured as its own part, preserving order and per-part mime type.""" + _, _, image_parts = _extract_content_parts( + [ + {"type": "image_url", "image_url": {"url": _data_url("AAA=", mime_type="image/png")}}, + {"type": "image_url", "image_url": {"url": _data_url("BBB=", mime_type="image/jpeg")}}, + ] + ) + assert image_parts == [ + {"mime_type": "image/png", "content": "AAA="}, + {"mime_type": "image/jpeg", "content": "BBB="}, + ] + + +def test_extract_content_parts_oversize_inline_image_keeps_marker_and_text(): + """An oversize inline image is dropped to a distinct marker; surrounding text is untouched.""" + text, _, image_parts = _extract_content_parts( + [ + {"type": "text", "text": "describe this"}, + {"type": "image_url", "image_url": {"url": _data_url(_OVERSIZE_B64)}}, + {"type": "text", "text": "in one word"}, + ] + ) + assert text == "describe this\n{}\nin one word".format(IMAGE_TOO_LARGE_MARKER) + assert image_parts == [] + + +def test_extract_content_parts_non_inline_image_keeps_generic_marker(): + """Remote URLs, missing URLs and malformed/non-image data URLs are not captured.""" + for image_url in ( + "https://example.com/x.png", + {"url": "https://example.com/x.png"}, + {"url": "data:image/png;base64,"}, # no payload + {"url": "data:image/png;base64, \n "}, # whitespace-only payload + {"url": "data:image/png;base64,not base64!!!"}, # payload isn't base64 + {"url": "data:application/pdf;base64,AAA="}, # not an image + {"url": "data:image/png,AAA="}, # not base64 + {"url": ""}, + {}, + None, + ): + text, _, image_parts = _extract_content_parts([{"type": "image_url", "image_url": image_url}]) + assert text == "[image]", image_url + assert image_parts == [], image_url + + +def test_extract_content_parts_wrapped_base64_payload_is_normalized(): + """Whitespace used to wrap base64 across lines is stripped, not carried into the part.""" + _, _, image_parts = _extract_content_parts( + [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA\nEC\nAw=="}}] + ) + assert image_parts == [{"mime_type": "image/png", "content": "AAECAw=="}] + + +def test_extract_content_parts_data_url_variants_captured(): + """Extra media-type params, uppercase scheme and svg+xml are all still inline base64.""" + for url, expected_mime in ( + ("data:image/png;charset=utf-8;base64,AAA=", "image/png"), + ("DATA:IMAGE/PNG;BASE64,AAA=", "image/png"), + ("data:image/svg+xml;base64,AAA=", "image/svg+xml"), + ): + _, _, image_parts = _extract_content_parts([{"type": "image_url", "image_url": {"url": url}}]) + assert image_parts == [{"mime_type": expected_mime, "content": "AAA="}], url + + +def test_multiple_in_budget_images_can_still_exceed_the_event_size_limit(): + """Pins the known limit of a PER-IMAGE guard: it does not bound the event. + + Two images that each pass the guard still serialize past the per-event limit, at which point + the writer drops the span's whole input and output. A cumulative per-request budget is the + deliberate follow-up; this test exists so that gap cannot be mistaken for a guarantee. + """ + half_budget_b64 = "A" * (_inline_image_budget() // 2) + + _, _, image_parts = _extract_content_parts( + [ + {"type": "text", "text": "compare these"}, + {"type": "image_url", "image_url": {"url": _data_url(half_budget_b64)}}, + {"type": "image_url", "image_url": {"url": _data_url(half_budget_b64)}}, + {"type": "image_url", "image_url": {"url": _data_url(half_budget_b64)}}, + ] + ) + assert len(image_parts) == 3 + assert len(safe_json(image_parts)) > DEFAULT_EVP_EVENT_SIZE_LIMIT + + +def test_capture_inline_image_rejects_oversize_at_every_scale(): + """Oversize inline images degrade to the marker whether barely or hugely over budget. + + Line-wrapped payloads must be measured after whitespace is stripped, so a wrapped image that + fits once normalized is still captured. + """ + assert _capture_inline_image(_data_url(_OVERSIZE_B64)) == (None, IMAGE_TOO_LARGE_MARKER) + huge = _data_url("A" * (16 * _inline_image_budget())) + assert _capture_inline_image(huge) == (None, IMAGE_TOO_LARGE_MARKER) + + wrapped = "\n".join(["A" * 76] * 8) # MIME-style line wrapping, comfortably in budget + part, marker = _capture_inline_image(_data_url(wrapped)) + assert marker is None + assert part == {"mime_type": "image/png", "content": "A" * (76 * 8)} + + +def test_is_data_url_detects_scheme_after_long_leading_whitespace(): + """Leading whitespace of any length must not hide the scheme and let the payload reach text.""" + payload = "A" * 4096 + assert _capture_inline_image(_data_url(payload))[0] is not None # unpadded still captures + + for pad in (" ", "\n" * 40, " \t\n" * 30): + url = pad + _data_url(payload) + # Whitespace defeats the regex's ^data: anchor, so this degrades to a marker rather than + # being captured -- what matters is that the payload never reaches the text. + assert _capture_inline_image(url) == (None, IMAGE_FALLBACK_MARKER), len(pad) + text, _, _ = _extract_content_parts([{"type": "image_url", "image_url": {"url": url}}]) + assert payload not in text, len(pad) + + +def test_inline_image_budget_follows_configured_event_size_limit(): + """The guard must track DD_LLMOBS_EVENT_SIZE_BYTES, not a fixed 4 MiB. + + A lower configured limit means an image the fixed budget would admit is larger than the whole + event allowance, so the writer would drop the span's entire input and output. + """ + small = "A" * 200_000 + part, marker = _capture_inline_image(_data_url(small)) + assert marker is None and part is not None + + with override_global_config(dict(_llmobs_event_size_limit=100_000)): + part, marker = _capture_inline_image(_data_url(small)) + assert (part, marker) == (None, IMAGE_TOO_LARGE_MARKER) + + +def test_oversize_marker_is_only_used_for_size(): + """The too-large marker must never stand in for another rejection reason. + + The shared guard also returns None for a bad mime or empty data; if the caller read size off + that None, a customer would be told an image was too large when it never was. + """ + for url in ("data:image/png;base64,", "data:application/pdf;base64,AAA=", "data:image/png,AAA="): + assert _capture_inline_image(url)[1] == IMAGE_FALLBACK_MARKER, url + + +def test_capture_inline_image_never_leaks_unparsed_data_url_as_text(): + """Invariant: a data URL we cannot parse degrades to a marker, never to the raw payload. + + Leaking it would put the whole base64 blob in the message content — the bug this guards. + """ + unparseable = "data:image/png;base64" + "A" * 64 # no "," separator + part, marker = _capture_inline_image(unparseable) + assert part is None + assert marker == "[image]" + _, _, image_parts = _extract_content_parts([{"type": "image_url", "image_url": {"url": unparseable}}]) + assert image_parts == [] + + def test_realtime_audio_format_to_mime_legacy_strings(): """Legacy string realtime formats map to MIME types.""" assert realtime_audio_format_to_mime("pcm16") == "audio/pcm" @@ -413,6 +602,71 @@ def __init__(self, file_url=None, file_id=None, filename=None, file_data=None): assert result["file_fallback"] == "[file]" +def test_output_image_generation_call_does_not_leak_base64(): + """A generated image's base64 result must not be stringified into the output message. + + Unhandled output item types fall back to str(item), which on the SDK's pydantic model + renders every field value -- including a multi-megabyte result. + """ + + from openai.types.responses.response_output_item import ImageGenerationCall + + item = ImageGenerationCall(id="ig_1", type="image_generation_call", status="completed", result="A" * 8192) + assert "A" * 64 in str(item) # the leak this guards: pydantic str() renders every field value + + processed, _, _ = _openai_parse_output_response_messages([item]) + assert processed == [{"content": "[image]", "role": "assistant"}] + + +def test_output_computer_call_screenshot_does_not_leak_base64(): + """A computer-use screenshot keeps a remote reference but never an inline data URL.""" + + class Screenshot: + def __init__(self, image_url=None, file_id=None): + self.type = "computer_screenshot" + self.image_url = image_url + self.file_id = file_id + + class ComputerCallOutput: + def __init__(self, output): + self.type = "computer_call_output" + self.output = output + + inline, _, _ = _openai_parse_output_response_messages([ComputerCallOutput(Screenshot(_data_url("A" * 8192)))]) + # role=user mirrors function_call_output: a tool result supplied to the model, not its own output. + assert inline == [{"content": "[image]", "role": "user"}] + + remote, _, _ = _openai_parse_output_response_messages( + [ComputerCallOutput(Screenshot(image_url="https://example.com/shot.png"))] + ) + assert remote == [{"content": "https://example.com/shot.png", "role": "user"}] + + +def test_normalize_prompt_variables_inline_image_degrades_to_marker(): + """A prompt variable holding an inline data URL must not put base64 on the span. + + Prompt variables are a plain string map, so there is nowhere to attach an ImagePart; the marker + keeps the payload off the event. Remote URLs and file_ids still keep their reference. + """ + + class ResponseInputImage: + def __init__(self, image_url=None, file_id=None): + self.type = "input_image" + self.image_url = image_url + self.file_id = file_id + + result = _normalize_prompt_variables( + { + "inline": ResponseInputImage(image_url=_data_url("A" * 4096)), + "remote": ResponseInputImage(image_url="https://example.com/img.png"), + "by_id": ResponseInputImage(file_id="file-123"), + } + ) + assert result["inline"] == "[image]" + assert result["remote"] == "https://example.com/img.png" + assert result["by_id"] == "file-123" + + def test_extract_chat_template_with_falsy_values(): """Test that falsy but valid values (0, False) are preserved in template extraction.""" @@ -589,6 +843,106 @@ class FakeResponseReasoningItem: assert processed[0]["role"] == "user" assert tool_call_ids == [] + def test_input_image_inline_base64_captured(self): + """An input_image data URL is captured as an ImagePart, not concatenated into the text.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + {"type": "input_image", "image_url": _data_url("AAECAw==")}, + ], + } + ] + processed, _ = _openai_parse_input_response_messages(messages) + assert processed == [ + { + "content": "what is this?", + "role": "user", + "image_parts": [{"mime_type": "image/png", "content": "AAECAw=="}], + } + ] + + def test_input_image_only_message_is_still_emitted(self): + """A message whose only content was a captured image must not be dropped.""" + messages = [{"role": "user", "content": [{"type": "input_image", "image_url": _data_url("AAA=")}]}] + processed, _ = _openai_parse_input_response_messages(messages) + assert processed == [ + {"content": "", "role": "user", "image_parts": [{"mime_type": "image/png", "content": "AAA="}]} + ] + + def test_input_image_oversize_keeps_marker_and_text(self): + """An oversize inline image degrades to a marker; the surrounding text survives.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "describe: "}, + {"type": "input_image", "image_url": _data_url(_OVERSIZE_B64)}, + ], + } + ] + processed, _ = _openai_parse_input_response_messages(messages) + assert processed == [{"content": "describe: {}".format(IMAGE_TOO_LARGE_MARKER), "role": "user"}] + + def test_input_image_remote_url_and_file_id_references_preserved(self): + """Capture is bytes-only: remote URLs and file_ids keep their existing reference text.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "input_image", "image_url": "https://example.com/x.png"}, + {"type": "input_image", "file_id": "file-abc123"}, + {"type": "input_image"}, + ], + } + ] + processed, _ = _openai_parse_input_response_messages(messages) + assert processed == [{"content": "https://example.com/x.pngfile-abc123[image]", "role": "user"}] + assert "image_parts" not in processed[0] + + def test_input_image_chat_shaped_url_object_does_not_leak(self): + """A chat-shaped nested image_url sent to the Responses parser must not reach the text. + + The SDK does not enforce the Responses shape at runtime, so this is what a user migrating + chat -> responses sends. It previously stringified the dict into the message content. + """ + payload = "A" * 4096 + messages = [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": {"url": _data_url(payload)}}], + } + ] + processed, _ = _openai_parse_input_response_messages(messages) + assert payload not in processed[0]["content"] # never in the message text + assert processed[0]["image_parts"] == [{"mime_type": "image/png", "content": payload}] + + def test_input_image_leading_whitespace_data_url_does_not_leak(self): + """Leading whitespace must not let a data URL bypass the inline check into message text.""" + payload = "B" * 4096 + messages = [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "\n " + _data_url(payload)}], + } + ] + processed, _ = _openai_parse_input_response_messages(messages) + assert payload not in safe_json(processed) + + def test_input_image_sdk_object_captured(self): + """SDK objects (attribute access, detail present) are captured the same as dicts.""" + + class ResponseInputImage: + type = "input_image" + detail = "auto" + file_id = None + image_url = _data_url("AAECAw==", mime_type="image/jpeg") + + messages = [{"role": "user", "content": [ResponseInputImage()]}] + processed, _ = _openai_parse_input_response_messages(messages) + assert processed[0]["image_parts"] == [{"mime_type": "image/jpeg", "content": "AAECAw=="}] + def _chunk(content=None, reasoning_content=None, role=None, finish_reason=None): delta = SimpleNamespace(content=content, reasoning_content=reasoning_content, role=role)