From 4ca5aaaa45cc42c2c74035db625c1780a2dff370 Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Tue, 1 Sep 2026 17:22:38 +0900 Subject: [PATCH 1/7] fix(genai-openai): preserve multimodal content-part arrays in input messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a chat message uses the OpenAI content-part array form ([{"type": "text", ...}, {"type": "image_url", ...}]), _is_text_part rejects it and _prepare_input_messages had no fallback branch, so the message was captured with empty parts — the text part was dropped along with the non-text parts, even with content capture opted in. Map "text" parts to TextPart and preserve any other part type ("image_url", "input_audio", ...) as a GenericPart carrying the provider-specific type discriminator and payload, per the GenericPart contract in opentelemetry-util-genai. Fixes #521 --- .../instrumentation/genai/openai/utils.py | 40 +++++++ .../tests/test_prepare_input_messages_unit.py | 107 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index 79997fd0c..1070d88d0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -23,6 +23,7 @@ ) from opentelemetry.util.genai.types import ( FunctionToolDefinition, + GenericPart, InputMessage, OutputMessage, TextPart, @@ -191,6 +192,41 @@ def _is_text_part(content: Any) -> bool: ) +def _as_plain_value(item: Any) -> Any: + """Return a JSON-serializable representation of a content part.""" + if isinstance(item, Mapping): + return dict(item) + model_dump = getattr(item, "model_dump", None) + if callable(model_dump): + return model_dump() + return str(item) + + +def _extract_content_parts(content: Iterable[Any]) -> list[Any]: + """Map an OpenAI content-part array (multimodal content) to message parts. + + ``text`` parts map to ``TextPart``. Any other part type (``image_url``, + ``input_audio``, ...) is preserved as a ``GenericPart`` carrying the + provider-specific type discriminator and payload, so opted-in content + capture does not silently drop the message. + """ + parts: list[Any] = [] + for item in content: + if isinstance(item, str): + parts.append(TextPart(content=item)) + continue + item_type = get_property_value(item, "type") + if item_type == "text": + parts.append( + TextPart(content=str(get_property_value(item, "text") or "")) + ) + else: + parts.append( + GenericPart(type=str(item_type), value=_as_plain_value(item)) + ) + return parts + + def _prepare_input_messages(messages) -> list[InputMessage]: chat_messages = [] for message in messages: @@ -206,6 +242,8 @@ def _prepare_input_messages(messages) -> list[InputMessage]: chat_message.parts += extract_tool_calls_new(tool_calls) if _is_text_part(content): chat_message.parts.append(TextPart(content=str(content))) + elif content is not None and isinstance(content, Iterable): + chat_message.parts += _extract_content_parts(content) elif role == "tool": tool_call_id = get_property_value(message, "tool_call_id") @@ -217,6 +255,8 @@ def _prepare_input_messages(messages) -> list[InputMessage]: # system, developer, user, fallback if _is_text_part(content): chat_message.parts.append(TextPart(content=str(content))) + elif content is not None and isinstance(content, Iterable): + chat_message.parts += _extract_content_parts(content) return chat_messages diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py new file mode 100644 index 000000000..445cdba26 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -0,0 +1,107 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for input message preparation, including multimodal content.""" + +from dataclasses import asdict + +from opentelemetry.instrumentation.genai.openai.utils import ( + _prepare_input_messages, +) +from opentelemetry.util.genai.types import GenericPart, TextPart +from opentelemetry.util.genai.utils import gen_ai_json_dumps + + +def test_string_content_maps_to_text_part(): + messages = [{"role": "user", "content": "hello"}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [TextPart(content="hello")] + + +def test_multimodal_content_keeps_text_part(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.png"}, + }, + ], + } + ] + + result = _prepare_input_messages(messages) + + parts = result[0].parts + assert parts[0] == TextPart(content="What is in this image?") + assert isinstance(parts[1], GenericPart) + assert parts[1].type == "image_url" + assert parts[1].value == { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.png"}, + } + + +def test_input_audio_content_preserved_as_generic_part(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Transcribe this"}, + { + "type": "input_audio", + "input_audio": {"data": "UklGRg==", "format": "wav"}, + }, + ], + } + ] + + result = _prepare_input_messages(messages) + + parts = result[0].parts + assert [type(part) for part in parts] == [TextPart, GenericPart] + assert parts[1].type == "input_audio" + + +def test_multimodal_assistant_history_content(): + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "It is a cat."}], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [TextPart(content="It is a cat.")] + + +def test_none_content_produces_no_parts(): + messages = [{"role": "assistant", "content": None}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [] + + +def test_multimodal_message_is_json_serializable(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": "https://e/x.png"}}, + ], + } + ] + + result = _prepare_input_messages(messages) + + serialized = gen_ai_json_dumps([asdict(message) for message in result]) + assert '"type": "image_url"'.replace(" ", "") in serialized.replace( + " ", "" + ) From a1fa5dabbaf60605fe34114d7a6b248e7ce3858b Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Tue, 1 Sep 2026 17:23:34 +0900 Subject: [PATCH 2/7] chore: add changelog fragment for #522 --- .../.changelog/522.fixed | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed new file mode 100644 index 000000000..fad9e028c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed @@ -0,0 +1,3 @@ +Preserve multimodal content-part arrays in captured input messages: ``text`` +parts map to ``TextPart`` and other part types (``image_url``, +``input_audio``, ...) are kept as ``GenericPart`` instead of being dropped. From c6030bb9278a6a91d9516183ecfead2480775318 Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Tue, 1 Sep 2026 17:31:08 +0900 Subject: [PATCH 3/7] fix(genai-openai): materialize content once before branching on its shape Address review feedback: _is_text_part could partially consume a single-pass iterable before _extract_content_parts iterated the remainder, and mappings fell into the content-part-array branch via key iteration. Route content through _content_to_parts, which materializes non-string, non-mapping iterables exactly once and keeps the previous behavior for strings and string-keyed mappings. --- .../instrumentation/genai/openai/utils.py | 37 +++++++++++++++---- .../tests/test_prepare_input_messages_unit.py | 23 ++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index 1070d88d0..004beee18 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -227,6 +227,33 @@ def _extract_content_parts(content: Iterable[Any]) -> list[Any]: return parts +def _content_to_parts(content: Any) -> list[Any]: + """Map message ``content`` to message parts. + + Strings map to a single ``TextPart``. Mappings keep the previous + ``_is_text_part`` behavior (string-keyed mappings are captured as + ``str(mapping)``) and are never treated as content-part arrays. Other + iterables are materialized exactly once — ``content`` may be a + single-pass iterable, and type-checking then re-iterating would silently + drop the items already consumed — then map to a ``TextPart`` when + string-only, or through ``_extract_content_parts`` otherwise. + """ + if isinstance(content, str): + return [TextPart(content=content)] + if content is None: + return [] + if isinstance(content, Mapping): + if all(isinstance(key, str) for key in content): + return [TextPart(content=str(content))] + return [] + if isinstance(content, Iterable): + items = list(content) + if all(isinstance(item, str) for item in items): + return [TextPart(content=str(items))] + return _extract_content_parts(items) + return [] + + def _prepare_input_messages(messages) -> list[InputMessage]: chat_messages = [] for message in messages: @@ -240,10 +267,7 @@ def _prepare_input_messages(messages) -> list[InputMessage]: tool_calls = get_property_value(message, "tool_calls") if tool_calls: chat_message.parts += extract_tool_calls_new(tool_calls) - if _is_text_part(content): - chat_message.parts.append(TextPart(content=str(content))) - elif content is not None and isinstance(content, Iterable): - chat_message.parts += _extract_content_parts(content) + chat_message.parts += _content_to_parts(content) elif role == "tool": tool_call_id = get_property_value(message, "tool_call_id") @@ -253,10 +277,7 @@ def _prepare_input_messages(messages) -> list[InputMessage]: else: # system, developer, user, fallback - if _is_text_part(content): - chat_message.parts.append(TextPart(content=str(content))) - elif content is not None and isinstance(content, Iterable): - chat_message.parts += _extract_content_parts(content) + chat_message.parts += _content_to_parts(content) return chat_messages diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py index 445cdba26..aa0008a39 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -88,6 +88,29 @@ def test_none_content_produces_no_parts(): assert result[0].parts == [] +def test_single_pass_iterable_content_is_not_partially_dropped(): + def content_parts(): + yield {"type": "text", "text": "hello"} + yield {"type": "image_url", "image_url": {"url": "https://e/x.png"}} + + messages = [{"role": "user", "content": content_parts()}] + + result = _prepare_input_messages(messages) + + parts = result[0].parts + assert [type(part) for part in parts] == [TextPart, GenericPart] + assert parts[0] == TextPart(content="hello") + + +def test_mapping_content_is_not_treated_as_content_part_array(): + content = {"unexpected": "shape"} + messages = [{"role": "user", "content": content}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [TextPart(content=str(content))] + + def test_multimodal_message_is_json_serializable(): messages = [ { From d7e6b9a92ff1d25ff6c780e39ea13ba097176dd8 Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Wed, 2 Sep 2026 09:46:57 +0900 Subject: [PATCH 4/7] fix(genai-openai): map multimodal parts to standard media parts Address review feedback on #522: - `image_url` parts map to `UriPart` / `BlobPart` via the shared `image_from_url` helper, `input_audio` to an audio `BlobPart`, and `file` to `FilePart` (file_id) / `BlobPart` (inline file_data). `GenericPart` is now used only for typed parts with no semconv representation. - Stop stringifying content: a bare mapping is treated as a single content part and string items become individual `TextPart`s instead of `str(list)` / `str(dict)` text parts. - Add a `MultimodalScenario` conformance test (text + external image URL + inline base64 image) recorded against gpt-4o-mini and validated with weaver live-check, asserting `text` / `uri` / `blob` parts round-trip onto the input message. Claude-Session: https://claude.ai/code/session_01QsXFvtQoFP6DwdBgyEA3xB --- .../.changelog/522.fixed | 7 +- .../instrumentation/genai/openai/utils.py | 137 ++++++++++---- .../cassettes/multimodal_conformance.yaml | 166 +++++++++++++++++ .../tests/conformance/multimodal.py | 129 +++++++++++++ .../tests/test_conformance.py | 2 + .../tests/test_prepare_input_messages_unit.py | 175 ++++++++++++------ 6 files changed, 513 insertions(+), 103 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed index fad9e028c..247770433 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed @@ -1,3 +1,4 @@ -Preserve multimodal content-part arrays in captured input messages: ``text`` -parts map to ``TextPart`` and other part types (``image_url``, -``input_audio``, ...) are kept as ``GenericPart`` instead of being dropped. +Capture multimodal content-part arrays on input messages instead of dropping +them: ``image_url`` parts map to ``UriPart``/``BlobPart``, ``input_audio`` to +an audio ``BlobPart``, ``file`` to ``FilePart``/``BlobPart``, and any other +typed part is preserved as a ``GenericPart``. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index 004beee18..14d6a117b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -4,7 +4,7 @@ from __future__ import annotations import json -from collections.abc import Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from typing import Any from urllib.parse import urlparse @@ -22,15 +22,19 @@ InferenceInvocation, ) from opentelemetry.util.genai.types import ( + BlobPart, + FilePart, FunctionToolDefinition, GenericPart, InputMessage, + MessagePart, OutputMessage, TextPart, ToolCallRequestPart, ToolCallResponsePart, ToolDefinition, ) +from opentelemetry.util.genai.utils import decode_base64, image_from_url _OpenAIOmit = getattr(openai, "Omit", None) @@ -192,6 +196,12 @@ def _is_text_part(content: Any) -> bool: ) +_AUDIO_MIME_TYPES: Mapping[str, str] = { + "mp3": "audio/mpeg", + "wav": "audio/wav", +} + + def _as_plain_value(item: Any) -> Any: """Return a JSON-serializable representation of a content part.""" if isinstance(item, Mapping): @@ -202,56 +212,101 @@ def _as_plain_value(item: Any) -> Any: return str(item) -def _extract_content_parts(content: Iterable[Any]) -> list[Any]: - """Map an OpenAI content-part array (multimodal content) to message parts. +def _image_url_part(item: Any) -> MessagePart | None: + """Map an ``image_url`` content part to a ``UriPart`` or ``BlobPart``.""" + image_url = get_property_value(item, "image_url") + url = ( + image_url + if isinstance(image_url, str) + else get_property_value(image_url, "url") + ) + if not isinstance(url, str) or not url: + return None + return image_from_url(url) + + +def _input_audio_part(item: Any) -> MessagePart | None: + """Map an ``input_audio`` content part to an audio ``BlobPart``.""" + input_audio = get_property_value(item, "input_audio") + data = get_property_value(input_audio, "data") + if not isinstance(data, str): + return None + content = decode_base64(data) + if content is None: + return None + audio_format = get_property_value(input_audio, "format") + mime_type = None + if isinstance(audio_format, str) and audio_format: + audio_format = audio_format.lower() + mime_type = _AUDIO_MIME_TYPES.get( + audio_format, f"audio/{audio_format}" + ) + return BlobPart(mime_type=mime_type, modality="audio", content=content) + + +def _file_part(item: Any) -> MessagePart | None: + """Map a ``file`` content part to a ``FilePart`` (``file_id``) or a + ``BlobPart`` (inline ``file_data`` data URL).""" + file_ref = get_property_value(item, "file") + file_id = get_property_value(file_ref, "file_id") + if isinstance(file_id, str) and file_id: + return FilePart(mime_type=None, modality="document", file_id=file_id) + file_data = get_property_value(file_ref, "file_data") + if isinstance(file_data, str) and file_data: + return image_from_url(file_data, modality="document") + return None + - ``text`` parts map to ``TextPart``. Any other part type (``image_url``, - ``input_audio``, ...) is preserved as a ``GenericPart`` carrying the - provider-specific type discriminator and payload, so opted-in content - capture does not silently drop the message. +_CONTENT_PART_CONVERTERS: Mapping[str, Callable[[Any], MessagePart | None]] = { + "image_url": _image_url_part, + "input_audio": _input_audio_part, + "file": _file_part, +} + + +def _convert_content_part(item: Any) -> MessagePart | None: + """Map one OpenAI content part to a semconv message part. + + ``text`` parts map to ``TextPart``; ``image_url``, ``input_audio`` and + ``file`` parts map to the standard media parts (``UriPart``, + ``BlobPart``, ``FilePart``). Any other typed part is preserved as a + ``GenericPart`` so opted-in content capture does not silently drop it. + Returns ``None`` for items without a type discriminator and for media + parts whose payload cannot be decoded. """ - parts: list[Any] = [] - for item in content: - if isinstance(item, str): - parts.append(TextPart(content=item)) - continue - item_type = get_property_value(item, "type") - if item_type == "text": - parts.append( - TextPart(content=str(get_property_value(item, "text") or "")) - ) - else: - parts.append( - GenericPart(type=str(item_type), value=_as_plain_value(item)) - ) - return parts + if isinstance(item, str): + return TextPart(content=item) + item_type = get_property_value(item, "type") + if not isinstance(item_type, str): + return None + if item_type == "text": + text = get_property_value(item, "text") + return TextPart(content=str(text) if text is not None else "") + converter = _CONTENT_PART_CONVERTERS.get(item_type) + if converter is not None: + return converter(item) + return GenericPart(type=item_type, value=_as_plain_value(item)) -def _content_to_parts(content: Any) -> list[Any]: +def _content_to_parts(content: Any) -> list[MessagePart]: """Map message ``content`` to message parts. - Strings map to a single ``TextPart``. Mappings keep the previous - ``_is_text_part`` behavior (string-keyed mappings are captured as - ``str(mapping)``) and are never treated as content-part arrays. Other - iterables are materialized exactly once — ``content`` may be a - single-pass iterable, and type-checking then re-iterating would silently - drop the items already consumed — then map to a ``TextPart`` when - string-only, or through ``_extract_content_parts`` otherwise. + A string is a single text part and a bare mapping is a single content + part. Any other iterable is an OpenAI content-part array; it is consumed + exactly once, so single-pass iterables are never partially dropped. """ - if isinstance(content, str): - return [TextPart(content=content)] if content is None: return [] - if isinstance(content, Mapping): - if all(isinstance(key, str) for key in content): - return [TextPart(content=str(content))] + if isinstance(content, (str, Mapping)): + content = [content] + if not isinstance(content, Iterable): return [] - if isinstance(content, Iterable): - items = list(content) - if all(isinstance(item, str) for item in items): - return [TextPart(content=str(items))] - return _extract_content_parts(items) - return [] + parts: list[MessagePart] = [] + for item in content: + part = _convert_content_part(item) + if part is not None: + parts.append(part) + return parts def _prepare_input_messages(messages) -> list[InputMessage]: diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml new file mode 100644 index 000000000..3d31cf031 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml @@ -0,0 +1,166 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe each image in one short sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png" + } + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + } + } + ] + } + ], + "model": "gpt-4o-mini", + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '414' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.1.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.1.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-EJTMD7Nxo79aeDCToRNNzHN5o3CLd", + "object": "chat.completion", + "created": 1788309789, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "1. The first image depicts a stylized security camera design in vibrant colors.\n2. The second image features a solid blue square with a uniform color.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 17015, + "completion_tokens": 31, + "total_tokens": 17046, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_f5dd3ac0b7" + } + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a3487a948c88ea01-ICN + connection: + - keep-alive + content-length: + - '964' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:43:10 GMT + openai-organization: test_openai_org_id + openai-processing-ms: + - '1318' + openai-project: test_openai_project_id + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: test_set_cookie + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998457' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_195d81bc125247db850711811edf3685 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py new file mode 100644 index 000000000..e35a220fa --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py @@ -0,0 +1,129 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: openai-v2 chat completion with multimodal input. + +Sends a content-part array carrying text, an external image URL and an inline +base64 image, and asserts each lands on the input message as the matching +semconv part (``text`` / ``uri`` / ``blob``). +""" + +from __future__ import annotations + +import json +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +IMAGE_URL = "https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png" +# 1x1 transparent PNG, sent inline as a base64 data URL. +IMAGE_DATA_URL = ( + "data:image/png;base64," + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA" + "60e6kgAAAABJRU5ErkJggg==" +) + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("multimodal_conformance.yaml"): + OpenAI().chat.completions.create( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe each image in one short sentence.", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_URL}, + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_DATA_URL}, + }, + ], + } + ], + model="gpt-4o-mini", + stream=False, + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + + # Weaver validates the shape of each part; assert the media parts + # actually round-tripped onto the input message with their modality. + chat_spans = [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and _attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + assert chat_spans, "no chat span emitted" + + input_parts = { + (part_type, modality) + for span in chat_spans + for part_type, modality in _part_fields( + _attr(span, "gen_ai.input.messages") + ) + } + assert ("text", None) in input_parts, ( + f"expected a text part on the input message, saw {input_parts}" + ) + assert ("uri", "image") in input_parts, ( + f"expected an image uri part on the input message, saw {input_parts}" + ) + assert ("blob", "image") in input_parts, ( + f"expected an image blob part on the input message, saw {input_parts}" + ) + + +def _attr(span: dict[str, Any], name: str) -> Any: + for attr in span["attributes"]: + if attr["name"] == name: + return attr["value"] + return None + + +def _part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + # gen_ai.input.messages is a JSON string of + # [{"role": ..., "parts": [{"type": ..., ...}]}]; keep modality so + # image/audio/document are distinguishable on blob/uri/file parts. + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py index 14c665488..d06ecb8e4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py @@ -26,6 +26,7 @@ from .conformance.embedding import EmbeddingScenario from .conformance.inference import InferenceScenario from .conformance.inference_streaming import InferenceStreamingScenario +from .conformance.multimodal import MultimodalScenario from .conformance.responses_conversation import ResponsesConversationScenario from .conformance.responses_fetch import ResponsesFetchScenario from .conformance.responses_stream import ResponsesStreamScenario @@ -40,6 +41,7 @@ InferenceStreamingScenario(), EmbeddingScenario(), ToolCallingScenario(), + MultimodalScenario(), ResponsesConversationScenario(), ResponsesFetchScenario(), ResponsesStreamScenario(), diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py index aa0008a39..c4a0fb013 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -3,68 +3,123 @@ """Unit tests for input message preparation, including multimodal content.""" +import base64 from dataclasses import asdict from opentelemetry.instrumentation.genai.openai.utils import ( _prepare_input_messages, ) -from opentelemetry.util.genai.types import GenericPart, TextPart +from opentelemetry.util.genai.types import ( + BlobPart, + FilePart, + GenericPart, + TextPart, + UriPart, +) from opentelemetry.util.genai.utils import gen_ai_json_dumps +IMAGE_URL = "https://example.com/cat.png" +# 1x1 transparent PNG. +PNG_BASE64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA" + "60e6kgAAAABJRU5ErkJggg==" +) +PNG_DATA_URL = f"data:image/png;base64,{PNG_BASE64}" + + +def _user_parts(content): + messages = [{"role": "user", "content": content}] + return _prepare_input_messages(messages)[0].parts + def test_string_content_maps_to_text_part(): - messages = [{"role": "user", "content": "hello"}] + assert _user_parts("hello") == [TextPart(content="hello")] + + +def test_image_url_maps_to_uri_part(): + parts = _user_parts( + [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": IMAGE_URL, "detail": "low"}, + }, + ] + ) - result = _prepare_input_messages(messages) + assert parts == [ + TextPart(content="What is in this image?"), + UriPart(mime_type=None, modality="image", uri=IMAGE_URL), + ] - assert result[0].parts == [TextPart(content="hello")] +def test_image_data_url_maps_to_blob_part(): + parts = _user_parts( + [{"type": "image_url", "image_url": {"url": PNG_DATA_URL}}] + ) -def test_multimodal_content_keeps_text_part(): - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this image?"}, - { - "type": "image_url", - "image_url": {"url": "https://example.com/cat.png"}, - }, - ], - } + assert parts == [ + BlobPart( + mime_type="image/png", + modality="image", + content=base64.b64decode(PNG_BASE64), + ) ] - result = _prepare_input_messages(messages) - parts = result[0].parts - assert parts[0] == TextPart(content="What is in this image?") - assert isinstance(parts[1], GenericPart) - assert parts[1].type == "image_url" - assert parts[1].value == { - "type": "image_url", - "image_url": {"url": "https://example.com/cat.png"}, - } +def test_input_audio_maps_to_audio_blob_part(): + parts = _user_parts( + [ + { + "type": "input_audio", + "input_audio": {"data": "UklGRg==", "format": "wav"}, + } + ] + ) + + assert parts == [ + BlobPart(mime_type="audio/wav", modality="audio", content=b"RIFF") + ] -def test_input_audio_content_preserved_as_generic_part(): - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Transcribe this"}, - { - "type": "input_audio", - "input_audio": {"data": "UklGRg==", "format": "wav"}, +def test_file_id_maps_to_file_part(): + parts = _user_parts([{"type": "file", "file": {"file_id": "file-abc"}}]) + + assert parts == [ + FilePart(mime_type=None, modality="document", file_id="file-abc") + ] + + +def test_file_data_maps_to_document_blob_part(): + parts = _user_parts( + [ + { + "type": "file", + "file": { + "filename": "doc.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", }, - ], - } + } + ] + ) + + assert parts == [ + BlobPart( + mime_type="application/pdf", modality="document", content=b"%PDF-" + ) ] - result = _prepare_input_messages(messages) - parts = result[0].parts - assert [type(part) for part in parts] == [TextPart, GenericPart] - assert parts[1].type == "input_audio" +def test_unknown_part_type_is_preserved_as_generic_part(): + item = {"type": "custom_widget", "custom_widget": {"id": 7}} + + assert _user_parts([item]) == [ + GenericPart(type="custom_widget", value=item) + ] + + +def test_untyped_part_is_dropped(): + assert _user_parts([{"unexpected": "shape"}]) == [] def test_multimodal_assistant_history_content(): @@ -88,27 +143,28 @@ def test_none_content_produces_no_parts(): assert result[0].parts == [] -def test_single_pass_iterable_content_is_not_partially_dropped(): - def content_parts(): - yield {"type": "text", "text": "hello"} - yield {"type": "image_url", "image_url": {"url": "https://e/x.png"}} - - messages = [{"role": "user", "content": content_parts()}] +def test_string_list_content_maps_each_item_to_text_part(): + assert _user_parts(["a", "b"]) == [ + TextPart(content="a"), + TextPart(content="b"), + ] - result = _prepare_input_messages(messages) - parts = result[0].parts - assert [type(part) for part in parts] == [TextPart, GenericPart] - assert parts[0] == TextPart(content="hello") +def test_mapping_content_is_treated_as_single_part(): + parts = _user_parts({"type": "text", "text": "hello"}) + assert parts == [TextPart(content="hello")] -def test_mapping_content_is_not_treated_as_content_part_array(): - content = {"unexpected": "shape"} - messages = [{"role": "user", "content": content}] - result = _prepare_input_messages(messages) +def test_single_pass_iterable_content_is_consumed_once(): + def content_parts(): + yield {"type": "text", "text": "hello"} + yield {"type": "image_url", "image_url": {"url": IMAGE_URL}} - assert result[0].parts == [TextPart(content=str(content))] + assert _user_parts(content_parts()) == [ + TextPart(content="hello"), + UriPart(mime_type=None, modality="image", uri=IMAGE_URL), + ] def test_multimodal_message_is_json_serializable(): @@ -117,7 +173,8 @@ def test_multimodal_message_is_json_serializable(): "role": "user", "content": [ {"type": "text", "text": "describe"}, - {"type": "image_url", "image_url": {"url": "https://e/x.png"}}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image_url", "image_url": {"url": PNG_DATA_URL}}, ], } ] @@ -125,6 +182,6 @@ def test_multimodal_message_is_json_serializable(): result = _prepare_input_messages(messages) serialized = gen_ai_json_dumps([asdict(message) for message in result]) - assert '"type": "image_url"'.replace(" ", "") in serialized.replace( - " ", "" - ) + assert IMAGE_URL in serialized + assert '"type": "uri"' in serialized or '"type":"uri"' in serialized + assert '"type": "blob"' in serialized or '"type":"blob"' in serialized From 7d339a8baf23aecf5126041d1fa65002d09f6d7a Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Wed, 2 Sep 2026 10:12:11 +0900 Subject: [PATCH 5/7] fix(genai-openai): never consume or break the caller's request while capturing content Self-review follow-ups on #522: - Only walk `Sequence` content as a content-part array. Iterating any other iterable (e.g. a generator, which the SDK accepts and materializes itself) drained the caller's input before the request was sent, so the SDK ended up sending empty content. Such content is now left untouched and not captured. - Content conversion never raises: each part is converted under a broad guard and skipped on failure, and `GenericPart.value` is made JSON-safe (pydantic `model_dump(mode="json")`, `json` round-trip with `default=str`) so span export cannot fail on `datetime`/`set`/`Enum` payloads. - `file_data` is base64 per the SDK: decode plain base64 into a document `BlobPart` (mime type from `filename`), keep data-URL support, and use `filename` for `FilePart.mime_type` too. - `input_audio.data` given as a data URL is decoded instead of dropped; unmapped audio formats get `mime_type=None` instead of a synthesized type. - `get_property_value` accepts any `Mapping`, so non-dict mappings are no longer accepted as content and then silently dropped. - Non-string `text` values are dropped rather than stringified. - Replace the dispatch table with an if-chain matching the sibling instrumentations and trim docstrings. Claude-Session: https://claude.ai/code/session_01QsXFvtQoFP6DwdBgyEA3xB --- .../instrumentation/genai/openai/utils.py | 110 ++++++++----- .../tests/test_prepare_input_messages_unit.py | 155 +++++++++++++++++- 2 files changed, 211 insertions(+), 54 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index 14d6a117b..7ffb7b1cc 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -4,7 +4,9 @@ from __future__ import annotations import json -from collections.abc import Callable, Iterable, Mapping +import logging +import mimetypes +from collections.abc import Iterable, Mapping, Sequence from typing import Any from urllib.parse import urlparse @@ -36,6 +38,8 @@ ) from opentelemetry.util.genai.utils import decode_base64, image_from_url +_logger = logging.getLogger(__name__) + _OpenAIOmit = getattr(openai, "Omit", None) SUPPORTED_RAPI_RESPONSE_HEADERS = ("x-ms-served-model",) @@ -59,7 +63,7 @@ def get_served_model(headers: Mapping[str, str] | None) -> str | None: def get_property_value(obj, property_name): - if isinstance(obj, dict): + if isinstance(obj, Mapping): return obj.get(property_name, None) return getattr(obj, property_name, None) @@ -203,13 +207,26 @@ def _is_text_part(content: Any) -> bool: def _as_plain_value(item: Any) -> Any: - """Return a JSON-serializable representation of a content part.""" - if isinstance(item, Mapping): - return dict(item) - model_dump = getattr(item, "model_dump", None) - if callable(model_dump): - return model_dump() - return str(item) + """Return a JSON-safe representation of a content part. + + Never raises: pydantic models are dumped in JSON mode, the result is + round-tripped through ``json`` so only JSON-native values remain, and any + failure falls back to ``str(item)``. + """ + try: + model_dump = getattr(item, "model_dump", None) + if callable(model_dump): + try: + value = model_dump(mode="json") + except Exception: # pylint: disable=broad-exception-caught + value = model_dump() # pydantic v1 has no ``mode`` + elif isinstance(item, Mapping): + value = dict(item) + else: + value = item + return json.loads(json.dumps(value, default=str)) + except Exception: # pylint: disable=broad-exception-caught + return str(item) def _image_url_part(item: Any) -> MessagePart | None: @@ -229,51 +246,47 @@ def _input_audio_part(item: Any) -> MessagePart | None: """Map an ``input_audio`` content part to an audio ``BlobPart``.""" input_audio = get_property_value(item, "input_audio") data = get_property_value(input_audio, "data") - if not isinstance(data, str): + if not isinstance(data, str) or not data: return None + if data.startswith("data:"): + return image_from_url(data, modality="audio") content = decode_base64(data) if content is None: return None audio_format = get_property_value(input_audio, "format") mime_type = None - if isinstance(audio_format, str) and audio_format: - audio_format = audio_format.lower() - mime_type = _AUDIO_MIME_TYPES.get( - audio_format, f"audio/{audio_format}" - ) + if isinstance(audio_format, str): + mime_type = _AUDIO_MIME_TYPES.get(audio_format.lower()) return BlobPart(mime_type=mime_type, modality="audio", content=content) def _file_part(item: Any) -> MessagePart | None: """Map a ``file`` content part to a ``FilePart`` (``file_id``) or a - ``BlobPart`` (inline ``file_data`` data URL).""" + document ``BlobPart`` (inline ``file_data``, base64 or data URL).""" file_ref = get_property_value(item, "file") + filename = get_property_value(file_ref, "filename") + mime_type = None + if isinstance(filename, str) and filename: + mime_type = mimetypes.guess_type(filename)[0] file_id = get_property_value(file_ref, "file_id") if isinstance(file_id, str) and file_id: - return FilePart(mime_type=None, modality="document", file_id=file_id) + return FilePart( + mime_type=mime_type, modality="document", file_id=file_id + ) file_data = get_property_value(file_ref, "file_data") - if isinstance(file_data, str) and file_data: + if not isinstance(file_data, str) or not file_data: + return None + if file_data.startswith("data:"): return image_from_url(file_data, modality="document") - return None - - -_CONTENT_PART_CONVERTERS: Mapping[str, Callable[[Any], MessagePart | None]] = { - "image_url": _image_url_part, - "input_audio": _input_audio_part, - "file": _file_part, -} + content = decode_base64(file_data) + if content is None: + return None + return BlobPart(mime_type=mime_type, modality="document", content=content) def _convert_content_part(item: Any) -> MessagePart | None: - """Map one OpenAI content part to a semconv message part. - - ``text`` parts map to ``TextPart``; ``image_url``, ``input_audio`` and - ``file`` parts map to the standard media parts (``UriPart``, - ``BlobPart``, ``FilePart``). Any other typed part is preserved as a - ``GenericPart`` so opted-in content capture does not silently drop it. - Returns ``None`` for items without a type discriminator and for media - parts whose payload cannot be decoded. - """ + """Map one OpenAI content part to a semconv message part; typed parts + with no semconv mapping become ``GenericPart`` rather than being dropped.""" if isinstance(item, str): return TextPart(content=item) item_type = get_property_value(item, "type") @@ -281,10 +294,13 @@ def _convert_content_part(item: Any) -> MessagePart | None: return None if item_type == "text": text = get_property_value(item, "text") - return TextPart(content=str(text) if text is not None else "") - converter = _CONTENT_PART_CONVERTERS.get(item_type) - if converter is not None: - return converter(item) + return TextPart(content=text) if isinstance(text, str) else None + if item_type == "image_url": + return _image_url_part(item) + if item_type == "input_audio": + return _input_audio_part(item) + if item_type == "file": + return _file_part(item) return GenericPart(type=item_type, value=_as_plain_value(item)) @@ -292,18 +308,22 @@ def _content_to_parts(content: Any) -> list[MessagePart]: """Map message ``content`` to message parts. A string is a single text part and a bare mapping is a single content - part. Any other iterable is an OpenAI content-part array; it is consumed - exactly once, so single-pass iterables are never partially dropped. + part. Only sequences are walked as content-part arrays: iterating any + other iterable (e.g. a generator) would consume the caller's input before + the SDK sends it, so those are left untouched and not captured. + Conversion never raises; a part that fails to convert is skipped. """ - if content is None: - return [] if isinstance(content, (str, Mapping)): content = [content] - if not isinstance(content, Iterable): + if not isinstance(content, Sequence): return [] parts: list[MessagePart] = [] for item in content: - part = _convert_content_part(item) + try: + part = _convert_content_part(item) + except Exception: # pylint: disable=broad-exception-caught + _logger.debug("Failed to convert content part", exc_info=True) + continue if part is not None: parts.append(part) return parts diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py index c4a0fb013..79adbae19 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -5,6 +5,8 @@ import base64 from dataclasses import asdict +from datetime import datetime, timezone +from types import MappingProxyType from opentelemetry.instrumentation.genai.openai.utils import ( _prepare_input_messages, @@ -25,6 +27,7 @@ "60e6kgAAAABJRU5ErkJggg==" ) PNG_DATA_URL = f"data:image/png;base64,{PNG_BASE64}" +PDF_BASE64 = "JVBERi0=" # b"%PDF-" def _user_parts(content): @@ -82,22 +85,66 @@ def test_input_audio_maps_to_audio_blob_part(): ] +def test_input_audio_data_url_maps_to_audio_blob_part(): + parts = _user_parts( + [ + { + "type": "input_audio", + "input_audio": { + "data": "data:audio/wav;base64,UklGRg==", + "format": "wav", + }, + } + ] + ) + + assert parts == [ + BlobPart(mime_type="audio/wav", modality="audio", content=b"RIFF") + ] + + +def test_unknown_audio_format_has_no_mime_type(): + parts = _user_parts( + [ + { + "type": "input_audio", + "input_audio": {"data": "UklGRg==", "format": "pcm16"}, + } + ] + ) + + assert parts == [ + BlobPart(mime_type=None, modality="audio", content=b"RIFF") + ] + + def test_file_id_maps_to_file_part(): - parts = _user_parts([{"type": "file", "file": {"file_id": "file-abc"}}]) + parts = _user_parts( + [ + { + "type": "file", + "file": {"file_id": "file-abc", "filename": "a.pdf"}, + } + ] + ) assert parts == [ - FilePart(mime_type=None, modality="document", file_id="file-abc") + FilePart( + mime_type="application/pdf", + modality="document", + file_id="file-abc", + ) ] -def test_file_data_maps_to_document_blob_part(): +def test_file_data_url_maps_to_document_blob_part(): parts = _user_parts( [ { "type": "file", "file": { "filename": "doc.pdf", - "file_data": "data:application/pdf;base64,JVBERi0=", + "file_data": f"data:application/pdf;base64,{PDF_BASE64}", }, } ] @@ -110,6 +157,31 @@ def test_file_data_maps_to_document_blob_part(): ] +def test_plain_base64_file_data_maps_to_document_blob_part(): + parts = _user_parts( + [ + { + "type": "file", + "file": {"filename": "doc.pdf", "file_data": PDF_BASE64}, + } + ] + ) + + assert parts == [ + BlobPart( + mime_type="application/pdf", modality="document", content=b"%PDF-" + ) + ] + + +def test_undecodable_file_data_is_dropped(): + parts = _user_parts( + [{"type": "file", "file": {"file_data": "not base64!!"}}] + ) + + assert parts == [] + + def test_unknown_part_type_is_preserved_as_generic_part(): item = {"type": "custom_widget", "custom_widget": {"id": 7}} @@ -118,10 +190,60 @@ def test_unknown_part_type_is_preserved_as_generic_part(): ] +def test_generic_part_value_is_json_safe(): + item = { + "type": "custom", + "when": datetime(2026, 1, 1, tzinfo=timezone.utc), + "tags": {"a"}, + "nested": {"n": (1, 2)}, + } + + parts = _user_parts([item]) + + assert parts == [ + GenericPart( + type="custom", + value={ + "type": "custom", + "when": "2026-01-01 00:00:00+00:00", + "tags": "{'a'}", + "nested": {"n": [1, 2]}, + }, + ) + ] + gen_ai_json_dumps(asdict(parts[0])) + + +def test_generic_part_value_does_not_alias_input(): + item = {"type": "custom", "nested": {"n": 1}} + + parts = _user_parts([item]) + item["nested"]["n"] = 2 + + assert parts[0].value == {"type": "custom", "nested": {"n": 1}} + + def test_untyped_part_is_dropped(): assert _user_parts([{"unexpected": "shape"}]) == [] +def test_non_string_text_part_is_dropped(): + assert _user_parts([{"type": "text", "text": {"a": 1}}]) == [] + + +def test_raising_part_is_skipped_without_breaking_the_message(): + class ExplodingPart: + @property + def type(self): + raise RuntimeError("boom") + + parts = _user_parts( + [ExplodingPart(), {"type": "text", "text": "still captured"}] + ) + + assert parts == [TextPart(content="still captured")] + + def test_multimodal_assistant_history_content(): messages = [ { @@ -150,21 +272,36 @@ def test_string_list_content_maps_each_item_to_text_part(): ] +def test_tuple_content_is_treated_as_content_part_array(): + assert _user_parts(({"type": "text", "text": "hello"},)) == [ + TextPart(content="hello") + ] + + def test_mapping_content_is_treated_as_single_part(): parts = _user_parts({"type": "text", "text": "hello"}) assert parts == [TextPart(content="hello")] -def test_single_pass_iterable_content_is_consumed_once(): +def test_non_dict_mapping_content_is_treated_as_single_part(): + content = MappingProxyType({"type": "text", "text": "hello"}) + + assert _user_parts(content) == [TextPart(content="hello")] + + +def test_generator_content_is_left_unconsumed_and_not_captured(): + # The OpenAI SDK accepts any iterable of content parts and materializes + # it itself; capturing it here would drain the caller's input before the + # request is sent. def content_parts(): yield {"type": "text", "text": "hello"} yield {"type": "image_url", "image_url": {"url": IMAGE_URL}} - assert _user_parts(content_parts()) == [ - TextPart(content="hello"), - UriPart(mime_type=None, modality="image", uri=IMAGE_URL), - ] + content = content_parts() + + assert _user_parts(content) == [] + assert len(list(content)) == 2 def test_multimodal_message_is_json_serializable(): From 4f9f343ae5eb39ac9c601eb3941f041676abe9b0 Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Thu, 3 Sep 2026 09:46:11 +0900 Subject: [PATCH 6/7] fix(genai-openai): do not capture inline audio payloads Per review: a single `input_audio` clip can be megabytes and would be base64-inlined into `gen_ai.input.messages`; util-genai has no inline size guard, so such a span can fail the whole OTLP export batch. `input_audio` parts are now explicitly skipped (rather than falling through to a `GenericPart` that would embed the base64 in `value`), while the text and image parts around them are still captured. Claude-Session: https://claude.ai/code/session_01QsXFvtQoFP6DwdBgyEA3xB --- .../.changelog/522.fixed | 7 +-- .../instrumentation/genai/openai/utils.py | 31 +++---------- .../tests/test_prepare_input_messages_unit.py | 44 +++---------------- 3 files changed, 15 insertions(+), 67 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed index 247770433..6409babdb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed @@ -1,4 +1,5 @@ Capture multimodal content-part arrays on input messages instead of dropping -them: ``image_url`` parts map to ``UriPart``/``BlobPart``, ``input_audio`` to -an audio ``BlobPart``, ``file`` to ``FilePart``/``BlobPart``, and any other -typed part is preserved as a ``GenericPart``. +them: ``image_url`` parts map to ``UriPart``/``BlobPart``, ``file`` to +``FilePart``/``BlobPart``, and any other typed part is preserved as a +``GenericPart``. Inline ``input_audio`` payloads are intentionally not +captured. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index 7ffb7b1cc..a3c1baf53 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -200,12 +200,6 @@ def _is_text_part(content: Any) -> bool: ) -_AUDIO_MIME_TYPES: Mapping[str, str] = { - "mp3": "audio/mpeg", - "wav": "audio/wav", -} - - def _as_plain_value(item: Any) -> Any: """Return a JSON-safe representation of a content part. @@ -242,24 +236,6 @@ def _image_url_part(item: Any) -> MessagePart | None: return image_from_url(url) -def _input_audio_part(item: Any) -> MessagePart | None: - """Map an ``input_audio`` content part to an audio ``BlobPart``.""" - input_audio = get_property_value(item, "input_audio") - data = get_property_value(input_audio, "data") - if not isinstance(data, str) or not data: - return None - if data.startswith("data:"): - return image_from_url(data, modality="audio") - content = decode_base64(data) - if content is None: - return None - audio_format = get_property_value(input_audio, "format") - mime_type = None - if isinstance(audio_format, str): - mime_type = _AUDIO_MIME_TYPES.get(audio_format.lower()) - return BlobPart(mime_type=mime_type, modality="audio", content=content) - - def _file_part(item: Any) -> MessagePart | None: """Map a ``file`` content part to a ``FilePart`` (``file_id``) or a document ``BlobPart`` (inline ``file_data``, base64 or data URL).""" @@ -286,7 +262,8 @@ def _file_part(item: Any) -> MessagePart | None: def _convert_content_part(item: Any) -> MessagePart | None: """Map one OpenAI content part to a semconv message part; typed parts - with no semconv mapping become ``GenericPart`` rather than being dropped.""" + with no semconv mapping become ``GenericPart`` rather than being dropped. + ``input_audio`` parts are the exception and are never captured.""" if isinstance(item, str): return TextPart(content=item) item_type = get_property_value(item, "type") @@ -298,7 +275,9 @@ def _convert_content_part(item: Any) -> MessagePart | None: if item_type == "image_url": return _image_url_part(item) if item_type == "input_audio": - return _input_audio_part(item) + # Inline audio is intentionally not captured: a single clip can be + # megabytes, and it would be base64-inlined into the span attribute. + return None if item_type == "file": return _file_part(item) return GenericPart(type=item_type, value=_as_plain_value(item)) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py index 79adbae19..3d8289610 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -70,52 +70,20 @@ def test_image_data_url_maps_to_blob_part(): ] -def test_input_audio_maps_to_audio_blob_part(): +def test_input_audio_part_is_not_captured(): + # Inline audio can be megabytes per part; it is deliberately left out of + # telemetry while the surrounding parts are still captured. parts = _user_parts( [ + {"type": "text", "text": "transcribe this"}, { "type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}, - } - ] - ) - - assert parts == [ - BlobPart(mime_type="audio/wav", modality="audio", content=b"RIFF") - ] - - -def test_input_audio_data_url_maps_to_audio_blob_part(): - parts = _user_parts( - [ - { - "type": "input_audio", - "input_audio": { - "data": "data:audio/wav;base64,UklGRg==", - "format": "wav", - }, - } - ] - ) - - assert parts == [ - BlobPart(mime_type="audio/wav", modality="audio", content=b"RIFF") - ] - - -def test_unknown_audio_format_has_no_mime_type(): - parts = _user_parts( - [ - { - "type": "input_audio", - "input_audio": {"data": "UklGRg==", "format": "pcm16"}, - } + }, ] ) - assert parts == [ - BlobPart(mime_type=None, modality="audio", content=b"RIFF") - ] + assert parts == [TextPart(content="transcribe this")] def test_file_id_maps_to_file_part(): From 4b0748a4b0c2228d651da1cd667d2c8350504242 Mon Sep 17 00:00:00 2001 From: feelkyoun Date: Thu, 3 Sep 2026 09:56:00 +0900 Subject: [PATCH 7/7] fix(genai-openai): do not capture inline file_data payloads either Same reasoning as for `input_audio`: an inline document can be megabytes and would be base64-inlined into `gen_ai.input.messages`. `file` parts are now captured only when they carry a `file_id` (as a `FilePart`, with the mime type guessed from `filename`); inline `file_data` is skipped while the surrounding parts are still captured. Claude-Session: https://claude.ai/code/session_01QsXFvtQoFP6DwdBgyEA3xB --- .../.changelog/522.fixed | 8 ++-- .../instrumentation/genai/openai/utils.py | 31 ++++++-------- .../tests/test_prepare_input_messages_unit.py | 41 +++---------------- 3 files changed, 22 insertions(+), 58 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed index 6409babdb..def15dbfe 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed @@ -1,5 +1,5 @@ Capture multimodal content-part arrays on input messages instead of dropping -them: ``image_url`` parts map to ``UriPart``/``BlobPart``, ``file`` to -``FilePart``/``BlobPart``, and any other typed part is preserved as a -``GenericPart``. Inline ``input_audio`` payloads are intentionally not -captured. +them: ``image_url`` parts map to ``UriPart``/``BlobPart``, ``file`` parts with +a ``file_id`` to ``FilePart``, and any other typed part is preserved as a +``GenericPart``. Inline ``input_audio`` and ``file_data`` payloads are +intentionally not captured. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index a3c1baf53..0f6b89a00 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -24,7 +24,6 @@ InferenceInvocation, ) from opentelemetry.util.genai.types import ( - BlobPart, FilePart, FunctionToolDefinition, GenericPart, @@ -36,7 +35,7 @@ ToolCallResponsePart, ToolDefinition, ) -from opentelemetry.util.genai.utils import decode_base64, image_from_url +from opentelemetry.util.genai.utils import image_from_url _logger = logging.getLogger(__name__) @@ -237,33 +236,27 @@ def _image_url_part(item: Any) -> MessagePart | None: def _file_part(item: Any) -> MessagePart | None: - """Map a ``file`` content part to a ``FilePart`` (``file_id``) or a - document ``BlobPart`` (inline ``file_data``, base64 or data URL).""" + """Map a ``file`` content part with a ``file_id`` to a ``FilePart``. + + Inline ``file_data`` is intentionally not captured: a single document can + be megabytes, and it would be base64-inlined into the span attribute. + """ file_ref = get_property_value(item, "file") + file_id = get_property_value(file_ref, "file_id") + if not isinstance(file_id, str) or not file_id: + return None filename = get_property_value(file_ref, "filename") mime_type = None if isinstance(filename, str) and filename: mime_type = mimetypes.guess_type(filename)[0] - file_id = get_property_value(file_ref, "file_id") - if isinstance(file_id, str) and file_id: - return FilePart( - mime_type=mime_type, modality="document", file_id=file_id - ) - file_data = get_property_value(file_ref, "file_data") - if not isinstance(file_data, str) or not file_data: - return None - if file_data.startswith("data:"): - return image_from_url(file_data, modality="document") - content = decode_base64(file_data) - if content is None: - return None - return BlobPart(mime_type=mime_type, modality="document", content=content) + return FilePart(mime_type=mime_type, modality="document", file_id=file_id) def _convert_content_part(item: Any) -> MessagePart | None: """Map one OpenAI content part to a semconv message part; typed parts with no semconv mapping become ``GenericPart`` rather than being dropped. - ``input_audio`` parts are the exception and are never captured.""" + Inline media payloads (``input_audio``, ``file_data``) are the exception + and are never captured.""" if isinstance(item, str): return TextPart(content=item) item_type = get_property_value(item, "type") diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py index 3d8289610..1063dd826 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -105,49 +105,20 @@ def test_file_id_maps_to_file_part(): ] -def test_file_data_url_maps_to_document_blob_part(): - parts = _user_parts( - [ - { - "type": "file", - "file": { - "filename": "doc.pdf", - "file_data": f"data:application/pdf;base64,{PDF_BASE64}", - }, - } - ] - ) - - assert parts == [ - BlobPart( - mime_type="application/pdf", modality="document", content=b"%PDF-" - ) - ] - - -def test_plain_base64_file_data_maps_to_document_blob_part(): +def test_inline_file_data_is_not_captured(): + # Inline documents can be megabytes per part; they are deliberately left + # out of telemetry while the surrounding parts are still captured. parts = _user_parts( [ + {"type": "text", "text": "summarize this"}, { "type": "file", "file": {"filename": "doc.pdf", "file_data": PDF_BASE64}, - } + }, ] ) - assert parts == [ - BlobPart( - mime_type="application/pdf", modality="document", content=b"%PDF-" - ) - ] - - -def test_undecodable_file_data_is_dropped(): - parts = _user_parts( - [{"type": "file", "file": {"file_data": "not base64!!"}}] - ) - - assert parts == [] + assert parts == [TextPart(content="summarize this")] def test_unknown_part_type_is_preserved_as_generic_part():