diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed new file mode 100644 index 000000000..def15dbfe --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/522.fixed @@ -0,0 +1,5 @@ +Capture multimodal content-part arrays on input messages instead of dropping +them: ``image_url`` parts map to ``UriPart``/``BlobPart``, ``file`` parts with +a ``file_id`` to ``FilePart``, and any other typed part is preserved as a +``GenericPart``. Inline ``input_audio`` and ``file_data`` payloads are +intentionally not captured. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index 79997fd0c..0f6b89a00 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -4,7 +4,9 @@ from __future__ import annotations import json -from collections.abc import Iterable, Mapping +import logging +import mimetypes +from collections.abc import Iterable, Mapping, Sequence from typing import Any from urllib.parse import urlparse @@ -22,14 +24,20 @@ InferenceInvocation, ) from opentelemetry.util.genai.types import ( + FilePart, FunctionToolDefinition, + GenericPart, InputMessage, + MessagePart, OutputMessage, TextPart, ToolCallRequestPart, ToolCallResponsePart, ToolDefinition, ) +from opentelemetry.util.genai.utils import image_from_url + +_logger = logging.getLogger(__name__) _OpenAIOmit = getattr(openai, "Omit", None) @@ -54,7 +62,7 @@ def get_served_model(headers: Mapping[str, str] | None) -> str | None: def get_property_value(obj, property_name): - if isinstance(obj, dict): + if isinstance(obj, Mapping): return obj.get(property_name, None) return getattr(obj, property_name, None) @@ -191,6 +199,108 @@ def _is_text_part(content: Any) -> bool: ) +def _as_plain_value(item: Any) -> Any: + """Return a JSON-safe representation of a content part. + + Never raises: pydantic models are dumped in JSON mode, the result is + round-tripped through ``json`` so only JSON-native values remain, and any + failure falls back to ``str(item)``. + """ + try: + model_dump = getattr(item, "model_dump", None) + if callable(model_dump): + try: + value = model_dump(mode="json") + except Exception: # pylint: disable=broad-exception-caught + value = model_dump() # pydantic v1 has no ``mode`` + elif isinstance(item, Mapping): + value = dict(item) + else: + value = item + return json.loads(json.dumps(value, default=str)) + except Exception: # pylint: disable=broad-exception-caught + return str(item) + + +def _image_url_part(item: Any) -> MessagePart | None: + """Map an ``image_url`` content part to a ``UriPart`` or ``BlobPart``.""" + image_url = get_property_value(item, "image_url") + url = ( + image_url + if isinstance(image_url, str) + else get_property_value(image_url, "url") + ) + if not isinstance(url, str) or not url: + return None + return image_from_url(url) + + +def _file_part(item: Any) -> MessagePart | None: + """Map a ``file`` content part with a ``file_id`` to a ``FilePart``. + + Inline ``file_data`` is intentionally not captured: a single document can + be megabytes, and it would be base64-inlined into the span attribute. + """ + file_ref = get_property_value(item, "file") + file_id = get_property_value(file_ref, "file_id") + if not isinstance(file_id, str) or not file_id: + return None + filename = get_property_value(file_ref, "filename") + mime_type = None + if isinstance(filename, str) and filename: + mime_type = mimetypes.guess_type(filename)[0] + return FilePart(mime_type=mime_type, modality="document", file_id=file_id) + + +def _convert_content_part(item: Any) -> MessagePart | None: + """Map one OpenAI content part to a semconv message part; typed parts + with no semconv mapping become ``GenericPart`` rather than being dropped. + Inline media payloads (``input_audio``, ``file_data``) are the exception + and are never captured.""" + if isinstance(item, str): + return TextPart(content=item) + item_type = get_property_value(item, "type") + if not isinstance(item_type, str): + return None + if item_type == "text": + text = get_property_value(item, "text") + return TextPart(content=text) if isinstance(text, str) else None + if item_type == "image_url": + return _image_url_part(item) + if item_type == "input_audio": + # Inline audio is intentionally not captured: a single clip can be + # megabytes, and it would be base64-inlined into the span attribute. + return None + if item_type == "file": + return _file_part(item) + return GenericPart(type=item_type, value=_as_plain_value(item)) + + +def _content_to_parts(content: Any) -> list[MessagePart]: + """Map message ``content`` to message parts. + + A string is a single text part and a bare mapping is a single content + part. Only sequences are walked as content-part arrays: iterating any + other iterable (e.g. a generator) would consume the caller's input before + the SDK sends it, so those are left untouched and not captured. + Conversion never raises; a part that fails to convert is skipped. + """ + if isinstance(content, (str, Mapping)): + content = [content] + if not isinstance(content, Sequence): + return [] + parts: list[MessagePart] = [] + for item in content: + try: + part = _convert_content_part(item) + except Exception: # pylint: disable=broad-exception-caught + _logger.debug("Failed to convert content part", exc_info=True) + continue + if part is not None: + parts.append(part) + return parts + + def _prepare_input_messages(messages) -> list[InputMessage]: chat_messages = [] for message in messages: @@ -204,8 +314,7 @@ def _prepare_input_messages(messages) -> list[InputMessage]: tool_calls = get_property_value(message, "tool_calls") if tool_calls: chat_message.parts += extract_tool_calls_new(tool_calls) - if _is_text_part(content): - chat_message.parts.append(TextPart(content=str(content))) + chat_message.parts += _content_to_parts(content) elif role == "tool": tool_call_id = get_property_value(message, "tool_call_id") @@ -215,8 +324,7 @@ def _prepare_input_messages(messages) -> list[InputMessage]: else: # system, developer, user, fallback - if _is_text_part(content): - chat_message.parts.append(TextPart(content=str(content))) + chat_message.parts += _content_to_parts(content) return chat_messages diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml new file mode 100644 index 000000000..3d31cf031 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/multimodal_conformance.yaml @@ -0,0 +1,166 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe each image in one short sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png" + } + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + } + } + ] + } + ], + "model": "gpt-4o-mini", + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '414' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.1.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.1.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-EJTMD7Nxo79aeDCToRNNzHN5o3CLd", + "object": "chat.completion", + "created": 1788309789, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "1. The first image depicts a stylized security camera design in vibrant colors.\n2. The second image features a solid blue square with a uniform color.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 17015, + "completion_tokens": 31, + "total_tokens": 17046, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_f5dd3ac0b7" + } + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a3487a948c88ea01-ICN + connection: + - keep-alive + content-length: + - '964' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:43:10 GMT + openai-organization: test_openai_org_id + openai-processing-ms: + - '1318' + openai-project: test_openai_project_id + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: test_set_cookie + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998457' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_195d81bc125247db850711811edf3685 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py new file mode 100644 index 000000000..e35a220fa --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/multimodal.py @@ -0,0 +1,129 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: openai-v2 chat completion with multimodal input. + +Sends a content-part array carrying text, an external image URL and an inline +base64 image, and asserts each lands on the input message as the matching +semconv part (``text`` / ``uri`` / ``blob``). +""" + +from __future__ import annotations + +import json +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +IMAGE_URL = "https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png" +# 1x1 transparent PNG, sent inline as a base64 data URL. +IMAGE_DATA_URL = ( + "data:image/png;base64," + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA" + "60e6kgAAAABJRU5ErkJggg==" +) + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("multimodal_conformance.yaml"): + OpenAI().chat.completions.create( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe each image in one short sentence.", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_URL}, + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_DATA_URL}, + }, + ], + } + ], + model="gpt-4o-mini", + stream=False, + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + + # Weaver validates the shape of each part; assert the media parts + # actually round-tripped onto the input message with their modality. + chat_spans = [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and _attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + assert chat_spans, "no chat span emitted" + + input_parts = { + (part_type, modality) + for span in chat_spans + for part_type, modality in _part_fields( + _attr(span, "gen_ai.input.messages") + ) + } + assert ("text", None) in input_parts, ( + f"expected a text part on the input message, saw {input_parts}" + ) + assert ("uri", "image") in input_parts, ( + f"expected an image uri part on the input message, saw {input_parts}" + ) + assert ("blob", "image") in input_parts, ( + f"expected an image blob part on the input message, saw {input_parts}" + ) + + +def _attr(span: dict[str, Any], name: str) -> Any: + for attr in span["attributes"]: + if attr["name"] == name: + return attr["value"] + return None + + +def _part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + # gen_ai.input.messages is a JSON string of + # [{"role": ..., "parts": [{"type": ..., ...}]}]; keep modality so + # image/audio/document are distinguishable on blob/uri/file parts. + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py index 14c665488..d06ecb8e4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py @@ -26,6 +26,7 @@ from .conformance.embedding import EmbeddingScenario from .conformance.inference import InferenceScenario from .conformance.inference_streaming import InferenceStreamingScenario +from .conformance.multimodal import MultimodalScenario from .conformance.responses_conversation import ResponsesConversationScenario from .conformance.responses_fetch import ResponsesFetchScenario from .conformance.responses_stream import ResponsesStreamScenario @@ -40,6 +41,7 @@ InferenceStreamingScenario(), EmbeddingScenario(), ToolCallingScenario(), + MultimodalScenario(), ResponsesConversationScenario(), ResponsesFetchScenario(), ResponsesStreamScenario(), diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py new file mode 100644 index 000000000..1063dd826 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -0,0 +1,263 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for input message preparation, including multimodal content.""" + +import base64 +from dataclasses import asdict +from datetime import datetime, timezone +from types import MappingProxyType + +from opentelemetry.instrumentation.genai.openai.utils import ( + _prepare_input_messages, +) +from opentelemetry.util.genai.types import ( + BlobPart, + FilePart, + GenericPart, + TextPart, + UriPart, +) +from opentelemetry.util.genai.utils import gen_ai_json_dumps + +IMAGE_URL = "https://example.com/cat.png" +# 1x1 transparent PNG. +PNG_BASE64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA" + "60e6kgAAAABJRU5ErkJggg==" +) +PNG_DATA_URL = f"data:image/png;base64,{PNG_BASE64}" +PDF_BASE64 = "JVBERi0=" # b"%PDF-" + + +def _user_parts(content): + messages = [{"role": "user", "content": content}] + return _prepare_input_messages(messages)[0].parts + + +def test_string_content_maps_to_text_part(): + assert _user_parts("hello") == [TextPart(content="hello")] + + +def test_image_url_maps_to_uri_part(): + parts = _user_parts( + [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": IMAGE_URL, "detail": "low"}, + }, + ] + ) + + assert parts == [ + TextPart(content="What is in this image?"), + UriPart(mime_type=None, modality="image", uri=IMAGE_URL), + ] + + +def test_image_data_url_maps_to_blob_part(): + parts = _user_parts( + [{"type": "image_url", "image_url": {"url": PNG_DATA_URL}}] + ) + + assert parts == [ + BlobPart( + mime_type="image/png", + modality="image", + content=base64.b64decode(PNG_BASE64), + ) + ] + + +def test_input_audio_part_is_not_captured(): + # Inline audio can be megabytes per part; it is deliberately left out of + # telemetry while the surrounding parts are still captured. + parts = _user_parts( + [ + {"type": "text", "text": "transcribe this"}, + { + "type": "input_audio", + "input_audio": {"data": "UklGRg==", "format": "wav"}, + }, + ] + ) + + assert parts == [TextPart(content="transcribe this")] + + +def test_file_id_maps_to_file_part(): + parts = _user_parts( + [ + { + "type": "file", + "file": {"file_id": "file-abc", "filename": "a.pdf"}, + } + ] + ) + + assert parts == [ + FilePart( + mime_type="application/pdf", + modality="document", + file_id="file-abc", + ) + ] + + +def test_inline_file_data_is_not_captured(): + # Inline documents can be megabytes per part; they are deliberately left + # out of telemetry while the surrounding parts are still captured. + parts = _user_parts( + [ + {"type": "text", "text": "summarize this"}, + { + "type": "file", + "file": {"filename": "doc.pdf", "file_data": PDF_BASE64}, + }, + ] + ) + + assert parts == [TextPart(content="summarize this")] + + +def test_unknown_part_type_is_preserved_as_generic_part(): + item = {"type": "custom_widget", "custom_widget": {"id": 7}} + + assert _user_parts([item]) == [ + GenericPart(type="custom_widget", value=item) + ] + + +def test_generic_part_value_is_json_safe(): + item = { + "type": "custom", + "when": datetime(2026, 1, 1, tzinfo=timezone.utc), + "tags": {"a"}, + "nested": {"n": (1, 2)}, + } + + parts = _user_parts([item]) + + assert parts == [ + GenericPart( + type="custom", + value={ + "type": "custom", + "when": "2026-01-01 00:00:00+00:00", + "tags": "{'a'}", + "nested": {"n": [1, 2]}, + }, + ) + ] + gen_ai_json_dumps(asdict(parts[0])) + + +def test_generic_part_value_does_not_alias_input(): + item = {"type": "custom", "nested": {"n": 1}} + + parts = _user_parts([item]) + item["nested"]["n"] = 2 + + assert parts[0].value == {"type": "custom", "nested": {"n": 1}} + + +def test_untyped_part_is_dropped(): + assert _user_parts([{"unexpected": "shape"}]) == [] + + +def test_non_string_text_part_is_dropped(): + assert _user_parts([{"type": "text", "text": {"a": 1}}]) == [] + + +def test_raising_part_is_skipped_without_breaking_the_message(): + class ExplodingPart: + @property + def type(self): + raise RuntimeError("boom") + + parts = _user_parts( + [ExplodingPart(), {"type": "text", "text": "still captured"}] + ) + + assert parts == [TextPart(content="still captured")] + + +def test_multimodal_assistant_history_content(): + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "It is a cat."}], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [TextPart(content="It is a cat.")] + + +def test_none_content_produces_no_parts(): + messages = [{"role": "assistant", "content": None}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [] + + +def test_string_list_content_maps_each_item_to_text_part(): + assert _user_parts(["a", "b"]) == [ + TextPart(content="a"), + TextPart(content="b"), + ] + + +def test_tuple_content_is_treated_as_content_part_array(): + assert _user_parts(({"type": "text", "text": "hello"},)) == [ + TextPart(content="hello") + ] + + +def test_mapping_content_is_treated_as_single_part(): + parts = _user_parts({"type": "text", "text": "hello"}) + + assert parts == [TextPart(content="hello")] + + +def test_non_dict_mapping_content_is_treated_as_single_part(): + content = MappingProxyType({"type": "text", "text": "hello"}) + + assert _user_parts(content) == [TextPart(content="hello")] + + +def test_generator_content_is_left_unconsumed_and_not_captured(): + # The OpenAI SDK accepts any iterable of content parts and materializes + # it itself; capturing it here would drain the caller's input before the + # request is sent. + def content_parts(): + yield {"type": "text", "text": "hello"} + yield {"type": "image_url", "image_url": {"url": IMAGE_URL}} + + content = content_parts() + + assert _user_parts(content) == [] + assert len(list(content)) == 2 + + +def test_multimodal_message_is_json_serializable(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image_url", "image_url": {"url": PNG_DATA_URL}}, + ], + } + ] + + result = _prepare_input_messages(messages) + + serialized = gen_ai_json_dumps([asdict(message) for message in result]) + assert IMAGE_URL in serialized + assert '"type": "uri"' in serialized or '"type":"uri"' in serialized + assert '"type": "blob"' in serialized or '"type":"blob"' in serialized