From 27ddfb80c1e74b15c18cb9d50a54ef74f242e944 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 3 Sep 2026 09:53:39 -0700 Subject: [PATCH 01/13] Capture Anthropic image and document inputs as GenAI `BlobPart`, `UriPart`, `FilePart`, and `GenericPart` message parts --- .../instrumentation/genai/anthropic/utils.py | 158 +++++++-- ...t_anthropic_multimodal_image_llm_call.yaml | 138 ++++++++ .../tests/conformance/multimodal.py | 234 ++++++++++++++ .../tests/test_async_messages.py | 170 ++++++++++ .../tests/test_conformance.py | 2 + .../tests/test_messages_extractors.py | 303 ++++++++++++++++++ .../tests/test_sync_messages.py | 167 ++++++++++ 7 files changed, 1153 insertions(+), 19 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index c497fdcb4..2d73d4f79 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -5,10 +5,10 @@ from __future__ import annotations -import base64 import json -from collections.abc import Mapping -from dataclasses import dataclass +from collections.abc import Iterable, Mapping +from dataclasses import asdict, dataclass +from os import PathLike from typing import TYPE_CHECKING, cast from anthropic.types import ( @@ -24,6 +24,8 @@ from opentelemetry.util.genai.types import ( BlobPart, + FilePart, + GenericPart, MessagePart, ReasoningPart, ServerToolCallPart, @@ -31,6 +33,7 @@ TextPart, ToolCallRequestPart, ToolCallResponsePart, + UriPart, ) _SERVER_TOOL_RESULT_TYPES = { @@ -41,10 +44,9 @@ "text_editor_code_execution_tool_result": "text_editor_code_execution", "tool_search_tool_result": "tool_search", } +from opentelemetry.util.genai.utils import decode_base64, image_from_url if TYPE_CHECKING: - from collections.abc import Iterable - from anthropic.types import ( ContentBlock, ContentBlockParam, @@ -102,26 +104,32 @@ def normalize_finish_reason(stop_reason: str | None) -> str | None: return normalized or stop_reason -def _decode_base64(data: str) -> bytes | None: - try: - return base64.b64decode(data) - except Exception: # pylint: disable=broad-exception-caught - return None - - -def _extract_base64_blob(source: object, modality: str) -> BlobPart | None: +def _extract_base64_blob(source: object, modality: str) -> MessagePart | None: """Extract a BlobPart from a base64-encoded source dict.""" if not isinstance(source, dict): return None - # source is a TypedDict (e.g. Base64ImageSourceParam) narrowed to dict; - # pyright cannot infer value types from isinstance-narrowed dicts. - data: object = source.get("data") # type: ignore[reportUnknownMemberType] + source_dict = cast(dict[str, object], source) + data = source_dict.get("data") if not isinstance(data, str): + if isinstance(data, PathLike) or callable(getattr(data, "read", None)): + media_type = source_dict.get("media_type") + return GenericPart( + type=modality, + value={ + "source_type": "base64_file", + "mime_type": media_type + if isinstance(media_type, str) + else None, + "input_type": "path" + if isinstance(data, PathLike) + else "stream", + }, + ) return None - decoded = _decode_base64(data) + decoded = decode_base64(data) if decoded is None: return None - media_type: object = source.get("media_type") # type: ignore[reportUnknownMemberType] + media_type = source_dict.get("media_type") return BlobPart( mime_type=media_type if isinstance(media_type, str) else None, modality=modality, @@ -129,6 +137,106 @@ def _extract_base64_blob(source: object, modality: str) -> BlobPart | None: ) +def _extract_image_source(source: object) -> MessagePart | None: + """Convert an Anthropic image source into a GenAI message part.""" + if not isinstance(source, dict): + return None + source_dict = cast(dict[str, object], source) + source_type = source_dict.get("type") + if source_type == "base64": + return _extract_base64_blob(source_dict, "image") + if source_type == "url": + url = source_dict.get("url") + if isinstance(url, str) and url: + return image_from_url(url) + if source_type == "file": + return _extract_file_source(source_dict, "image") + return None + + +def _extract_file_source( + source: Mapping[str, object], modality: str +) -> FilePart | None: + file_id = source.get("file_id") + if not isinstance(file_id, str) or not file_id: + return None + return FilePart(mime_type=None, modality=modality, file_id=file_id) + + +def _extract_document_source(source: object) -> list[MessagePart]: + """Convert an Anthropic document source into GenAI message parts.""" + if not isinstance(source, dict): + return [] + source_dict = cast(dict[str, object], source) + source_type = source_dict.get("type") + if source_type == "base64": + part = _extract_base64_blob(source_dict, "document") + return [part] if part is not None else [] + if source_type == "url": + url = source_dict.get("url") + if isinstance(url, str) and url: + return [ + UriPart( + mime_type="application/pdf", + modality="document", + uri=url, + ) + ] + return [] + if source_type == "text": + data = source_dict.get("data") + if isinstance(data, str): + return [ + BlobPart( + mime_type="text/plain", + modality="document", + content=data.encode(), + ) + ] + return [] + if source_type == "content": + content = source_dict.get("content") + if isinstance(content, str): + return [TextPart(content=content)] + if isinstance(content, Iterable): + return convert_content_to_parts( + cast("Iterable[ContentBlock | ContentBlockParam]", content) + ) + if source_type == "file": + part = _extract_file_source(source_dict, "document") + return [part] if part is not None else [] + return [] + + +def _convert_document_block(block: Mapping[str, Any]) -> list[MessagePart]: + parts = _extract_document_source(block.get("source")) + metadata = { + key: block[key] + for key in ("title", "context", "citations") + if block.get(key) is not None + } + source = block.get("source") + source_mapping = ( + cast(Mapping[str, object], source) + if isinstance(source, Mapping) + else None + ) + is_nested = ( + source_mapping is not None and source_mapping.get("type") == "content" + ) + if metadata or (is_nested and parts): + return [ + GenericPart( + type="document", + value={ + "parts": [asdict(part) for part in parts], + **metadata, + }, + ) + ] + return parts + + def _convert_dict_block_to_part( block: Mapping[str, object], ) -> MessagePart | None: @@ -189,7 +297,14 @@ def _convert_dict_block_to_part( content=str(thinking) if thinking is not None else "" ) - if block_type in ("image", "audio", "video", "document", "file"): + if block_type == "image": + return _extract_image_source(block.get("source")) + + if block_type == "document": + parts = _convert_document_block(block) + return parts[0] if len(parts) == 1 else None + + if block_type in ("audio", "video", "file"): return _extract_base64_blob(block.get("source"), str(block_type)) return None @@ -236,6 +351,11 @@ def convert_content_to_parts( return [TextPart(content=content)] parts: list[MessagePart] = [] for item in content: + if isinstance(item, Mapping): + item_mapping = cast(Mapping[str, Any], item) + if item_mapping.get("type") == "document": + parts.extend(_convert_document_block(item_mapping)) + continue part = _convert_content_block_to_part(item) if part is not None: parts.append(part) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..e35af8d98 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml @@ -0,0 +1,138 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these images." + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD" + } + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png" + } + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD" + } + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf" + } + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text" + } + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": { + "enabled": true + }, + "source": { + "type": "content", + "content": [ + { + "type": "text", + "text": "Nested text" + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/nested.png" + } + } + ] + } + }, + { + "type": "image", + "source": { + "type": "file", + "file_id": "file-image" + } + }, + { + "type": "document", + "source": { + "type": "file", + "file_id": "file-document" + } + } + ] + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + anthropic-version: + - '2023-06-01' + content-type: + - application/json + host: + - api.anthropic.com + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_multimodal_conformance", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "The images were received." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 7 + } + } + headers: + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py new file mode 100644 index 000000000..4346ad3d6 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py @@ -0,0 +1,234 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: Anthropic multimodal chat input.""" + +from __future__ import annotations + +import json +import os +from typing import Any +from unittest import mock + +from anthropic import Anthropic + +from opentelemetry.instrumentation.genai.anthropic import AnthropicInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("ANTHROPIC_API_KEY") + else {"ANTHROPIC_API_KEY": "test_anthropic_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + AnthropicInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette( + "test_chat_anthropic_multimodal_image_llm_call.yaml" + ): + Anthropic().messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these images.", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + }, + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text", + }, + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + "source": { + "type": "content", + "content": [ + { + "type": "text", + "text": "Nested text", + }, + { + "type": "image", + "source": { + "type": "url", + "url": ( + "https://example.com/nested.png" + ), + }, + }, + ], + }, + }, + { + "type": "image", + "source": { + "type": "file", + "file_id": "file-image", + }, + }, + { + "type": "document", + "source": { + "type": "file", + "file_id": "file-document", + }, + }, + ], + } + ], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + chat_spans = [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and _attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + assert chat_spans, "no chat span emitted" + + input_messages = json.loads( + _attr(chat_spans[0], "gen_ai.input.messages") + ) + assert len(input_messages) == 1 + assert input_messages[0]["role"] == "user" + + parts = input_messages[0]["parts"] + assert len(parts) == 9 + assert parts[0] == { + "type": "text", + "content": "Describe these images.", + } + assert parts[1] == { + "type": "blob", + "mime_type": "image/png", + "modality": "image", + "content": "QUJD", + } + assert parts[2] == { + "type": "uri", + "mime_type": None, + "modality": "image", + "uri": "https://example.com/image.png", + } + assert parts[3] == { + "type": "blob", + "mime_type": "application/pdf", + "modality": "document", + "content": "QUJD", + } + assert parts[4] == { + "type": "uri", + "mime_type": "application/pdf", + "modality": "document", + "uri": "https://example.com/document.pdf", + } + assert parts[5] == { + "type": "blob", + "mime_type": "text/plain", + "modality": "document", + "content": "RG9jdW1lbnQgdGV4dA==", + } + assert parts[6] == { + "type": "document", + "value": { + "parts": [ + {"content": "Nested text", "type": "text"}, + { + "mime_type": None, + "modality": "image", + "uri": "https://example.com/nested.png", + "type": "uri", + }, + ], + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + }, + } + assert parts[7] == { + "type": "file", + "mime_type": None, + "modality": "image", + "file_id": "file-image", + } + assert parts[8] == { + "type": "file", + "mime_type": None, + "modality": "document", + "file_id": "file-document", + } + + +def _attr(span: dict[str, Any], name: str) -> Any: + for attr in span["attributes"]: + if attr["name"] == name: + return attr["value"] + return None diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py index 2aeb1d3da..73fc950b4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py @@ -107,6 +107,132 @@ def _assert_weather_tool_definitions(span): ] +def _multimodal_input_message(): + return { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these images."}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + }, + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text", + }, + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "Nested text"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/nested.png", + }, + }, + ], + }, + }, + ], + } + + +def _assert_multimodal_input(span): + messages = _load_span_messages(span, GenAIAttributes.GEN_AI_INPUT_MESSAGES) + assert len(messages) == 1 + assert messages[0]["role"] == "user" + + parts = messages[0]["parts"] + assert len(parts) == 7 + assert parts[0] == { + "type": "text", + "content": "Describe these images.", + } + assert parts[1] == { + "type": "blob", + "mime_type": "image/png", + "modality": "image", + "content": "QUJD", + } + assert parts[2] == { + "type": "uri", + "mime_type": None, + "modality": "image", + "uri": "https://example.com/image.png", + } + assert parts[3] == { + "type": "blob", + "mime_type": "application/pdf", + "modality": "document", + "content": "QUJD", + } + assert parts[4] == { + "type": "uri", + "mime_type": "application/pdf", + "modality": "document", + "uri": "https://example.com/document.pdf", + } + assert parts[5] == { + "type": "blob", + "mime_type": "text/plain", + "modality": "document", + "content": "RG9jdW1lbnQgdGV4dA==", + } + assert parts[6] == { + "type": "document", + "value": { + "parts": [ + {"content": "Nested text", "type": "text"}, + { + "mime_type": None, + "modality": "image", + "uri": "https://example.com/nested.png", + "type": "uri", + }, + ], + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + }, + } + + class _AsyncErrorInjectingStreamDelegate: def __init__(self, inner): self._inner = inner @@ -245,6 +371,25 @@ async def test_async_messages_create_captures_content( assert output_messages[0]["parts"][0]["type"] == "text" +@pytest.mark.asyncio +async def test_async_messages_create_captures_multimodal_content( + span_exporter, + async_anthropic_client, + instrument_with_content, + vcr, +): + with vcr.use_cassette("test_async_messages_create_captures_content.yaml"): + await async_anthropic_client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[_multimodal_input_message()], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + _assert_multimodal_input(spans[0]) + + @pytest.mark.asyncio @pytest.mark.vcr() async def test_async_messages_create_with_all_params( @@ -454,6 +599,31 @@ async def test_async_messages_create_streaming_captures_content( assert output_messages[0]["parts"] +@pytest.mark.asyncio +async def test_async_messages_create_streaming_captures_multimodal_content( + span_exporter, + async_anthropic_client, + instrument_with_content, + vcr, +): + with vcr.use_cassette( + "test_async_messages_create_streaming_captures_content.yaml" + ): + stream = await async_anthropic_client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[_multimodal_input_message()], + stream=True, + ) + async with stream: + async for _ in stream: + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + _assert_multimodal_input(spans[0]) + + @pytest.mark.asyncio @pytest.mark.vcr() async def test_async_messages_create_streaming_iteration( diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py index 04ea70d42..e598caf25 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py @@ -25,6 +25,7 @@ ) from .conformance.inference_streaming import InferenceStreamingScenario from .conformance.server_tool_calling import ServerToolCallingScenario +from .conformance.multimodal import MultimodalScenario from .conformance.tool_calling import ToolCallingScenario @@ -32,6 +33,7 @@ "scenario", [ InferenceScenario(), + MultimodalScenario(), InferenceStreamingScenario(), InferenceRawResponseScenario(), InferenceRawResponseStreamingScenario(), diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index f1f780465..0b6e88ae0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -3,6 +3,11 @@ """Tests for Anthropic message parameter extraction.""" +from io import BytesIO +from pathlib import Path + +import pytest + from types import SimpleNamespace from unittest.mock import MagicMock @@ -16,6 +21,18 @@ from opentelemetry.instrumentation.genai.anthropic.messages_extractors import ( extract_params, get_tool_definitions, + get_input_messages, +) +from opentelemetry.instrumentation.genai.anthropic.utils import ( + _convert_dict_block_to_part, + convert_content_to_parts, +) +from opentelemetry.util.genai.types import ( + BlobPart, + FilePart, + GenericPart, + TextPart, + UriPart, set_invocation_response_attributes, ) from opentelemetry.instrumentation.genai.anthropic.utils import ( @@ -70,6 +87,292 @@ def test_extract_params_ignores_non_mapping_extra_body(): assert params.top_k is None +def test_base64_image_source_converts_to_blob_part(): + part = _convert_dict_block_to_part( + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD", + }, + } + ) + + assert isinstance(part, BlobPart) + assert part.content == b"ABC" + assert part.mime_type == "image/png" + assert part.modality == "image" + + +def test_url_image_source_converts_to_uri_part(): + part = _convert_dict_block_to_part( + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + } + ) + + assert isinstance(part, UriPart) + assert part.uri == "https://example.com/image.png" + assert part.mime_type is None + assert part.modality == "image" + + +def test_file_image_source_converts_to_file_part(): + part = _convert_dict_block_to_part( + { + "type": "image", + "source": {"type": "file", "file_id": "file-image"}, + } + ) + + assert isinstance(part, FilePart) + assert part.file_id == "file-image" + assert part.mime_type is None + assert part.modality == "image" + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "image/png", "data": "%%%"}, + {"type": "base64", "media_type": "image/png"}, + {"type": "url"}, + {"type": "url", "url": ""}, + {"type": "file"}, + {"type": "file", "file_id": ""}, + {"type": "unknown", "data": "QUJD"}, + None, + ], +) +def test_invalid_image_source_is_ignored(source): + assert ( + _convert_dict_block_to_part({"type": "image", "source": source}) + is None + ) + + +def test_mixed_text_and_image_parts_preserve_order(): + parts = convert_content_to_parts( + [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + }, + ] + ) + + assert len(parts) == 2 + assert isinstance(parts[0], TextPart) + assert parts[0].content == "Describe this image." + assert isinstance(parts[1], UriPart) + assert parts[1].uri == "https://example.com/image.png" + + +def test_image_only_input_message_is_preserved(): + messages = get_input_messages( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD", + }, + } + ], + } + ] + ) + + assert len(messages) == 1 + assert messages[0].role == "user" + assert len(messages[0].parts) == 1 + assert isinstance(messages[0].parts[0], BlobPart) + + +def test_base64_document_source_converts_to_blob_part(): + part = _convert_dict_block_to_part( + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + } + ) + + assert isinstance(part, BlobPart) + assert part.content == b"ABC" + assert part.mime_type == "application/pdf" + assert part.modality == "document" + + +def test_url_document_source_converts_to_uri_part(): + part = _convert_dict_block_to_part( + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + }, + } + ) + + assert isinstance(part, UriPart) + assert part.uri == "https://example.com/document.pdf" + assert part.mime_type == "application/pdf" + assert part.modality == "document" + + +def test_file_document_source_converts_to_file_part(): + part = _convert_dict_block_to_part( + { + "type": "document", + "source": {"type": "file", "file_id": "file-document"}, + } + ) + + assert isinstance(part, FilePart) + assert part.file_id == "file-document" + assert part.mime_type is None + assert part.modality == "document" + + +def test_plain_text_document_source_converts_to_blob_part(): + part = _convert_dict_block_to_part( + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text", + }, + } + ) + + assert isinstance(part, BlobPart) + assert part.content == b"Document text" + assert part.mime_type == "text/plain" + assert part.modality == "document" + + +def test_nested_content_document_source_preserves_part_order(): + parts = convert_content_to_parts( + [ + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "Nested text"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + }, + ], + }, + } + ] + ) + + assert len(parts) == 1 + assert isinstance(parts[0], GenericPart) + assert parts[0].type == "document" + assert parts[0].value == { + "parts": [ + {"content": "Nested text", "type": "text"}, + { + "mime_type": None, + "modality": "image", + "uri": "https://example.com/image.png", + "type": "uri", + }, + ], + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + } + + +@pytest.mark.parametrize( + ("block_type", "media_type", "data", "input_type"), + [ + ("image", "image/png", Path("private/image.png"), "path"), + ("image", "image/png", BytesIO(b"image"), "stream"), + ( + "document", + "application/pdf", + Path("private/document.pdf"), + "path", + ), + ("document", "application/pdf", BytesIO(b"document"), "stream"), + ], +) +def test_file_backed_base64_source_is_preserved_without_reading( + block_type, media_type, data, input_type +): + initial_position = data.tell() if isinstance(data, BytesIO) else None + part = _convert_dict_block_to_part( + { + "type": block_type, + "source": { + "type": "base64", + "media_type": media_type, + "data": data, + }, + } + ) + + assert isinstance(part, GenericPart) + assert part.type == block_type + assert part.value == { + "source_type": "base64_file", + "mime_type": media_type, + "input_type": input_type, + } + if initial_position is not None: + assert data.tell() == initial_position + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "application/pdf", "data": "%%%"}, + {"type": "url"}, + {"type": "text"}, + {"type": "content", "content": None}, + {"type": "file"}, + {"type": "file", "file_id": ""}, + {"type": "unknown"}, + None, + ], +) +def test_invalid_document_source_is_ignored(source): + assert ( + convert_content_to_parts([{"type": "document", "source": source}]) + == [] + ) + + def test_set_invocation_response_attributes_records_cache_tokens(): invocation = MagicMock() message = SimpleNamespace( diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py index ca1b71b03..8d34d3694 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py @@ -236,6 +236,132 @@ def _assert_weather_tool_definitions(span): ] +def _multimodal_input_message(): + return { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these images."}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + }, + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text", + }, + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "Nested text"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/nested.png", + }, + }, + ], + }, + }, + ], + } + + +def _assert_multimodal_input(span): + messages = _load_span_messages(span, GenAIAttributes.GEN_AI_INPUT_MESSAGES) + assert len(messages) == 1 + assert messages[0]["role"] == "user" + + parts = messages[0]["parts"] + assert len(parts) == 7 + assert parts[0] == { + "type": "text", + "content": "Describe these images.", + } + assert parts[1] == { + "type": "blob", + "mime_type": "image/png", + "modality": "image", + "content": "QUJD", + } + assert parts[2] == { + "type": "uri", + "mime_type": None, + "modality": "image", + "uri": "https://example.com/image.png", + } + assert parts[3] == { + "type": "blob", + "mime_type": "application/pdf", + "modality": "document", + "content": "QUJD", + } + assert parts[4] == { + "type": "uri", + "mime_type": "application/pdf", + "modality": "document", + "uri": "https://example.com/document.pdf", + } + assert parts[5] == { + "type": "blob", + "mime_type": "text/plain", + "modality": "document", + "content": "RG9jdW1lbnQgdGV4dA==", + } + assert parts[6] == { + "type": "document", + "value": { + "parts": [ + {"content": "Nested text", "type": "text"}, + { + "mime_type": None, + "modality": "image", + "uri": "https://example.com/nested.png", + "type": "uri", + }, + ], + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + }, + } + + def _skip_if_cassette_missing_and_no_real_key(request): cassette_path = ( Path(__file__).parent / "cassettes" / f"{request.node.name}.yaml" @@ -499,6 +625,24 @@ def test_sync_messages_create_captures_content( assert output_messages[0]["parts"][0]["type"] == "text" +def test_sync_messages_create_captures_multimodal_content( + span_exporter, + anthropic_client, + instrument_with_content, + vcr, +): + with vcr.use_cassette("test_sync_messages_create_captures_content.yaml"): + anthropic_client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[_multimodal_input_message()], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + _assert_multimodal_input(spans[0]) + + @pytest.mark.vcr() def test_sync_messages_create_with_all_params( span_exporter, anthropic_client, instrument_no_content @@ -846,6 +990,29 @@ def test_sync_messages_stream_records_tool_definitions( _assert_weather_tool_definitions(spans[0]) +def test_sync_messages_create_streaming_captures_multimodal_content( + span_exporter, + anthropic_client, + instrument_with_content, + vcr, +): + with vcr.use_cassette( + "test_sync_messages_create_streaming_captures_content.yaml" + ): + with anthropic_client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[_multimodal_input_message()], + stream=True, + ) as stream: + for _ in stream: + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + _assert_multimodal_input(spans[0]) + + @pytest.mark.vcr() def test_sync_messages_stream( # pylint: disable=too-many-locals request, span_exporter, anthropic_client, instrument_no_content From 2600f861d9f0f082d7fd31217949262e82091584 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 3 Sep 2026 09:56:26 -0700 Subject: [PATCH 02/13] Add CHANGELOG --- .../.changelog/589.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/589.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/589.added b/instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/589.added new file mode 100644 index 000000000..91a08b9bb --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/589.added @@ -0,0 +1 @@ +Capture Anthropic image and document inputs as GenAI ``BlobPart``, ``UriPart``, ``FilePart``, and ``GenericPart`` message parts From 529099e0c995194ab9f42c828f9b720e1ce2a73f Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 3 Sep 2026 10:06:52 -0700 Subject: [PATCH 03/13] Add expected violations to multimodal scenario tests --- .../tests/conformance/multimodal.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py index 4346ad3d6..91c662106 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py @@ -17,7 +17,10 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.test.weaver_live_check import LiveCheckReport -from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) from opentelemetry.test_util_genai.instrumentor import instrument @@ -27,6 +30,12 @@ class MultimodalScenario(Scenario): "gen_ai.client.operation.duration", "gen_ai.client.token.usage", ) + expected_violations = ( + ExpectedViolation( + advice_id="missing_attribute", + message_substring="gen_ai.usage.cache_creation.input_tokens", + ), + ) def run( self, From d7f59af7114ba333039020bc1a6e9891939a8666 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Fri, 4 Sep 2026 13:40:52 -0700 Subject: [PATCH 04/13] Fix formatting --- .../instrumentation/genai/anthropic/utils.py | 32 ++++---- .../tests/test_messages_extractors.py | 78 +++++++++++-------- .../tests/test_sync_messages.py | 57 ++++++++++++++ 3 files changed, 117 insertions(+), 50 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index 2d73d4f79..f417e8b76 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -6,7 +6,7 @@ from __future__ import annotations import json -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from dataclasses import asdict, dataclass from os import PathLike from typing import TYPE_CHECKING, cast @@ -198,6 +198,8 @@ def _extract_document_source(source: object) -> list[MessagePart]: content = source_dict.get("content") if isinstance(content, str): return [TextPart(content=content)] + if isinstance(content, Iterator): + return [] if isinstance(content, Iterable): return convert_content_to_parts( cast("Iterable[ContentBlock | ContentBlockParam]", content) @@ -208,7 +210,7 @@ def _extract_document_source(source: object) -> list[MessagePart]: return [] -def _convert_document_block(block: Mapping[str, Any]) -> list[MessagePart]: +def _convert_document_block(block: Mapping[str, Any]) -> MessagePart | None: parts = _extract_document_source(block.get("source")) metadata = { key: block[key] @@ -225,16 +227,14 @@ def _convert_document_block(block: Mapping[str, Any]) -> list[MessagePart]: source_mapping is not None and source_mapping.get("type") == "content" ) if metadata or (is_nested and parts): - return [ - GenericPart( - type="document", - value={ - "parts": [asdict(part) for part in parts], - **metadata, - }, - ) - ] - return parts + return GenericPart( + type="document", + value={ + "parts": [asdict(part) for part in parts], + **metadata, + }, + ) + return parts[0] if parts else None def _convert_dict_block_to_part( @@ -301,8 +301,7 @@ def _convert_dict_block_to_part( return _extract_image_source(block.get("source")) if block_type == "document": - parts = _convert_document_block(block) - return parts[0] if len(parts) == 1 else None + return _convert_document_block(block) if block_type in ("audio", "video", "file"): return _extract_base64_blob(block.get("source"), str(block_type)) @@ -351,11 +350,6 @@ def convert_content_to_parts( return [TextPart(content=content)] parts: list[MessagePart] = [] for item in content: - if isinstance(item, Mapping): - item_mapping = cast(Mapping[str, Any], item) - if item_mapping.get("type") == "document": - parts.extend(_convert_document_block(item_mapping)) - continue part = _convert_content_block_to_part(item) if part is not None: parts.append(part) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index 0b6e88ae0..0e3171f57 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -203,17 +203,21 @@ def test_image_only_input_message_is_preserved(): def test_base64_document_source_converts_to_blob_part(): - part = _convert_dict_block_to_part( - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "QUJD", - }, - } + parts = convert_content_to_parts( + [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + } + ] ) + assert len(parts) == 1 + part = parts[0] assert isinstance(part, BlobPart) assert part.content == b"ABC" assert part.mime_type == "application/pdf" @@ -221,16 +225,20 @@ def test_base64_document_source_converts_to_blob_part(): def test_url_document_source_converts_to_uri_part(): - part = _convert_dict_block_to_part( - { - "type": "document", - "source": { - "type": "url", - "url": "https://example.com/document.pdf", - }, - } + parts = convert_content_to_parts( + [ + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + }, + } + ] ) + assert len(parts) == 1 + part = parts[0] assert isinstance(part, UriPart) assert part.uri == "https://example.com/document.pdf" assert part.mime_type == "application/pdf" @@ -238,13 +246,17 @@ def test_url_document_source_converts_to_uri_part(): def test_file_document_source_converts_to_file_part(): - part = _convert_dict_block_to_part( - { - "type": "document", - "source": {"type": "file", "file_id": "file-document"}, - } + parts = convert_content_to_parts( + [ + { + "type": "document", + "source": {"type": "file", "file_id": "file-document"}, + } + ] ) + assert len(parts) == 1 + part = parts[0] assert isinstance(part, FilePart) assert part.file_id == "file-document" assert part.mime_type is None @@ -252,17 +264,21 @@ def test_file_document_source_converts_to_file_part(): def test_plain_text_document_source_converts_to_blob_part(): - part = _convert_dict_block_to_part( - { - "type": "document", - "source": { - "type": "text", - "media_type": "text/plain", - "data": "Document text", - }, - } + parts = convert_content_to_parts( + [ + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text", + }, + } + ] ) + assert len(parts) == 1 + part = parts[0] assert isinstance(part, BlobPart) assert part.content == b"Document text" assert part.mime_type == "text/plain" diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py index 8d34d3694..96861cb28 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py @@ -643,6 +643,63 @@ def test_sync_messages_create_captures_multimodal_content( _assert_multimodal_input(spans[0]) +def test_sync_messages_create_preserves_generator_document_content( + instrument_with_content, +): + received_content = None + + def handle_request(request): + nonlocal received_content + body = json.loads(request.content) + received_content = body["messages"][0]["content"][0]["source"][ + "content" + ] + return httpx.Response( + 200, + json={ + "id": "msg_generator_content", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [{"type": "text", "text": "Received."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + def document_content(): + yield {"type": "text", "text": "First"} + yield {"type": "text", "text": "Second"} + + transport = httpx.MockTransport(handle_request) + with httpx.Client(transport=transport) as http_client: + client = Anthropic(http_client=http_client) + client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[ + { + "role": "user", + "content": [ + { + "type": "document", + "source": { + "type": "content", + "content": document_content(), + }, + } + ], + } + ], + ) + + assert received_content == [ + {"type": "text", "text": "First"}, + {"type": "text", "text": "Second"}, + ] + + @pytest.mark.vcr() def test_sync_messages_create_with_all_params( span_exporter, anthropic_client, instrument_no_content From 421579974411e3afcf91bcbcdae1e0090c131515 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Fri, 4 Sep 2026 16:22:24 -0700 Subject: [PATCH 05/13] Remove type references from GenericPart to match the updated utils --- .../instrumentation/genai/anthropic/utils.py | 24 ++------------- .../tests/conformance/multimodal.py | 14 --------- .../tests/test_async_messages.py | 14 --------- .../tests/test_messages_extractors.py | 30 ++++--------------- .../tests/test_sync_messages.py | 14 --------- 5 files changed, 8 insertions(+), 88 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index f417e8b76..a83898c4a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -7,7 +7,7 @@ import json from collections.abc import Iterable, Iterator, Mapping -from dataclasses import asdict, dataclass +from dataclasses import dataclass from os import PathLike from typing import TYPE_CHECKING, cast @@ -112,19 +112,7 @@ def _extract_base64_blob(source: object, modality: str) -> MessagePart | None: data = source_dict.get("data") if not isinstance(data, str): if isinstance(data, PathLike) or callable(getattr(data, "read", None)): - media_type = source_dict.get("media_type") - return GenericPart( - type=modality, - value={ - "source_type": "base64_file", - "mime_type": media_type - if isinstance(media_type, str) - else None, - "input_type": "path" - if isinstance(data, PathLike) - else "stream", - }, - ) + return GenericPart(type=modality) return None decoded = decode_base64(data) if decoded is None: @@ -227,13 +215,7 @@ def _convert_document_block(block: Mapping[str, Any]) -> MessagePart | None: source_mapping is not None and source_mapping.get("type") == "content" ) if metadata or (is_nested and parts): - return GenericPart( - type="document", - value={ - "parts": [asdict(part) for part in parts], - **metadata, - }, - ) + return GenericPart(type="document") return parts[0] if parts else None diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py index 91c662106..08b1ce33a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py @@ -207,20 +207,6 @@ def validate(self, report: LiveCheckReport) -> None: } assert parts[6] == { "type": "document", - "value": { - "parts": [ - {"content": "Nested text", "type": "text"}, - { - "mime_type": None, - "modality": "image", - "uri": "https://example.com/nested.png", - "type": "uri", - }, - ], - "title": "Reference", - "context": "Use the nested content.", - "citations": {"enabled": True}, - }, } assert parts[7] == { "type": "file", diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py index 73fc950b4..0e530351c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py @@ -216,20 +216,6 @@ def _assert_multimodal_input(span): } assert parts[6] == { "type": "document", - "value": { - "parts": [ - {"content": "Nested text", "type": "text"}, - { - "mime_type": None, - "modality": "image", - "uri": "https://example.com/nested.png", - "type": "uri", - }, - ], - "title": "Reference", - "context": "Use the nested content.", - "citations": {"enabled": True}, - }, } diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index 0e3171f57..4c0c16443 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -313,38 +313,23 @@ def test_nested_content_document_source_preserves_part_order(): assert len(parts) == 1 assert isinstance(parts[0], GenericPart) assert parts[0].type == "document" - assert parts[0].value == { - "parts": [ - {"content": "Nested text", "type": "text"}, - { - "mime_type": None, - "modality": "image", - "uri": "https://example.com/image.png", - "type": "uri", - }, - ], - "title": "Reference", - "context": "Use the nested content.", - "citations": {"enabled": True}, - } @pytest.mark.parametrize( - ("block_type", "media_type", "data", "input_type"), + ("block_type", "media_type", "data"), [ - ("image", "image/png", Path("private/image.png"), "path"), - ("image", "image/png", BytesIO(b"image"), "stream"), + ("image", "image/png", Path("private/image.png")), + ("image", "image/png", BytesIO(b"image")), ( "document", "application/pdf", Path("private/document.pdf"), - "path", ), - ("document", "application/pdf", BytesIO(b"document"), "stream"), + ("document", "application/pdf", BytesIO(b"document")), ], ) def test_file_backed_base64_source_is_preserved_without_reading( - block_type, media_type, data, input_type + block_type, media_type, data ): initial_position = data.tell() if isinstance(data, BytesIO) else None part = _convert_dict_block_to_part( @@ -360,11 +345,6 @@ def test_file_backed_base64_source_is_preserved_without_reading( assert isinstance(part, GenericPart) assert part.type == block_type - assert part.value == { - "source_type": "base64_file", - "mime_type": media_type, - "input_type": input_type, - } if initial_position is not None: assert data.tell() == initial_position diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py index 96861cb28..7b28a9314 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py @@ -345,20 +345,6 @@ def _assert_multimodal_input(span): } assert parts[6] == { "type": "document", - "value": { - "parts": [ - {"content": "Nested text", "type": "text"}, - { - "mime_type": None, - "modality": "image", - "uri": "https://example.com/nested.png", - "type": "uri", - }, - ], - "title": "Reference", - "context": "Use the nested content.", - "citations": {"enabled": True}, - }, } From 1598796d8153c33bba1820b28dcd33bd7d40d1a9 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 9 Sep 2026 15:34:06 -0700 Subject: [PATCH 06/13] Address feedback --- ...es_create_captures_multimodal_content.yaml | 124 ++++++++++++++++++ ...es_create_captures_multimodal_content.yaml | 124 ++++++++++++++++++ .../tests/test_async_messages.py | 4 +- .../tests/test_sync_messages.py | 4 +- 4 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_captures_multimodal_content.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_captures_multimodal_content.yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_captures_multimodal_content.yaml b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_captures_multimodal_content.yaml new file mode 100644 index 000000000..b8c661986 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_captures_multimodal_content.yaml @@ -0,0 +1,124 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these images." + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD" + } + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png" + } + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD" + } + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf" + } + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text" + } + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": { + "enabled": true + }, + "source": { + "type": "content", + "content": [ + { + "type": "text", + "text": "Nested text" + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/nested.png" + } + } + ] + } + } + ] + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + anthropic-version: + - '2023-06-01' + content-type: + - application/json + host: + - api.anthropic.com + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_async_multimodal", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "The images and documents were received." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 7 + } + } + headers: + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_captures_multimodal_content.yaml b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_captures_multimodal_content.yaml new file mode 100644 index 000000000..d4a446f1d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_captures_multimodal_content.yaml @@ -0,0 +1,124 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these images." + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD" + } + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png" + } + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD" + } + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf" + } + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text" + } + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": { + "enabled": true + }, + "source": { + "type": "content", + "content": [ + { + "type": "text", + "text": "Nested text" + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/nested.png" + } + } + ] + } + } + ] + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + anthropic-version: + - '2023-06-01' + content-type: + - application/json + host: + - api.anthropic.com + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_sync_multimodal", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "The images and documents were received." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 7 + } + } + headers: + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py index 0e530351c..a7fd52dd1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py @@ -364,7 +364,9 @@ async def test_async_messages_create_captures_multimodal_content( instrument_with_content, vcr, ): - with vcr.use_cassette("test_async_messages_create_captures_content.yaml"): + with vcr.use_cassette( + "test_async_messages_create_captures_multimodal_content.yaml" + ): await async_anthropic_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=100, diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py index 7b28a9314..680c81698 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py @@ -617,7 +617,9 @@ def test_sync_messages_create_captures_multimodal_content( instrument_with_content, vcr, ): - with vcr.use_cassette("test_sync_messages_create_captures_content.yaml"): + with vcr.use_cassette( + "test_sync_messages_create_captures_multimodal_content.yaml" + ): anthropic_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=100, From 60a7762452b7def436bf627844c6f0fc47faa2de Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 10 Sep 2026 09:12:35 -0700 Subject: [PATCH 07/13] Fix formatting --- .../tests/test_messages_extractors.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index 4c0c16443..214fd66f0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -5,9 +5,6 @@ from io import BytesIO from pathlib import Path - -import pytest - from types import SimpleNamespace from unittest.mock import MagicMock @@ -18,6 +15,7 @@ WebSearchToolResultError, ) + from opentelemetry.instrumentation.genai.anthropic.messages_extractors import ( extract_params, get_tool_definitions, From 09d9a89f23114787461979884fbbd6db2b906ed8 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 10 Sep 2026 09:47:43 -0700 Subject: [PATCH 08/13] Fix incorrect import --- .../tests/test_messages_extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index 214fd66f0..b4a18d007 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -20,6 +20,7 @@ extract_params, get_tool_definitions, get_input_messages, + set_invocation_response_attributes, ) from opentelemetry.instrumentation.genai.anthropic.utils import ( _convert_dict_block_to_part, @@ -31,7 +32,6 @@ GenericPart, TextPart, UriPart, - set_invocation_response_attributes, ) from opentelemetry.instrumentation.genai.anthropic.utils import ( _convert_content_block_to_part, From 4a958d6b20e1dbc49bfe4b48ef9adafa9a5678e4 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 10 Sep 2026 10:55:49 -0700 Subject: [PATCH 09/13] Fix test --- .../tests/conformance/multimodal.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py index 08b1ce33a..433e1a84e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py @@ -17,10 +17,7 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.test.weaver_live_check import LiveCheckReport -from opentelemetry.test_util_genai.conformance import ( - ExpectedViolation, - Scenario, -) +from opentelemetry.test_util_genai.conformance import Scenario from opentelemetry.test_util_genai.instrumentor import instrument @@ -30,12 +27,6 @@ class MultimodalScenario(Scenario): "gen_ai.client.operation.duration", "gen_ai.client.token.usage", ) - expected_violations = ( - ExpectedViolation( - advice_id="missing_attribute", - message_substring="gen_ai.usage.cache_creation.input_tokens", - ), - ) def run( self, From a9803fbb46269723b37dc9b6225bb5ebe83b5d2c Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 10 Sep 2026 13:36:59 -0700 Subject: [PATCH 10/13] Address feedback --- .../instrumentation/genai/anthropic/utils.py | 6 +- ...streaming_captures_multimodal_content.yaml | 72 +++++++++++++++++++ ...streaming_captures_multimodal_content.yaml | 72 +++++++++++++++++++ .../tests/test_async_messages.py | 2 +- .../tests/test_sync_messages.py | 2 +- 5 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_streaming_captures_multimodal_content.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_streaming_captures_multimodal_content.yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index a83898c4a..86ebcbdec 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -105,7 +105,11 @@ def normalize_finish_reason(stop_reason: str | None) -> str | None: def _extract_base64_blob(source: object, modality: str) -> MessagePart | None: - """Extract a BlobPart from a base64-encoded source dict.""" + """Convert an Anthropic base64 source to a GenAI message part. + + String data becomes a ``BlobPart``; file-backed data is represented as a + ``GenericPart`` without being read. + """ if not isinstance(source, dict): return None source_dict = cast(dict[str, object], source) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_streaming_captures_multimodal_content.yaml b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_streaming_captures_multimodal_content.yaml new file mode 100644 index 000000000..522d54af2 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_async_messages_create_streaming_captures_multimodal_content.yaml @@ -0,0 +1,72 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these images."}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "QUJD"}}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/image.png"}}, + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "QUJD"}}, + {"type": "document", "source": {"type": "url", "url": "https://example.com/document.pdf"}}, + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Document text"}}, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": true}, + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "Nested text"}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/nested.png"}} + ] + } + } + ] + } + ], + "model": "claude-sonnet-4-20250514", + "stream": true + } + headers: + content-type: + - application/json + host: + - api.anthropic.com + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_async_streaming_multimodal","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"}}} + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Received."}} + + event: content_block_stop + data: {"type":"content_block_stop","index":0} + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}} + + event: message_stop + data: {"type":"message_stop"} + + headers: + content-type: + - text/event-stream; charset=utf-8 + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_streaming_captures_multimodal_content.yaml b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_streaming_captures_multimodal_content.yaml new file mode 100644 index 000000000..d03d97e71 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/test_sync_messages_create_streaming_captures_multimodal_content.yaml @@ -0,0 +1,72 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these images."}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "QUJD"}}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/image.png"}}, + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "QUJD"}}, + {"type": "document", "source": {"type": "url", "url": "https://example.com/document.pdf"}}, + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Document text"}}, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": true}, + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "Nested text"}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/nested.png"}} + ] + } + } + ] + } + ], + "model": "claude-sonnet-4-20250514", + "stream": true + } + headers: + content-type: + - application/json + host: + - api.anthropic.com + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_sync_streaming_multimodal","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":2,"service_tier":"standard","inference_geo":"not_available"}}} + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Received."}} + + event: content_block_stop + data: {"type":"content_block_stop","index":0} + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}} + + event: message_stop + data: {"type":"message_stop"} + + headers: + content-type: + - text/event-stream; charset=utf-8 + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py index a7fd52dd1..976c22ea2 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py @@ -595,7 +595,7 @@ async def test_async_messages_create_streaming_captures_multimodal_content( vcr, ): with vcr.use_cassette( - "test_async_messages_create_streaming_captures_content.yaml" + "test_async_messages_create_streaming_captures_multimodal_content.yaml" ): stream = await async_anthropic_client.messages.create( model="claude-sonnet-4-20250514", diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py index 680c81698..4e9ef96d7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py @@ -1042,7 +1042,7 @@ def test_sync_messages_create_streaming_captures_multimodal_content( vcr, ): with vcr.use_cassette( - "test_sync_messages_create_streaming_captures_content.yaml" + "test_sync_messages_create_streaming_captures_multimodal_content.yaml" ): with anthropic_client.messages.create( model="claude-sonnet-4-20250514", From ebf3a8956a65f5a78dfc3f43108dc16c4c130126 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 10 Sep 2026 16:29:37 -0700 Subject: [PATCH 11/13] Address feedback --- .../instrumentation/genai/anthropic/utils.py | 2 +- .../tests/test_messages_extractors.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index 86ebcbdec..c26ce2707 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -191,7 +191,7 @@ def _extract_document_source(source: object) -> list[MessagePart]: if isinstance(content, str): return [TextPart(content=content)] if isinstance(content, Iterator): - return [] + return [GenericPart(type="document")] if isinstance(content, Iterable): return convert_content_to_parts( cast("Iterable[ContentBlock | ContentBlockParam]", content) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index b4a18d007..f83cf3c8d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -313,6 +313,28 @@ def test_nested_content_document_source_preserves_part_order(): assert parts[0].type == "document" +def test_iterator_document_source_is_preserved_without_consuming(): + def document_content(): + yield {"type": "text", "text": "First"} + yield {"type": "text", "text": "Second"} + + content = document_content() + parts = convert_content_to_parts( + [ + { + "type": "document", + "source": {"type": "content", "content": content}, + } + ] + ) + + assert parts == [GenericPart(type="document")] + assert list(content) == [ + {"type": "text", "text": "First"}, + {"type": "text", "text": "Second"}, + ] + + @pytest.mark.parametrize( ("block_type", "media_type", "data"), [ From c5f28a06a17c6869794dcd69c725cf31ce303d4d Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 14 Sep 2026 11:55:05 -0700 Subject: [PATCH 12/13] Address feedback --- .../instrumentation/genai/anthropic/utils.py | 38 ++---- .../tests/conformance/multimodal.py | 15 +- .../tests/conftest.py | 128 ++++++++++++++++++ .../tests/test_async_messages.py | 125 ++--------------- .../tests/test_messages_extractors.py | 43 +++++- .../tests/test_sync_messages.py | 125 ++--------------- 6 files changed, 209 insertions(+), 265 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index c26ce2707..a4e30daa2 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -116,7 +116,7 @@ def _extract_base64_blob(source: object, modality: str) -> MessagePart | None: data = source_dict.get("data") if not isinstance(data, str): if isinstance(data, PathLike) or callable(getattr(data, "read", None)): - return GenericPart(type=modality) + return GenericPart(type="blob") return None decoded = decode_base64(data) if decoded is None: @@ -167,9 +167,12 @@ def _extract_document_source(source: object) -> list[MessagePart]: if source_type == "url": url = source_dict.get("url") if isinstance(url, str) and url: + media_type = source_dict.get("media_type") return [ UriPart( - mime_type="application/pdf", + mime_type=media_type + if isinstance(media_type, str) + else None, modality="document", uri=url, ) @@ -191,7 +194,7 @@ def _extract_document_source(source: object) -> list[MessagePart]: if isinstance(content, str): return [TextPart(content=content)] if isinstance(content, Iterator): - return [GenericPart(type="document")] + return [GenericPart(type="blob")] if isinstance(content, Iterable): return convert_content_to_parts( cast("Iterable[ContentBlock | ContentBlockParam]", content) @@ -202,25 +205,8 @@ def _extract_document_source(source: object) -> list[MessagePart]: return [] -def _convert_document_block(block: Mapping[str, Any]) -> MessagePart | None: - parts = _extract_document_source(block.get("source")) - metadata = { - key: block[key] - for key in ("title", "context", "citations") - if block.get(key) is not None - } - source = block.get("source") - source_mapping = ( - cast(Mapping[str, object], source) - if isinstance(source, Mapping) - else None - ) - is_nested = ( - source_mapping is not None and source_mapping.get("type") == "content" - ) - if metadata or (is_nested and parts): - return GenericPart(type="document") - return parts[0] if parts else None +def _convert_document_block(block: Mapping[str, Any]) -> list[MessagePart]: + return _extract_document_source(block.get("source")) def _convert_dict_block_to_part( @@ -287,7 +273,8 @@ def _convert_dict_block_to_part( return _extract_image_source(block.get("source")) if block_type == "document": - return _convert_document_block(block) + parts = _convert_document_block(block) + return parts[0] if parts else None if block_type in ("audio", "video", "file"): return _extract_base64_blob(block.get("source"), str(block_type)) @@ -336,6 +323,11 @@ def convert_content_to_parts( return [TextPart(content=content)] parts: list[MessagePart] = [] for item in content: + if hasattr(item, "get"): + item_mapping = cast(Mapping[str, Any], item) + if item_mapping.get("type") == "document": + parts.extend(_convert_document_block(item_mapping)) + continue part = _convert_content_block_to_part(item) if part is not None: parts.append(part) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py index 433e1a84e..7ee9f04f7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/multimodal.py @@ -161,7 +161,7 @@ def validate(self, report: LiveCheckReport) -> None: assert input_messages[0]["role"] == "user" parts = input_messages[0]["parts"] - assert len(parts) == 9 + assert len(parts) == 10 assert parts[0] == { "type": "text", "content": "Describe these images.", @@ -186,7 +186,7 @@ def validate(self, report: LiveCheckReport) -> None: } assert parts[4] == { "type": "uri", - "mime_type": "application/pdf", + "mime_type": None, "modality": "document", "uri": "https://example.com/document.pdf", } @@ -197,15 +197,22 @@ def validate(self, report: LiveCheckReport) -> None: "content": "RG9jdW1lbnQgdGV4dA==", } assert parts[6] == { - "type": "document", + "type": "text", + "content": "Nested text", } assert parts[7] == { + "type": "uri", + "mime_type": None, + "modality": "image", + "uri": "https://example.com/nested.png", + } + assert parts[8] == { "type": "file", "mime_type": None, "modality": "image", "file_id": "file-image", } - assert parts[8] == { + assert parts[9] == { "type": "file", "mime_type": None, "modality": "document", diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conftest.py index c708fd602..5c6d81d67 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conftest.py @@ -4,6 +4,7 @@ """Test configuration and fixtures for Anthropic instrumentation tests.""" # pylint: disable=redefined-outer-name +import json import os import pytest @@ -11,6 +12,10 @@ from anthropic import Anthropic, AsyncAnthropic from opentelemetry.instrumentation.genai.anthropic import AnthropicInstrumentor +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.test_util_genai.vcr import scrub_response_headers @@ -20,6 +25,129 @@ ] +def multimodal_input_message(): + return { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these images."}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "QUJD", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png", + }, + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + }, + }, + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "Document text", + }, + }, + { + "type": "document", + "title": "Reference", + "context": "Use the nested content.", + "citations": {"enabled": True}, + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "Nested text"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/nested.png", + }, + }, + ], + }, + }, + ], + } + + +def assert_multimodal_input(span: ReadableSpan) -> None: + value = span.attributes.get(GenAIAttributes.GEN_AI_INPUT_MESSAGES) + assert value is not None + assert isinstance(value, str) + messages = json.loads(value) + assert isinstance(messages, list) + assert len(messages) == 1 + assert messages[0]["role"] == "user" + + parts = messages[0]["parts"] + assert len(parts) == 8 + assert parts[0] == { + "type": "text", + "content": "Describe these images.", + } + assert parts[1] == { + "type": "blob", + "mime_type": "image/png", + "modality": "image", + "content": "QUJD", + } + assert parts[2] == { + "type": "uri", + "mime_type": None, + "modality": "image", + "uri": "https://example.com/image.png", + } + assert parts[3] == { + "type": "blob", + "mime_type": "application/pdf", + "modality": "document", + "content": "QUJD", + } + assert parts[4] == { + "type": "uri", + "mime_type": None, + "modality": "document", + "uri": "https://example.com/document.pdf", + } + assert parts[5] == { + "type": "blob", + "mime_type": "text/plain", + "modality": "document", + "content": "RG9jdW1lbnQgdGV4dA==", + } + assert parts[6] == { + "type": "text", + "content": "Nested text", + } + assert parts[7] == { + "type": "uri", + "mime_type": None, + "modality": "image", + "uri": "https://example.com/nested.png", + } + + @pytest.fixture(autouse=True) def environment(): """Set up environment variables for testing.""" diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py index 976c22ea2..37826da40 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_async_messages.py @@ -44,6 +44,11 @@ ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from .conftest import ( + assert_multimodal_input, + multimodal_input_message, +) + _create_params = set(inspect.signature(_AsyncMessages.create).parameters) _has_tools_param = "tools" in _create_params _has_thinking_param = "thinking" in _create_params @@ -107,118 +112,6 @@ def _assert_weather_tool_definitions(span): ] -def _multimodal_input_message(): - return { - "role": "user", - "content": [ - {"type": "text", "text": "Describe these images."}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "QUJD", - }, - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/image.png", - }, - }, - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "QUJD", - }, - }, - { - "type": "document", - "source": { - "type": "url", - "url": "https://example.com/document.pdf", - }, - }, - { - "type": "document", - "source": { - "type": "text", - "media_type": "text/plain", - "data": "Document text", - }, - }, - { - "type": "document", - "title": "Reference", - "context": "Use the nested content.", - "citations": {"enabled": True}, - "source": { - "type": "content", - "content": [ - {"type": "text", "text": "Nested text"}, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/nested.png", - }, - }, - ], - }, - }, - ], - } - - -def _assert_multimodal_input(span): - messages = _load_span_messages(span, GenAIAttributes.GEN_AI_INPUT_MESSAGES) - assert len(messages) == 1 - assert messages[0]["role"] == "user" - - parts = messages[0]["parts"] - assert len(parts) == 7 - assert parts[0] == { - "type": "text", - "content": "Describe these images.", - } - assert parts[1] == { - "type": "blob", - "mime_type": "image/png", - "modality": "image", - "content": "QUJD", - } - assert parts[2] == { - "type": "uri", - "mime_type": None, - "modality": "image", - "uri": "https://example.com/image.png", - } - assert parts[3] == { - "type": "blob", - "mime_type": "application/pdf", - "modality": "document", - "content": "QUJD", - } - assert parts[4] == { - "type": "uri", - "mime_type": "application/pdf", - "modality": "document", - "uri": "https://example.com/document.pdf", - } - assert parts[5] == { - "type": "blob", - "mime_type": "text/plain", - "modality": "document", - "content": "RG9jdW1lbnQgdGV4dA==", - } - assert parts[6] == { - "type": "document", - } - - class _AsyncErrorInjectingStreamDelegate: def __init__(self, inner): self._inner = inner @@ -370,12 +263,12 @@ async def test_async_messages_create_captures_multimodal_content( await async_anthropic_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=100, - messages=[_multimodal_input_message()], + messages=[multimodal_input_message()], ) spans = span_exporter.get_finished_spans() assert len(spans) == 1 - _assert_multimodal_input(spans[0]) + assert_multimodal_input(spans[0]) @pytest.mark.asyncio @@ -600,7 +493,7 @@ async def test_async_messages_create_streaming_captures_multimodal_content( stream = await async_anthropic_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=100, - messages=[_multimodal_input_message()], + messages=[multimodal_input_message()], stream=True, ) async with stream: @@ -609,7 +502,7 @@ async def test_async_messages_create_streaming_captures_multimodal_content( spans = span_exporter.get_finished_spans() assert len(spans) == 1 - _assert_multimodal_input(spans[0]) + assert_multimodal_input(spans[0]) @pytest.mark.asyncio diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index f83cf3c8d..77102e6b4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -222,6 +222,32 @@ def test_base64_document_source_converts_to_blob_part(): assert part.modality == "document" +def test_document_metadata_preserves_base64_source(): + parts = convert_content_to_parts( + [ + { + "type": "document", + "title": "Report", + "context": "Quarterly results", + "citations": {"enabled": True}, + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "QUJD", + }, + } + ] + ) + + assert parts == [ + BlobPart( + mime_type="application/pdf", + modality="document", + content=b"ABC", + ) + ] + + def test_url_document_source_converts_to_uri_part(): parts = convert_content_to_parts( [ @@ -239,7 +265,7 @@ def test_url_document_source_converts_to_uri_part(): part = parts[0] assert isinstance(part, UriPart) assert part.uri == "https://example.com/document.pdf" - assert part.mime_type == "application/pdf" + assert part.mime_type is None assert part.modality == "document" @@ -308,9 +334,14 @@ def test_nested_content_document_source_preserves_part_order(): ] ) - assert len(parts) == 1 - assert isinstance(parts[0], GenericPart) - assert parts[0].type == "document" + assert parts == [ + TextPart(content="Nested text"), + UriPart( + mime_type=None, + modality="image", + uri="https://example.com/image.png", + ), + ] def test_iterator_document_source_is_preserved_without_consuming(): @@ -328,7 +359,7 @@ def document_content(): ] ) - assert parts == [GenericPart(type="document")] + assert parts == [GenericPart(type="blob")] assert list(content) == [ {"type": "text", "text": "First"}, {"type": "text", "text": "Second"}, @@ -364,7 +395,7 @@ def test_file_backed_base64_source_is_preserved_without_reading( ) assert isinstance(part, GenericPart) - assert part.type == block_type + assert part.type == "blob" if initial_position is not None: assert data.tell() == initial_position diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py index 4e9ef96d7..0f91c01bb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_sync_messages.py @@ -53,6 +53,11 @@ ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from .conftest import ( + assert_multimodal_input, + multimodal_input_message, +) + # Detect whether the installed anthropic SDK supports tools / thinking params. # Older SDK versions (e.g. 0.16.0) do not accept these keyword arguments. _create_params = set(inspect.signature(_Messages.create).parameters) @@ -236,118 +241,6 @@ def _assert_weather_tool_definitions(span): ] -def _multimodal_input_message(): - return { - "role": "user", - "content": [ - {"type": "text", "text": "Describe these images."}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "QUJD", - }, - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/image.png", - }, - }, - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "QUJD", - }, - }, - { - "type": "document", - "source": { - "type": "url", - "url": "https://example.com/document.pdf", - }, - }, - { - "type": "document", - "source": { - "type": "text", - "media_type": "text/plain", - "data": "Document text", - }, - }, - { - "type": "document", - "title": "Reference", - "context": "Use the nested content.", - "citations": {"enabled": True}, - "source": { - "type": "content", - "content": [ - {"type": "text", "text": "Nested text"}, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/nested.png", - }, - }, - ], - }, - }, - ], - } - - -def _assert_multimodal_input(span): - messages = _load_span_messages(span, GenAIAttributes.GEN_AI_INPUT_MESSAGES) - assert len(messages) == 1 - assert messages[0]["role"] == "user" - - parts = messages[0]["parts"] - assert len(parts) == 7 - assert parts[0] == { - "type": "text", - "content": "Describe these images.", - } - assert parts[1] == { - "type": "blob", - "mime_type": "image/png", - "modality": "image", - "content": "QUJD", - } - assert parts[2] == { - "type": "uri", - "mime_type": None, - "modality": "image", - "uri": "https://example.com/image.png", - } - assert parts[3] == { - "type": "blob", - "mime_type": "application/pdf", - "modality": "document", - "content": "QUJD", - } - assert parts[4] == { - "type": "uri", - "mime_type": "application/pdf", - "modality": "document", - "uri": "https://example.com/document.pdf", - } - assert parts[5] == { - "type": "blob", - "mime_type": "text/plain", - "modality": "document", - "content": "RG9jdW1lbnQgdGV4dA==", - } - assert parts[6] == { - "type": "document", - } - - def _skip_if_cassette_missing_and_no_real_key(request): cassette_path = ( Path(__file__).parent / "cassettes" / f"{request.node.name}.yaml" @@ -623,12 +516,12 @@ def test_sync_messages_create_captures_multimodal_content( anthropic_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=100, - messages=[_multimodal_input_message()], + messages=[multimodal_input_message()], ) spans = span_exporter.get_finished_spans() assert len(spans) == 1 - _assert_multimodal_input(spans[0]) + assert_multimodal_input(spans[0]) def test_sync_messages_create_preserves_generator_document_content( @@ -1047,7 +940,7 @@ def test_sync_messages_create_streaming_captures_multimodal_content( with anthropic_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=100, - messages=[_multimodal_input_message()], + messages=[multimodal_input_message()], stream=True, ) as stream: for _ in stream: @@ -1055,7 +948,7 @@ def test_sync_messages_create_streaming_captures_multimodal_content( spans = span_exporter.get_finished_spans() assert len(spans) == 1 - _assert_multimodal_input(spans[0]) + assert_multimodal_input(spans[0]) @pytest.mark.vcr() From e8d66ddc083d37b1732d498ce20d31497ad5b8c2 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 14 Sep 2026 11:55:28 -0700 Subject: [PATCH 13/13] Add tests --- .../tests/test_messages_extractors.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index 77102e6b4..178aaca59 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -269,6 +269,29 @@ def test_url_document_source_converts_to_uri_part(): assert part.modality == "document" +def test_url_document_source_preserves_media_type(): + parts = convert_content_to_parts( + [ + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/document.pdf", + "media_type": "application/pdf", + }, + } + ] + ) + + assert parts == [ + UriPart( + mime_type="application/pdf", + modality="document", + uri="https://example.com/document.pdf", + ) + ] + + def test_file_document_source_converts_to_file_part(): parts = convert_content_to_parts( [