diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/699.fixed b/instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/699.fixed new file mode 100644 index 000000000..4ac1d95bc --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/.changelog/699.fixed @@ -0,0 +1 @@ +Represent Anthropic server tool calls and results with server tool message parts. diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py index 033201edc..c497fdcb4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/src/opentelemetry/instrumentation/genai/anthropic/utils.py @@ -9,7 +9,7 @@ import json from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, cast from anthropic.types import ( InputJSONDelta, @@ -20,18 +20,28 @@ ThinkingBlock, ThinkingDelta, ToolUseBlock, - WebSearchToolResultBlock, ) from opentelemetry.util.genai.types import ( BlobPart, MessagePart, ReasoningPart, + ServerToolCallPart, + ServerToolCallResponsePart, TextPart, ToolCallRequestPart, ToolCallResponsePart, ) +_SERVER_TOOL_RESULT_TYPES = { + "web_search_tool_result": "web_search", + "web_fetch_tool_result": "web_fetch", + "code_execution_tool_result": "code_execution", + "bash_code_execution_tool_result": "bash_code_execution", + "text_editor_code_execution_tool_result": "text_editor_code_execution", + "tool_search_tool_result": "tool_search", +} + if TYPE_CHECKING: from collections.abc import Iterable @@ -120,7 +130,7 @@ def _extract_base64_blob(source: object, modality: str) -> BlobPart | None: def _convert_dict_block_to_part( - block: Mapping[str, Any], + block: Mapping[str, object], ) -> MessagePart | None: """Convert a request-param content block (TypedDict/dict) to a MessagePart.""" block_type = block.get("type") @@ -137,12 +147,42 @@ def _convert_dict_block_to_part( id=str(block.get("id", "")), ) + if block_type == "server_tool_use": + name = str(block.get("name", "")) + server_tool_call = { + key: value + for key, value in block.items() + if key not in ("id", "name", "type") + } + server_tool_call["type"] = name + block_id = block.get("id") + return ServerToolCallPart( + name=name, + server_tool_call=server_tool_call, + id=str(block_id) if block_id is not None else None, + ) + if block_type == "tool_result": return ToolCallResponsePart( response=block.get("content"), id=str(block.get("tool_use_id", "")), ) + if isinstance(block_type, str) and ( + server_tool_name := _SERVER_TOOL_RESULT_TYPES.get(block_type) + ): + server_tool_call_response = { + key: value + for key, value in block.items() + if key not in ("tool_use_id", "type") + } + server_tool_call_response["type"] = server_tool_name + tool_use_id = block.get("tool_use_id") + return ServerToolCallResponsePart( + server_tool_call_response=server_tool_call_response, + id=str(tool_use_id) if tool_use_id is not None else None, + ) + if block_type in ("thinking", "redacted_thinking"): thinking = block.get("thinking") or block.get("data") return ReasoningPart( @@ -159,29 +199,32 @@ def _convert_content_block_to_part( block: ContentBlock | ContentBlockParam, ) -> MessagePart | None: """Convert an Anthropic content block to a MessagePart.""" + if isinstance(block, Mapping): + return _convert_dict_block_to_part(cast(Mapping[str, object], block)) + if isinstance(block, TextBlock): return TextPart(content=block.text) - if isinstance(block, (ToolUseBlock, ServerToolUseBlock)): + if isinstance(block, ToolUseBlock): return ToolCallRequestPart( arguments=block.input, name=block.name, id=block.id ) + if isinstance(block, ServerToolUseBlock): + return _convert_dict_block_to_part(block.model_dump(exclude_none=True)) + if isinstance(block, (ThinkingBlock, RedactedThinkingBlock)): content = ( block.thinking if isinstance(block, ThinkingBlock) else block.data ) return ReasoningPart(content=content) - if isinstance(block, WebSearchToolResultBlock): - return ToolCallResponsePart( - response=block.model_dump().get("content"), - id=block.tool_use_id, + if block.type in _SERVER_TOOL_RESULT_TYPES: + return _convert_dict_block_to_part( + cast(Mapping[str, object], block.model_dump(exclude_none=True)) ) - if not hasattr(block, "get"): - return None - return _convert_dict_block_to_part(cast(Mapping[str, Any], block)) + return None def convert_content_to_parts( diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/server_tool_calling_conformance.yaml b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/server_tool_calling_conformance.yaml new file mode 100644 index 000000000..65c487a0a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/cassettes/server_tool_calling_conformance.yaml @@ -0,0 +1,58 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + {"max_tokens":256,"messages":[{"role":"user","content":"Search for OpenTelemetry."}],"model":"claude-sonnet-4-20250514","tools":[{"type":"web_search_20250305","name":"web_search","max_uses":1}]} + headers: + accept: + - application/json + content-type: + - application/json + host: + - api.anthropic.com + x-api-key: + - test_anthropic_api_key + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_server_tool", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "OpenTelemetry"} + }, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": { + "type": "web_search_tool_result_error", + "error_code": "unavailable" + } + }, + { + "type": "text", + "text": "Search was unavailable." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 20, + "output_tokens": 15 + } + } + headers: + Content-Type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/server_tool_calling.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/server_tool_calling.py new file mode 100644 index 000000000..654fa76f6 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/server_tool_calling.py @@ -0,0 +1,102 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: Anthropic chat with server-side tool calls.""" + +from __future__ import annotations + +import json +import os +from typing import Any +from unittest import mock + +from anthropic import Anthropic + +from opentelemetry.instrumentation.genai.anthropic import AnthropicInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class ServerToolCallingScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_messages = [ + json.loads(attribute["value"]) + for entry in report["samples"] + if "span" in entry + for attribute in entry["span"]["attributes"] + if attribute["name"] == "gen_ai.output.messages" + ] + assert len(output_messages) == 1 + assert output_messages[0][0]["parts"][:2] == [ + { + "name": "web_search", + "server_tool_call": { + "input": {"query": "OpenTelemetry"}, + "type": "web_search", + }, + "id": "srvtoolu_01", + "type": "server_tool_call", + }, + { + "server_tool_call_response": { + "content": { + "error_code": "unavailable", + "type": "web_search_tool_result_error", + }, + "type": "web_search", + }, + "id": "srvtoolu_01", + "type": "server_tool_call_response", + }, + ] + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("ANTHROPIC_API_KEY") + else {"ANTHROPIC_API_KEY": "test_anthropic_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + AnthropicInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("server_tool_calling_conformance.yaml"): + Anthropic().messages.create( + model="claude-sonnet-4-20250514", + max_tokens=256, + messages=[ + { + "role": "user", + "content": "Search for OpenTelemetry.", + } + ], + tools=[ + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 1, + } + ], + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py index 3b2d33954..04ea70d42 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py @@ -24,6 +24,7 @@ InferenceRawResponseStreamingScenario, ) from .conformance.inference_streaming import InferenceStreamingScenario +from .conformance.server_tool_calling import ServerToolCallingScenario from .conformance.tool_calling import ToolCallingScenario @@ -35,6 +36,7 @@ InferenceRawResponseScenario(), InferenceRawResponseStreamingScenario(), ToolCallingScenario(), + ServerToolCallingScenario(), ], ids=lambda s: type(s).__name__, ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py index 541b17081..1b986609d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_messages_extractors.py @@ -6,10 +6,23 @@ from types import SimpleNamespace from unittest.mock import MagicMock +from anthropic.types import ( + ServerToolUseBlock, + WebSearchToolResultBlock, + WebSearchToolResultError, +) + from opentelemetry.instrumentation.genai.anthropic.messages_extractors import ( extract_params, set_invocation_response_attributes, ) +from opentelemetry.instrumentation.genai.anthropic.utils import ( + _convert_content_block_to_part, +) +from opentelemetry.util.genai.types import ( + ServerToolCallPart, + ServerToolCallResponsePart, +) def test_extract_params_reads_sampling_params_from_extra_body(): @@ -73,3 +86,80 @@ def test_set_invocation_response_attributes_records_cache_tokens(): assert invocation.output_tokens == 20 assert invocation.cache_write_input_tokens == 15 assert invocation.cache_read_input_tokens == 5 + + +def test_convert_server_tool_use_block(): + part = _convert_content_block_to_part( + ServerToolUseBlock( + id="srvtoolu_123", + input={"query": "OpenTelemetry"}, + name="web_search", + type="server_tool_use", + ) + ) + + assert isinstance(part, ServerToolCallPart) + assert part.id == "srvtoolu_123" + assert part.name == "web_search" + assert part.server_tool_call == { + "input": {"query": "OpenTelemetry"}, + "type": "web_search", + } + + +def test_convert_server_tool_result_block(): + part = _convert_content_block_to_part( + WebSearchToolResultBlock( + content=WebSearchToolResultError( + error_code="unavailable", + type="web_search_tool_result_error", + ), + tool_use_id="srvtoolu_123", + type="web_search_tool_result", + ) + ) + + assert isinstance(part, ServerToolCallResponsePart) + assert part.id == "srvtoolu_123" + assert part.server_tool_call_response == { + "content": { + "error_code": "unavailable", + "type": "web_search_tool_result_error", + }, + "type": "web_search", + } + + +def test_convert_server_tool_dicts(): + call = _convert_content_block_to_part( + { + "type": "server_tool_use", + "id": "srvtoolu_123", + "name": "web_fetch", + "input": {"url": "https://opentelemetry.io"}, + } + ) + response = _convert_content_block_to_part( + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_123", + "content": { + "type": "web_fetch_result", + "url": "https://opentelemetry.io", + }, + } + ) + + assert isinstance(call, ServerToolCallPart) + assert call.server_tool_call == { + "input": {"url": "https://opentelemetry.io"}, + "type": "web_fetch", + } + assert isinstance(response, ServerToolCallResponsePart) + assert response.server_tool_call_response == { + "content": { + "type": "web_fetch_result", + "url": "https://opentelemetry.io", + }, + "type": "web_fetch", + }