diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py index 79c974826..fcf4f9cce 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py @@ -36,7 +36,7 @@ ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, + LocalAgentInvocation, ToolInvocation, WorkflowInvocation, ) @@ -201,7 +201,7 @@ def _set_tool_invocation_output( def _set_invocation_input( - invocation: AgentInvocation | WorkflowInvocation, + invocation: LocalAgentInvocation | WorkflowInvocation, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any], @@ -225,7 +225,7 @@ def _extract_finish_reason(result: object) -> str: def _set_invocation_output( - invocation: AgentInvocation | WorkflowInvocation, + invocation: LocalAgentInvocation | WorkflowInvocation, result: object | None, capture_content: bool, ) -> None: @@ -249,7 +249,7 @@ def _start_agent_invocation( args: tuple[Any, ...], kwargs: dict[str, Any], capture_content: bool, -) -> AgentInvocation: +) -> LocalAgentInvocation: agent_name = getattr(instance, "name", None) invocation = handler.invoke_local_agent(agent_name=agent_name) _set_invocation_input(invocation, instance, args, kwargs, capture_content) diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py index a94177ee5..5b3697404 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py @@ -12,7 +12,7 @@ format_content, ) from opentelemetry.util.genai.invocation import ( - AgentInvocation, + LocalAgentInvocation, WorkflowInvocation, ) from opentelemetry.util.genai.stream import ( @@ -37,7 +37,7 @@ def _extract_chunk_content(chunk: Any) -> str | None: class _AgentStreamMixin: - _self_agent_invocation: AgentInvocation + _self_agent_invocation: LocalAgentInvocation _self_capture_content: bool _self_content_parts: list[str] _self_completed_content: str | None @@ -56,14 +56,6 @@ def _process_chunk(self, chunk: Any) -> None: self._self_agent_invocation.output_tokens = ( metrics.output_tokens ) - if getattr(metrics, "cache_read_tokens", None) is not None: - self._self_agent_invocation.cache_read_input_tokens = ( - metrics.cache_read_tokens - ) - if getattr(metrics, "cache_write_tokens", None) is not None: - self._self_agent_invocation.cache_write_input_tokens = ( - metrics.cache_write_tokens - ) event_name = str(getattr(chunk, "event", "")) chunk_type = type(chunk).__name__ @@ -159,7 +151,7 @@ class AgnoAgentStreamWrapper(_AgentStreamMixin, SyncStreamWrapper[Any]): def __init__( self, stream: Any, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, capture_content: bool, ) -> None: super().__init__(stream) @@ -176,7 +168,7 @@ class AsyncAgnoAgentStreamWrapper(_AgentStreamMixin, AsyncStreamWrapper[Any]): def __init__( self, stream: Any, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, capture_content: bool, ) -> None: super().__init__(stream) diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py index be17a87c3..0f9a1dbd9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py @@ -23,7 +23,10 @@ class AgentScenario(Scenario): expected_spans = {"invoke_agent": 1, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py index 4f3109b22..1eb8867b9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py +++ b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py @@ -52,7 +52,10 @@ def __call__(self, **kwargs: object) -> dict[str, str]: class ReActScenario(Scenario): expected_spans = {"invoke_agent": 1, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) expected_violations = ( ExpectedViolation( advice_id="genai_expected_attribute_missing", diff --git a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py index 1246dafa6..a04b7ef82 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py +++ b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py @@ -58,7 +58,10 @@ class SubmitPred: class ReActV2Scenario(Scenario): expected_spans = {"invoke_agent": 1, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) expected_violations = ( ExpectedViolation( advice_id="genai_expected_attribute_missing", diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed new file mode 100644 index 000000000..70d5a1799 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`, and stop setting span attribute `gen_ai.agent.id` on internal agent spans. diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index ceb4d387d..8b6c66867 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -173,7 +173,6 @@ def on_chain_start( agent.input_messages = make_input_message(inputs) if metadata: - agent.agent_id = metadata.get("agent_id") agent.agent_description = metadata.get( "agent_description" ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py index 802c1a5f0..d1980dd0a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py @@ -44,6 +44,8 @@ class AgentScenario(Scenario): expected_metrics = ( "gen_ai.client.operation.duration", "gen_ai.client.token.usage", + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", ) # langchain can't populate server.address on chat spans. # invoke_agent provider is unknown at span creation; ls_provider is only diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index c4c0632f0..214dea68e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -79,6 +79,7 @@ def _make_agent_inv_mock() -> mock.MagicMock: # agent_name is an instance attribute set in AgentInvocation.__init__ via the # constructor arg; pre-configure it so spec-restricted attribute access works. agent_inv.agent_name = None + agent_inv.agent_id = None return agent_inv @@ -256,7 +257,7 @@ def test_agent_metadata_set(self): }, ) - assert agent_inv.agent_id == "agent-123" + assert agent_inv.agent_id is None assert agent_inv.agent_description == "does math" assert agent_inv.conversation_id == "thread-abc" diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index c5da051e3..c08e5dbe8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -39,8 +39,8 @@ from opentelemetry.trace import set_span_in_context from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, GenAIInvocation, + LocalAgentInvocation, ToolInvocation, WorkflowInvocation, ) @@ -70,13 +70,13 @@ _MEMBER_AGENT_CONTEXTS: MutableMapping[ - AgentInvocation, contextvars.Context + LocalAgentInvocation, contextvars.Context ] = WeakKeyDictionary() def _start_member_agent( - start: Callable[[], AgentInvocation], -) -> AgentInvocation: + start: Callable[[], LocalAgentInvocation], +) -> LocalAgentInvocation: """Open a member-agent invocation inside a context this module owns. The invocation stays open across workflow steps, and each step runs in its @@ -91,7 +91,7 @@ def _start_member_agent( def _finish_member_agent( - invocation: AgentInvocation, error: BaseException | None = None + invocation: LocalAgentInvocation, error: BaseException | None = None ) -> None: """Finish a member-agent invocation in the context that started it.""" @@ -420,7 +420,7 @@ def _workflow_tool_definitions( return definitions or None -def _set_agent_output(invocation: AgentInvocation, result: Any) -> None: +def _set_agent_output(invocation: LocalAgentInvocation, result: Any) -> None: """Copy the final chat response out of LlamaIndex's workflow result.""" output = getattr(result, "result", None) response = getattr(output, "response", None) @@ -428,14 +428,16 @@ def _set_agent_output(invocation: AgentInvocation, result: Any) -> None: invocation.output_messages = [_output_message(response)] -def _set_agent_step_output(invocation: AgentInvocation, result: Any) -> None: +def _set_agent_step_output( + invocation: LocalAgentInvocation, result: Any +) -> None: """Copy a member agent's response out of an AgentWorkflow step.""" if isinstance(result, AgentOutput): invocation.output_messages = [_output_message(result.response)] def _set_return_direct_agent_output( - invocation: AgentInvocation, tool_output: ToolOutput + invocation: LocalAgentInvocation, tool_output: ToolOutput ) -> None: """Record the response synthesized by a return-direct tool execution.""" invocation.output_messages = [ @@ -507,11 +509,13 @@ class _LlamaIndexInvocation(BaseSpan): ) = PrivateAttr() _workflow_agents: dict[str, BaseWorkflowAgent] = PrivateAttr() _workflow_agents_by_run_id: dict[str, BaseWorkflowAgent] = PrivateAttr() - _workflow_invocations_by_run_id: dict[str, AgentInvocation] = PrivateAttr() - _workflow_invocations_by_key: dict[tuple[str, str], AgentInvocation] = ( + _workflow_invocations_by_run_id: dict[str, LocalAgentInvocation] = ( PrivateAttr() ) - _workflow_agent_invocation: AgentInvocation | None = PrivateAttr() + _workflow_invocations_by_key: dict[ + tuple[str, str], LocalAgentInvocation + ] = PrivateAttr() + _workflow_agent_invocation: LocalAgentInvocation | None = PrivateAttr() _workflow_handoff: bool = PrivateAttr() _workflow_agent_context_token: Token[Context] | None = PrivateAttr() _tool_parent_context_token: Token[Context] | None = PrivateAttr() @@ -522,7 +526,7 @@ class _LlamaIndexInvocation(BaseSpan): _workflow_tool_counts: dict[str, int] = PrivateAttr() _workflow_return_direct_runs: set[str] = PrivateAttr() _workflow_tool_errors: dict[str, BaseException] = PrivateAttr() - _workflow_pending_handoffs: dict[str, AgentInvocation] = PrivateAttr() + _workflow_pending_handoffs: dict[str, LocalAgentInvocation] = PrivateAttr() def __init__( self, @@ -539,7 +543,7 @@ def __init__( workflow_agents: Mapping[str, BaseWorkflowAgent] | None = None, workflow_run_id: str | None = None, workflow_agent: BaseWorkflowAgent | None = None, - workflow_agent_invocation: AgentInvocation | None = None, + workflow_agent_invocation: LocalAgentInvocation | None = None, workflow_handoff: bool = False, tool_parent_context_token: Token[Context] | None = None, ) -> None: @@ -591,7 +595,7 @@ def workflow_agent_for_run_id( def workflow_invocation_for_run_id( self, run_id: str | None, agent_name: str | None = None - ) -> AgentInvocation | None: + ) -> LocalAgentInvocation | None: """Return the reusable member-agent invocation for a workflow run.""" if self._workflow_agent_invocation is not None: return self._workflow_agent_invocation @@ -602,13 +606,15 @@ def workflow_invocation_for_run_id( return self._workflow_invocations_by_run_id.get(run_id) def register_workflow_invocation( - self, run_id: str, agent_name: str, invocation: AgentInvocation + self, run_id: str, agent_name: str, invocation: LocalAgentInvocation ) -> None: """Keep one member-agent invocation open across workflow turns.""" self._workflow_invocations_by_run_id[run_id] = invocation self._workflow_invocations_by_key[(run_id, agent_name)] = invocation - def remove_workflow_invocation(self, invocation: AgentInvocation) -> None: + def remove_workflow_invocation( + self, invocation: LocalAgentInvocation + ) -> None: """Forget a completed member invocation so a later turn can restart it.""" for key, value in list(self._workflow_invocations_by_key.items()): if value is invocation: @@ -666,7 +672,7 @@ def expect_workflow_tools(self, run_id: str, count: int) -> None: def set_return_direct_agent_output( self, run_id: str, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, tool_output: ToolOutput, ) -> None: """Keep the first successful return-direct result to arrive.""" @@ -698,12 +704,12 @@ def release_workflow_tool(self, run_id: str | None) -> bool: return True def set_pending_handoff( - self, run_id: str, invocation: AgentInvocation + self, run_id: str, invocation: LocalAgentInvocation ) -> None: """Hold a handing-off agent open until its whole turn has drained.""" self._workflow_pending_handoffs[run_id] = invocation - def take_pending_handoff(self, run_id: str) -> AgentInvocation | None: + def take_pending_handoff(self, run_id: str) -> LocalAgentInvocation | None: """Claim the handing-off agent owed a close, if there is one.""" return self._workflow_pending_handoffs.pop(run_id, None) @@ -739,7 +745,7 @@ def finalize_workflow_agents( if name == agent_name: _set_agent_step_output(invocation, output) break - invocations: list[AgentInvocation] = [] + invocations: list[LocalAgentInvocation] = [] for candidate in self._workflow_invocations_by_key.values(): if all(candidate is not existing for existing in invocations): invocations.append(candidate) @@ -791,7 +797,7 @@ def new_span( workflow_agents: Mapping[str, BaseWorkflowAgent] | None = None workflow_run_id: str | None = None workflow_agent: BaseWorkflowAgent | None = None - workflow_agent_invocation: AgentInvocation | None = None + workflow_agent_invocation: LocalAgentInvocation | None = None workflow_handoff = False member_agent_step = False workflow_tool_token: ( @@ -926,7 +932,7 @@ def new_span( else None ) if active_invocation is None and parent is not None: - if isinstance(parent._invocation, AgentInvocation): + if isinstance(parent._invocation, LocalAgentInvocation): active_invocation = parent._invocation workflow_run_id = ( tags.get("llamaindex.run_id") if tags is not None else None @@ -1050,7 +1056,7 @@ def _expect_workflow_tools( def _release_workflow_invocation( self, span: _LlamaIndexInvocation, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, ) -> None: """Drop a finished member invocation so a later turn opens a new span.""" parent = self.open_spans.get(span.parent_id or "") @@ -1119,7 +1125,7 @@ def prepare_to_exit_span( span.finalize_workflow_agents( result=result if capture_content else None ) - elif isinstance(span._invocation, AgentInvocation): + elif isinstance(span._invocation, LocalAgentInvocation): span.reset_tool_attributes() span.reset_workflow_tool() if self._handler.should_capture_content(): @@ -1202,14 +1208,14 @@ def prepare_to_drop_span( span._invocation.stop() else: span._invocation.fail(err) - elif isinstance(span._invocation, AgentInvocation): + elif isinstance(span._invocation, LocalAgentInvocation): _finish_member_agent(span._invocation, err) elif err is None: span._invocation.stop() else: span._invocation.fail(err) span.reset_tool_parent_context() - if isinstance(span._invocation, AgentInvocation): + if isinstance(span._invocation, LocalAgentInvocation): span.reset_workflow_agent() self._release_workflow_invocation(span, span._invocation) elif isinstance(span._invocation, ToolInvocation): diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py index f91082026..acc9250fb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py @@ -28,7 +28,10 @@ class AgentScenario(Scenario): expected_spans = {"invoke_agent": 3, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py index 0d60c3603..0f666f306 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py @@ -31,8 +31,9 @@ class WorkflowScenario(Scenario): "execute_tool": 1, } expected_metrics = ( - "gen_ai.client.operation.duration", "gen_ai.invoke_workflow.duration", + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", ) def run( diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py index bc6d94f7d..390c6ec20 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py @@ -76,7 +76,8 @@ class OrchestrationScenario(Scenario): "execute_tool": 1, } expected_metrics = ( - "gen_ai.client.operation.duration", + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", "gen_ai.invoke_workflow.duration", ) expected_violations = ( diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py index e0c382510..e1f817a03 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py @@ -27,7 +27,7 @@ find_tool_call_id, ) from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.invocation import AgentInvocation +from opentelemetry.util.genai.invocation import LocalAgentInvocation from opentelemetry.util.genai.stream import SyncStreamWrapper @@ -41,7 +41,7 @@ class _AgentRunStreamWrapper(SyncStreamWrapper[Any]): def __init__( self, stream: Any, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, capture_content: bool, ) -> None: super().__init__(stream) diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py index 136b89c25..ae551d96c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, cast from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.invocation import AgentInvocation +from opentelemetry.util.genai.invocation import LocalAgentInvocation from opentelemetry.util.genai.types import ( InputMessage, MessagePart, @@ -269,7 +269,7 @@ def create_agent_invocation( handler: TelemetryHandler, agent_instance: Any, messages: QwenMessage | list[QwenMessage] | None, -) -> AgentInvocation: +) -> LocalAgentInvocation: """Create and start an AgentInvocation for Agent.run().""" llm_instance = getattr(agent_instance, "llm", None) agent_name = ( diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py index f6a7dd708..34b50a84a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py @@ -54,7 +54,10 @@ class InvokeAgentScenario(Scenario): # invoke_agent span. LLM call (chat) spans are left to the underlying # model client library's instrumentation and are not emitted here. expected_spans = {"invoke_agent": 2, "execute_tool": 2} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed new file mode 100644 index 000000000..2b72bfc3f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed @@ -0,0 +1 @@ +Record metric `gen_ai.invoke_agent.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 0889ca845..7509ae115 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -34,9 +34,9 @@ ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, GenAIInvocation, InferenceInvocation, + LocalAgentInvocation, ) from opentelemetry.util.genai.stream import SyncStreamWrapper from opentelemetry.util.genai.types import OutputMessage, Role, TextPart @@ -422,7 +422,7 @@ def wrapper( def _record_run_answer( - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, output: object, *, @@ -454,7 +454,7 @@ class _AgentRunStreamWrapper(SyncStreamWrapper[_RunStreamChunk]): def __init__( self, stream: Generator[_RunStreamChunk, None, None], - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, *, capture_content: bool, @@ -522,7 +522,7 @@ def _on_stream_error(self, error: BaseException) -> None: def _record_agent( - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, bound: dict[str, Any], *, @@ -540,7 +540,7 @@ def _record_agent( def _record_agent_run( - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, bound: dict[str, Any], result: object, diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py index 2cffd623e..c17fafaf6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py @@ -23,7 +23,7 @@ class AgentScenario(Scenario): # The model call goes to the OpenAI SDK, whose instrumentation is not # enabled here. Tool calls are not instrumented yet. expected_spans = {"invoke_agent": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ("gen_ai.invoke_agent.duration",) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py index e82476419..6048291ff 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py @@ -353,7 +353,7 @@ def test_streaming_run_records_no_chunk_metrics( list(agent.run("Test question", stream=True)) metrics = metrics_by_name(metric_reader) - assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert "gen_ai.invoke_agent.duration" in metrics assert ( gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK not in metrics @@ -389,9 +389,9 @@ def test_agent_run_metrics( agent.run("Test question") metrics = metrics_by_name(metric_reader) - duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + duration = metrics["gen_ai.invoke_agent.duration"] assert { - GenAI.GEN_AI_OPERATION_NAME: "invoke_agent", + GenAI.GEN_AI_AGENT_NAME: "CodeAgent", GenAI.GEN_AI_REQUEST_MODEL: "fake-model", } in data_point_attributes(duration) # A run reports no token counts of its own: each model call records its diff --git a/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py b/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py index a4ce7d7ff..b5459eaf5 100644 --- a/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py +++ b/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py @@ -86,8 +86,8 @@ class Stream: ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, InferenceInvocation, + RemoteAgentInvocation, ) from opentelemetry.util.genai.stream import ( AsyncStreamWrapper, @@ -135,7 +135,7 @@ def _set_co_filename(wrapped: object) -> None: def _apply_interaction_response_attributes( response: Interaction, - invocation: InferenceInvocation | AgentInvocation, + invocation: InferenceInvocation | RemoteAgentInvocation, telemetry_handler: TelemetryHandler, ) -> None: if isinstance(invocation, InferenceInvocation): @@ -297,7 +297,7 @@ class InteractionsStreamWrapper(SyncStreamWrapper[InteractionSSEEvent]): def __init__( self, stream: Iterable[InteractionSSEEvent], - invocation: InferenceInvocation | AgentInvocation, + invocation: InferenceInvocation | RemoteAgentInvocation, telemetry_handler: TelemetryHandler, ) -> None: super().__init__(stream) @@ -329,7 +329,7 @@ class AsyncInteractionsStreamWrapper(AsyncStreamWrapper[InteractionSSEEvent]): def __init__( self, stream: AsyncIterable[InteractionSSEEvent], - invocation: InferenceInvocation | AgentInvocation, + invocation: InferenceInvocation | RemoteAgentInvocation, telemetry_handler: TelemetryHandler, ) -> None: super().__init__(stream) @@ -414,7 +414,7 @@ def _start_interactions_invocation( telemetry_handler: TelemetryHandler, instance: InteractionsResource | AsyncInteractionsResource, kwargs: dict[str, Any], -) -> InferenceInvocation | AgentInvocation: +) -> InferenceInvocation | RemoteAgentInvocation: # Vertex AI does not support the interactions API yet, but eventually will. # SDK will raise an exception if model or agent is not passed or if input data is not passed. is_vertex, server_address = _get_client_info(instance) @@ -424,7 +424,7 @@ def _start_interactions_invocation( else GenAIAttributes.GenAiSystemValues.GEMINI.value ) if agent := kwargs.get("agent"): - invocation: InferenceInvocation | AgentInvocation = ( + invocation: InferenceInvocation | RemoteAgentInvocation = ( telemetry_handler.invoke_remote_agent( provider=provider, request_model=kwargs.get("model"), diff --git a/util/opentelemetry-util-genai/.changelog/616.added b/util/opentelemetry-util-genai/.changelog/616.added new file mode 100644 index 000000000..8b5f55a88 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/616.added @@ -0,0 +1 @@ +Add `LocalAgentInvocation` and `RemoteAgentInvocation` for in-process and remote agent invocations. diff --git a/util/opentelemetry-util-genai/.changelog/616.changed b/util/opentelemetry-util-genai/.changelog/616.changed new file mode 100644 index 000000000..7aad107a4 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/616.changed @@ -0,0 +1 @@ +Record `gen_ai.invoke_agent.duration` on local agent invocations. diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index c95e83535..a8bbb45ff 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -3,6 +3,10 @@ from __future__ import annotations +import timeit +from abc import ABC, abstractmethod +from typing import Final + from opentelemetry._logs import Logger from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, @@ -26,16 +30,23 @@ from opentelemetry.util.genai.utils import ContentCapturingMode from opentelemetry.util.types import AttributeValue +_GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS: Final = ( + "gen_ai.usage.cache_write.input_tokens" +) +_GEN_AI_REQUEST_PREVIOUS_RESPONSE_ID: Final = ( + "gen_ai.request.previous_response.id" +) + -class AgentInvocation(GenAIInvocation): - """Represents a single agent invocation (invoke_agent span). +class AgentInvocation(GenAIInvocation, ABC): + """Base class representing a GenAI agent invocation (invoke_agent span). Use handler.invoke_local_agent() or handler.invoke_remote_agent() rather than constructing this directly. Reference: - Client span: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span - Internal span: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span + Client span: https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span + Internal span: https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span """ def __init__( @@ -45,15 +56,11 @@ def __init__( logger: Logger, completion_hook: CompletionHook, *, - provider: str | None = None, - span_kind: SpanKind = SpanKind.INTERNAL, + span_kind: SpanKind, request_model: str | None = None, - server_address: str | None = None, - server_port: int | None = None, agent_name: str | None = None, content_capturing_mode: ContentCapturingMode | None = None, ) -> None: - """Use handler.invoke_local_agent() or handler.invoke_remote_agent() instead of calling this directly.""" _operation_name = GenAI.GenAiOperationNameValues.INVOKE_AGENT.value super().__init__( tracer, @@ -67,15 +74,9 @@ def __init__( span_kind=span_kind, content_capturing_mode=content_capturing_mode, ) - self._provider: str | None = provider self._request_model: str | None = request_model - self._server_address: str | None = server_address - self._server_port: int | None = server_port - self._agent_name: str | None = agent_name - self.agent_id: str | None = None self.agent_description: str | None = None - self.agent_version: str | None = None self.conversation_id: str | None = None self.data_source_id: str | None = None @@ -94,8 +95,6 @@ def __init__( self.input_tokens: int | None = None self.output_tokens: int | None = None - self._cache_write_input_tokens: int | None = None - self._cache_read_input_tokens: int | None = None self.input_messages: list[InputMessage] = [] self.output_messages: list[OutputMessage] = [] @@ -105,72 +104,14 @@ def __init__( """System instructions for the agent. Passing ``MessagePart`` is deprecated; use ``SystemInstructionPart``.""" self.tool_definitions: list[ToolDefinition] | None = None - self._start(self._get_start_attributes()) - - @property - def cache_write_input_tokens(self) -> int | None: - """The number of cache write input tokens. - - .. deprecated:: 1.3b0 - Cache tokens are not reported on internal agent spans per semantic conventions. - """ - return self._cache_write_input_tokens - - @cache_write_input_tokens.setter - def cache_write_input_tokens(self, value: int | None) -> None: - self._cache_write_input_tokens = value - - @property - def cache_creation_input_tokens(self) -> int | None: - """The number of cache creation input tokens. - - .. deprecated:: 1.3b0 - Cache tokens are not reported on internal agent spans per semantic conventions. - """ - return self._cache_write_input_tokens - - @cache_creation_input_tokens.setter - def cache_creation_input_tokens(self, value: int | None) -> None: - self._cache_write_input_tokens = value - - @property - def cache_read_input_tokens(self) -> int | None: - """The number of cache read input tokens. - - .. deprecated:: 1.3b0 - Cache tokens are not reported on internal agent spans per semantic conventions. - """ - return self._cache_read_input_tokens - - @cache_read_input_tokens.setter - def cache_read_input_tokens(self, value: int | None) -> None: - self._cache_read_input_tokens = value - @property def agent_name(self) -> str | None: """The agent name provided at construction time.""" return self._agent_name - def _get_start_attributes(self) -> dict[str, AttributeValue]: - """Return sampling-relevant attributes available at span creation time.""" - optional_attrs = ( - (GenAI.GEN_AI_REQUEST_MODEL, self._request_model), - (GenAI.GEN_AI_AGENT_NAME, self._agent_name), - (server_attributes.SERVER_ADDRESS, self._server_address), - (server_attributes.SERVER_PORT, self._server_port), - (GenAI.GEN_AI_PROVIDER_NAME, self._provider), - ) - return { - GenAI.GEN_AI_OPERATION_NAME: self._operation_name, - **{k: v for k, v in optional_attrs if v is not None}, - } - def _get_agent_attributes(self) -> dict[str, AttributeValue]: - """Return agent attributes not known at span creation time.""" optional_attrs = ( - (GenAI.GEN_AI_AGENT_ID, self.agent_id), (GenAI.GEN_AI_AGENT_DESCRIPTION, self.agent_description), - (GenAI.GEN_AI_AGENT_VERSION, self.agent_version), ) return {k: v for k, v in optional_attrs if v is not None} @@ -212,6 +153,204 @@ def _get_content_attributes_for_span(self) -> dict[str, AttributeValue]: content_capturing_mode=self._content_capturing_mode, ) + def _apply_finish(self, error: Error | None = None) -> None: + if error is not None: + self._apply_error_attributes(error) + + attributes: dict[str, AttributeValue] = {} + attributes.update(self._get_agent_attributes()) + attributes.update(self._get_request_attributes()) + attributes.update(self._get_response_attributes()) + attributes.update(self._get_usage_attributes()) + attributes.update(self._get_content_attributes_for_span()) + attributes.update(self.attributes) + self.span.set_attributes(attributes) + self._call_completion_hook( + inputs=self.input_messages, + outputs=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + ) + self._record_metrics() + + @abstractmethod + def _record_metrics(self) -> None: + """Record invocation metrics.""" + + +class LocalAgentInvocation(AgentInvocation): + """Represents an in-process agent invocation (INTERNAL span kind). + + Use handler.invoke_local_agent() rather than constructing this directly. + + Reference: + https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span + """ + + def __init__( + self, + tracer: Tracer, + instruments: _Instruments, + logger: Logger, + completion_hook: CompletionHook, + *, + request_model: str | None = None, + agent_name: str | None = None, + content_capturing_mode: ContentCapturingMode | None = None, + ) -> None: + super().__init__( + tracer, + instruments, + logger, + completion_hook, + span_kind=SpanKind.INTERNAL, + request_model=request_model, + agent_name=agent_name, + content_capturing_mode=content_capturing_mode, + ) + self._start(self._get_start_attributes()) + + def _get_start_attributes(self) -> dict[str, AttributeValue]: + optional_attrs = ( + (GenAI.GEN_AI_REQUEST_MODEL, self._request_model), + (GenAI.GEN_AI_AGENT_NAME, self._agent_name), + ) + return { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + **{k: v for k, v in optional_attrs if v is not None}, + } + + def _get_metric_attributes(self) -> dict[str, AttributeValue]: + attrs: dict[str, AttributeValue] = {} + if self._agent_name is not None: + attrs[GenAI.GEN_AI_AGENT_NAME] = self._agent_name + if self._request_model is not None: + attrs[GenAI.GEN_AI_REQUEST_MODEL] = self._request_model + attrs.update(self.metric_attributes) + return attrs + + def _record_metrics(self) -> None: + duration_seconds = max( + timeit.default_timer() - self._monotonic_start_s, + 0.0, + ) + self._instruments.invoke_agent_duration.record( + duration_seconds, + attributes=self._get_metric_attributes(), + context=self._span_context, + ) + + +class RemoteAgentInvocation(AgentInvocation): + """Represents a remote agent invocation (CLIENT span kind). + + Use handler.invoke_remote_agent() rather than constructing this directly. + + Reference: + https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span + """ + + def __init__( + self, + tracer: Tracer, + instruments: _Instruments, + logger: Logger, + completion_hook: CompletionHook, + provider: str, + *, + request_model: str | None = None, + server_address: str | None = None, + server_port: int | None = None, + agent_name: str | None = None, + agent_id: str | None = None, + agent_version: str | None = None, + content_capturing_mode: ContentCapturingMode | None = None, + ) -> None: + super().__init__( + tracer, + instruments, + logger, + completion_hook, + span_kind=SpanKind.CLIENT, + request_model=request_model, + agent_name=agent_name, + content_capturing_mode=content_capturing_mode, + ) + self._provider: str = provider + self._server_address: str | None = server_address + self._server_port: int | None = server_port + + self.agent_id: str | None = agent_id + self.agent_version: str | None = agent_version + self.previous_response_id: str | None = None + self._cache_write_input_tokens: int | None = None + self.cache_read_input_tokens: int | None = None + + self._start(self._get_start_attributes()) + + @property + def cache_write_input_tokens(self) -> int | None: + """The number of cache write input tokens.""" + return self._cache_write_input_tokens + + @cache_write_input_tokens.setter + def cache_write_input_tokens(self, value: int | None) -> None: + self._cache_write_input_tokens = value + + @property + def cache_creation_input_tokens(self) -> int | None: + """The number of cache creation input tokens. + + .. deprecated:: 1.3b0 + Use :attr:`cache_write_input_tokens` instead. + """ + return self._cache_write_input_tokens + + @cache_creation_input_tokens.setter + def cache_creation_input_tokens(self, value: int | None) -> None: + self._cache_write_input_tokens = value + + def _get_start_attributes(self) -> dict[str, AttributeValue]: + optional_attrs = ( + (GenAI.GEN_AI_REQUEST_MODEL, self._request_model), + (GenAI.GEN_AI_AGENT_NAME, self._agent_name), + (server_attributes.SERVER_ADDRESS, self._server_address), + (server_attributes.SERVER_PORT, self._server_port), + (GenAI.GEN_AI_PROVIDER_NAME, self._provider), + ) + return { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + **{k: v for k, v in optional_attrs if v is not None}, + } + + def _get_agent_attributes(self) -> dict[str, AttributeValue]: + optional_attrs = ( + (GenAI.GEN_AI_AGENT_ID, self.agent_id), + (GenAI.GEN_AI_AGENT_DESCRIPTION, self.agent_description), + (GenAI.GEN_AI_AGENT_VERSION, self.agent_version), + ) + return {k: v for k, v in optional_attrs if v is not None} + + def _get_request_attributes(self) -> dict[str, AttributeValue]: + attrs = super()._get_request_attributes() + if self.previous_response_id is not None: + attrs[_GEN_AI_REQUEST_PREVIOUS_RESPONSE_ID] = ( + self.previous_response_id + ) + return attrs + + def _get_usage_attributes(self) -> dict[str, AttributeValue]: + attrs = super()._get_usage_attributes() + if self.cache_write_input_tokens is not None: + attrs[_GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS] = ( + self.cache_write_input_tokens + ) + if self.cache_read_input_tokens is not None: + attrs[GenAI.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = ( + self.cache_read_input_tokens + ) + return attrs + def _get_metric_attributes(self) -> dict[str, AttributeValue]: optional_attrs = ( (GenAI.GEN_AI_PROVIDER_NAME, self._provider), @@ -236,22 +375,5 @@ def _get_metric_token_counts(self) -> dict[str, int]: ) return counts - def _apply_finish(self, error: Error | None = None) -> None: - if error is not None: - self._apply_error_attributes(error) - - attributes: dict[str, AttributeValue] = {} - attributes.update(self._get_agent_attributes()) - attributes.update(self._get_request_attributes()) - attributes.update(self._get_response_attributes()) - attributes.update(self._get_usage_attributes()) - attributes.update(self._get_content_attributes_for_span()) - attributes.update(self.attributes) - self.span.set_attributes(attributes) - self._call_completion_hook( - inputs=self.input_messages, - outputs=self.output_messages, - system_instruction=self.system_instruction, - tool_definitions=self.tool_definitions, - ) + def _record_metrics(self) -> None: self._record_client_metrics() diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py index f3ab9dbde..ed43e85a0 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py @@ -62,6 +62,22 @@ _GEN_AI_EXECUTE_TOOL_DURATION: Final = "gen_ai.execute_tool.duration" _GEN_AI_INVOKE_WORKFLOW_DURATION: Final = "gen_ai.invoke_workflow.duration" +_GEN_AI_INVOKE_AGENT_DURATION: Final = "gen_ai.invoke_agent.duration" +_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS: Final = [ + 0.1, + 0.2, + 0.4, + 0.8, + 1.6, + 3.2, + 6.4, + 12.8, + 25.6, + 51.2, + 102.4, + 204.8, + 409.6, +] class _Instruments: @@ -104,6 +120,12 @@ def __init__(self, meter: Meter) -> None: unit="s", explicit_bucket_boundaries_advisory=_GEN_AI_INVOKE_WORKFLOW_DURATION_BUCKETS, ) + self.invoke_agent_duration: Histogram = meter.create_histogram( + name=_GEN_AI_INVOKE_AGENT_DURATION, + description="Measures the duration of an in-process agent invocation.", + unit="s", + explicit_bucket_boundaries_advisory=_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS, + ) __all__ = ["_Instruments"] diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py index a9db0dafa..f36304a7e 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py @@ -44,11 +44,9 @@ from opentelemetry.metrics import Meter, MeterProvider, get_meter from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import ( - SpanKind, TracerProvider, get_tracer, ) -from opentelemetry.util.genai._agent_invocation import AgentInvocation from opentelemetry.util.genai._inference_invocation import LLMInvocation from opentelemetry.util.genai._instruments import _Instruments from opentelemetry.util.genai._invocation import Error @@ -57,9 +55,12 @@ _NoOpCompletionHook, ) from opentelemetry.util.genai.invocation import ( + AgentInvocation, EmbeddingInvocation, FetchResponseInvocation, InferenceInvocation, + LocalAgentInvocation, + RemoteAgentInvocation, RetrievalInvocation, ToolInvocation, WorkflowInvocation, @@ -462,12 +463,11 @@ def start_invoke_local_agent( Set remaining attributes (agent_name, etc.) on the returned invocation, then call invocation.stop() or invocation.fail(). """ - return AgentInvocation( + return LocalAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, - span_kind=SpanKind.INTERNAL, request_model=request_model, agent_name=agent_name, content_capturing_mode=self._content_capturing_mode, @@ -492,13 +492,12 @@ def start_invoke_remote_agent( Set remaining attributes (agent_name, etc.) on the returned invocation, then call invocation.stop() or invocation.fail(). """ - return AgentInvocation( + return RemoteAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, provider=provider, - span_kind=SpanKind.CLIENT, request_model=request_model, agent_name=agent_name, server_address=server_address, @@ -511,7 +510,7 @@ def invoke_local_agent( *, request_model: str | None = None, agent_name: str | None = None, - ) -> AgentInvocation: + ) -> LocalAgentInvocation: """Returns an agent invocation (INTERNAL span kind). Starts span when called. Returned object can be used as a ContextManager which automatically calls `stop` or `fail` @@ -522,12 +521,11 @@ def invoke_local_agent( Only set data attributes on the invocation object, do not modify the span or context. """ - return AgentInvocation( + return LocalAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, - span_kind=SpanKind.INTERNAL, request_model=request_model, agent_name=agent_name, content_capturing_mode=self._content_capturing_mode, @@ -541,7 +539,9 @@ def invoke_remote_agent( server_address: str | None = None, server_port: int | None = None, agent_name: str | None = None, - ) -> AgentInvocation: + agent_id: str | None = None, + agent_version: str | None = None, + ) -> RemoteAgentInvocation: """Returns an agent invocation (CLIENT span kind). Starts span when called. Returned object can be used as a ContextManager which automatically calls `stop` or `fail` @@ -552,15 +552,16 @@ def invoke_remote_agent( Only set data attributes on the invocation object, do not modify the span or context. """ - return AgentInvocation( + return RemoteAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, provider=provider, - span_kind=SpanKind.CLIENT, request_model=request_model, agent_name=agent_name, + agent_id=agent_id, + agent_version=agent_version, server_address=server_address, server_port=server_port, content_capturing_mode=self._content_capturing_mode, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py index 37b27823d..3d345d3fc 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py @@ -17,7 +17,11 @@ ) """ -from opentelemetry.util.genai._agent_invocation import AgentInvocation +from opentelemetry.util.genai._agent_invocation import ( + AgentInvocation, + LocalAgentInvocation, + RemoteAgentInvocation, +) from opentelemetry.util.genai._embedding_invocation import EmbeddingInvocation from opentelemetry.util.genai._fetch_response_invocation import ( FetchResponseInvocation, @@ -40,6 +44,8 @@ "FetchResponseInvocation", "GenAIInvocation", "InferenceInvocation", + "LocalAgentInvocation", + "RemoteAgentInvocation", "RetrievalInvocation", "ToolInvocation", "WorkflowInvocation", diff --git a/util/opentelemetry-util-genai/tests/test_handler_agent.py b/util/opentelemetry-util-genai/tests/test_handler_agent.py index eb3ad3c19..18a0c5db6 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_agent.py +++ b/util/opentelemetry-util-genai/tests/test_handler_agent.py @@ -23,6 +23,11 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, ) from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import ( + AgentInvocation, + LocalAgentInvocation, + RemoteAgentInvocation, +) from opentelemetry.util.genai.types import ( ContentCapturingMode, Error, @@ -47,6 +52,8 @@ def test_start_stop_creates_span(self): request_model="gpt-4", agent_name="Math Tutor", ) + assert isinstance(invocation, LocalAgentInvocation) + assert isinstance(invocation, AgentInvocation) invocation.stop() spans = self.span_exporter.get_finished_spans() @@ -96,9 +103,7 @@ def test_all_attributes(self): request_model="gpt-4", agent_name="Full Agent", ) - invocation.agent_id = "agent-123" invocation.agent_description = "A test agent" - invocation.agent_version = "1.0.0" invocation.conversation_id = "conv-456" invocation.data_source_id = "ds-789" invocation.output_type = "text" @@ -117,9 +122,9 @@ def test_all_attributes(self): attrs = self.span_exporter.get_finished_spans()[0].attributes assert attrs[GenAI.GEN_AI_AGENT_NAME] == "Full Agent" - assert attrs[GenAI.GEN_AI_AGENT_ID] == "agent-123" + assert GenAI.GEN_AI_AGENT_ID not in attrs assert attrs[GenAI.GEN_AI_AGENT_DESCRIPTION] == "A test agent" - assert attrs[GenAI.GEN_AI_AGENT_VERSION] == "1.0.0" + assert GenAI.GEN_AI_AGENT_VERSION not in attrs assert attrs[GenAI.GEN_AI_USAGE_INPUT_TOKENS] == 100 assert attrs[GenAI.GEN_AI_USAGE_OUTPUT_TOKENS] == 200 assert attrs[GenAI.GEN_AI_CONVERSATION_ID] == "conv-456" @@ -206,16 +211,21 @@ def test_context_manager_default_invocation(self): def test_default_values(self): invocation = self.handler.invoke_local_agent() invocation.stop() + assert isinstance(invocation, LocalAgentInvocation) + assert isinstance(invocation, AgentInvocation) assert invocation._operation_name == "invoke_agent" assert invocation.agent_name is None assert invocation._request_model is None assert not invocation.input_messages assert not invocation.output_messages assert invocation.tool_definitions is None - assert invocation.cache_creation_input_tokens is None - assert invocation.cache_read_input_tokens is None assert invocation.span is not INVALID_SPAN assert not invocation.attributes + assert not hasattr(invocation, "agent_id") + assert not hasattr(invocation, "agent_version") + assert not hasattr(invocation, "previous_response_id") + assert not hasattr(invocation, "cache_write_input_tokens") + assert not hasattr(invocation, "cache_read_input_tokens") def test_with_messages(self): invocation = self.handler.invoke_local_agent() @@ -457,11 +467,56 @@ def setUp(self): def test_span_kind_client(self): invocation = self.handler.invoke_remote_agent("openai") + assert isinstance(invocation, RemoteAgentInvocation) + assert isinstance(invocation, AgentInvocation) invocation.stop() assert ( self.span_exporter.get_finished_spans()[0].kind == SpanKind.CLIENT ) + def test_default_values(self): + invocation = self.handler.invoke_remote_agent("openai") + invocation.stop() + assert isinstance(invocation, RemoteAgentInvocation) + assert isinstance(invocation, AgentInvocation) + assert invocation._operation_name == "invoke_agent" + assert invocation.agent_name is None + assert invocation._request_model is None + assert invocation.agent_id is None + assert invocation.agent_version is None + assert invocation.previous_response_id is None + assert invocation.cache_write_input_tokens is None + assert invocation.cache_creation_input_tokens is None + assert invocation.cache_read_input_tokens is None + assert not invocation.input_messages + assert not invocation.output_messages + assert invocation.tool_definitions is None + assert invocation.span is not INVALID_SPAN + assert not invocation.attributes + + def test_constructor_agent_id_and_version(self): + invocation = self.handler.invoke_remote_agent( + "openai", + agent_id="agent-999", + agent_version="2.0.0", + ) + invocation.stop() + attrs = self.span_exporter.get_finished_spans()[0].attributes + assert attrs[GenAI.GEN_AI_AGENT_ID] == "agent-999" + assert attrs[GenAI.GEN_AI_AGENT_VERSION] == "2.0.0" + + def test_cache_token_attributes(self): + invocation = self.handler.invoke_remote_agent("openai") + invocation.input_tokens = 100 + invocation.cache_creation_input_tokens = 25 + invocation.cache_read_input_tokens = 50 + invocation.stop() + + attrs = self.span_exporter.get_finished_spans()[0].attributes + assert attrs[GenAI.GEN_AI_USAGE_INPUT_TOKENS] == 100 + assert attrs["gen_ai.usage.cache_write.input_tokens"] == 25 + assert attrs[GenAI.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 50 + def test_server_attributes(self): invocation = self.handler.invoke_remote_agent( "openai", @@ -484,8 +539,11 @@ def test_all_attributes(self): invocation.agent_id = "agent-123" invocation.agent_description = "A remote test agent" invocation.agent_version = "1.0.0" + invocation.previous_response_id = "resp_123" invocation.input_tokens = 100 invocation.output_tokens = 200 + invocation.cache_write_input_tokens = 30 + invocation.cache_read_input_tokens = 15 invocation.stop() attrs = self.span_exporter.get_finished_spans()[0].attributes @@ -493,8 +551,11 @@ def test_all_attributes(self): assert attrs[GenAI.GEN_AI_AGENT_ID] == "agent-123" assert attrs[GenAI.GEN_AI_AGENT_DESCRIPTION] == "A remote test agent" assert attrs[GenAI.GEN_AI_AGENT_VERSION] == "1.0.0" + assert attrs["gen_ai.request.previous_response.id"] == "resp_123" assert attrs[GenAI.GEN_AI_USAGE_INPUT_TOKENS] == 100 assert attrs[GenAI.GEN_AI_USAGE_OUTPUT_TOKENS] == 200 + assert attrs["gen_ai.usage.cache_write.input_tokens"] == 30 + assert attrs[GenAI.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 15 assert attrs[GenAI.GEN_AI_REQUEST_MODEL] == "gpt-4" def test_fail_sets_error_status(self): @@ -589,45 +650,26 @@ def test_local_agent_records_duration_and_tokens(self) -> None: meter_provider=self.meter_provider, ) with patch("timeit.default_timer", return_value=1000.0): - invocation = handler.invoke_local_agent(request_model="model") - invocation.input_tokens = 5 - invocation.output_tokens = 7 + invocation = handler.invoke_local_agent( + request_model="model", agent_name="LocalAgent" + ) with patch("timeit.default_timer", return_value=1002.0): invocation.stop() metrics = self._harvest_metrics() - self.assertIn("gen_ai.client.operation.duration", metrics) - duration_points = metrics["gen_ai.client.operation.duration"] + self.assertIn("gen_ai.invoke_agent.duration", metrics) + duration_points = metrics["gen_ai.invoke_agent.duration"] self.assertEqual(len(duration_points), 1) duration_point = duration_points[0] self.assertEqual( - duration_point.attributes[GenAI.GEN_AI_OPERATION_NAME], - GenAI.GenAiOperationNameValues.INVOKE_AGENT.value, + duration_point.attributes[GenAI.GEN_AI_AGENT_NAME], "LocalAgent" ) self.assertEqual( duration_point.attributes[GenAI.GEN_AI_REQUEST_MODEL], "model" ) self.assertAlmostEqual(duration_point.sum, 2.0, places=3) - self.assertIn("gen_ai.client.token.usage", metrics) - token_points = metrics["gen_ai.client.token.usage"] - token_by_type = { - point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point - for point in token_points - } - self.assertEqual(len(token_by_type), 2) - self.assertAlmostEqual( - token_by_type[GenAI.GenAiTokenTypeValues.INPUT.value].sum, - 5.0, - places=3, - ) - self.assertAlmostEqual( - token_by_type[GenAI.GenAiTokenTypeValues.OUTPUT.value].sum, - 7.0, - places=3, - ) - def test_remote_agent_records_duration_with_server_attrs(self) -> None: handler = TelemetryHandler( tracer_provider=self.tracer_provider, @@ -636,19 +678,33 @@ def test_remote_agent_records_duration_with_server_attrs(self) -> None: invocation = handler.invoke_remote_agent( "prov", request_model="model", + agent_name="RemoteAgent", server_address="agent.example.com", server_port=443, ) - invocation.input_tokens = 10 invocation.stop() metrics = self._harvest_metrics() self.assertIn("gen_ai.client.operation.duration", metrics) duration_point = metrics["gen_ai.client.operation.duration"][0] self.assertEqual( - duration_point.attributes["server.address"], "agent.example.com" + duration_point.attributes[GenAI.GEN_AI_OPERATION_NAME], + "invoke_agent", + ) + self.assertEqual( + duration_point.attributes[GenAI.GEN_AI_PROVIDER_NAME], "prov" + ) + self.assertNotIn(GenAI.GEN_AI_AGENT_NAME, duration_point.attributes) + self.assertEqual( + duration_point.attributes[GenAI.GEN_AI_REQUEST_MODEL], "model" + ) + self.assertEqual( + duration_point.attributes[server_attributes.SERVER_ADDRESS], + "agent.example.com", + ) + self.assertEqual( + duration_point.attributes[server_attributes.SERVER_PORT], 443 ) - self.assertEqual(duration_point.attributes["server.port"], 443) def test_fail_agent_records_error_metric(self) -> None: handler = TelemetryHandler( @@ -656,19 +712,23 @@ def test_fail_agent_records_error_metric(self) -> None: meter_provider=self.meter_provider, ) with patch("timeit.default_timer", return_value=2000.0): - invocation = handler.invoke_local_agent(request_model="err-model") - invocation.input_tokens = 11 + invocation = handler.invoke_local_agent( + request_model="err-model", agent_name="ErrAgent" + ) error = Error(message="boom", type="ValueError") with patch("timeit.default_timer", return_value=2001.0): invocation.fail(error) metrics = self._harvest_metrics() - self.assertIn("gen_ai.client.operation.duration", metrics) - duration_point = metrics["gen_ai.client.operation.duration"][0] + self.assertIn("gen_ai.invoke_agent.duration", metrics) + duration_point = metrics["gen_ai.invoke_agent.duration"][0] self.assertEqual( duration_point.attributes.get("error.type"), "ValueError" ) + self.assertEqual( + duration_point.attributes.get(GenAI.GEN_AI_AGENT_NAME), "ErrAgent" + ) self.assertAlmostEqual(duration_point.sum, 1.0, places=3) def _harvest_metrics(self):