From caf4ec1ba9644d1408819bf0bd85d217d17793d9 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 3 Sep 2026 18:44:15 -0700 Subject: [PATCH 1/9] Split AgentInvocation into LocalAgentInvocation and RemoteAgentInvocation --- .../instrumentation/genai/agno/stream.py | 18 +- .../genai/langchain/callback_handler.py | 1 - .../tests/conformance/agent.py | 2 + .../tests/test_callback_handler.py | 3 +- .../tests/conformance/invoke_agent.py | 5 +- .../util/genai/_agent_invocation.py | 278 +++++++++++++----- .../opentelemetry/util/genai/_instruments.py | 22 ++ .../src/opentelemetry/util/genai/handler.py | 25 +- .../opentelemetry/util/genai/invocation.py | 8 +- .../tests/test_handler_agent.py | 136 ++++++--- 10 files changed, 364 insertions(+), 134 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py index a94177ee5..becd20bbb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py @@ -13,6 +13,7 @@ ) from opentelemetry.util.genai.invocation import ( AgentInvocation, + RemoteAgentInvocation, WorkflowInvocation, ) from opentelemetry.util.genai.stream import ( @@ -56,14 +57,15 @@ def _process_chunk(self, chunk: Any) -> None: self._self_agent_invocation.output_tokens = ( metrics.output_tokens ) - if getattr(metrics, "cache_read_tokens", None) is not None: - self._self_agent_invocation.cache_read_input_tokens = ( - metrics.cache_read_tokens - ) - if getattr(metrics, "cache_write_tokens", None) is not None: - self._self_agent_invocation.cache_write_input_tokens = ( - metrics.cache_write_tokens - ) + if isinstance(self._self_agent_invocation, RemoteAgentInvocation): + if getattr(metrics, "cache_read_tokens", None) is not None: + self._self_agent_invocation.cache_read_input_tokens = ( + metrics.cache_read_tokens + ) + if getattr(metrics, "cache_write_tokens", None) is not None: + self._self_agent_invocation.cache_write_input_tokens = ( + metrics.cache_write_tokens + ) event_name = str(getattr(chunk, "event", "")) chunk_type = type(chunk).__name__ diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index ceb4d387d..8b6c66867 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -173,7 +173,6 @@ def on_chain_start( agent.input_messages = make_input_message(inputs) if metadata: - agent.agent_id = metadata.get("agent_id") agent.agent_description = metadata.get( "agent_description" ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py index 802c1a5f0..d1980dd0a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py @@ -44,6 +44,8 @@ class AgentScenario(Scenario): expected_metrics = ( "gen_ai.client.operation.duration", "gen_ai.client.token.usage", + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", ) # langchain can't populate server.address on chat spans. # invoke_agent provider is unknown at span creation; ls_provider is only diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index c4c0632f0..214dea68e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -79,6 +79,7 @@ def _make_agent_inv_mock() -> mock.MagicMock: # agent_name is an instance attribute set in AgentInvocation.__init__ via the # constructor arg; pre-configure it so spec-restricted attribute access works. agent_inv.agent_name = None + agent_inv.agent_id = None return agent_inv @@ -256,7 +257,7 @@ def test_agent_metadata_set(self): }, ) - assert agent_inv.agent_id == "agent-123" + assert agent_inv.agent_id is None assert agent_inv.agent_description == "does math" assert agent_inv.conversation_id == "thread-abc" diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py index f6a7dd708..34b50a84a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/tests/conformance/invoke_agent.py @@ -54,7 +54,10 @@ class InvokeAgentScenario(Scenario): # invoke_agent span. LLM call (chat) spans are left to the underlying # model client library's instrumentation and are not emitted here. expected_spans = {"invoke_agent": 2, "execute_tool": 2} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) def run( self, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index c95e83535..567ae1935 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -3,6 +3,9 @@ from __future__ import annotations +import timeit +from typing import Final + from opentelemetry._logs import Logger from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, @@ -26,9 +29,16 @@ from opentelemetry.util.genai.utils import ContentCapturingMode from opentelemetry.util.types import AttributeValue +_GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS: Final = ( + "gen_ai.usage.cache_write.input_tokens" +) +_GEN_AI_REQUEST_PREVIOUS_RESPONSE_ID: Final = ( + "gen_ai.request.previous_response.id" +) + class AgentInvocation(GenAIInvocation): - """Represents a single agent invocation (invoke_agent span). + """Base class representing a GenAI agent invocation (invoke_agent span). Use handler.invoke_local_agent() or handler.invoke_remote_agent() rather than constructing this directly. @@ -45,15 +55,11 @@ def __init__( logger: Logger, completion_hook: CompletionHook, *, - provider: str | None = None, - span_kind: SpanKind = SpanKind.INTERNAL, + span_kind: SpanKind, request_model: str | None = None, - server_address: str | None = None, - server_port: int | None = None, agent_name: str | None = None, content_capturing_mode: ContentCapturingMode | None = None, ) -> None: - """Use handler.invoke_local_agent() or handler.invoke_remote_agent() instead of calling this directly.""" _operation_name = GenAI.GenAiOperationNameValues.INVOKE_AGENT.value super().__init__( tracer, @@ -67,15 +73,9 @@ def __init__( span_kind=span_kind, content_capturing_mode=content_capturing_mode, ) - self._provider: str | None = provider self._request_model: str | None = request_model - self._server_address: str | None = server_address - self._server_port: int | None = server_port - self._agent_name: str | None = agent_name - self.agent_id: str | None = None self.agent_description: str | None = None - self.agent_version: str | None = None self.conversation_id: str | None = None self.data_source_id: str | None = None @@ -94,8 +94,6 @@ def __init__( self.input_tokens: int | None = None self.output_tokens: int | None = None - self._cache_write_input_tokens: int | None = None - self._cache_read_input_tokens: int | None = None self.input_messages: list[InputMessage] = [] self.output_messages: list[OutputMessage] = [] @@ -105,72 +103,14 @@ def __init__( """System instructions for the agent. Passing ``MessagePart`` is deprecated; use ``SystemInstructionPart``.""" self.tool_definitions: list[ToolDefinition] | None = None - self._start(self._get_start_attributes()) - - @property - def cache_write_input_tokens(self) -> int | None: - """The number of cache write input tokens. - - .. deprecated:: 1.3b0 - Cache tokens are not reported on internal agent spans per semantic conventions. - """ - return self._cache_write_input_tokens - - @cache_write_input_tokens.setter - def cache_write_input_tokens(self, value: int | None) -> None: - self._cache_write_input_tokens = value - - @property - def cache_creation_input_tokens(self) -> int | None: - """The number of cache creation input tokens. - - .. deprecated:: 1.3b0 - Cache tokens are not reported on internal agent spans per semantic conventions. - """ - return self._cache_write_input_tokens - - @cache_creation_input_tokens.setter - def cache_creation_input_tokens(self, value: int | None) -> None: - self._cache_write_input_tokens = value - - @property - def cache_read_input_tokens(self) -> int | None: - """The number of cache read input tokens. - - .. deprecated:: 1.3b0 - Cache tokens are not reported on internal agent spans per semantic conventions. - """ - return self._cache_read_input_tokens - - @cache_read_input_tokens.setter - def cache_read_input_tokens(self, value: int | None) -> None: - self._cache_read_input_tokens = value - @property def agent_name(self) -> str | None: """The agent name provided at construction time.""" return self._agent_name - def _get_start_attributes(self) -> dict[str, AttributeValue]: - """Return sampling-relevant attributes available at span creation time.""" - optional_attrs = ( - (GenAI.GEN_AI_REQUEST_MODEL, self._request_model), - (GenAI.GEN_AI_AGENT_NAME, self._agent_name), - (server_attributes.SERVER_ADDRESS, self._server_address), - (server_attributes.SERVER_PORT, self._server_port), - (GenAI.GEN_AI_PROVIDER_NAME, self._provider), - ) - return { - GenAI.GEN_AI_OPERATION_NAME: self._operation_name, - **{k: v for k, v in optional_attrs if v is not None}, - } - def _get_agent_attributes(self) -> dict[str, AttributeValue]: - """Return agent attributes not known at span creation time.""" optional_attrs = ( - (GenAI.GEN_AI_AGENT_ID, self.agent_id), (GenAI.GEN_AI_AGENT_DESCRIPTION, self.agent_description), - (GenAI.GEN_AI_AGENT_VERSION, self.agent_version), ) return {k: v for k, v in optional_attrs if v is not None} @@ -212,6 +152,200 @@ def _get_content_attributes_for_span(self) -> dict[str, AttributeValue]: content_capturing_mode=self._content_capturing_mode, ) + +class LocalAgentInvocation(AgentInvocation): + """Represents an in-process agent invocation (INTERNAL span kind). + + Use handler.invoke_local_agent() rather than constructing this directly. + + Reference: + https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span + """ + + def __init__( + self, + tracer: Tracer, + instruments: _Instruments, + logger: Logger, + completion_hook: CompletionHook, + *, + request_model: str | None = None, + agent_name: str | None = None, + content_capturing_mode: ContentCapturingMode | None = None, + ) -> None: + super().__init__( + tracer, + instruments, + logger, + completion_hook, + span_kind=SpanKind.INTERNAL, + request_model=request_model, + agent_name=agent_name, + content_capturing_mode=content_capturing_mode, + ) + self._start(self._get_start_attributes()) + + def _get_start_attributes(self) -> dict[str, AttributeValue]: + optional_attrs = ( + (GenAI.GEN_AI_REQUEST_MODEL, self._request_model), + (GenAI.GEN_AI_AGENT_NAME, self._agent_name), + ) + return { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + **{k: v for k, v in optional_attrs if v is not None}, + } + + def _get_metric_attributes(self) -> dict[str, AttributeValue]: + attrs: dict[str, AttributeValue] = {} + if self._agent_name is not None: + attrs[GenAI.GEN_AI_AGENT_NAME] = self._agent_name + if self._request_model is not None: + attrs[GenAI.GEN_AI_REQUEST_MODEL] = self._request_model + attrs.update(self.metric_attributes) + return attrs + + def _apply_finish(self, error: Error | None = None) -> None: + if error is not None: + self._apply_error_attributes(error) + + attributes: dict[str, AttributeValue] = {} + attributes.update(self._get_agent_attributes()) + attributes.update(self._get_request_attributes()) + attributes.update(self._get_response_attributes()) + attributes.update(self._get_usage_attributes()) + attributes.update(self._get_content_attributes_for_span()) + attributes.update(self.attributes) + self.span.set_attributes(attributes) + self._call_completion_hook( + inputs=self.input_messages, + outputs=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + ) + self._record_metrics() + + def _record_metrics(self) -> None: + duration_seconds = max( + timeit.default_timer() - self._monotonic_start_s, + 0.0, + ) + self._instruments.invoke_agent_duration.record( + duration_seconds, + attributes=self._get_metric_attributes(), + context=self._span_context, + ) + + +class RemoteAgentInvocation(AgentInvocation): + """Represents a remote agent invocation (CLIENT span kind). + + Use handler.invoke_remote_agent() rather than constructing this directly. + + Reference: + https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span + """ + + def __init__( + self, + tracer: Tracer, + instruments: _Instruments, + logger: Logger, + completion_hook: CompletionHook, + provider: str, + *, + request_model: str | None = None, + server_address: str | None = None, + server_port: int | None = None, + agent_name: str | None = None, + agent_id: str | None = None, + agent_version: str | None = None, + content_capturing_mode: ContentCapturingMode | None = None, + ) -> None: + super().__init__( + tracer, + instruments, + logger, + completion_hook, + span_kind=SpanKind.CLIENT, + request_model=request_model, + agent_name=agent_name, + content_capturing_mode=content_capturing_mode, + ) + self._provider: str = provider + self._server_address: str | None = server_address + self._server_port: int | None = server_port + + self.agent_id: str | None = agent_id + self.agent_version: str | None = agent_version + self.previous_response_id: str | None = None + self._cache_write_input_tokens: int | None = None + self.cache_read_input_tokens: int | None = None + + self._start(self._get_start_attributes()) + + @property + def cache_write_input_tokens(self) -> int | None: + """The number of cache write input tokens.""" + return self._cache_write_input_tokens + + @cache_write_input_tokens.setter + def cache_write_input_tokens(self, value: int | None) -> None: + self._cache_write_input_tokens = value + + @property + def cache_creation_input_tokens(self) -> int | None: + """The number of cache creation input tokens. + + .. deprecated:: 1.3b0 + Use :attr:`cache_write_input_tokens` instead. + """ + return self._cache_write_input_tokens + + @cache_creation_input_tokens.setter + def cache_creation_input_tokens(self, value: int | None) -> None: + self._cache_write_input_tokens = value + + def _get_start_attributes(self) -> dict[str, AttributeValue]: + optional_attrs = ( + (GenAI.GEN_AI_REQUEST_MODEL, self._request_model), + (GenAI.GEN_AI_AGENT_NAME, self._agent_name), + (server_attributes.SERVER_ADDRESS, self._server_address), + (server_attributes.SERVER_PORT, self._server_port), + (GenAI.GEN_AI_PROVIDER_NAME, self._provider), + ) + return { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + **{k: v for k, v in optional_attrs if v is not None}, + } + + def _get_agent_attributes(self) -> dict[str, AttributeValue]: + optional_attrs = ( + (GenAI.GEN_AI_AGENT_ID, self.agent_id), + (GenAI.GEN_AI_AGENT_DESCRIPTION, self.agent_description), + (GenAI.GEN_AI_AGENT_VERSION, self.agent_version), + ) + return {k: v for k, v in optional_attrs if v is not None} + + def _get_request_attributes(self) -> dict[str, AttributeValue]: + attrs = dict(super()._get_request_attributes()) + if self.previous_response_id is not None: + attrs[_GEN_AI_REQUEST_PREVIOUS_RESPONSE_ID] = ( + self.previous_response_id + ) + return attrs + + def _get_usage_attributes(self) -> dict[str, AttributeValue]: + attrs = dict(super()._get_usage_attributes()) + if self.cache_write_input_tokens is not None: + attrs[_GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS] = ( + self.cache_write_input_tokens + ) + if self.cache_read_input_tokens is not None: + attrs[GenAI.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = ( + self.cache_read_input_tokens + ) + return attrs + def _get_metric_attributes(self) -> dict[str, AttributeValue]: optional_attrs = ( (GenAI.GEN_AI_PROVIDER_NAME, self._provider), diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py index f3ab9dbde..ed43e85a0 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py @@ -62,6 +62,22 @@ _GEN_AI_EXECUTE_TOOL_DURATION: Final = "gen_ai.execute_tool.duration" _GEN_AI_INVOKE_WORKFLOW_DURATION: Final = "gen_ai.invoke_workflow.duration" +_GEN_AI_INVOKE_AGENT_DURATION: Final = "gen_ai.invoke_agent.duration" +_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS: Final = [ + 0.1, + 0.2, + 0.4, + 0.8, + 1.6, + 3.2, + 6.4, + 12.8, + 25.6, + 51.2, + 102.4, + 204.8, + 409.6, +] class _Instruments: @@ -104,6 +120,12 @@ def __init__(self, meter: Meter) -> None: unit="s", explicit_bucket_boundaries_advisory=_GEN_AI_INVOKE_WORKFLOW_DURATION_BUCKETS, ) + self.invoke_agent_duration: Histogram = meter.create_histogram( + name=_GEN_AI_INVOKE_AGENT_DURATION, + description="Measures the duration of an in-process agent invocation.", + unit="s", + explicit_bucket_boundaries_advisory=_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS, + ) __all__ = ["_Instruments"] diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py index a9db0dafa..f36304a7e 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py @@ -44,11 +44,9 @@ from opentelemetry.metrics import Meter, MeterProvider, get_meter from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import ( - SpanKind, TracerProvider, get_tracer, ) -from opentelemetry.util.genai._agent_invocation import AgentInvocation from opentelemetry.util.genai._inference_invocation import LLMInvocation from opentelemetry.util.genai._instruments import _Instruments from opentelemetry.util.genai._invocation import Error @@ -57,9 +55,12 @@ _NoOpCompletionHook, ) from opentelemetry.util.genai.invocation import ( + AgentInvocation, EmbeddingInvocation, FetchResponseInvocation, InferenceInvocation, + LocalAgentInvocation, + RemoteAgentInvocation, RetrievalInvocation, ToolInvocation, WorkflowInvocation, @@ -462,12 +463,11 @@ def start_invoke_local_agent( Set remaining attributes (agent_name, etc.) on the returned invocation, then call invocation.stop() or invocation.fail(). """ - return AgentInvocation( + return LocalAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, - span_kind=SpanKind.INTERNAL, request_model=request_model, agent_name=agent_name, content_capturing_mode=self._content_capturing_mode, @@ -492,13 +492,12 @@ def start_invoke_remote_agent( Set remaining attributes (agent_name, etc.) on the returned invocation, then call invocation.stop() or invocation.fail(). """ - return AgentInvocation( + return RemoteAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, provider=provider, - span_kind=SpanKind.CLIENT, request_model=request_model, agent_name=agent_name, server_address=server_address, @@ -511,7 +510,7 @@ def invoke_local_agent( *, request_model: str | None = None, agent_name: str | None = None, - ) -> AgentInvocation: + ) -> LocalAgentInvocation: """Returns an agent invocation (INTERNAL span kind). Starts span when called. Returned object can be used as a ContextManager which automatically calls `stop` or `fail` @@ -522,12 +521,11 @@ def invoke_local_agent( Only set data attributes on the invocation object, do not modify the span or context. """ - return AgentInvocation( + return LocalAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, - span_kind=SpanKind.INTERNAL, request_model=request_model, agent_name=agent_name, content_capturing_mode=self._content_capturing_mode, @@ -541,7 +539,9 @@ def invoke_remote_agent( server_address: str | None = None, server_port: int | None = None, agent_name: str | None = None, - ) -> AgentInvocation: + agent_id: str | None = None, + agent_version: str | None = None, + ) -> RemoteAgentInvocation: """Returns an agent invocation (CLIENT span kind). Starts span when called. Returned object can be used as a ContextManager which automatically calls `stop` or `fail` @@ -552,15 +552,16 @@ def invoke_remote_agent( Only set data attributes on the invocation object, do not modify the span or context. """ - return AgentInvocation( + return RemoteAgentInvocation( self._tracer, self._instruments, self._logger, self._completion_hook, provider=provider, - span_kind=SpanKind.CLIENT, request_model=request_model, agent_name=agent_name, + agent_id=agent_id, + agent_version=agent_version, server_address=server_address, server_port=server_port, content_capturing_mode=self._content_capturing_mode, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py index 37b27823d..3d345d3fc 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py @@ -17,7 +17,11 @@ ) """ -from opentelemetry.util.genai._agent_invocation import AgentInvocation +from opentelemetry.util.genai._agent_invocation import ( + AgentInvocation, + LocalAgentInvocation, + RemoteAgentInvocation, +) from opentelemetry.util.genai._embedding_invocation import EmbeddingInvocation from opentelemetry.util.genai._fetch_response_invocation import ( FetchResponseInvocation, @@ -40,6 +44,8 @@ "FetchResponseInvocation", "GenAIInvocation", "InferenceInvocation", + "LocalAgentInvocation", + "RemoteAgentInvocation", "RetrievalInvocation", "ToolInvocation", "WorkflowInvocation", diff --git a/util/opentelemetry-util-genai/tests/test_handler_agent.py b/util/opentelemetry-util-genai/tests/test_handler_agent.py index eb3ad3c19..18a0c5db6 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_agent.py +++ b/util/opentelemetry-util-genai/tests/test_handler_agent.py @@ -23,6 +23,11 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, ) from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import ( + AgentInvocation, + LocalAgentInvocation, + RemoteAgentInvocation, +) from opentelemetry.util.genai.types import ( ContentCapturingMode, Error, @@ -47,6 +52,8 @@ def test_start_stop_creates_span(self): request_model="gpt-4", agent_name="Math Tutor", ) + assert isinstance(invocation, LocalAgentInvocation) + assert isinstance(invocation, AgentInvocation) invocation.stop() spans = self.span_exporter.get_finished_spans() @@ -96,9 +103,7 @@ def test_all_attributes(self): request_model="gpt-4", agent_name="Full Agent", ) - invocation.agent_id = "agent-123" invocation.agent_description = "A test agent" - invocation.agent_version = "1.0.0" invocation.conversation_id = "conv-456" invocation.data_source_id = "ds-789" invocation.output_type = "text" @@ -117,9 +122,9 @@ def test_all_attributes(self): attrs = self.span_exporter.get_finished_spans()[0].attributes assert attrs[GenAI.GEN_AI_AGENT_NAME] == "Full Agent" - assert attrs[GenAI.GEN_AI_AGENT_ID] == "agent-123" + assert GenAI.GEN_AI_AGENT_ID not in attrs assert attrs[GenAI.GEN_AI_AGENT_DESCRIPTION] == "A test agent" - assert attrs[GenAI.GEN_AI_AGENT_VERSION] == "1.0.0" + assert GenAI.GEN_AI_AGENT_VERSION not in attrs assert attrs[GenAI.GEN_AI_USAGE_INPUT_TOKENS] == 100 assert attrs[GenAI.GEN_AI_USAGE_OUTPUT_TOKENS] == 200 assert attrs[GenAI.GEN_AI_CONVERSATION_ID] == "conv-456" @@ -206,16 +211,21 @@ def test_context_manager_default_invocation(self): def test_default_values(self): invocation = self.handler.invoke_local_agent() invocation.stop() + assert isinstance(invocation, LocalAgentInvocation) + assert isinstance(invocation, AgentInvocation) assert invocation._operation_name == "invoke_agent" assert invocation.agent_name is None assert invocation._request_model is None assert not invocation.input_messages assert not invocation.output_messages assert invocation.tool_definitions is None - assert invocation.cache_creation_input_tokens is None - assert invocation.cache_read_input_tokens is None assert invocation.span is not INVALID_SPAN assert not invocation.attributes + assert not hasattr(invocation, "agent_id") + assert not hasattr(invocation, "agent_version") + assert not hasattr(invocation, "previous_response_id") + assert not hasattr(invocation, "cache_write_input_tokens") + assert not hasattr(invocation, "cache_read_input_tokens") def test_with_messages(self): invocation = self.handler.invoke_local_agent() @@ -457,11 +467,56 @@ def setUp(self): def test_span_kind_client(self): invocation = self.handler.invoke_remote_agent("openai") + assert isinstance(invocation, RemoteAgentInvocation) + assert isinstance(invocation, AgentInvocation) invocation.stop() assert ( self.span_exporter.get_finished_spans()[0].kind == SpanKind.CLIENT ) + def test_default_values(self): + invocation = self.handler.invoke_remote_agent("openai") + invocation.stop() + assert isinstance(invocation, RemoteAgentInvocation) + assert isinstance(invocation, AgentInvocation) + assert invocation._operation_name == "invoke_agent" + assert invocation.agent_name is None + assert invocation._request_model is None + assert invocation.agent_id is None + assert invocation.agent_version is None + assert invocation.previous_response_id is None + assert invocation.cache_write_input_tokens is None + assert invocation.cache_creation_input_tokens is None + assert invocation.cache_read_input_tokens is None + assert not invocation.input_messages + assert not invocation.output_messages + assert invocation.tool_definitions is None + assert invocation.span is not INVALID_SPAN + assert not invocation.attributes + + def test_constructor_agent_id_and_version(self): + invocation = self.handler.invoke_remote_agent( + "openai", + agent_id="agent-999", + agent_version="2.0.0", + ) + invocation.stop() + attrs = self.span_exporter.get_finished_spans()[0].attributes + assert attrs[GenAI.GEN_AI_AGENT_ID] == "agent-999" + assert attrs[GenAI.GEN_AI_AGENT_VERSION] == "2.0.0" + + def test_cache_token_attributes(self): + invocation = self.handler.invoke_remote_agent("openai") + invocation.input_tokens = 100 + invocation.cache_creation_input_tokens = 25 + invocation.cache_read_input_tokens = 50 + invocation.stop() + + attrs = self.span_exporter.get_finished_spans()[0].attributes + assert attrs[GenAI.GEN_AI_USAGE_INPUT_TOKENS] == 100 + assert attrs["gen_ai.usage.cache_write.input_tokens"] == 25 + assert attrs[GenAI.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 50 + def test_server_attributes(self): invocation = self.handler.invoke_remote_agent( "openai", @@ -484,8 +539,11 @@ def test_all_attributes(self): invocation.agent_id = "agent-123" invocation.agent_description = "A remote test agent" invocation.agent_version = "1.0.0" + invocation.previous_response_id = "resp_123" invocation.input_tokens = 100 invocation.output_tokens = 200 + invocation.cache_write_input_tokens = 30 + invocation.cache_read_input_tokens = 15 invocation.stop() attrs = self.span_exporter.get_finished_spans()[0].attributes @@ -493,8 +551,11 @@ def test_all_attributes(self): assert attrs[GenAI.GEN_AI_AGENT_ID] == "agent-123" assert attrs[GenAI.GEN_AI_AGENT_DESCRIPTION] == "A remote test agent" assert attrs[GenAI.GEN_AI_AGENT_VERSION] == "1.0.0" + assert attrs["gen_ai.request.previous_response.id"] == "resp_123" assert attrs[GenAI.GEN_AI_USAGE_INPUT_TOKENS] == 100 assert attrs[GenAI.GEN_AI_USAGE_OUTPUT_TOKENS] == 200 + assert attrs["gen_ai.usage.cache_write.input_tokens"] == 30 + assert attrs[GenAI.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 15 assert attrs[GenAI.GEN_AI_REQUEST_MODEL] == "gpt-4" def test_fail_sets_error_status(self): @@ -589,45 +650,26 @@ def test_local_agent_records_duration_and_tokens(self) -> None: meter_provider=self.meter_provider, ) with patch("timeit.default_timer", return_value=1000.0): - invocation = handler.invoke_local_agent(request_model="model") - invocation.input_tokens = 5 - invocation.output_tokens = 7 + invocation = handler.invoke_local_agent( + request_model="model", agent_name="LocalAgent" + ) with patch("timeit.default_timer", return_value=1002.0): invocation.stop() metrics = self._harvest_metrics() - self.assertIn("gen_ai.client.operation.duration", metrics) - duration_points = metrics["gen_ai.client.operation.duration"] + self.assertIn("gen_ai.invoke_agent.duration", metrics) + duration_points = metrics["gen_ai.invoke_agent.duration"] self.assertEqual(len(duration_points), 1) duration_point = duration_points[0] self.assertEqual( - duration_point.attributes[GenAI.GEN_AI_OPERATION_NAME], - GenAI.GenAiOperationNameValues.INVOKE_AGENT.value, + duration_point.attributes[GenAI.GEN_AI_AGENT_NAME], "LocalAgent" ) self.assertEqual( duration_point.attributes[GenAI.GEN_AI_REQUEST_MODEL], "model" ) self.assertAlmostEqual(duration_point.sum, 2.0, places=3) - self.assertIn("gen_ai.client.token.usage", metrics) - token_points = metrics["gen_ai.client.token.usage"] - token_by_type = { - point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point - for point in token_points - } - self.assertEqual(len(token_by_type), 2) - self.assertAlmostEqual( - token_by_type[GenAI.GenAiTokenTypeValues.INPUT.value].sum, - 5.0, - places=3, - ) - self.assertAlmostEqual( - token_by_type[GenAI.GenAiTokenTypeValues.OUTPUT.value].sum, - 7.0, - places=3, - ) - def test_remote_agent_records_duration_with_server_attrs(self) -> None: handler = TelemetryHandler( tracer_provider=self.tracer_provider, @@ -636,19 +678,33 @@ def test_remote_agent_records_duration_with_server_attrs(self) -> None: invocation = handler.invoke_remote_agent( "prov", request_model="model", + agent_name="RemoteAgent", server_address="agent.example.com", server_port=443, ) - invocation.input_tokens = 10 invocation.stop() metrics = self._harvest_metrics() self.assertIn("gen_ai.client.operation.duration", metrics) duration_point = metrics["gen_ai.client.operation.duration"][0] self.assertEqual( - duration_point.attributes["server.address"], "agent.example.com" + duration_point.attributes[GenAI.GEN_AI_OPERATION_NAME], + "invoke_agent", + ) + self.assertEqual( + duration_point.attributes[GenAI.GEN_AI_PROVIDER_NAME], "prov" + ) + self.assertNotIn(GenAI.GEN_AI_AGENT_NAME, duration_point.attributes) + self.assertEqual( + duration_point.attributes[GenAI.GEN_AI_REQUEST_MODEL], "model" + ) + self.assertEqual( + duration_point.attributes[server_attributes.SERVER_ADDRESS], + "agent.example.com", + ) + self.assertEqual( + duration_point.attributes[server_attributes.SERVER_PORT], 443 ) - self.assertEqual(duration_point.attributes["server.port"], 443) def test_fail_agent_records_error_metric(self) -> None: handler = TelemetryHandler( @@ -656,19 +712,23 @@ def test_fail_agent_records_error_metric(self) -> None: meter_provider=self.meter_provider, ) with patch("timeit.default_timer", return_value=2000.0): - invocation = handler.invoke_local_agent(request_model="err-model") - invocation.input_tokens = 11 + invocation = handler.invoke_local_agent( + request_model="err-model", agent_name="ErrAgent" + ) error = Error(message="boom", type="ValueError") with patch("timeit.default_timer", return_value=2001.0): invocation.fail(error) metrics = self._harvest_metrics() - self.assertIn("gen_ai.client.operation.duration", metrics) - duration_point = metrics["gen_ai.client.operation.duration"][0] + self.assertIn("gen_ai.invoke_agent.duration", metrics) + duration_point = metrics["gen_ai.invoke_agent.duration"][0] self.assertEqual( duration_point.attributes.get("error.type"), "ValueError" ) + self.assertEqual( + duration_point.attributes.get(GenAI.GEN_AI_AGENT_NAME), "ErrAgent" + ) self.assertAlmostEqual(duration_point.sum, 1.0, places=3) def _harvest_metrics(self): From 88a62c8911ed9ffb9d4746ae006e4af6ee4c1123 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 3 Sep 2026 18:44:32 -0700 Subject: [PATCH 2/9] Add changelog fragments for #616 --- .../.changelog/616.changed | 1 + .../.changelog/616.changed | 1 + util/opentelemetry-util-genai/.changelog/616.added | 1 + util/opentelemetry-util-genai/.changelog/616.changed | 1 + 4 files changed, 4 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed create mode 100644 instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed create mode 100644 util/opentelemetry-util-genai/.changelog/616.added create mode 100644 util/opentelemetry-util-genai/.changelog/616.changed diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed new file mode 100644 index 000000000..70d5a1799 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`, and stop setting span attribute `gen_ai.agent.id` on internal agent spans. diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/util/opentelemetry-util-genai/.changelog/616.added b/util/opentelemetry-util-genai/.changelog/616.added new file mode 100644 index 000000000..8b5f55a88 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/616.added @@ -0,0 +1 @@ +Add `LocalAgentInvocation` and `RemoteAgentInvocation` for in-process and remote agent invocations. diff --git a/util/opentelemetry-util-genai/.changelog/616.changed b/util/opentelemetry-util-genai/.changelog/616.changed new file mode 100644 index 000000000..7aad107a4 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/616.changed @@ -0,0 +1 @@ +Record `gen_ai.invoke_agent.duration` on local agent invocations. From d18b865b6542d7a716c11c1f8e59ef4fd4627412 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 3 Sep 2026 18:55:33 -0700 Subject: [PATCH 3/9] Update agent conformance scenarios to expect invoke_agent and execute_tool duration metrics --- .../.changelog/616.changed | 1 + .../tests/conformance/agent.py | 5 ++++- .../.changelog/616.changed | 1 + .../tests/conformance/agent.py | 5 ++++- .../.changelog/616.changed | 1 + .../tests/conformance/orchestration.py | 3 ++- 6 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed create mode 100644 instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py index be17a87c3..0f9a1dbd9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/tests/conformance/agent.py @@ -23,7 +23,10 @@ class AgentScenario(Scenario): expected_spans = {"invoke_agent": 1, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py index f91082026..acc9250fb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/agent.py @@ -28,7 +28,10 @@ class AgentScenario(Scenario): expected_spans = {"invoke_agent": 3, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py index bc6d94f7d..390c6ec20 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py @@ -76,7 +76,8 @@ class OrchestrationScenario(Scenario): "execute_tool": 1, } expected_metrics = ( - "gen_ai.client.operation.duration", + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", "gen_ai.invoke_workflow.duration", ) expected_violations = ( From e6f1079af4dfeb5a0ce7afa72c8063fe7ff967e7 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 3 Sep 2026 23:10:45 -0700 Subject: [PATCH 4/9] Fix semconv URLs to point to semantic-conventions-genai in agent invocations --- .../src/opentelemetry/util/genai/_agent_invocation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index 567ae1935..ffdc182dc 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -44,8 +44,8 @@ class AgentInvocation(GenAIInvocation): rather than constructing this directly. Reference: - Client span: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span - Internal span: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span + Client span: https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span + Internal span: https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span """ def __init__( @@ -159,7 +159,7 @@ class LocalAgentInvocation(AgentInvocation): Use handler.invoke_local_agent() rather than constructing this directly. Reference: - https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span + https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-internal-span """ def __init__( @@ -242,7 +242,7 @@ class RemoteAgentInvocation(AgentInvocation): Use handler.invoke_remote_agent() rather than constructing this directly. Reference: - https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span + https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-client-span """ def __init__( From de65f5f0184021d8145501b73e0d54cdcaaaba7a Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 10 Sep 2026 08:33:12 -0700 Subject: [PATCH 5/9] Address review comments on agent split --- .../instrumentation/genai/llama_index/_handler.py | 6 +++--- .../instrumentation/genai/qwen_agent/patch.py | 4 ++-- .../instrumentation/genai/qwen_agent/utils.py | 4 ++-- .../instrumentation/google_genai/interactions.py | 12 ++++++------ .../opentelemetry/util/genai/_agent_invocation.py | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py index 87801c306..0276d3620 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/src/opentelemetry/instrumentation/genai/llama_index/_handler.py @@ -32,8 +32,8 @@ from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, GenAIInvocation, + LocalAgentInvocation, ToolInvocation, ) from opentelemetry.util.genai.types import ( @@ -316,7 +316,7 @@ def _tool_definitions(agent: BaseWorkflowAgent) -> list[ToolDefinition] | None: return definitions or None -def _set_agent_output(invocation: AgentInvocation, result: Any) -> None: +def _set_agent_output(invocation: LocalAgentInvocation, result: Any) -> None: """Copy the final chat response out of LlamaIndex's workflow result.""" output = getattr(result, "result", None) response = getattr(output, "response", None) @@ -512,7 +512,7 @@ def prepare_to_exit_span( span = self.open_spans.get(id_) if span is None: return None - if isinstance(span._invocation, AgentInvocation): + if isinstance(span._invocation, LocalAgentInvocation): span.reset_tool_attributes() if self._handler.should_capture_content(): _set_agent_output(span._invocation, result) diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py index b011b349d..349ffb67d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/patch.py @@ -27,7 +27,7 @@ find_tool_call_id, ) from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.invocation import AgentInvocation +from opentelemetry.util.genai.invocation import LocalAgentInvocation from opentelemetry.util.genai.stream import SyncStreamWrapper @@ -41,7 +41,7 @@ class _AgentRunStreamWrapper(SyncStreamWrapper[Any]): def __init__( self, stream: Any, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, capture_content: bool, ) -> None: super().__init__(stream) diff --git a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py index 136b89c25..ae551d96c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-qwen-agent/src/opentelemetry/instrumentation/genai/qwen_agent/utils.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, cast from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.invocation import AgentInvocation +from opentelemetry.util.genai.invocation import LocalAgentInvocation from opentelemetry.util.genai.types import ( InputMessage, MessagePart, @@ -269,7 +269,7 @@ def create_agent_invocation( handler: TelemetryHandler, agent_instance: Any, messages: QwenMessage | list[QwenMessage] | None, -) -> AgentInvocation: +) -> LocalAgentInvocation: """Create and start an AgentInvocation for Agent.run().""" llm_instance = getattr(agent_instance, "llm", None) agent_name = ( diff --git a/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py b/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py index 468fa26a8..31b4736fa 100644 --- a/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py +++ b/instrumentation/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/interactions.py @@ -86,8 +86,8 @@ class Stream: ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, InferenceInvocation, + RemoteAgentInvocation, ) from opentelemetry.util.genai.stream import ( AsyncStreamWrapper, @@ -135,7 +135,7 @@ def _set_co_filename(wrapped: object) -> None: def _apply_interaction_response_attributes( response: Interaction, - invocation: InferenceInvocation | AgentInvocation, + invocation: InferenceInvocation | RemoteAgentInvocation, telemetry_handler: TelemetryHandler, ) -> None: if isinstance(invocation, InferenceInvocation): @@ -297,7 +297,7 @@ class InteractionsStreamWrapper(SyncStreamWrapper[InteractionSSEEvent]): def __init__( self, stream: Iterable[InteractionSSEEvent], - invocation: InferenceInvocation | AgentInvocation, + invocation: InferenceInvocation | RemoteAgentInvocation, telemetry_handler: TelemetryHandler, ) -> None: super().__init__(stream) @@ -329,7 +329,7 @@ class AsyncInteractionsStreamWrapper(AsyncStreamWrapper[InteractionSSEEvent]): def __init__( self, stream: AsyncIterable[InteractionSSEEvent], - invocation: InferenceInvocation | AgentInvocation, + invocation: InferenceInvocation | RemoteAgentInvocation, telemetry_handler: TelemetryHandler, ) -> None: super().__init__(stream) @@ -414,7 +414,7 @@ def _start_interactions_invocation( telemetry_handler: TelemetryHandler, instance: InteractionsResource | AsyncInteractionsResource, kwargs: dict[str, Any], -) -> InferenceInvocation | AgentInvocation: +) -> InferenceInvocation | RemoteAgentInvocation: # Vertex AI does not support the interactions API yet, but eventually will. # SDK will raise an exception if model or agent is not passed or if input data is not passed. is_vertex, server_address = _get_client_info(instance) @@ -424,7 +424,7 @@ def _start_interactions_invocation( else GenAIAttributes.GenAiSystemValues.GEMINI.value ) if agent := kwargs.get("agent"): - invocation: InferenceInvocation | AgentInvocation = ( + invocation: InferenceInvocation | RemoteAgentInvocation = ( telemetry_handler.invoke_remote_agent( provider=provider, request_model=kwargs.get("model"), diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index ffdc182dc..606032448 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -327,7 +327,7 @@ def _get_agent_attributes(self) -> dict[str, AttributeValue]: return {k: v for k, v in optional_attrs if v is not None} def _get_request_attributes(self) -> dict[str, AttributeValue]: - attrs = dict(super()._get_request_attributes()) + attrs = super()._get_request_attributes() if self.previous_response_id is not None: attrs[_GEN_AI_REQUEST_PREVIOUS_RESPONSE_ID] = ( self.previous_response_id @@ -335,7 +335,7 @@ def _get_request_attributes(self) -> dict[str, AttributeValue]: return attrs def _get_usage_attributes(self) -> dict[str, AttributeValue]: - attrs = dict(super()._get_usage_attributes()) + attrs = super()._get_usage_attributes() if self.cache_write_input_tokens is not None: attrs[_GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS] = ( self.cache_write_input_tokens From 868e01d9bc5dc4ebec093f3913dedae161ba788f Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 10 Sep 2026 10:21:26 -0700 Subject: [PATCH 6/9] Update DSPy and smolagents agent metric expectations for agent duration --- .../.changelog/616.changed | 1 + .../tests/conformance/react.py | 5 ++++- .../tests/conformance/react_v2.py | 5 ++++- .../.changelog/616.changed | 1 + .../tests/conformance/agent.py | 2 +- .../tests/test_agents.py | 6 +++--- 6 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed diff --git a/instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed new file mode 100644 index 000000000..55b40f6e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-dspy/.changelog/616.changed @@ -0,0 +1 @@ +Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py index 4f3109b22..1eb8867b9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py +++ b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react.py @@ -52,7 +52,10 @@ def __call__(self, **kwargs: object) -> dict[str, str]: class ReActScenario(Scenario): expected_spans = {"invoke_agent": 1, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) expected_violations = ( ExpectedViolation( advice_id="genai_expected_attribute_missing", diff --git a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py index 1246dafa6..a04b7ef82 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py +++ b/instrumentation/opentelemetry-instrumentation-genai-dspy/tests/conformance/react_v2.py @@ -58,7 +58,10 @@ class SubmitPred: class ReActV2Scenario(Scenario): expected_spans = {"invoke_agent": 1, "execute_tool": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ( + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", + ) expected_violations = ( ExpectedViolation( advice_id="genai_expected_attribute_missing", diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed new file mode 100644 index 000000000..2b72bfc3f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/616.changed @@ -0,0 +1 @@ +Record metric `gen_ai.invoke_agent.duration` instead of `gen_ai.client.operation.duration`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py index 2cffd623e..c17fafaf6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py @@ -23,7 +23,7 @@ class AgentScenario(Scenario): # The model call goes to the OpenAI SDK, whose instrumentation is not # enabled here. Tool calls are not instrumented yet. expected_spans = {"invoke_agent": 1} - expected_metrics = ("gen_ai.client.operation.duration",) + expected_metrics = ("gen_ai.invoke_agent.duration",) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py index e82476419..6048291ff 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py @@ -353,7 +353,7 @@ def test_streaming_run_records_no_chunk_metrics( list(agent.run("Test question", stream=True)) metrics = metrics_by_name(metric_reader) - assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert "gen_ai.invoke_agent.duration" in metrics assert ( gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK not in metrics @@ -389,9 +389,9 @@ def test_agent_run_metrics( agent.run("Test question") metrics = metrics_by_name(metric_reader) - duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + duration = metrics["gen_ai.invoke_agent.duration"] assert { - GenAI.GEN_AI_OPERATION_NAME: "invoke_agent", + GenAI.GEN_AI_AGENT_NAME: "CodeAgent", GenAI.GEN_AI_REQUEST_MODEL: "fake-model", } in data_point_attributes(duration) # A run reports no token counts of its own: each model call records its From a0f4264aaa79c0dd3a6797e105292f9c2c5f1e4a Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 10 Sep 2026 10:37:36 -0700 Subject: [PATCH 7/9] Deduplicate agent finish logic and narrow agent types in agno and smolagents --- .../instrumentation/genai/agno/patch.py | 8 +-- .../instrumentation/genai/agno/stream.py | 18 ++--- .../instrumentation/genai/smolagents/patch.py | 10 +-- .../util/genai/_agent_invocation.py | 65 ++++++++----------- 4 files changed, 40 insertions(+), 61 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py index 79c974826..fcf4f9cce 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/patch.py @@ -36,7 +36,7 @@ ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, + LocalAgentInvocation, ToolInvocation, WorkflowInvocation, ) @@ -201,7 +201,7 @@ def _set_tool_invocation_output( def _set_invocation_input( - invocation: AgentInvocation | WorkflowInvocation, + invocation: LocalAgentInvocation | WorkflowInvocation, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any], @@ -225,7 +225,7 @@ def _extract_finish_reason(result: object) -> str: def _set_invocation_output( - invocation: AgentInvocation | WorkflowInvocation, + invocation: LocalAgentInvocation | WorkflowInvocation, result: object | None, capture_content: bool, ) -> None: @@ -249,7 +249,7 @@ def _start_agent_invocation( args: tuple[Any, ...], kwargs: dict[str, Any], capture_content: bool, -) -> AgentInvocation: +) -> LocalAgentInvocation: agent_name = getattr(instance, "name", None) invocation = handler.invoke_local_agent(agent_name=agent_name) _set_invocation_input(invocation, instance, args, kwargs, capture_content) diff --git a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py index becd20bbb..5b3697404 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py +++ b/instrumentation/opentelemetry-instrumentation-genai-agno/src/opentelemetry/instrumentation/genai/agno/stream.py @@ -12,8 +12,7 @@ format_content, ) from opentelemetry.util.genai.invocation import ( - AgentInvocation, - RemoteAgentInvocation, + LocalAgentInvocation, WorkflowInvocation, ) from opentelemetry.util.genai.stream import ( @@ -38,7 +37,7 @@ def _extract_chunk_content(chunk: Any) -> str | None: class _AgentStreamMixin: - _self_agent_invocation: AgentInvocation + _self_agent_invocation: LocalAgentInvocation _self_capture_content: bool _self_content_parts: list[str] _self_completed_content: str | None @@ -57,15 +56,6 @@ def _process_chunk(self, chunk: Any) -> None: self._self_agent_invocation.output_tokens = ( metrics.output_tokens ) - if isinstance(self._self_agent_invocation, RemoteAgentInvocation): - if getattr(metrics, "cache_read_tokens", None) is not None: - self._self_agent_invocation.cache_read_input_tokens = ( - metrics.cache_read_tokens - ) - if getattr(metrics, "cache_write_tokens", None) is not None: - self._self_agent_invocation.cache_write_input_tokens = ( - metrics.cache_write_tokens - ) event_name = str(getattr(chunk, "event", "")) chunk_type = type(chunk).__name__ @@ -161,7 +151,7 @@ class AgnoAgentStreamWrapper(_AgentStreamMixin, SyncStreamWrapper[Any]): def __init__( self, stream: Any, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, capture_content: bool, ) -> None: super().__init__(stream) @@ -178,7 +168,7 @@ class AsyncAgnoAgentStreamWrapper(_AgentStreamMixin, AsyncStreamWrapper[Any]): def __init__( self, stream: Any, - invocation: AgentInvocation, + invocation: LocalAgentInvocation, capture_content: bool, ) -> None: super().__init__(stream) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index 0889ca845..7509ae115 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -34,9 +34,9 @@ ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( - AgentInvocation, GenAIInvocation, InferenceInvocation, + LocalAgentInvocation, ) from opentelemetry.util.genai.stream import SyncStreamWrapper from opentelemetry.util.genai.types import OutputMessage, Role, TextPart @@ -422,7 +422,7 @@ def wrapper( def _record_run_answer( - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, output: object, *, @@ -454,7 +454,7 @@ class _AgentRunStreamWrapper(SyncStreamWrapper[_RunStreamChunk]): def __init__( self, stream: Generator[_RunStreamChunk, None, None], - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, *, capture_content: bool, @@ -522,7 +522,7 @@ def _on_stream_error(self, error: BaseException) -> None: def _record_agent( - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, bound: dict[str, Any], *, @@ -540,7 +540,7 @@ def _record_agent( def _record_agent_run( - invocation: AgentInvocation, + invocation: LocalAgentInvocation, agent: MultiStepAgent, bound: dict[str, Any], result: object, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index 606032448..ad36168d1 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -4,6 +4,7 @@ from __future__ import annotations import timeit +from abc import abstractmethod from typing import Final from opentelemetry._logs import Logger @@ -152,6 +153,30 @@ def _get_content_attributes_for_span(self) -> dict[str, AttributeValue]: content_capturing_mode=self._content_capturing_mode, ) + def _apply_finish(self, error: Error | None = None) -> None: + if error is not None: + self._apply_error_attributes(error) + + attributes: dict[str, AttributeValue] = {} + attributes.update(self._get_agent_attributes()) + attributes.update(self._get_request_attributes()) + attributes.update(self._get_response_attributes()) + attributes.update(self._get_usage_attributes()) + attributes.update(self._get_content_attributes_for_span()) + attributes.update(self.attributes) + self.span.set_attributes(attributes) + self._call_completion_hook( + inputs=self.input_messages, + outputs=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + ) + self._record_metrics() + + @abstractmethod + def _record_metrics(self) -> None: + """Record invocation metrics.""" + class LocalAgentInvocation(AgentInvocation): """Represents an in-process agent invocation (INTERNAL span kind). @@ -204,26 +229,6 @@ def _get_metric_attributes(self) -> dict[str, AttributeValue]: attrs.update(self.metric_attributes) return attrs - def _apply_finish(self, error: Error | None = None) -> None: - if error is not None: - self._apply_error_attributes(error) - - attributes: dict[str, AttributeValue] = {} - attributes.update(self._get_agent_attributes()) - attributes.update(self._get_request_attributes()) - attributes.update(self._get_response_attributes()) - attributes.update(self._get_usage_attributes()) - attributes.update(self._get_content_attributes_for_span()) - attributes.update(self.attributes) - self.span.set_attributes(attributes) - self._call_completion_hook( - inputs=self.input_messages, - outputs=self.output_messages, - system_instruction=self.system_instruction, - tool_definitions=self.tool_definitions, - ) - self._record_metrics() - def _record_metrics(self) -> None: duration_seconds = max( timeit.default_timer() - self._monotonic_start_s, @@ -370,22 +375,6 @@ def _get_metric_token_counts(self) -> dict[str, int]: ) return counts - def _apply_finish(self, error: Error | None = None) -> None: - if error is not None: - self._apply_error_attributes(error) - - attributes: dict[str, AttributeValue] = {} - attributes.update(self._get_agent_attributes()) - attributes.update(self._get_request_attributes()) - attributes.update(self._get_response_attributes()) - attributes.update(self._get_usage_attributes()) - attributes.update(self._get_content_attributes_for_span()) - attributes.update(self.attributes) - self.span.set_attributes(attributes) - self._call_completion_hook( - inputs=self.input_messages, - outputs=self.output_messages, - system_instruction=self.system_instruction, - tool_definitions=self.tool_definitions, - ) + def _record_metrics(self) -> None: self._record_client_metrics() + From 7e9ad047de6816d139e701888af5b35c3fbbbdb3 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 10 Sep 2026 17:27:32 -0700 Subject: [PATCH 8/9] Fix llama-index agent types after merging main --- .../tests/conformance/workflow.py | 3 ++- .../src/opentelemetry/util/genai/_agent_invocation.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py index 0d60c3603..0f666f306 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py +++ b/instrumentation/opentelemetry-instrumentation-genai-llama-index/tests/conformance/workflow.py @@ -31,8 +31,9 @@ class WorkflowScenario(Scenario): "execute_tool": 1, } expected_metrics = ( - "gen_ai.client.operation.duration", "gen_ai.invoke_workflow.duration", + "gen_ai.invoke_agent.duration", + "gen_ai.execute_tool.duration", ) def run( diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index ad36168d1..d7789c256 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -377,4 +377,3 @@ def _get_metric_token_counts(self) -> dict[str, int]: def _record_metrics(self) -> None: self._record_client_metrics() - From 0a177ab84cd3b525f53e757fe112d3c4f77baa3d Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 11 Sep 2026 14:29:25 -0700 Subject: [PATCH 9/9] Explicitly inherit from ABC in AgentInvocation --- .../src/opentelemetry/util/genai/_agent_invocation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index d7789c256..a8bbb45ff 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -4,7 +4,7 @@ from __future__ import annotations import timeit -from abc import abstractmethod +from abc import ABC, abstractmethod from typing import Final from opentelemetry._logs import Logger @@ -38,7 +38,7 @@ ) -class AgentInvocation(GenAIInvocation): +class AgentInvocation(GenAIInvocation, ABC): """Base class representing a GenAI agent invocation (invoke_agent span). Use handler.invoke_local_agent() or handler.invoke_remote_agent()