Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`.
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
)
from opentelemetry.util.genai.handler import TelemetryHandler
from opentelemetry.util.genai.invocation import (
AgentInvocation,
LocalAgentInvocation,
ToolInvocation,
WorkflowInvocation,
)
Expand Down Expand Up @@ -201,7 +201,7 @@ def _set_tool_invocation_output(


def _set_invocation_input(
invocation: AgentInvocation | WorkflowInvocation,
invocation: LocalAgentInvocation | WorkflowInvocation,
instance: Any,
args: tuple[Any, ...],
kwargs: dict[str, Any],
Expand All @@ -225,7 +225,7 @@ def _extract_finish_reason(result: object) -> str:


def _set_invocation_output(
invocation: AgentInvocation | WorkflowInvocation,
invocation: LocalAgentInvocation | WorkflowInvocation,
result: object | None,
capture_content: bool,
) -> None:
Expand All @@ -249,7 +249,7 @@ def _start_agent_invocation(
args: tuple[Any, ...],
kwargs: dict[str, Any],
capture_content: bool,
) -> AgentInvocation:
) -> LocalAgentInvocation:
agent_name = getattr(instance, "name", None)
invocation = handler.invoke_local_agent(agent_name=agent_name)
_set_invocation_input(invocation, instance, args, kwargs, capture_content)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
format_content,
)
from opentelemetry.util.genai.invocation import (
AgentInvocation,
LocalAgentInvocation,
WorkflowInvocation,
)
from opentelemetry.util.genai.stream import (
Expand All @@ -37,7 +37,7 @@ def _extract_chunk_content(chunk: Any) -> str | None:


class _AgentStreamMixin:
_self_agent_invocation: AgentInvocation
_self_agent_invocation: LocalAgentInvocation
_self_capture_content: bool
_self_content_parts: list[str]
_self_completed_content: str | None
Expand All @@ -56,14 +56,6 @@ def _process_chunk(self, chunk: Any) -> None:
self._self_agent_invocation.output_tokens = (
metrics.output_tokens
)
if getattr(metrics, "cache_read_tokens", None) is not None:
self._self_agent_invocation.cache_read_input_tokens = (
metrics.cache_read_tokens
)
if getattr(metrics, "cache_write_tokens", None) is not None:
self._self_agent_invocation.cache_write_input_tokens = (
metrics.cache_write_tokens
)

event_name = str(getattr(chunk, "event", ""))
chunk_type = type(chunk).__name__
Expand Down Expand Up @@ -159,7 +151,7 @@ class AgnoAgentStreamWrapper(_AgentStreamMixin, SyncStreamWrapper[Any]):
def __init__(
self,
stream: Any,
invocation: AgentInvocation,
invocation: LocalAgentInvocation,
capture_content: bool,
) -> None:
super().__init__(stream)
Expand All @@ -176,7 +168,7 @@ class AsyncAgnoAgentStreamWrapper(_AgentStreamMixin, AsyncStreamWrapper[Any]):
def __init__(
self,
stream: Any,
invocation: AgentInvocation,
invocation: LocalAgentInvocation,
capture_content: bool,
) -> None:
super().__init__(stream)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@

class AgentScenario(Scenario):
expected_spans = {"invoke_agent": 1, "execute_tool": 1}
expected_metrics = ("gen_ai.client.operation.duration",)
expected_metrics = (
"gen_ai.invoke_agent.duration",
"gen_ai.execute_tool.duration",
)

def run(
self,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`.
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ def __call__(self, **kwargs: object) -> dict[str, str]:

class ReActScenario(Scenario):
expected_spans = {"invoke_agent": 1, "execute_tool": 1}
expected_metrics = ("gen_ai.client.operation.duration",)
expected_metrics = (
"gen_ai.invoke_agent.duration",
"gen_ai.execute_tool.duration",
)
expected_violations = (
ExpectedViolation(
advice_id="genai_expected_attribute_missing",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@ class SubmitPred:

class ReActV2Scenario(Scenario):
expected_spans = {"invoke_agent": 1, "execute_tool": 1}
expected_metrics = ("gen_ai.client.operation.duration",)
expected_metrics = (
"gen_ai.invoke_agent.duration",
"gen_ai.execute_tool.duration",
)
expected_violations = (
ExpectedViolation(
advice_id="genai_expected_attribute_missing",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`, and stop setting span attribute `gen_ai.agent.id` on internal agent spans.
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,6 @@ def on_chain_start(
agent.input_messages = make_input_message(inputs)

if metadata:
agent.agent_id = metadata.get("agent_id")
agent.agent_description = metadata.get(
"agent_description"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ class AgentScenario(Scenario):
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
"gen_ai.invoke_agent.duration",
"gen_ai.execute_tool.duration",
)
# langchain can't populate server.address on chat spans.
# invoke_agent provider is unknown at span creation; ls_provider is only
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def _make_agent_inv_mock() -> mock.MagicMock:
# agent_name is an instance attribute set in AgentInvocation.__init__ via the
# constructor arg; pre-configure it so spec-restricted attribute access works.
agent_inv.agent_name = None
agent_inv.agent_id = None
return agent_inv


Expand Down Expand Up @@ -256,7 +257,7 @@ def test_agent_metadata_set(self):
},
)

assert agent_inv.agent_id == "agent-123"
assert agent_inv.agent_id is None
assert agent_inv.agent_description == "does math"
assert agent_inv.conversation_id == "thread-abc"

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Record metrics `gen_ai.invoke_agent.duration` and `gen_ai.execute_tool.duration` instead of `gen_ai.client.operation.duration`.
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
from opentelemetry.trace import set_span_in_context
from opentelemetry.util.genai.handler import TelemetryHandler
from opentelemetry.util.genai.invocation import (
AgentInvocation,
GenAIInvocation,
LocalAgentInvocation,
ToolInvocation,
WorkflowInvocation,
)
Expand Down Expand Up @@ -70,13 +70,13 @@


_MEMBER_AGENT_CONTEXTS: MutableMapping[
AgentInvocation, contextvars.Context
LocalAgentInvocation, contextvars.Context
] = WeakKeyDictionary()


def _start_member_agent(
start: Callable[[], AgentInvocation],
) -> AgentInvocation:
start: Callable[[], LocalAgentInvocation],
) -> LocalAgentInvocation:
"""Open a member-agent invocation inside a context this module owns.

The invocation stays open across workflow steps, and each step runs in its
Expand All @@ -91,7 +91,7 @@ def _start_member_agent(


def _finish_member_agent(
invocation: AgentInvocation, error: BaseException | None = None
invocation: LocalAgentInvocation, error: BaseException | None = None
) -> None:
"""Finish a member-agent invocation in the context that started it."""

Expand Down Expand Up @@ -420,22 +420,24 @@ def _workflow_tool_definitions(
return definitions or None


def _set_agent_output(invocation: AgentInvocation, result: Any) -> None:
def _set_agent_output(invocation: LocalAgentInvocation, result: Any) -> None:
"""Copy the final chat response out of LlamaIndex's workflow result."""
output = getattr(result, "result", None)
response = getattr(output, "response", None)
if isinstance(response, ChatMessage):
invocation.output_messages = [_output_message(response)]


def _set_agent_step_output(invocation: AgentInvocation, result: Any) -> None:
def _set_agent_step_output(
invocation: LocalAgentInvocation, result: Any
) -> None:
"""Copy a member agent's response out of an AgentWorkflow step."""
if isinstance(result, AgentOutput):
invocation.output_messages = [_output_message(result.response)]


def _set_return_direct_agent_output(
invocation: AgentInvocation, tool_output: ToolOutput
invocation: LocalAgentInvocation, tool_output: ToolOutput
) -> None:
"""Record the response synthesized by a return-direct tool execution."""
invocation.output_messages = [
Expand Down Expand Up @@ -507,11 +509,13 @@ class _LlamaIndexInvocation(BaseSpan):
) = PrivateAttr()
_workflow_agents: dict[str, BaseWorkflowAgent] = PrivateAttr()
_workflow_agents_by_run_id: dict[str, BaseWorkflowAgent] = PrivateAttr()
_workflow_invocations_by_run_id: dict[str, AgentInvocation] = PrivateAttr()
_workflow_invocations_by_key: dict[tuple[str, str], AgentInvocation] = (
_workflow_invocations_by_run_id: dict[str, LocalAgentInvocation] = (
PrivateAttr()
)
_workflow_agent_invocation: AgentInvocation | None = PrivateAttr()
_workflow_invocations_by_key: dict[
tuple[str, str], LocalAgentInvocation
] = PrivateAttr()
_workflow_agent_invocation: LocalAgentInvocation | None = PrivateAttr()
_workflow_handoff: bool = PrivateAttr()
_workflow_agent_context_token: Token[Context] | None = PrivateAttr()
_tool_parent_context_token: Token[Context] | None = PrivateAttr()
Expand All @@ -522,7 +526,7 @@ class _LlamaIndexInvocation(BaseSpan):
_workflow_tool_counts: dict[str, int] = PrivateAttr()
_workflow_return_direct_runs: set[str] = PrivateAttr()
_workflow_tool_errors: dict[str, BaseException] = PrivateAttr()
_workflow_pending_handoffs: dict[str, AgentInvocation] = PrivateAttr()
_workflow_pending_handoffs: dict[str, LocalAgentInvocation] = PrivateAttr()

def __init__(
self,
Expand All @@ -539,7 +543,7 @@ def __init__(
workflow_agents: Mapping[str, BaseWorkflowAgent] | None = None,
workflow_run_id: str | None = None,
workflow_agent: BaseWorkflowAgent | None = None,
workflow_agent_invocation: AgentInvocation | None = None,
workflow_agent_invocation: LocalAgentInvocation | None = None,
workflow_handoff: bool = False,
tool_parent_context_token: Token[Context] | None = None,
) -> None:
Expand Down Expand Up @@ -591,7 +595,7 @@ def workflow_agent_for_run_id(

def workflow_invocation_for_run_id(
self, run_id: str | None, agent_name: str | None = None
) -> AgentInvocation | None:
) -> LocalAgentInvocation | None:
"""Return the reusable member-agent invocation for a workflow run."""
if self._workflow_agent_invocation is not None:
return self._workflow_agent_invocation
Expand All @@ -602,13 +606,15 @@ def workflow_invocation_for_run_id(
return self._workflow_invocations_by_run_id.get(run_id)

def register_workflow_invocation(
self, run_id: str, agent_name: str, invocation: AgentInvocation
self, run_id: str, agent_name: str, invocation: LocalAgentInvocation
) -> None:
"""Keep one member-agent invocation open across workflow turns."""
self._workflow_invocations_by_run_id[run_id] = invocation
self._workflow_invocations_by_key[(run_id, agent_name)] = invocation

def remove_workflow_invocation(self, invocation: AgentInvocation) -> None:
def remove_workflow_invocation(
self, invocation: LocalAgentInvocation
) -> None:
"""Forget a completed member invocation so a later turn can restart it."""
for key, value in list(self._workflow_invocations_by_key.items()):
if value is invocation:
Expand Down Expand Up @@ -666,7 +672,7 @@ def expect_workflow_tools(self, run_id: str, count: int) -> None:
def set_return_direct_agent_output(
self,
run_id: str,
invocation: AgentInvocation,
invocation: LocalAgentInvocation,
tool_output: ToolOutput,
) -> None:
"""Keep the first successful return-direct result to arrive."""
Expand Down Expand Up @@ -698,12 +704,12 @@ def release_workflow_tool(self, run_id: str | None) -> bool:
return True

def set_pending_handoff(
self, run_id: str, invocation: AgentInvocation
self, run_id: str, invocation: LocalAgentInvocation
) -> None:
"""Hold a handing-off agent open until its whole turn has drained."""
self._workflow_pending_handoffs[run_id] = invocation

def take_pending_handoff(self, run_id: str) -> AgentInvocation | None:
def take_pending_handoff(self, run_id: str) -> LocalAgentInvocation | None:
"""Claim the handing-off agent owed a close, if there is one."""
return self._workflow_pending_handoffs.pop(run_id, None)

Expand Down Expand Up @@ -739,7 +745,7 @@ def finalize_workflow_agents(
if name == agent_name:
_set_agent_step_output(invocation, output)
break
invocations: list[AgentInvocation] = []
invocations: list[LocalAgentInvocation] = []
for candidate in self._workflow_invocations_by_key.values():
if all(candidate is not existing for existing in invocations):
invocations.append(candidate)
Expand Down Expand Up @@ -791,7 +797,7 @@ def new_span(
workflow_agents: Mapping[str, BaseWorkflowAgent] | None = None
workflow_run_id: str | None = None
workflow_agent: BaseWorkflowAgent | None = None
workflow_agent_invocation: AgentInvocation | None = None
workflow_agent_invocation: LocalAgentInvocation | None = None
workflow_handoff = False
member_agent_step = False
workflow_tool_token: (
Expand Down Expand Up @@ -926,7 +932,7 @@ def new_span(
else None
)
if active_invocation is None and parent is not None:
if isinstance(parent._invocation, AgentInvocation):
if isinstance(parent._invocation, LocalAgentInvocation):
active_invocation = parent._invocation
workflow_run_id = (
tags.get("llamaindex.run_id") if tags is not None else None
Expand Down Expand Up @@ -1050,7 +1056,7 @@ def _expect_workflow_tools(
def _release_workflow_invocation(
self,
span: _LlamaIndexInvocation,
invocation: AgentInvocation,
invocation: LocalAgentInvocation,
) -> None:
"""Drop a finished member invocation so a later turn opens a new span."""
parent = self.open_spans.get(span.parent_id or "")
Expand Down Expand Up @@ -1119,7 +1125,7 @@ def prepare_to_exit_span(
span.finalize_workflow_agents(
result=result if capture_content else None
)
elif isinstance(span._invocation, AgentInvocation):
elif isinstance(span._invocation, LocalAgentInvocation):
span.reset_tool_attributes()
span.reset_workflow_tool()
if self._handler.should_capture_content():
Expand Down Expand Up @@ -1202,14 +1208,14 @@ def prepare_to_drop_span(
span._invocation.stop()
else:
span._invocation.fail(err)
elif isinstance(span._invocation, AgentInvocation):
elif isinstance(span._invocation, LocalAgentInvocation):
_finish_member_agent(span._invocation, err)
elif err is None:
span._invocation.stop()
else:
span._invocation.fail(err)
span.reset_tool_parent_context()
if isinstance(span._invocation, AgentInvocation):
if isinstance(span._invocation, LocalAgentInvocation):
span.reset_workflow_agent()
self._release_workflow_invocation(span, span._invocation)
elif isinstance(span._invocation, ToolInvocation):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@

class AgentScenario(Scenario):
expected_spans = {"invoke_agent": 3, "execute_tool": 1}
expected_metrics = ("gen_ai.client.operation.duration",)
expected_metrics = (
"gen_ai.invoke_agent.duration",
"gen_ai.execute_tool.duration",
)

def run(
self,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ class WorkflowScenario(Scenario):
"execute_tool": 1,
}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.invoke_workflow.duration",
"gen_ai.invoke_agent.duration",
"gen_ai.execute_tool.duration",
)

def run(
Expand Down
Loading
Loading