Skip to content

Add agent call count metrics on AgentInvocation (#597) #634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dlowzzxx wants to merge 1 commit into
base: main
Choose a base branch
from
+196 −1
Open

Add agent call count metrics on AgentInvocation (#597) #634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions util/opentelemetry-util-genai/.changelog/634.added
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add agent call count metrics on AgentInvocation.
30 changes: 29 additions & 1 deletion util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ def __init__(
self.input_tokens: int | None = None
self.output_tokens: int | None = None

self.inference_calls: int | None = None
self.tool_calls: int | None = None

self.input_messages: list[InputMessage] = []
self.output_messages: list[OutputMessage] = []
self.system_instruction: (
Expand Down Expand Up @@ -173,6 +176,22 @@ def _apply_finish(self, error: Error | None = None) -> None:
)
self._record_metrics()

def _record_call_metrics(
self, attributes: dict[str, AttributeValue]
) -> None:
if self.inference_calls is not None:
self._instruments.invoke_agent_inference_calls.record(
self.inference_calls,
attributes=attributes,
context=self._span_context,
)
if self.tool_calls is not None:
self._instruments.invoke_agent_tool_calls.record(
self.tool_calls,
attributes=attributes,
context=self._span_context,
)

@abstractmethod
def _record_metrics(self) -> None:
"""Record invocation metrics."""
Expand Down Expand Up @@ -234,11 +253,13 @@ def _record_metrics(self) -> None:
timeit.default_timer() - self._monotonic_start_s,
0.0,
)
attributes = self._get_metric_attributes()
self._instruments.invoke_agent_duration.record(
duration_seconds,
attributes=self._get_metric_attributes(),
attributes=attributes,
context=self._span_context,
)
self._record_call_metrics(attributes)


class RemoteAgentInvocation(AgentInvocation):
Expand Down Expand Up @@ -375,5 +396,12 @@ def _get_metric_token_counts(self) -> dict[str, int]:
)
return counts

def _get_call_metric_attributes(self) -> dict[str, AttributeValue]:
attrs = dict(self._get_metric_attributes())
if self._agent_name is not None:
attrs[GenAI.GEN_AI_AGENT_NAME] = self._agent_name
return attrs

def _record_metrics(self) -> None:
self._record_client_metrics()
self._record_call_metrics(self._get_call_metric_attributes())
20 changes: 20 additions & 0 deletions util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@
_GEN_AI_EXECUTE_TOOL_DURATION: Final = "gen_ai.execute_tool.duration"
_GEN_AI_INVOKE_WORKFLOW_DURATION: Final = "gen_ai.invoke_workflow.duration"
_GEN_AI_INVOKE_AGENT_DURATION: Final = "gen_ai.invoke_agent.duration"
_GEN_AI_INVOKE_AGENT_INFERENCE_CALLS: Final = (
"gen_ai.invoke_agent.inference_calls"
)
_GEN_AI_INVOKE_AGENT_TOOL_CALLS: Final = "gen_ai.invoke_agent.tool_calls"
_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS: Final = [
0.1,
0.2,
Expand Down Expand Up @@ -126,6 +130,22 @@ def __init__(self, meter: Meter) -> None:
unit="s",
explicit_bucket_boundaries_advisory=_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS,
)
self.invoke_agent_inference_calls: Histogram = meter.create_histogram(
name=_GEN_AI_INVOKE_AGENT_INFERENCE_CALLS,
description=(
"The number of inference (model) calls a GenAI agent makes"
" during a single invocation."
),
unit="{inference_call}",
)
self.invoke_agent_tool_calls: Histogram = meter.create_histogram(
name=_GEN_AI_INVOKE_AGENT_TOOL_CALLS,
description=(
"The number of tool calls a GenAI agent makes during a single"
" invocation."
),
unit="{tool_call}",
)


__all__ = ["_Instruments"]
146 changes: 146 additions & 0 deletions util/opentelemetry-util-genai/tests/test_handler_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,152 @@ def test_fail_agent_records_error_metric(self) -> None:
)
self.assertAlmostEqual(duration_point.sum, 1.0, places=3)

def test_agent_records_inference_and_tool_calls(self) -> None:
handler = TelemetryHandler(
tracer_provider=self.tracer_provider,
meter_provider=self.meter_provider,
)
invocation = handler.invoke_local_agent(
agent_name="SearchAgent",
request_model="gpt-4o",
)
invocation.inference_calls = 3
invocation.tool_calls = 5
invocation.stop()

metrics = self._harvest_metrics()

self.assertIn("gen_ai.invoke_agent.inference_calls", metrics)
inf_points = metrics["gen_ai.invoke_agent.inference_calls"]
self.assertEqual(len(inf_points), 1)
self.assertAlmostEqual(inf_points[0].sum, 3.0, places=3)
self.assertEqual(
inf_points[0].attributes[GenAI.GEN_AI_AGENT_NAME],
"SearchAgent",
)
self.assertEqual(
inf_points[0].attributes[GenAI.GEN_AI_REQUEST_MODEL],
"gpt-4o",
)

self.assertIn("gen_ai.invoke_agent.tool_calls", metrics)
tool_points = metrics["gen_ai.invoke_agent.tool_calls"]
self.assertEqual(len(tool_points), 1)
self.assertAlmostEqual(tool_points[0].sum, 5.0, places=3)
self.assertEqual(
tool_points[0].attributes[GenAI.GEN_AI_AGENT_NAME],
"SearchAgent",
)
self.assertEqual(
tool_points[0].attributes[GenAI.GEN_AI_REQUEST_MODEL],
"gpt-4o",
)

def test_agent_without_calls_does_not_record_call_metrics(self) -> None:
handler = TelemetryHandler(
tracer_provider=self.tracer_provider,
meter_provider=self.meter_provider,
)
invocation = handler.invoke_local_agent(agent_name="MinimalAgent")
invocation.stop()

metrics = self._harvest_metrics()
self.assertIn("gen_ai.invoke_agent.duration", metrics)
self.assertNotIn("gen_ai.invoke_agent.inference_calls", metrics)
self.assertNotIn("gen_ai.invoke_agent.tool_calls", metrics)

def test_agent_records_zero_calls(self) -> None:
handler = TelemetryHandler(
tracer_provider=self.tracer_provider,
meter_provider=self.meter_provider,
)
invocation = handler.invoke_local_agent(agent_name="ZeroAgent")
invocation.inference_calls = 1
invocation.tool_calls = 0
invocation.stop()

metrics = self._harvest_metrics()
self.assertIn("gen_ai.invoke_agent.inference_calls", metrics)
self.assertAlmostEqual(
metrics["gen_ai.invoke_agent.inference_calls"][0].sum,
1.0,
places=3,
)
self.assertIn("gen_ai.invoke_agent.tool_calls", metrics)
self.assertAlmostEqual(
metrics["gen_ai.invoke_agent.tool_calls"][0].sum,
0.0,
places=3,
)

def test_remote_agent_records_calls_with_server_attrs(self) -> None:
handler = TelemetryHandler(
tracer_provider=self.tracer_provider,
meter_provider=self.meter_provider,
)
invocation = handler.invoke_remote_agent(
provider="openai",
request_model="gpt-4o",
server_address="api.openai.com",
server_port=443,
agent_name="RemoteAgent",
)
invocation.inference_calls = 2
invocation.tool_calls = 1
invocation.stop()

metrics = self._harvest_metrics()
inf_points = metrics["gen_ai.invoke_agent.inference_calls"]
self.assertEqual(len(inf_points), 1)
self.assertEqual(
inf_points[0].attributes["server.address"],
"api.openai.com",
)
self.assertEqual(inf_points[0].attributes["server.port"], 443)
self.assertEqual(
inf_points[0].attributes[GenAI.GEN_AI_PROVIDER_NAME],
"openai",
)
self.assertEqual(
inf_points[0].attributes[GenAI.GEN_AI_AGENT_NAME],
"RemoteAgent",
)
self.assertEqual(
inf_points[0].attributes[GenAI.GEN_AI_OPERATION_NAME],
"invoke_agent",
)
tool_points = metrics["gen_ai.invoke_agent.tool_calls"]
self.assertEqual(len(tool_points), 1)
self.assertAlmostEqual(tool_points[0].sum, 1.0, places=3)

def test_failed_agent_records_calls_with_error_type(self) -> None:
handler = TelemetryHandler(
tracer_provider=self.tracer_provider,
meter_provider=self.meter_provider,
)
invocation = handler.invoke_local_agent(agent_name="FailingAgent")
invocation.inference_calls = 2
invocation.fail(Error(message="boom", type="ValueError"))

metrics = self._harvest_metrics()
self.assertIn("gen_ai.invoke_agent.inference_calls", metrics)
inf_point = metrics["gen_ai.invoke_agent.inference_calls"][0]
self.assertEqual(inf_point.attributes.get("error.type"), "ValueError")
self.assertAlmostEqual(inf_point.sum, 2.0, places=3)

def test_agent_name_omitted_from_metrics_when_none(self) -> None:
handler = TelemetryHandler(
tracer_provider=self.tracer_provider,
meter_provider=self.meter_provider,
)
invocation = handler.invoke_local_agent()
invocation.inference_calls = 1
invocation.stop()

metrics = self._harvest_metrics()
inf_point = metrics["gen_ai.invoke_agent.inference_calls"][0]
self.assertNotIn(GenAI.GEN_AI_AGENT_NAME, inf_point.attributes)

def _harvest_metrics(self):
metrics = self.get_sorted_metrics()
metrics_by_name = {}
Expand Down
Loading