diff --git a/util/opentelemetry-util-genai/.changelog/634.added b/util/opentelemetry-util-genai/.changelog/634.added new file mode 100644 index 000000000..70a580f69 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/634.added @@ -0,0 +1 @@ +Add agent call count metrics on AgentInvocation. diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py index a8bbb45ff..0584dab59 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_agent_invocation.py @@ -96,6 +96,9 @@ def __init__( self.input_tokens: int | None = None self.output_tokens: int | None = None + self.inference_calls: int | None = None + self.tool_calls: int | None = None + self.input_messages: list[InputMessage] = [] self.output_messages: list[OutputMessage] = [] self.system_instruction: ( @@ -173,6 +176,22 @@ def _apply_finish(self, error: Error | None = None) -> None: ) self._record_metrics() + def _record_call_metrics( + self, attributes: dict[str, AttributeValue] + ) -> None: + if self.inference_calls is not None: + self._instruments.invoke_agent_inference_calls.record( + self.inference_calls, + attributes=attributes, + context=self._span_context, + ) + if self.tool_calls is not None: + self._instruments.invoke_agent_tool_calls.record( + self.tool_calls, + attributes=attributes, + context=self._span_context, + ) + @abstractmethod def _record_metrics(self) -> None: """Record invocation metrics.""" @@ -234,11 +253,13 @@ def _record_metrics(self) -> None: timeit.default_timer() - self._monotonic_start_s, 0.0, ) + attributes = self._get_metric_attributes() self._instruments.invoke_agent_duration.record( duration_seconds, - attributes=self._get_metric_attributes(), + attributes=attributes, context=self._span_context, ) + self._record_call_metrics(attributes) class RemoteAgentInvocation(AgentInvocation): @@ -375,5 +396,12 @@ def _get_metric_token_counts(self) -> dict[str, int]: ) return counts + def _get_call_metric_attributes(self) -> dict[str, AttributeValue]: + attrs = dict(self._get_metric_attributes()) + if self._agent_name is not None: + attrs[GenAI.GEN_AI_AGENT_NAME] = self._agent_name + return attrs + def _record_metrics(self) -> None: self._record_client_metrics() + self._record_call_metrics(self._get_call_metric_attributes()) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py index ed43e85a0..5068440ed 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_instruments.py @@ -63,6 +63,10 @@ _GEN_AI_EXECUTE_TOOL_DURATION: Final = "gen_ai.execute_tool.duration" _GEN_AI_INVOKE_WORKFLOW_DURATION: Final = "gen_ai.invoke_workflow.duration" _GEN_AI_INVOKE_AGENT_DURATION: Final = "gen_ai.invoke_agent.duration" +_GEN_AI_INVOKE_AGENT_INFERENCE_CALLS: Final = ( + "gen_ai.invoke_agent.inference_calls" +) +_GEN_AI_INVOKE_AGENT_TOOL_CALLS: Final = "gen_ai.invoke_agent.tool_calls" _GEN_AI_INVOKE_AGENT_DURATION_BUCKETS: Final = [ 0.1, 0.2, @@ -126,6 +130,22 @@ def __init__(self, meter: Meter) -> None: unit="s", explicit_bucket_boundaries_advisory=_GEN_AI_INVOKE_AGENT_DURATION_BUCKETS, ) + self.invoke_agent_inference_calls: Histogram = meter.create_histogram( + name=_GEN_AI_INVOKE_AGENT_INFERENCE_CALLS, + description=( + "The number of inference (model) calls a GenAI agent makes" + " during a single invocation." + ), + unit="{inference_call}", + ) + self.invoke_agent_tool_calls: Histogram = meter.create_histogram( + name=_GEN_AI_INVOKE_AGENT_TOOL_CALLS, + description=( + "The number of tool calls a GenAI agent makes during a single" + " invocation." + ), + unit="{tool_call}", + ) __all__ = ["_Instruments"] diff --git a/util/opentelemetry-util-genai/tests/test_handler_agent.py b/util/opentelemetry-util-genai/tests/test_handler_agent.py index 18a0c5db6..558debdc2 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_agent.py +++ b/util/opentelemetry-util-genai/tests/test_handler_agent.py @@ -731,6 +731,152 @@ def test_fail_agent_records_error_metric(self) -> None: ) self.assertAlmostEqual(duration_point.sum, 1.0, places=3) + def test_agent_records_inference_and_tool_calls(self) -> None: + handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + invocation = handler.invoke_local_agent( + agent_name="SearchAgent", + request_model="gpt-4o", + ) + invocation.inference_calls = 3 + invocation.tool_calls = 5 + invocation.stop() + + metrics = self._harvest_metrics() + + self.assertIn("gen_ai.invoke_agent.inference_calls", metrics) + inf_points = metrics["gen_ai.invoke_agent.inference_calls"] + self.assertEqual(len(inf_points), 1) + self.assertAlmostEqual(inf_points[0].sum, 3.0, places=3) + self.assertEqual( + inf_points[0].attributes[GenAI.GEN_AI_AGENT_NAME], + "SearchAgent", + ) + self.assertEqual( + inf_points[0].attributes[GenAI.GEN_AI_REQUEST_MODEL], + "gpt-4o", + ) + + self.assertIn("gen_ai.invoke_agent.tool_calls", metrics) + tool_points = metrics["gen_ai.invoke_agent.tool_calls"] + self.assertEqual(len(tool_points), 1) + self.assertAlmostEqual(tool_points[0].sum, 5.0, places=3) + self.assertEqual( + tool_points[0].attributes[GenAI.GEN_AI_AGENT_NAME], + "SearchAgent", + ) + self.assertEqual( + tool_points[0].attributes[GenAI.GEN_AI_REQUEST_MODEL], + "gpt-4o", + ) + + def test_agent_without_calls_does_not_record_call_metrics(self) -> None: + handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + invocation = handler.invoke_local_agent(agent_name="MinimalAgent") + invocation.stop() + + metrics = self._harvest_metrics() + self.assertIn("gen_ai.invoke_agent.duration", metrics) + self.assertNotIn("gen_ai.invoke_agent.inference_calls", metrics) + self.assertNotIn("gen_ai.invoke_agent.tool_calls", metrics) + + def test_agent_records_zero_calls(self) -> None: + handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + invocation = handler.invoke_local_agent(agent_name="ZeroAgent") + invocation.inference_calls = 1 + invocation.tool_calls = 0 + invocation.stop() + + metrics = self._harvest_metrics() + self.assertIn("gen_ai.invoke_agent.inference_calls", metrics) + self.assertAlmostEqual( + metrics["gen_ai.invoke_agent.inference_calls"][0].sum, + 1.0, + places=3, + ) + self.assertIn("gen_ai.invoke_agent.tool_calls", metrics) + self.assertAlmostEqual( + metrics["gen_ai.invoke_agent.tool_calls"][0].sum, + 0.0, + places=3, + ) + + def test_remote_agent_records_calls_with_server_attrs(self) -> None: + handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + invocation = handler.invoke_remote_agent( + provider="openai", + request_model="gpt-4o", + server_address="api.openai.com", + server_port=443, + agent_name="RemoteAgent", + ) + invocation.inference_calls = 2 + invocation.tool_calls = 1 + invocation.stop() + + metrics = self._harvest_metrics() + inf_points = metrics["gen_ai.invoke_agent.inference_calls"] + self.assertEqual(len(inf_points), 1) + self.assertEqual( + inf_points[0].attributes["server.address"], + "api.openai.com", + ) + self.assertEqual(inf_points[0].attributes["server.port"], 443) + self.assertEqual( + inf_points[0].attributes[GenAI.GEN_AI_PROVIDER_NAME], + "openai", + ) + self.assertEqual( + inf_points[0].attributes[GenAI.GEN_AI_AGENT_NAME], + "RemoteAgent", + ) + self.assertEqual( + inf_points[0].attributes[GenAI.GEN_AI_OPERATION_NAME], + "invoke_agent", + ) + tool_points = metrics["gen_ai.invoke_agent.tool_calls"] + self.assertEqual(len(tool_points), 1) + self.assertAlmostEqual(tool_points[0].sum, 1.0, places=3) + + def test_failed_agent_records_calls_with_error_type(self) -> None: + handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + invocation = handler.invoke_local_agent(agent_name="FailingAgent") + invocation.inference_calls = 2 + invocation.fail(Error(message="boom", type="ValueError")) + + metrics = self._harvest_metrics() + self.assertIn("gen_ai.invoke_agent.inference_calls", metrics) + inf_point = metrics["gen_ai.invoke_agent.inference_calls"][0] + self.assertEqual(inf_point.attributes.get("error.type"), "ValueError") + self.assertAlmostEqual(inf_point.sum, 2.0, places=3) + + def test_agent_name_omitted_from_metrics_when_none(self) -> None: + handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + invocation = handler.invoke_local_agent() + invocation.inference_calls = 1 + invocation.stop() + + metrics = self._harvest_metrics() + inf_point = metrics["gen_ai.invoke_agent.inference_calls"][0] + self.assertNotIn(GenAI.GEN_AI_AGENT_NAME, inf_point.attributes) + def _harvest_metrics(self): metrics = self.get_sorted_metrics() metrics_by_name = {}