diff --git a/manifests/python.yml b/manifests/python.yml index 775bb6a74de..ab2c4c81451 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1081,15 +1081,8 @@ manifest: tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_apm.py::TestAnthropicApmMessages: v3.16.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages: v3.16.0 - tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create: missing_feature # ephemeral cache TTL metrics not yet released - tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_content_block: missing_feature # ephemeral cache TTL metrics not yet released tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) - ? tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_multiple_system_prompts - : missing_feature # ephemeral cache TTL metrics not yet released - tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_redact_image_input: missing_feature # ephemeral cache TTL metrics not yet released - tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_stream_method: missing_feature # ephemeral cache TTL metrics not yet released tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_tool_result: bug (MLOB-1234) - tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_with_tools: missing_feature # ephemeral cache TTL metrics not yet released tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py::TestGoogleGenAiEmbedContent: v3.11.0 tests/integration_frameworks/llm/google_genai/test_google_genai_apm.py::TestGoogleGenAiGenerateContent: v3.11.0 tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiEmbedContent: v3.13.0 diff --git a/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py b/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py index c74a58e9bb9..1bf59e370d2 100644 --- a/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py +++ b/tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py @@ -81,6 +81,8 @@ def test_create(self, test_agent: TestAgentAPI, test_client: FrameworkTestClient "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -124,6 +126,8 @@ def test_create_stream_method(self, test_agent: TestAgentAPI, test_client: Frame "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -181,6 +185,8 @@ def test_create_content_block(self, test_agent: TestAgentAPI, test_client: Frame "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -318,6 +324,8 @@ def test_create_multiple_system_prompts( "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -384,6 +392,8 @@ def test_create_with_tools(self, test_agent: TestAgentAPI, test_client: Framewor "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -501,6 +511,8 @@ def test_create_tool_result(self, test_agent: TestAgentAPI, test_client: Framewo "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -574,6 +586,8 @@ def test_create_redact_image_input( "total_tokens": mock.ANY, "cache_read_input_tokens": mock.ANY, "cache_write_input_tokens": mock.ANY, + "ephemeral_1h_input_tokens": mock.ANY, + "ephemeral_5m_input_tokens": mock.ANY, }, ) @@ -630,4 +644,5 @@ def test_create_prompt_caching( write_span_event, read_span_event = span_events assert write_span_event["metrics"]["cache_write_input_tokens"] == 6163 + assert write_span_event["metrics"]["ephemeral_5m_input_tokens"] == 6163 assert read_span_event["metrics"]["cache_read_input_tokens"] == 6163