From 9a24b51760a136adb46d6467afdef78f57fdbdca Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 23 Sep 2026 16:16:49 -0700 Subject: [PATCH 01/16] Address feedback --- .../genai/langchain/callback_handler.py | 152 ++++++++- .../genai/langchain/invocation_manager.py | 49 ++- ...test_chat_openai_chat_prompt_template.yaml | 199 +++++++++++ .../test_chat_openai_prompt_template.yaml | 205 +++++++++++ .../tests/test_callback_handler.py | 319 +++++++++++++++++- .../tests/test_invocation_manager.py | 54 +++ .../tests/test_llm_call.py | 193 +++++++++++ 7 files changed, 1168 insertions(+), 3 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_chat_prompt_template.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_prompt_template.yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index a9876ebba..565cbfc8e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -22,6 +22,7 @@ ) from opentelemetry.instrumentation.genai.langchain.invocation_manager import ( _InvocationManager, + _PromptContext, ) from opentelemetry.instrumentation.genai.langchain.operation_mapping import ( OperationName, @@ -61,6 +62,7 @@ object_to_any_value, tool_arguments_to_any_value, ) +from opentelemetry.util.genai.utils import gen_ai_json_dumps SUPPORTED_RAPI_RESPONSE_HEADERS = ("x-ms-served-model",) @@ -71,6 +73,135 @@ "conversation_id", ) +_PROMPT_TEMPLATE_TYPES = {"ChatPromptTemplate", "PromptTemplate"} +_INVALID_PROMPT_VALUE = object() + + +def _prompt_template_type(serialized: Mapping[str, Any]) -> str | None: + serialized_id = serialized.get("id") + if not isinstance(serialized_id, Sequence) or isinstance( + serialized_id, str + ): + return None + serialized_id = cast(Sequence[object], serialized_id) + template_type = serialized_id[-1] if serialized_id else None + if not isinstance(template_type, str): + return None + return template_type if template_type in _PROMPT_TEMPLATE_TYPES else None + + +def _string_sequence(value: object) -> list[str]: + if not isinstance(value, Sequence) or isinstance(value, str): + return [] + return [ + item + for item in cast(Sequence[object], value) + if isinstance(item, str) + ] + + +def _prompt_variable_value(value: object) -> object: + if isinstance(value, BaseMessage): + value = value.model_dump() + elif isinstance(value, Mapping): + value = cast(Mapping[object, object], value) + if value.get("type") == "not_implemented": + return _INVALID_PROMPT_VALUE + normalized_mapping: dict[object, object] = {} + for key, item in value.items(): + normalized_item = _prompt_variable_value(item) + if normalized_item is _INVALID_PROMPT_VALUE: + return _INVALID_PROMPT_VALUE + normalized_mapping[key] = normalized_item + value = normalized_mapping + elif isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ): + normalized_sequence: list[object] = [] + for item in cast(Sequence[object], value): + normalized_item = _prompt_variable_value(item) + if normalized_item is _INVALID_PROMPT_VALUE: + return _INVALID_PROMPT_VALUE + normalized_sequence.append(normalized_item) + value = normalized_sequence + elif callable(value): + return _INVALID_PROMPT_VALUE + + try: + gen_ai_json_dumps(value) + except (TypeError, ValueError, OverflowError): + return _INVALID_PROMPT_VALUE + return value + + +def _extract_prompt_context( + serialized: Mapping[str, Any], + inputs: object, + metadata: Mapping[str, Any] | None, + *, + capture_variables: bool = True, +) -> _PromptContext | None: + template_type = _prompt_template_type(serialized) + if template_type is None: + return None + + raw_serialized_kwargs = serialized.get("kwargs") + if not isinstance(raw_serialized_kwargs, Mapping): + serialized_kwargs: Mapping[str, object] = {} + else: + serialized_kwargs = cast( + Mapping[str, object], raw_serialized_kwargs + ) + + name = (metadata or {}).get("prompt_name") + if not isinstance(name, str): + name = serialized_kwargs.get("name") + if not isinstance(name, str): + name = serialized.get("name") + if not isinstance(name, str) or name in _PROMPT_TEMPLATE_TYPES: + name = None + + if not capture_variables: + return _PromptContext(name=name, variables={}) + + required_variables = _string_sequence( + serialized_kwargs.get("input_variables") + ) + declared_variables = set(required_variables) + declared_variables.update( + _string_sequence(serialized_kwargs.get("optional_variables")) + ) + + partial_variables = serialized_kwargs.get("partial_variables") + if isinstance(partial_variables, Mapping): + partial_variables = cast(Mapping[object, object], partial_variables) + declared_variables.update( + key for key in partial_variables if isinstance(key, str) + ) + + if isinstance(inputs, Mapping): + runtime_variables: Mapping[object, object] = cast( + Mapping[object, object], inputs + ) + elif len(required_variables) == 1: + runtime_variables = {required_variables[0]: inputs} + else: + runtime_variables = {} + + effective_variables: dict[str, object] = {} + for source in (partial_variables, runtime_variables): + if not isinstance(source, Mapping): + continue + source = cast(Mapping[object, object], source) + for key, value in source.items(): + if not isinstance(key, str) or key not in declared_variables: + continue + normalized_value = _prompt_variable_value(value) + if normalized_value is not _INVALID_PROMPT_VALUE: + effective_variables[key] = normalized_value + + return _PromptContext(name=name, variables=effective_variables) + def _conversation_id(metadata: dict[str, Any] | None) -> str | None: """Return the conversation id from a run's own metadata.""" @@ -178,7 +309,7 @@ def __init__( def on_chain_start( self, - serialized: dict[str, Any], + serialized: dict[str, Any] | None, inputs: dict[str, Any], *, run_id: UUID, @@ -187,6 +318,7 @@ def on_chain_start( metadata: dict[str, Any] | None = None, **kwargs: Any, ) -> Any: + serialized = serialized or {} parent_agent_name, ancestor_agent_names = self._find_agent_context( parent_run_id ) @@ -298,6 +430,16 @@ def on_chain_start( run_id, parent_run_id, None ) + if prompt_context := _extract_prompt_context( + serialized, + inputs, + metadata, + capture_variables=capture_content, + ): + self._invocation_manager.set_prompt_context( + run_id, prompt_context + ) + def on_chain_end( self, outputs: dict[str, Any], @@ -306,6 +448,7 @@ def on_chain_end( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + self._invocation_manager.publish_prompt_context(run_id) invocation = self._invocation_manager.get_invocation(run_id=run_id) if invocation is None or not isinstance( invocation, (WorkflowInvocation, LocalAgentInvocation) @@ -350,6 +493,9 @@ def on_chat_model_start( metadata: dict[str, Any] | None = None, **kwargs: Any, ) -> None: + prompt_context = self._invocation_manager.consume_prompt_context( + parent_run_id + ) if "invocation_params" in kwargs: params = ( kwargs["invocation_params"].get("params") @@ -442,6 +588,10 @@ def on_chat_model_start( conversation_id=_conversation_id(metadata), _attach_to_context=False, ) + if prompt_context is not None: + llm_invocation.prompt_name = prompt_context.name + llm_invocation.prompt_variables = prompt_context.variables + llm_invocation.conversation_id = _conversation_id(metadata) llm_invocation.input_messages = input_messages llm_invocation.top_p = top_p llm_invocation.top_k = top_k diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py index d4a0fa0fe..c7c3da334 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py @@ -1,6 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +from collections.abc import Mapping from dataclasses import dataclass, field from typing import TypeAlias from uuid import UUID @@ -14,7 +15,7 @@ WorkflowInvocation, ) -__all__ = ["_InvocationManager"] +__all__ = ["_InvocationManager", "_PromptContext"] _AnyInvocation: TypeAlias = ( InferenceInvocation @@ -25,6 +26,12 @@ ) +@dataclass(frozen=True) +class _PromptContext: + name: str | None + variables: Mapping[str, object] + + @dataclass class _InvocationState: invocation: _AnyInvocation | None @@ -32,6 +39,10 @@ class _InvocationState: parent_run_id: UUID | None = None ended: bool = False agent_name: str | None = None + prompt_context: _PromptContext | None = None + pending_prompt_contexts: list[_PromptContext] = field( + default_factory=lambda: [] + ) class _InvocationManager: @@ -80,6 +91,42 @@ def get_parent_context(self, parent_run_id: UUID | None) -> Context | None: current = self.get_parent_run_id(current) return None + def set_prompt_context( + self, run_id: UUID, prompt_context: _PromptContext + ) -> None: + invocation_state = self._invocations.get(run_id) + if invocation_state is not None: + invocation_state.prompt_context = prompt_context + + def publish_prompt_context(self, run_id: UUID) -> None: + invocation_state = self._invocations.get(run_id) + if invocation_state is None or invocation_state.prompt_context is None: + return + parent_run_id = invocation_state.parent_run_id + parent_state = ( + self._invocations.get(parent_run_id) + if parent_run_id is not None + else None + ) + if parent_state is not None: + parent_state.pending_prompt_contexts.append( + invocation_state.prompt_context + ) + invocation_state.prompt_context = None + + def consume_prompt_context( + self, parent_run_id: UUID | None + ) -> _PromptContext | None: + current_run_id = parent_run_id + while current_run_id is not None: + invocation_state = self._invocations.get(current_run_id) + if invocation_state is None: + return None + if invocation_state.pending_prompt_contexts: + return invocation_state.pending_prompt_contexts.pop(0) + current_run_id = invocation_state.parent_run_id + return None + def delete_invocation_state(self, run_id: UUID) -> None: invocation_state = self._invocations.get(run_id) if not invocation_state: diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_chat_prompt_template.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_chat_prompt_template.yaml new file mode 100644 index 000000000..b8d94ac7c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_chat_prompt_template.yaml @@ -0,0 +1,199 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "Answer weather questions in a concise style.", + "role": "system" + }, + { + "content": "What's the weather like in Seattle?", + "role": "user" + } + ], + "model": "gpt-4.1", + "max_completion_tokens": 100, + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '214' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.2.0 + x-stainless-arch: + - other:amd64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Windows + x-stainless-package-version: + - 3.2.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.15 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "choices": [ + { + "content_filter_result": { + "error": { + "code": "content_filter_error", + "message": "The contents are not filtered" + } + }, + "content_filter_results": { + "defender_for_ai": { + "detected": false, + "filtered": false, + "details": {} + }, + "protected_material_code": { + "detected": false, + "filtered": false + }, + "protected_material_text": { + "detected": false, + "filtered": false + } + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Seattle typically has mild, cloudy weather with some rain. In June, expect temperatures around 60\u201375\u00b0F (16\u201324\u00b0C), with occasional showers and partly cloudy skies. For current conditions, please check a live weather service.", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 1789507840, + "id": "chatcmpl-EOV1c1Tpo0m0fRufLteTkNK2ijMaY", + "model": "gpt-4.1-2025-04-14", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "defender_for_ai": { + "detected": false, + "filtered": false, + "details": {} + }, + "hate": { + "filtered": false, + "severity": "safe" + }, + "jailbreak": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "routing": { + "serving_pipereplica": "d20260822223544-9f100e74-default-r2-dp0-default" + }, + "service_tier": "default", + "system_fingerprint": "fp_861cf8a3da", + "usage": { + "completion_tokens": 47, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "latency_checkpoint": { + "engine_tbt_ms": 7, + "engine_ttft_ms": 30, + "engine_ttlt_ms": 355, + "pre_inference_ms": 255, + "service_tbt_ms": 7, + "service_ttft_ms": 655, + "service_ttlt_ms": 973, + "user_visible_ttft_ms": 400 + }, + "prompt_tokens": 26, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 73 + } + } + headers: + content-length: + - '1795' + content-type: + - application/json + date: + - Tue, 15 Sep 2026 21:30:41 GMT + skip-error-remapping: + - 'true' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-ratelimit-abusepenalty-active: + - 'False' + x-ratelimit-key: + - gpt-4.1 + x-ratelimit-limit-requests: + - '130' + x-ratelimit-limit-tokens: + - '130000' + x-ratelimit-remaining-requests: + - '129' + x-ratelimit-remaining-tokens: + - '129974' + x-ratelimit-renewalperiod-requests: + - '60' + x-ratelimit-renewalperiod-tokens: + - '60' + x-ratelimit-reset-requests: + - '0' + x-ratelimit-reset-tokens: + - '0' + x-request-id: + - f4c019da-4898-4485-af07-df675f3496c9 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_prompt_template.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_prompt_template.yaml new file mode 100644 index 000000000..141563e4a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_prompt_template.yaml @@ -0,0 +1,205 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "Answer this weather question briefly: What's the weather like in Seattle?", + "role": "user" + } + ], + "model": "gpt-4.1", + "max_completion_tokens": 100, + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.2.0 + x-stainless-arch: + - other:amd64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Windows + x-stainless-package-version: + - 3.2.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.15 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "choices": [ + { + "content_filter_results": { + "defender_for_ai": { + "detected": false, + "filtered": false, + "details": {} + }, + "hate": { + "filtered": false, + "severity": "safe" + }, + "protected_material_code": { + "detected": false, + "filtered": false + }, + "protected_material_text": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Seattle's weather is typically cool and cloudy, with a chance of rain. As of early summer, expect mild temperatures (60s\u201370s \u00b0F) and occasional sunny days. Check a local forecast for the latest conditions.", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 1789507838, + "id": "chatcmpl-EOV1axDKQtoU9haVagJyiP0tLgYEk", + "model": "gpt-4.1-2025-04-14", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "defender_for_ai": { + "detected": false, + "filtered": false, + "details": {} + }, + "hate": { + "filtered": false, + "severity": "safe" + }, + "jailbreak": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "routing": { + "serving_pipereplica": "d20260822223544-9f100e74-default-r0-dp0-default" + }, + "service_tier": "default", + "system_fingerprint": "fp_861cf8a3da", + "usage": { + "completion_tokens": 46, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "latency_checkpoint": { + "engine_tbt_ms": 12, + "engine_ttft_ms": 39, + "engine_ttlt_ms": 594, + "pre_inference_ms": 204, + "service_tbt_ms": 12, + "service_ttft_ms": 597, + "service_ttlt_ms": 1136, + "user_visible_ttft_ms": 393 + }, + "prompt_tokens": 20, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 66 + } + } + headers: + content-length: + - '1856' + content-type: + - application/json + date: + - Tue, 15 Sep 2026 21:30:39 GMT + skip-error-remapping: + - 'true' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-ratelimit-abusepenalty-active: + - 'False' + x-ratelimit-key: + - gpt-4.1 + x-ratelimit-limit-requests: + - '130' + x-ratelimit-limit-tokens: + - '130000' + x-ratelimit-remaining-requests: + - '129' + x-ratelimit-remaining-tokens: + - '129980' + x-ratelimit-renewalperiod-requests: + - '60' + x-ratelimit-renewalperiod-tokens: + - '60' + x-ratelimit-reset-requests: + - '0' + x-ratelimit-reset-tokens: + - '0' + x-request-id: + - a640ace2-8c0c-4ff7-8fcd-08d3ce5df557 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 13b3a809c..7e031c80e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -9,9 +9,11 @@ import json import math import uuid +from unittest import mock import pytest from langchain_core.documents import Document +from langchain_core.language_models.fake_chat_models import FakeListChatModel from langchain_core.messages import ( AIMessage, AIMessageChunk, @@ -33,7 +35,7 @@ LLMResult, ) from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnableConfig, RunnableLambda +from langchain_core.runnables import RunnableConfig, RunnableLambda, RunnableParallel from tests._api import ( FakeChatModel, FakeRetriever, @@ -48,6 +50,7 @@ OpenTelemetryLangChainCallbackHandler, _document_to_retrieval_document, _extract_document_score, + _extract_prompt_context, ) from opentelemetry.instrumentation.genai.langchain.utils import ( _legacy_function_call_request, @@ -75,6 +78,11 @@ from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.trace import StatusCode from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import ( + InferenceInvocation, + LocalAgentInvocation, + WorkflowInvocation, +) from opentelemetry.util.genai.types import ( BlobPart, ContentCapturingMode, @@ -88,6 +96,147 @@ ) +def _make_agent_inv_mock() -> mock.MagicMock: + agent_inv = mock.MagicMock(spec=LocalAgentInvocation) + agent_inv.agent_id = None + return agent_inv + + +def _make_handler(): + telemetry = mock.MagicMock() + workflow_inv = mock.MagicMock(spec=WorkflowInvocation) + telemetry.workflow.return_value = workflow_inv + agent_inv = _make_agent_inv_mock() + telemetry.invoke_local_agent.return_value = agent_inv + handler = OpenTelemetryLangChainCallbackHandler(telemetry) + return handler, telemetry, workflow_inv, agent_inv + + +def _run_id(): + return uuid.uuid4() + + +class TestExtractPromptContext: + @pytest.mark.parametrize( + "template_type", ["PromptTemplate", "ChatPromptTemplate"] + ) + def test_extracts_name_and_declared_variables( + self, template_type: str + ) -> None: + context = _extract_prompt_context( + { + "id": ["langchain", "prompts", template_type], + "name": "weather", + "kwargs": { + "input_variables": ["question"], + "optional_variables": ["history"], + "partial_variables": {"language": "French"}, + }, + }, + { + "question": "Will it rain?", + "history": [HumanMessage(content="Earlier question")], + "unused": "ignored", + }, + None, + ) + + assert context is not None + assert context.name == "weather" + assert context.variables == { + "question": "Will it rain?", + "history": [ + HumanMessage(content="Earlier question").model_dump() + ], + "language": "French", + } + + def test_name_precedence_and_generic_name_exclusion(self): + serialized = { + "id": ["langchain", "prompts", "PromptTemplate"], + "name": "PromptTemplate", + "kwargs": {"name": "serialized-name"}, + } + + context = _extract_prompt_context( + serialized, {}, {"prompt_name": "metadata-name"} + ) + assert context is not None + assert context.name == "metadata-name" + + unnamed = _extract_prompt_context( + {**serialized, "kwargs": {}}, {}, None + ) + assert unnamed is not None + assert unnamed.name is None + + def test_runtime_values_override_partials_and_scalar_is_mapped(self): + serialized = { + "id": ["langchain", "prompts", "PromptTemplate"], + "kwargs": { + "input_variables": ["question"], + "optional_variables": ["history"], + "partial_variables": {"language": "French"}, + }, + } + + context = _extract_prompt_context(serialized, "runtime", None) + + assert context is not None + assert context.variables == { + "question": "runtime", + "language": "French", + } + + def test_preserves_none_as_json_null(self): + context = _extract_prompt_context( + { + "id": ["langchain", "prompts", "PromptTemplate"], + "kwargs": {"input_variables": ["value"]}, + }, + {"value": None}, + None, + ) + + assert context is not None + assert context.variables == {"value": None} + + def test_skips_unresolved_and_unserializable_values(self): + serialized = { + "id": ["langchain", "prompts", "PromptTemplate"], + "kwargs": { + "partial_variables": { + "callable": lambda: "value", + "unresolved": { + "lc": 1, + "type": "not_implemented", + }, + }, + "input_variables": ["unsupported"], + }, + } + + context = _extract_prompt_context( + serialized, {"unsupported": object()}, None + ) + + assert context is not None + assert context.variables == {} + + def test_ignores_non_prompt_runs(self): + assert ( + _extract_prompt_context( + { + "id": ["langchain", "schema", "RunnableSequence"], + "name": "PromptTemplate", + }, + {"question": "ignored"}, + {"prompt_name": "ignored"}, + ) + is None + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize( @@ -840,6 +989,174 @@ async def aretrieve( ) +class TestPromptContextLifecycle: + @pytest.mark.parametrize("with_transform", [False, True]) + def test_real_lcel_prompt_context_reaches_model( + self, with_transform: bool + ) -> None: + handler, telemetry, _, _ = _make_handler() + telemetry.should_capture_content.return_value = True + prompt = ChatPromptTemplate.from_messages( + [("human", "Hello {name}")] + ).model_copy(update={"name": "greeting"}) + model = FakeListChatModel(responses=["Hello Ada"]).with_config( + metadata={"ls_model_name": "fake-model"} + ) + if with_transform: + (prompt | RunnableLambda(lambda value: value) | model).invoke( + {"name": "Ada"}, config={"callbacks": [handler]} + ) + else: + (prompt | model).invoke( + {"name": "Ada"}, config={"callbacks": [handler]} + ) + + invocation = telemetry.inference.return_value + assert invocation.prompt_name == "greeting" + assert invocation.prompt_variables == {"name": "Ada"} + + def test_real_parallel_lcel_keeps_prompt_contexts_isolated(self) -> None: + handler, telemetry, _, _ = _make_handler() + telemetry.should_capture_content.return_value = True + invocations = [ + mock.MagicMock(spec=InferenceInvocation), + mock.MagicMock(spec=InferenceInvocation), + ] + telemetry.inference.side_effect = invocations + + def branch(prompt_name: str, response: str): + prompt = ChatPromptTemplate.from_messages( + [("human", "{value}")] + ).model_copy(update={"name": prompt_name}) + model = FakeListChatModel(responses=[response]).with_config( + metadata={"ls_model_name": "fake-model"} + ) + return prompt | model + + chain = RunnableParallel( + first=branch("first-prompt", "first-response"), + second=branch("second-prompt", "second-response"), + ) + + chain.invoke({"value": "shared"}, config={"callbacks": [handler]}) + + contexts = { + (invocation.prompt_name, invocation.prompt_variables["value"]) + for invocation in invocations + } + assert contexts == { + ("first-prompt", "shared"), + ("second-prompt", "shared"), + } + + def test_successful_prompt_context_is_applied_to_next_model(self): + handler, telemetry, _, _ = _make_handler() + telemetry.should_capture_content.return_value = True + sequence_id = _run_id() + prompt_id = _run_id() + handler._invocation_manager.add_invocation_state( + sequence_id, None, None + ) + handler.on_chain_start( + serialized={ + "id": ["langchain", "prompts", "PromptTemplate"], + "name": "greeting", + "kwargs": {"input_variables": ["name"]}, + }, + inputs={"name": "Ada"}, + run_id=prompt_id, + parent_run_id=sequence_id, + ) + handler.on_chain_end( + outputs={"text": "Hello Ada"}, run_id=prompt_id + ) + + handler.on_chat_model_start( + serialized={"name": "ChatOpenAI"}, + messages=[[HumanMessage(content="Hello Ada")]], + run_id=_run_id(), + parent_run_id=sequence_id, + metadata={"ls_provider": "openai"}, + invocation_params={"model_name": "gpt-4"}, + ) + + invocation = telemetry.inference.return_value + assert invocation.prompt_name == "greeting" + assert invocation.prompt_variables == {"name": "Ada"} + assert ( + handler._invocation_manager.consume_prompt_context(sequence_id) + is None + ) + + def test_prompt_variables_are_not_processed_when_content_disabled(self): + handler, telemetry, _, _ = _make_handler() + telemetry.should_capture_content.return_value = False + sequence_id = _run_id() + prompt_id = _run_id() + handler._invocation_manager.add_invocation_state( + sequence_id, None, None + ) + handler.on_chain_start( + serialized={ + "id": ["langchain", "prompts", "PromptTemplate"], + "name": "greeting", + "kwargs": {"input_variables": ["name"]}, + }, + inputs={"name": object()}, + run_id=prompt_id, + parent_run_id=sequence_id, + ) + handler.on_chain_end(outputs={}, run_id=prompt_id) + + handler.on_chat_model_start( + serialized={"name": "ChatOpenAI"}, + messages=[[HumanMessage(content="Hello")]], + run_id=_run_id(), + parent_run_id=sequence_id, + metadata={"ls_provider": "openai"}, + invocation_params={"model_name": "gpt-4"}, + ) + + invocation = telemetry.inference.return_value + assert invocation.prompt_name == "greeting" + assert invocation.prompt_variables == {} + + def test_failed_prompt_context_is_not_applied(self): + handler, telemetry, _, _ = _make_handler() + telemetry.should_capture_content.return_value = True + telemetry.inference.return_value.prompt_name = None + telemetry.inference.return_value.prompt_variables = None + sequence_id = _run_id() + prompt_id = _run_id() + handler._invocation_manager.add_invocation_state( + sequence_id, None, None + ) + handler.on_chain_start( + serialized={ + "id": ["langchain", "prompts", "PromptTemplate"], + "name": "failed", + "kwargs": {"input_variables": ["value"]}, + }, + inputs={"value": "secret"}, + run_id=prompt_id, + parent_run_id=sequence_id, + ) + handler.on_chain_error(RuntimeError("format failed"), run_id=prompt_id) + + handler.on_chat_model_start( + serialized={"name": "ChatOpenAI"}, + messages=[[HumanMessage(content="unrelated")]], + run_id=_run_id(), + parent_run_id=sequence_id, + metadata={"ls_provider": "openai"}, + invocation_params={"model_name": "gpt-4"}, + ) + + invocation = telemetry.inference.return_value + assert invocation.prompt_name is None + assert invocation.prompt_variables is None + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) async def test_multimodal_input_content_capture( diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py index bebf6127d..05eed6cd8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py @@ -9,6 +9,7 @@ from opentelemetry.instrumentation.genai.langchain.invocation_manager import ( _InvocationManager, + _PromptContext, ) from opentelemetry.util.genai.invocation import InferenceInvocation @@ -242,3 +243,56 @@ def test_none_invocation_can_be_stored_and_retrieved(invocation_manager): def test_delete_nonexistent_run_id_does_not_raise(invocation_manager): invocation_manager.delete_invocation_state(uuid.uuid4()) # must not raise + + +def test_prompt_context_publishes_to_parent_and_is_consumed_once( + invocation_manager, +): + parent_id = uuid.uuid4() + prompt_id = uuid.uuid4() + context = _PromptContext("greeting", {"name": "Ada"}) + invocation_manager.add_invocation_state(parent_id, None, None) + invocation_manager.add_invocation_state(prompt_id, parent_id, None) + + invocation_manager.set_prompt_context(prompt_id, context) + invocation_manager.publish_prompt_context(prompt_id) + + assert invocation_manager.consume_prompt_context(parent_id) == context + assert invocation_manager.consume_prompt_context(parent_id) is None + + +def test_prompt_context_is_consumed_from_nearest_ancestor(invocation_manager): + root_id = uuid.uuid4() + branch_id = uuid.uuid4() + model_parent_id = uuid.uuid4() + root_context = _PromptContext("root", {}) + branch_context = _PromptContext("branch", {}) + invocation_manager.add_invocation_state(root_id, None, None) + invocation_manager.add_invocation_state(branch_id, root_id, None) + invocation_manager.add_invocation_state(model_parent_id, branch_id, None) + invocation_manager._invocations[root_id].pending_prompt_contexts.append( + root_context + ) + invocation_manager._invocations[branch_id].pending_prompt_contexts.append( + branch_context + ) + + assert ( + invocation_manager.consume_prompt_context(model_parent_id) + == branch_context + ) + assert invocation_manager.consume_prompt_context(root_id) == root_context + + +def test_unpublished_prompt_context_is_discarded_on_delete(invocation_manager): + parent_id = uuid.uuid4() + prompt_id = uuid.uuid4() + invocation_manager.add_invocation_state(parent_id, None, None) + invocation_manager.add_invocation_state(prompt_id, parent_id, None) + invocation_manager.set_prompt_context( + prompt_id, _PromptContext("failed", {"value": "secret"}) + ) + + invocation_manager.delete_invocation_state(prompt_id) + + assert invocation_manager.consume_prompt_context(parent_id) is None diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index d421041d3..8cee46c35 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -20,6 +20,7 @@ SystemMessageChunk, ) from langchain_core.outputs import ChatGenerationChunk +from langchain_core.prompts import ChatPromptTemplate, PromptTemplate from langchain_core.tools import tool from langchain_openai import ChatOpenAI from openai import AuthenticationError @@ -82,6 +83,198 @@ def _langchain_openai_version() -> tuple: _ANTHROPIC_FILE_ID = "file_011CNhaGCM5eyZmDsFmQJVQe" +@pytest.mark.skipif( + _langchain_openai_version() < (1, 0, 0), + reason="cassettes were recorded with langchain-openai 1.x", +) +@pytest.mark.vcr() +def test_chat_openai_prompt_template( + span_exporter, tracer_provider, meter_provider, logger_provider, vcr +): + model = ChatOpenAI(model="gpt-4.1", max_tokens=100) + prompt = PromptTemplate.from_template( + "Answer this weather question briefly: {question}" + ) + + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("test_chat_openai_prompt_template.yaml"): + response = (prompt | model).invoke( + {"question": "What's the weather like in Seattle?"}, + config={"metadata": {"prompt_name": "weather_prompt"}}, + ) + + assert response.content + chat_span = next( + span + for span in span_exporter.get_finished_spans() + if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == "chat" + ) + assert ( + chat_span.attributes[gen_ai_attributes.GEN_AI_PROMPT_NAME] + == "weather_prompt" + ) + assert ( + chat_span.attributes["gen_ai.prompt.variable.question"] + == "What's the weather like in Seattle?" + ) + + +@pytest.mark.asyncio +async def test_async_prompt_template_propagates_prompt_context( + span_exporter, tracer_provider, meter_provider, logger_provider +): + model = FakeMessagesListChatModel( + responses=[AIMessage(content="It is raining.")] + ) + prompt = PromptTemplate.from_template( + "Answer this weather question briefly: {question}", + name="weather_prompt", + ) + + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture="SPAN_ONLY", + ): + response = await (prompt | model).ainvoke( + {"question": "What's the weather like in Seattle?"}, + config={"metadata": {"ls_model_name": "test-chat-model"}}, + ) + + assert response.content == "It is raining." + chat_span = next( + span + for span in span_exporter.get_finished_spans() + if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == "chat" + ) + assert ( + chat_span.attributes[gen_ai_attributes.GEN_AI_PROMPT_NAME] + == "weather_prompt" + ) + assert ( + chat_span.attributes["gen_ai.prompt.variable.question"] + == "What's the weather like in Seattle?" + ) + + +def test_prompt_template_accepts_scalar_input( + span_exporter, tracer_provider, meter_provider, logger_provider +): + model = FakeMessagesListChatModel(responses=[AIMessage(content="Hello!")]) + prompt = PromptTemplate.from_template("Hello {name}") + + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture="SPAN_ONLY", + ): + response = (prompt | model).invoke( + "Ada", + config={"metadata": {"ls_model_name": "test-chat-model"}}, + ) + + assert response.content == "Hello!" + chat_span = next( + span + for span in span_exporter.get_finished_spans() + if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == "chat" + ) + assert chat_span.attributes["gen_ai.prompt.variable.name"] == "Ada" + + +def test_prompt_template_omits_non_serializable_variable( + span_exporter, tracer_provider, meter_provider, logger_provider +): + model = FakeMessagesListChatModel( + responses=[AIMessage(content="It is raining.")] + ) + prompt = PromptTemplate.from_template("Input: {value}") + + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture="SPAN_ONLY", + ): + response = (prompt | model).invoke( + {"value": object()}, + config={"metadata": {"ls_model_name": "test-chat-model"}}, + ) + + assert response.content == "It is raining." + chat_span = next( + span + for span in span_exporter.get_finished_spans() + if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == "chat" + ) + assert "gen_ai.prompt.variable.value" not in chat_span.attributes + + +@pytest.mark.skipif( + _langchain_openai_version() < (1, 0, 0), + reason="cassettes were recorded with langchain-openai 1.x", +) +@pytest.mark.vcr() +def test_chat_openai_chat_prompt_template( + span_exporter, tracer_provider, meter_provider, logger_provider, vcr +): + model = ChatOpenAI(model="gpt-4.1", max_tokens=100) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", "Answer weather questions in a {style} style."), + ("human", "{question}"), + ] + ) + + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("test_chat_openai_chat_prompt_template.yaml"): + response = (prompt | model).invoke( + { + "question": "What's the weather like in Seattle?", + "style": "concise", + }, + config={"metadata": {"prompt_name": "weather_chat_prompt"}}, + ) + + assert response.content + chat_span = next( + span + for span in span_exporter.get_finished_spans() + if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) + == "chat" + ) + assert ( + chat_span.attributes[gen_ai_attributes.GEN_AI_PROMPT_NAME] + == "weather_chat_prompt" + ) + assert ( + chat_span.attributes["gen_ai.prompt.variable.question"] + == "What's the weather like in Seattle?" + ) + assert chat_span.attributes["gen_ai.prompt.variable.style"] == "concise" + + # span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py @pytest.mark.parametrize( "capture_content", From 88c1c628a9006b5e1d653832cfb182ef49361e47 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 23 Sep 2026 16:30:51 -0700 Subject: [PATCH 02/16] Add CHANGELOG --- .../.changelog/730.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/730.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/730.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/730.added new file mode 100644 index 000000000..849c635af --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/730.added @@ -0,0 +1 @@ +Capture prompt template name - `gen_ai.prompt.name` and variables - `gen_ai.prompt.variable.` on chat From 5b757d7242816d3d5c21c01dff3dd193db7830ca Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 23 Sep 2026 16:48:04 -0700 Subject: [PATCH 03/16] Fix lint --- .../genai/langchain/callback_handler.py | 12 +++--------- .../genai/langchain/invocation_manager.py | 4 +--- .../tests/test_callback_handler.py | 8 ++------ 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 565cbfc8e..5ed6e0f6b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -94,9 +94,7 @@ def _string_sequence(value: object) -> list[str]: if not isinstance(value, Sequence) or isinstance(value, str): return [] return [ - item - for item in cast(Sequence[object], value) - if isinstance(item, str) + item for item in cast(Sequence[object], value) if isinstance(item, str) ] @@ -149,9 +147,7 @@ def _extract_prompt_context( if not isinstance(raw_serialized_kwargs, Mapping): serialized_kwargs: Mapping[str, object] = {} else: - serialized_kwargs = cast( - Mapping[str, object], raw_serialized_kwargs - ) + serialized_kwargs = cast(Mapping[str, object], raw_serialized_kwargs) name = (metadata or {}).get("prompt_name") if not isinstance(name, str): @@ -436,9 +432,7 @@ def on_chain_start( metadata, capture_variables=capture_content, ): - self._invocation_manager.set_prompt_context( - run_id, prompt_context - ) + self._invocation_manager.set_prompt_context(run_id, prompt_context) def on_chain_end( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py index c7c3da334..fa934d845 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py @@ -40,9 +40,7 @@ class _InvocationState: ended: bool = False agent_name: str | None = None prompt_context: _PromptContext | None = None - pending_prompt_contexts: list[_PromptContext] = field( - default_factory=lambda: [] - ) + pending_prompt_contexts: list[_PromptContext] = field(default_factory=list) class _InvocationManager: diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 7e031c80e..8e8179cbb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -145,9 +145,7 @@ def test_extracts_name_and_declared_variables( assert context.name == "weather" assert context.variables == { "question": "Will it rain?", - "history": [ - HumanMessage(content="Earlier question").model_dump() - ], + "history": [HumanMessage(content="Earlier question").model_dump()], "language": "French", } @@ -1067,9 +1065,7 @@ def test_successful_prompt_context_is_applied_to_next_model(self): run_id=prompt_id, parent_run_id=sequence_id, ) - handler.on_chain_end( - outputs={"text": "Hello Ada"}, run_id=prompt_id - ) + handler.on_chain_end(outputs={"text": "Hello Ada"}, run_id=prompt_id) handler.on_chat_model_start( serialized={"name": "ChatOpenAI"}, From df7972c8ebe5fd6ad2940708b34f75af4c53f1b0 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 24 Sep 2026 09:01:32 -0700 Subject: [PATCH 04/16] Fix typecheck --- .../instrumentation/genai/langchain/invocation_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py index fa934d845..6e6b068c5 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py @@ -40,7 +40,9 @@ class _InvocationState: ended: bool = False agent_name: str | None = None prompt_context: _PromptContext | None = None - pending_prompt_contexts: list[_PromptContext] = field(default_factory=list) + pending_prompt_contexts: list[_PromptContext] = field( + default_factory=lambda: list[_PromptContext]() + ) class _InvocationManager: From 8849425b6ba045647c84fbe63882e8e7c978f44d Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 24 Sep 2026 11:27:37 -0700 Subject: [PATCH 05/16] Address feedback --- .../genai/langchain/callback_handler.py | 6 +-- .../genai/langchain/invocation_manager.py | 4 +- .../tests/test_callback_handler.py | 42 +++++++++++++++++++ .../tests/test_invocation_manager.py | 23 ++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 5ed6e0f6b..9d1391bca 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -487,9 +487,6 @@ def on_chat_model_start( metadata: dict[str, Any] | None = None, **kwargs: Any, ) -> None: - prompt_context = self._invocation_manager.consume_prompt_context( - parent_run_id - ) if "invocation_params" in kwargs: params = ( kwargs["invocation_params"].get("params") @@ -575,6 +572,9 @@ def on_chat_model_start( parent_context = self._invocation_manager.get_parent_context( parent_run_id ) + prompt_context = self._invocation_manager.consume_prompt_context( + parent_run_id + ) llm_invocation = self._telemetry_handler.inference( provider, request_model=request_model, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py index 6e6b068c5..022107199 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/invocation_manager.py @@ -123,7 +123,9 @@ def consume_prompt_context( if invocation_state is None: return None if invocation_state.pending_prompt_contexts: - return invocation_state.pending_prompt_contexts.pop(0) + prompt_context = invocation_state.pending_prompt_contexts[-1] + invocation_state.pending_prompt_contexts.clear() + return prompt_context current_run_id = invocation_state.parent_run_id return None diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 8e8179cbb..4b8ce492b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -1084,6 +1084,48 @@ def test_successful_prompt_context_is_applied_to_next_model(self): is None ) + def test_unsupported_model_does_not_consume_prompt_context(self): + handler, telemetry, _, _ = _make_handler() + telemetry.should_capture_content.return_value = True + sequence_id = _run_id() + prompt_id = _run_id() + handler._invocation_manager.add_invocation_state( + sequence_id, None, None + ) + handler.on_chain_start( + serialized={ + "id": ["langchain", "prompts", "PromptTemplate"], + "name": "greeting", + "kwargs": {"input_variables": ["name"]}, + }, + inputs={"name": "Ada"}, + run_id=prompt_id, + parent_run_id=sequence_id, + ) + handler.on_chain_end(outputs={"text": "Hello Ada"}, run_id=prompt_id) + + handler.on_chat_model_start( + serialized={"name": "UnsupportedModel"}, + messages=[[HumanMessage(content="Hello Ada")]], + run_id=_run_id(), + parent_run_id=sequence_id, + ) + + telemetry.inference.assert_not_called() + + handler.on_chat_model_start( + serialized={"name": "ChatOpenAI"}, + messages=[[HumanMessage(content="Hello Ada")]], + run_id=_run_id(), + parent_run_id=sequence_id, + metadata={"ls_provider": "openai"}, + invocation_params={"model_name": "gpt-4"}, + ) + + invocation = telemetry.inference.return_value + assert invocation.prompt_name == "greeting" + assert invocation.prompt_variables == {"name": "Ada"} + def test_prompt_variables_are_not_processed_when_content_disabled(self): handler, telemetry, _, _ = _make_handler() telemetry.should_capture_content.return_value = False diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py index 05eed6cd8..bb0524c93 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_invocation_manager.py @@ -261,6 +261,29 @@ def test_prompt_context_publishes_to_parent_and_is_consumed_once( assert invocation_manager.consume_prompt_context(parent_id) is None +def test_most_recent_prompt_context_supersedes_earlier_context( + invocation_manager, +): + parent_id = uuid.uuid4() + first_prompt_id = uuid.uuid4() + second_prompt_id = uuid.uuid4() + first_context = _PromptContext("first", {"value": "first"}) + second_context = _PromptContext("second", {"value": "second"}) + invocation_manager.add_invocation_state(parent_id, None, None) + invocation_manager.add_invocation_state(first_prompt_id, parent_id, None) + invocation_manager.add_invocation_state(second_prompt_id, parent_id, None) + + invocation_manager.set_prompt_context(first_prompt_id, first_context) + invocation_manager.publish_prompt_context(first_prompt_id) + invocation_manager.set_prompt_context(second_prompt_id, second_context) + invocation_manager.publish_prompt_context(second_prompt_id) + + assert ( + invocation_manager.consume_prompt_context(parent_id) == second_context + ) + assert invocation_manager.consume_prompt_context(parent_id) is None + + def test_prompt_context_is_consumed_from_nearest_ancestor(invocation_manager): root_id = uuid.uuid4() branch_id = uuid.uuid4() From a0437cec6a0002010ee3b8ad4cda002788665aa2 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 24 Sep 2026 12:55:01 -0700 Subject: [PATCH 06/16] Retrigger CI/CD pipeline From c614206ea903babafbba0819c9c9d3ec782dde9e Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 24 Sep 2026 13:55:14 -0700 Subject: [PATCH 07/16] Address feedback --- .../genai/langchain/callback_handler.py | 46 ++++++++++++++----- .../tests/test_callback_handler.py | 23 ++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 9d1391bca..9a77e6f29 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -98,29 +98,53 @@ def _string_sequence(value: object) -> list[str]: ] -def _prompt_variable_value(value: object) -> object: +def _prompt_variable_value( + value: object, active_container_ids: set[int] | None = None +) -> object: + if active_container_ids is None: + active_container_ids = set() + if isinstance(value, BaseMessage): value = value.model_dump() elif isinstance(value, Mapping): value = cast(Mapping[object, object], value) if value.get("type") == "not_implemented": return _INVALID_PROMPT_VALUE + container_id = id(value) + if container_id in active_container_ids: + return _INVALID_PROMPT_VALUE + active_container_ids.add(container_id) normalized_mapping: dict[object, object] = {} - for key, item in value.items(): - normalized_item = _prompt_variable_value(item) - if normalized_item is _INVALID_PROMPT_VALUE: - return _INVALID_PROMPT_VALUE - normalized_mapping[key] = normalized_item + try: + for key, item in value.items(): + normalized_item = _prompt_variable_value( + item, active_container_ids + ) + if normalized_item is _INVALID_PROMPT_VALUE: + return _INVALID_PROMPT_VALUE + normalized_mapping[key] = normalized_item + finally: + active_container_ids.remove(container_id) value = normalized_mapping elif isinstance(value, Sequence) and not isinstance( value, (str, bytes, bytearray) ): + value = cast(Sequence[object], value) + container_id = id(value) + if container_id in active_container_ids: + return _INVALID_PROMPT_VALUE + active_container_ids.add(container_id) normalized_sequence: list[object] = [] - for item in cast(Sequence[object], value): - normalized_item = _prompt_variable_value(item) - if normalized_item is _INVALID_PROMPT_VALUE: - return _INVALID_PROMPT_VALUE - normalized_sequence.append(normalized_item) + try: + for item in value: + normalized_item = _prompt_variable_value( + item, active_container_ids + ) + if normalized_item is _INVALID_PROMPT_VALUE: + return _INVALID_PROMPT_VALUE + normalized_sequence.append(normalized_item) + finally: + active_container_ids.remove(container_id) value = normalized_sequence elif callable(value): return _INVALID_PROMPT_VALUE diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 4b8ce492b..cf84d610f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -221,6 +221,29 @@ def test_skips_unresolved_and_unserializable_values(self): assert context is not None assert context.variables == {} + def test_skips_cyclic_values(self): + cyclic_list = [] + cyclic_list.append(cyclic_list) + cyclic_mapping = {} + cyclic_mapping["self"] = cyclic_mapping + + context = _extract_prompt_context( + { + "id": ["langchain", "prompts", "PromptTemplate"], + "kwargs": { + "input_variables": ["list_value", "mapping_value"] + }, + }, + { + "list_value": cyclic_list, + "mapping_value": cyclic_mapping, + }, + None, + ) + + assert context is not None + assert context.variables == {} + def test_ignores_non_prompt_runs(self): assert ( _extract_prompt_context( From 78598e236ec020b13de0255a5fb0db3f5f2fd3dd Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 24 Sep 2026 14:04:13 -0700 Subject: [PATCH 08/16] Fix lint --- .../tests/test_callback_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index cf84d610f..66f3ff92e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -230,9 +230,7 @@ def test_skips_cyclic_values(self): context = _extract_prompt_context( { "id": ["langchain", "prompts", "PromptTemplate"], - "kwargs": { - "input_variables": ["list_value", "mapping_value"] - }, + "kwargs": {"input_variables": ["list_value", "mapping_value"]}, }, { "list_value": cyclic_list, From e842a58dcbadd932c36e4e8f8ac2705e785fea65 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 30 Sep 2026 09:59:59 -0700 Subject: [PATCH 09/16] Address feedback --- .../genai/langchain/callback_handler.py | 110 ++++-------------- .../tests/test_callback_handler.py | 58 ++++----- 2 files changed, 56 insertions(+), 112 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 9a77e6f29..b4bdb5c97 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -62,7 +62,6 @@ object_to_any_value, tool_arguments_to_any_value, ) -from opentelemetry.util.genai.utils import gen_ai_json_dumps SUPPORTED_RAPI_RESPONSE_HEADERS = ("x-ms-served-model",) @@ -74,17 +73,14 @@ ) _PROMPT_TEMPLATE_TYPES = {"ChatPromptTemplate", "PromptTemplate"} -_INVALID_PROMPT_VALUE = object() -def _prompt_template_type(serialized: Mapping[str, Any]) -> str | None: - serialized_id = serialized.get("id") - if not isinstance(serialized_id, Sequence) or isinstance( - serialized_id, str - ): +def _prompt_template_type(runnable: Mapping[str, Any]) -> str | None: + runnable_id = runnable.get("id") + if not isinstance(runnable_id, Sequence) or isinstance(runnable_id, str): return None - serialized_id = cast(Sequence[object], serialized_id) - template_type = serialized_id[-1] if serialized_id else None + runnable_id = cast(Sequence[str], runnable_id) + template_type = runnable_id[-1] if runnable_id else None if not isinstance(template_type, str): return None return template_type if template_type in _PROMPT_TEMPLATE_TYPES else None @@ -98,101 +94,47 @@ def _string_sequence(value: object) -> list[str]: ] -def _prompt_variable_value( - value: object, active_container_ids: set[int] | None = None -) -> object: - if active_container_ids is None: - active_container_ids = set() - - if isinstance(value, BaseMessage): - value = value.model_dump() - elif isinstance(value, Mapping): - value = cast(Mapping[object, object], value) - if value.get("type") == "not_implemented": - return _INVALID_PROMPT_VALUE - container_id = id(value) - if container_id in active_container_ids: - return _INVALID_PROMPT_VALUE - active_container_ids.add(container_id) - normalized_mapping: dict[object, object] = {} - try: - for key, item in value.items(): - normalized_item = _prompt_variable_value( - item, active_container_ids - ) - if normalized_item is _INVALID_PROMPT_VALUE: - return _INVALID_PROMPT_VALUE - normalized_mapping[key] = normalized_item - finally: - active_container_ids.remove(container_id) - value = normalized_mapping - elif isinstance(value, Sequence) and not isinstance( - value, (str, bytes, bytearray) - ): - value = cast(Sequence[object], value) - container_id = id(value) - if container_id in active_container_ids: - return _INVALID_PROMPT_VALUE - active_container_ids.add(container_id) - normalized_sequence: list[object] = [] - try: - for item in value: - normalized_item = _prompt_variable_value( - item, active_container_ids - ) - if normalized_item is _INVALID_PROMPT_VALUE: - return _INVALID_PROMPT_VALUE - normalized_sequence.append(normalized_item) - finally: - active_container_ids.remove(container_id) - value = normalized_sequence - elif callable(value): - return _INVALID_PROMPT_VALUE - - try: - gen_ai_json_dumps(value) - except (TypeError, ValueError, OverflowError): - return _INVALID_PROMPT_VALUE - return value - - def _extract_prompt_context( - serialized: Mapping[str, Any], + runnable: Mapping[str, Any], inputs: object, metadata: Mapping[str, Any] | None, *, capture_variables: bool = True, ) -> _PromptContext | None: - template_type = _prompt_template_type(serialized) + template_type = _prompt_template_type(runnable) if template_type is None: return None - raw_serialized_kwargs = serialized.get("kwargs") - if not isinstance(raw_serialized_kwargs, Mapping): - serialized_kwargs: Mapping[str, object] = {} + raw_template_kwargs = runnable.get("kwargs") + if not isinstance(raw_template_kwargs, Mapping): + template_kwargs: Mapping[str, object] = {} else: - serialized_kwargs = cast(Mapping[str, object], raw_serialized_kwargs) + template_kwargs = cast(Mapping[str, object], raw_template_kwargs) name = (metadata or {}).get("prompt_name") if not isinstance(name, str): - name = serialized_kwargs.get("name") + name = template_kwargs.get("name") if not isinstance(name, str): - name = serialized.get("name") - if not isinstance(name, str) or name in _PROMPT_TEMPLATE_TYPES: - name = None + runnable_name = runnable.get("name") + name = ( + runnable_name + if isinstance(runnable_name, str) + and runnable_name not in _PROMPT_TEMPLATE_TYPES + else None + ) if not capture_variables: return _PromptContext(name=name, variables={}) required_variables = _string_sequence( - serialized_kwargs.get("input_variables") + template_kwargs.get("input_variables") ) declared_variables = set(required_variables) declared_variables.update( - _string_sequence(serialized_kwargs.get("optional_variables")) + _string_sequence(template_kwargs.get("optional_variables")) ) - partial_variables = serialized_kwargs.get("partial_variables") + partial_variables = template_kwargs.get("partial_variables") if isinstance(partial_variables, Mapping): partial_variables = cast(Mapping[object, object], partial_variables) declared_variables.update( @@ -200,8 +142,8 @@ def _extract_prompt_context( ) if isinstance(inputs, Mapping): - runtime_variables: Mapping[object, object] = cast( - Mapping[object, object], inputs + runtime_variables: Mapping[str, object] = cast( + Mapping[str, object], inputs ) elif len(required_variables) == 1: runtime_variables = {required_variables[0]: inputs} @@ -216,9 +158,7 @@ def _extract_prompt_context( for key, value in source.items(): if not isinstance(key, str) or key not in declared_variables: continue - normalized_value = _prompt_variable_value(value) - if normalized_value is not _INVALID_PROMPT_VALUE: - effective_variables[key] = normalized_value + effective_variables[key] = value return _PromptContext(name=name, variables=effective_variables) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 66f3ff92e..c88e884e0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -123,6 +123,7 @@ class TestExtractPromptContext: def test_extracts_name_and_declared_variables( self, template_type: str ) -> None: + history = [HumanMessage(content="Earlier question")] context = _extract_prompt_context( { "id": ["langchain", "prompts", template_type], @@ -135,7 +136,7 @@ def test_extracts_name_and_declared_variables( }, { "question": "Will it rain?", - "history": [HumanMessage(content="Earlier question")], + "history": history, "unused": "ignored", }, None, @@ -145,7 +146,7 @@ def test_extracts_name_and_declared_variables( assert context.name == "weather" assert context.variables == { "question": "Will it rain?", - "history": [HumanMessage(content="Earlier question").model_dump()], + "history": history, "language": "French", } @@ -168,6 +169,24 @@ def test_name_precedence_and_generic_name_exclusion(self): assert unnamed is not None assert unnamed.name is None + @pytest.mark.parametrize("source", ["metadata", "kwargs"]) + def test_explicit_template_class_name_is_preserved(self, source: str): + serialized = { + "id": ["langchain", "prompts", "PromptTemplate"], + "name": "PromptTemplate", + "kwargs": {}, + } + metadata = None + if source == "metadata": + metadata = {"prompt_name": "PromptTemplate"} + else: + serialized["kwargs"] = {"name": "PromptTemplate"} + + context = _extract_prompt_context(serialized, {}, metadata) + + assert context is not None + assert context.name == "PromptTemplate" + def test_runtime_values_override_partials_and_scalar_is_mapped(self): serialized = { "id": ["langchain", "prompts", "PromptTemplate"], @@ -199,12 +218,14 @@ def test_preserves_none_as_json_null(self): assert context is not None assert context.variables == {"value": None} - def test_skips_unresolved_and_unserializable_values(self): + def test_passes_declared_values_without_serializing(self): + callable_value = lambda: "value" + unsupported_value = object() serialized = { "id": ["langchain", "prompts", "PromptTemplate"], "kwargs": { "partial_variables": { - "callable": lambda: "value", + "callable": callable_value, "unresolved": { "lc": 1, "type": "not_implemented", @@ -215,32 +236,15 @@ def test_skips_unresolved_and_unserializable_values(self): } context = _extract_prompt_context( - serialized, {"unsupported": object()}, None + serialized, {"unsupported": unsupported_value}, None ) assert context is not None - assert context.variables == {} - - def test_skips_cyclic_values(self): - cyclic_list = [] - cyclic_list.append(cyclic_list) - cyclic_mapping = {} - cyclic_mapping["self"] = cyclic_mapping - - context = _extract_prompt_context( - { - "id": ["langchain", "prompts", "PromptTemplate"], - "kwargs": {"input_variables": ["list_value", "mapping_value"]}, - }, - { - "list_value": cyclic_list, - "mapping_value": cyclic_mapping, - }, - None, - ) - - assert context is not None - assert context.variables == {} + assert context.variables == { + "callable": callable_value, + "unresolved": {"lc": 1, "type": "not_implemented"}, + "unsupported": unsupported_value, + } def test_ignores_non_prompt_runs(self): assert ( From b117965a92c075ed1ceadbfbb56786c6219f2456 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 30 Sep 2026 10:51:00 -0700 Subject: [PATCH 10/16] Remove the uninteded conversation id statement --- .../instrumentation/genai/langchain/callback_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index b4bdb5c97..900b3951c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -549,7 +549,6 @@ def on_chat_model_start( if prompt_context is not None: llm_invocation.prompt_name = prompt_context.name llm_invocation.prompt_variables = prompt_context.variables - llm_invocation.conversation_id = _conversation_id(metadata) llm_invocation.input_messages = input_messages llm_invocation.top_p = top_p llm_invocation.top_k = top_k From 2d15f44383e2041dcb0db3c2142924e8843984e9 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Fri, 2 Oct 2026 10:01:34 -0700 Subject: [PATCH 11/16] Rework this when dependent PR is merged --- .../tests/test_llm_call.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 8cee46c35..4a772b4d8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -195,13 +195,23 @@ def test_prompt_template_accepts_scalar_input( assert chat_span.attributes["gen_ai.prompt.variable.name"] == "Ada" -def test_prompt_template_omits_non_serializable_variable( - span_exporter, tracer_provider, meter_provider, logger_provider +@pytest.mark.xfail( + strict=True, + reason="Blocked by opentelemetry-python-genai#821", +) +def test_prompt_template_with_non_serializable_partial_finishes_telemetry( + span_exporter, + metric_reader, + tracer_provider, + meter_provider, + logger_provider, ): model = FakeMessagesListChatModel( responses=[AIMessage(content="It is raining.")] ) - prompt = PromptTemplate.from_template("Input: {value}") + prompt = PromptTemplate.from_template("Input: {value}").partial( + value={"lc": object()} + ) with instrument( LangChainInstrumentor(), @@ -211,7 +221,7 @@ def test_prompt_template_omits_non_serializable_variable( content_capture="SPAN_ONLY", ): response = (prompt | model).invoke( - {"value": object()}, + {}, config={"metadata": {"ls_model_name": "test-chat-model"}}, ) @@ -223,6 +233,21 @@ def test_prompt_template_omits_non_serializable_variable( == "chat" ) assert "gen_ai.prompt.variable.value" not in chat_span.attributes + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES in chat_span.attributes + assert ( + gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS + in chat_span.attributes + ) + + metrics_data = metric_reader.get_metrics_data() + assert metrics_data is not None + metric_names = { + metric.name + for resource_metrics in metrics_data.resource_metrics + for scope_metrics in resource_metrics.scope_metrics + for metric in scope_metrics.metrics + } + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metric_names @pytest.mark.skipif( From 6913d6f4d83dff188ad11a7096adf43862a458e0 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 12 Oct 2026 09:12:46 -0700 Subject: [PATCH 12/16] Fix test assertion --- .../tests/test_llm_call.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 4a772b4d8..edea16a71 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -195,10 +195,6 @@ def test_prompt_template_accepts_scalar_input( assert chat_span.attributes["gen_ai.prompt.variable.name"] == "Ada" -@pytest.mark.xfail( - strict=True, - reason="Blocked by opentelemetry-python-genai#821", -) def test_prompt_template_with_non_serializable_partial_finishes_telemetry( span_exporter, metric_reader, @@ -232,7 +228,9 @@ def test_prompt_template_with_non_serializable_partial_finishes_telemetry( if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == "chat" ) - assert "gen_ai.prompt.variable.value" not in chat_span.attributes + assert chat_span.attributes["gen_ai.prompt.variable.value"] == ( + '{"__lc_escaped__":{}}' + ) assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES in chat_span.attributes assert ( gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS From 4bc25f7edc34d89923d1e35793946f968eedbfa5 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 13 Oct 2026 09:56:32 -0700 Subject: [PATCH 13/16] Fix test assertion --- .../tests/test_llm_call.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index edea16a71..cc7265182 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -228,9 +228,6 @@ def test_prompt_template_with_non_serializable_partial_finishes_telemetry( if span.attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == "chat" ) - assert chat_span.attributes["gen_ai.prompt.variable.value"] == ( - '{"__lc_escaped__":{}}' - ) assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES in chat_span.attributes assert ( gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS From dbc6e300f0499d3c8dcc604756ebe206fea8349d Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 13 Oct 2026 13:01:52 -0700 Subject: [PATCH 14/16] Address feedback --- .../genai/langchain/callback_handler.py | 7 +++++++ .../tests/test_callback_handler.py | 1 - .../tests/test_llm_call.py | 21 +++++++++++-------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 900b3951c..97e2d4f36 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -158,6 +158,13 @@ def _extract_prompt_context( for key, value in source.items(): if not isinstance(key, str) or key not in declared_variables: continue + if ( + source is partial_variables + and isinstance(value, Mapping) + and value.get("lc") == 1 + and value.get("type") == "not_implemented" + ): + continue effective_variables[key] = value return _PromptContext(name=name, variables=effective_variables) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index c88e884e0..ba888630b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -242,7 +242,6 @@ def test_passes_declared_values_without_serializing(self): assert context is not None assert context.variables == { "callable": callable_value, - "unresolved": {"lc": 1, "type": "not_implemented"}, "unsupported": unsupported_value, } diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index cc7265182..635dc22bc 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -127,8 +127,9 @@ def test_chat_openai_prompt_template( @pytest.mark.asyncio +@pytest.mark.parametrize("capture_content", ["SPAN_ONLY", "NO_CONTENT"]) async def test_async_prompt_template_propagates_prompt_context( - span_exporter, tracer_provider, meter_provider, logger_provider + span_exporter, tracer_provider, meter_provider, logger_provider, capture_content ): model = FakeMessagesListChatModel( responses=[AIMessage(content="It is raining.")] @@ -137,19 +138,17 @@ async def test_async_prompt_template_propagates_prompt_context( "Answer this weather question briefly: {question}", name="weather_prompt", ) - with instrument( LangChainInstrumentor(), tracer_provider=tracer_provider, meter_provider=meter_provider, logger_provider=logger_provider, - content_capture="SPAN_ONLY", + content_capture=capture_content, ): response = await (prompt | model).ainvoke( {"question": "What's the weather like in Seattle?"}, config={"metadata": {"ls_model_name": "test-chat-model"}}, ) - assert response.content == "It is raining." chat_span = next( span @@ -161,10 +160,13 @@ async def test_async_prompt_template_propagates_prompt_context( chat_span.attributes[gen_ai_attributes.GEN_AI_PROMPT_NAME] == "weather_prompt" ) - assert ( - chat_span.attributes["gen_ai.prompt.variable.question"] - == "What's the weather like in Seattle?" - ) + if capture_content == "SPAN_ONLY": + assert ( + chat_span.attributes["gen_ai.prompt.variable.question"] + == "What's the weather like in Seattle?" + ) + else: + assert "gen_ai.prompt.variable.question" not in chat_span.attributes def test_prompt_template_accepts_scalar_input( @@ -206,7 +208,7 @@ def test_prompt_template_with_non_serializable_partial_finishes_telemetry( responses=[AIMessage(content="It is raining.")] ) prompt = PromptTemplate.from_template("Input: {value}").partial( - value={"lc": object()} + value=object() ) with instrument( @@ -229,6 +231,7 @@ def test_prompt_template_with_non_serializable_partial_finishes_telemetry( == "chat" ) assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES in chat_span.attributes + assert "gen_ai.prompt.variable.value" not in chat_span.attributes assert ( gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS in chat_span.attributes From 19fe1d389c021ff9f344665a73cd78674229f7e7 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 13 Oct 2026 13:54:18 -0700 Subject: [PATCH 15/16] Fix lint and format --- .../genai/langchain/callback_handler.py | 11 ++++++++--- .../tests/test_llm_call.py | 6 +++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 97e2d4f36..c543ae05f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -158,11 +158,16 @@ def _extract_prompt_context( for key, value in source.items(): if not isinstance(key, str) or key not in declared_variables: continue + serialized_value = ( + cast(Mapping[str, object], value) + if isinstance(value, Mapping) + else None + ) if ( source is partial_variables - and isinstance(value, Mapping) - and value.get("lc") == 1 - and value.get("type") == "not_implemented" + and serialized_value is not None + and serialized_value.get("lc") == 1 + and serialized_value.get("type") == "not_implemented" ): continue effective_variables[key] = value diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 635dc22bc..8d81a3391 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -129,7 +129,11 @@ def test_chat_openai_prompt_template( @pytest.mark.asyncio @pytest.mark.parametrize("capture_content", ["SPAN_ONLY", "NO_CONTENT"]) async def test_async_prompt_template_propagates_prompt_context( - span_exporter, tracer_provider, meter_provider, logger_provider, capture_content + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + capture_content, ): model = FakeMessagesListChatModel( responses=[AIMessage(content="It is raining.")] From 3dcd2b7a3903df2f5b25689bc463dfd43e01fb0f Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 14 Oct 2026 10:08:47 -0700 Subject: [PATCH 16/16] Fix lint --- .../tests/test_callback_handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index ba888630b..280125cc1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -35,7 +35,11 @@ LLMResult, ) from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnableConfig, RunnableLambda, RunnableParallel +from langchain_core.runnables import ( + RunnableConfig, + RunnableLambda, + RunnableParallel, +) from tests._api import ( FakeChatModel, FakeRetriever,