diff --git a/CHANGELOG.md b/CHANGELOG.md index 938a017..a31d132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Features Added +- Emit `gen_ai.output.type` and available GenAI request parameters on LangChain chat spans. [#213](https://github.com/microsoft/opentelemetry-distro-javascript/pull/213) + ### Bugs Fixed - Fix duplicate Bunyan logs, missing HTTP duration metrics, duplicate request filtering, and incorrect performance-counter values. [#212](https://github.com/microsoft/opentelemetry-distro-javascript/pull/212) diff --git a/src/genai/instrumentations/langchain/tracer.ts b/src/genai/instrumentations/langchain/tracer.ts index 421eab6..68ee8c1 100644 --- a/src/genai/instrumentations/langchain/tracer.ts +++ b/src/genai/instrumentations/langchain/tracer.ts @@ -228,6 +228,7 @@ export class LangChainTracer extends BaseTracer { } Utils.setModelAttribute(run, span); Utils.setChoiceCountAttribute(run, span); + Utils.setRequestAttributes(run, span); Utils.setResponseIdAttribute(run, span); Utils.setFinishReasonsAttribute(run, span); Utils.setProviderNameAttribute(run, span); diff --git a/src/genai/instrumentations/langchain/utils.ts b/src/genai/instrumentations/langchain/utils.ts index 79fa804..eec1c16 100644 --- a/src/genai/instrumentations/langchain/utils.ts +++ b/src/genai/instrumentations/langchain/utils.ts @@ -9,10 +9,20 @@ import { ATTR_GEN_AI_CONVERSATION_ID, ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_OUTPUT_TYPE, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_PROVIDER_NAME, ATTR_GEN_AI_REQUEST_CHOICE_COUNT, + ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY, + ATTR_GEN_AI_REQUEST_MAX_TOKENS, ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY, + ATTR_GEN_AI_REQUEST_SEED, + ATTR_GEN_AI_REQUEST_STOP_SEQUENCES, + ATTR_GEN_AI_REQUEST_STREAM, + ATTR_GEN_AI_REQUEST_TEMPERATURE, + ATTR_GEN_AI_REQUEST_TOP_K, + ATTR_GEN_AI_REQUEST_TOP_P, ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_RESPONSE_FINISH_REASONS, ATTR_GEN_AI_RESPONSE_MODEL, @@ -566,6 +576,124 @@ export function setChoiceCountAttribute(run: Run, span: Span) { } } +// LangChain exposes provider-specific invocation params as unknown values. Match the upstream +// OpenTelemetry GenAI instrumentations by accepting typed SDK values without coercing strings. +function firstDefined(params: Record, keys: string[]): unknown { + for (const key of keys) { + if (params[key] !== undefined && params[key] !== null) { + return params[key]; + } + } + return undefined; +} + +function integer(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) + ? value + : undefined; +} + +function stringArray(value: unknown): string[] | undefined { + if (isString(value) && value.length > 0) return [value]; + if (!Array.isArray(value)) return undefined; + const strings = value.filter((item): item is string => isString(item) && item.length > 0); + return strings.length === value.length && strings.length > 0 ? strings : undefined; +} + +const OUTPUT_TYPES: readonly string[] = ["text", "json", "image"]; + +const OUTPUT_TYPE_ALIASES: Readonly> = { + json_object: "json", + json_schema: "json", + b64_json: "image", + url: "image", +}; + +function normalizeOutputType(value: unknown): string | undefined { + if (!isString(value)) return undefined; + const normalized = value.trim().toLowerCase(); + return OUTPUT_TYPES.includes(normalized) ? normalized : OUTPUT_TYPE_ALIASES[normalized]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function getOutputType(params: Record): string | undefined { + const explicit = firstDefined(params, ["output_type", "outputType"]); + const explicitType = normalizeOutputType(explicit); + if (explicitType) return explicitType; + + const responseFormat = firstDefined(params, ["response_format", "responseFormat"]); + if (isRecord(responseFormat)) { + const formatType = normalizeOutputType(responseFormat.type); + if (formatType) return formatType; + } + const responseFormatType = normalizeOutputType(responseFormat); + if (responseFormatType) return responseFormatType; + + const text = params.text; + if (isRecord(text)) { + const format = text.format; + if (isRecord(format)) { + return normalizeOutputType(format.type); + } + } + return undefined; +} + +// Request attributes surfaced by LangChain under extra.invocation_params. +// Both OpenAI-style snake_case and LangChain/provider camelCase aliases are +// accepted because callback payloads vary by model integration and API path. +export function setRequestAttributes(run: Run, span: Span): void { + const params = run.extra?.invocation_params; + if (!isRecord(params)) return; + + const outputType = getOutputType(params); + if (outputType) span.setAttribute(ATTR_GEN_AI_OUTPUT_TYPE, outputType); + + const numberAttributes: Array<[string, string[]]> = [ + [ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY, ["frequency_penalty", "frequencyPenalty"]], + [ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY, ["presence_penalty", "presencePenalty"]], + [ATTR_GEN_AI_REQUEST_TEMPERATURE, ["temperature"]], + [ATTR_GEN_AI_REQUEST_TOP_P, ["top_p", "topP"]], + ]; + for (const [attribute, keys] of numberAttributes) { + const value = firstDefined(params, keys); + if (typeof value === "number" && Number.isFinite(value)) { + span.setAttribute(attribute, value); + } + } + + const maxTokens = integer( + firstDefined(params, [ + "max_tokens", + "maxTokens", + "max_completion_tokens", + "maxCompletionTokens", + "max_output_tokens", + "maxOutputTokens", + ]), + ); + if (maxTokens !== undefined) { + span.setAttribute(ATTR_GEN_AI_REQUEST_MAX_TOKENS, maxTokens); + } + + const seed = integer(firstDefined(params, ["seed"])); + if (seed !== undefined) span.setAttribute(ATTR_GEN_AI_REQUEST_SEED, seed); + + const topK = integer(firstDefined(params, ["top_k", "topK"])); + if (topK !== undefined) span.setAttribute(ATTR_GEN_AI_REQUEST_TOP_K, topK); + + const stopSequences = stringArray( + firstDefined(params, ["stop", "stop_sequences", "stopSequences"]), + ); + if (stopSequences) span.setAttribute(ATTR_GEN_AI_REQUEST_STOP_SEQUENCES, stopSequences); + + const stream = firstDefined(params, ["stream", "streaming"]); + if (typeof stream === "boolean") span.setAttribute(ATTR_GEN_AI_REQUEST_STREAM, stream); +} + // Response identifier - Helper to extract the unique response id returned by // the underlying provider (e.g. OpenAI chat completion id). LangChain.js // typically surfaces this as the AIMessage id (top-level for v1, nested diff --git a/src/genai/semconv.ts b/src/genai/semconv.ts index dc7c4d0..0936a34 100644 --- a/src/genai/semconv.ts +++ b/src/genai/semconv.ts @@ -26,8 +26,18 @@ export const ATTR_ERROR_MESSAGE = "error.message" as const; // GenAI core export const ATTR_GEN_AI_OPERATION_NAME = "gen_ai.operation.name" as const; +export const ATTR_GEN_AI_OUTPUT_TYPE = "gen_ai.output.type" as const; export const ATTR_GEN_AI_REQUEST_MODEL = "gen_ai.request.model" as const; export const ATTR_GEN_AI_REQUEST_CHOICE_COUNT = "gen_ai.request.choice.count" as const; +export const ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" as const; +export const ATTR_GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" as const; +export const ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" as const; +export const ATTR_GEN_AI_REQUEST_SEED = "gen_ai.request.seed" as const; +export const ATTR_GEN_AI_REQUEST_STOP_SEQUENCES = "gen_ai.request.stop_sequences" as const; +export const ATTR_GEN_AI_REQUEST_STREAM = "gen_ai.request.stream" as const; +export const ATTR_GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" as const; +export const ATTR_GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k" as const; +export const ATTR_GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" as const; export const ATTR_GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" as const; export const ATTR_GEN_AI_RESPONSE_ID = "gen_ai.response.id" as const; export const ATTR_GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons" as const; diff --git a/test/internal/unit/a365/agent365Exporter.test.ts b/test/internal/unit/a365/agent365Exporter.test.ts index 5c92c0d..dbbe0a6 100644 --- a/test/internal/unit/a365/agent365Exporter.test.ts +++ b/test/internal/unit/a365/agent365Exporter.test.ts @@ -374,6 +374,35 @@ describe("Agent365Exporter", () => { assert.deepStrictEqual(exportedSpan.attributes["number_array_attr"], [1, 2, 3]); }); + it("should preserve GenAI request attributes in the A365 payload", async () => { + const { exportedSpan } = await exportAndGetPayload(fetchSpy, { + "gen_ai.output.type": "json", + "gen_ai.request.frequency_penalty": 0.1, + "gen_ai.request.max_tokens": 512, + "gen_ai.request.presence_penalty": -0.2, + "gen_ai.request.seed": 42, + "gen_ai.request.stop_sequences": ["DONE", "STOP"], + "gen_ai.request.stream": false, + "gen_ai.request.temperature": 0.2, + "gen_ai.request.top_k": 40, + "gen_ai.request.top_p": 0.8, + }); + + assert.strictEqual(exportedSpan.attributes["gen_ai.output.type"], "json"); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.frequency_penalty"], 0.1); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.max_tokens"], 512); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.presence_penalty"], -0.2); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.seed"], 42); + assert.deepStrictEqual(exportedSpan.attributes["gen_ai.request.stop_sequences"], [ + "DONE", + "STOP", + ]); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.stream"], false); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.temperature"], 0.2); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.top_k"], 40); + assert.strictEqual(exportedSpan.attributes["gen_ai.request.top_p"], 0.8); + }); + it("should partition spans by identity and export separately", async () => { const exporter = new Agent365Exporter({ tokenResolver: () => "test-token", diff --git a/test/internal/unit/genai/langchain/tracer.test.ts b/test/internal/unit/genai/langchain/tracer.test.ts index 17065f1..754ba09 100644 --- a/test/internal/unit/genai/langchain/tracer.test.ts +++ b/test/internal/unit/genai/langchain/tracer.test.ts @@ -16,10 +16,13 @@ import { ATTR_ERROR_MESSAGE, ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_OUTPUT_TYPE, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_PROVIDER_NAME, ATTR_GEN_AI_REQUEST_CHOICE_COUNT, ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_REQUEST_TEMPERATURE, + ATTR_GEN_AI_REQUEST_TOP_P, ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_RESPONSE_MODEL, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, @@ -426,7 +429,13 @@ describe("LangChainTracer", () => { }, extra: { metadata: { ls_model_name: "deployment-o4-mini", ls_provider: "openai" }, - invocation_params: { model: "deployment-o4-mini", n: 3 }, + invocation_params: { + model: "deployment-o4-mini", + n: 3, + temperature: 0.2, + top_p: 0.8, + response_format: { type: "json_object" }, + }, }, inputs: { messages: [[{ role: "user", content: "hello" }]], @@ -481,6 +490,9 @@ describe("LangChainTracer", () => { 3, "request choice count should come from invocation_params.n when >1", ); + assert.strictEqual(got(ATTR_GEN_AI_REQUEST_TEMPERATURE), 0.2, "request temperature"); + assert.strictEqual(got(ATTR_GEN_AI_REQUEST_TOP_P), 0.8, "request top_p"); + assert.strictEqual(got(ATTR_GEN_AI_OUTPUT_TYPE), "json", "requested output type"); assert.strictEqual( got(ATTR_GEN_AI_RESPONSE_MODEL), "o4-mini-2025-04-16", diff --git a/test/internal/unit/genai/langchain/utils.test.ts b/test/internal/unit/genai/langchain/utils.test.ts index a5909a6..4b6da85 100644 --- a/test/internal/unit/genai/langchain/utils.test.ts +++ b/test/internal/unit/genai/langchain/utils.test.ts @@ -13,6 +13,7 @@ import { setOutputMessagesAttribute, setModelAttribute, setChoiceCountAttribute, + setRequestAttributes, setProviderNameAttribute, setResponseIdAttribute, setFinishReasonsAttribute, @@ -25,10 +26,20 @@ import { ATTR_GEN_AI_AGENT_NAME, ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OPERATION_NAME, + ATTR_GEN_AI_OUTPUT_TYPE, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_PROVIDER_NAME, ATTR_GEN_AI_REQUEST_CHOICE_COUNT, + ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY, + ATTR_GEN_AI_REQUEST_MAX_TOKENS, ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY, + ATTR_GEN_AI_REQUEST_SEED, + ATTR_GEN_AI_REQUEST_STOP_SEQUENCES, + ATTR_GEN_AI_REQUEST_STREAM, + ATTR_GEN_AI_REQUEST_TEMPERATURE, + ATTR_GEN_AI_REQUEST_TOP_K, + ATTR_GEN_AI_REQUEST_TOP_P, ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_RESPONSE_FINISH_REASONS, ATTR_GEN_AI_RESPONSE_MODEL, @@ -801,6 +812,183 @@ describe("setChoiceCountAttribute", () => { }); }); +describe("setRequestAttributes", () => { + it("sets GenAI request parameters from OpenAI-style invocation params", () => { + const span = makeSpan(); + const run = makeRun({ + extra: { + invocation_params: { + temperature: 0.2, + top_p: 0.8, + top_k: 40, + max_completion_tokens: 512, + frequency_penalty: 0.1, + presence_penalty: -0.2, + seed: 42, + stop: ["DONE", "STOP"], + stream: true, + response_format: { type: "json_schema" }, + }, + }, + }); + + setRequestAttributes(run, span); + + assert.deepStrictEqual(span.attrs, { + [ATTR_GEN_AI_OUTPUT_TYPE]: "json", + [ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY]: 0.1, + [ATTR_GEN_AI_REQUEST_MAX_TOKENS]: 512, + [ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY]: -0.2, + [ATTR_GEN_AI_REQUEST_TEMPERATURE]: 0.2, + [ATTR_GEN_AI_REQUEST_TOP_K]: 40, + [ATTR_GEN_AI_REQUEST_TOP_P]: 0.8, + [ATTR_GEN_AI_REQUEST_SEED]: 42, + [ATTR_GEN_AI_REQUEST_STOP_SEQUENCES]: ["DONE", "STOP"], + [ATTR_GEN_AI_REQUEST_STREAM]: true, + }); + }); + + it("supports typed camelCase provider aliases and normalizes a single stop sequence", () => { + const span = makeSpan(); + const run = makeRun({ + extra: { + invocation_params: { + maxOutputTokens: 256, + frequencyPenalty: 0.25, + presencePenalty: 0, + topP: 0.9, + topK: 10, + stopSequences: "END", + streaming: false, + responseFormat: "text", + }, + }, + }); + + setRequestAttributes(run, span); + + assert.strictEqual(span.attrs[ATTR_GEN_AI_OUTPUT_TYPE], "text"); + assert.strictEqual(span.attrs[ATTR_GEN_AI_REQUEST_MAX_TOKENS], 256); + assert.strictEqual(span.attrs[ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY], 0.25); + assert.strictEqual(span.attrs[ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY], 0); + assert.strictEqual(span.attrs[ATTR_GEN_AI_REQUEST_TOP_P], 0.9); + assert.strictEqual(span.attrs[ATTR_GEN_AI_REQUEST_TOP_K], 10); + assert.deepStrictEqual(span.attrs[ATTR_GEN_AI_REQUEST_STOP_SEQUENCES], ["END"]); + assert.strictEqual(span.attrs[ATTR_GEN_AI_REQUEST_STREAM], false); + }); + + it("preserves whitespace stop sequences", () => { + for (const stop of ["\n", ["STOP", "\n\n"]]) { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { stop } } }); + + setRequestAttributes(run, span); + + assert.deepStrictEqual( + span.attrs[ATTR_GEN_AI_REQUEST_STOP_SEQUENCES], + Array.isArray(stop) ? stop : [stop], + ); + } + }); + + it("rejects empty stop sequences", () => { + for (const stop of ["", ["DONE", ""]]) { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params: { stop } } }); + + setRequestAttributes(run, span); + + assert.deepStrictEqual(span.attrs, {}); + } + }); + + it("ignores invocation params that are not objects", () => { + for (const invocation_params of [null, "invalid", 1, true, []]) { + const span = makeSpan(); + const run = makeRun({ extra: { invocation_params } }); + + setRequestAttributes(run, span); + + assert.deepStrictEqual(span.attrs, {}); + } + }); + + it("reads Responses API output type from text.format", () => { + const span = makeSpan(); + const run = makeRun({ + extra: { + invocation_params: { + text: { format: { type: "json_schema" } }, + }, + }, + }); + + setRequestAttributes(run, span); + + assert.strictEqual(span.attrs[ATTR_GEN_AI_OUTPUT_TYPE], "json"); + }); + + it("normalizes canonical and provider-specific output types", () => { + const outputTypes = [ + ["text", "text"], + ["json", "json"], + ["image", "image"], + ["json_object", "json"], + ["json_schema", "json"], + ["b64_json", "image"], + ["url", "image"], + ]; + + for (const [outputType, expected] of outputTypes) { + const span = makeSpan(); + const run = makeRun({ + extra: { invocation_params: { output_type: outputType } }, + }); + + setRequestAttributes(run, span); + + assert.strictEqual(span.attrs[ATTR_GEN_AI_OUTPUT_TYPE], expected); + } + }); + + it("ignores output types not documented by the semantic conventions", () => { + for (const outputType of ["speech", "audio"]) { + const span = makeSpan(); + const run = makeRun({ + extra: { invocation_params: { output_type: outputType } }, + }); + + setRequestAttributes(run, span); + + assert.strictEqual(span.attrs[ATTR_GEN_AI_OUTPUT_TYPE], undefined); + } + }); + + it("ignores absent or invalid request parameters", () => { + const span = makeSpan(); + const run = makeRun({ + extra: { + invocation_params: { + temperature: Number.NaN, + top_p: "not-a-number", + top_k: 1.5, + max_tokens: 1.5, + maxOutputTokens: "256", + frequencyPenalty: "0.25", + seed: {}, + stop: ["valid", 1], + stream: "false", + response_format: { type: "unsupported" }, + }, + }, + }); + + setRequestAttributes(run, span); + + assert.deepStrictEqual(span.attrs, {}); + }); +}); + describe("setResponseIdAttribute", () => { it("extracts response id from AIMessage.id (v1)", () => { const span = makeSpan();