From 5118379885c11576c9383df3e8dd6a85cba7a2f2 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 1 Oct 2026 13:06:56 -0600 Subject: [PATCH 1/2] feat(api): upgrade AI SDK to v7 and trace AI calls with GenAI semantic conventions AI SDK v7 no longer emits OpenTelemetry spans by itself, so the API now registers @ai-sdk/otel's OpenTelemetry integration once at startup. Spans follow the GenAI semantic conventions (invoke_agent / chat / execute_tool, gen_ai.* attributes) instead of the legacy ai.* shape. v7 also dropped telemetry.metadata. Team/user/session attribution now goes through runtimeContext, and an enrichSpan hook stamps hyperdx.team.id, user.id and gen_ai.conversation.id on every span. llmTelemetry() lists the same keys in includeRuntimeContext, which v7 requires before integrations can see them. The v7 packages are ESM-only: compiled CJS loads them via require(esm), Jest transpiles them, and hdx-eval's tsconfig moves to module node20. hdx-eval's judge passes `instructions` instead of the deprecated `system`. --- .changeset/ai-sdk-v7-genai-spans.md | 7 + packages/api/jest.config.js | 5 + packages/api/jest.int.config.js | 3 +- packages/api/package.json | 8 +- packages/api/src/api-app.ts | 3 + .../api/src/controllers/__tests__/ai.test.ts | 20 +- packages/api/src/controllers/ai.ts | 22 -- packages/api/src/routers/api/ai.ts | 8 +- .../src/utils/__tests__/aiTelemetry.test.ts | 209 ++++++++++++++++++ packages/api/src/utils/aiTelemetry.ts | 76 +++++++ packages/hdx-eval/jest.config.js | 9 +- packages/hdx-eval/package.json | 6 +- .../hdx-eval/src/__tests__/judge.sdk.test.ts | 72 ++++++ packages/hdx-eval/src/__tests__/judge.test.ts | 33 +++ packages/hdx-eval/src/grading/judge.ts | 2 +- packages/hdx-eval/tsconfig.json | 5 +- yarn.lock | 159 +++++++------ 17 files changed, 525 insertions(+), 122 deletions(-) create mode 100644 .changeset/ai-sdk-v7-genai-spans.md create mode 100644 packages/api/src/utils/__tests__/aiTelemetry.test.ts create mode 100644 packages/api/src/utils/aiTelemetry.ts create mode 100644 packages/hdx-eval/src/__tests__/judge.sdk.test.ts diff --git a/.changeset/ai-sdk-v7-genai-spans.md b/.changeset/ai-sdk-v7-genai-spans.md new file mode 100644 index 0000000000..24e2d5ad58 --- /dev/null +++ b/.changeset/ai-sdk-v7-genai-spans.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +feat: upgrade the AI SDK to v7 and trace AI calls with GenAI semantic conventions + +AI assistant calls now emit OpenTelemetry spans that follow the GenAI semantic conventions (`invoke_agent` / `chat` / `execute_tool` spans with `gen_ai.*` attributes) instead of the legacy `ai.*` span shape. Team and user attribution moves from `ai.telemetry.metadata.*` to `hyperdx.team.id` and `user.id` (plus `gen_ai.conversation.id` when a session is known). Saved searches, alerts or dashboards keyed on the old `ai.*` span names or attributes need updating. diff --git a/packages/api/jest.config.js b/packages/api/jest.config.js index d8d938caa1..d34b88a93a 100644 --- a/packages/api/jest.config.js +++ b/packages/api/jest.config.js @@ -5,6 +5,9 @@ const tsJestTransformCfg = createJsWithTsPreset({ // TypeScript 6 requires an explicit rootDir when compiling a subset of // files (ts-jest compiles per-file), otherwise it errors with TS5011. rootDir: './src', + // The AI SDK (v7+) ships ESM-only builds that Jest's CJS loader can't + // parse, so they are transpiled like source (see transformIgnorePatterns). + allowJs: true, }, }); @@ -22,4 +25,6 @@ module.exports = { moduleNameMapper: { '@/(.*)$': '/$1', }, + // Keep in sync with jest.int.config.js. + transformIgnorePatterns: ['/node_modules/(?!(ai|@ai-sdk|@workflow/serde)/)'], }; diff --git a/packages/api/jest.int.config.js b/packages/api/jest.int.config.js index 30584352dd..0e8fe29158 100644 --- a/packages/api/jest.int.config.js +++ b/packages/api/jest.int.config.js @@ -21,8 +21,9 @@ module.exports = { '^.+\\.mjs$': esmDepsTransformCfg.transform['^.+\\.[tj]sx?$'], }, moduleFileExtensions: ['js', 'mjs', 'json', 'ts', 'tsx', 'node'], + // Superset of the unit config's list (AI SDK packages) plus the proxy deps. transformIgnorePatterns: [ - '/node_modules/(?!(http-proxy-middleware|httpxy|is-plain-obj)/)', + '/node_modules/(?!(ai|@ai-sdk|@workflow/serde|http-proxy-middleware|httpxy|is-plain-obj)/)', ], testMatch: ['**/__tests__/**/*.int.test.ts?(x)'], // Override the unit config's ignore list: it excludes `.int.test.ts`, which diff --git a/packages/api/package.json b/packages/api/package.json index a72eb1ea25..e385a6263d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -16,8 +16,9 @@ } }, "dependencies": { - "@ai-sdk/anthropic": "^3.0.58", - "@ai-sdk/openai": "^3.0.47", + "@ai-sdk/anthropic": "^4.0.65", + "@ai-sdk/openai": "^4.0.78", + "@ai-sdk/otel": "^1.0.116", "@braintree/sanitize-url": "^7.1.1", "@clickhouse/client-common": "1.23.0-head.fae5998.1", "@esm2cjs/p-queue": "^7.3.0", @@ -30,7 +31,7 @@ "@opentelemetry/sdk-metrics": "^1.30.1", "@slack/webhook": "^7.0.0", "@types/node": "^22.15.18", - "ai": "^6.0.116", + "ai": "^7.0.116", "aws4": "^1.13.2", "chrono-node": "^2.9.0", "compression": "^1.7.4", @@ -65,6 +66,7 @@ "zod-express-middleware": "^1.4.0" }, "devDependencies": { + "@opentelemetry/sdk-trace-base": "^1.30.1", "@stoplight/spectral-cli": "^6.15.0", "@types/compression": "^1.7.3", "@types/cors": "^2.8.14", diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index 5e336d0d8d..c2d9028a15 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -19,9 +19,12 @@ import savedSearchRouter from './routers/api/savedSearch'; import sourcesRouter from './routers/api/sources'; import externalRoutersV2 from './routers/external-api/v2'; import usageStats from './tasks/usageStats'; +import { registerAITelemetry } from './utils/aiTelemetry'; import logger, { expressLogger } from './utils/logger'; import passport from './utils/passport'; +registerAITelemetry(); + const app: express.Application = express(); const sess: session.SessionOptions & { cookie: session.CookieOptions } = { diff --git a/packages/api/src/controllers/__tests__/ai.test.ts b/packages/api/src/controllers/__tests__/ai.test.ts index 165612cd71..2f7d3a048f 100644 --- a/packages/api/src/controllers/__tests__/ai.test.ts +++ b/packages/api/src/controllers/__tests__/ai.test.ts @@ -48,7 +48,7 @@ function setConfig(overrides: Record) { Object.assign(mockConfig, overrides); } -import { getAIModel, llmTelemetry } from '@/controllers/ai'; +import { getAIModel } from '@/controllers/ai'; beforeEach(() => { setConfig({}); @@ -252,21 +252,3 @@ describe('openai provider', () => { }); }); }); - -describe('llmTelemetry', () => { - it('enables telemetry and passes through string metadata', () => { - expect( - llmTelemetry({ sessionId: 'notebook-1', teamId: 't1', userId: 'u1' }), - ).toEqual({ - isEnabled: true, - metadata: { sessionId: 'notebook-1', teamId: 't1', userId: 'u1' }, - }); - }); - - it('drops nullish metadata values', () => { - expect(llmTelemetry({ sessionId: undefined, teamId: 't1' })).toEqual({ - isEnabled: true, - metadata: { teamId: 't1' }, - }); - }); -}); diff --git a/packages/api/src/controllers/ai.ts b/packages/api/src/controllers/ai.ts index f7b2a6f5a4..eeaeae13bc 100644 --- a/packages/api/src/controllers/ai.ts +++ b/packages/api/src/controllers/ai.ts @@ -73,28 +73,6 @@ export function getAIModel(): LanguageModel { } } -/** - * Vercel AI SDK telemetry settings for LLM calls. The SDK flattens each - * metadata entry onto every span of the call as - * `ai.telemetry.metadata.`, which the LLM dashboard reads: - * `sessionId` groups the call's spans into a session on the Sessions tab - * (pass a stable conversation-scoped id, e.g. a notebook id) and `userId` - * feeds per-user attribution. Nullish values are dropped. - */ -export function llmTelemetry(metadata: Record): { - isEnabled: boolean; - metadata: Record; -} { - return { - isEnabled: true, - metadata: Object.fromEntries( - Object.entries(metadata).filter( - (entry): entry is [string, string] => entry[1] != null, - ), - ), - }; -} - export async function getAIMetadata(source: ISource) { const connectionId = source.connection.toString(); diff --git a/packages/api/src/routers/api/ai.ts b/packages/api/src/routers/api/ai.ts index bd6419f2ac..5f55fbd61f 100644 --- a/packages/api/src/routers/api/ai.ts +++ b/packages/api/src/routers/api/ai.ts @@ -11,10 +11,10 @@ import { getAIMetadata, getAIModel, getChartConfigFromResolvedConfig, - llmTelemetry, } from '@/controllers/ai'; import { getSource } from '@/controllers/sources'; import { getNonNullUserWithTeam } from '@/middleware/auth'; +import { llmTelemetry } from '@/utils/aiTelemetry'; import { Api404Error, Api500Error } from '@/utils/errors'; import { withOperationMetrics } from '@/utils/instrumentation'; import logger from '@/utils/logger'; @@ -106,11 +106,7 @@ ${JSON.stringify(allFieldsWithKeys.slice(0, 200).map(f => ({ field: f.key, type: output: Output.object({ schema: AssistantLineTableConfigSchema, }), - // teamId/userId flatten to ai.telemetry.metadata.* span - // attributes for attribution on the LLM dashboard. Callers - // with a conversation-scoped id should also pass sessionId so - // their calls group into a session (see llmTelemetry). - experimental_telemetry: llmTelemetry({ + ...llmTelemetry({ teamId: teamId.toString(), userId: userId.toString(), }), diff --git a/packages/api/src/utils/__tests__/aiTelemetry.test.ts b/packages/api/src/utils/__tests__/aiTelemetry.test.ts new file mode 100644 index 0000000000..1f66d2eb01 --- /dev/null +++ b/packages/api/src/utils/__tests__/aiTelemetry.test.ts @@ -0,0 +1,209 @@ +import { trace } from '@opentelemetry/api'; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import { generateText, isStepCount, tool } from 'ai'; +import { MockLanguageModelV4 } from 'ai/test'; +import { z } from 'zod'; + +import { + createAITelemetry, + enrichLLMSpan, + type LLMAttribution, + llmTelemetry, + registerAITelemetry, +} from '@/utils/aiTelemetry'; + +const usage: Awaited>['usage'] = { + inputTokens: { + total: 10, + noCache: 10, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, +}; + +// Step 1 calls a tool, step 2 answers, so the call produces operation, step, +// chat and execute_tool spans. +const createModel = () => { + let call = 0; + return new MockLanguageModelV4({ + provider: 'anthropic', + modelId: 'mock-model', + doGenerate: async () => { + call += 1; + return call === 1 + ? { + content: [ + { + type: 'tool-call', + toolCallId: 'tc-1', + toolName: 'lookup', + input: '{"q":"x"}', + }, + ], + finishReason: { unified: 'tool-calls', raw: undefined }, + usage, + warnings: [], + } + : { + content: [{ type: 'text', text: 'done' }], + finishReason: { unified: 'stop', raw: undefined }, + usage, + warnings: [], + }; + }, + }); +}; + +const runToolCall = (telemetry: ReturnType) => + generateText({ + model: createModel(), + prompt: 'hi', + tools: { + lookup: tool({ + inputSchema: z.object({ q: z.string() }), + execute: async ({ q }) => ({ q }), + }), + }, + stopWhen: isStepCount(2), + ...telemetry, + }); + +const runTracedCall = async (attribution: LLMAttribution) => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + await runToolCall( + llmTelemetry(attribution, { + functionId: 'test.fn', + integrations: createAITelemetry({ tracer: provider.getTracer('test') }), + }), + ); + return exporter.getFinishedSpans(); +}; + +const operationNames = (spans: { attributes: Record }[]) => + spans.map(s => s.attributes['gen_ai.operation.name']).sort(); + +describe('AI SDK OpenTelemetry integration', () => { + it('stamps attribution onto every span of the call', async () => { + const spans = await runTracedCall({ + sessionId: 'session-1', + teamId: 'team-1', + userId: 'user-1', + }); + + const operations = spans.map(s => s.attributes['gen_ai.operation.name']); + expect(operations).toEqual( + expect.arrayContaining(['chat', 'execute_tool']), + ); + for (const span of spans) { + expect(span.attributes).toMatchObject({ + 'gen_ai.conversation.id': 'session-1', + 'hyperdx.team.id': 'team-1', + 'user.id': 'user-1', + }); + } + }); + + it('omits the session attribute when only team and user are known', async () => { + // The chart assistant has no conversation-scoped id. + const spans = await runTracedCall({ teamId: 'team-1', userId: 'user-1' }); + + expect(spans.length).toBeGreaterThan(0); + for (const span of spans) { + expect(span.attributes).toMatchObject({ + 'hyperdx.team.id': 'team-1', + 'user.id': 'user-1', + }); + expect(span.attributes).not.toHaveProperty('gen_ai.conversation.id'); + } + }); + + it('emits GenAI semconv attributes the LLM dashboard reads', async () => { + const spans = await runTracedCall({ sessionId: 'session-1' }); + + const chat = spans.find( + s => s.attributes['gen_ai.operation.name'] === 'chat', + ); + expect(chat?.attributes).toMatchObject({ + 'gen_ai.provider.name': 'anthropic', + 'gen_ai.request.model': 'mock-model', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 5, + }); + expect( + spans.some(s => s.attributes['gen_ai.agent.name'] === 'test.fn'), + ).toBe(true); + + // Message content feeds the dashboard's Messages tab. + expect(chat?.attributes['gen_ai.input.messages']).toEqual( + expect.stringContaining('"content":"hi"'), + ); + expect(chat?.attributes['gen_ai.output.messages']).toEqual( + expect.stringContaining('"name":"lookup"'), + ); + const toolSpan = spans.find( + s => s.attributes['gen_ai.operation.name'] === 'execute_tool', + ); + expect(toolSpan?.attributes).toMatchObject({ + 'gen_ai.tool.name': 'lookup', + 'gen_ai.tool.call.arguments': expect.stringContaining('"q":"x"'), + }); + }); +}); + +describe('enrichLLMSpan', () => { + const enrich = (runtimeContext: Record | undefined) => + enrichLLMSpan({ + spanType: 'languageModel', + operationId: 'ai.generateText', + callId: 'call-1', + runtimeContext, + }); + + it('skips missing, empty and non-string values', () => { + expect(enrich({ sessionId: '', teamId: 42, userId: undefined })).toEqual( + {}, + ); + expect(enrich(undefined)).toEqual({}); + }); + + it('ignores unrelated runtime context keys', () => { + expect(enrich({ sessionId: 's1', other: 'x' })).toEqual({ + 'gen_ai.conversation.id': 's1', + }); + }); +}); + +describe('registerAITelemetry', () => { + afterAll(() => trace.disable()); + + it('emits each span once through the global tracer provider, even if registered twice', async () => { + const perCall = operationNames( + await runTracedCall({ sessionId: 'session-1' }), + ); + + const exporter = new InMemorySpanExporter(); + trace.setGlobalTracerProvider( + new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }), + ); + registerAITelemetry(); + registerAITelemetry(); + await runToolCall(llmTelemetry({ sessionId: 'session-global' })); + + const spans = exporter.getFinishedSpans(); + expect(perCall.length).toBeGreaterThan(0); + expect(operationNames(spans)).toEqual(perCall); + for (const span of spans) { + expect(span.attributes['gen_ai.conversation.id']).toBe('session-global'); + } + }); +}); diff --git a/packages/api/src/utils/aiTelemetry.ts b/packages/api/src/utils/aiTelemetry.ts new file mode 100644 index 0000000000..ed4192804e --- /dev/null +++ b/packages/api/src/utils/aiTelemetry.ts @@ -0,0 +1,76 @@ +import { type EnrichSpan, OpenTelemetry } from '@ai-sdk/otel'; +import { registerTelemetry, type TelemetryOptions } from 'ai'; + +// GenAI/OTel semconv keys where one exists; team matches setBusinessContext. +const ATTRIBUTION_SPAN_ATTRIBUTES = { + sessionId: 'gen_ai.conversation.id', + teamId: 'hyperdx.team.id', + userId: 'user.id', +} as const; + +/** + * Who an LLM call is attributed to on the LLM dashboard. `sessionId` groups + * spans into a session on the Sessions tab (use a stable conversation-scoped + * id) and `userId` feeds per-user attribution. + */ +export type LLMAttribution = { + [K in keyof typeof ATTRIBUTION_SPAN_ATTRIBUTES]?: string; +}; + +// The SDK hides runtimeContext keys from telemetry integrations unless they +// are listed here; `satisfies` keeps this in lockstep with the map above. +const INCLUDE_ATTRIBUTION = { + sessionId: true, + teamId: true, + userId: true, +} as const satisfies Record; + +const SPAN_ATTRIBUTE_BY_KEY: ReadonlyMap = new Map( + Object.entries(ATTRIBUTION_SPAN_ATTRIBUTES), +); + +/** + * `generateText` options that carry `attribution` onto every span of the + * call. + */ +export function llmTelemetry( + attribution: LLMAttribution, + telemetry: Omit = {}, +) { + return { + runtimeContext: attribution, + telemetry: { ...telemetry, includeRuntimeContext: INCLUDE_ATTRIBUTION }, + } satisfies { + runtimeContext: LLMAttribution; + telemetry: TelemetryOptions; + }; +} + +export const enrichLLMSpan: EnrichSpan = ({ runtimeContext }) => + Object.fromEntries( + Object.entries(runtimeContext ?? {}).flatMap(([key, value]) => { + const attribute = SPAN_ATTRIBUTE_BY_KEY.get(key); + return attribute && typeof value === 'string' && value !== '' + ? [[attribute, value]] + : []; + }), + ); + +export function createAITelemetry( + options: ConstructorParameters[0] = {}, +): OpenTelemetry { + return new OpenTelemetry({ enrichSpan: enrichLLMSpan, ...options }); +} + +let registered = false; + +/** + * AI SDK v7 only emits spans through a registered integration. + * `registerTelemetry` appends, so repeat calls here are no-ops rather than + * duplicating every span. + */ +export function registerAITelemetry(): void { + if (registered) return; + registered = true; + registerTelemetry(createAITelemetry()); +} diff --git a/packages/hdx-eval/jest.config.js b/packages/hdx-eval/jest.config.js index f184265f85..4eec3651bb 100644 --- a/packages/hdx-eval/jest.config.js +++ b/packages/hdx-eval/jest.config.js @@ -1,6 +1,12 @@ const { createJsWithTsPreset } = require('ts-jest'); -const tsJestTransformCfg = createJsWithTsPreset(); +const tsJestTransformCfg = createJsWithTsPreset({ + tsconfig: { + // The AI SDK (v7+) ships ESM-only builds that Jest's CJS loader can't + // parse, so they are transpiled like source (see transformIgnorePatterns). + allowJs: true, + }, +}); /** @type {import("jest").Config} **/ module.exports = { @@ -12,6 +18,7 @@ module.exports = { moduleNameMapper: { '@/(.*)$': '/$1', }, + transformIgnorePatterns: ['/node_modules/(?!(ai|@ai-sdk|@workflow/serde)/)'], // Coverage floor pinned just below measured reality so coverage can only // ratchet up. Decay below these numbers fails the build. Raise them // deliberately as coverage improves; never lower them silently. diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json index 9553a7dffc..c012565af2 100644 --- a/packages/hdx-eval/package.json +++ b/packages/hdx-eval/package.json @@ -13,10 +13,10 @@ "dist/*" ], "dependencies": { - "@ai-sdk/anthropic": "^3.0.58", - "@ai-sdk/openai": "^3.0.47", + "@ai-sdk/anthropic": "^4.0.65", + "@ai-sdk/openai": "^4.0.78", "@clickhouse/client": "1.23.0-head.fae5998.1", - "ai": "^6.0.116", + "ai": "^7.0.116", "commander": "^12.1.0", "ms": "^2.1.3", "zod": "3.25" diff --git a/packages/hdx-eval/src/__tests__/judge.sdk.test.ts b/packages/hdx-eval/src/__tests__/judge.sdk.test.ts new file mode 100644 index 0000000000..40f4e6d38a --- /dev/null +++ b/packages/hdx-eval/src/__tests__/judge.sdk.test.ts @@ -0,0 +1,72 @@ +/** + * Runs the judge against the real AI SDK with a mock model (judge.test.ts + * mocks generateObject itself), so a broken option binding such as the + * rubric not reaching the model as its system message fails here. + */ +import { MockLanguageModelV4 } from 'ai/test'; + +import { judgeTrajectory } from '@/grading/judge'; +import type { Rubric } from '@/grading/types'; + +const RUBRIC: Rubric = { + programmatic: [], + judge: { + criteria: [ + { id: 'accuracy', weight: 1, description: 'Is the answer correct?' }, + ], + }, +}; + +describe('judgeTrajectory with the real AI SDK', () => { + it('sends the rubric to the model as the system message', async () => { + const model = new MockLanguageModelV4({ + doGenerate: async () => ({ + content: [ + { + type: 'text', + text: JSON.stringify({ + scores: { accuracy: { score: 5, rationale: 'correct' } }, + }), + }, + ], + finishReason: { unified: 'stop', raw: undefined }, + usage: { + inputTokens: { + total: 10, + noCache: 5, + cacheRead: 3, + cacheWrite: 2, + }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, + warnings: [], + }), + }); + + const result = await judgeTrajectory({ + scenarioName: 'test-scenario', + scenarioPrompt: 'Why did it break?', + groundTruth: { rubric: RUBRIC }, + rubric: RUBRIC, + finalAnswer: 'Because the database timed out.', + model, + }); + + expect(result.error).toBeUndefined(); + expect(result.scores.accuracy.score).toBe(5); + // Cost tracking depends on the v7 usage shape (inputTokenDetails). + expect(result.tokens).toEqual({ + input: 10, + output: 5, + cacheCreation: 2, + cacheRead: 3, + }); + + const [system, ...rest] = model.doGenerateCalls[0].prompt; + expect(system.role).toBe('system'); + expect(system.content).toContain( + '- "accuracy" (weight 1): Is the answer correct?', + ); + expect(JSON.stringify(rest)).toContain('Because the database timed out.'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/judge.test.ts b/packages/hdx-eval/src/__tests__/judge.test.ts index bf03e614ef..c4c6b9d1c2 100644 --- a/packages/hdx-eval/src/__tests__/judge.test.ts +++ b/packages/hdx-eval/src/__tests__/judge.test.ts @@ -97,6 +97,39 @@ describe('judgeTrajectory', () => { expect(result.scores.clarity.score).toBe(5); }); + it('sends the rubric as instructions and the answer as the prompt', async () => { + // Checks the option name only; judge.sdk.test.ts verifies the real SDK + // delivers it to the model as the system message. + mockJudgeResult({ + object: { + scores: { + accuracy: { score: 3, rationale: '' }, + clarity: { score: 3, rationale: '' }, + }, + }, + }); + + await judgeTrajectory(baseOpts()); + + const callArg = mockGenerateObject.mock.calls[0][0] as { + instructions?: unknown; + system?: unknown; + prompt?: unknown; + }; + expect(callArg.instructions).toEqual( + expect.stringContaining( + '- "accuracy" (weight 2): Is the answer correct?', + ), + ); + expect(callArg.instructions).toEqual( + expect.stringContaining('SCENARIO: test-scenario'), + ); + expect(callArg.system).toBeUndefined(); + expect(callArg.prompt).toEqual( + expect.stringContaining('Because the database timed out.'), + ); + }); + it('builds a CLOSED schema keyed by rubric criteria (OpenAI strict-mode safe)', async () => { // OpenAI's strict structured-output mode rejects open `z.record` maps and // any object property missing from `required`. The judge must therefore diff --git a/packages/hdx-eval/src/grading/judge.ts b/packages/hdx-eval/src/grading/judge.ts index 43f7faf58b..41399f41f6 100644 --- a/packages/hdx-eval/src/grading/judge.ts +++ b/packages/hdx-eval/src/grading/judge.ts @@ -226,7 +226,7 @@ async function callJudge(args: { model: args.model, schema: args.schema, maxOutputTokens: args.maxOutputTokens, - system: args.systemPrompt, + instructions: args.systemPrompt, prompt: args.userPrompt, }); return { diff --git a/packages/hdx-eval/tsconfig.json b/packages/hdx-eval/tsconfig.json index 716f0be586..db81487879 100644 --- a/packages/hdx-eval/tsconfig.json +++ b/packages/hdx-eval/tsconfig.json @@ -1,7 +1,10 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "module": "node16", + // node20, not node16: CJS output may import ESM-only deps (AI SDK v7) via + // Node's require(esm). There is no node20 moduleResolution; node16 is the + // resolver TypeScript pairs with it. + "module": "node20", "moduleResolution": "node16", "paths": { "@/*": ["./src/*"] diff --git a/yarn.lock b/yarn.lock index 9e30b4f777..35fa310f60 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,75 +19,75 @@ __metadata: languageName: node linkType: hard -"@ai-sdk/anthropic@npm:^3.0.58": - version: 3.0.60 - resolution: "@ai-sdk/anthropic@npm:3.0.60" +"@ai-sdk/anthropic@npm:^4.0.65": + version: 4.0.65 + resolution: "@ai-sdk/anthropic@npm:4.0.65" dependencies: - "@ai-sdk/provider": "npm:3.0.8" - "@ai-sdk/provider-utils": "npm:4.0.20" + "@ai-sdk/provider": "npm:4.0.18" + "@ai-sdk/provider-utils": "npm:5.0.49" peerDependencies: zod: ^3.25.76 || ^4.1.8 - checksum: 10c0/127ccf2f4b8a8bb54ef3d448317190680e90d9f27747b96c23c5e87a5959698a877b0cbf3611c258e79036d1f867fe82687c256a1bd312714ed29e85d2230999 + checksum: 10c0/2cb1de00e180c020aa0a8cfd2f5df85a60be6a555594e9e228eacb74eaad0125a0fa5a0ad33607653e8aa77205b52fc0d7865a33278e696edd431f5a97afb48e languageName: node linkType: hard -"@ai-sdk/gateway@npm:3.0.73": - version: 3.0.73 - resolution: "@ai-sdk/gateway@npm:3.0.73" +"@ai-sdk/gateway@npm:4.0.94": + version: 4.0.94 + resolution: "@ai-sdk/gateway@npm:4.0.94" dependencies: - "@ai-sdk/provider": "npm:3.0.8" - "@ai-sdk/provider-utils": "npm:4.0.20" - "@vercel/oidc": "npm:3.1.0" + "@ai-sdk/provider": "npm:4.0.18" + "@ai-sdk/provider-utils": "npm:5.0.49" + "@vercel/oidc": "npm:3.2.0" peerDependencies: zod: ^3.25.76 || ^4.1.8 - checksum: 10c0/4b7f38dc8860863a1336068b6379111c1ede122f794e11f2ac18cdf47674cf4640ff454c2dbe5d006fe743db297435f79079abf2b871e24a2613483abea5b1f1 + checksum: 10c0/7a729f976e063c580e5a89840a58eea0f947191a96213e1b2fec37bf234cfd138e8ac33bff12ae6eb3921c58d9769d17459af67441f9ae87f06d23b501286536 languageName: node linkType: hard -"@ai-sdk/openai@npm:^3.0.47": - version: 3.0.48 - resolution: "@ai-sdk/openai@npm:3.0.48" +"@ai-sdk/openai@npm:^4.0.78": + version: 4.0.78 + resolution: "@ai-sdk/openai@npm:4.0.78" dependencies: - "@ai-sdk/provider": "npm:3.0.8" - "@ai-sdk/provider-utils": "npm:4.0.21" + "@ai-sdk/provider": "npm:4.0.18" + "@ai-sdk/provider-utils": "npm:5.0.49" peerDependencies: zod: ^3.25.76 || ^4.1.8 - checksum: 10c0/2aaf85fba5ec98e892a41b276a6d2dff9f728f1f149293f437f15daae06a894886730cc9a6d9be39f3f8ba1679d07635c9e9dac5b0013ac3549658a4ffad6638 + checksum: 10c0/1c61a639614e488aa27829f581c27c1712601e5844177c2eee69b59b6b0599bff2089de6ad6e4f8ce60b1b39482451e0947b8c12beb4dd7322dfbc89909ce737 languageName: node linkType: hard -"@ai-sdk/provider-utils@npm:4.0.20": - version: 4.0.20 - resolution: "@ai-sdk/provider-utils@npm:4.0.20" +"@ai-sdk/otel@npm:^1.0.116": + version: 1.0.116 + resolution: "@ai-sdk/otel@npm:1.0.116" dependencies: - "@ai-sdk/provider": "npm:3.0.8" - "@standard-schema/spec": "npm:^1.1.0" - eventsource-parser: "npm:^3.0.6" - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - checksum: 10c0/749d9f91b2b098f716246bbb92a801d0f8a952fed4481e48b7d36a1c026cfe96b9d956f34b95270bc4628048ad57edd7b19026c167e8ded288e741aa289d4cdd + "@ai-sdk/provider": "npm:4.0.18" + "@opentelemetry/api": "npm:1.9.1" + ai: "npm:7.0.116" + checksum: 10c0/284fbef5661fe0d54538357c53e582707e6f6c14267b8e735a142ae053578eee246246753e6c949b301b2d9d52d06e632187321c001f9acd3a06dcf574859f8d languageName: node linkType: hard -"@ai-sdk/provider-utils@npm:4.0.21": - version: 4.0.21 - resolution: "@ai-sdk/provider-utils@npm:4.0.21" +"@ai-sdk/provider-utils@npm:5.0.49": + version: 5.0.49 + resolution: "@ai-sdk/provider-utils@npm:5.0.49" dependencies: - "@ai-sdk/provider": "npm:3.0.8" + "@ai-sdk/provider": "npm:4.0.18" "@standard-schema/spec": "npm:^1.1.0" - eventsource-parser: "npm:^3.0.6" + "@workflow/serde": "npm:4.1.0" + eventsource-parser: "npm:^3.0.8" + undici: "npm:^7.29.0" peerDependencies: zod: ^3.25.76 || ^4.1.8 - checksum: 10c0/70d19cfefad32865f098d0e6e53342408363929d30151eeb072f90c2d7661b24e4a9bd9ac39d75352aeeee8dffeb931c49b85cafcd953c67336c895497c71cfa + checksum: 10c0/c8335d1c0c6b784ca3a757e1868bef9e8170ee27747afbfde5b2a95199f1496334da6f1d0c006006b0b9f20aa82e6db732ae07550ca9cd284586863c0d54cc48 languageName: node linkType: hard -"@ai-sdk/provider@npm:3.0.8": - version: 3.0.8 - resolution: "@ai-sdk/provider@npm:3.0.8" +"@ai-sdk/provider@npm:4.0.18": + version: 4.0.18 + resolution: "@ai-sdk/provider@npm:4.0.18" dependencies: json-schema: "npm:^0.4.0" - checksum: 10c0/c68637c0139a6ce8af17bac1d7d539f531860026237c5c971dcecda2daa8b1e42d8c05e1e664ece60c15edb325c0253fd5b091ee54d32f870a750a493acbb0b7 + checksum: 10c0/bfc6d708afd4e1f22daa8741e7fb517364708c369ec54f0b2aa187291cbc85164198c92e8334f2883123a2440656caef35339242c4b01f2a62faecd16e002d82 languageName: node linkType: hard @@ -4620,8 +4620,9 @@ __metadata: version: 0.0.0-use.local resolution: "@hyperdx/api@workspace:packages/api" dependencies: - "@ai-sdk/anthropic": "npm:^3.0.58" - "@ai-sdk/openai": "npm:^3.0.47" + "@ai-sdk/anthropic": "npm:^4.0.65" + "@ai-sdk/openai": "npm:^4.0.78" + "@ai-sdk/otel": "npm:^1.0.116" "@braintree/sanitize-url": "npm:^7.1.1" "@clickhouse/client-common": "npm:1.23.0-head.fae5998.1" "@esm2cjs/p-queue": "npm:^7.3.0" @@ -4632,6 +4633,7 @@ __metadata: "@opentelemetry/api": "npm:^1.8.0" "@opentelemetry/host-metrics": "npm:^0.35.5" "@opentelemetry/sdk-metrics": "npm:^1.30.1" + "@opentelemetry/sdk-trace-base": "npm:^1.30.1" "@slack/webhook": "npm:^7.0.0" "@stoplight/spectral-cli": "npm:^6.15.0" "@types/compression": "npm:^1.7.3" @@ -4648,7 +4650,7 @@ __metadata: "@types/supertest": "npm:^2.0.12" "@types/swagger-jsdoc": "npm:^6" "@types/uuid": "npm:^8.3.4" - ai: "npm:^6.0.116" + ai: "npm:^7.0.116" ajv: "npm:^8.20.0" aws4: "npm:^1.13.2" chrono-node: "npm:^2.9.0" @@ -4910,14 +4912,14 @@ __metadata: version: 0.0.0-use.local resolution: "@hyperdx/hdx-eval@workspace:packages/hdx-eval" dependencies: - "@ai-sdk/anthropic": "npm:^3.0.58" - "@ai-sdk/openai": "npm:^3.0.47" + "@ai-sdk/anthropic": "npm:^4.0.65" + "@ai-sdk/openai": "npm:^4.0.78" "@clickhouse/client": "npm:1.23.0-head.fae5998.1" "@dotenvx/dotenvx": "npm:^1.51.1" "@types/jest": "npm:^29.5.14" "@types/ms": "npm:^0.7.31" "@types/node": "npm:^22.15.18" - ai: "npm:^6.0.116" + ai: "npm:^7.0.116" commander: "npm:^12.1.0" jest: "npm:^30.2.0" ms: "npm:^2.1.3" @@ -6838,10 +6840,10 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:1.9.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.8.0": - version: 1.9.0 - resolution: "@opentelemetry/api@npm:1.9.0" - checksum: 10c0/9aae2fe6e8a3a3eeb6c1fdef78e1939cf05a0f37f8a4fae4d6bf2e09eb1e06f966ece85805626e01ba5fab48072b94f19b835449e58b6d26720ee19a58298add +"@opentelemetry/api@npm:1.9.1": + version: 1.9.1 + resolution: "@opentelemetry/api@npm:1.9.1" + checksum: 10c0/c608485fc8b5a91e1f7e05e843b45b509307456b31cd2ad365933d90813e40ebfedf179f1451c762037e82d7c76aa8500e95d2da3609f640a1206cde5322cd14 languageName: node linkType: hard @@ -6852,6 +6854,13 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.8.0": + version: 1.9.0 + resolution: "@opentelemetry/api@npm:1.9.0" + checksum: 10c0/9aae2fe6e8a3a3eeb6c1fdef78e1939cf05a0f37f8a4fae4d6bf2e09eb1e06f966ece85805626e01ba5fab48072b94f19b835449e58b6d26720ee19a58298add + languageName: node + linkType: hard + "@opentelemetry/auto-instrumentations-node@npm:^0.56.1": version: 0.56.1 resolution: "@opentelemetry/auto-instrumentations-node@npm:0.56.1" @@ -11498,10 +11507,10 @@ __metadata: languageName: node linkType: hard -"@vercel/oidc@npm:3.1.0": - version: 3.1.0 - resolution: "@vercel/oidc@npm:3.1.0" - checksum: 10c0/f57278ed4b4c022c7ca85e8baa5f9bdb2623397abfa0e5dbfd75de283c8e5dc534d64dac1364b5ad8c96d00eb2d469886e6f7b640f6f195def5766950ad8ce71 +"@vercel/oidc@npm:3.2.0": + version: 3.2.0 + resolution: "@vercel/oidc@npm:3.2.0" + checksum: 10c0/98318d3236f58c296616c8c2e1655b268c7bf58525bcd985adac7af6d900e05fc610f6f03ce2ff4bdcd3df7885a40c0ca44fdc761f122dcfe15a78c2756b0243 languageName: node linkType: hard @@ -11717,6 +11726,13 @@ __metadata: languageName: node linkType: hard +"@workflow/serde@npm:4.1.0": + version: 4.1.0 + resolution: "@workflow/serde@npm:4.1.0" + checksum: 10c0/8ec68cd9448b486bc8244d1f2b33d64d4af4b0fc18d33417699ce29345b21e4359d880c5ea5d66872def4576c0cddcfa2a63282d1f4d661c455663c075595e81 + languageName: node + linkType: hard + "@xstate/fsm@npm:^1.4.0": version: 1.6.5 resolution: "@xstate/fsm@npm:1.6.5" @@ -11942,17 +11958,16 @@ __metadata: languageName: node linkType: hard -"ai@npm:^6.0.116": - version: 6.0.127 - resolution: "ai@npm:6.0.127" +"ai@npm:7.0.116, ai@npm:^7.0.116": + version: 7.0.116 + resolution: "ai@npm:7.0.116" dependencies: - "@ai-sdk/gateway": "npm:3.0.73" - "@ai-sdk/provider": "npm:3.0.8" - "@ai-sdk/provider-utils": "npm:4.0.20" - "@opentelemetry/api": "npm:1.9.0" + "@ai-sdk/gateway": "npm:4.0.94" + "@ai-sdk/provider": "npm:4.0.18" + "@ai-sdk/provider-utils": "npm:5.0.49" peerDependencies: zod: ^3.25.76 || ^4.1.8 - checksum: 10c0/8334afb0c62f6aaadf9a124aecea9a4b86363b1ecba1f6827091ee30c83dc0ec50ddeca6b0869eda3da03431593a2aebe55a463e71703af5f96e1a9049b45062 + checksum: 10c0/a53e89e6f70a6458da1bae93593e5d1b5e93a0067607057d37e30ca120389adfb959fcb412492a17a6a2567cbfcfc678ba2c8672eb0524ae03125be0e37677f5 languageName: node linkType: hard @@ -16746,13 +16761,20 @@ __metadata: languageName: node linkType: hard -"eventsource-parser@npm:^3.0.0, eventsource-parser@npm:^3.0.1, eventsource-parser@npm:^3.0.6": +"eventsource-parser@npm:^3.0.0, eventsource-parser@npm:^3.0.1": version: 3.0.6 resolution: "eventsource-parser@npm:3.0.6" checksum: 10c0/70b8ccec7dac767ef2eca43f355e0979e70415701691382a042a2df8d6a68da6c2fca35363669821f3da876d29c02abe9b232964637c1b6635c940df05ada78a languageName: node linkType: hard +"eventsource-parser@npm:^3.0.8": + version: 3.1.1 + resolution: "eventsource-parser@npm:3.1.1" + checksum: 10c0/dd3236c61140587253dcaaf947de528ac0e7371caffdc53c77afaeee36a17661f00e251a6ea16af56c99be5ac138303e67b5d750630e7c41b02e3564e8ad8c44 + languageName: node + linkType: hard + "eventsource@npm:^3.0.2": version: 3.0.7 resolution: "eventsource@npm:3.0.7" @@ -17467,8 +17489,8 @@ __metadata: asynckit: "npm:^0.4.0" combined-stream: "npm:^1.0.8" es-set-tostringtag: "npm:^2.1.0" - hasown: "npm:^2.0.4" - mime-types: "npm:^2.1.35" + hasown: "npm:^2.0.2" + mime-types: "npm:^2.1.12" checksum: 10c0/43947a77bf0ff45c6ceed789778982d47a3f3e720a74b71721174ebf3310a5f1a8be1d6b38a3ee3688e8a18a2c4273073ec0844cd37efda3eaf46d41c9c318ff languageName: node linkType: hard @@ -18270,7 +18292,7 @@ __metadata: languageName: node linkType: hard -"hasown@npm:2.0.4, hasown@npm:^2.0.4": +"hasown@npm:2.0.4": version: 2.0.4 resolution: "hasown@npm:2.0.4" dependencies: @@ -22230,7 +22252,7 @@ __metadata: languageName: node linkType: hard -"mime-types@npm:2.1.35, mime-types@npm:^2.1.27, mime-types@npm:^2.1.31, mime-types@npm:^2.1.35, mime-types@npm:~2.1.24, mime-types@npm:~2.1.34": +"mime-types@npm:2.1.35, mime-types@npm:^2.1.12, mime-types@npm:^2.1.27, mime-types@npm:^2.1.31, mime-types@npm:~2.1.24, mime-types@npm:~2.1.34": version: 2.1.35 resolution: "mime-types@npm:2.1.35" dependencies: @@ -29290,6 +29312,13 @@ __metadata: languageName: node linkType: hard +"undici@npm:^7.29.0": + version: 7.30.0 + resolution: "undici@npm:7.30.0" + checksum: 10c0/011a321a9198532b0f0504740faa43b9ab8fcc0e4f60104e5c9cb808a61faeb03e4be4aec4f0c65f47913d7c7d9d91591793a6cb9ab843f98ca2c3d711c4a34c + languageName: node + linkType: hard + "unicode-canonical-property-names-ecmascript@npm:^2.0.0": version: 2.0.0 resolution: "unicode-canonical-property-names-ecmascript@npm:2.0.0" From f86235fff2862c923fb62f181b5660af16a96f75 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 1 Oct 2026 13:15:35 -0600 Subject: [PATCH 2/2] chore: refresh yarn.lock form-data 4.0.6 dependency ranges --- yarn.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index 35fa310f60..577364a13a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17489,8 +17489,8 @@ __metadata: asynckit: "npm:^0.4.0" combined-stream: "npm:^1.0.8" es-set-tostringtag: "npm:^2.1.0" - hasown: "npm:^2.0.2" - mime-types: "npm:^2.1.12" + hasown: "npm:^2.0.4" + mime-types: "npm:^2.1.35" checksum: 10c0/43947a77bf0ff45c6ceed789778982d47a3f3e720a74b71721174ebf3310a5f1a8be1d6b38a3ee3688e8a18a2c4273073ec0844cd37efda3eaf46d41c9c318ff languageName: node linkType: hard @@ -18292,7 +18292,7 @@ __metadata: languageName: node linkType: hard -"hasown@npm:2.0.4": +"hasown@npm:2.0.4, hasown@npm:^2.0.4": version: 2.0.4 resolution: "hasown@npm:2.0.4" dependencies: @@ -22252,7 +22252,7 @@ __metadata: languageName: node linkType: hard -"mime-types@npm:2.1.35, mime-types@npm:^2.1.12, mime-types@npm:^2.1.27, mime-types@npm:^2.1.31, mime-types@npm:~2.1.24, mime-types@npm:~2.1.34": +"mime-types@npm:2.1.35, mime-types@npm:^2.1.27, mime-types@npm:^2.1.31, mime-types@npm:^2.1.35, mime-types@npm:~2.1.24, mime-types@npm:~2.1.34": version: 2.1.35 resolution: "mime-types@npm:2.1.35" dependencies: