From 08cbb42e879818044e760e573440940192546916 Mon Sep 17 00:00:00 2001 From: sebi Date: Sun, 11 Oct 2026 22:13:28 -0500 Subject: [PATCH 1/3] fix(ai): no catalogue read before the gateway is asked; a lazily registered catalogue answers its first call 27.10.0 set `llm.sampling.ignored` at the top of agent() and llm() through ignoredSampling -> modelSpec, before the gateway was consulted and whether or not a knob was asked. An app whose gateway registers the model rows when it is first built had every first call refused X_AI_MODEL_UNKNOWN by a span attribute. The attribute is now set when the run ends; a call asking no knob reads no row; an unregistered model leaves the attribute out. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- packages/ai/src/agent.ts | 42 ++++++----- packages/ai/src/lazy-catalogue.test.ts | 100 +++++++++++++++++++++++++ packages/ai/src/llm.ts | 26 ++++--- packages/ai/src/sampling.ts | 23 +++++- 5 files changed, 167 insertions(+), 30 deletions(-) create mode 100644 packages/ai/src/lazy-catalogue.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ece903394..6d7642860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,11 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major ## [Unreleased] -Nothing yet. +### Fixed + +Tier 4 — ai. + +- **An app that registers its model catalogue when its gateway is first used is no longer refused `X_AI_MODEL_UNKNOWN` on its first call.** 27.10.0 read the model's row at the top of `agent()` and `llm()` to set the `llm.sampling.ignored` span attribute (`ignoredSampling` → `modelSpec`), before the gateway was asked anything, and whether or not the declaration named a `temperature` or `seed`. A gateway built lazily from the env, which registers the rows as it builds, had not run yet, so every first call in a process failed on a span attribute. Until 27.9.0 the gateway's own call was the first read of the catalogue, and it is again: the attribute is set when the run ends (answered or failed), a call that asks for no knob reads no row for it, and a model still unregistered by then leaves the attribute out instead of throwing over the call's own failure. ## 27.10.0 - 2026-10-12 diff --git a/packages/ai/src/agent.ts b/packages/ai/src/agent.ts index a9ad430f1..cef59b9ce 100644 --- a/packages/ai/src/agent.ts +++ b/packages/ai/src/agent.ts @@ -60,7 +60,7 @@ import { RESPOND, respondFor } from './respond'; import { runMeter } from './run-meter'; import { aiGateway, aiRedactor, installedGateway } from './runtime'; import type { Sampling } from './sampling'; -import { ignoredSampling, screenSampling } from './sampling'; +import { ignoredSamplingAttribute, screenSampling } from './sampling'; import type { AgentTool, LlmTool, LlmToolResult, ProjectableAction } from './tools'; import { asProjectableAction, isOfferedTool, runLlmToolCall, toLlmTools, toolLabel } from './tools'; @@ -196,12 +196,15 @@ async function run< 'agent.max_turns': maxTurns, 'llm.redacted': rendered !== rawPrompt || !sameSystem(system, rawSystem), 'llm.prompt_cache': promptCache !== undefined, - 'llm.sampling.ignored': ignoredSampling( - model, - bounds.sampling, - installedGateway()?.samplingWires?.(model), - ).join(','), }); + // Read when the run ends, not here: the gateway may be what registers the model's row. + const reportSampling = (): void => { + span.setAttributes( + ignoredSamplingAttribute(model, bounds.sampling, () => + installedGateway()?.samplingWires?.(model), + ), + ); + }; // Rooted in the GATEWAY's own ceilings when no scope is open — an empty root ignored them. const gateway = aiGateway(name); @@ -371,18 +374,21 @@ async function run< throw new AgentMaxTurnsError({ agent: name, turns: maxTurns, calls }); }; return withBudget(ledger, () => - meter.within(loop).catch((error: unknown) => { - const failed = meter.fail(error); - // A failed `.run()` / `.continue()` leaves a handle too: the transcript up to the request - // the failed turn was answering, with `turns` and `usage` counting what this call was - // billed. Without it a retry resumes from the OLD handle, which never heard of that spend - // — six truncated continuations billed six times against a ceiling that saw none of them. - if (session !== undefined && at.messages.length > 0) { - const { usage } = runTotal(model, prior?.usage, meter.billed()); - withTranscript(failed, sealTranscript(binding, { ...at, usage })); - } - throw failed; - }), + meter + .within(loop) + .catch((error: unknown) => { + const failed = meter.fail(error); + // A failed `.run()` / `.continue()` leaves a handle too: the transcript up to the request + // the failed turn was answering, with `turns` and `usage` counting what this call was + // billed. Without it a retry resumes from the OLD handle, which never heard of that spend + // — six truncated continuations billed six times against a ceiling that saw none of them. + if (session !== undefined && at.messages.length > 0) { + const { usage } = runTotal(model, prior?.usage, meter.billed()); + withTranscript(failed, sealTranscript(binding, { ...at, usage })); + } + throw failed; + }) + .finally(reportSampling), ); }); } diff --git a/packages/ai/src/lazy-catalogue.test.ts b/packages/ai/src/lazy-catalogue.test.ts new file mode 100644 index 000000000..398ff6847 --- /dev/null +++ b/packages/ai/src/lazy-catalogue.test.ts @@ -0,0 +1,100 @@ +/** + * A catalogue the app registers when its GATEWAY is first used. The failure first: 27.10 read the + * model's row at the top of `agent()` and `llm()` — to report `llm.sampling.ignored` on the span — + * before the gateway was asked anything, so an app whose gateway is built (and registers its + * models) on first use had every first call refused `X_AI_MODEL_UNKNOWN`, sampling asked or not. + * Until 27.9 the first read of the catalogue was the gateway's own. A span attribute is a report: + * it is read once the gateway has been consulted, and never fails the call it describes. + */ + +import { beforeEach, describe, expect, test } from 'bun:test'; +import { anonymousCtx } from '@ultimat3/action'; +import type { ReadableSpan } from '@ultimat3/core'; +import { configureTelemetry, memoryExporter, resetTelemetry } from '@ultimat3/core'; +import { allow } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import { agent } from './agent'; +import type { Gateway } from './gateway'; +import { providerGateway } from './gateway'; +import { ANSWER, declare, POST_ID, promptFor, stub } from './llm-fixture'; +import { FIXTURE_MODEL, registerFixtureModels } from './model-fixture'; +import { resetModels } from './models'; +import type { Provider } from './provider'; +import { configureAi, resetAiRuntime } from './runtime'; + +const spans = memoryExporter(); + +beforeEach(() => { + resetAiRuntime(); + resetModels(); + resetTelemetry(); + spans.reset(); + configureTelemetry({ exporter: spans }); +}); + +/** The app's shape: every method builds the real gateway on first use, and the build registers. */ +function lazyGateway(provider: Provider): Gateway { + let built: Gateway | undefined; + const real = (): Gateway => { + if (built === undefined) { + registerFixtureModels(); + built = providerGateway({ providers: [provider] }); + } + return built; + }; + return { + defaultModel: undefined, + generate: (request) => real().generate(request), + stream: (request) => real().stream(request), + scope: (input, run) => real().scope(input, run), + spent: () => real().spent(), + callLedger: (keys) => real().callLedger(keys), + }; +} + +const attribute = (name: string): unknown => + spans.spans.find((span: ReadableSpan) => span.name === name)?.attributes['llm.sampling.ignored']; + +function lazyAgent(sampling: { temperature?: number; seed?: number } = {}) { + return agent({ + input: t.object({ postId: t.uuid }), + output: t.object({ answer: t.string }), + prompt: promptFor(), + model: FIXTURE_MODEL, + vars: ({ input }) => ({ postId: input.postId }), + tools: [], + policy: allow(), + ...sampling, + }); +} + +describe('a catalogue registered by the gateway on first use', () => { + test('agent(): the first call answers', async () => { + configureAi({ gateway: lazyGateway(stub({ answer: 'done' }).provider) }); + const run = lazyAgent().named('lazyCatalogueAgent'); + expect(await run({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual({ answer: 'done' }); + expect(attribute('ai.agent')).toBe(''); + }); + + test('llm(): the first call answers', async () => { + configureAi({ gateway: lazyGateway(stub(ANSWER).provider) }); + const call = declare(promptFor(), { model: FIXTURE_MODEL }); + expect(await call({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual(ANSWER); + expect(attribute('ai.llm')).toBe(''); + }); + + test('a knob the row does not take is still reported, once the row is there', async () => { + configureAi({ gateway: lazyGateway(stub({ answer: 'done' }).provider) }); + // The fixture rows declare no `sampling`, so both knobs are left out of the request. + const run = lazyAgent({ temperature: 0, seed: 7 }).named('lazyCatalogueSampledAgent'); + expect(await run({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual({ answer: 'done' }); + expect(attribute('ai.agent')).toBe('temperature,seed'); + }); + + test('llm() with knobs: answered, and the span names what was left out', async () => { + configureAi({ gateway: lazyGateway(stub(ANSWER).provider) }); + const call = declare(promptFor(), { model: FIXTURE_MODEL, temperature: 0 }); + expect(await call({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual(ANSWER); + expect(attribute('ai.llm')).toBe('temperature'); + }); +}); diff --git a/packages/ai/src/llm.ts b/packages/ai/src/llm.ts index 0a78120fd..ef6d28aa1 100644 --- a/packages/ai/src/llm.ts +++ b/packages/ai/src/llm.ts @@ -50,7 +50,7 @@ import type { RunMeter } from './run-meter'; import { runMeter } from './run-meter'; import { aiGateway, aiRedactor, installedGateway } from './runtime'; import type { Sampling } from './sampling'; -import { ignoredSampling, screenSampling } from './sampling'; +import { ignoredSamplingAttribute, screenSampling } from './sampling'; /** Two attempts total: the answer, then one repair turn. See `LlmOutputInvalidError`. */ const ATTEMPTS = 2; @@ -182,12 +182,13 @@ async function generate< // an audit asks evidence for, and a redactor that silently stopped matching looks identical // to one that had nothing to remove until this attribute separates them. 'llm.redacted': redacted, - 'llm.sampling.ignored': ignoredSampling( - model, - sampling, - installedGateway()?.samplingWires?.(model), - ).join(','), }); + // Read when the call ends, not here: the gateway may be what registers the model's row. + const reportSampling = (): void => { + span.setAttributes( + ignoredSamplingAttribute(model, sampling, () => installedGateway()?.samplingWires?.(model)), + ); + }; // A cached answer is still data of unknown provenance, so it goes through the schema like // any other. One that no longer fits — the schema moved under it — is a miss, not a @@ -201,7 +202,10 @@ async function generate< }); const hit = await accept(def.output, await cache?.lookup()); span.setAttribute('llm.cache.hit', hit !== undefined); - if (hit !== undefined) return hit.value; + if (hit !== undefined) { + reportSampling(); + return hit.value; + } // No `tools` yet: the `respond` projection belongs to the non-streaming path alone. A tool // call is emitted whole, so forcing the answer through one leaves a stream with nothing to @@ -292,9 +296,11 @@ async function generate< }); }; return withBudget(ledger, () => - answer().catch((error: unknown) => { - throw meter.fail(error); - }), + answer() + .catch((error: unknown) => { + throw meter.fail(error); + }) + .finally(reportSampling), ); }); } diff --git a/packages/ai/src/sampling.ts b/packages/ai/src/sampling.ts index 5f2628dd2..11191e524 100644 --- a/packages/ai/src/sampling.ts +++ b/packages/ai/src/sampling.ts @@ -11,7 +11,7 @@ import { AiRequestInvalidError } from './errors'; import type { ModelId } from './models'; -import { modelSpec } from './models'; +import { isModelRegistered, modelSpec } from './models'; export interface Sampling { /** 0 to 2. `0` is the most repeatable a model offers; it is not a guarantee of determinism. */ @@ -74,6 +74,8 @@ export function ignoredSampling( /** The wire formats of the providers serving `model`, when the gateway can say. */ wires?: readonly SamplingWire[], ): readonly string[] { + // Nothing asked, nothing ignored — and no row read: a model none asked of needs none here. + if (request.temperature === undefined && request.seed === undefined) return []; const accepts = modelSpec(model).sampling; // A knob no serving wire has a word for is dropped too — `seed` on the Messages API. const sendable = (knob: 'temperature' | 'seed'): boolean => @@ -84,3 +86,22 @@ export function ignoredSampling( (knob) => request[knob] !== undefined && (accepts?.[knob] !== true || !sendable(knob)), ); } + +/** + * The `llm.sampling.ignored` span attribute, set when a run ENDS — answered or failed — never at + * its start. The catalogue is the app's, and an app may register it when its gateway is first used + * (a gateway built lazily from the env): read at the top of `agent()` / `llm()`, before the gateway + * was asked anything, the row was not there yet and a REPORT refused the call `X_AI_MODEL_UNKNOWN`. + * A model still unregistered by then has no row to report against; the call's own failure says so. + */ +export function ignoredSamplingAttribute( + model: ModelId, + request: Sampling, + wires: () => readonly SamplingWire[] | undefined, +): Record<'llm.sampling.ignored', string> | Record { + if (request.temperature === undefined && request.seed === undefined) { + return { 'llm.sampling.ignored': '' }; + } + if (!isModelRegistered(model)) return {}; + return { 'llm.sampling.ignored': ignoredSampling(model, request, wires()).join(',') }; +} From 2325ec3b90aedcb829dd26af57cdb7bfd117336d Mon Sep 17 00:00:00 2001 From: sebi Date: Sun, 11 Oct 2026 22:32:44 -0500 Subject: [PATCH 2/3] fix(ai): the sampling report cannot replace the call's result; failed and unlisted-model runs tested Co-Authored-By: Claude Opus 5.5 --- packages/ai/src/lazy-catalogue.test.ts | 43 ++++++++++++++++++++++++++ packages/ai/src/sampling.ts | 8 ++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/lazy-catalogue.test.ts b/packages/ai/src/lazy-catalogue.test.ts index 398ff6847..1dfc8aaa7 100644 --- a/packages/ai/src/lazy-catalogue.test.ts +++ b/packages/ai/src/lazy-catalogue.test.ts @@ -98,3 +98,46 @@ describe('a catalogue registered by the gateway on first use', () => { expect(attribute('ai.llm')).toBe('temperature'); }); }); + +describe('the report never changes the call it describes', () => { + test('a gateway whose samplingWires throws: the answer stands, the attribute is left out', async () => { + const inner = lazyGateway(stub({ answer: 'done' }).provider); + const throwing = { + ...inner, + samplingWires: (): never => { + throw new Error('samplingWires broke'); + }, + } as Gateway; + configureAi({ gateway: throwing }); + const run = lazyAgent({ temperature: 0 }).named('lazyCatalogueThrowingWiresAgent'); + expect(await run({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual({ answer: 'done' }); + expect(attribute('ai.agent')).toBeUndefined(); + }); + + test('a failed run still reports what was left out', async () => { + configureAi({ gateway: lazyGateway(stub({ wrong: 1 }).provider) }); + const run = lazyAgent({ seed: 7 }).named('lazyCatalogueFailingAgent'); + await expect(run({ postId: POST_ID }, { ctx: anonymousCtx() })).rejects.toBeDefined(); + expect(attribute('ai.agent')).toBe('seed'); + }); + + test('a model the catalogue never names: the call keeps its own error, no attribute', async () => { + configureAi({ gateway: lazyGateway(stub({ answer: 'done' }).provider) }); + const run = agent({ + input: t.object({ postId: t.uuid }), + output: t.object({ answer: t.string }), + prompt: promptFor(), + model: 'nobody/unlisted-model', + vars: ({ input }) => ({ postId: input.postId }), + tools: [], + policy: allow(), + temperature: 0, + }).named('lazyCatalogueUnlistedAgent'); + const error = await run({ postId: POST_ID }, { ctx: anonymousCtx() }).then( + () => undefined, + (thrown: unknown) => thrown, + ); + expect(error).toBeDefined(); + expect(attribute('ai.agent')).toBeUndefined(); + }); +}); diff --git a/packages/ai/src/sampling.ts b/packages/ai/src/sampling.ts index 11191e524..8f10ca838 100644 --- a/packages/ai/src/sampling.ts +++ b/packages/ai/src/sampling.ts @@ -103,5 +103,11 @@ export function ignoredSamplingAttribute( return { 'llm.sampling.ignored': '' }; } if (!isModelRegistered(model)) return {}; - return { 'llm.sampling.ignored': ignoredSampling(model, request, wires()).join(',') }; + // A report never fails the call it describes: an installed gateway may throw from + // `samplingWires`, and the attribute is then left out rather than replacing the call's result. + try { + return { 'llm.sampling.ignored': ignoredSampling(model, request, wires()).join(',') }; + } catch { + return {}; + } } From d271f63647587569aaaef74cc6f62ba9ff1163e2 Mon Sep 17 00:00:00 2001 From: sebi Date: Sun, 11 Oct 2026 22:42:50 -0500 Subject: [PATCH 3/3] test(ai): the lazy-catalogue tests throw coded errors and prove the call's own error survives by identity Co-Authored-By: Claude Opus 5.5 --- packages/ai/src/lazy-catalogue.test.ts | 31 ++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/ai/src/lazy-catalogue.test.ts b/packages/ai/src/lazy-catalogue.test.ts index 1dfc8aaa7..cdb1a9289 100644 --- a/packages/ai/src/lazy-catalogue.test.ts +++ b/packages/ai/src/lazy-catalogue.test.ts @@ -10,7 +10,7 @@ import { beforeEach, describe, expect, test } from 'bun:test'; import { anonymousCtx } from '@ultimat3/action'; import type { ReadableSpan } from '@ultimat3/core'; -import { configureTelemetry, memoryExporter, resetTelemetry } from '@ultimat3/core'; +import { configureTelemetry, memoryExporter, resetTelemetry, UltimateError } from '@ultimat3/core'; import { allow } from '@ultimat3/policy'; import { t } from '@ultimat3/schema'; import { agent } from './agent'; @@ -105,7 +105,11 @@ describe('the report never changes the call it describes', () => { const throwing = { ...inner, samplingWires: (): never => { - throw new Error('samplingWires broke'); + throw new UltimateError({ + code: 'X_TEST_SAMPLING_WIRES', + cause: 'samplingWires broke', + fix: 'nothing: the test throws on purpose', + }); }, } as Gateway; configureAi({ gateway: throwing }); @@ -122,7 +126,26 @@ describe('the report never changes the call it describes', () => { }); test('a model the catalogue never names: the call keeps its own error, no attribute', async () => { - configureAi({ gateway: lazyGateway(stub({ answer: 'done' }).provider) }); + // The call's own error is the gateway's: identity proves the report did not replace it + // (27.10.0 read the row first and threw X_AI_MODEL_UNKNOWN before the gateway was asked). + const own = new UltimateError({ + code: 'X_TEST_GATEWAY_REFUSED', + cause: 'the gateway refused the call', + fix: 'nothing: the test refuses on purpose', + }); + const refuse = (): never => { + throw own; + }; + configureAi({ + gateway: { + defaultModel: undefined, + generate: refuse, + stream: refuse, + scope: refuse, + spent: refuse, + callLedger: refuse, + } as unknown as Gateway, + }); const run = agent({ input: t.object({ postId: t.uuid }), output: t.object({ answer: t.string }), @@ -137,7 +160,7 @@ describe('the report never changes the call it describes', () => { () => undefined, (thrown: unknown) => thrown, ); - expect(error).toBeDefined(); + expect(error).toBe(own); expect(attribute('ai.agent')).toBeUndefined(); }); });