fix(ai): reporting ignored sampling never reads the catalogue before the gateway is asked

CHANGELOG.md

@@ -8,7 +8,11 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major 
 
 ## [Unreleased]
 
-Nothing yet.
+### Fixed
+
+Tier 4 — ai.
+
+- **An app that registers its model catalogue when its gateway is first used is no longer refused `X_AI_MODEL_UNKNOWN` on its first call.** 27.10.0 read the model's row at the top of `agent()` and `llm()` to set the `llm.sampling.ignored` span attribute (`ignoredSampling` → `modelSpec`), before the gateway was asked anything, and whether or not the declaration named a `temperature` or `seed`. A gateway built lazily from the env, which registers the rows as it builds, had not run yet, so every first call in a process failed on a span attribute. Until 27.9.0 the gateway's own call was the first read of the catalogue, and it is again: the attribute is set when the run ends (answered or failed), a call that asks for no knob reads no row for it, and a model still unregistered by then leaves the attribute out instead of throwing over the call's own failure.
 
 ## 27.10.0 - 2026-10-12
 

packages/ai/src/agent.ts

@@ -60,7 +60,7 @@ import { RESPOND, respondFor } from './respond';
 import { runMeter } from './run-meter';
 import { aiGateway, aiRedactor, installedGateway } from './runtime';
 import type { Sampling } from './sampling';
-import { ignoredSampling, screenSampling } from './sampling';
+import { ignoredSamplingAttribute, screenSampling } from './sampling';
 import type { AgentTool, LlmTool, LlmToolResult, ProjectableAction } from './tools';
 import { asProjectableAction, isOfferedTool, runLlmToolCall, toLlmTools, toolLabel } from './tools';
 
@@ -196,12 +196,15 @@ async function run<
       'agent.max_turns': maxTurns,
       'llm.redacted': rendered !== rawPrompt || !sameSystem(system, rawSystem),
       'llm.prompt_cache': promptCache !== undefined,
-      'llm.sampling.ignored': ignoredSampling(
-        model,
-        bounds.sampling,
-        installedGateway()?.samplingWires?.(model),
-      ).join(','),
     });
+    // Read when the run ends, not here: the gateway may be what registers the model's row.
+    const reportSampling = (): void => {
+      span.setAttributes(
+        ignoredSamplingAttribute(model, bounds.sampling, () =>
+          installedGateway()?.samplingWires?.(model),
+        ),
+      );
+    };
 
     // Rooted in the GATEWAY's own ceilings when no scope is open — an empty root ignored them.
     const gateway = aiGateway(name);
@@ -371,18 +374,21 @@ async function run<
       throw new AgentMaxTurnsError({ agent: name, turns: maxTurns, calls });
     };
     return withBudget(ledger, () =>
-      meter.within(loop).catch((error: unknown) => {
-        const failed = meter.fail(error);
-        // A failed `.run()` / `.continue()` leaves a handle too: the transcript up to the request
-        // the failed turn was answering, with `turns` and `usage` counting what this call was
-        // billed. Without it a retry resumes from the OLD handle, which never heard of that spend
-        // — six truncated continuations billed six times against a ceiling that saw none of them.
-        if (session !== undefined && at.messages.length > 0) {
-          const { usage } = runTotal(model, prior?.usage, meter.billed());
-          withTranscript(failed, sealTranscript(binding, { ...at, usage }));
-        }
-        throw failed;
-      }),
+      meter
+        .within(loop)
+        .catch((error: unknown) => {
+          const failed = meter.fail(error);
+          // A failed `.run()` / `.continue()` leaves a handle too: the transcript up to the request
+          // the failed turn was answering, with `turns` and `usage` counting what this call was
+          // billed. Without it a retry resumes from the OLD handle, which never heard of that spend
+          // — six truncated continuations billed six times against a ceiling that saw none of them.
+          if (session !== undefined && at.messages.length > 0) {
+            const { usage } = runTotal(model, prior?.usage, meter.billed());
+            withTranscript(failed, sealTranscript(binding, { ...at, usage }));
+          }
+          throw failed;
+        })
+        .finally(reportSampling),
     );
   });
 }

packages/ai/src/lazy-catalogue.test.ts

@@ -0,0 +1,166 @@
+/**
+ * A catalogue the app registers when its GATEWAY is first used. The failure first: 27.10 read the
+ * model's row at the top of `agent()` and `llm()` — to report `llm.sampling.ignored` on the span —
+ * before the gateway was asked anything, so an app whose gateway is built (and registers its
+ * models) on first use had every first call refused `X_AI_MODEL_UNKNOWN`, sampling asked or not.
+ * Until 27.9 the first read of the catalogue was the gateway's own. A span attribute is a report:
+ * it is read once the gateway has been consulted, and never fails the call it describes.
+ */
+
+import { beforeEach, describe, expect, test } from 'bun:test';
+import { anonymousCtx } from '@ultimat3/action';
+import type { ReadableSpan } from '@ultimat3/core';
+import { configureTelemetry, memoryExporter, resetTelemetry, UltimateError } from '@ultimat3/core';
+import { allow } from '@ultimat3/policy';
+import { t } from '@ultimat3/schema';
+import { agent } from './agent';
+import type { Gateway } from './gateway';
+import { providerGateway } from './gateway';
+import { ANSWER, declare, POST_ID, promptFor, stub } from './llm-fixture';
+import { FIXTURE_MODEL, registerFixtureModels } from './model-fixture';
+import { resetModels } from './models';
+import type { Provider } from './provider';
+import { configureAi, resetAiRuntime } from './runtime';
+
+const spans = memoryExporter();
+
+beforeEach(() => {
+  resetAiRuntime();
+  resetModels();
+  resetTelemetry();
+  spans.reset();
+  configureTelemetry({ exporter: spans });
+});
+
+/** The app's shape: every method builds the real gateway on first use, and the build registers. */
+function lazyGateway(provider: Provider): Gateway {
+  let built: Gateway | undefined;
+  const real = (): Gateway => {
+    if (built === undefined) {
+      registerFixtureModels();
+      built = providerGateway({ providers: [provider] });
+    }
+    return built;
+  };
+  return {
+    defaultModel: undefined,
+    generate: (request) => real().generate(request),
+    stream: (request) => real().stream(request),
+    scope: (input, run) => real().scope(input, run),
+    spent: () => real().spent(),
+    callLedger: (keys) => real().callLedger(keys),
+  };
+}
+
+const attribute = (name: string): unknown =>
+  spans.spans.find((span: ReadableSpan) => span.name === name)?.attributes['llm.sampling.ignored'];
+
+function lazyAgent(sampling: { temperature?: number; seed?: number } = {}) {
+  return agent({
+    input: t.object({ postId: t.uuid }),
+    output: t.object({ answer: t.string }),
+    prompt: promptFor(),
+    model: FIXTURE_MODEL,
+    vars: ({ input }) => ({ postId: input.postId }),
+    tools: [],
+    policy: allow(),
+    ...sampling,
+  });
+}
+
+describe('a catalogue registered by the gateway on first use', () => {
+  test('agent(): the first call answers', async () => {
+    configureAi({ gateway: lazyGateway(stub({ answer: 'done' }).provider) });
+    const run = lazyAgent().named('lazyCatalogueAgent');
+    expect(await run({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual({ answer: 'done' });
+    expect(attribute('ai.agent')).toBe('');
+  });
+
+  test('llm(): the first call answers', async () => {
+    configureAi({ gateway: lazyGateway(stub(ANSWER).provider) });
+    const call = declare(promptFor(), { model: FIXTURE_MODEL });
+    expect(await call({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual(ANSWER);
+    expect(attribute('ai.llm')).toBe('');
+  });
+
+  test('a knob the row does not take is still reported, once the row is there', async () => {
+    configureAi({ gateway: lazyGateway(stub({ answer: 'done' }).provider) });
+    // The fixture rows declare no `sampling`, so both knobs are left out of the request.
+    const run = lazyAgent({ temperature: 0, seed: 7 }).named('lazyCatalogueSampledAgent');
+    expect(await run({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual({ answer: 'done' });
+    expect(attribute('ai.agent')).toBe('temperature,seed');
+  });
+
+  test('llm() with knobs: answered, and the span names what was left out', async () => {
+    configureAi({ gateway: lazyGateway(stub(ANSWER).provider) });
+    const call = declare(promptFor(), { model: FIXTURE_MODEL, temperature: 0 });
+    expect(await call({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual(ANSWER);
+    expect(attribute('ai.llm')).toBe('temperature');
+  });
+});
+
+describe('the report never changes the call it describes', () => {
+  test('a gateway whose samplingWires throws: the answer stands, the attribute is left out', async () => {
+    const inner = lazyGateway(stub({ answer: 'done' }).provider);
+    const throwing = {
+      ...inner,
+      samplingWires: (): never => {
+        throw new UltimateError({
+          code: 'X_TEST_SAMPLING_WIRES',
+          cause: 'samplingWires broke',
+          fix: 'nothing: the test throws on purpose',
+        });
+      },
+    } as Gateway;
+    configureAi({ gateway: throwing });
+    const run = lazyAgent({ temperature: 0 }).named('lazyCatalogueThrowingWiresAgent');
+    expect(await run({ postId: POST_ID }, { ctx: anonymousCtx() })).toEqual({ answer: 'done' });
+    expect(attribute('ai.agent')).toBeUndefined();
+  });
+
+  test('a failed run still reports what was left out', async () => {
+    configureAi({ gateway: lazyGateway(stub({ wrong: 1 }).provider) });
+    const run = lazyAgent({ seed: 7 }).named('lazyCatalogueFailingAgent');
+    await expect(run({ postId: POST_ID }, { ctx: anonymousCtx() })).rejects.toBeDefined();
+    expect(attribute('ai.agent')).toBe('seed');
+  });
+
+  test('a model the catalogue never names: the call keeps its own error, no attribute', async () => {
+    // The call's own error is the gateway's: identity proves the report did not replace it
+    // (27.10.0 read the row first and threw X_AI_MODEL_UNKNOWN before the gateway was asked).
+    const own = new UltimateError({
+      code: 'X_TEST_GATEWAY_REFUSED',
+      cause: 'the gateway refused the call',
+      fix: 'nothing: the test refuses on purpose',
+    });
+    const refuse = (): never => {
+      throw own;
+    };
+    configureAi({
+      gateway: {
+        defaultModel: undefined,
+        generate: refuse,
+        stream: refuse,
+        scope: refuse,
+        spent: refuse,
+        callLedger: refuse,
+      } as unknown as Gateway,
+    });
+    const run = agent({
+      input: t.object({ postId: t.uuid }),
+      output: t.object({ answer: t.string }),
+      prompt: promptFor(),
+      model: 'nobody/unlisted-model',
+      vars: ({ input }) => ({ postId: input.postId }),
+      tools: [],
+      policy: allow(),
+      temperature: 0,
+    }).named('lazyCatalogueUnlistedAgent');
+    const error = await run({ postId: POST_ID }, { ctx: anonymousCtx() }).then(
+      () => undefined,
+      (thrown: unknown) => thrown,
+    );
+    expect(error).toBe(own);
+    expect(attribute('ai.agent')).toBeUndefined();
+  });
+});

packages/ai/src/llm.ts

@@ -50,7 +50,7 @@ import type { RunMeter } from './run-meter';
 import { runMeter } from './run-meter';
 import { aiGateway, aiRedactor, installedGateway } from './runtime';
 import type { Sampling } from './sampling';
-import { ignoredSampling, screenSampling } from './sampling';
+import { ignoredSamplingAttribute, screenSampling } from './sampling';
 
 /** Two attempts total: the answer, then one repair turn. See `LlmOutputInvalidError`. */
 const ATTEMPTS = 2;
@@ -182,12 +182,13 @@ async function generate<
       // an audit asks evidence for, and a redactor that silently stopped matching looks identical
       // to one that had nothing to remove until this attribute separates them.
       'llm.redacted': redacted,
-      'llm.sampling.ignored': ignoredSampling(
-        model,
-        sampling,
-        installedGateway()?.samplingWires?.(model),
-      ).join(','),
     });
+    // Read when the call ends, not here: the gateway may be what registers the model's row.
+    const reportSampling = (): void => {
+      span.setAttributes(
+        ignoredSamplingAttribute(model, sampling, () => installedGateway()?.samplingWires?.(model)),
+      );
+    };
 
     // A cached answer is still data of unknown provenance, so it goes through the schema like
     // any other. One that no longer fits — the schema moved under it — is a miss, not a
@@ -201,7 +202,10 @@ async function generate<
     });
     const hit = await accept(def.output, await cache?.lookup());
     span.setAttribute('llm.cache.hit', hit !== undefined);
-    if (hit !== undefined) return hit.value;
+    if (hit !== undefined) {
+      reportSampling();
+      return hit.value;
+    }
 
     // No `tools` yet: the `respond` projection belongs to the non-streaming path alone. A tool
     // call is emitted whole, so forcing the answer through one leaves a stream with nothing to
@@ -292,9 +296,11 @@ async function generate<
       });
     };
     return withBudget(ledger, () =>
-      answer().catch((error: unknown) => {
-        throw meter.fail(error);
-      }),
+      answer()
+        .catch((error: unknown) => {
+          throw meter.fail(error);
+        })
+        .finally(reportSampling),
     );
   });
 }

packages/ai/src/sampling.ts

@@ -11,7 +11,7 @@
 
 import { AiRequestInvalidError } from './errors';
 import type { ModelId } from './models';
-import { modelSpec } from './models';
+import { isModelRegistered, modelSpec } from './models';
 
 export interface Sampling {
   /** 0 to 2. `0` is the most repeatable a model offers; it is not a guarantee of determinism. */
@@ -74,6 +74,8 @@ export function ignoredSampling(
   /** The wire formats of the providers serving `model`, when the gateway can say. */
   wires?: readonly SamplingWire[],
 ): readonly string[] {
+  // Nothing asked, nothing ignored — and no row read: a model none asked of needs none here.
+  if (request.temperature === undefined && request.seed === undefined) return [];
   const accepts = modelSpec(model).sampling;
   // A knob no serving wire has a word for is dropped too — `seed` on the Messages API.
   const sendable = (knob: 'temperature' | 'seed'): boolean =>
@@ -84,3 +86,28 @@ export function ignoredSampling(
     (knob) => request[knob] !== undefined && (accepts?.[knob] !== true || !sendable(knob)),
   );
 }
+
+/**
+ * The `llm.sampling.ignored` span attribute, set when a run ENDS — answered or failed — never at
+ * its start. The catalogue is the app's, and an app may register it when its gateway is first used
+ * (a gateway built lazily from the env): read at the top of `agent()` / `llm()`, before the gateway
+ * was asked anything, the row was not there yet and a REPORT refused the call `X_AI_MODEL_UNKNOWN`.
+ * A model still unregistered by then has no row to report against; the call's own failure says so.
+ */
+export function ignoredSamplingAttribute(
+  model: ModelId,
+  request: Sampling,
+  wires: () => readonly SamplingWire[] | undefined,
+): Record<'llm.sampling.ignored', string> | Record<string, never> {
+  if (request.temperature === undefined && request.seed === undefined) {
+    return { 'llm.sampling.ignored': '' };
+  }
+  if (!isModelRegistered(model)) return {};
+  // A report never fails the call it describes: an installed gateway may throw from
+  // `samplingWires`, and the attribute is then left out rather than replacing the call's result.
+  try {
+    return { 'llm.sampling.ignored': ignoredSampling(model, request, wires()).join(',') };
+  } catch {
+    return {};
+  }
+}