fix: expose fallback model and refusals

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@alphaxiv/agents",
-  "version": "0.7.9",
+  "version": "0.7.10",
   "license": "MIT",
   "fmt": {
     "lineWidth": 120

mod.ts

@@ -14,6 +14,7 @@ export {
   ERROR_KINDS,
   FirstTokenTimeoutError,
   InvalidAttachmentError,
+  ModelRefusalError,
 } from "./src/errors.ts";
 export type { ClassifiedError, ErrorKind } from "./src/errors.ts";
 

src/adapters/anthropic/adapter.ts

@@ -199,7 +199,13 @@ ${JSON.stringify(structuredOutput.originalJsonSchema, null, 2)}
 
       const parts: ChatItem[] = [];
       const reasoningSignatures = new Map<number, string>();
+      // finalMessage can drop stop_details emitted in message_delta.
+      let refusalCategory: string | null = null;
       for await (const part of response) {
+        if (part.type === "message_delta") {
+          refusalCategory = part.delta.stop_details?.category ?? refusalCategory;
+        }
+
         if (part.type === "content_block_delta") {
           const { delta } = part;
           if (delta.type === "text_delta") {
@@ -319,6 +325,8 @@ ${JSON.stringify(structuredOutput.originalJsonSchema, null, 2)}
 
       const final = await response.finalMessage();
       return {
+        stopReason: final.stop_reason,
+        refusalCategory: final.stop_details?.category ?? refusalCategory,
         outputTokens: final.usage.output_tokens,
         inputTokens: final.usage.input_tokens,
         cacheReadTokens: final.usage.cache_read_input_tokens ?? null,

src/agent.ts

@@ -5,7 +5,13 @@ import type { Adapter } from "./adapters/adapter.ts";
 import { type AdapterLike, resolveModel } from "./adapters/model_resolver.ts";
 import { addStreamItem } from "./client.ts";
 import { createStructuredOutputRetryFeedback } from "./constants.ts";
-import { type ClassifiedError, classifyError, createClassifiedError, FirstTokenTimeoutError } from "./errors.ts";
+import {
+  type ClassifiedError,
+  classifyError,
+  createClassifiedError,
+  FirstTokenTimeoutError,
+  ModelRefusalError,
+} from "./errors.ts";
 import {
   determineRetryBehavior,
   isDeterministicModelError,
@@ -60,6 +66,7 @@ export interface AgentRunResult<zO> {
   output: zO;
   history: WithTraceId<ChatItem>[];
   usage: TokenUsage;
+  model: ModelInfo;
   get outputText(): string;
 }
 
@@ -365,7 +372,7 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
         signal.throwIfAborted();
 
         // Phase 1: Stream from a model (dispatches tool calls eagerly)
-        const { turnItems, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, trace } = yield* this
+        const { turnItems, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, trace, model } = yield* this
           .#invokeModel({
             signal,
             initialHistory,
@@ -412,6 +419,7 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
               history: [...history, ...turnItems],
               output: modelOutput.output as ResolveAgentOutput<zO, Tools>,
               usage,
+              model,
             });
           }
 
@@ -442,6 +450,7 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
           history,
           output: result.output,
           usage,
+          model,
         });
       }
 
@@ -469,7 +478,7 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
     deadModels: Set<number>;
   }): AsyncGenerator<
     WithTraceId<StreamItem>,
-    ModelCallTokens & { turnItems: WithTraceId<ChatItem>[]; trace: string }
+    ModelCallTokens & { turnItems: WithTraceId<ChatItem>[]; trace: string; model: ModelInfo }
   > {
     const { signal, history, pendingTools, toolSignal, agentTrace, turn } = options;
     let { modelCallReason } = options;
@@ -591,7 +600,7 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
             // summaries appear before the model's output in history (correct chronological order).
             turnItems.unshift(...compactionItems);
 
-            return { turnItems, ...result };
+            return { turnItems, ...result, model: currentModel };
           } catch (error) {
             messageTracer.cancel();
             modelTrace.error(error);
@@ -610,6 +619,8 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
               options.deadModels.add(currentModelIndex);
             }
 
+            if (classified.kind === "content_filtered") turnItems.length = 0;
+
             const behavior = determineRetryBehavior(classified, this.#retryStrategy, sameModelRetries);
 
             // Always try handleModelError if provided - let the user decide what errors to handle
@@ -732,12 +743,17 @@ export class Agent<zO = unknown, zI = unknown, const Tools extends AnyTool[] = [
       if (done) {
         disarmTimer();
         messageTracer.endMessageTraceIfStarted();
-        modelTrace.success({
+        modelTrace.update({
+          stopReason: part.stopReason,
+          refusalCategory: part.refusalCategory,
           inputTokens: part.inputTokens,
           outputTokens: part.outputTokens,
           cacheReadTokens: part.cacheReadTokens ?? null,
           cacheWriteTokens: part.cacheWriteTokens ?? null,
         });
+        if (part.stopReason === "refusal") throw new ModelRefusalError(part.refusalCategory ?? null, adapter.model);
+
+        modelTrace.success();
         return {
           inputTokens: part.inputTokens ?? 0,
           outputTokens: part.outputTokens ?? 0,
@@ -960,12 +976,14 @@ function createRunResult<T>(completion: {
   history: WithTraceId<ChatItem>[];
   usage: TokenUsage;
   output: T;
+  model: ModelInfo;
 }): AgentRunResult<T> {
-  const { output, history, usage } = completion;
+  const { output, history, usage, model } = completion;
   return {
     history,
     output,
     usage,
+    model,
     // using a getter so that if you console.log this object, you
     // don't see all the data twice.
     get outputText() {

src/errors.ts

@@ -201,6 +201,13 @@ export class FirstTokenTimeoutError extends Error {
   }
 }
 
+export class ModelRefusalError extends Error {
+  override readonly name = "ModelRefusalError";
+  constructor(readonly category: string | null = null, readonly model: string | null = null) {
+    super("Model refused the request");
+  }
+}
+
 export class InvalidAttachmentError extends Error {
   constructor(readonly url: string, message: string) {
     super(message);
@@ -236,7 +243,9 @@ export function classifyError(error: unknown, status?: number): ClassifiedError
 
   let kind: ErrorKind = "unknown";
 
-  if (error instanceof InvalidAttachmentError) {
+  if (error instanceof ModelRefusalError) {
+    kind = "content_filtered";
+  } else if (error instanceof InvalidAttachmentError) {
     kind = "invalid_attachment";
   } else if (isLikelyAttachmentRejected(message)) {
     kind = "attachment_rejected";

src/tracing.ts

@@ -105,6 +105,8 @@ export interface ModelTraceEvent extends BaseTraceEvent {
     provider: string;
     /** The model given to the provider */
     model: string;
+    stopReason?: string | null;
+    refusalCategory?: string | null;
     /**
      * The millisecond timestamp at which the adapter sent the request to the provider, after any preparation.
      * `null` if the adapter did not report it.

src/types.ts

@@ -214,6 +214,8 @@ export type AgentStreamIterator<T = unknown> = AsyncGenerator<
 export type Awaitable<T> = T | Promise<T>;
 
 export interface ProviderStreamMetadata {
+  stopReason?: string | null;
+  refusalCategory?: string | null;
   /**
    * Count of input tokens billed at the full rate. Providers that report prompt
    * caching exclude cached tokens here, so the prompt size is

tests/adapters/anthropic.test.ts

@@ -1,7 +1,15 @@
-import type Anthropic from "@anthropic-ai/sdk";
+import Anthropic from "@anthropic-ai/sdk";
 import { assert, assertEquals, assertRejects } from "@std/assert";
 import z from "zod";
-import { addStreamItem, Agent, type AnyTool, type ChatItem, type StreamItem, Tool } from "../../mod.ts";
+import {
+  addStreamItem,
+  Agent,
+  type AnyTool,
+  type ChatItem,
+  ModelRefusalError,
+  type StreamItem,
+  Tool,
+} from "../../mod.ts";
 import { anthropicModel } from "../../src/adapters/anthropic/adapter.ts";
 import { applyAnthropicCacheBreakpoint, getAnthropicHistory } from "../../src/adapters/anthropic/history.ts";
 import { getAnthropicMessagesStreamConfig } from "../../src/adapters/anthropic/models.ts";
@@ -17,9 +25,124 @@ import {
   runStructuredToolParameterStreamingTest,
 } from "./shared.ts";
 import type { Adapter } from "../../src/adapters/adapter.ts";
+import type { TraceEvent } from "../../src/tracing.ts";
 
 const HAS_ANTHROPIC_KEY = Boolean(Deno.env.get("ANTHROPIC_API_KEY"));
 
+function createStreamingAnthropicClient(stopReason: "end_turn" | "refusal", category: string | null = null) {
+  const events = [
+    {
+      type: "message_start",
+      message: {
+        id: "msg_test",
+        type: "message",
+        role: "assistant",
+        model: "claude-opus-5-5",
+        content: [],
+        stop_reason: null,
+        stop_sequence: null,
+        stop_details: null,
+        usage: { input_tokens: 3, output_tokens: 0 },
+      },
+    },
+    { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
+    { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Partial answer" } },
+    { type: "content_block_stop", index: 0 },
+    {
+      type: "message_delta",
+      delta: {
+        stop_reason: stopReason,
+        stop_sequence: null,
+        stop_details: category === null ? null : { category },
+      },
+      usage: { output_tokens: 4 },
+    },
+    { type: "message_stop" },
+  ];
+  const body = events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join("");
+
+  return new Anthropic({
+    apiKey: "test-api-key",
+    fetch: () => Promise.resolve(new Response(body, { headers: { "content-type": "text/event-stream" } })),
+  });
+}
+
+Deno.test("Anthropic reports completion metadata through the SDK stream", async () => {
+  const adapter = anthropicModel({ model: "claude-opus-5-5", client: createStreamingAnthropicClient("end_turn") });
+  const { metadata } = await streamOnce(adapter);
+
+  assertEquals(metadata, {
+    stopReason: "end_turn",
+    refusalCategory: null,
+    inputTokens: 3,
+    outputTokens: 4,
+    cacheReadTokens: null,
+    cacheWriteTokens: null,
+  });
+});
+
+Deno.test("Anthropic refusal details reach fallback and traces without retaining refused output", async () => {
+  const events: TraceEvent[] = [];
+  const fallback: Adapter<unknown, unknown> = {
+    provider: "fallback",
+    model: "requested-backup",
+    async *stream({ history }) {
+      assertEquals(history, [{ type: "input_text", content: "hi" }]);
+      yield { type: "delta_output_text", index: 0, delta: "Accepted answer" };
+      return { inputTokens: 5, outputTokens: 6 };
+    },
+  };
+  const refusals: ModelRefusalError[] = [];
+  const agent = new Agent({
+    model: [
+      anthropicModel({ model: "claude-opus-5-5", client: createStreamingAnthropicClient("refusal", "bio") }),
+      fallback,
+    ],
+    instructions: "Answer the question.",
+    retryStrategy: {
+      customHandler({ kind, original }) {
+        assertEquals(kind, "content_filtered");
+        assert(original instanceof ModelRefusalError);
+        refusals.push(original);
+        return "switch-model";
+      },
+    },
+  });
+  const result = await agent.run("hi", { tracers: [{ event: (event) => events.push(event) }] });
+
+  assertEquals(refusals.length, 1);
+  const [refusal] = refusals;
+  assertEquals(refusal.category, "bio");
+  assertEquals(refusal.model, "claude-opus-5-5");
+  assertEquals(result.outputText, "Accepted answer");
+  assertEquals(result.history.length, 1);
+  assertEquals(result.model, { provider: "fallback", model: "requested-backup" });
+
+  const models = events.filter((event) => event.type === "model");
+  assertEquals(models.length, 2);
+  assertEquals(models[0].errorObject, refusal);
+  assertEquals(models[0].content.model, "claude-opus-5-5");
+  assertEquals(models[0].content.stopReason, "refusal");
+  assertEquals(models[0].content.refusalCategory, "bio");
+  assertEquals(models[0].content.inputTokens, 3);
+  assertEquals(models[0].content.outputTokens, 4);
+  assertEquals(models[1].errorMessage, null);
+  assertEquals(models[1].content.model, "requested-backup");
+});
+
+for (const category of [null, "new-category"]) {
+  Deno.test(`Anthropic refusals preserve category ${category} and fail without an explicit retry`, async () => {
+    const agent = new Agent({
+      model: anthropicModel({ model: "claude-opus-5-5", client: createStreamingAnthropicClient("refusal", category) }),
+      instructions: "Answer the question.",
+    });
+    const error = await assertRejects(() => agent.run("hi"), ModelRefusalError);
+
+    assertEquals(error.category, category);
+    assertEquals(error.model, "claude-opus-5-5");
+  });
+}
+
 Deno.test({
   name: "AnthropicAdapter streams a parameterized tool call (claude-opus-5, adaptive)",
   ignore: !HAS_ANTHROPIC_KEY,
@@ -750,7 +873,11 @@ Deno.test("Anthropic ignores signature deltas when thinking display omits reason
               yield { type: "content_block_stop", index: 1 };
             },
             finalMessage() {
-              return { usage: { input_tokens: 0, output_tokens: 0 } };
+              return {
+                stop_reason: "end_turn",
+                stop_details: null,
+                usage: { input_tokens: 0, output_tokens: 0 },
+              };
             },
           };
         },
@@ -777,7 +904,14 @@ Deno.test("Anthropic ignores signature deltas when thinking display omits reason
     delta: "done",
     index: 1,
   }]);
-  assertEquals(metadata, { inputTokens: 0, outputTokens: 0, cacheReadTokens: null, cacheWriteTokens: null });
+  assertEquals(metadata, {
+    stopReason: "end_turn",
+    refusalCategory: null,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: null,
+    cacheWriteTokens: null,
+  });
 });
 
 Deno.test("Anthropic attaches signatures after reasoning block completion", async () => {
@@ -955,6 +1089,8 @@ function createCapturingAnthropicClient(usage: Partial<Anthropic.Beta.BetaUsage>
             (async function* () {})(),
             {
               finalMessage: () => ({
+                stop_reason: "end_turn",
+                stop_details: null,
                 usage: {
                   input_tokens: 10,
                   output_tokens: 20,
@@ -1114,6 +1250,8 @@ Deno.test("Anthropic reports cache token usage separately from uncached input",
   const { metadata } = await streamOnce(anthropicModel({ model: "claude-opus-4-8", cache: true, client }));
 
   assertEquals(metadata, {
+    stopReason: "end_turn",
+    refusalCategory: null,
     inputTokens: 10,
     outputTokens: 20,
     cacheReadTokens: 4096,

tests/simple/agent.test.ts

@@ -18,6 +18,7 @@ Deno.test("Basic input out of agents works", async () => {
     instructions: "You are a friendly assistant",
   });
   const run = await agent.run("Hello!");
+  assertEquals(run.model, { provider: "deterministic", model: "deterministic" });
   assertEquals(run.history.length, 1);
   assertObjectMatch(run.history[0], {
     type: "output_text",

tests/simple/model-output.test.ts

@@ -23,6 +23,7 @@ Deno.test("ModelOutput from a tool terminates agent run and returns the value",
   const run = await agent.run("Call output tool");
   run.output satisfies { ids: string[] } | undefined;
   assertEquals(run.output, { ids: ["a", "b", "c"] });
+  assertEquals(run.model, { provider: "deterministic", model: "deterministic" });
 });
 
 Deno.test("ModelOutput outputText is JSON-stringified for non-string values", async () => {