diff --git a/src/llm/capabilities.ts b/src/llm/capabilities.ts index 7fb3b993..863e843f 100644 --- a/src/llm/capabilities.ts +++ b/src/llm/capabilities.ts @@ -139,6 +139,19 @@ export function registerWireRejectionEfforts( }); } +export function learnRejectedEffort( + provider: ProviderId, + model: string, + effort: ReasoningEffort, +): void { + if (!model.trim()) return; + const effective = displayReasoningEfforts(provider, model); + if (!effective?.length) return; + const reduced = effective.filter((value) => value !== effort); + if (reduced.length === 0 || reduced.length === effective.length) return; + registerWireRejectionEfforts(provider, model, reduced); +} + export function markReasoningMandatory( provider: ProviderId, model: string, diff --git a/src/llm/routing/attempt-complete.ts b/src/llm/routing/attempt-complete.ts index 24e4945d..695a0ada 100644 --- a/src/llm/routing/attempt-complete.ts +++ b/src/llm/routing/attempt-complete.ts @@ -6,6 +6,7 @@ import type { } from "../../types.js"; import { learnModelVisionCapability, + learnRejectedEffort, markReasoningMandatory, markReasoningUnsupported, registerWireRejectionEfforts, @@ -139,7 +140,9 @@ export async function tryCompleteOnce( thinking: candidate, }; try { - return await runAttempt(retryRequest, "adaptation"); + const result = await runAttempt(retryRequest, "adaptation"); + learnRejectedEffort(providerId, model, thinking.effort); + return result; } catch (retryError) { if (!shouldContinueEffortLadder(retryError)) throw retryError; } diff --git a/src/llm/routing/attempt-request.ts b/src/llm/routing/attempt-request.ts index 1b8684cc..4bef433a 100644 --- a/src/llm/routing/attempt-request.ts +++ b/src/llm/routing/attempt-request.ts @@ -14,6 +14,8 @@ import { } from "../capabilities.js"; import { buildReasoningPayload, stripImagesFromMessages } from "../http.js"; import type { ReasoningStyle } from "../http.js"; +import { isBuiltInProviderId } from "../provider-profile.js"; +import { resolveBuiltInProfile } from "../provider-profiles.js"; import { isOperationPolicyError } from "../operation-ledger.js"; import { runGenerationAttempt } from "../operation-usage.js"; import { isModelNotFoundError } from "./error-classification.js"; @@ -60,8 +62,14 @@ export function reasoningWireKey( model: string, providerId: ProviderId, ): string { + const control = isBuiltInProviderId(providerId) + ? { + profile: resolveBuiltInProfile({ provider: providerId, model }), + willReplayReasoning: false, + } + : undefined; return JSON.stringify( - buildReasoningPayload(thinking, style, model, providerId), + buildReasoningPayload(thinking, style, model, providerId, control), ); } diff --git a/src/llm/routing/attempt-stream.ts b/src/llm/routing/attempt-stream.ts index 30f5a396..d75b44ee 100644 --- a/src/llm/routing/attempt-stream.ts +++ b/src/llm/routing/attempt-stream.ts @@ -8,6 +8,7 @@ import type { } from "../../types.js"; import { learnModelVisionCapability, + learnRejectedEffort, markReasoningMandatory, markReasoningUnsupported, registerWireRejectionEfforts, @@ -277,7 +278,9 @@ export async function tryStreamOnce( thinking: candidate, }; try { - return await runAttempt(retryRequest, "adaptation"); + const result = await runAttempt(retryRequest, "adaptation"); + learnRejectedEffort(providerId, model, thinking.effort); + return result; } catch (retryError) { if (!shouldContinueEffortLadder(retryError)) { throw markStreamEmittedBytes( diff --git a/src/llm/routing/error-classification.ts b/src/llm/routing/error-classification.ts index ae5e5e8c..27b7a30a 100644 --- a/src/llm/routing/error-classification.ts +++ b/src/llm/routing/error-classification.ts @@ -64,6 +64,34 @@ function isReasoningRelatedServerError(error: unknown): boolean { return mentionsReasoning(error); } +const UNIVERSAL_EFFORTS: ReadonlySet = new Set([ + "none", + "low", + "medium", + "high", +]); + +const OPAQUE_PARAMETER_REJECTION_RE = + /invalid request|invalid parameter|unsupported parameter|unknown parameter|unrecognized|extra inputs are not permitted|not a valid/i; + +const NON_EFFORT_REJECTION_RE = + /model is not supported|model is unavailable|model not found|no such model|unknown model|rate limit|quota|insufficient|authentication|authorization|permission/i; + +function isOpaqueParameterRejection( + error: unknown, + effort: ReasoningEffort | undefined, +): boolean { + if (!effort || UNIVERSAL_EFFORTS.has(effort)) return false; + if (!(error instanceof ProviderError)) return false; + const status = error.status ?? 0; + const parameterRejected = + (status === 400 || status === 422) && + OPAQUE_PARAMETER_REJECTION_RE.test(`${error.message}\n${error.body ?? ""}`); + const upstreamCrashed = status >= 500 && status <= 504; + if (!parameterRejected && !upstreamCrashed) return false; + return !NON_EFFORT_REJECTION_RE.test(`${error.message}\n${error.body ?? ""}`); +} + export function shouldContinueEffortLadder(error: unknown): boolean { return ( isReasoningUnsupportedError(error) || isReasoningRelatedServerError(error) @@ -88,9 +116,17 @@ export function effortCandidatesFor( .reasoning.acceptedEfforts; if (declared.length === 0) return fallbackEffortsFor(requested); const nearest = nearestAcceptedEffort(requested, declared); - if (nearest === undefined || nearest === requested) return []; - const scaled = EFFORT_SCALE.find((effort) => effort === nearest); - return scaled ? [scaled] : []; + if (nearest !== undefined && nearest !== requested) { + const scaled = EFFORT_SCALE.find((effort) => effort === nearest); + return scaled ? [scaled] : []; + } + const requestedIndex = EFFORT_SCALE.indexOf(requested); + const lower = declared + .map((effort) => EFFORT_SCALE.findIndex((scaled) => scaled === effort)) + .filter((index) => index >= 0 && requestedIndex >= 0 && index < requestedIndex) + .sort((a, b) => b - a)[0]; + if (lower === undefined) return []; + return [EFFORT_SCALE[lower]!]; } export function shouldEnterEffortLadder( @@ -104,7 +140,8 @@ export function shouldEnterEffortLadder( if (singleDispatch) return false; if (!thinking?.enabled) return false; if (isReasoningUnsupported(providerId, model)) return false; - return isReasoningRelatedServerError(error); + if (isReasoningRelatedServerError(error)) return true; + return isOpaqueParameterRejection(error, thinking.effort); } export function isCacheOnlyColdError(error: unknown): boolean { diff --git a/test/llm/effort-ladder-opaque.test.ts b/test/llm/effort-ladder-opaque.test.ts new file mode 100644 index 00000000..32c12922 --- /dev/null +++ b/test/llm/effort-ladder-opaque.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + clearReasoningUnsupported, + displayReasoningEfforts, + resetReasoningKnowledge, +} from "../../src/llm/capabilities.js"; +import { ProviderError } from "../../src/llm/http.js"; +import { streamWithProvider } from "../../src/llm/router.js"; +import { + effortCandidatesFor, + shouldEnterEffortLadder, +} from "../../src/llm/routing/error-classification.js"; +import { installTransport } from "../conformance/fake-transport.js"; +import { + jsonResponse, + textStreamResponse, +} from "../conformance/wire-fixtures.js"; +import type { ChatMessage, ReasoningEffort } from "../../src/types.js"; + +vi.mock("../../src/store/keys.js", async (importOriginal) => { + const actual = await importOriginal< + typeof import("../../src/store/keys.js") + >(); + return { + ...actual, + getProviderKeys: async (provider: string) => ({ + keys: [{ id: "env", value: `sk-${provider}-testkey`, createdAt: 0 }], + activeIndex: 0, + source: "env" as const, + }), + }; +}); + +const messages: ChatMessage[] = [{ role: "user", content: "hi" }]; +const MODEL = "free-1/mimo-v2.5-free"; + +const ZEN_PARAMETER_REJECTION = { + error: { + type: "server_error", + message: + "Error from provider (Console): Upstream request failed: [400] Invalid request parameters", + }, +}; + +function okStream(): Response { + return textStreamResponse([ + `data: ${JSON.stringify({ + choices: [{ index: 0, delta: { role: "assistant", reasoning_content: "hm" } }], + })}\n\n`, + `data: ${JSON.stringify({ + choices: [{ index: 0, delta: { content: "ok" } }], + })}\n\n`, + `data: ${JSON.stringify({ + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + })}\n\n`, + "data: [DONE]\n\n", + ]); +} + +function zenGatewayRejectingExtendedEfforts() { + return installTransport((req) => { + const body = (req.body ?? {}) as Record; + const effort = body["reasoning_effort"]; + if (effort === "xhigh" || effort === "minimal") { + return jsonResponse(ZEN_PARAMETER_REJECTION, 400); + } + return okStream(); + }); +} + +const effortsOf = (t: ReturnType): unknown[] => + t.generations.map((g) => (g.body as Record)["reasoning_effort"]); + +const thinking = (effort: ReasoningEffort) => ({ enabled: true, effort }); + +beforeEach(() => { + resetReasoningKnowledge(); +}); + +afterEach(() => { + clearReasoningUnsupported(); + resetReasoningKnowledge(); + vi.unstubAllGlobals(); +}); + +describe("effort ladder entry for opaque gateway rejections", () => { + const zen400 = new ProviderError( + "Free (model=mimo-v2.5-free): Provider request failed with HTTP 400 — Error from provider (Console): Upstream request failed: [400] Invalid request parameters", + 400, + JSON.stringify(ZEN_PARAMETER_REJECTION), + ); + const zen500 = new ProviderError( + "Free (model=mimo-v2.5-free): Provider request failed with HTTP 500 — Internal server error", + 500, + "Internal server error", + ); + + it("enters the ladder when an extended effort meets an opaque rejection", () => { + expect(shouldEnterEffortLadder(zen400, thinking("xhigh"), "free", MODEL, false)).toBe(true); + expect(shouldEnterEffortLadder(zen500, thinking("xhigh"), "free", MODEL, false)).toBe(true); + expect(shouldEnterEffortLadder(zen400, thinking("minimal"), "free", MODEL, false)).toBe(true); + }); + + it("stays out of the ladder for universal efforts and unrelated failures", () => { + expect(shouldEnterEffortLadder(zen400, thinking("high"), "free", MODEL, false)).toBe(false); + expect(shouldEnterEffortLadder(zen500, thinking("medium"), "free", MODEL, false)).toBe(false); + expect(shouldEnterEffortLadder(zen400, undefined, "free", MODEL, false)).toBe(false); + expect( + shouldEnterEffortLadder( + new ProviderError("rate limited", 429), + thinking("xhigh"), + "free", + MODEL, + false, + ), + ).toBe(false); + expect( + shouldEnterEffortLadder( + new ProviderError("model not supported", 401), + thinking("xhigh"), + "free", + MODEL, + false, + ), + ).toBe(false); + expect( + shouldEnterEffortLadder( + new ProviderError( + "Provider request failed with HTTP 400 — Model is unavailable", + 400, + "Model is unavailable", + ), + thinking("xhigh"), + "free", + MODEL, + false, + ), + ).toBe(false); + }); + + it("steps down within the declared efforts when the gateway rejects one", () => { + expect(effortCandidatesFor("free", MODEL, "xhigh")).toEqual(["high"]); + expect(effortCandidatesFor("free", MODEL, "minimal")).toEqual(["none"]); + }); +}); + +describe("zen free model rejecting extended efforts", () => { + it("recovers the turn through the ladder and learns the route", async () => { + const transport = zenGatewayRejectingExtendedEfforts(); + const statuses: string[] = []; + + const result = await streamWithProvider( + { + provider: "free", + model: MODEL, + messages, + thinking: thinking("xhigh"), + }, + () => {}, + (message) => statuses.push(message), + ); + + expect(result.text).toContain("ok"); + expect(effortsOf(transport)).toEqual(["xhigh", "high"]); + expect(statuses.some((message) => /retrying with high/.test(message))).toBe(true); + expect(displayReasoningEfforts("free", MODEL)).not.toContain("xhigh"); + }); + + it("maps the rejected effort before the wire on later turns", async () => { + zenGatewayRejectingExtendedEfforts(); + await streamWithProvider( + { + provider: "free", + model: MODEL, + messages, + thinking: thinking("xhigh"), + }, + () => {}, + ); + + const second = zenGatewayRejectingExtendedEfforts(); + const result = await streamWithProvider( + { + provider: "free", + model: MODEL, + messages, + thinking: thinking("xhigh"), + }, + () => {}, + ); + + expect(result.text).toContain("ok"); + expect(effortsOf(second)).toEqual(["high"]); + }); + + it("falls back to disabling reasoning when no lower effort exists", async () => { + const transport = zenGatewayRejectingExtendedEfforts(); + + const result = await streamWithProvider( + { + provider: "free", + model: MODEL, + messages, + thinking: thinking("minimal"), + }, + () => {}, + ); + + expect(result.text).toContain("ok"); + expect(effortsOf(transport)).toEqual(["minimal", "none"]); + expect(displayReasoningEfforts("free", MODEL)).not.toContain("minimal"); + }); +}); diff --git a/test/llm/effort-single-hop.test.ts b/test/llm/effort-single-hop.test.ts index b405a737..1f68196b 100644 --- a/test/llm/effort-single-hop.test.ts +++ b/test/llm/effort-single-hop.test.ts @@ -49,7 +49,7 @@ function effortOf(body: unknown): unknown { } describe("a route with a declared effort vocabulary takes one hop, not a ladder", () => { - it("retries exactly once when the metadata turns out to be wrong", async () => { + it("retries without re-sending a payload the emitter already chose", async () => { let chatCalls = 0; const transport = installTransport((record) => { if (record.url.endsWith("/responses")) { @@ -74,7 +74,8 @@ describe("a route with a declared effort vocabulary takes one hop, not a ladder" generation.url.includes("/chat/completions"), ); expect(chatGenerations).toHaveLength(2); - expect(effortOf(chatGenerations[1]?.body)).toBe("high"); + expect(effortOf(chatGenerations[0]?.body)).toBe("high"); + expect(effortOf(chatGenerations[1]?.body)).toBeUndefined(); }); it("gives up after that single hop instead of walking the whole scale", async () => { @@ -143,9 +144,9 @@ describe("candidate selection", () => { ).toEqual(["high"]); }); - it("offers nothing when the declared vocabulary already contains the request", () => { + it("steps down one rung when the declared vocabulary contains the request", () => { expect(effortCandidatesFor("tokenrouter", "moonshotai/kimi-k3", "high")).toEqual( - [], + ["low"], ); });