From ffbcdbf1e6479335d3eccfd2bb8e7ea1cd66e16e Mon Sep 17 00:00:00 2001 From: xiaobaifly7 <196511160+xiaobaifly7@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:24:43 +0800 Subject: [PATCH] fix(dsh): size each llm/stream dispatch against its own model and await a cold window headersFor stamped x-bili-plugin-model / -context-window for the model the session header names (#2381), but an llm/stream dispatch can stream to a different model: auxiliary calls (compaction, title, subagent) running on another model carried the session model's numbers, so the proxy's exact-match gate discarded the header channel and the request was sized against the fallback window. The window resolve was also fire-and-forget, so a model's first dispatch always went out with no window header. The dispatch scope now carries the model the dispatch streams to (GenerateOptions.provider + model, falling back to the session header and the global default). A dispatch whose model has no cached window waits for the in-flight resolve before its first fetch, bounded to 2s from the resolve's start so a slow or hung resolver can never stall the stream. Co-Authored-By: Claude Opus 5.5 --- src/agent/dsh-native.ts | 45 ++++++-- tests/dsh-dispatch-model-window.test.ts | 134 ++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 tests/dsh-dispatch-model-window.test.ts diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index 5d83069fb5..92c8159355 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -529,12 +529,15 @@ function modelInfoRetryCooldownMs(): number { const parsed = raw === undefined ? Number.NaN : Number(raw); return Number.isFinite(parsed) && parsed >= 0 ? parsed : MODEL_INFO_RETRY_COOLDOWN_MS; } +// How long after a resolve starts a dispatch may wait on it before its first +// fetch; a slow or hung resolver must never stall the stream beyond this. +const MODEL_INFO_DISPATCH_WAIT_MS = 2_000; const modelInfo: { byKey: Map; - refreshingKeys: Set; + refreshingKeys: Map>; retryAtByKey: Map; services?: { llm?: PluginContext["llm"]; agentDefaultModel?: PluginContext["agentDefaultModel"] }; -} = { byKey: new Map(), refreshingKeys: new Set(), retryAtByKey: new Map() }; +} = { byKey: new Map(), refreshingKeys: new Map(), retryAtByKey: new Map() }; function modelInfoKey(provider: string, model: string): string { return `${provider}\u0000${model}`; @@ -559,8 +562,12 @@ function normalizeModelTarget(provider: unknown, model: unknown): { provider: st // #2381: the model THIS request routes to — the session's own live header // first (exact at fetch time, correct across per-session model switches), the // host's global default selection only as fallback. Any miss degrades to -// pre-#2381 behavior; it never yields a wrong model. +// pre-#2381 behavior; it never yields a wrong model. Inside an llm/stream +// dispatch the model that dispatch itself streams to wins (auxiliary calls +// may run on a different model than the session's). function sessionModelTargetOf(ctx: PluginContext): { provider: string; model: string } | undefined { + const scoped = dispatchScope.getStore()?.modelTarget; + if (scoped !== undefined) return scoped; try { const init = ctx.agents?.currentInitiator?.(); const header = init?.session?.requestHeader?.(); @@ -618,7 +625,10 @@ export function bodyOverWindowDetail( return sharedBodyOverWindowDetail(body?.bytes, rawBodyStringOf(init), declaredWindow); } -function refreshModelInfo(origin: string | undefined, target?: { provider: string; model: string }): void { +// Returns a wait handle for the target's in-flight resolve (settles when it +// lands, or MODEL_INFO_DISPATCH_WAIT_MS after it started), undefined when +// there is nothing to wait for. +function refreshModelInfo(origin: string | undefined, target?: { provider: string; model: string }): Promise | undefined { const svc = modelInfo.services; if (svc === undefined) return; const sel = target ?? globalModelTarget(svc); @@ -633,7 +643,7 @@ function refreshModelInfo(origin: string | undefined, target?: { provider: strin if (cached !== undefined && cached.contextWindow !== undefined) return; const retryAt = modelInfo.retryAtByKey.get(key); if (cached !== undefined && retryAt !== undefined && Date.now() < retryAt) return; - if (modelInfo.refreshingKeys.has(key)) return; + if (modelInfo.refreshingKeys.has(key)) return modelInfo.refreshingKeys.get(key); // The host registers llm as a service object, so resolveModelInfo needs its // receiver; called detached below, this threw // "Cannot read properties of undefined (reading 'resolveModelInfoFor')" @@ -645,9 +655,8 @@ function refreshModelInfo(origin: string | undefined, target?: { provider: strin evictModelInfoEntries(); return; } - modelInfo.refreshingKeys.add(key); modelInfo.retryAtByKey.set(key, Date.now() + modelInfoRetryCooldownMs()); - void Promise.resolve() + const flight = Promise.resolve() .then(() => resolve(provider, model)) .then((info) => { // Per-key commit (#2381): the entry belongs to its OWN key, so a @@ -687,6 +696,9 @@ function refreshModelInfo(origin: string | undefined, target?: { provider: strin }).catch(() => {}); } }); + const waitable = Promise.race([flight, new Promise((done) => { setTimeout(done, MODEL_INFO_DISPATCH_WAIT_MS).unref?.(); })]); + modelInfo.refreshingKeys.set(key, waitable); + return waitable; } function errMessage(err: unknown): string { @@ -1252,6 +1264,7 @@ interface LlmDispatchScope { sid: string; msgIds: string[] | undefined; purpose?: string; + modelTarget?: { provider: string; model: string }; } const dispatchScope = new AsyncLocalStorage(); @@ -2084,7 +2097,7 @@ export async function apply(ctx: PluginContext): Promise { if (typeof ctx.inject === "function" && typeof ctx.on === "function") { const onEvent = ctx.on.bind(ctx); ctx.inject(["llm"], () => { - onEvent("llm/stream", (options: { sessionId?: unknown; messages?: unknown; purpose?: unknown }, next: () => AsyncIterable) => { + onEvent("llm/stream", (options: { sessionId?: unknown; messages?: unknown; purpose?: unknown; provider?: unknown; model?: unknown }, next: () => AsyncIterable) => { const sid = typeof options?.sessionId === "string" && options.sessionId.length > 0 ? options.sessionId : undefined; if (sid === undefined) return next(); // #4491: capture the host message ids riding the same options @@ -2094,7 +2107,12 @@ export async function apply(ctx: PluginContext): Promise { // (dsh sets it on every auxiliary call; conversation leaves it // unset). Absent/odd values simply stay off the turn. const purpose = typeof options?.purpose === "string" && options.purpose.length > 0 && options.purpose.length <= 64 && /^[\x21-\x7e]+$/.test(options.purpose) ? options.purpose : undefined; - const scope: LlmDispatchScope = { sid, msgIds: hostMsgIdsOf(options?.messages), purpose }; + // Size the dispatch against the model IT streams to (auxiliary + // calls may use another model than the session header names). + // GenerateOptions declares provider + model at the top level. + const modelTarget = normalizeModelTarget(options?.provider, options?.model) + ?? sessionModelTargetOf(ctx) ?? globalModelTarget(modelInfo.services); + const scope: LlmDispatchScope = { sid, msgIds: hostMsgIdsOf(options?.messages), purpose, modelTarget }; // #5403/#5418 v3: feed the per-slot id sequence into the // watermark channel (the fetch chokepoint emits count + // tail-ids; core = pf-channel.ts) — "?" slots ride id-less @@ -2108,7 +2126,14 @@ export async function apply(ctx: PluginContext): Promise { // latter; each also clears inherited fetch-chain re-entry // state so a new operation never looks like mid-descent // re-entry (#5614). - return runLlmDispatch(scope, () => wrapScopedIterable(next(), scope)); + // A model's first dispatch would otherwise fetch before its + // window resolved and go out without it; wait (bounded) first. + const pending = refreshModelInfo(register.base ?? state.origin, modelTarget); + const stream: () => AsyncIterable = pending === undefined ? next : async function* () { + await pending; + yield* next(); + }; + return runLlmDispatch(scope, () => wrapScopedIterable(stream(), scope)); }); }); } diff --git a/tests/dsh-dispatch-model-window.test.ts b/tests/dsh-dispatch-model-window.test.ts new file mode 100644 index 0000000000..c6ebe4a268 --- /dev/null +++ b/tests/dsh-dispatch-model-window.test.ts @@ -0,0 +1,134 @@ +/** + * dsh llm/stream dispatches are sized against the model THEY stream to, and a + * model's first dispatch carries its window: the window header used to come + * from the session header's model (wrong for auxiliary calls on another + * model) and from a fire-and-forget resolve (absent on a model's first fetch). + */ +import { strict as assert } from "node:assert"; +import { mkdtempSync, rmSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +delete process.env.BILI_PROVIDER_REWRITES; +delete process.env.BILI_NATIVE_DSH; +delete process.env.BILLION_CONTEXT_PLUGIN; + +import { apply, _resetRegisterForTest, _settleNativeForTest, _stateHeadersForTest, type PluginContext } from "../src/agent/dsh-native.ts"; +import { __resetConversationRecallForTests } from "../src/agent/conversation-recall.ts"; + +const MODEL_URL = "https://api.anthropic.com/v1/messages"; +type Listener = (payload: unknown, next: () => unknown) => unknown; +type Resolve = (provider: string, model: string) => Promise<{ context?: { contextWindow?: number } } | undefined>; + +async function startMockProxy(): Promise<{ origin: string; server: Server }> { + const server = createServer((req, res) => { + res.writeHead(req.url === "/__bili/plugin/manifest" || req.url === "/__bili/plugin/session-name" ? 200 : 404, { "content-type": "application/json" }); + res.end(JSON.stringify(req.url === "/__bili/plugin/manifest" + ? { version: "0.1.119", tools: { anthropic: [{ name: "compress", description: "c", input_schema: { type: "object", properties: {} } }] } } + : {})); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (address === null || typeof address !== "object") throw new Error("no port"); + return { origin: `http://127.0.0.1:${address.port}`, server }; +} + +async function withDsh(resolve: Resolve, sessionModel: string, fn: (dispatch: (options: Record) => Promise>) => Promise): Promise { + __resetConversationRecallForTests(); + const home = mkdtempSync(join(tmpdir(), "dsh-dispatch-model-")); + const saved = { BILLION_CONTEXT_PROXY: process.env.BILLION_CONTEXT_PROXY, DSH_HOME: process.env.DSH_HOME }; + const proxy = await startMockProxy(); + process.env.BILLION_CONTEXT_PROXY = proxy.origin; + process.env.DSH_HOME = home; + _resetRegisterForTest(proxy.origin); + const listeners: Listener[] = []; + const registered: unknown[] = []; + const services = { llm: { resolveModelInfo: resolve }, agentDefaultModel: { currentSelection: () => ({ provider: "deepseek", model: "main-model" }) } }; + const ctx = { + inject: (deps: readonly string[], cb: (sub: unknown) => void) => { if (deps.includes("llm")) cb(services); }, + on: (name: string, listener: Listener) => { if (name === "llm/stream") listeners.push(listener); }, + emit: () => undefined, + logger: { info: () => undefined, warn: () => undefined, error: () => undefined }, + tools: { register: (tool: unknown) => void registered.push(tool) }, + commands: { register: () => undefined }, + agents: { currentInitiator: () => ({ session: { id: "ambient", requestHeader: () => ({ config: { provider: "deepseek", model: sessionModel } }) } }) }, + } as unknown as PluginContext; + // One adapter fetch per dispatch: the headers stamped on its FIRST request. + const dispatch = async (options: Record): Promise> => { + let stamped: Record = {}; + const base = async function* (): AsyncIterable { + stamped = _stateHeadersForTest()?.(MODEL_URL) ?? {}; + yield 1; + }; + const composed = listeners.reduceRight((acc, l) => (p, n) => l(p, () => acc(p, n)), () => base()); + for await (const _ of composed(options, () => base()) as AsyncIterable) { /* drain */ } + return stamped; + }; + try { + await apply(ctx); + const deadline = Date.now() + 15_000; + while (registered.length === 0) { + if (Date.now() > deadline) throw new Error("timeout waiting for tool registration"); + await new Promise((r) => setTimeout(r, 20)); + } + await fn(dispatch); + } finally { + await _settleNativeForTest(); + proxy.server.close(); + proxy.server.closeIdleConnections?.(); + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + rmSync(home, { recursive: true, force: true }); + _resetRegisterForTest(undefined); + } +} + +const WINDOWS: Record = { "main-model": 1_000_000, "aux-model": 65_536, "fresh-model": 131_072 }; +// The session's own model resolves at boot; every other model resolves late. +const slowResolve: Resolve = (_provider, model) => model === "main-model" + ? Promise.resolve({ context: { contextWindow: WINDOWS[model] } }) + : new Promise((r) => setTimeout(() => r({ context: { contextWindow: WINDOWS[model] } }), 100)); + +test("auxiliary dispatch on another model stamps THAT model's window on its first request", async () => { + await withDsh(slowResolve, "main-model", async (dispatch) => { + const deadline = Date.now() + 15_000; + while (_stateHeadersForTest()?.(MODEL_URL)?.["x-bili-plugin-context-window"] !== "1000000") { + if (Date.now() > deadline) throw new Error("timeout waiting for the session model's window"); + await new Promise((r) => setTimeout(r, 20)); + } + const headers = await dispatch({ sessionId: "sess-aux", purpose: "compaction", provider: "deepseek", model: "aux-model" }); + assert.equal(headers["x-bili-plugin-conversation"], "sess-aux"); + assert.equal(headers["x-bili-plugin-model"], "aux-model"); + assert.equal(headers["x-bili-plugin-context-window"], "65536"); + // A dispatch on the session's own model keeps that model's window. + const main = await dispatch({ sessionId: "sess-aux", provider: "deepseek", model: "main-model" }); + assert.equal(main["x-bili-plugin-model"], "main-model"); + assert.equal(main["x-bili-plugin-context-window"], "1000000"); + }); +}); + +test("a model's first dispatch waits for its window instead of going out without it", async () => { + await withDsh(slowResolve, "fresh-model", async (dispatch) => { + const headers = await dispatch({ sessionId: "sess-fresh" }); + assert.equal(headers["x-bili-plugin-model"], "fresh-model"); + assert.equal(headers["x-bili-plugin-context-window"], "131072"); + }); +}); + +test("a hung resolver delays only a model's first dispatch, and only by the bound", async () => { + const hung: Resolve = (_provider, model) => model === "aux-model" ? new Promise(() => {}) : Promise.resolve({ context: { contextWindow: WINDOWS[model] } }); + await withDsh(hung, "main-model", async (dispatch) => { + let t0 = Date.now(); + const first = await dispatch({ sessionId: "sess-hung", provider: "deepseek", model: "aux-model" }); + const firstMs = Date.now() - t0; + assert.equal(first["x-bili-plugin-context-window"], undefined); + assert.ok(firstMs >= 1_500 && firstMs < 5_000, `first dispatch waited ${firstMs}ms`); + t0 = Date.now(); + await dispatch({ sessionId: "sess-hung", provider: "deepseek", model: "aux-model" }); + assert.ok(Date.now() - t0 < 500, "a later dispatch does not wait on the hung resolve again"); + }); +});