Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 35 additions & 10 deletions src/agent/dsh-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -529,12 +529,15 @@ function modelInfoRetryCooldownMs(): number {
const parsed = raw === undefined ? Number.NaN : Number(raw);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : MODEL_INFO_RETRY_COOLDOWN_MS;
}
// How long after a resolve starts a dispatch may wait on it before its first
// fetch; a slow or hung resolver must never stall the stream beyond this.
const MODEL_INFO_DISPATCH_WAIT_MS = 2_000;
const modelInfo: {
byKey: Map<string, ModelInfoEntry>;
refreshingKeys: Set<string>;
refreshingKeys: Map<string, Promise<void>>;
retryAtByKey: Map<string, number>;
services?: { llm?: PluginContext["llm"]; agentDefaultModel?: PluginContext["agentDefaultModel"] };
} = { byKey: new Map(), refreshingKeys: new Set(), retryAtByKey: new Map() };
} = { byKey: new Map(), refreshingKeys: new Map(), retryAtByKey: new Map() };

function modelInfoKey(provider: string, model: string): string {
return `${provider}\u0000${model}`;
Expand All @@ -559,8 +562,12 @@ function normalizeModelTarget(provider: unknown, model: unknown): { provider: st
// #2381: the model THIS request routes to — the session's own live header
// first (exact at fetch time, correct across per-session model switches), the
// host's global default selection only as fallback. Any miss degrades to
// pre-#2381 behavior; it never yields a wrong model.
// pre-#2381 behavior; it never yields a wrong model. Inside an llm/stream
// dispatch the model that dispatch itself streams to wins (auxiliary calls
// may run on a different model than the session's).
function sessionModelTargetOf(ctx: PluginContext): { provider: string; model: string } | undefined {
const scoped = dispatchScope.getStore()?.modelTarget;
if (scoped !== undefined) return scoped;
try {
const init = ctx.agents?.currentInitiator?.();
const header = init?.session?.requestHeader?.();
Expand Down Expand Up @@ -618,7 +625,10 @@ export function bodyOverWindowDetail(
return sharedBodyOverWindowDetail(body?.bytes, rawBodyStringOf(init), declaredWindow);
}

function refreshModelInfo(origin: string | undefined, target?: { provider: string; model: string }): void {
// Returns a wait handle for the target's in-flight resolve (settles when it
// lands, or MODEL_INFO_DISPATCH_WAIT_MS after it started), undefined when
// there is nothing to wait for.
function refreshModelInfo(origin: string | undefined, target?: { provider: string; model: string }): Promise<void> | undefined {
const svc = modelInfo.services;
if (svc === undefined) return;
const sel = target ?? globalModelTarget(svc);
Expand All @@ -633,7 +643,7 @@ function refreshModelInfo(origin: string | undefined, target?: { provider: strin
if (cached !== undefined && cached.contextWindow !== undefined) return;
const retryAt = modelInfo.retryAtByKey.get(key);
if (cached !== undefined && retryAt !== undefined && Date.now() < retryAt) return;
if (modelInfo.refreshingKeys.has(key)) return;
if (modelInfo.refreshingKeys.has(key)) return modelInfo.refreshingKeys.get(key);
// The host registers llm as a service object, so resolveModelInfo needs its
// receiver; called detached below, this threw
// "Cannot read properties of undefined (reading 'resolveModelInfoFor')"
Expand All @@ -645,9 +655,8 @@ function refreshModelInfo(origin: string | undefined, target?: { provider: strin
evictModelInfoEntries();
return;
}
modelInfo.refreshingKeys.add(key);
modelInfo.retryAtByKey.set(key, Date.now() + modelInfoRetryCooldownMs());
void Promise.resolve()
const flight = Promise.resolve()
.then(() => resolve(provider, model))
.then((info) => {
// Per-key commit (#2381): the entry belongs to its OWN key, so a
Expand Down Expand Up @@ -687,6 +696,9 @@ function refreshModelInfo(origin: string | undefined, target?: { provider: strin
}).catch(() => {});
}
});
const waitable = Promise.race([flight, new Promise<void>((done) => { setTimeout(done, MODEL_INFO_DISPATCH_WAIT_MS).unref?.(); })]);
modelInfo.refreshingKeys.set(key, waitable);
return waitable;
}

function errMessage(err: unknown): string {
Expand Down Expand Up @@ -1252,6 +1264,7 @@ interface LlmDispatchScope {
sid: string;
msgIds: string[] | undefined;
purpose?: string;
modelTarget?: { provider: string; model: string };
}
const dispatchScope = new AsyncLocalStorage<LlmDispatchScope>();

Expand Down Expand Up @@ -2084,7 +2097,7 @@ export async function apply(ctx: PluginContext): Promise<void> {
if (typeof ctx.inject === "function" && typeof ctx.on === "function") {
const onEvent = ctx.on.bind(ctx);
ctx.inject(["llm"], () => {
onEvent("llm/stream", (options: { sessionId?: unknown; messages?: unknown; purpose?: unknown }, next: () => AsyncIterable<unknown>) => {
onEvent("llm/stream", (options: { sessionId?: unknown; messages?: unknown; purpose?: unknown; provider?: unknown; model?: unknown }, next: () => AsyncIterable<unknown>) => {
const sid = typeof options?.sessionId === "string" && options.sessionId.length > 0 ? options.sessionId : undefined;
if (sid === undefined) return next();
// #4491: capture the host message ids riding the same options
Expand All @@ -2094,7 +2107,12 @@ export async function apply(ctx: PluginContext): Promise<void> {
// (dsh sets it on every auxiliary call; conversation leaves it
// unset). Absent/odd values simply stay off the turn.
const purpose = typeof options?.purpose === "string" && options.purpose.length > 0 && options.purpose.length <= 64 && /^[\x21-\x7e]+$/.test(options.purpose) ? options.purpose : undefined;
const scope: LlmDispatchScope = { sid, msgIds: hostMsgIdsOf(options?.messages), purpose };
// Size the dispatch against the model IT streams to (auxiliary
// calls may use another model than the session header names).
// GenerateOptions declares provider + model at the top level.
const modelTarget = normalizeModelTarget(options?.provider, options?.model)
?? sessionModelTargetOf(ctx) ?? globalModelTarget(modelInfo.services);
const scope: LlmDispatchScope = { sid, msgIds: hostMsgIdsOf(options?.messages), purpose, modelTarget };
// #5403/#5418 v3: feed the per-slot id sequence into the
// watermark channel (the fetch chokepoint emits count +
// tail-ids; core = pf-channel.ts) — "?" slots ride id-less
Expand All @@ -2108,7 +2126,14 @@ export async function apply(ctx: PluginContext): Promise<void> {
// latter; each also clears inherited fetch-chain re-entry
// state so a new operation never looks like mid-descent
// re-entry (#5614).
return runLlmDispatch(scope, () => wrapScopedIterable(next(), scope));
// A model's first dispatch would otherwise fetch before its
// window resolved and go out without it; wait (bounded) first.
const pending = refreshModelInfo(register.base ?? state.origin, modelTarget);
const stream: () => AsyncIterable<unknown> = pending === undefined ? next : async function* () {
await pending;
yield* next();
};
return runLlmDispatch(scope, () => wrapScopedIterable(stream(), scope));
});
});
}
Expand Down
134 changes: 134 additions & 0 deletions tests/dsh-dispatch-model-window.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/**
* dsh llm/stream dispatches are sized against the model THEY stream to, and a
* model's first dispatch carries its window: the window header used to come
* from the session header's model (wrong for auxiliary calls on another
* model) and from a fire-and-forget resolve (absent on a model's first fetch).
*/
import { strict as assert } from "node:assert";
import { mkdtempSync, rmSync } from "node:fs";
import { createServer, type Server } from "node:http";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";

delete process.env.BILI_PROVIDER_REWRITES;
delete process.env.BILI_NATIVE_DSH;
delete process.env.BILLION_CONTEXT_PLUGIN;

import { apply, _resetRegisterForTest, _settleNativeForTest, _stateHeadersForTest, type PluginContext } from "../src/agent/dsh-native.ts";
import { __resetConversationRecallForTests } from "../src/agent/conversation-recall.ts";

const MODEL_URL = "https://api.anthropic.com/v1/messages";
type Listener = (payload: unknown, next: () => unknown) => unknown;
type Resolve = (provider: string, model: string) => Promise<{ context?: { contextWindow?: number } } | undefined>;

async function startMockProxy(): Promise<{ origin: string; server: Server }> {
const server = createServer((req, res) => {
res.writeHead(req.url === "/__bili/plugin/manifest" || req.url === "/__bili/plugin/session-name" ? 200 : 404, { "content-type": "application/json" });
res.end(JSON.stringify(req.url === "/__bili/plugin/manifest"
? { version: "0.1.119", tools: { anthropic: [{ name: "compress", description: "c", input_schema: { type: "object", properties: {} } }] } }
: {}));
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (address === null || typeof address !== "object") throw new Error("no port");
return { origin: `http://127.0.0.1:${address.port}`, server };
}

async function withDsh(resolve: Resolve, sessionModel: string, fn: (dispatch: (options: Record<string, unknown>) => Promise<Record<string, string>>) => Promise<void>): Promise<void> {
__resetConversationRecallForTests();
const home = mkdtempSync(join(tmpdir(), "dsh-dispatch-model-"));
const saved = { BILLION_CONTEXT_PROXY: process.env.BILLION_CONTEXT_PROXY, DSH_HOME: process.env.DSH_HOME };
const proxy = await startMockProxy();
process.env.BILLION_CONTEXT_PROXY = proxy.origin;
process.env.DSH_HOME = home;
_resetRegisterForTest(proxy.origin);
const listeners: Listener[] = [];
const registered: unknown[] = [];
const services = { llm: { resolveModelInfo: resolve }, agentDefaultModel: { currentSelection: () => ({ provider: "deepseek", model: "main-model" }) } };
const ctx = {
inject: (deps: readonly string[], cb: (sub: unknown) => void) => { if (deps.includes("llm")) cb(services); },
on: (name: string, listener: Listener) => { if (name === "llm/stream") listeners.push(listener); },
emit: () => undefined,
logger: { info: () => undefined, warn: () => undefined, error: () => undefined },
tools: { register: (tool: unknown) => void registered.push(tool) },
commands: { register: () => undefined },
agents: { currentInitiator: () => ({ session: { id: "ambient", requestHeader: () => ({ config: { provider: "deepseek", model: sessionModel } }) } }) },
} as unknown as PluginContext;
// One adapter fetch per dispatch: the headers stamped on its FIRST request.
const dispatch = async (options: Record<string, unknown>): Promise<Record<string, string>> => {
let stamped: Record<string, string> = {};
const base = async function* (): AsyncIterable<number> {
stamped = _stateHeadersForTest()?.(MODEL_URL) ?? {};
yield 1;
};
const composed = listeners.reduceRight<Listener>((acc, l) => (p, n) => l(p, () => acc(p, n)), () => base());
for await (const _ of composed(options, () => base()) as AsyncIterable<number>) { /* drain */ }
return stamped;
};
try {
await apply(ctx);
const deadline = Date.now() + 15_000;
while (registered.length === 0) {
if (Date.now() > deadline) throw new Error("timeout waiting for tool registration");
await new Promise((r) => setTimeout(r, 20));
}
await fn(dispatch);
} finally {
await _settleNativeForTest();
proxy.server.close();
proxy.server.closeIdleConnections?.();
for (const [key, value] of Object.entries(saved)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
rmSync(home, { recursive: true, force: true });
_resetRegisterForTest(undefined);
}
}

const WINDOWS: Record<string, number> = { "main-model": 1_000_000, "aux-model": 65_536, "fresh-model": 131_072 };
// The session's own model resolves at boot; every other model resolves late.
const slowResolve: Resolve = (_provider, model) => model === "main-model"
? Promise.resolve({ context: { contextWindow: WINDOWS[model] } })
: new Promise((r) => setTimeout(() => r({ context: { contextWindow: WINDOWS[model] } }), 100));

test("auxiliary dispatch on another model stamps THAT model's window on its first request", async () => {
await withDsh(slowResolve, "main-model", async (dispatch) => {
const deadline = Date.now() + 15_000;
while (_stateHeadersForTest()?.(MODEL_URL)?.["x-bili-plugin-context-window"] !== "1000000") {
if (Date.now() > deadline) throw new Error("timeout waiting for the session model's window");
await new Promise((r) => setTimeout(r, 20));
}
const headers = await dispatch({ sessionId: "sess-aux", purpose: "compaction", provider: "deepseek", model: "aux-model" });
assert.equal(headers["x-bili-plugin-conversation"], "sess-aux");
assert.equal(headers["x-bili-plugin-model"], "aux-model");
assert.equal(headers["x-bili-plugin-context-window"], "65536");
// A dispatch on the session's own model keeps that model's window.
const main = await dispatch({ sessionId: "sess-aux", provider: "deepseek", model: "main-model" });
assert.equal(main["x-bili-plugin-model"], "main-model");
assert.equal(main["x-bili-plugin-context-window"], "1000000");
});
});

test("a model's first dispatch waits for its window instead of going out without it", async () => {
await withDsh(slowResolve, "fresh-model", async (dispatch) => {
const headers = await dispatch({ sessionId: "sess-fresh" });
assert.equal(headers["x-bili-plugin-model"], "fresh-model");
assert.equal(headers["x-bili-plugin-context-window"], "131072");
});
});

test("a hung resolver delays only a model's first dispatch, and only by the bound", async () => {
const hung: Resolve = (_provider, model) => model === "aux-model" ? new Promise(() => {}) : Promise.resolve({ context: { contextWindow: WINDOWS[model] } });
await withDsh(hung, "main-model", async (dispatch) => {
let t0 = Date.now();
const first = await dispatch({ sessionId: "sess-hung", provider: "deepseek", model: "aux-model" });
const firstMs = Date.now() - t0;
assert.equal(first["x-bili-plugin-context-window"], undefined);
assert.ok(firstMs >= 1_500 && firstMs < 5_000, `first dispatch waited ${firstMs}ms`);
t0 = Date.now();
await dispatch({ sessionId: "sess-hung", provider: "deepseek", model: "aux-model" });
assert.ok(Date.now() - t0 < 500, "a later dispatch does not wait on the hung resolve again");
});
});
Loading