diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index 8c95387313..ba0f627d1a 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -66,6 +66,21 @@ function retryIntervalMs(): number { return Number.isFinite(parsed) && parsed > 0 ? parsed : RETRY_INTERVAL_MS; } +// #3380 P2: a registration-failure streak doubles the back-off per step up to +// a hard cap — the incident retried an identical doomed failure on a flat 10s +// wall 145 times in an hour while the user saw nothing change. The cap (not a +// stop) keeps the loop alive for transient faults: a network blip or a slow +// manifest must still heal without user action. +const REG_RETRY_MAX_MS = 5 * 60 * 1000; +// #3380 P2: after this many consecutive failures, say it once in the user's +// face with the recovery action — silence is how the incident stayed hidden. +const REG_GIVE_UP_NOTICE_STREAK = 5; +/** #3380: capped exponential back-off for the registration-failure streak. */ +function regBackoffMs(failStreak: number): number { + const steps = Math.max(0, Math.min(failStreak - 1, 16)); + return Math.min(retryIntervalMs() * 2 ** steps, REG_RETRY_MAX_MS); +} + type AgentLike = { session?: { id?: unknown; @@ -174,13 +189,31 @@ export function planNativeDsh(env: NodeJS.ProcessEnv): { mode: "off" | "attach" const state: NativeInterceptState = { origin: undefined, ready: Promise.resolve(undefined) }; -// One registration lifecycle per process. toolsReady is the ONLY gate for -// header stamping (pi.ts discipline): stamped headers flip the proxy into +// One registration lifecycle per HOST CONTEXT. toolsReady is the ONLY gate +// for header stamping (pi.ts discipline): stamped headers flip the proxy into // plugin mode, which suppresses wire tool injection — stamping before the // local tools exist would send a tool-less request. -type RegisterState = { base: string | undefined; toolsReady: boolean; dead: boolean; retryAt: number; pending: Promise | undefined }; +// #3380: registrations live in the host process's tool layer and die ONLY +// with their context (the apply() re-arm below is the sole reset). Proxy +// death merely moves the origin — the execute closures read the LIVE base +// (#983) — so proxy-death paths must NOT clear toolsReady: doing so forced a +// doomed re-registration that DSH rejects ("tool X is already registered"), +// locked the latch false forever, and silently stripped every request of its +// identity headers (wire mode → anonymous zero-fold ledgers → 413 on large +// sessions). +type RegisterState = { + base: string | undefined; + toolsReady: boolean; + dead: boolean; + retryAt: number; + pending: Promise | undefined; + // #3380 P2: consecutive-failure bookkeeping — capped exponential back-off + // (regBackoffMs) + the one-time "restart dsh" notice. + failStreak: number; + lastRegistrationError: string | undefined; +}; -const register: RegisterState = { base: undefined, toolsReady: false, dead: false, retryAt: 0, pending: undefined }; +const register: RegisterState = { base: undefined, toolsReady: false, dead: false, retryAt: 0, pending: undefined, failStreak: 0, lastRegistrationError: undefined }; // #1797: attach/recovery chains still in flight after apply() returned — a late // chain would clobber the shared register mid-run of whatever executes next. @@ -199,6 +232,10 @@ function trackChain(p: Promise): Promise { // single time even though apply() may run again after context re-arming. let webProfileWarned = false; +// #3380: once-per-process — the idempotent-adoption convergence notice must +// not repeat per attempt (same discipline as webProfileWarned). +let regAdoptLogged = false; + /** #1590/#1809: origin of the proxy as currently reachable — register.base * once bound (attach synchronously, spawn after bootstrap), else the preset * env origin. Shared by the index-inject row and the live /bili/origin route. */ @@ -520,9 +557,10 @@ async function verifyAttachAndRecover(attachOrigin: string): Promise { persistClientEvent("proxy respawn gave up — model traffic goes direct (uncompressed)"); + // #3380: only the origin dies here — the host-context tool + // registrations survive, so toolsReady stays put. register.base = undefined; - register.toolsReady = false; }; const landed = start().then((origin) => { if (origin === undefined) { if (landingOwnsRegister(attachOrigin)) { register.base = undefined; - register.toolsReady = false; // A back-off armed against the stale base must die with it: // the unfreeze is a one-shot transition to a fresh base-less // world and must not inherit the old base's 10s wall (#1783). @@ -613,8 +651,27 @@ async function registerTools(ctx: PluginContext): Promise { // flight — a re-activation mid-fetch must not register into (or die // on) the deactivated context captured at call time. const target = activeCtx ?? ctx; - for (const tool of tools) target.tools.register(toolDefinition(tool)); + let adopted = 0; + for (const tool of tools) { + try { + target.tools.register(toolDefinition(tool)); + } catch (err) { + // #3380: idempotent adoption — a name this process ALREADY + // holds in the host tool layer throws "already registered"; + // that registration is exactly what we want, so skip it and + // register the rest (also converges a partial registration + // left by an earlier interrupted flight). + if (!errMessage(err).includes("already registered")) throw err; + adopted += 1; + } + } + if (adopted > 0 && !regAdoptLogged) { + regAdoptLogged = true; + persistClientEvent(`tool registration adopted ${adopted} pre-existing registration(s) from this process — continuing`); + } register.toolsReady = true; + register.failStreak = 0; + register.lastRegistrationError = undefined; })() .catch((err: unknown) => { // Cordis deactivates the plugin context while the host tears down @@ -625,12 +682,22 @@ async function registerTools(ctx: PluginContext): Promise { register.dead = true; return; } + // #3380 P2: consecutive failures escalate — capped exponential + // back-off (a flat wall retried the incident's identical doomed + // failure 145×/h) and a one-time loud notice with the recovery + // action. Retries NEVER stop: a transient fault must still heal + // (#983/#2082 — permanent non-recovery was the original bug class). + register.failStreak += 1; + register.lastRegistrationError = errMessage(err); + if (register.failStreak === REG_GIVE_UP_NOTICE_STREAK) { + persistClientEvent(`bili tools still unavailable after ${register.failStreak} failed registrations (last: ${register.lastRegistrationError}) — model requests ride bili WITHOUT identity headers (wire mode: no compression, anonymous sessions). Restart dsh to restore; retries continue in the background.`); + } // Scope the back-off to the base that actually failed: if the // register moved on while this manifest fetch was in flight (the // apply chain unfroze a dead preset, maybeRetry healed elsewhere), // an armed retryAt would gate the NEXT base's first heal behind a // 10s wall — exactly the windows-22 CI deadlock (#1783). - if (register.base === base) register.retryAt = Date.now() + retryIntervalMs(); + if (register.base === base) register.retryAt = Date.now() + regBackoffMs(register.failStreak); persistClientEvent(`manifest registration failed (${errMessage(err)}) — retrying; requests stay in wire mode until it succeeds`); }) .finally(() => { @@ -679,8 +746,12 @@ function armRecoveryTimer(): void { function maybeRetry(ctx: PluginContext): void { activeCtx = ctx; - if (!register.dead && !register.toolsReady) armRecoveryTimer(); - if (register.dead || register.toolsReady) return; + // #3380: the base-less state heals through the RESPAWN branch below even + // when toolsReady is true (registrations survive proxy death) — gating on + // toolsReady alone would strand a given-up register with no driver left. + if (!register.dead && (register.base === undefined || !register.toolsReady)) armRecoveryTimer(); + if (register.dead) return; + if (register.base !== undefined && register.toolsReady) return; if (register.pending !== undefined) return; if (Date.now() < register.retryAt) return; if (register.base === undefined) { @@ -822,12 +893,24 @@ function latestSessionNote(status: Record, requestedSid: string async function statusOutcome(ctx: PluginContext, invocation?: CommandInvocation): Promise { const base = register.base; if (!base) { + // #3380 P3: toolsReady implies a proxy WAS serving this session — a + // missing origin is then a transient outage with recovery running, + // not a missing install. return { kind: "error", - text: "bili: no proxy detected — install via `bili plugin install dsh` or launch through `bili dsh`.", + text: register.toolsReady + ? "bili: proxy temporarily unreachable — it was serving this session before and recovery is running in the background; compression resumes automatically when it answers." + : "bili: no proxy detected — install via `bili plugin install dsh` or launch through `bili dsh`.", }; } maybeRetry(ctx); + // #3380 P3: wire-mode exposure — the proxy answers but the local tools are + // down, so every model request rides bili WITHOUT identity headers + // (anonymous zero-fold sessions, no compression). Say so in the panel the + // user already looks at instead of letting it stay silent. + const wireNote = !register.toolsReady && !register.dead + ? `⚠️ bili tools NOT registered — model requests ride bili WITHOUT identity headers (wire mode: no compression, anonymous sessions). Last error: ${register.lastRegistrationError ?? "unknown"}. Restart dsh to restore; retries continue in the background.\n\n` + : ""; // #1677: the command executor hands us the invoking session via the invocation — // DSH never establishes the AsyncLocalStorage boundary on the command path, so // currentInitiator() is always empty here and relying on it made /acp silently @@ -844,7 +927,7 @@ async function statusOutcome(ctx: PluginContext, invocation?: CommandInvocation) if (status && typeof panel === "string" && panel.length > 0) { // A panel reached through the latest-fallback belongs to some OTHER session — // say so instead of presenting foreign data as the caller's own (#1677). - return { kind: "success", text: fellBackToLatest ? `${latestSessionNote(status, sid)}\n\n${panel}` : panel }; + return { kind: "success", text: `${wireNote}${fellBackToLatest ? `${latestSessionNote(status, sid)}\n\n` : ""}${panel}` }; } // #955: pre-first-request view — the proxy answers from the runtime-info // table this plugin populated at bootstrap, so /acp shows the client's @@ -858,14 +941,14 @@ async function statusOutcome(ctx: PluginContext, invocation?: CommandInvocation) const version = await fetchProxyVersion(base); return { kind: "success", - text: `billion-context${version ? `@${version}` : ""} — proxy connected, compression armed. Runtime info${typeof ri.source === "string" ? ` (${ri.source})` : ""}: ${parts.join(" ")}. No model request yet; send one, then run /acp again for the full panel.`, + text: `${wireNote}billion-context${version ? `@${version}` : ""} — proxy connected, compression armed. Runtime info${typeof ri.source === "string" ? ` (${ri.source})` : ""}: ${parts.join(" ")}. No model request yet; send one, then run /acp again for the full panel.`, }; } const version = await fetchProxyVersion(base); if (version) { return { kind: "success", - text: `billion-context@${version} — proxy connected, compression armed. No model request seen yet; send one, then run /acp again.`, + text: `${wireNote}billion-context@${version} — proxy connected, compression armed. No model request seen yet; send one, then run /acp again.`, }; } return { @@ -960,6 +1043,12 @@ export function apply(ctx: PluginContext): void { register.retryAt = 0; } register.dead = false; + // #3380: a fresh activation starts a clean registration cycle — stale + // failure bookkeeping from the previous context must not carry over + // (back-off would resume mid-ladder, the notice would stay spent). + register.failStreak = 0; + register.lastRegistrationError = undefined; + regAdoptLogged = false; activeCtx = ctx; // #1590: the dsh web-profile settings panel shows a "bili设置" entry @@ -1039,8 +1128,9 @@ export function apply(ctx: PluginContext): void { state.respawn = start; state.onGiveUp = () => { persistClientEvent("proxy respawn gave up — model traffic goes direct (uncompressed)"); + // #3380: only the origin dies here — the host-context tool + // registrations survive, so toolsReady stays put. register.base = undefined; - register.toolsReady = false; }; // #1365: late routed evidence — if model traffic later arrives baked // against a DIFFERENT origin than the one we attached to, rebind the @@ -1066,8 +1156,9 @@ export function apply(ctx: PluginContext): void { state.respawn = start; state.onGiveUp = () => { persistClientEvent("proxy respawn gave up — model traffic goes direct (uncompressed)"); + // #3380: only the origin dies here — the host-context tool + // registrations survive, so toolsReady stays put. register.base = undefined; - register.toolsReady = false; }; state.ready = trackChain(start()); } @@ -1533,6 +1624,9 @@ export function _resetRegisterForTest(base: string | undefined): void { register.dead = false; register.retryAt = 0; register.pending = undefined; + register.failStreak = 0; + register.lastRegistrationError = undefined; + regAdoptLogged = false; modelInfo.byKey.clear(); modelInfo.refreshingKeys.clear(); modelInfo.retryAtByKey.clear(); @@ -1582,6 +1676,22 @@ export function _stateRespawnForTest(): (() => Promise) | un return state.respawn; } +/** Test hook (#3380): expose the armed give-up seam so a suite can fire the + * exact callback the interceptor's death branch drives. */ +export function _stateOnGiveUpForTest(): (() => void) | undefined { + return state.onGiveUp; +} + +/** Test hook (#3380): read-only snapshot of the registration lifecycle. */ +export function _registerStateForTest(): { base: string | undefined; toolsReady: boolean; dead: boolean; retryAt: number; failStreak: number; lastRegistrationError: string | undefined } { + return { base: register.base, toolsReady: register.toolsReady, dead: register.dead, retryAt: register.retryAt, failStreak: register.failStreak, lastRegistrationError: register.lastRegistrationError }; +} + +/** Test hook (#3380): the capped exponential registration back-off schedule. */ +export function _regBackoffForTest(failStreak: number): number { + return regBackoffMs(failStreak); +} + /** Test hook (#1365): record routed-channel evidence exactly as the fetch * patch would (the patch is not installed under NODE_TEST_CONTEXT). */ export function _noteRoutedForTest(url: string): void { diff --git a/tests/issue3380-dsh-rereg-selflock.test.ts b/tests/issue3380-dsh-rereg-selflock.test.ts new file mode 100644 index 0000000000..4f956e3d5b --- /dev/null +++ b/tests/issue3380-dsh-rereg-selflock.test.ts @@ -0,0 +1,395 @@ +// #3380: plugin tool registration state self-lock. +// +// Incident (bili v0.1.197, DSH desktop): a healthy session lost its proxy at +// runtime; the respawn gave up once (onGiveUp), which cleared BOTH +// register.base AND register.toolsReady. Tools had been registered into the +// host process's tool layer hours earlier and survived the proxy death — so +// every later re-registration attempt threw DSH's duplicate error +// ('tool "compress" is already registered'), the retry loop burned the same +// doomed failure 145×/h on a flat 10s wall, toolsReady stayed false forever, +// headersFor stopped stamping x-bili-plugin*, and every model request rode +// bili anonymously (wire mode → zero-fold pfa-* ledgers → 413 +// preflight_compress_failed on large sessions) until the host restarted. +// +// Root cause: conflation of two lifecycles — tool registrations belong to the +// HOST CONTEXT (they die only with their context; execute closures read the +// live base, #983), while the proxy-death paths treated them as proxy-owned. +// Fix under test: +// 1. proxy-death paths no longer clear toolsReady (identity survives a +// respawn; NO doomed re-registration ever runs again); +// 2. registration is idempotent — a name this process already holds is +// adopted, not thrown on (converges partial/dirty states); +// 3. consecutive failures escalate: capped exponential back-off + one loud +// notice with the recovery action (retries never stop — #983/#2082); +// 4. /acp exposes wire-mode identity loss instead of staying silent. + +import test, { afterEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import { + apply, + maybeRetryForTest, + _resetRegisterForTest, + _setSpawnForTest, + _settleNativeForTest, + _stateHeadersForTest, + _stateOnGiveUpForTest, + _registerStateForTest, + _regBackoffForTest, + type PluginContext, +} from "../src/agent/dsh-native.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// The node:test runner sets NODE_TEST_CONTEXT itself; set defensively so a +// standalone file run behaves identically (dsh-native reads it lazily). +process.env.NODE_TEST_CONTEXT = process.env.NODE_TEST_CONTEXT ?? "1"; +process.env.BILI_PERSIST = "0"; + +afterEach(() => { void _settleNativeForTest(); }); + +type RegisteredTool = { + name: string; + description?: string; + parameters: unknown; + output: { schema: unknown; render: (args: unknown, value: unknown) => Array<{ type: string; text: string }> }; + execute: (args: Record, exec: { agent?: { session?: { id?: unknown } }; signal?: AbortSignal }) => Promise; +}; +type CommandSpec = { + name: string; + description?: string; + handler: (invocation?: { agent?: { session?: { id?: unknown } } }) => Promise<{ kind: string; text: string }>; +}; + +// DSH's root tool layer: ONE registry per host process. Re-registering a name +// this process already holds throws the verbatim incident error — this is +// exactly what turned #3380's post-respawn re-registration into a self-lock. +const rootLayer = new Map(); +const DSH_DUP_ERROR = (name: string): Error => + new Error(`tool "${name}" is already registered (for a per-agent variant, register through that agent's \`agent.ctx\` instead)`); + +function resetRootLayer(): void { + rootLayer.clear(); +} + +async function startMockProxy(toolNames: string[]): Promise<{ origin: string; close: () => void; manifestRequests: () => number }> { + const counters = { manifestRequests: 0 }; + const server = http.createServer((req, res) => { + const url = req.url ?? ""; + if (url === "/__bili/plugin/manifest") { + counters.manifestRequests += 1; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + version: "0.1.119", + tools: { anthropic: toolNames.map((name) => ({ name, description: `${name} (test)`, input_schema: { type: "object", properties: {} } })) }, + })); + return; + } + res.writeHead(404); + res.end("{}"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + const addr = server.address() as { port: number }; + return { + origin: `http://127.0.0.1:${addr.port}`, + close: () => { server.close(); server.closeAllConnections?.(); }, + manifestRequests: () => counters.manifestRequests, + }; +} + +type HostCtx = Parameters[0]; + +function mockCtx(opts: { registerError?: () => Error | undefined } = {}) { + const tools: RegisteredTool[] = []; + const commands: CommandSpec[] = []; + let initiator: { session?: { id?: unknown } } | undefined = undefined; + return { + tools: { + register: (t: RegisteredTool) => { + const err = opts.registerError?.(); + if (err !== undefined) throw err; + if (rootLayer.has(t.name)) throw DSH_DUP_ERROR(t.name); + rootLayer.set(t.name, true); + tools.push(t); + }, + }, + commands: { register: (c: CommandSpec) => commands.push(c) }, + agents: { currentInitiator: () => initiator }, + setInitiator: (i: { session?: { id?: unknown } } | undefined) => (initiator = i), + registeredTools: tools, + registeredCommands: commands, + inject: (_deps: readonly string[], _cb: (sub: HostCtx) => void) => {}, + }; +} + +function mockCtxTyped(ctx: ReturnType): PluginContext { + return ctx as unknown as PluginContext; +} + +async function withEnv(env: Record, fn: () => Promise | T): Promise { + const saved: Record = {}; + for (const k of Object.keys(env)) { + saved[k] = process.env[k]; + const v = env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + return await fn(); + } finally { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +async function waitFor(cond: () => boolean, what: string): Promise { + const deadline = Date.now() + 15000; + while (!cond()) { + if (Date.now() > deadline) throw new Error(`timeout waiting for ${what}`); + await new Promise((r) => setTimeout(r, 10)); + } +} + +function logFileOf(home: string): string { + return path.join(home, "billion-context", "bili.log"); +} + +test("#3380 incident repro: identity survives proxy death + respawn — no doomed re-registration ever runs", async () => { + const proxyA = await startMockProxy(["compress", "decompress"]); + const proxyB = await startMockProxy(["compress", "decompress"]); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-incident-")); + const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-incident-state-")); + try { + // grace window shrunk so the dead-attach probe falls through to the + // spawn seam quickly instead of waiting the production evidence grace + await withEnv({ DSH_HOME: home, BILLION_CONTEXT_PROXY: proxyA.origin, BILI_ATTACH_EVIDENCE_GRACE_MS: "50", XDG_STATE_HOME: stateHome }, async () => { + _resetRegisterForTest(undefined); + resetRootLayer(); + const raw = mockCtx(); + const ctx = mockCtxTyped(raw); + raw.setInitiator({ session: { id: "i3380" } }); + apply(ctx); + await waitFor(() => rootLayer.size === 2, "initial registration"); + const h1 = _stateHeadersForTest()?.("http://example.test/v1/messages"); + assert.equal(h1?.["x-bili-plugin"], "dsh", "phase 1: identity stamped"); + assert.equal(h1?.["x-bili-plugin-conversation"], "i3380"); + + // Phase 2: the proxy dies and its respawn gives up. Old code + // cleared toolsReady here — the self-lock's ignition. + proxyA.close(); + const giveUp = _stateOnGiveUpForTest(); + assert.ok(giveUp !== undefined, "give-up seam armed by the attach plan"); + giveUp!(); + await _settleNativeForTest(); + assert.equal(_registerStateForTest().base, undefined, "the origin is gone"); + assert.equal(_registerStateForTest().toolsReady, true, "the latch SURVIVES proxy death (registrations belong to the host context)"); + + // Phase 3: a NEW origin lands through the respawn seam. + _setSpawnForTest(async () => proxyB.origin); + try { + maybeRetryForTest(ctx); + await waitFor(() => _registerStateForTest().base === proxyB.origin, "respawn landing on the new origin"); + assert.equal(proxyB.manifestRequests(), 0, "NO doomed re-registration flight ran against the new proxy"); + assert.equal(rootLayer.size, 2, "no duplicate registration reached the host tool layer"); + const h2 = _stateHeadersForTest()?.("http://example.test/v1/messages"); + assert.equal(h2?.["x-bili-plugin"], "dsh", "identity restored on the new origin without re-registration"); + assert.equal(h2?.["x-bili-plugin-conversation"], "i3380", "same conversation identity carried across the respawn"); + } finally { + _setSpawnForTest(undefined); + } + }); + } finally { + proxyA.close(); + proxyB.close(); + rmrf(home); + rmrf(stateHome); + _resetRegisterForTest(undefined); + resetRootLayer(); + } +}); + +test("#3380 idempotent adoption: a pre-existing registration is adopted, the rest still lands", async () => { + const proxy = await startMockProxy(["compress", "decompress"]); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-adopt-")); + const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-adopt-state-")); + try { + await withEnv({ DSH_HOME: home, BILLION_CONTEXT_PROXY: proxy.origin, XDG_STATE_HOME: stateHome }, async () => { + _resetRegisterForTest(undefined); + resetRootLayer(); + // Dirty state: an earlier interrupted flight left "compress" held + // in the process's tool layer. + rootLayer.set("compress", true); + const raw = mockCtx(); + const ctx = mockCtxTyped(raw); + raw.setInitiator({ session: { id: "i3380-adopt" } }); + apply(ctx); + await waitFor(() => _registerStateForTest().toolsReady, "registration converges despite the held name"); + assert.ok(rootLayer.has("compress") && rootLayer.has("decompress"), "both tools present in the host layer"); + assert.equal(raw.registeredTools.length, 1, "only the missing tool was registered fresh"); + const h = _stateHeadersForTest()?.("http://example.test/v1/messages"); + assert.equal(h?.["x-bili-plugin"], "dsh", "plugin mode restored after adoption"); + await waitFor(() => fs.existsSync(logFileOf(stateHome)) && fs.readFileSync(logFileOf(stateHome), "utf8").includes("adopted 1 pre-existing registration"), "adoption convergence logged once"); + }); + } finally { + proxy.close(); + rmrf(home); + rmrf(stateHome); + _resetRegisterForTest(undefined); + resetRootLayer(); + } +}); + +test("#3380 P2 back-off schedule: capped exponential, streak resets on success", async () => { + const saved = process.env.BILI_DSH_RETRY_INTERVAL_MS; + process.env.BILI_DSH_RETRY_INTERVAL_MS = "20"; + try { + assert.equal(_regBackoffForTest(1), 20, "first failure: base interval"); + assert.equal(_regBackoffForTest(2), 40, "second failure: doubled"); + assert.equal(_regBackoffForTest(3), 80, "third failure: doubled again"); + const plateau = _regBackoffForTest(100); + assert.equal(plateau, 5 * 60 * 1000, "hard cap at 5 minutes"); + assert.equal(_regBackoffForTest(101), plateau, "stays capped"); + } finally { + if (saved === undefined) delete process.env.BILI_DSH_RETRY_INTERVAL_MS; + else process.env.BILI_DSH_RETRY_INTERVAL_MS = saved; + } +}); + +test("#3380 P2 integration: failures escalate on the ladder, then heal clean", async () => { + const proxy = await startMockProxy(["compress"]); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-backoff-")); + const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-backoff-state-")); + const savedInterval = process.env.BILI_DSH_RECOVERY_INTERVAL_MS; + const savedRetry = process.env.BILI_DSH_RETRY_INTERVAL_MS; + process.env.BILI_DSH_RECOVERY_INTERVAL_MS = "20"; + process.env.BILI_DSH_RETRY_INTERVAL_MS = "20"; + try { + await withEnv({ DSH_HOME: home, BILLION_CONTEXT_PROXY: proxy.origin, XDG_STATE_HOME: stateHome }, async () => { + _resetRegisterForTest(undefined); + resetRootLayer(); + let failing = true; + const raw = mockCtx({ registerError: () => (failing ? new Error("manifest down (transient)") : undefined) }); + const ctx = mockCtxTyped(raw); + apply(ctx); + // Third consecutive failure arms the third ladder step (4×base). + await waitFor(() => _registerStateForTest().failStreak >= 3, "three consecutive failures"); + const snap = _registerStateForTest(); + assert.equal(snap.lastRegistrationError, "manifest down (transient)", "last error recorded for status surfaces"); + const delta = snap.retryAt - Date.now(); + assert.ok(delta >= 30 && delta <= 95, `retryAt sits on the 80ms ladder step (got ${delta}ms)`); + // Heal: the transient fault clears, the next attempt succeeds, and + // the bookkeeping resets. + failing = false; + await waitFor(() => _registerStateForTest().toolsReady, "heal after the fault clears"); + const healed = _registerStateForTest(); + assert.equal(healed.failStreak, 0, "streak resets on success"); + assert.equal(healed.lastRegistrationError, undefined, "stale error cleared"); + }); + } finally { + for (const [k, v] of [["BILI_DSH_RECOVERY_INTERVAL_MS", savedInterval], ["BILI_DSH_RETRY_INTERVAL_MS", savedRetry]] as const) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + proxy.close(); + rmrf(home); + rmrf(stateHome); + _resetRegisterForTest(undefined); + resetRootLayer(); + } +}); + +test("#3380 P2 notice: exactly one loud restart hint across a long failure streak", async () => { + const proxy = await startMockProxy(["compress"]); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-notice-")); + const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-notice-state-")); + const savedInterval = process.env.BILI_DSH_RECOVERY_INTERVAL_MS; + const savedRetry = process.env.BILI_DSH_RETRY_INTERVAL_MS; + process.env.BILI_DSH_RECOVERY_INTERVAL_MS = "20"; + process.env.BILI_DSH_RETRY_INTERVAL_MS = "20"; + try { + await withEnv({ DSH_HOME: home, BILLION_CONTEXT_PROXY: proxy.origin, XDG_STATE_HOME: stateHome }, async () => { + _resetRegisterForTest(undefined); + resetRootLayer(); + const raw = mockCtx({ registerError: () => new Error("manifest down (persistent)") }); + apply(mockCtxTyped(raw)); + const MARKER = "bili tools still unavailable after"; + await waitFor(() => { + if (_registerStateForTest().failStreak < 7) return false; + const log = fs.existsSync(logFileOf(stateHome)) ? fs.readFileSync(logFileOf(stateHome), "utf8") : ""; + return log.split(MARKER).length - 1 >= 1; + }, "streak past the notice threshold with the notice logged"); + const log = fs.readFileSync(logFileOf(stateHome), "utf8"); + assert.equal(log.split(MARKER).length - 1, 1, "the notice fires exactly once, not per attempt"); + assert.ok(log.includes("Restart dsh to restore"), "carries the recovery action"); + assert.ok(log.includes("manifest down (persistent)"), "carries the last error"); + }); + } finally { + for (const [k, v] of [["BILI_DSH_RECOVERY_INTERVAL_MS", savedInterval], ["BILI_DSH_RETRY_INTERVAL_MS", savedRetry]] as const) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + proxy.close(); + rmrf(home); + rmrf(stateHome); + _resetRegisterForTest(undefined); + resetRootLayer(); + } +}); + +test("#3380 P3 /acp: wire-mode identity loss is exposed, not silent", async () => { + const proxy = await startMockProxy(["compress"]); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-acp-")); + const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-3380-acp-state-")); + const savedInterval = process.env.BILI_DSH_RECOVERY_INTERVAL_MS; + const savedRetry = process.env.BILI_DSH_RETRY_INTERVAL_MS; + process.env.BILI_DSH_RECOVERY_INTERVAL_MS = "20"; + process.env.BILI_DSH_RETRY_INTERVAL_MS = "20"; + try { + await withEnv({ DSH_HOME: home, BILLION_CONTEXT_PROXY: proxy.origin, XDG_STATE_HOME: stateHome }, async () => { + _resetRegisterForTest(undefined); + resetRootLayer(); + let failing = true; + const raw = mockCtx({ registerError: () => (failing ? new Error("manifest down (transient)") : undefined) }); + const ctx = mockCtxTyped(raw); + apply(ctx); + const acp = raw.registeredCommands.find((c) => c.name === "acp"); + assert.ok(acp !== undefined, "acp command registered"); + const invocation = { agent: { session: { id: "i3380-acp" } } }; + + // (a) proxy answers but tools are down → wire-mode warning. + await waitFor(() => _registerStateForTest().failStreak >= 1, "first registration failure"); + const wire = await acp!.handler(invocation); + assert.equal(wire.kind, "success"); + assert.ok(wire.text.includes("WITHOUT identity headers (wire mode"), "wire-mode loss named explicitly"); + assert.ok(wire.text.includes("manifest down (transient)"), "last error surfaced"); + assert.ok(wire.text.includes("Restart dsh to restore"), "recovery action surfaced"); + + // (b) heal, then lose the proxy entirely → recovering, not "install". + failing = false; + await waitFor(() => _registerStateForTest().toolsReady, "heal"); + proxy.close(); + const giveUp = _stateOnGiveUpForTest(); + assert.ok(giveUp !== undefined); + giveUp!(); + await _settleNativeForTest(); + const out = await acp!.handler(invocation); + assert.equal(out.kind, "error"); + assert.ok(out.text.includes("proxy temporarily unreachable"), "a previously-serving proxy reads as an outage with recovery running"); + assert.ok(!out.text.includes("no proxy detected — install"), "not misread as a missing install"); + }); + } finally { + for (const [k, v] of [["BILI_DSH_RECOVERY_INTERVAL_MS", savedInterval], ["BILI_DSH_RETRY_INTERVAL_MS", savedRetry]] as const) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + proxy.close(); + rmrf(home); + rmrf(stateHome); + _resetRegisterForTest(undefined); + resetRootLayer(); + } +});