Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions src/agent/dsh-native-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,16 @@ export const inject = ["slots", "locale"];

const NS = "bili";

// #1809: live-origin probe cadence — first attempt immediate, then a retry
// every POLL_INTERVAL_MS up to POLL_MAX_ATTEMPTS total (~30s of coverage for
// a slow spawn-mode bootstrap, bounded so an absent host costs no more).
// #1809/#2187: live-origin probe cadence — first attempt immediate, then a
// retry every POLL_INTERVAL_MS for POLL_MAX_ATTEMPTS total (~30s), after
// which probing CONTINUES at SLOW_POLL_INTERVAL_MS until the origin arrives
// or the page unmounts. #2187: the server side now heals a failed spawn-mode
// bootstrap in the background and a slow Windows boot can take minutes, so
// stopping at ~30s stranded the entry on "not bound" for the whole session.
const ORIGIN_PATH = "/bili/origin";
const POLL_INTERVAL_MS = 3000;
const POLL_MAX_ATTEMPTS = 10;
const SLOW_POLL_INTERVAL_MS = 10000;

const zh: Dict = {
"nav": "bili设置",
Expand Down Expand Up @@ -75,9 +79,9 @@ function openExternal(url: string): void {
if (typeof w.open === "function") w.open(url, "_blank", "noopener,noreferrer");
}

/** #1809: poll the host's live origin route until one arrives; returns the
* cancel used as the effect cleanup (no fetch ⇒ no-op, older hosts without
* the route simply stay degraded after the attempts are exhausted). */
/** #1809/#2187: poll the host's live origin route until one arrives; returns
* the cancel used as the effect cleanup (no fetch ⇒ no-op, older hosts
* without the route simply stay degraded). */
function probeOrigin(onOrigin: (origin: string) => void): () => void {
if (typeof fetch !== "function") return () => {};
let cancelled = false;
Expand All @@ -99,7 +103,10 @@ function probeOrigin(onOrigin: (origin: string) => void): () => void {
}
if (resolved) return;
attempts += 1;
if (!cancelled && attempts < POLL_MAX_ATTEMPTS) timer = setTimeout(() => void poll(), POLL_INTERVAL_MS);
// #2187: fast phase for the first POLL_MAX_ATTEMPTS, then an endless
// slow phase — the origin can land at any time (background heal).
const delay = attempts < POLL_MAX_ATTEMPTS ? POLL_INTERVAL_MS : SLOW_POLL_INTERVAL_MS;
if (!cancelled) timer = setTimeout(() => void poll(), delay);
};
void poll();
return () => {
Expand Down
15 changes: 15 additions & 0 deletions src/agent/dsh-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,16 @@ export function apply(ctx: PluginContext): void {
state.ready = trackChain(start());
}

// #2187: arm the recovery loop at PLAN time, not on first traffic. After
// a boot-time spawn failure NO caller can reach maybeRetry — headersFor
// is only consulted once an origin has resolved, /acp early-returns while
// base-less, and the recovery timer below arms itself only from inside
// maybeRetry — so the lane stayed degraded for the whole session. Happy
// path: single-flight joins the bootstrap started above (no second
// spawn); failure path: the timer re-fires state.respawn() every
// RETRY_INTERVAL_MS until a proxy lands.
maybeRetry(ctx);

// #1158 L2: a refusal sends model traffic DIRECT. First refusal per
// endpoint logs once; same-state refusals accumulate silently and re-print
// only on an attribution STATE change (none↔threw), always carrying the
Expand Down Expand Up @@ -1096,6 +1106,11 @@ export function _resetRegisterForTest(base: string | undefined): void {
recoveryTimer = undefined;
}
activeCtx = undefined;
// #2187: apply() now reaches the respawn arming on every plan, so a stale
// single-flight wrapper (capturing a previous test's spawn stub) must not
// survive a reset.
state.respawn = undefined;
state.onGiveUp = undefined;
register.base = base;
register.toolsReady = false;
register.dead = false;
Expand Down
43 changes: 32 additions & 11 deletions src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,20 @@ export type BaseClientName = "claude" | "codex" | "pi" | "omp" | "opencode" | "h
const HEALTH_PATH = "/__bili/health";
const HEALTH_POLL_INTERVAL_MS = 200;
const SPAWN_WAIT_MS = 20000;
// #2187: grace beyond the initial wait for a LIVE child still starting.
// Spawn-window startups have been observed taking 26-45s on slow Windows
// machines (AV / first-run disk activity); the one-shot 20s budget misread
// that as failure and left the lane degraded for the whole session.
const SLOW_START_GRACE_MS = 40_000;
// Total wait for a live child: initial window + grace. A child that EXITS or
// fails to spawn still fails fast (the poll loop breaks on exit/error).
export const SPAWN_BUDGET_MS = SPAWN_WAIT_MS + SLOW_START_GRACE_MS;
const PROBE_TIMEOUT_MS = 1500;
// #707: max age of a starting marker still treated as an in-progress bring-up.
// A well-behaved starter resolves within SPAWN_WAIT_MS; the slack covers slow
// disks and client teardown before it clears the marker.
const STARTING_MARKER_TTL_MS = SPAWN_WAIT_MS + 30_000;
// #707/#2187: max age of a starting marker still treated as an in-progress
// bring-up. A well-behaved starter resolves within SPAWN_BUDGET_MS (a live
// child may take the whole budget); the slack covers slow disks and client
// teardown before it clears the marker.
const STARTING_MARKER_TTL_MS = SPAWN_BUDGET_MS + 30_000;
// #1903: budget for the post-exit re-discovery below. A spawned child dying
// before becoming healthy is evidence the port it wanted is HELD — but our
// one-shot discovery snapshot may predate the holder publishing its identity
Expand Down Expand Up @@ -3439,8 +3448,10 @@ function pickAttachable(
}

/** #707: wait for another launcher's in-flight bring-up to produce a live
* instance. Bounded by SPAWN_WAIT_MS; breaks early when the starting marker
* disappears (starter gave up / crashed). The final probe closes the
* instance. Bounded by SPAWN_BUDGET_MS — the starter's own live-child wait
* has the same envelope (#2187); bailing earlier would double-spawn against
* a still-starting child; breaks early when the starting marker disappears
* (starter gave up / crashed). The final probe closes the
* deadline-boundary sliver: the starter's own poll window ends ~now, and its
* success path clears the marker — indistinguishable from a failure bail
* without one last look. */
Expand All @@ -3455,7 +3466,7 @@ async function waitForStarterInstance(
refusedLog: Set<string>,
diag?: (msg: string) => void,
): Promise<ProxyInstanceFile | undefined> {
const deadline = now() + SPAWN_WAIT_MS;
const deadline = now() + SPAWN_BUDGET_MS;
const probe = async (): Promise<ProxyInstanceFile | undefined> =>
pickAttachable(await probeLiveInstances(readInstance, fetchHealthInfo, diag), opts, codeFingerprint, attachExternal, refusedLog, diag);
let inst: ProxyInstanceFile | undefined;
Expand Down Expand Up @@ -3764,7 +3775,7 @@ export async function ensureProxyRunning(
};
// #1225: an in-flight starter of a DIFFERENT declared lane can never
// produce an instance we may attach to — waiting would only stall this
// launch behind its SPAWN_WAIT_MS window. Undeclared lanes wildcard.
// launch behind its SPAWN_BUDGET_MS window. Undeclared lanes wildcard.
const starterLaneMatches = (m: ProxyStartingMarker): boolean =>
opts.lane === undefined || m.lane === undefined || m.lane === opts.lane;
const marker = readStartingMarker();
Expand Down Expand Up @@ -3876,7 +3887,7 @@ export async function ensureProxyRunning(

// #401/#480: fail fast when OUR spawned child dies before becoming
// healthy — otherwise a startup crash (bad config, missing upstream, …)
// burns the whole SPAWN_WAIT_MS poll window before erroring.
// burns the whole spawn wait budget before erroring.
let childExit: { code: number | null; signal: string | null } | undefined;
// #809/D: an async spawn failure (EACCES/ENOENT on the resolved runtime)
// emits 'error', not 'exit'. Unhandled, it becomes an uncaughtException
Expand All @@ -3895,9 +3906,19 @@ export async function ensureProxyRunning(
childError = rest[0];
});

const deadline = now() + SPAWN_WAIT_MS;
const deadline = now() + SPAWN_BUDGET_MS;
// #2187: announce the phase boundary once — a live child past the
// initial window is still being waited for, and the log is the only
// place to see that the budget was extended instead of failing.
let slowStartNoticed = false;
while (now() < deadline) {
if (childExit || childError !== undefined) break;
if (!slowStartNoticed && now() >= deadline - SLOW_START_GRACE_MS) {
slowStartNoticed = true;
console.error(
`bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (log: ${logPath})`,
);
}
await sleepImpl(HEALTH_POLL_INTERVAL_MS);
const inst = readInstance();
if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) {
Expand Down Expand Up @@ -3979,7 +4000,7 @@ export async function ensureProxyRunning(
: childExit.signal ? `signal ${childExit.signal}` : "unknown reason";
throw new Error(`bili: proxy child exited before becoming healthy (${detail}) (log: ${logPath})`);
}
throw new Error(`bili: proxy did not become healthy within ${SPAWN_WAIT_MS}ms (log: ${logPath})`);
throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms (log: ${logPath})`);
} finally {
if (claimed) clearStartingMarker(launchToken);
}
Expand Down
86 changes: 85 additions & 1 deletion tests/dsh-settings-ui.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ test("#1590/#2125: client bundle registers the settings.section entry bili AND t
assert.ok(degBundleTexts.some((t) => t.includes("/acp")), "the bundle degraded hint points at /acp");
});

test("#1809/#2125: client polls /bili/origin while unresolved — upgrades on success, stays degraded and cancels when absent (both slots)", async () => {
test("#1809/#2125: client polls /bili/origin while unresolved — upgrades on success, stays degraded and cancels when absent (both slots); #2187 keeps probing in a slow phase past the fast attempts", async () => {
const { build } = await import("esbuild");
const configs = ((await import("../tsup.config.ts")).default) as unknown as Array<{
entry: Record<string, string>;
Expand Down Expand Up @@ -595,6 +595,90 @@ test("#1809/#2125: client polls /bili/origin while unresolved — upgrades on su
assert.deepEqual(opened, ["http://127.0.0.1:9999/__bili/"]);
m.runCleanups();
}

{
// #2187: past the fast phase, probing CONTINUES at the slow cadence
// instead of stopping — a late-arriving origin (background heal) must
// still upgrade the entry. Manual clock: capture each scheduled retry
// and fire it by hand so the test never waits in wall time.
const pending: Array<{ id: number; delay: number; fn: () => void }> = [];
let nextId = 1;
let polls = 0;
let landed = false;
const fakeSetTimeout = ((fn: () => void, delay?: number): unknown => {
const id = nextId++;
pending.push({ id, delay: delay ?? 0, fn });
return id;
}) as unknown as typeof setTimeout;
const fakeClearTimeout = ((id: unknown): void => {
const i = pending.findIndex((p) => p.id === id);
if (i >= 0) pending.splice(i, 1);
}) as unknown as typeof clearTimeout;
const m = mount({
fetch: async () => {
polls += 1;
if (polls >= 13) landed = true;
return { ok: true, json: async () => ({ origin: polls >= 13 ? "http://127.0.0.1:9997" : null }) };
},
setTimeout: fakeSetTimeout,
clearTimeout: fakeClearTimeout,
});
m.resetHooks();
const first = m.component() as ElementNode;
assert.equal(findButton(first), undefined, "first paint before the probe resolves is still degraded");
const delays: number[] = [];
for (;;) {
await tick();
if (landed) break;
if (pending.length === 0) throw new Error("polling stopped before the origin arrived");
const t = pending.shift()!;
delays.push(t.delay);
t.fn();
}
const second = m.component() as ElementNode;
const button = findButton(second);
assert.ok(button !== undefined, "a late-arriving origin upgrades the entry without a reload");
assert.deepEqual(delays.slice(0, 9), Array(9).fill(3000), "fast phase keeps the original cadence");
assert.ok(delays.length > 9, `probing must continue past the fast phase, got ${delays.length} retries`);
assert.deepEqual(delays.slice(9), Array(delays.length - 9).fill(10000), "slow phase uses the reduced cadence");
m.runCleanups();
}

{
// #2187: unmount still cancels the pending retry even in the slow phase
const pending: Array<{ id: number; delay: number; fn: () => void }> = [];
let nextId = 1;
let cleared = 0;
const fakeSetTimeout = ((fn: () => void, delay?: number): unknown => {
const id = nextId++;
pending.push({ id, delay: delay ?? 0, fn });
return id;
}) as unknown as typeof setTimeout;
const fakeClearTimeout = ((id: unknown): void => {
cleared += 1;
const i = pending.findIndex((p) => p.id === id);
if (i >= 0) pending.splice(i, 1);
}) as unknown as typeof clearTimeout;
const m = mount({
fetch: async () => ({ ok: true, json: async () => ({ origin: null }) }),
setTimeout: fakeSetTimeout,
clearTimeout: fakeClearTimeout,
});
m.resetHooks();
m.component();
const delays: number[] = [];
for (let i = 0; i < 12; i++) {
await tick();
if (pending.length === 0) break;
const t = pending.shift()!;
delays.push(t.delay);
t.fn();
}
assert.ok(delays.includes(10000), `reached the slow phase, got delays ${JSON.stringify(delays)}`);
m.runCleanups();
assert.ok(cleared >= 1, "the pending retry is cancelled on unmount");
assert.equal(pending.length, 0, "no timers survive unmount");
}
});

test("#2125: the manifest declares no dsh.client.inject (pre-0.2 dsh compat)", () => {
Expand Down
Loading
Loading