Skip to content
176 changes: 157 additions & 19 deletions src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,12 @@ interface ProxyHandle {
* proxy will outlive every session and ignore config edits until killed;
* host-native bootstraps must surface this instead of staying silent. */
refusedWatcher?: boolean;
/** #5641: the OUTCOME of the owner-watcher registration on attach.
* `attached`/transport-ready must not imply the shared proxy's watchdog
* acknowledged us — a transient failure leaves "failed" here even though
* traffic flows fine, and only a later acknowledged heartbeat proves
* recovery. Consumers reasoning about lifecycle guarantees read THIS. */
watcherRegistration?: WatcherRegistration;
}

export interface LauncherDeps {
Expand Down Expand Up @@ -4024,14 +4030,49 @@ async function fetchHealthInfoDefault(origin: string, opts?: { noKeepAlive?: boo
/** Outcome of a session-watcher registration attempt (#1190). */
export type WatcherRegistration = "ok" | "refused" | "failed";

/** #5641: an actionable rendering of a registration failure — error name and
* message plus the FIRST nested cause that carries one (the transport-level
* reason behind a bare "fetch failed", e.g. ETIMEDOUT/ECONNREFUSED). Some
* failure classes expose no structured `code` (DNS: the errno rides in the
* cause MESSAGE), so the first cause with either is used. Name/message/code
* only: never request bodies or credentials. Exported for the #5641 suite. */
export function formatWatcherError(err: unknown): string {
const head = err instanceof Error
? (err.name !== "Error" ? `${err.name}: ` : "") + err.message
: String(err);
let detail = head;
let cur: unknown = err instanceof Error ? err.cause : undefined;
for (let i = 0; i < 3 && cur !== undefined && cur !== null; i++) {
const c = cur as Record<string, unknown>;
if (typeof c.code === "string" || typeof c.code === "number") {
detail += ` [cause: ${c.code}]`;
return detail;
}
if (typeof c.message === "string" && c.message.trim() !== "") {
detail += ` [cause: ${c.message}]`;
return detail;
}
cur = c.cause;
}
return detail;
}

/** #1190: an ATTACHED shared proxy belongs to whoever SPAWNED it — its
* parent-gone watchdog tracks THAT owner's pid, so without registration the
* first owner's exit kills every attached session (#7/#1183). Registering our
* own owner makes the proxy die only after the LAST owner exits. Never throws:
* a failed registration degrades to the pre-fix single-owner behavior.
* #1322: returns the outcome so callers can surface a refusal — attaching to
* a daemon proxy means the "dies with the session" contract is void. */
* a daemon proxy means the "dies with the session" contract is void.
* #5641: failures warn with origin/pid and the nested cause code so the log
* line is actionable instead of a bare "fetch failed". */
export async function registerWatcherDefault(origin: string, pid: number): Promise<WatcherRegistration> {
// The recovery path depends on WHOSE pid was registered: our own gets an
// owner heartbeat that re-registers every beat (and reports recovery); a
// foreign pid has no heartbeat, so bounded retries are its whole chance.
const tail = pid === process.pid
? "a later heartbeat will re-register us and report recovery"
: "no heartbeat runs for a foreign owner pid — registration stays unconfirmed unless a retry succeeds";
try {
const res = await fetch(`${origin}/__bili/watcher`, {
method: "POST",
Expand All @@ -4043,12 +4084,12 @@ export async function registerWatcherDefault(origin: string, pid: number): Promi
// silence stays at this layer, the handle carries the flag instead.
if (!res.ok) {
if (res.status === 409) return "refused";
console.error(`bili: watcher registration returned HTTP ${res.status} — the shared proxy may exit when its first owner does`);
console.error(`bili: watcher registration at ${origin} (owner pid ${pid}) returned HTTP ${res.status} — the shared proxy may exit when its first owner does; ${tail}`);
return "failed";
}
return "ok";
} catch (err) {
console.error(`bili: watcher registration failed — the shared proxy may exit when its first owner does (${err instanceof Error ? err.message : String(err)})`);
console.error(`bili: watcher registration at ${origin} (owner pid ${pid}) failed — the shared proxy may exit when its first owner does (${formatWatcherError(err)}); ${tail}`);
return "failed";
}
}
Expand Down Expand Up @@ -4757,6 +4798,13 @@ const STALE_ROTATION_WAIT_MS = 12_000;
const SHUTDOWN_REQUEST_TIMEOUT_MS = 3_000;
const OWNER_HEARTBEAT_INTERVAL_MS = 5_000;
const OWNER_HEARTBEAT_POST_TIMEOUT_MS = 2_000;
// #5641: bounded retry budget for a TRANSIENT owner-registration failure on
// attach. A foreign owner pid has no heartbeat to repair the registration
// later, so these attempts are its whole chance; worst case ≈ ATTEMPTS × 2s
// fetch timeout + (ATTEMPTS−1) × backoff ≈ 7s. A 409 refusal is a policy
// answer, never retried.
const WATCHER_REG_ATTEMPTS = 3;
const WATCHER_REG_RETRY_BACKOFF_MS = 500;

export async function requestInstanceShutdown(origin: string): Promise<boolean> {
try {
Expand Down Expand Up @@ -4868,8 +4916,10 @@ export function startOwnerHeartbeat(
//
// The `post` seam is preserved verbatim: an overriding caller (the #3544
// tests record beats without a live server) keeps the in-thread loop, so
// existing contracts and testability are untouched.
if (post !== undefined) return runOwnerHeartbeatLoop(post, pid, intervalMs);
// existing contracts and testability are untouched. #5641: the origin is
// passed through so an injected post resolving TRUE reports an
// acknowledged beat exactly like the default sender does.
if (post !== undefined) return runOwnerHeartbeatLoop(post, pid, intervalMs, origin);
return runOwnerHeartbeatWorker(origin, pid, intervalMs);
}

Expand All @@ -4882,11 +4932,39 @@ export function startOwnerHeartbeat(
* above), so one host attests exactly one pid to a given proxy. */
const ownerHeartbeatWorkers = new Map<string, { stop: () => void; refs: number }>();

/** #5641: origins whose initial owner-watcher registration ended in a
* TRANSIENT failure — an acknowledged beat (an HTTP 2xx from
* /__bili/watcher, which re-adds the pid server-side) promotes them to
* recovered with exactly ONE host-visible notice through the diagnostic
* channel captured at attach time. A resolved fetch returning 409/5xx never
* counts as recovery. Entries die on recovery, on a direct "ok"
* re-registration, or when the origin's last heartbeat stops; a
* foreign-owner-pid entry (no heartbeat runs for it) simply lives until
* process exit or a later direct-"ok" attach to the same origin. Exported
* alongside noteWatcherHeartbeatAck so the regression suite can exercise the
* transition without a full proxy bring-up. */
const watcherPendingRecovery = new Map<string, { pid: number; diag: (msg: string) => void }>();

export function markWatcherPendingRecovery(origin: string, pid: number, diag: (msg: string) => void): void {
watcherPendingRecovery.set(origin, { pid, diag });
}

export function noteWatcherHeartbeatAck(origin: string): void {
const pending = watcherPendingRecovery.get(origin);
if (pending === undefined) return;
watcherPendingRecovery.delete(origin);
pending.diag(
`bili: watcher registration recovered at ${origin} (owner pid ${pending.pid}) — the shared proxy's watchdog now tracks this owner again; ` +
`the earlier "may exit when its first owner does" warning is cleared`,
);
}

/** In-thread beat loop — the historical shape, kept for injected `post`. */
function runOwnerHeartbeatLoop(
post: (body: string) => Promise<unknown>,
pid: number,
intervalMs: number,
origin?: string,
): () => void {
let stopped = false;
const beat = async (): Promise<void> => {
Expand All @@ -4897,7 +4975,13 @@ function runOwnerHeartbeatLoop(
});
if (stopped) return;
try {
await post(JSON.stringify({ pid, hb: true }));
// #5641: a post resolving TRUE means the server ACKNOWLEDGED
// the beat (defaultOwnerPost resolves res.ok; only a 2xx
// re-adds the pid). That is the sole input to the
// pending→recovered transition — a resolved non-ok response
// must not flip it.
const result = await post(JSON.stringify({ pid, hb: true }));
if (result === true && origin !== undefined) noteWatcherHeartbeatAck(origin);
} catch {
// transient failure — the next beat retries and the server-side
// grace window absorbs single misses
Expand All @@ -4919,16 +5003,22 @@ const { origin, pid, intervalMs, timeoutMs } = workerData;
let stopped = false;
parentPort.on("message", (m) => { if (m === "stop") stopped = true; });
const beat = async () => {
let acked = false;
while (!stopped) {
await new Promise((r) => { const t = setTimeout(r, intervalMs); t.unref?.(); });
if (stopped) return;
try {
await fetch(origin + "/__bili/watcher", {
const res = await fetch(origin + "/__bili/watcher", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ pid, hb: true }),
signal: AbortSignal.timeout(timeoutMs),
});
// #5641: report the FIRST acknowledged beat (a 2xx re-adds our pid
// server-side) so a pending registration failure can flip to
// recovered. Once only — no per-heartbeat traffic or log spam.
// 409/5xx never count: only res.ok does.
if (res.ok && !acked) { acked = true; parentPort.postMessage("ack"); }
} catch {
// transient — the next beat retries; the server-side grace window
// absorbs single misses
Expand All @@ -4940,9 +5030,11 @@ void beat();
`;

/** #5152: the beat-worker surface this module drives — injectable so the
* crash-fallback path can be exercised without a real thread death. */
* crash-fallback path can be exercised without a real thread death. #5641:
* also surfaces "message" — the worker's one-shot "ack" for its first
* acknowledged beat (#5641 recovery notice). */
export interface OwnerBeatWorker {
on(event: "error", listener: (err: Error) => void): unknown;
on(event: "error" | "message", listener: (arg: Error | string) => void): unknown;
unref(): void;
postMessage(message: string): void;
terminate(): Promise<number | undefined>;
Expand Down Expand Up @@ -4975,24 +5067,41 @@ export function runOwnerHeartbeatWorker(
let worker: OwnerBeatWorker | undefined;
let fallbackStop: (() => void) | undefined;
try {
worker = spawn(OWNER_HEARTBEAT_WORKER_SOURCE, { origin, pid, intervalMs, timeoutMs: OWNER_HEARTBEAT_POST_TIMEOUT_MS });
worker.unref();
worker.on("error", () => {
const beatWorker = spawn(OWNER_HEARTBEAT_WORKER_SOURCE, { origin, pid, intervalMs, timeoutMs: OWNER_HEARTBEAT_POST_TIMEOUT_MS });
beatWorker.on("message", (message) => {
// #5641: the worker's one-shot "ack" (first 2xx beat) drives the
// pending→recovered transition. "stopped" and anything else are
// ignored — no per-heartbeat logging.
if (message === "ack") noteWatcherHeartbeatAck(origin);
// A "message" listener RE-REFS the worker's port (Node MessagePort
// semantics) — re-assert the unref contract (#5152: the beat must
// never hold the host process open) after every delivery.
beatWorker.unref();
});
beatWorker.on("error", () => {
// Construction failure is covered by the catch below; this covers a
// thread that dies MID-FLIGHT: keep attesting from the in-thread
// loop (its fetch failures keep looping where a dead thread cannot).
if (!stopped && fallbackStop === undefined) {
fallbackStop = runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs);
fallbackStop = runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs, origin);
}
});
// AFTER all listener registrations: attaching "message" re-refs the
// port, so an earlier unref would be undone by the attachment itself.
beatWorker.unref();
worker = beatWorker;
} catch {
return runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs);
return runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs, origin);
}
const entry = { refs: 1, stop: (): void => { /* replaced below */ } };
const stop = (): void => {
if (stopped) return;
stopped = true;
ownerHeartbeatWorkers.delete(origin);
// #5641: no more beats will come for this origin from this process, so
// a pending registration failure can never recover now — drop the
// entry; a fresh attach re-marks it.
watcherPendingRecovery.delete(origin);
// Best-effort: tell the worker to stop, then tear it down regardless so
// a wedged worker can never outlive the host that owns it.
try { worker?.postMessage("stop"); } catch { /* already gone */ }
Expand All @@ -5010,15 +5119,18 @@ export function runOwnerHeartbeatWorker(
};
}

/** The default in-thread beat sender (fallback path). */
function defaultOwnerPost(origin: string): (body: string) => Promise<unknown> {
/** The default in-thread beat sender (fallback path). #5641: resolves
* res.ok — TRUE is the loop's ack signal for the pending→recovered
* transition; a resolved 409/5xx response stays falsy and never counts. */
function defaultOwnerPost(origin: string): (body: string) => Promise<boolean> {
return async (body: string) => {
await fetch(`${origin}/__bili/watcher`, {
const res = await fetch(`${origin}/__bili/watcher`, {
method: "POST",
headers: { "content-type": "application/json" },
body,
signal: AbortSignal.timeout(OWNER_HEARTBEAT_POST_TIMEOUT_MS),
});
return res.ok;
};
}

Expand Down Expand Up @@ -5059,12 +5171,25 @@ export async function ensureProxyRunning(
// (#7/#1183). Awaited before returning: a spawner that dies right after a
// second session attaches must not beat the registration to the grace
// window.
// #5641: a TRANSIENT failure (transport fault) is retried a small bounded
// number of times within WATCHER_REG_ATTEMPTS × 2s + backoffs — for a
// foreign owner pid there is no heartbeat to repair it later, and even
// for our own pid each retry shortens the unregistered window. "refused"
// (409) is a policy answer and is never retried.
const registerOwner = async (inst: ProxyInstanceFile): Promise<WatcherRegistration> => {
let reg = await registerWatcher(inst.origin, watchPid);
for (let attempt = 1; reg === "failed" && attempt < WATCHER_REG_ATTEMPTS; attempt++) {
await sleepImpl(WATCHER_REG_RETRY_BACKOFF_MS);
reg = await registerWatcher(inst.origin, watchPid);
}
return reg;
};
const attachTo = async (inst: ProxyInstanceFile): Promise<ProxyHandle> => {
// #3404: host-safe channel (#3200) — a raw console.* here lands on the
// physical screen bypassing an in-host TUI renderer (host-log.ts),
// e.g. into Pi's input draft after herdr session restore.
attachDiag(`bili: attaching to running proxy at ${inst.origin} (pid ${inst.pid})`);
const reg = await registerWatcher(inst.origin, watchPid);
const reg = await registerOwner(inst);
// #3544: keep re-asserting our liveness so the proxy's watchdog ages us
// out by HEARTBEAT expiry, not just kill(pid,0) (identity-blind under
// Windows pid reuse). No-op unless we own the pid.
Expand All @@ -5073,8 +5198,21 @@ export async function ensureProxyRunning(
// watchdog (started without BILI_PARENT_PID, e.g. manually on a stable
// port) — it will outlive every session; host-native bootstraps must
// say so instead of silently serving a voided lifecycle contract.
const handle: ProxyHandle = { origin: inst.origin, port: inst.port, attached: true };
// #5641: the handle carries the OUTCOME for every case — attached
// traffic does not imply the watchdog acknowledged us.
const handle: ProxyHandle = { origin: inst.origin, port: inst.port, attached: true, watcherRegistration: reg };
if (reg === "refused") handle.refusedWatcher = true;
if (reg === "failed") {
// The shared proxy may be healthy again on the next beat: remember
// the pending failure so the first ACKNOWLEDGED heartbeat flips it
// to recovered with one host-visible notice (host-safe channel —
// never a raw console.* on the attach path).
markWatcherPendingRecovery(inst.origin, watchPid, attachDiag);
} else {
// A direct "ok" proves registration right now — drop any stale
// pending entry for this origin (e.g. a sibling lane failed first).
watcherPendingRecovery.delete(inst.origin);
}
return handle;
};
const script = deps.scriptPath ?? process.argv[1];
Expand Down
11 changes: 11 additions & 0 deletions tests/e2e/e2e-native-pi.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,17 @@ function piRun(
cwd: ctx.piCwd,
env: {
...cleanEnv(),
// This suite documents the SPAWN-posture package-native contract:
// instance record + proxy origin, per-request wire stamps, the
// shared-proxy delegate-child ACP channel (#2268), /__bili/plugin/status
// over HTTP. #5428 flipped the pi lane DEFAULT to inline (in-process
// engine, no instance record) and this suite was not updated with it:
// primary-lane tests then pass for the wrong reason while the
// instance-record and delegate-child stamp assertions fail structurally
// (CI sentinel incident, #4599). Pin the legacy posture so this
// coverage stays deterministic; inline posture is covered by
// tests/pi-inline-ws.test.ts and tests/pi-native.test.ts.
BILI_PI_INLINE: "0",
PI_CODING_AGENT_DIR: ctx.piAgentDir,
XDG_CONFIG_HOME: ctx.xdg.config,
XDG_CACHE_HOME: ctx.xdg.cache,
Expand Down
Loading
Loading