From 9d1eb15bd8bac17fc6c49c8cddcb5b561a7fd2cc Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 4 Nov 2026 02:07:47 +0800 Subject: [PATCH 1/6] test(ci): scope the wire-exit-matrix abort-teardown budget to win32 (#4599) Master 9dc4c945 red on test (windows-latest, 22, 2) only: the two SSE abort cells exhausted the 10s teardown poll (closed=0, ~11s each) while the same code went green on ubuntu22/24 and windows24 shard-2 siblings (pass durations 23-352ms). The audit mandated by the constant's own note found no teardown hang: relay.ts arms res.on('close') -> clientAbort.abort() synchronously with no linger/backstop on this path, and no commit since d244ffff touched destroy/abort wiring. Residual latency is runner-side event-loop starvation on loaded Windows runners (same family as the recorded 3s/6s eras). Platform-scoped response: win32 doubles to 20s; every other platform keeps the tight 10s gate so a genuine hang still fails fast. --- tests/wire-exit-matrix.test.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/wire-exit-matrix.test.ts b/tests/wire-exit-matrix.test.ts index 6611f3156b..2de048cf3e 100644 --- a/tests/wire-exit-matrix.test.ts +++ b/tests/wire-exit-matrix.test.ts @@ -35,9 +35,19 @@ const SLOW_MS = 1500; // Raised to 10s after 6s still flaked three consecutive runs on windows-latest // node22 shard 2 (master c78ecb4b + ci-fixes c49bf8b1/12b988ff, all closed=0, // same cell, sibling shards green — load latency, not a teardown regression; -// recorded under #4599). If it flakes again at 10s, investigate a real teardown -// hang instead of raising further. -const ABORT_TEARDOWN_BUDGET_MS = 10000; +// recorded under #4599). Flaked once more at 10s (master 9dc4c945, run +// 41686972484, 2026-11-03: openai-sse + anthropic-sse closed=0 after the full +// poll, ~11s each; same code green on the ubuntu22/24 and windows24 shard-2 +// siblings, pass durations 23–352ms). The mandated "investigate a real +// teardown hang" audit found none: relay.ts arms res.on("close") -> +// clientAbort.abort() synchronously with no linger/backstop on this path, and +// no commit since d244ffff touched destroy/abort wiring — the residual latency +// is runner-side event-loop starvation on loaded Windows runners. Hence a +// platform-scoped budget instead of another flat raise: win32 doubles to 20s +// (every recorded starvation instance was windows-latest — node22+24, multiple +// shards, #3665/#4347 era — ubuntu never showed the signature), keeping the +// tight 10s gate everywhere else so a genuine teardown hang still fails fast. +const ABORT_TEARDOWN_BUDGET_MS = process.platform === "win32" ? 20000 : 10000; async function awaitUpstreamTeardown(socketsClosed: () => number): Promise { const deadline = Date.now() + ABORT_TEARDOWN_BUDGET_MS; From 0365c22a934cbc8879e30e9dac08686fcb6680e0 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 4 Nov 2026 11:38:31 +0800 Subject: [PATCH 2/6] test(e2e): pin the pi-codex-ws shared-proxy scenario to the pre-#5428 posture (#4599) --- tests/e2e/e2e-pi-codex-ws.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/e2e/e2e-pi-codex-ws.test.ts b/tests/e2e/e2e-pi-codex-ws.test.ts index e73027ad06..a8df48ae7c 100644 --- a/tests/e2e/e2e-pi-codex-ws.test.ts +++ b/tests/e2e/e2e-pi-codex-ws.test.ts @@ -722,7 +722,14 @@ test("E: two subagent-style sessions share one proxy — distinct state, identic assert.ok(r.stdout.includes("WS_E2E_OK")); }; - ok("X warmup", await piRun(ctxX, "alpha marker ALPHA_X_SEED")); + // #5428 flipped the pi lane's DEFAULT posture to inline (in-process proxy): + // left unpinned, X's proxy would die with its pi process and unregister its + // instance record — nothing left to discover, nothing for Y to attach to. + // This scenario pins the legacy spawn/attach posture it was built on; + // scenarios A–D intentionally ride the new inline default. + const legacyPosture: NodeJS.ProcessEnv = { BILI_PI_INLINE: "0" }; + + ok("X warmup", await piRun(ctxX, "alpha marker ALPHA_X_SEED", { extraEnv: legacyPosture })); const origin = proxyOriginOf(ctxX); assert.ok(origin, "X's proxy origin discovered from its instance record"); // Y attaches to X's proxy (real subagent posture: inherited @@ -733,7 +740,7 @@ test("E: two subagent-style sessions share one proxy — distinct state, identic // Same tool args ({}) from both sessions, interleaved. const [xStatus, yStatus] = await Promise.all([ - piRun(ctxX, "WS_STATUS alpha status", { resume: true }), + piRun(ctxX, "WS_STATUS alpha status", { resume: true, extraEnv: legacyPosture }), piRun(ctxY, "WS_STATUS beta status", { resume: true, extraEnv: yEnv }), ]); ok("X status", xStatus); @@ -763,7 +770,7 @@ test("E: two subagent-style sessions share one proxy — distinct state, identic // Concurrent plain turns, same proxy, no cross-talk afterwards. const [xFin, yFin] = await Promise.all([ - piRun(ctxX, "concurrent X tail", { resume: true }), + piRun(ctxX, "concurrent X tail", { resume: true, extraEnv: legacyPosture }), piRun(ctxY, "concurrent Y tail", { resume: true, extraEnv: yEnv }), ]); ok("X concurrent", xFin); From c8270e75cb6f33b000007d199f147d3f30d53c7f Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 12:58:14 +0800 Subject: [PATCH 3/6] fix: surface failed watcher registration cause/state and report heartbeat recovery (#5641) A transient POST /__bili/watcher failure on attach printed a bare "fetch failed" line, dropped the outcome from the returned handle (only 409 was retained), and then self-repaired silently via the owner heartbeat with no trace. - registerWatcherDefault diagnostics carry sanitized origin + owner pid, the error name/message plus the first structured cause code (or cause message for undici errors that carry no code), and an outcome-specific tail (own pid: a later heartbeat re-registers and reports recovery; foreign pid: stays unconfirmed unless a retry succeeds). - ProxyHandle gains watcherRegistration ("ok"|"refused"|"failed") for every attach outcome; refusedWatcher is kept for existing consumers. - attachTo retries transient failures only (never 409): up to 3 attempts, 500ms backoff, worst-case ~7s extra attach latency. - The heartbeat worker inspects res.ok and posts a one-shot "ack" after the first 2xx beat; the parent transitions pending->recovered exactly once through the host-safe diagnostic channel. The in-thread fallback path acks via the post() boolean result. A resolved fetch returning 409/5xx never counts as recovery. Attaching the "message" listener re-refs the worker port (Node MessagePort semantics), so unref is re-asserted at attachment and after every delivery to keep the #5152 liveness contract. No watchdog death/expiry policy change; no spawn/restart/fallback added; no config surface change. Tests: tests/issue5641-watcher-registration-recovery.test.ts covers the issue six-row regression table. --- src/launcher.ts | 174 ++++++- ...5641-watcher-registration-recovery.test.ts | 459 ++++++++++++++++++ 2 files changed, 614 insertions(+), 19 deletions(-) create mode 100644 tests/issue5641-watcher-registration-recovery.test.ts diff --git a/src/launcher.ts b/src/launcher.ts index da099db8f8..d76ae6f341 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -262,6 +262,12 @@ interface ProxyHandle { * proxy will outlive every session and ignore config edits until killed; * host-native bootstraps must surface this instead of staying silent. */ refusedWatcher?: boolean; + /** #5641: the OUTCOME of the owner-watcher registration on attach. + * `attached`/transport-ready must not imply the shared proxy's watchdog + * acknowledged us — a transient failure leaves "failed" here even though + * traffic flows fine, and only a later acknowledged heartbeat proves + * recovery. Consumers reasoning about lifecycle guarantees read THIS. */ + watcherRegistration?: WatcherRegistration; } export interface LauncherDeps { @@ -4024,14 +4030,49 @@ async function fetchHealthInfoDefault(origin: string, opts?: { noKeepAlive?: boo /** Outcome of a session-watcher registration attempt (#1190). */ export type WatcherRegistration = "ok" | "refused" | "failed"; +/** #5641: an actionable rendering of a registration failure — error name and + * message plus the FIRST nested cause that carries one (the transport-level + * reason behind a bare "fetch failed", e.g. ETIMEDOUT/ECONNREFUSED). Some + * failure classes expose no structured `code` (DNS: the errno rides in the + * cause MESSAGE), so the first cause with either is used. Name/message/code + * only: never request bodies or credentials. Exported for the #5641 suite. */ +export function formatWatcherError(err: unknown): string { + const head = err instanceof Error + ? (err.name !== "Error" ? `${err.name}: ` : "") + err.message + : String(err); + let detail = head; + let cur: unknown = err instanceof Error ? err.cause : undefined; + for (let i = 0; i < 3 && cur !== undefined && cur !== null; i++) { + const c = cur as Record; + if (typeof c.code === "string" || typeof c.code === "number") { + detail += ` [cause: ${c.code}]`; + return detail; + } + if (typeof c.message === "string" && c.message.trim() !== "") { + detail += ` [cause: ${c.message}]`; + return detail; + } + cur = c.cause; + } + return detail; +} + /** #1190: an ATTACHED shared proxy belongs to whoever SPAWNED it — its * parent-gone watchdog tracks THAT owner's pid, so without registration the * first owner's exit kills every attached session (#7/#1183). Registering our * own owner makes the proxy die only after the LAST owner exits. Never throws: * a failed registration degrades to the pre-fix single-owner behavior. * #1322: returns the outcome so callers can surface a refusal — attaching to - * a daemon proxy means the "dies with the session" contract is void. */ + * a daemon proxy means the "dies with the session" contract is void. + * #5641: failures warn with origin/pid and the nested cause code so the log + * line is actionable instead of a bare "fetch failed". */ export async function registerWatcherDefault(origin: string, pid: number): Promise { + // The recovery path depends on WHOSE pid was registered: our own gets an + // owner heartbeat that re-registers every beat (and reports recovery); a + // foreign pid has no heartbeat, so bounded retries are its whole chance. + const tail = pid === process.pid + ? "a later heartbeat will re-register us and report recovery" + : "no heartbeat runs for a foreign owner pid — registration stays unconfirmed unless a retry succeeds"; try { const res = await fetch(`${origin}/__bili/watcher`, { method: "POST", @@ -4043,12 +4084,12 @@ export async function registerWatcherDefault(origin: string, pid: number): Promi // silence stays at this layer, the handle carries the flag instead. if (!res.ok) { if (res.status === 409) return "refused"; - console.error(`bili: watcher registration returned HTTP ${res.status} — the shared proxy may exit when its first owner does`); + console.error(`bili: watcher registration at ${origin} (owner pid ${pid}) returned HTTP ${res.status} — the shared proxy may exit when its first owner does; ${tail}`); return "failed"; } return "ok"; } catch (err) { - console.error(`bili: watcher registration failed — the shared proxy may exit when its first owner does (${err instanceof Error ? err.message : String(err)})`); + console.error(`bili: watcher registration at ${origin} (owner pid ${pid}) failed — the shared proxy may exit when its first owner does (${formatWatcherError(err)}); ${tail}`); return "failed"; } } @@ -4757,6 +4798,13 @@ const STALE_ROTATION_WAIT_MS = 12_000; const SHUTDOWN_REQUEST_TIMEOUT_MS = 3_000; const OWNER_HEARTBEAT_INTERVAL_MS = 5_000; const OWNER_HEARTBEAT_POST_TIMEOUT_MS = 2_000; +// #5641: bounded retry budget for a TRANSIENT owner-registration failure on +// attach. A foreign owner pid has no heartbeat to repair the registration +// later, so these attempts are its whole chance; worst case ≈ ATTEMPTS × 2s +// fetch timeout + (ATTEMPTS−1) × backoff ≈ 7s. A 409 refusal is a policy +// answer, never retried. +const WATCHER_REG_ATTEMPTS = 3; +const WATCHER_REG_RETRY_BACKOFF_MS = 500; export async function requestInstanceShutdown(origin: string): Promise { try { @@ -4868,8 +4916,10 @@ export function startOwnerHeartbeat( // // The `post` seam is preserved verbatim: an overriding caller (the #3544 // tests record beats without a live server) keeps the in-thread loop, so - // existing contracts and testability are untouched. - if (post !== undefined) return runOwnerHeartbeatLoop(post, pid, intervalMs); + // existing contracts and testability are untouched. #5641: the origin is + // passed through so an injected post resolving TRUE reports an + // acknowledged beat exactly like the default sender does. + if (post !== undefined) return runOwnerHeartbeatLoop(post, pid, intervalMs, origin); return runOwnerHeartbeatWorker(origin, pid, intervalMs); } @@ -4882,11 +4932,37 @@ export function startOwnerHeartbeat( * above), so one host attests exactly one pid to a given proxy. */ const ownerHeartbeatWorkers = new Map void; refs: number }>(); +/** #5641: origins whose initial owner-watcher registration ended in a + * TRANSIENT failure — an acknowledged beat (an HTTP 2xx from + * /__bili/watcher, which re-adds the pid server-side) promotes them to + * recovered with exactly ONE host-visible notice through the diagnostic + * channel captured at attach time. A resolved fetch returning 409/5xx never + * counts as recovery. Entries die on recovery, on a direct "ok" + * re-registration, or when the origin's last heartbeat stops. Exported + * alongside noteWatcherHeartbeatAck so the regression suite can exercise the + * transition without a full proxy bring-up. */ +const watcherPendingRecovery = new Map void }>(); + +export function markWatcherPendingRecovery(origin: string, pid: number, diag: (msg: string) => void): void { + watcherPendingRecovery.set(origin, { pid, diag }); +} + +export function noteWatcherHeartbeatAck(origin: string): void { + const pending = watcherPendingRecovery.get(origin); + if (pending === undefined) return; + watcherPendingRecovery.delete(origin); + pending.diag( + `bili: watcher registration recovered at ${origin} (owner pid ${pending.pid}) — the shared proxy's watchdog now tracks this owner again; ` + + `the earlier "may exit when its first owner does" warning is cleared`, + ); +} + /** In-thread beat loop — the historical shape, kept for injected `post`. */ function runOwnerHeartbeatLoop( post: (body: string) => Promise, pid: number, intervalMs: number, + origin?: string, ): () => void { let stopped = false; const beat = async (): Promise => { @@ -4897,7 +4973,13 @@ function runOwnerHeartbeatLoop( }); if (stopped) return; try { - await post(JSON.stringify({ pid, hb: true })); + // #5641: a post resolving TRUE means the server ACKNOWLEDGED + // the beat (defaultOwnerPost resolves res.ok; only a 2xx + // re-adds the pid). That is the sole input to the + // pending→recovered transition — a resolved non-ok response + // must not flip it. + const result = await post(JSON.stringify({ pid, hb: true })); + if (result === true && origin !== undefined) noteWatcherHeartbeatAck(origin); } catch { // transient failure — the next beat retries and the server-side // grace window absorbs single misses @@ -4919,16 +5001,22 @@ const { origin, pid, intervalMs, timeoutMs } = workerData; let stopped = false; parentPort.on("message", (m) => { if (m === "stop") stopped = true; }); const beat = async () => { + let acked = false; while (!stopped) { await new Promise((r) => { const t = setTimeout(r, intervalMs); t.unref?.(); }); if (stopped) return; try { - await fetch(origin + "/__bili/watcher", { + const res = await fetch(origin + "/__bili/watcher", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ pid, hb: true }), signal: AbortSignal.timeout(timeoutMs), }); + // #5641: report the FIRST acknowledged beat (a 2xx re-adds our pid + // server-side) so a pending registration failure can flip to + // recovered. Once only — no per-heartbeat traffic or log spam. + // 409/5xx never count: only res.ok does. + if (res.ok && !acked) { acked = true; parentPort.postMessage("ack"); } } catch { // transient — the next beat retries; the server-side grace window // absorbs single misses @@ -4940,9 +5028,11 @@ void beat(); `; /** #5152: the beat-worker surface this module drives — injectable so the - * crash-fallback path can be exercised without a real thread death. */ + * crash-fallback path can be exercised without a real thread death. #5641: + * also surfaces "message" — the worker's one-shot "ack" for its first + * acknowledged beat (#5641 recovery notice). */ export interface OwnerBeatWorker { - on(event: "error", listener: (err: Error) => void): unknown; + on(event: "error" | "message", listener: (arg: Error | string) => void): unknown; unref(): void; postMessage(message: string): void; terminate(): Promise; @@ -4975,24 +5065,41 @@ export function runOwnerHeartbeatWorker( let worker: OwnerBeatWorker | undefined; let fallbackStop: (() => void) | undefined; try { - worker = spawn(OWNER_HEARTBEAT_WORKER_SOURCE, { origin, pid, intervalMs, timeoutMs: OWNER_HEARTBEAT_POST_TIMEOUT_MS }); - worker.unref(); - worker.on("error", () => { + const beatWorker = spawn(OWNER_HEARTBEAT_WORKER_SOURCE, { origin, pid, intervalMs, timeoutMs: OWNER_HEARTBEAT_POST_TIMEOUT_MS }); + beatWorker.on("message", (message) => { + // #5641: the worker's one-shot "ack" (first 2xx beat) drives the + // pending→recovered transition. "stopped" and anything else are + // ignored — no per-heartbeat logging. + if (message === "ack") noteWatcherHeartbeatAck(origin); + // A "message" listener RE-REFS the worker's port (Node MessagePort + // semantics) — re-assert the unref contract (#5152: the beat must + // never hold the host process open) after every delivery. + beatWorker.unref(); + }); + beatWorker.on("error", () => { // Construction failure is covered by the catch below; this covers a // thread that dies MID-FLIGHT: keep attesting from the in-thread // loop (its fetch failures keep looping where a dead thread cannot). if (!stopped && fallbackStop === undefined) { - fallbackStop = runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs); + fallbackStop = runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs, origin); } }); + // AFTER all listener registrations: attaching "message" re-refs the + // port, so an earlier unref would be undone by the attachment itself. + beatWorker.unref(); + worker = beatWorker; } catch { - return runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs); + return runOwnerHeartbeatLoop(defaultOwnerPost(origin), pid, intervalMs, origin); } const entry = { refs: 1, stop: (): void => { /* replaced below */ } }; const stop = (): void => { if (stopped) return; stopped = true; ownerHeartbeatWorkers.delete(origin); + // #5641: no more beats will come for this origin from this process, so + // a pending registration failure can never recover now — drop the + // entry; a fresh attach re-marks it. + watcherPendingRecovery.delete(origin); // Best-effort: tell the worker to stop, then tear it down regardless so // a wedged worker can never outlive the host that owns it. try { worker?.postMessage("stop"); } catch { /* already gone */ } @@ -5010,15 +5117,18 @@ export function runOwnerHeartbeatWorker( }; } -/** The default in-thread beat sender (fallback path). */ -function defaultOwnerPost(origin: string): (body: string) => Promise { +/** The default in-thread beat sender (fallback path). #5641: resolves + * res.ok — TRUE is the loop's ack signal for the pending→recovered + * transition; a resolved 409/5xx response stays falsy and never counts. */ +function defaultOwnerPost(origin: string): (body: string) => Promise { return async (body: string) => { - await fetch(`${origin}/__bili/watcher`, { + const res = await fetch(`${origin}/__bili/watcher`, { method: "POST", headers: { "content-type": "application/json" }, body, signal: AbortSignal.timeout(OWNER_HEARTBEAT_POST_TIMEOUT_MS), }); + return res.ok; }; } @@ -5059,12 +5169,25 @@ export async function ensureProxyRunning( // (#7/#1183). Awaited before returning: a spawner that dies right after a // second session attaches must not beat the registration to the grace // window. + // #5641: a TRANSIENT failure (transport fault) is retried a small bounded + // number of times within WATCHER_REG_ATTEMPTS × 2s + backoffs — for a + // foreign owner pid there is no heartbeat to repair it later, and even + // for our own pid each retry shortens the unregistered window. "refused" + // (409) is a policy answer and is never retried. + const registerOwner = async (inst: ProxyInstanceFile): Promise => { + let reg = await registerWatcher(inst.origin, watchPid); + for (let attempt = 1; reg === "failed" && attempt < WATCHER_REG_ATTEMPTS; attempt++) { + await sleepImpl(WATCHER_REG_RETRY_BACKOFF_MS); + reg = await registerWatcher(inst.origin, watchPid); + } + return reg; + }; const attachTo = async (inst: ProxyInstanceFile): Promise => { // #3404: host-safe channel (#3200) — a raw console.* here lands on the // physical screen bypassing an in-host TUI renderer (host-log.ts), // e.g. into Pi's input draft after herdr session restore. attachDiag(`bili: attaching to running proxy at ${inst.origin} (pid ${inst.pid})`); - const reg = await registerWatcher(inst.origin, watchPid); + const reg = await registerOwner(inst); // #3544: keep re-asserting our liveness so the proxy's watchdog ages us // out by HEARTBEAT expiry, not just kill(pid,0) (identity-blind under // Windows pid reuse). No-op unless we own the pid. @@ -5073,8 +5196,21 @@ export async function ensureProxyRunning( // watchdog (started without BILI_PARENT_PID, e.g. manually on a stable // port) — it will outlive every session; host-native bootstraps must // say so instead of silently serving a voided lifecycle contract. - const handle: ProxyHandle = { origin: inst.origin, port: inst.port, attached: true }; + // #5641: the handle carries the OUTCOME for every case — attached + // traffic does not imply the watchdog acknowledged us. + const handle: ProxyHandle = { origin: inst.origin, port: inst.port, attached: true, watcherRegistration: reg }; if (reg === "refused") handle.refusedWatcher = true; + if (reg === "failed") { + // The shared proxy may be healthy again on the next beat: remember + // the pending failure so the first ACKNOWLEDGED heartbeat flips it + // to recovered with one host-visible notice (host-safe channel — + // never a raw console.* on the attach path). + markWatcherPendingRecovery(inst.origin, watchPid, attachDiag); + } else { + // A direct "ok" proves registration right now — drop any stale + // pending entry for this origin (e.g. a sibling lane failed first). + watcherPendingRecovery.delete(inst.origin); + } return handle; }; const script = deps.scriptPath ?? process.argv[1]; diff --git a/tests/issue5641-watcher-registration-recovery.test.ts b/tests/issue5641-watcher-registration-recovery.test.ts new file mode 100644 index 0000000000..ba43436da0 --- /dev/null +++ b/tests/issue5641-watcher-registration-recovery.test.ts @@ -0,0 +1,459 @@ +// #5641: a transient owner-watcher registration failure used to be invisible +// after the fact — the warning dropped the underlying cause, the returned +// handle kept only "refused", and a heartbeat that later repaired the +// registration (the server re-adds the pid on every accepted hb) produced no +// signal at all. The user was left staring at a permanent +// "may exit when its first owner does" warning while the state had already +// recovered. +// +// This suite pins the reporter's scenario table: +// 1. transient failure + later acknowledged beat → diagnosable failure, +// pending state clears ONLY on an ack, exactly one recovery notice, no +// extra proxy spawned; +// 2. persistent network failure → bounded attempts, stays unconfirmed, no +// false success; +// 3. HTTP 409 → refused semantics preserved, never retried, no false +// recovered; +// 4. heartbeat HTTP 500/409 → a resolved fetch is NOT recovery; +// 5. foreign owner pid → no heartbeat started, failure retained on the +// handle, retries still happen (they are the only chance); +// 6. worker death → in-thread fallback carries the same ack semantics. +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { + ensureProxyRunning, + formatWatcherError, + registerWatcherDefault, + runOwnerHeartbeatWorker, + startOwnerHeartbeat, + markWatcherPendingRecovery, + noteWatcherHeartbeatAck, + type LauncherDeps, + type OwnerBeatWorker, + type WatcherRegistration, +} from "../src/launcher.js"; +import { type ProxyInstanceFile as InstanceFile } from "../src/instance.js"; + +async function wait(ms: number): Promise { + await new Promise((resolve) => { setTimeout(resolve, ms); }); +} + +/** Poll until `check()` passes or the deadline expires. */ +async function until(check: () => boolean, deadlineMs: number, stepMs = 50): Promise { + const end = Date.now() + deadlineMs; + while (Date.now() < end) { + if (check()) return true; + await wait(stepMs); + } + return check(); +} + +/** A controllable /__bili/watcher stand-in recording every beat body. */ +interface WatcherStub { + origin: string; + bodies: string[]; + close(): Promise; +} + +function stubWatcher(status: number): Promise { + const bodies: string[] = []; + const server = http.createServer((req, res) => { + if (req.method === "POST" && req.url === "/__bili/watcher") { + let b = ""; + req.on("data", (c) => (b += c)); + req.on("end", () => { + bodies.push(b); + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(status === 200 ? { ok: true, watchers: 1 } : { ok: false, error: "stub" })); + }); + return; + } + res.writeHead(404); + res.end(); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + resolve({ + origin: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + bodies, + close: () => new Promise((r) => server.close(() => r())), + }); + }); + }); +} + +/** Capture console.error lines for the duration of a test. */ +function captureConsoleError(): { lines: string[]; restore(): void } { + const lines: string[] = []; + const orig = console.error; + console.error = (...args: unknown[]): void => { lines.push(args.map(String).join(" ")); }; + return { + lines, + restore(): void { console.error = orig; }, + }; +} + +// #1225-style fingerprint fixture: the attaching side hashes the script it +// WOULD spawn; the fake instance records the same hash so the attach gate +// lines up. +const FP_SCRIPT = path.join(os.tmpdir(), `bili-5641-fp-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// #5641 fingerprint fixture\n"); +const FP_HASH = createHash("sha256").update(fs.readFileSync(FP_SCRIPT)).digest("hex"); + +function recordedInstance(origin: string, port: number): InstanceFile { + return { + origin, + instanceId: "inst-5641", + pid: process.pid, + startedAt: Date.now(), + host: "127.0.0.1", + port, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + codeFingerprint: FP_HASH, + }; +} + +/** Deps that force ensureProxyRunning down the ATTACH path against a virtual + * (or real) instance — nothing here spawns or touches the network beyond + * what the caller wires. */ +function attachDeps(opts: { + origin: string; + port: number; + registerWatcher: (origin: string, pid: number) => Promise; + diag: (msg: string) => void; +}): LauncherDeps { + return { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, instanceId: "inst-5641", watchdog: { armed: true } }), + readInstanceFile: () => recordedInstance(opts.origin, opts.port), + isConnectable: async () => true, + registerWatcher: opts.registerWatcher, + attachDiag: opts.diag, + scriptPath: FP_SCRIPT, + spawnImpl: () => { throw new Error("attach path must not spawn"); }, + }; +} + +function launchWith(deps: LauncherDeps, port: number, parentPid?: number) { + return ensureProxyRunning( + { host: "127.0.0.1", port, passthrough: false, debug: false, ...(parentPid !== undefined ? { parentPid } : {}) }, + deps, + ); +} + +// ── 1. Diagnostics: the failure line must be actionable ───────────────────── + +test("#5641 registerWatcherDefault: HTTP failure warns with origin/pid/status and the recovery path", async () => { + const stub = await stubWatcher(500); + const cap = captureConsoleError(); + try { + const reg = await registerWatcherDefault(stub.origin, process.pid); + assert.equal(reg, "failed"); + assert.equal(cap.lines.length, 1, "one warning line per attempt"); + const line = cap.lines[0]; + assert.ok(line.includes(stub.origin), `origin present: ${line}`); + assert.ok(line.includes(`owner pid ${process.pid}`), `owner pid present: ${line}`); + assert.ok(line.includes("HTTP 500"), `status present: ${line}`); + assert.ok(line.includes("the shared proxy may exit when its first owner does"), line); + assert.ok(line.includes("a later heartbeat will re-register us"), "own-pid tail names the recovery path"); + } finally { + cap.restore(); + await stub.close(); + } +}); + +test("#5641 registerWatcherDefault: transport failure surfaces the nested cause and the foreign-pid tail", async () => { + // Deterministic refused port: bind, close, then hit it — port 1 would be + // blocked by undici's bad-port list before any socket is even opened. + const probe = http.createServer(() => {}); + await new Promise((r) => probe.listen(0, "127.0.0.1", () => r())); + const port = (probe.address() as AddressInfo).port; + await new Promise((r) => probe.close(() => r())); + const cap = captureConsoleError(); + try { + const reg = await registerWatcherDefault(`http://127.0.0.1:${port}`, 999_999); + assert.equal(reg, "failed"); + assert.equal(cap.lines.length, 1); + const line = cap.lines[0]; + assert.ok(line.includes("ECONNREFUSED"), `transport-level reason present: ${line}`); + assert.ok(line.includes("no heartbeat runs for a foreign owner pid"), "foreign-pid tail says so explicitly"); + assert.ok(!line.includes("a later heartbeat will re-register us"), "foreign pid gets no own-heartbeat promise"); + } finally { + cap.restore(); + } +}); + +test("#5641 formatWatcherError: first carrying cause wins, message fallback, bare errors pass through", () => { + // Structured code (undici ConnectError shape): the code wins over deeper causes. + const outer = new TypeError("fetch failed"); + const causeErr = Object.assign(new Error("connect ETIMEDOUT 127.0.0.1:8787"), { code: "ETIMEDOUT" }); + const inner = Object.assign(new Error("inner"), { code: "ECONNRESET" }); + causeErr.cause = inner; + outer.cause = causeErr; + assert.equal(formatWatcherError(outer), "TypeError: fetch failed [cause: ETIMEDOUT]"); + + // No structured code (Node 22 DNS failures carry the errno in the message). + const msgOnly = new TypeError("fetch failed"); + msgOnly.cause = new Error("getaddrinfo ENOTFOUND upstream.invalid"); + assert.equal(formatWatcherError(msgOnly), "TypeError: fetch failed [cause: getaddrinfo ENOTFOUND upstream.invalid]"); + + // No cause at all: name+message only; generic Error name stays unprefixed. + assert.equal(formatWatcherError(new RangeError("boom")), "RangeError: boom"); + assert.equal(formatWatcherError(new Error("plain")), "plain"); + // Non-Error input survives. + assert.equal(formatWatcherError("string failure"), "string failure"); +}); + +// ── 2. Handle state: attached never implies registered ────────────────────── + +test("#5641 handle.watcherRegistration carries every outcome; refused keeps its flag; no extra spawn", async () => { + const deadOrigin = "http://127.0.0.1:1"; + const diag: string[] = []; + + let calls = 0; + const okHandle = await launchWith(attachDeps({ + origin: deadOrigin, + port: 1, + registerWatcher: async () => { calls++; return "ok"; }, + diag: (m) => diag.push(m), + }), 1); + assert.equal(okHandle.attached, true); + assert.equal(okHandle.watcherRegistration, "ok"); + assert.equal(okHandle.refusedWatcher, undefined); + assert.equal(calls, 1, "a clean registration is not retried"); + + calls = 0; + const refusedHandle = await launchWith(attachDeps({ + origin: deadOrigin, + port: 1, + registerWatcher: async () => { calls++; return "refused"; }, + diag: (m) => diag.push(m), + }), 1); + assert.equal(refusedHandle.watcherRegistration, "refused"); + assert.equal(refusedHandle.refusedWatcher, true, "#1322 flag preserved"); + assert.equal(calls, 1, "a 409 refusal is a policy answer — NEVER retried (#5641 row 3)"); +}); + +test("#5641 transient failure: bounded retries, retained 'failed', no false recovery, no spawn", async () => { + const deadOrigin = "http://127.0.0.1:1"; + const diag: string[] = []; + const sleeps: number[] = []; + let calls = 0; + const deps = attachDeps({ + origin: deadOrigin, + port: 1, + registerWatcher: async () => { calls++; return "failed"; }, + diag: (m) => diag.push(m), + }); + deps.sleep = async (ms: number) => { sleeps.push(ms); }; + + const handle = await launchWith(deps, 1); + assert.equal(handle.attached, true, "traffic still flows — attachment is not rejected"); + assert.equal(handle.watcherRegistration, "failed", "the unconfirmed state is RETAINED on the handle"); + assert.notEqual(handle.refusedWatcher, true, "a transport fault is not a refusal"); + assert.equal(calls, 3, "bounded: initial + 2 retries, then stop (#5641 row 2)"); + assert.deepEqual(sleeps, [500, 500], "explicit backoff budget between attempts"); + + // Persistent failure stays unconfirmed: nothing ever reports recovery. + await wait(300); + assert.equal(diag.filter((m) => m.includes("recovered")).length, 0, "no false success"); +}); + +test("#5641 retry succeeds on attempt two: final outcome 'ok', no pending state", async () => { + const deadOrigin = "http://127.0.0.1:1"; + const diag: string[] = []; + const results: WatcherRegistration[] = ["failed", "ok"]; + let i = 0; + const handle = await launchWith(attachDeps({ + origin: deadOrigin, + port: 1, + registerWatcher: async () => results[i++], + diag: (m) => diag.push(m), + }), 1); + assert.equal(i, 2, "stops retrying once the outcome is conclusive"); + assert.equal(handle.watcherRegistration, "ok"); + await wait(200); + assert.equal(diag.filter((m) => m.includes("recovered")).length, 0, "direct success needs no recovery notice"); +}); + +test("#5641 foreign owner pid: retries still run (their only chance), failure stays retained", async () => { + const deadOrigin = "http://127.0.0.1:1"; + const diag: string[] = []; + let calls = 0; + const handle = await launchWith(attachDeps({ + origin: deadOrigin, + port: 1, + registerWatcher: async () => { calls++; return "failed"; }, + diag: (m) => diag.push(m), + }), 1, 999_999); + assert.equal(handle.watcherRegistration, "failed", "row 5: the unconfirmed fact survives on the handle"); + assert.equal(calls, 3, "row 5: bounded retries are the foreign pid's only repair path"); + await wait(300); + assert.equal(diag.filter((m) => m.includes("recovered")).length, 0); +}); + +// ── 3. Recovery: an acknowledged beat flips pending → recovered, once ─────── + +test("#5641 noteWatcherHeartbeatAck: one notice per episode, idempotent afterwards", () => { + const diag: string[] = []; + const origin = "http://127.0.0.1:5641-unit"; + markWatcherPendingRecovery(origin, 4242, (m) => diag.push(m)); + noteWatcherHeartbeatAck(origin); + noteWatcherHeartbeatAck(origin); + assert.equal(diag.length, 1, "exactly ONE host-visible recovery notice"); + assert.ok(diag[0].includes(`watcher registration recovered at ${origin}`), diag[0]); + assert.ok(diag[0].includes("owner pid 4242"), diag[0]); + // An origin that never failed has nothing to recover. + noteWatcherHeartbeatAck("http://127.0.0.1:5641-quiet"); + assert.equal(diag.length, 1, "acks without a pending failure stay silent"); +}); + +test("#5641 worker path: first 2xx beat acknowledges, later beats add no noise", async () => { + const stub = await stubWatcher(200); + const diag: string[] = []; + markWatcherPendingRecovery(stub.origin, process.pid, (m) => diag.push(m)); + const stop = runOwnerHeartbeatWorker(stub.origin, process.pid, 200); + assert.equal(typeof stop, "function"); + try { + assert.ok(await until(() => diag.length >= 1, 8_000), `recovery notice arrives via the host-safe channel (got: ${JSON.stringify(diag)})`); + const settled = diag.length; + await wait(700); + assert.equal(diag.length, settled, "no per-heartbeat log spam after recovery"); + assert.ok(stub.bodies.length >= 1, "beats actually reached the proxy"); + assert.deepEqual(JSON.parse(stub.bodies[0]), { pid: process.pid, hb: true }, "wire shape unchanged"); + } finally { + stop(); + await stub.close(); + } +}); + +test("#5641 row 4: a resolved fetch returning 500 is NOT recovery", async () => { + const stub = await stubWatcher(500); + const diag: string[] = []; + markWatcherPendingRecovery(stub.origin, process.pid, (m) => diag.push(m)); + const stop = runOwnerHeartbeatWorker(stub.origin, process.pid, 150); + try { + await wait(900); + assert.ok(stub.bodies.length >= 1, "beats reached the proxy even while it answers 500"); + assert.equal(diag.length, 0, "a resolved non-2xx response must not flip pending→recovered"); + } finally { + stop(); + await stub.close(); + } +}); + +test("#5641 rows 3/4: a resolved fetch returning 409 is NOT recovery either", async () => { + const stub = await stubWatcher(409); + const diag: string[] = []; + markWatcherPendingRecovery(stub.origin, process.pid, (m) => diag.push(m)); + const stop = runOwnerHeartbeatWorker(stub.origin, process.pid, 150); + try { + await wait(900); + assert.ok(stub.bodies.length >= 1, "beats keep going (liveness attestation is unchanged)"); + assert.equal(diag.length, 0, "409 means 'watchdog unarmed' — nothing to recover"); + } finally { + stop(); + await stub.close(); + } +}); + +test("#5641 row 6: a worker that dies mid-flight falls back to the in-thread loop WITH ack semantics", async () => { + const stub = await stubWatcher(200); + const diag: string[] = []; + markWatcherPendingRecovery(stub.origin, process.pid, (m) => diag.push(m)); + let errorListener: ((err: Error) => void) | undefined; + const fakeWorker: OwnerBeatWorker = { + on(event, listener) { if (event === "error") errorListener = listener; }, + unref() {}, + postMessage() {}, + terminate() { return Promise.resolve(undefined); }, + }; + const stop = runOwnerHeartbeatWorker(stub.origin, process.pid, 150, () => fakeWorker); + try { + assert.equal(typeof errorListener, "function", "error handler armed before the crash"); + errorListener!(new Error("simulated thread death")); + assert.ok(await until(() => diag.length >= 1, 8_000), `in-thread fallback must report recovery too (got: ${JSON.stringify(diag)})`); + assert.equal(diag.length, 1, "still exactly one notice"); + stop(); + const frozen = stub.bodies.length; + await wait(400); + assert.equal(stub.bodies.length, frozen, "stop() silences the fallback together with the entry"); + } finally { + stop(); + await stub.close(); + } +}); + +test("#5641 injected-post seam: resolving TRUE acks, anything else does not", async () => { + const origin = "http://127.0.0.1:5641-injected"; + const diag: string[] = []; + markWatcherPendingRecovery(origin, process.pid, (m) => diag.push(m)); + let posts = 0; + const stop = startOwnerHeartbeat(origin, process.pid, 120, async () => { + posts++; + return posts >= 2; // first beat unresolved-ish, second is an ack + }); + assert.equal(typeof stop, "function"); + try { + assert.ok(await until(() => diag.length >= 1, 5_000), `injected post resolving true drives the transition (posts=${posts})`); + const settled = diag.length; + await wait(400); + assert.equal(diag.length, settled, "no repeat notices"); + } finally { + stop!(); + } + + // Negative direction: a post that never resolves TRUE never recovers. + const quiet: string[] = []; + const origin2 = "http://127.0.0.1:5641-injected-neg"; + markWatcherPendingRecovery(origin2, process.pid, (m) => quiet.push(m)); + const stop2 = startOwnerHeartbeat(origin2, process.pid, 120, async () => { /* records nothing */ }); + try { + await wait(500); + assert.equal(quiet.length, 0, "falsy/undefined post results are not acknowledgements"); + } finally { + stop2?.(); + } +}); + +// ── 4. End-to-end through ensureProxyRunning (the reported scenario) ──────── + +test("#5641 end-to-end: attach with failed registration, real heartbeat recovers with one notice, zero spawns", { timeout: 30_000 }, async () => { + const stub = await stubWatcher(200); + const diag: string[] = []; + let regCalls = 0; + const port = Number(new URL(stub.origin).port); + const handle = await launchWith(attachDeps({ + origin: stub.origin, + port, + registerWatcher: async () => { regCalls++; return "failed"; }, + diag: (m) => diag.push(m), + }), port); + try { + assert.equal(handle.attached, true); + assert.equal(handle.watcherRegistration, "failed", "handle admits the registration is unconfirmed"); + assert.equal(regCalls, 3, "bounded retries ran during attach"); + assert.ok(diag.some((m) => m.includes("attaching to running proxy")), "attach itself proceeded normally"); + + // The internal startOwnerHeartbeat uses the 5s production interval — + // the first acknowledged beat lands ~5s out. Generous deadline like + // the #5152 suite (worker cold start under CI load). + assert.ok( + await until(() => diag.some((m) => m.includes("watcher registration recovered")), 12_000), + `the recovery notice must reach the host-safe diagnostic channel (diag=${JSON.stringify(diag)})`, + ); + assert.equal(diag.filter((m) => m.includes("watcher registration recovered")).length, 1, "exactly one recovery notice"); + assert.ok(stub.bodies.length >= 1, "the owning heartbeat did the re-registering — no extra proxy involved"); + assert.deepEqual(JSON.parse(stub.bodies[0]), { pid: process.pid, hb: true }); + } finally { + await stub.close(); + } +}); From c37293b691003955194f30167e846208cc0dcff5 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 4 Nov 2026 13:03:31 +0800 Subject: [PATCH 4/6] test(e2e): pin e2e-native-pi to the pre-#5428 spawn posture (#4599) #5428 made inline the pi lane default; this suite documents the spawn contract (instance record + proxy origin, per-request stamps, the shared-proxy delegate-child channel, /__bili/plugin/status over HTTP) and was not updated with the flip. Under the inline default its delegate-child stamp assertions get 0 stamped child requests (child traffic reaches upstream unstamped, conv=null) and the /acp instance- record assertion fails structurally. Pin BILI_PI_INLINE=0 for the whole suite so the spawn-posture coverage stays deterministic; inline posture is covered by tests/pi-inline-ws.test.ts / tests/pi-native.test.ts. --- tests/e2e/e2e-native-pi.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/e2e/e2e-native-pi.test.ts b/tests/e2e/e2e-native-pi.test.ts index e71682096d..7f0ee66eb3 100644 --- a/tests/e2e/e2e-native-pi.test.ts +++ b/tests/e2e/e2e-native-pi.test.ts @@ -458,6 +458,17 @@ function piRun( cwd: ctx.piCwd, env: { ...cleanEnv(), + // This suite documents the SPAWN-posture package-native contract: + // instance record + proxy origin, per-request wire stamps, the + // shared-proxy delegate-child ACP channel (#2268), /__bili/plugin/status + // over HTTP. #5428 flipped the pi lane DEFAULT to inline (in-process + // engine, no instance record) and this suite was not updated with it: + // primary-lane tests then pass for the wrong reason while the + // instance-record and delegate-child stamp assertions fail structurally + // (CI sentinel incident, #4599). Pin the legacy posture so this + // coverage stays deterministic; inline posture is covered by + // tests/pi-inline-ws.test.ts and tests/pi-native.test.ts. + BILI_PI_INLINE: "0", PI_CODING_AGENT_DIR: ctx.piAgentDir, XDG_CONFIG_HOME: ctx.xdg.config, XDG_CACHE_HOME: ctx.xdg.cache, From 30b18d15683df0c3526532639547c6fbe54995fa Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 13:57:26 +0800 Subject: [PATCH 5/6] fix: document the foreign-pid lifetime of watcherPendingRecovery entries (#5641) Review nit (comment-only): the doc comment listed three ways a pending-recovery entry dies, but a foreign-owner-pid entry has no heartbeat to stop it and lives until process exit or a later direct-ok attach to the same origin. State that so the residual is visible in-code. No behavior change. --- src/launcher.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/launcher.ts b/src/launcher.ts index d76ae6f341..336077c5c6 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -4938,7 +4938,9 @@ const ownerHeartbeatWorkers = new Map void; refs: number } * recovered with exactly ONE host-visible notice through the diagnostic * channel captured at attach time. A resolved fetch returning 409/5xx never * counts as recovery. Entries die on recovery, on a direct "ok" - * re-registration, or when the origin's last heartbeat stops. Exported + * re-registration, or when the origin's last heartbeat stops; a + * foreign-owner-pid entry (no heartbeat runs for it) simply lives until + * process exit or a later direct-"ok" attach to the same origin. Exported * alongside noteWatcherHeartbeatAck so the regression suite can exercise the * transition without a full proxy bring-up. */ const watcherPendingRecovery = new Map void }>(); From 50318a49b0334cd9a80b0bc311e0c19230e666cf Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 14:26:19 +0800 Subject: [PATCH 6/6] test: make msgid-lane-flow settle deterministic via afterNextFlush (#5644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit win22-s2 red on head 30b18d156: 'a delta re-claiming a live canonical id' read ['u1'] where ['a1','u1','u2'] was expected — the HTTP turn flow was correct (200/400/appliedSeq all asserted fine); only the durable rows were missing. Mechanism: the lane wrap-up arms a debounced save AFTER the ack (incremental-lane.ts), whose async write chain fsyncs behind the test's fixed 60ms settle sleep; under loaded windows-latest runners the fsync outran the budget and rows() saw the pre-turn snapshot. Same code green on the previous head (71c267515) and on every other platform/shard. Replace the fixed sleep with the store's own landing verdict (afterNextFlush, #5155/#4788 D2 — the same signal the production pointer upkeep already awaits): deterministic when the save is slow, and a false verdict fails loud at the verdict instead of as a confusing missing-row diff if a save genuinely fails. --- tests/msgid-lane-flow.test.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/msgid-lane-flow.test.ts b/tests/msgid-lane-flow.test.ts index 30e0798d13..8fed5c7817 100644 --- a/tests/msgid-lane-flow.test.ts +++ b/tests/msgid-lane-flow.test.ts @@ -54,9 +54,10 @@ async function harness(pin: boolean): Promise { resetToolRingForTest(); __resetIncrementalLaneForTests(); // #5155: the wrap-up schedules a debounced save instead of a synchronous - // writeNow per turn — rows settle shortly AFTER the ack. A short debounce - // plus settle() keeps the row reads deterministic; the identity contracts - // under test are unchanged. + // writeNow per turn — rows settle shortly AFTER the ack. settle() awaits + // the store's own landing verdict (afterNextFlush), not a fixed sleep, so + // the row reads stay deterministic even when the fsync outruns a sleep + // budget under load; the identity contracts under test are unchanged. const store = new SessionStore({ dir: dir + "/sessions", enabled: true, debounceMs: 15 }); _setStoreForTest(store); _setForTest({}); @@ -100,7 +101,16 @@ async function harness(pin: boolean): Promise { db.close(); } }; - return { turn, forwarded, rows, settle: async () => { await new Promise((resolve) => setTimeout(resolve, 60)); }, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; + const settle = async (): Promise => { + // A fixed sleep here raced the debounced save's fsync on loaded + // windows-latest runners (the row read saw the pre-turn snapshot — + // PR #5644 win22-s2 red). Await the store's landing verdict instead; + // a false verdict means a save genuinely failed — fail loud there, + // not on the missing row below. + const ok = await store.afterNextFlush("conv-msgid-1"); + assert.ok(ok, "debounced row save landed on disk before the row read"); + }; + return { turn, forwarded, rows, settle, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; } const wmsg = (text: string) => ({ role: "user", content: [{ type: "text", text: text.repeat(200) }] });