From 3718483a7e476d787d5b4e4a4c6f32a99309833d Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 7 Nov 2026 12:44:00 +0800 Subject: [PATCH 1/6] fix(retry): transient-4xx classifier honors upstream machine retry declaration (#3606 side-lane part) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isTransientUpstreamError (src/fetch-util.ts) gains two OR'd true-paths for 4xx bodies: canonical wording markers (please retry / 请重试; bare 'retry' deliberately NOT added — 'do not retry' false-positive pinned by negative test) and bodyDeclaresRetryable(): bounded JSON scan (64 KiB cap, 4096-node budget) for actions:[...RETRY] or retryable:true, unwrapping string fields that parse as JSON (the code-11101 envelope behind #3606). Behavior change: side lanes (acp-loop replay, compress-loop-responses, preflight summarization) now retry ≤ network.replayRetryMax on such bodies instead of fail-fast. Main forwarding path untouched by design (#1688); sub-defect A of #3606 (main-path HTTP-level retry) still awaits owner decision — this PR must NOT close #3606. Picked up from PR #3610. Lock-file version drift from its base no longer present against current master. --- src/fetch-util.ts | 68 +++++++++++++++++++++++++++++++++++++++-- tests/loop-core.test.ts | 50 ++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/fetch-util.ts b/src/fetch-util.ts index 08854ec230..979f16abac 100644 --- a/src/fetch-util.ts +++ b/src/fetch-util.ts @@ -316,7 +316,10 @@ export class UpstreamHttpError extends Error { * rate-limit rejection rather than a genuine client error. GLM Coding Plan * returns 400 {"code":3007,"msg":"captcha verify failed"} ~1s after large * context rewrites (issue #189); every observed case recovered on retry, - * so such bodies are retried while plain 4xx (bad model, bad params) fail fast. */ + * so such bodies are retried while plain 4xx (bad model, bad params) fail + * fast. "please retry" / "请重试" cover the canonical "just retry" phrasing of + * risk-control gateways (issue #3606). Bare "retry" is deliberately absent — + * "do not retry" / "请勿重试" must stay non-transient. */ const TRANSIENT_BODY_MARKERS = [ "captcha", "verify failed", @@ -325,13 +328,74 @@ const TRANSIENT_BODY_MARKERS = [ "rate limit", "too many requests", "try again", + "please retry", + "请重试", ]; +// #3606: an explicit machine self-declaration outranks prose sniffing — a +// gateway that says its rejection is retryable means it, and enumerating +// every vendor's wording will never converge. Bodies past this cap are too +// large to be JSON error envelopes; skip the parse. +const DECLARATION_BODY_CAP = 65_536; +// Node visit budget for the declaration scan: bounds worst-case work on +// pathological (but still small) bodies. +const DECLARATION_NODE_BUDGET = 4_096; + export function isTransientUpstreamError(status: number, body: string): boolean { if (status === 429 || status >= 500) return true; if (status < 400) return false; const lower = body.toLowerCase(); - return TRANSIENT_BODY_MARKERS.some((marker) => lower.includes(marker)); + if (TRANSIENT_BODY_MARKERS.some((marker) => lower.includes(marker))) return true; + return bodyDeclaresRetryable(body); +} + +// #3606: honor the upstream's explicit retryability declaration — an "actions" +// array containing "RETRY" (case-insensitive) or a boolean "retryable": true +// anywhere in the error JSON. The scan walks nested objects AND string values +// that themselves parse as JSON, because gateways wrap their envelope in a +// string field ({"error":{"message":"{...}"}}, the shape behind issue #3606's +// code 11101 bodies). A string "true" or an actions list without RETRY is NOT +// a declaration — only unambiguous machine signals count. +function bodyDeclaresRetryable(body: string): boolean { + if (body.length > DECLARATION_BODY_CAP) return false; + const root = parseJsonObject(body); + if (root === undefined) return false; + const queue: unknown[] = [root]; + let head = 0; + let visited = 0; + while (head < queue.length && visited < DECLARATION_NODE_BUDGET) { + const node = queue[head++]; + visited += 1; + if (typeof node === "string") { + const nested = parseJsonObject(node); + if (nested !== undefined) queue.push(nested); + } else if (Array.isArray(node)) { + for (const item of node) queue.push(item); + } else if (node !== null && typeof node === "object") { + for (const [key, value] of Object.entries(node)) { + const loweredKey = key.toLowerCase(); + if (loweredKey === "actions" && Array.isArray(value) + && value.some((item) => typeof item === "string" && item.toLowerCase() === "retry")) { + return true; + } + if (loweredKey === "retryable" && value === true) { + return true; + } + if (value !== null && typeof value === "object") queue.push(value); + else if (typeof value === "string" && /^[{\[]/.test(value.trimStart())) queue.push(value); + } + } + } + return false; +} + +function parseJsonObject(text: string): object | undefined { + try { + const value: unknown = JSON.parse(text); + return typeof value === "object" && value !== null ? value : undefined; + } catch { + return undefined; + } } /** Total requests per replay attempt (initial + retries). */ diff --git a/tests/loop-core.test.ts b/tests/loop-core.test.ts index e8bc5679d4..b0012ca38e 100644 --- a/tests/loop-core.test.ts +++ b/tests/loop-core.test.ts @@ -564,6 +564,56 @@ test("isTransientUpstreamError: classifier matrix", () => { assert.equal(REPLAY_MAX_ATTEMPTS, 3); }); +// Regression guard for #3606: risk-control gateways return 400 whose body +// machine-declares retryability (code 11101, actions:["RETRY"]) or uses the +// canonical "Please retry" / 「请重试」 phrasing — none of which hit the +// original seven markers, so every such rejection failed fast instead of +// being replayed. +const RETRY_DECLARED_400_BODY = JSON.stringify({ error: { message: JSON.stringify({ + code: 11101, + msg: "Parse message failed: unsupported content type at index 0: tool_use", + requestId: "req-test", + displayMsg: { en: "Bad request.", zh: "请求不合法" }, + actions: ["RETRY", "SUBMIT_FEEDBACK"], +}) } }); + +test("isTransientUpstreamError: explicit retry declarations (#3606)", () => { + assert.equal(isTransientUpstreamError(400, RETRY_DECLARED_400_BODY), true, 'JSON-string-wrapped actions:["RETRY"] 400 is transient'); + assert.equal(isTransientUpstreamError(400, '{"code":11101,"actions":["RETRY","SUBMIT_FEEDBACK"]}'), true, "flat actions:[RETRY] 400 is transient"); + assert.equal(isTransientUpstreamError(400, '{"error":{"retryable":true,"message":"temporary"}}'), true, "nested retryable:true 400 is transient"); + assert.equal(isTransientUpstreamError(400, '{"message":"please retry later"}'), true, '"please retry" wording is transient'); + assert.equal(isTransientUpstreamError(400, '{"displayMsg":{"en":"Invalid request. Please retry","zh":"请求内容不合法,请重试"},"msg":"x"}'), true, "Please retry / 请重试 wording is transient"); + // negatives: only unambiguous machine signals or canonical phrasing count + assert.equal(isTransientUpstreamError(400, '{"message":"invalid api_key, do not retry"}'), false, '"do not retry" stays non-transient'); + assert.equal(isTransientUpstreamError(400, '{"retryable":"true"}'), false, 'string "true" is not a boolean declaration'); + assert.equal(isTransientUpstreamError(400, '{"actions":["SUBMIT_FEEDBACK"]}'), false, "actions without RETRY is not transient"); + assert.equal(isTransientUpstreamError(400, "not json at all"), false, "non-JSON body without markers is not transient"); +}); + +test("replay retry: upstream-declared transient 400 then success → retried (#3606)", async () => { + process.env.BILI_REPLAY_RETRY_BASE_MS = "1"; + let fetchCalls = 0; + const orig = globalThis.fetch; + globalThis.fetch = (async () => { + fetchCalls++; + if (fetchCalls === 1) return new Response(RETRY_DECLARED_400_BODY, { status: 400 }); + return new Response(COMPLETED, { status: 200 }); + }) as typeof fetch; + try { + const out = await drain( + new Response(compressRound(), { status: 200 }).body!, + replayCtx(), + { model: "gpt-4o", input: [], stream: true }, + { url: "http://mock", headers: {} }, + ); + assert.equal(fetchCalls, 2, "declared-transient 400 retried once"); + assert.ok(!out.includes("upstream error"), "no upstream error surfaced to client"); + } finally { + delete process.env.BILI_REPLAY_RETRY_BASE_MS; + globalThis.fetch = orig; + } +}); + test("replayBackoffMs: exponential from env-tunable base", () => { process.env.BILI_REPLAY_RETRY_BASE_MS = "100"; try { From 7a923f8bb74da7681c4548138bcfbcea09226c32 Mon Sep 17 00:00:00 2001 From: Aldo Adirajasa Fathoni Date: Sat, 7 Nov 2026 12:46:40 +0800 Subject: [PATCH 2/6] fix(served-markers): ignore prior container generations with same PID on restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In containers processes virtually always run as PID 1; after a restart the old writer marker (pid: 1) still passes isPidAlive() against the new process, so findContestedWriters false-positives [split-brain] on every boot. Fix: record startedAt (epoch ms) on WriterIdentity/WriterMarkerFile; a marker sharing our PID whose startedAt (or mtime for older builds without startedAt) predates this process boot by more than SAME_PID_GENERATION_MARGIN_MS is skipped as a prior generation. Integrated onto master's contestedServedAt helper (#4626); both mechanisms keep working. Original commit c115e351 of PR #5028 (external contribution by @fathonix; later review-fix commits of the same PR condensed into this one — full credit retained via git author). --- src/served-markers.ts | 26 +++++++++++++++++++++++++- src/server.ts | 4 ++-- tests/served-markers.test.ts | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/served-markers.ts b/src/served-markers.ts index 0b29bd11b8..c79e93a6f2 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -29,6 +29,7 @@ export interface WriterIdentity { /** #3923/#3959: this instance's lane (e.g. "pi", "dsh") — additive; * markers written by older builds carry none and stay valid. */ lane?: string; + startedAt?: number; } interface WriterMarkerFile { @@ -38,6 +39,7 @@ interface WriterMarkerFile { origin: string; version: string; lane?: string; + startedAt?: number; /** Set only by a graceful dispose(): this instance recorded its FINAL * marker and will never commit again. Peers skip such markers outright — * a positively-known dead predecessor is not a concurrent writer, even @@ -92,11 +94,21 @@ const PEER_FILE_FRESH_MS = 30_000; /** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is * older than this. */ const STALE_REAP_MS = 60 * 60_000; +/** Same-PID markers that predate this process's boot by more than this margin + * belong to an earlier container generation (the PID was reused, not shared) + * — the margin absorbs clock/rounding skew between generations (#3923). */ +const SAME_PID_GENERATION_MARGIN_MS = 1_000; export function writerMarkerDir(stateDirPath: string): string { return path.join(stateDirPath, "writers"); } +/** This process's boot time (ms epoch), derived from uptime against the + * caller's clock so tests can pin it. */ +export function processStartMs(now: number): number { + return now - Math.round(process.uptime() * 1000); +} + let tmpSeq = 0; function durableAtomicWrite(file: string, text: string): void { @@ -157,6 +169,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, origin: identity.origin, version: identity.version, ...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }), + ...(typeof identity.startedAt === "number" ? { startedAt: identity.startedAt } : {}), ...(stoppedAt === undefined ? {} : { stoppedAt }), servedAt: Object.fromEntries(servedAt), ...(final ? { disposedAt: now } : {}), @@ -250,7 +263,9 @@ function contestedServedAt(raw: Partial, selfInstanceId: strin * window by a LIVE foreign instance (pid alive or file fresh). Markers * carrying a graceful-stop stamp (`stoppedAt`, or the `disposedAt` alias, * #4626) are skipped unconditionally — a restart flow's predecessor died - * seconds ago and must not raise an alarm. Malformed or tearing files are + * seconds ago and must not raise an alarm. Same-PID markers from an earlier + * container generation (booted before this process) are likewise skipped — + * the PID was reused, not shared (#3923). Malformed or tearing files are * skipped silently — a marker is observability, not data. */ export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array { let entries: fs.Dirent[]; @@ -266,6 +281,14 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; const mtimeMs = fs.statSync(file).mtimeMs; + // #3923: in containers PID is typically 1 across restarts. A marker + // that shares our PID but predates this process's boot belongs to a + // prior container generation, not a live twin (older builds carry no + // startedAt — the marker mtime then stands in for their last write). + const samePid = typeof raw.pid === "number" && raw.pid === process.pid; + const peerStartedAt = typeof raw.startedAt === "number" ? raw.startedAt : mtimeMs; + if (samePid && peerStartedAt < processStartMs(now) - SAME_PID_GENERATION_MARGIN_MS) continue; + const at = contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now); if (at === null || typeof raw.instanceId !== "string") continue; out.push({ @@ -274,6 +297,7 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio origin: typeof raw.origin === "string" ? raw.origin : "unknown", version: typeof raw.version === "string" ? raw.version : "unknown", lane: typeof raw.lane === "string" && raw.lane !== "" ? raw.lane : undefined, + ...(typeof raw.startedAt === "number" ? { startedAt: raw.startedAt } : {}), servedAt: at, }); } catch { diff --git a/src/server.ts b/src/server.ts index 1b1e2da049..7059a79cbd 100644 --- a/src/server.ts +++ b/src/server.ts @@ -99,7 +99,7 @@ import { noteRequest } from "./attach-heartbeat.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, sessionsDir, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryShape, findPresentUnverifiedSameLaneTwin, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, markPortReleased, probePid, readProxyInstanceFile, registerInstanceAndWarn, resolveOwnFingerprint, sameInstanceShape, SPAWN_FINGERPRINT_ENV, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; -import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; +import { createServedMarkerWriter, peerStillContests, processStartMs, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js"; import { holderProbeOrigin, mayClobberPointer, probeHolderForHint, probeOwnerInfo, resolvePortClaim, verifyEntryOwnsPort } from "./port-owner.js"; import { armEventLoopWatchdog } from "./event-loop-watchdog.js"; @@ -1347,7 +1347,7 @@ export async function startServer(opts: ProxyOptions): Promise { { const markerDir = writerMarkerDir(stateDir()); servedWriter = createServedMarkerWriter( - { instanceId, pid: process.pid, origin, version: VERSION, lane: launcherLane }, + { instanceId, pid: process.pid, origin, version: VERSION, lane: launcherLane, startedAt: processStartMs(Date.now()) }, markerDir, (level, msg) => log(level, msg), { diff --git a/tests/served-markers.test.ts b/tests/served-markers.test.ts index e3245bcb67..d0829b351d 100644 --- a/tests/served-markers.test.ts +++ b/tests/served-markers.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { createServedMarkerWriter, findContestedWriters, reapStaleWriterMarkers, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; +import { createServedMarkerWriter, findContestedWriters, processStartMs, reapStaleWriterMarkers, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; // #3923: per-session writer markers — session-level split-brain visibility. // Two live bili processes sharing one sessions directory already warn at @@ -197,3 +197,36 @@ test("reapStaleWriterMarkers removes dead+stale files, keeps live and recent one assert.equal(fs.existsSync(deadFile), false, "provably-dead + beyond reap horizon → removed"); assert.equal(fs.existsSync(liveFile), true, "live pid → kept regardless of age"); }); + +test("container restart with same PID (pid 1) does not falsely contest prior generation", () => { + const dir = writerMarkerDir(freshDir("bili-container-pid1-")); + // Anchor to THIS process's boot so the assertion holds no matter how long + // the test process has been alive (a fixed offset from Date.now() only + // works while uptime stays under ~29 s). + const selfBoot = processStartMs(Date.now()); + const priorPeer: WriterIdentity = { + instanceId: "prior-container-gen", + pid: process.pid, + origin: "http://127.0.0.1:8787", + version: "0.1.204", + startedAt: selfBoot - 30_000, + }; + writePeerMarker(dir, priorPeer, { "sess-container": Date.now() - 5000 }); + const found = findContestedWriters(dir, SELF.instanceId, "sess-container", Date.now()); + assert.equal(found.length, 0, "prior container generation with same PID is not a live peer"); +}); + +test("same-PID peer that booted after this process still contests (generation gate is one-way)", () => { + const dir = writerMarkerDir(freshDir("bili-container-pid1-young-")); + const selfBoot = processStartMs(Date.now()); + const youngerTwin: WriterIdentity = { + instanceId: "younger-twin-gen", + pid: process.pid, + origin: "http://127.0.0.1:8788", + version: "0.1.204", + startedAt: selfBoot + 5_000, + }; + writePeerMarker(dir, youngerTwin, { "sess-container": Date.now() - 5000 }); + const found = findContestedWriters(dir, SELF.instanceId, "sess-container", Date.now()); + assert.equal(found.length, 1, "a same-PID twin that booted later than us is still a live peer"); +}); From 145c5673bd023ec62d84a3d7fc90105e1506863c Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 7 Nov 2026 12:46:46 +0800 Subject: [PATCH 3/6] fix(cache-keepalive): treat ttl-less cache_control breakpoints as 5m tier so keepalive actually arms (#5665) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit detectKeepaliveTier only armed when a breakpoint carried an explicit ttl of 1h/5m; the default form cache_control:{type:ephemeral} (no ttl) returned undefined and armInternal exited silently — contradicting CONFIGURATION.md (which said: 1h when any breakpoint declares ttl:1h, else 5m). Now any breakpoint without explicit ttl is 5m-tier; explicit ttl:1h still lifts to 1h; intervalMs still wins. Picked up from PR #5669 (issue #5665). --- src/cache-keepalive.ts | 5 ++++- tests/cache-keepalive.test.ts | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/cache-keepalive.ts b/src/cache-keepalive.ts index 333332c53c..63c4c0edf8 100644 --- a/src/cache-keepalive.ts +++ b/src/cache-keepalive.ts @@ -132,7 +132,10 @@ export function detectKeepaliveTier(body: unknown): "5m" | "1h" | undefined { found = "1h"; return; } - if (ttl === "5m") found = "5m"; + // A cache_control breakpoint without an explicit ttl uses Anthropic's + // default 5-minute TTL, so it is a 5m-tier breakpoint (#5667); only an + // explicit "1h" lifts the tier. + found = "5m"; } for (const v of Object.values(o)) walk(v); }; diff --git a/tests/cache-keepalive.test.ts b/tests/cache-keepalive.test.ts index a1146d0e8e..9281b65d74 100644 --- a/tests/cache-keepalive.test.ts +++ b/tests/cache-keepalive.test.ts @@ -161,6 +161,14 @@ const BODY_NO_BP = JSON.stringify({ system: "plain system", messages: [{ role: "user", content: "hi" }], }); +// #5667: a breakpoint with no explicit ttl — Anthropic's default ephemeral TTL is 5m. +const BODY_NO_TTL = JSON.stringify({ + model: "claude-opus-test", + max_tokens: 1024, + stream: true, + system: [{ type: "text", text: "sys", cache_control: { type: "ephemeral" } }], + messages: [{ role: "user", content: "hi" }], +}); let timers: FakeTimers; let sid: string; @@ -213,6 +221,19 @@ test("detectKeepaliveTier: 5m / 1h / mixed / nested / none", () => { messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral", ttl: "1h" } }] }], }; assert.equal(detectKeepaliveTier(mixed), "1h"); + // #5667: a breakpoint without an explicit ttl is Anthropic's default 5m tier, + // NOT "no breakpoint detected". Only bodies with zero cache_control are undefined. + assert.equal(detectKeepaliveTier(JSON.parse(BODY_NO_TTL)), "5m"); + const mixedNoTtl = { + system: [{ text: "a", cache_control: { type: "ephemeral", ttl: "5m" } }], + messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral" } }] }], + }; + assert.equal(detectKeepaliveTier(mixedNoTtl), "5m"); + const oneHourAmongNoTtl = { + system: [{ text: "a", cache_control: { type: "ephemeral" } }], + messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral", ttl: "1h" } }] }], + }; + assert.equal(detectKeepaliveTier(oneHourAmongNoTtl), "1h"); assert.equal(detectKeepaliveTier(JSON.parse(BODY_NO_BP)), undefined); assert.equal(detectKeepaliveTier(null), undefined); }); @@ -297,6 +318,17 @@ test("arm: 1h breakpoints arm at 1h×0.9", () => { assert.equal(_keepaliveEntryForTest(sid)?.cadenceMs, 3_240_000); }); +// #5667 regression: a breakpoint with no explicit ttl used to return tier +// undefined and silently never arm; it must now arm at the 5m cadence. +test("arm: no-ttl breakpoints arm at the default 5m tier", () => { + noteSuccessfulForward(armInput({ wireBody: BODY_NO_TTL })); + const e = _keepaliveEntryForTest(sid); + assert.ok(e, "session armed from a no-ttl breakpoint"); + assert.equal(e!.cadenceMs, 270_000); + assert.equal(e!.tierLabel, "5m"); + assert.deepEqual(_keepaliveArmedForTest(), [sid]); +}); + for (const [name, over] of [ ["non-anthropic protocol never arms", { protocol: "openai" as const }], ["GET never arms", { reqMethod: "GET" }], From cbad22e0fdd75bda34a091abcbb4f39a20207b9e Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 7 Nov 2026 12:46:57 +0800 Subject: [PATCH 4/6] fix(cache-keepalive): unref the keepalive timers so short-lived inline-lane hosts can exit (#5672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recurring keepalive setTimeout had no .unref(), so on an inline lane (proxy inside host process) a short-lived host (pi -p / sub-agent print-json run) kept living until maxDurationMs (default 24h), pinging upstream the whole time — the armed timer was typically the only live handle left after the turn ended. Default scheduler and test-reset now route through one realSchedule() helper calling t.unref?.() at every arm/reschedule site (initial arm, busy-reschedule, post-ping reschedule), matching the existing repo idiom (~49 other unref call sites). Picked up from PR #5672. --- scripts/live-files.mjs | 6 +++++ src/cache-keepalive.ts | 15 +++++++++-- tests/cache-keepalive.test.ts | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index d7a8f03a48..21faf81617 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -111,6 +111,12 @@ export const LIVE_FILES = [ // (SIGTERM/SIGKILL/clean shutdown). Private mkdtemp homes per lane-arm, // probed-free ports re-probed on every restart — no band reuse. "tests/issue5631-restart-survival.test.ts", + // #5666: process-liveness regression — spawns ONE bare node child (--import + // tsx) that arms a keepalive through the real scheduler and asserts it exits + // on its own; the child runs under a per-run mkdtemp XDG sandbox, does no + // network I/O and binds no port, so no zone band is needed — the entry is for + // the tripwire alone, like #3890/#5114 + "tests/cache-keepalive.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/src/cache-keepalive.ts b/src/cache-keepalive.ts index 63c4c0edf8..38138f5d31 100644 --- a/src/cache-keepalive.ts +++ b/src/cache-keepalive.ts @@ -175,8 +175,19 @@ interface KeepaliveEntry { const registry = new Map(); +/** Real-timer scheduler: every handle is unref'd (#5666) so an armed cadence + * timer never holds the event loop open. Interactive hosts stay alive for + * their own reasons and the pings still fire on schedule; a short-lived + * process (pi -p, a sub-agent) whose only remaining handle is the armed + * keepalive now exits cleanly instead of limping along until maxDurationMs. */ +function realSchedule(fn: () => void, ms: number): ReturnType { + const t = setTimeout(fn, ms); + t.unref?.(); + return t; +} + let clock: () => number = () => Date.now(); -let scheduleFn: (fn: () => void, ms: number) => ReturnType = (fn, ms) => setTimeout(fn, ms); +let scheduleFn: (fn: () => void, ms: number) => ReturnType = realSchedule; let cancelFn: (id: ReturnType) => void = (id) => clearTimeout(id); export interface KeepaliveArmInput { @@ -457,7 +468,7 @@ export function _setKeepaliveSchedulerForTest(sched: FakeScheduler | undefined): scheduleFn = (fn, ms) => sched.schedule(fn, ms) as ReturnType; cancelFn = (id) => sched.cancel(id as unknown); } else { - scheduleFn = (fn, ms) => setTimeout(fn, ms); + scheduleFn = realSchedule; cancelFn = (id) => clearTimeout(id); } } diff --git a/tests/cache-keepalive.test.ts b/tests/cache-keepalive.test.ts index 9281b65d74..cbab1b97ac 100644 --- a/tests/cache-keepalive.test.ts +++ b/tests/cache-keepalive.test.ts @@ -6,6 +6,11 @@ import { test, before, after, beforeEach } from "node:test"; import assert from "node:assert"; import http from "node:http"; +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { buildPingBody, @@ -539,3 +544,49 @@ test("keepaliveStatusFor: mirrors armed state, ping history and disarm", async ( await settle(() => _keepaliveArmedForTest().length === 0); assert.equal(keepaliveStatusFor(sid), undefined, "disarmed sessions report no state"); }); + +// ---- process-liveness (real timers) --------------------------------------- + +test("arm: the real scheduler unrefs its timer, so short-lived processes exit (#5666)", async () => { + // A child arms a keepalive through the DEFAULT (non-injected) scheduler and + // then does nothing else. The armed cadence timer is that process's only + // remaining handle — exactly the inline-lane pi -p / sub-agent shape. With + // the fix the timer is unref'd, so the event loop drains and the process + // exits on its own; without it the ref'd timer holds the loop open and the + // child hangs until the (270 s) timer fires. Hermetic: no network (the ping + // URL is never reached before the process is gone). + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ka-exit-")); + for (const d of ["cfg", "state", "data"]) fs.mkdirSync(path.join(tmp, d), { recursive: true }); + const env = { + ...process.env, + XDG_CONFIG_HOME: path.join(tmp, "cfg"), + XDG_STATE_HOME: path.join(tmp, "state"), + XDG_DATA_HOME: path.join(tmp, "data"), + ACP_LOG_FILE: "off", + }; + const kaUrl = pathToFileURL(fileURLToPath(new URL("../src/cache-keepalive.ts", import.meta.url))).href; + const code = ` +process.env.BILI_CACHE_KEEPALIVE = "1"; +process.env.BILI_CACHE_KEEPALIVE_ASSUME_TTL = "5m"; +const { noteSuccessfulForward } = await import(${JSON.stringify(kaUrl)}); +noteSuccessfulForward({ + sessionId: "ka-exit", protocol: "anthropic", reqMethod: "POST", sidePassthrough: false, resignLane: false, + wireBody: JSON.stringify({ model: "claude-opus-test", max_tokens: 4096, stream: true, system: [{ type: "text", text: "s", cache_control: { type: "ephemeral", ttl: "5m" } }], messages: [{ role: "user", content: "hi" }] }), + upstreamUrl: "http://127.0.0.1:9/v1/messages", headers: { "content-type": "application/json" }, dispatcher: undefined, routes: undefined, +}); +`; + const child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { + cwd: fileURLToPath(new URL("../", import.meta.url)), + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let out = ""; + child.stdout?.on("data", (c) => (out += String(c))); + child.stderr?.on("data", (c) => (out += String(c))); + const exited = new Promise((resolve) => child.on("close", (c) => resolve(c))); + const timedOut = new Promise((resolve) => setTimeout(() => resolve(null), 10_000).unref()); + const exitCode = await Promise.race([exited, timedOut]); + child.kill(); + fs.rmSync(tmp, { recursive: true, force: true }); + assert.equal(exitCode, 0, `an armed keepalive must not hold the event loop open (got ${exitCode === null ? "TIMEOUT — the cadence timer kept the process alive" : `exit code ${exitCode}`})\n--- child output ---\n${out}`); +}); From 151efcacfb86c521129f5ddcade010cfc1389f05 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 7 Nov 2026 13:26:21 +0800 Subject: [PATCH 5/6] test(#5719): realign two stale pins after #5734 changed the re-anchor rung order (30a8bc2d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream 30a8bc2d (merged via #5734 on Nov 6) made an authored-checkpoint conv declaration re-anchor on the raw lane BEFORE the store-overlap rung, and even when the store view is null — its own commit message documents 'this shape went from forks to re-anchors'. It added tests/issue5719-self-checkpoint-reanchor.test.ts but did not realign T1/T8 of this older suite, leaving every old-base PR (and the main branch itself) red on exactly those two pins. T1 now pins the self-declaration reason string (same outcome asserts kept: key raw + action reanchor); T8 now expects reanchor on the raw key where it previously expected the byte-evidence degrade to derived-persona/sub-fork. Marker-less shapes untouched (their pins still pass unmodified). --- tests/issue5719-msgid-lane-identity.test.ts | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/issue5719-msgid-lane-identity.test.ts b/tests/issue5719-msgid-lane-identity.test.ts index b3a700ef96..a67f971bed 100644 --- a/tests/issue5719-msgid-lane-identity.test.ts +++ b/tests/issue5719-msgid-lane-identity.test.ts @@ -112,8 +112,13 @@ test("#5719 T1: composite field failure + overlapping host ids re-anchor the raw ); assert.equal(res.key, raw, `expected raw re-anchor, got ${res.key} (${res.action})`); assert.equal(res.action, "reanchor"); - assert.match(res.reason ?? "", /msgid identity \(#5719\)/); - assert.match(res.reason ?? "", /1 host message id\(s\) match/); + // #5734 realignment (after 30a8bc2d): this incident cell carries the authored + // checkpoint's conv declaration, so the self-declaration rung now fires BEFORE + // the store-overlap rung — same outcome (reanchor on the raw key), earlier + // reason. Pre-#5734 the pin read /msgid identity \(#5719\)/ + + // /1 host message id\(s\) match/. + assert.match(res.reason ?? "", /self-authored checkpoint declares this raw id/); + assert.match(res.reason ?? "", /#5719/); }); test("#5719 T2: same composite failure but DISJOINT ids still forks (owner rule 3 guard rail)", () => { @@ -261,12 +266,15 @@ test("#5719 T7: personaConversationOrder forwards observer + ids + lazy lookup", assert.ok(lookups <= 1, `lazy contract: store queried at most once (got ${lookups})`); }); -test("#5719 T8: store unavailable (lookup → null) degrades to byte evidence only → fork", () => { +test("#5719 T8: store unavailable (lookup → null) — self-declared checkpoint still re-anchors (#5734)", () => { resetSessionOrderForTest(); const raw = "session-5719-t8"; assert.equal(resolveSessionOrder(raw, chain(10, "h"), SYSTEM_A).key, raw); - // Overlapping ids WOULD prove identity — but the unified store view is - // null (latch-off / legacy epoch, #4407/#5536): degrade, never guess. + // Overlapping ids WOULD prove identity — and the unified store view is + // null (latch-off / legacy epoch, #4407/#5536). Pre-#5734 this degraded to + // byte evidence only → fork (action "derived-persona", key `${raw}|sub:`). + // #5734 (30a8bc2d): the checkpoint's conv declaration rides the WIRE, so the + // self-declaration rung re-anchors on the raw key even with a null store. const res = resolveSessionOrder( raw, incidentHistory(raw, uid("h", 3), "v".repeat(120)), @@ -277,6 +285,7 @@ test("#5719 T8: store unavailable (lookup → null) degrades to byte evidence on ["s-1", "s-2", uid("h", 3), "s-3"], () => null, ); - assert.equal(res.action, "derived-persona"); - assert.ok(res.key.startsWith(`${raw}|sub:`), res.key); + assert.equal(res.action, "reanchor"); + assert.equal(res.key, raw); + assert.match(res.reason ?? "", /self-authored checkpoint declares this raw id/); }); From e5c1dcd45cbce1a9046eb193f888182d7ffaa48b Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 13:57:20 +0800 Subject: [PATCH 6/6] =?UTF-8?q?docs:=20correct=20no-ttl=20breakpoint=20iss?= =?UTF-8?q?ue=20reference=20(#5667=20=E2=86=92=20#5665)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache-keepalive.ts | 2 +- tests/cache-keepalive.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cache-keepalive.ts b/src/cache-keepalive.ts index 38138f5d31..d14606747a 100644 --- a/src/cache-keepalive.ts +++ b/src/cache-keepalive.ts @@ -133,7 +133,7 @@ export function detectKeepaliveTier(body: unknown): "5m" | "1h" | undefined { return; } // A cache_control breakpoint without an explicit ttl uses Anthropic's - // default 5-minute TTL, so it is a 5m-tier breakpoint (#5667); only an + // default 5-minute TTL, so it is a 5m-tier breakpoint (#5665); only an // explicit "1h" lifts the tier. found = "5m"; } diff --git a/tests/cache-keepalive.test.ts b/tests/cache-keepalive.test.ts index cbab1b97ac..6c2988ad15 100644 --- a/tests/cache-keepalive.test.ts +++ b/tests/cache-keepalive.test.ts @@ -166,7 +166,7 @@ const BODY_NO_BP = JSON.stringify({ system: "plain system", messages: [{ role: "user", content: "hi" }], }); -// #5667: a breakpoint with no explicit ttl — Anthropic's default ephemeral TTL is 5m. +// #5665: a breakpoint with no explicit ttl — Anthropic's default ephemeral TTL is 5m. const BODY_NO_TTL = JSON.stringify({ model: "claude-opus-test", max_tokens: 1024, @@ -226,7 +226,7 @@ test("detectKeepaliveTier: 5m / 1h / mixed / nested / none", () => { messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral", ttl: "1h" } }] }], }; assert.equal(detectKeepaliveTier(mixed), "1h"); - // #5667: a breakpoint without an explicit ttl is Anthropic's default 5m tier, + // #5665: a breakpoint without an explicit ttl is Anthropic's default 5m tier, // NOT "no breakpoint detected". Only bodies with zero cache_control are undefined. assert.equal(detectKeepaliveTier(JSON.parse(BODY_NO_TTL)), "5m"); const mixedNoTtl = { @@ -323,7 +323,7 @@ test("arm: 1h breakpoints arm at 1h×0.9", () => { assert.equal(_keepaliveEntryForTest(sid)?.cadenceMs, 3_240_000); }); -// #5667 regression: a breakpoint with no explicit ttl used to return tier +// #5665 regression: a breakpoint with no explicit ttl used to return tier // undefined and silently never arm; it must now arm at the 5m cadence. test("arm: no-ttl breakpoints arm at the default 5m tier", () => { noteSuccessfulForward(armInput({ wireBody: BODY_NO_TTL }));