Skip to content
6 changes: 6 additions & 0 deletions scripts/live-files.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,12 @@ export const LIVE_FILES = [
// (SIGTERM/SIGKILL/clean shutdown). Private mkdtemp homes per lane-arm,
// probed-free ports re-probed on every restart — no band reuse.
"tests/issue5631-restart-survival.test.ts",
// #5666: process-liveness regression — spawns ONE bare node child (--import
// tsx) that arms a keepalive through the real scheduler and asserts it exits
// on its own; the child runs under a per-run mkdtemp XDG sandbox, does no
// network I/O and binds no port, so no zone band is needed — the entry is for
// the tripwire alone, like #3890/#5114
"tests/cache-keepalive.test.ts",
];

/** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry;
Expand Down
20 changes: 17 additions & 3 deletions src/cache-keepalive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,10 @@ export function detectKeepaliveTier(body: unknown): "5m" | "1h" | undefined {
found = "1h";
return;
}
if (ttl === "5m") found = "5m";
// A cache_control breakpoint without an explicit ttl uses Anthropic's
// default 5-minute TTL, so it is a 5m-tier breakpoint (#5665); only an
// explicit "1h" lifts the tier.
found = "5m";
}
for (const v of Object.values(o)) walk(v);
};
Expand Down Expand Up @@ -172,8 +175,19 @@ interface KeepaliveEntry {

const registry = new Map<string, KeepaliveEntry>();

/** Real-timer scheduler: every handle is unref'd (#5666) so an armed cadence
* timer never holds the event loop open. Interactive hosts stay alive for
* their own reasons and the pings still fire on schedule; a short-lived
* process (pi -p, a sub-agent) whose only remaining handle is the armed
* keepalive now exits cleanly instead of limping along until maxDurationMs. */
function realSchedule(fn: () => void, ms: number): ReturnType<typeof setTimeout> {
const t = setTimeout(fn, ms);
t.unref?.();
return t;
}

let clock: () => number = () => Date.now();
let scheduleFn: (fn: () => void, ms: number) => ReturnType<typeof setTimeout> = (fn, ms) => setTimeout(fn, ms);
let scheduleFn: (fn: () => void, ms: number) => ReturnType<typeof setTimeout> = realSchedule;
let cancelFn: (id: ReturnType<typeof setTimeout>) => void = (id) => clearTimeout(id);

export interface KeepaliveArmInput {
Expand Down Expand Up @@ -454,7 +468,7 @@ export function _setKeepaliveSchedulerForTest(sched: FakeScheduler | undefined):
scheduleFn = (fn, ms) => sched.schedule(fn, ms) as ReturnType<typeof setTimeout>;
cancelFn = (id) => sched.cancel(id as unknown);
} else {
scheduleFn = (fn, ms) => setTimeout(fn, ms);
scheduleFn = realSchedule;
cancelFn = (id) => clearTimeout(id);
}
}
Expand Down
68 changes: 66 additions & 2 deletions src/fetch-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,10 @@ export class UpstreamHttpError extends Error {
* rate-limit rejection rather than a genuine client error. GLM Coding Plan
* returns 400 {"code":3007,"msg":"captcha verify failed"} ~1s after large
* context rewrites (issue #189); every observed case recovered on retry,
* so such bodies are retried while plain 4xx (bad model, bad params) fail fast. */
* so such bodies are retried while plain 4xx (bad model, bad params) fail
* fast. "please retry" / "请重试" cover the canonical "just retry" phrasing of
* risk-control gateways (issue #3606). Bare "retry" is deliberately absent —
* "do not retry" / "请勿重试" must stay non-transient. */
const TRANSIENT_BODY_MARKERS = [
"captcha",
"verify failed",
Expand All @@ -325,13 +328,74 @@ const TRANSIENT_BODY_MARKERS = [
"rate limit",
"too many requests",
"try again",
"please retry",
"请重试",
];

// #3606: an explicit machine self-declaration outranks prose sniffing — a
// gateway that says its rejection is retryable means it, and enumerating
// every vendor's wording will never converge. Bodies past this cap are too
// large to be JSON error envelopes; skip the parse.
const DECLARATION_BODY_CAP = 65_536;
// Node visit budget for the declaration scan: bounds worst-case work on
// pathological (but still small) bodies.
const DECLARATION_NODE_BUDGET = 4_096;

export function isTransientUpstreamError(status: number, body: string): boolean {
if (status === 429 || status >= 500) return true;
if (status < 400) return false;
const lower = body.toLowerCase();
return TRANSIENT_BODY_MARKERS.some((marker) => lower.includes(marker));
if (TRANSIENT_BODY_MARKERS.some((marker) => lower.includes(marker))) return true;
return bodyDeclaresRetryable(body);
}

// #3606: honor the upstream's explicit retryability declaration — an "actions"
// array containing "RETRY" (case-insensitive) or a boolean "retryable": true
// anywhere in the error JSON. The scan walks nested objects AND string values
// that themselves parse as JSON, because gateways wrap their envelope in a
// string field ({"error":{"message":"{...}"}}, the shape behind issue #3606's
// code 11101 bodies). A string "true" or an actions list without RETRY is NOT
// a declaration — only unambiguous machine signals count.
function bodyDeclaresRetryable(body: string): boolean {
if (body.length > DECLARATION_BODY_CAP) return false;
const root = parseJsonObject(body);
if (root === undefined) return false;
const queue: unknown[] = [root];
let head = 0;
let visited = 0;
while (head < queue.length && visited < DECLARATION_NODE_BUDGET) {
const node = queue[head++];
visited += 1;
if (typeof node === "string") {
const nested = parseJsonObject(node);
if (nested !== undefined) queue.push(nested);
} else if (Array.isArray(node)) {
for (const item of node) queue.push(item);
} else if (node !== null && typeof node === "object") {
for (const [key, value] of Object.entries(node)) {
const loweredKey = key.toLowerCase();
if (loweredKey === "actions" && Array.isArray(value)
&& value.some((item) => typeof item === "string" && item.toLowerCase() === "retry")) {
return true;
}
if (loweredKey === "retryable" && value === true) {
return true;
}
if (value !== null && typeof value === "object") queue.push(value);
else if (typeof value === "string" && /^[{\[]/.test(value.trimStart())) queue.push(value);
}
}
}
return false;
}

function parseJsonObject(text: string): object | undefined {
try {
const value: unknown = JSON.parse(text);
return typeof value === "object" && value !== null ? value : undefined;
} catch {
return undefined;
}
}

/** Total requests per replay attempt (initial + retries). */
Expand Down
26 changes: 25 additions & 1 deletion src/served-markers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export interface WriterIdentity {
/** #3923/#3959: this instance's lane (e.g. "pi", "dsh") — additive;
* markers written by older builds carry none and stay valid. */
lane?: string;
startedAt?: number;
}

interface WriterMarkerFile {
Expand All @@ -38,6 +39,7 @@ interface WriterMarkerFile {
origin: string;
version: string;
lane?: string;
startedAt?: number;
/** Set only by a graceful dispose(): this instance recorded its FINAL
* marker and will never commit again. Peers skip such markers outright —
* a positively-known dead predecessor is not a concurrent writer, even
Expand Down Expand Up @@ -92,11 +94,21 @@ const PEER_FILE_FRESH_MS = 30_000;
/** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is
* older than this. */
const STALE_REAP_MS = 60 * 60_000;
/** Same-PID markers that predate this process's boot by more than this margin
* belong to an earlier container generation (the PID was reused, not shared)
* — the margin absorbs clock/rounding skew between generations (#3923). */
const SAME_PID_GENERATION_MARGIN_MS = 1_000;

export function writerMarkerDir(stateDirPath: string): string {
return path.join(stateDirPath, "writers");
}

/** This process's boot time (ms epoch), derived from uptime against the
* caller's clock so tests can pin it. */
export function processStartMs(now: number): number {
return now - Math.round(process.uptime() * 1000);
}

let tmpSeq = 0;

function durableAtomicWrite(file: string, text: string): void {
Expand Down Expand Up @@ -157,6 +169,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
origin: identity.origin,
version: identity.version,
...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }),
...(typeof identity.startedAt === "number" ? { startedAt: identity.startedAt } : {}),
...(stoppedAt === undefined ? {} : { stoppedAt }),
servedAt: Object.fromEntries(servedAt),
...(final ? { disposedAt: now } : {}),
Expand Down Expand Up @@ -250,7 +263,9 @@ function contestedServedAt(raw: Partial<WriterMarkerFile>, selfInstanceId: strin
* window by a LIVE foreign instance (pid alive or file fresh). Markers
* carrying a graceful-stop stamp (`stoppedAt`, or the `disposedAt` alias,
* #4626) are skipped unconditionally — a restart flow's predecessor died
* seconds ago and must not raise an alarm. Malformed or tearing files are
* seconds ago and must not raise an alarm. Same-PID markers from an earlier
* container generation (booted before this process) are likewise skipped —
* the PID was reused, not shared (#3923). Malformed or tearing files are
* skipped silently — a marker is observability, not data. */
export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array<WriterIdentity & { servedAt: number }> {
let entries: fs.Dirent[];
Expand All @@ -266,6 +281,14 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio
try {
const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial<WriterMarkerFile>;
const mtimeMs = fs.statSync(file).mtimeMs;
// #3923: in containers PID is typically 1 across restarts. A marker
// that shares our PID but predates this process's boot belongs to a
// prior container generation, not a live twin (older builds carry no
// startedAt — the marker mtime then stands in for their last write).
const samePid = typeof raw.pid === "number" && raw.pid === process.pid;
const peerStartedAt = typeof raw.startedAt === "number" ? raw.startedAt : mtimeMs;
if (samePid && peerStartedAt < processStartMs(now) - SAME_PID_GENERATION_MARGIN_MS) continue;

const at = contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now);
if (at === null || typeof raw.instanceId !== "string") continue;
out.push({
Expand All @@ -274,6 +297,7 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio
origin: typeof raw.origin === "string" ? raw.origin : "unknown",
version: typeof raw.version === "string" ? raw.version : "unknown",
lane: typeof raw.lane === "string" && raw.lane !== "" ? raw.lane : undefined,
...(typeof raw.startedAt === "number" ? { startedAt: raw.startedAt } : {}),
servedAt: at,
});
} catch {
Expand Down
4 changes: 2 additions & 2 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ import { noteRequest } from "./attach-heartbeat.js";
import { queryLogLines } from "./web/logs-query.js";
import { configFile, defaultLogFile, sessionsDir, stateDir } from "./paths.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryShape, findPresentUnverifiedSameLaneTwin, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, markPortReleased, probePid, readProxyInstanceFile, registerInstanceAndWarn, resolveOwnFingerprint, sameInstanceShape, SPAWN_FINGERPRINT_ENV, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js";
import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js";
import { createServedMarkerWriter, peerStillContests, processStartMs, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js";
import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js";
import { holderProbeOrigin, mayClobberPointer, probeHolderForHint, probeOwnerInfo, resolvePortClaim, verifyEntryOwnsPort } from "./port-owner.js";
import { armEventLoopWatchdog } from "./event-loop-watchdog.js";
Expand Down Expand Up @@ -1347,7 +1347,7 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
{
const markerDir = writerMarkerDir(stateDir());
servedWriter = createServedMarkerWriter(
{ instanceId, pid: process.pid, origin, version: VERSION, lane: launcherLane },
{ instanceId, pid: process.pid, origin, version: VERSION, lane: launcherLane, startedAt: processStartMs(Date.now()) },
markerDir,
(level, msg) => log(level, msg),
{
Expand Down
83 changes: 83 additions & 0 deletions tests/cache-keepalive.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
import { test, before, after, beforeEach } from "node:test";
import assert from "node:assert";
import http from "node:http";
import { spawn } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";

import {
buildPingBody,
Expand Down Expand Up @@ -161,6 +166,14 @@ const BODY_NO_BP = JSON.stringify({
system: "plain system",
messages: [{ role: "user", content: "hi" }],
});
// #5665: a breakpoint with no explicit ttl — Anthropic's default ephemeral TTL is 5m.
const BODY_NO_TTL = JSON.stringify({
model: "claude-opus-test",
max_tokens: 1024,
stream: true,
system: [{ type: "text", text: "sys", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: "hi" }],
});

let timers: FakeTimers;
let sid: string;
Expand Down Expand Up @@ -213,6 +226,19 @@ test("detectKeepaliveTier: 5m / 1h / mixed / nested / none", () => {
messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral", ttl: "1h" } }] }],
};
assert.equal(detectKeepaliveTier(mixed), "1h");
// #5665: a breakpoint without an explicit ttl is Anthropic's default 5m tier,
// NOT "no breakpoint detected". Only bodies with zero cache_control are undefined.
assert.equal(detectKeepaliveTier(JSON.parse(BODY_NO_TTL)), "5m");
const mixedNoTtl = {
system: [{ text: "a", cache_control: { type: "ephemeral", ttl: "5m" } }],
messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral" } }] }],
};
assert.equal(detectKeepaliveTier(mixedNoTtl), "5m");
const oneHourAmongNoTtl = {
system: [{ text: "a", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: [{ type: "text", text: "b", cache_control: { type: "ephemeral", ttl: "1h" } }] }],
};
assert.equal(detectKeepaliveTier(oneHourAmongNoTtl), "1h");
assert.equal(detectKeepaliveTier(JSON.parse(BODY_NO_BP)), undefined);
assert.equal(detectKeepaliveTier(null), undefined);
});
Expand Down Expand Up @@ -297,6 +323,17 @@ test("arm: 1h breakpoints arm at 1h×0.9", () => {
assert.equal(_keepaliveEntryForTest(sid)?.cadenceMs, 3_240_000);
});

// #5665 regression: a breakpoint with no explicit ttl used to return tier
// undefined and silently never arm; it must now arm at the 5m cadence.
test("arm: no-ttl breakpoints arm at the default 5m tier", () => {
noteSuccessfulForward(armInput({ wireBody: BODY_NO_TTL }));
const e = _keepaliveEntryForTest(sid);
assert.ok(e, "session armed from a no-ttl breakpoint");
assert.equal(e!.cadenceMs, 270_000);
assert.equal(e!.tierLabel, "5m");
assert.deepEqual(_keepaliveArmedForTest(), [sid]);
});

for (const [name, over] of [
["non-anthropic protocol never arms", { protocol: "openai" as const }],
["GET never arms", { reqMethod: "GET" }],
Expand Down Expand Up @@ -507,3 +544,49 @@ test("keepaliveStatusFor: mirrors armed state, ping history and disarm", async (
await settle(() => _keepaliveArmedForTest().length === 0);
assert.equal(keepaliveStatusFor(sid), undefined, "disarmed sessions report no state");
});

// ---- process-liveness (real timers) ---------------------------------------

test("arm: the real scheduler unrefs its timer, so short-lived processes exit (#5666)", async () => {
// A child arms a keepalive through the DEFAULT (non-injected) scheduler and
// then does nothing else. The armed cadence timer is that process's only
// remaining handle — exactly the inline-lane pi -p / sub-agent shape. With
// the fix the timer is unref'd, so the event loop drains and the process
// exits on its own; without it the ref'd timer holds the loop open and the
// child hangs until the (270 s) timer fires. Hermetic: no network (the ping
// URL is never reached before the process is gone).
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ka-exit-"));
for (const d of ["cfg", "state", "data"]) fs.mkdirSync(path.join(tmp, d), { recursive: true });
const env = {
...process.env,
XDG_CONFIG_HOME: path.join(tmp, "cfg"),
XDG_STATE_HOME: path.join(tmp, "state"),
XDG_DATA_HOME: path.join(tmp, "data"),
ACP_LOG_FILE: "off",
};
const kaUrl = pathToFileURL(fileURLToPath(new URL("../src/cache-keepalive.ts", import.meta.url))).href;
const code = `
process.env.BILI_CACHE_KEEPALIVE = "1";
process.env.BILI_CACHE_KEEPALIVE_ASSUME_TTL = "5m";
const { noteSuccessfulForward } = await import(${JSON.stringify(kaUrl)});
noteSuccessfulForward({
sessionId: "ka-exit", protocol: "anthropic", reqMethod: "POST", sidePassthrough: false, resignLane: false,
wireBody: JSON.stringify({ model: "claude-opus-test", max_tokens: 4096, stream: true, system: [{ type: "text", text: "s", cache_control: { type: "ephemeral", ttl: "5m" } }], messages: [{ role: "user", content: "hi" }] }),
upstreamUrl: "http://127.0.0.1:9/v1/messages", headers: { "content-type": "application/json" }, dispatcher: undefined, routes: undefined,
});
`;
const child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], {
cwd: fileURLToPath(new URL("../", import.meta.url)),
env,
stdio: ["ignore", "pipe", "pipe"],
});
let out = "";
child.stdout?.on("data", (c) => (out += String(c)));
child.stderr?.on("data", (c) => (out += String(c)));
const exited = new Promise<number | null>((resolve) => child.on("close", (c) => resolve(c)));
const timedOut = new Promise<null>((resolve) => setTimeout(() => resolve(null), 10_000).unref());
const exitCode = await Promise.race([exited, timedOut]);
child.kill();
fs.rmSync(tmp, { recursive: true, force: true });
assert.equal(exitCode, 0, `an armed keepalive must not hold the event loop open (got ${exitCode === null ? "TIMEOUT — the cadence timer kept the process alive" : `exit code ${exitCode}`})\n--- child output ---\n${out}`);
});
Loading
Loading