Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion src/agent/dsh-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ import { adminRelayEnabled, resolveResignSettings } from "../config.js";
import { decodeRequestBody } from "../content-encoding.js";
import { dshContextTabEnabled, dshContextTabLabel, limitsDecodedBytes } from "../knobs.js";
import { MIN_ASSUMED_WINDOW_TOKENS } from "../util.js";
import { tcpConnectable } from "../liveness.js";
import { markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, singleFlight } from "./native-bootstrap.js";
import { bodyOverWindowDetail as sharedBodyOverWindowDetail, getDeclaredModelEndpoints, installNativeFetchIntercept, liveStateSlotEffect, noteRoutedOrigin, observeRoutedOrigin, routedBiliModelUrl, setDeclaredModelEndpoints, type GateVerdict, type NativeInterceptState } from "./native-intercept.js";
import { matchModelEndpoint, loadDeclaredModelEndpoints } from "../model-endpoints.js";
Expand Down Expand Up @@ -714,6 +715,12 @@ async function bootstrap(): Promise<string | undefined> {
if (gen !== originGen) return handle.origin;
state.origin = handle.origin;
register.base = handle.origin;
// #6881: a fresh-spawn landing must clear the old base's backoff — a
// stale retryAt/failStreak from a dead base would gate the NEW base's
// first registration behind the old base's wall (the incident's
// backoff carried into the new world, delaying the first registration).
register.retryAt = 0;
register.failStreak = 0;
// #983: do NOT write BILLION_CONTEXT_PROXY — dsh has no reader for it
// here (tools use register.base, /acp uses it too), and a frozen env
// turns a later same-process re-apply (cordis deactivate/reactivate)
Expand Down Expand Up @@ -871,6 +878,11 @@ async function spawnFallback(attachOrigin: string, note: string, gen: number): P
if (landingOwnsRegister(attachOrigin)) {
register.base = origin;
state.origin = origin;
// #6881: a fresh-spawn landing clears the old base's backoff so the
// new base's first registration is not gated behind the dead base's
// wall (the incident's backoff carried into the new world).
register.retryAt = 0;
register.failStreak = 0;
}
return origin;
});
Expand Down Expand Up @@ -1073,7 +1085,7 @@ async function registerTools(ctx: PluginContext): Promise<void> {
register.failStreak = 0;
register.lastRegistrationError = undefined;
})()
.catch((err: unknown) => {
.catch(async (err: unknown) => {
if (owner !== toolsOwner || owner.closed) return;
register.toolsReady = false;
// Cordis deactivates the plugin context while the host tears down
Expand All @@ -1094,6 +1106,23 @@ async function registerTools(ctx: PluginContext): Promise<void> {
if (register.failStreak === REG_GIVE_UP_NOTICE_STREAK) {
persistClientEvent(`bili tools still unavailable after ${register.failStreak} failed registrations (last: ${register.lastRegistrationError}) — model requests ride bili WITHOUT identity headers (wire mode: no compression, anonymous sessions). Restart dsh to restore; retries continue in the background.`);
}
// #6881: a manifest failure against a base that no longer accepts
// connections is a DEAD base, not a transient fault — and backing
// off against it strands the register: maybeRetry's respawn branch
// only runs when register.base is undefined, so a dead-but-set base
// re-fetches the dead manifest on every tick and never respawns
// (the first model request then rides wire mode into an anonymous
// session). Probe the listening side directly: a refused connect
// (a draining instance closed its listener) clears the base so the
// respawn self-heal takes over; a BUSY instance (accept queue still
// drains while its loop is inside a large fold) stays put (#3654).
if (register.base === base && !(await tcpConnectable(base))) {
persistClientEvent(`manifest base ${base} stopped accepting connections — clearing the dead base so the respawn self-heal takes over (#6881)`);
register.base = undefined;
register.retryAt = 0;
armRecoveryTimer();
return;
}
// Scope the back-off to the base that actually failed: if the
// register moved on while this manifest fetch was in flight (the
// apply chain unfroze a dead preset, maybeRetry healed elsewhere),
Expand Down Expand Up @@ -1189,6 +1218,12 @@ function maybeRetry(ctx: PluginContext): void {
if (gen !== originGen) return;
register.base = origin;
state.origin = origin;
// #6881: a fresh-spawn landing clears the old base's backoff so
// the new base's first registration is not gated behind the
// dead base's wall (the incident's backoff carried into the new
// world, delaying the first registration).
register.retryAt = 0;
register.failStreak = 0;
void registerTools(ctx).catch(() => {});
})
.catch(() => {});
Expand Down
8 changes: 8 additions & 0 deletions src/instance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -792,6 +792,14 @@ export function discoverLiveInstances(): ProxyInstanceFile[] {
const out: ProxyInstanceFile[] = [];
for (const e of readAllRegistryEntries()) {
if (!isPidAlive(e.pid)) continue;
// #6881: a draining instance stamped portReleasedAt when its listener
// closed (#4982) — its pid still lives but the socket no longer does,
// so it must not surface as a live attach candidate (mirrors
// classifySameLanePeer's skip: a stale "I hold port N" marker would
// otherwise let discovery attach to a dead address). The fresh-conn
// health probe (#6881) is the robust catch for the single-origin-file
// path that never stamps this; this is the cheap registry-path guard.
if (e.portReleasedAt !== undefined) continue;
out.push({
origin: e.origin,
instanceId: e.instanceId,
Expand Down
25 changes: 23 additions & 2 deletions src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4229,7 +4229,7 @@ function instanceCompatible(inst: ProxyInstanceFile, opts: LaunchOptions, codeFi
* health-probes each candidate in parallel. */
async function probeLiveInstances(
readInstance: () => ProxyInstanceFile | { origin: string } | undefined,
fetchHealthInfo: (origin: string) => Promise<HealthInfo | undefined>,
fetchHealthInfo: (origin: string, opts?: { noKeepAlive?: boolean }) => Promise<HealthInfo | undefined>,
diag?: (msg: string) => void,
isConnectable: (origin: string) => Promise<boolean> = tcpConnectable,
): Promise<Array<{ inst: ProxyInstanceFile; health: HealthInfo | undefined }>> {
Expand All @@ -4240,7 +4240,16 @@ async function probeLiveInstances(
const checked = await Promise.all(
[...seen.values()].map(async (c): Promise<{ inst: ProxyInstanceFile; health: HealthInfo | undefined } | undefined> => {
if (!isPidAlive(c.pid)) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): owner process gone`); return undefined; }
const health = await fetchHealthInfo(c.origin);
// #6881: probe on a FRESH connection (Connection: close) — a
// draining instance has closed its listener but still answers its
// established keep-alive connections, so a pooled socket would
// report it "alive" and discovery would attach to a dead address
// (the incident: H2 attached to P1's draining tail). A fresh
// connect is refused once the listener is closed, so the instance
// is correctly dropped. A BUSY instance (accept queue still drains)
// still TCP-connects on a fresh socket, so the #3654/#4811 BUSY
// keep below is unaffected.
const health = await fetchHealthInfo(c.origin, { noKeepAlive: true });
if (!health || !health.ok) {
// #3654: a refused connection and a saturated event loop both
// fail this probe — tell them apart in the log: connect-but-
Expand Down Expand Up @@ -5621,6 +5630,18 @@ export async function ensureProxyRunning(
// e.g. into Pi's input draft after herdr session restore.
attachDiag(`bili: attaching to running proxy at ${inst.origin} (pid ${inst.pid})`);
const reg = await registerOwner(inst);
// #6881: the watcher registration result was invisible in bili.log —
// success logged nothing and failure only hit console.error (a raw
// console.* on the attach path, #3404). Log it on the host-safe
// channel so a "failed" (watchdog not active) or "refused" (no
// watchdog) attach is visible right next to the attach line above.
if (reg === "ok") {
attachDiag(`bili: session-watcher registered at ${inst.origin} (pid ${inst.pid}) — lifecycle watchdog active`);
} else if (reg === "failed") {
attachDiag(`bili: session-watcher registration FAILED at ${inst.origin} (pid ${inst.pid}) — lifecycle watchdog NOT active; the proxy will outlive this session's cleanup until a heartbeat recovers it`);
} else {
attachDiag(`bili: session-watcher registration refused (409) at ${inst.origin} (pid ${inst.pid}) — proxy has no session-lifecycle watchdog (started without BILI_PARENT_PID)`);
}
// #3544: keep re-asserting our liveness so the proxy's watchdog ages us
// out by HEARTBEAT expiry, not just kill(pid,0) (identity-blind under
// Windows pid reuse). No-op unless we own the pid.
Expand Down
Loading
Loading