Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 151 additions & 35 deletions src/agent/native-intercept.ts

Large diffs are not rendered by default.

22 changes: 17 additions & 5 deletions src/agent/opencode-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ import { ACP_TOOLS_OPENAI } from "../compress-tool.js";
import { ensureProxyRunning, LAUNCHER_DEFAULT_HOST, unwrapUpstream, wrapUpstream } from "../launcher.js";
import { createAcpCommandHooks, showAcpText } from "./opencode-acp-command.js";
import { markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, singleFlight } from "./native-bootstrap.js";
import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, replaceRequestPayload, replaceRequestTarget, routedBiliModelUrl, type LiveOriginResolverDeps, type NativeInterceptState } from "./native-intercept.js";
import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, replaceRequestPayload, replaceRequestTarget, routedBiliModelUrl, type LiveOriginResolverDeps, type NativeInterceptState, type UnavailableReason } from "./native-intercept.js";
import { clearIncrementalProbe, createPrepareDirectSend, INCREMENTAL_TURN_MARKER_HEADER, INCREMENTAL_TURN_ID_HEADER, newIncrementalTurnId, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn, probeIncrementalTurns, rememberOriginalBody, sweepStale, wireMessagesLookAnthropic, type IncrementalWireMessage } from "./incremental-client.js";
import { createOpencodeV2Setup, type V2HttpRequestEvent, type V2State } from "./opencode-v2.js";
import { asciiHeaderValue, fetchProxyVersion, postIdentityRegister, reportRuntimeInfoOnChange, waitForProxyVersion, type ForwardedToolResult } from "./shared.js";
Expand Down Expand Up @@ -117,7 +117,7 @@ export type OpencodeNativeRouteDeps = LiveOriginResolverDeps;
* model, unknown window, unreadable or absent body all return undefined and
* keep the documented fail-open direct send. The size is measured through a
* CLONE — the original Request must stay sendable (#2496 discipline). */
async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State): Promise<Error | undefined> {
async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State, reason?: UnavailableReason): Promise<Error | undefined> {
const model = e.model;
if (model === undefined || typeof model.providerID !== "string" || typeof model.id !== "string") return undefined;
const window = s.windows?.get(`${model.providerID}/${model.id}`);
Expand All @@ -132,6 +132,14 @@ async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State): Promise
}
const floorTokens = Math.ceil(bytes / 4);
if (floorTokens <= window) return undefined;
// #3654: a busy-exhausted refusal must NOT claim "proxy unavailable …
// guaranteed upstream 400" — the proxy IS up and would serve the request
// once its event loop drains; the honest fact is that we stopped waiting.
if (reason === "busy-exhausted") {
return new Error(
`bili: proxy alive but unresponsive — it accepted connections but never answered its health probe within the wait budget (event loop saturated, typically a large compression in flight); this request cannot be sent uncompressed (body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window})), so it is refused locally instead of queued indefinitely. The proxy is likely to serve it once the fold finishes (#3654); retry shortly.`,
);
}
return new Error(
`bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. Recovery is automatic (#3308); retry shortly, or start a fresh session if the local history has outgrown the window.`,
);
Expand Down Expand Up @@ -189,12 +197,16 @@ export function createNativeRoute(state: NativeInterceptState, deps: OpencodeNat
if (target === undefined) {
// #3308: a direct send that provably cannot fit the window is not
// "availability" — refuse it locally (#2924 ruling) instead of
// handing upstream a request that can only 400.
const refusal = await oversizeDirectRefusal(e, s);
// handing upstream a request that can only 400. #3654: pass WHY
// nothing was routable so a saturated-but-alive proxy gets its own
// honest wording instead of "proxy unavailable … guaranteed 400".
const refusal = await oversizeDirectRefusal(e, s, state.lastUnavailableReason);
if (refusal !== undefined) {
if (!refusedNoticed) {
refusedNoticed = true;
logHostEvent("error", "[native-client]", `bili-native-opencode: proxy unavailable — model request refused locally instead of going direct into a guaranteed upstream 400 (#3308): ${refusal.message}`);
logHostEvent("error", "[native-client]", state.lastUnavailableReason === "busy-exhausted"
? `bili-native-opencode: proxy alive but saturated past the wait budget — model request refused locally instead of queued indefinitely (#3654): ${refusal.message}`
: `bili-native-opencode: proxy unavailable — model request refused locally instead of going direct into a guaranteed upstream 400 (#3308): ${refusal.message}`);
}
s.proxyBase = undefined;
throw refusal;
Expand Down
16 changes: 16 additions & 0 deletions src/instance.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { resolveZonePortBase } from "./config.js";
import { stateDir } from "./paths.js";
Expand Down Expand Up @@ -134,6 +135,21 @@ export function writeZonePort(lane: string, port: number, file: string = portZon
* base (BILI_ZONE_PORT override). Explicit user overrides
* (BILI_CLAUDE_NATIVE_PORT / BILI_ZCODE_PORT) are resolved separately by
* the lanes and imply strict-port launches. */
/** #3654/#3116: append a diagnostic line to the spawner-owned per-port log
* (%TMPDIR%/bili-proxy-<port>.log) — the exact file a spawner failure message
* points users at. Post-#2864 the child mutes its stderr leg into bili.log as
* soon as its logger is live, so a child that dies during listen setup leaves
* that file EMPTY (the "0 bytes" dead-end); this writes the child-side truth
* there. Named by the PREFERRED port (what the spawner uses for the file
* name); the line itself names the port actually tried. Line grammar mirrors
* logger.ts so mixed writers stay greppable. Best-effort: a failed append
* never affects the exit. */
export function writeSpawnPortLog(preferredPort: number, line: string): void {
try {
fs.appendFileSync(path.join(os.tmpdir(), `bili-proxy-${preferredPort}.log`), `${line}\n`, "utf8");
} catch {}
}

export function lanePreferredPort(lane: string, env: NodeJS.ProcessEnv = process.env, file: string = portZoneFilePath()): number {
return readZonePort(lane, file) ?? resolveZonePortBase(env);
}
Expand Down
55 changes: 53 additions & 2 deletions src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ import {
type ProxyInstanceFile,
type ProxyStartingMarker,
} from "./instance.js";
import { TWIN_ADOPT_EXIT_CODE } from "./listen-policy.js";
import { tcpConnectable } from "./liveness.js";
import { selfPackageRoot, isBiliPiEntry, ompPluginLoadedFrom, dshNativeInstalled, claudeNativeInstalled, piPackageStrings } from "./plugin-install.js";
import { applyOmpFirstEventTimeout } from "./agent/native-bootstrap.js";
import { logHostEvent } from "./agent/host-log.js";
Expand Down Expand Up @@ -316,6 +318,13 @@ export interface LauncherDeps {
* discoverLiveInstances, instance.ts). Tests inject a fake so the
* clobbered-proxy-origin scenario never touches the real state dir. */
listRegistryInstances?: () => ProxyInstanceFile[];
/** #3654: TCP pre-probe distinguishing "refused" from "saturated" for
* diagnostics and twin adoption (default: tcpConnectable). Tests inject
* a fake so no real sockets are needed. */
isConnectable?: (origin: string) => Promise<boolean>;
/** #3654: registry view behind serving-twin adoption after exit code 3
* (default: discoverLiveInstances, instance.ts). */
listAdoptCandidates?: () => ProxyInstanceFile[];
}

export function isLaunchClient(value: string): value is ClientName {
Expand Down Expand Up @@ -3647,6 +3656,7 @@ async function probeLiveInstances(
readInstance: () => ProxyInstanceFile | { origin: string } | undefined,
fetchHealthInfo: (origin: string) => Promise<HealthInfo | undefined>,
diag?: (msg: string) => void,
isConnectable: (origin: string) => Promise<boolean> = tcpConnectable,
): Promise<Array<{ inst: ProxyInstanceFile; health: HealthInfo }>> {
const seen = new Map<string, ProxyInstanceFile>();
const inst = readInstance();
Expand All @@ -3656,7 +3666,17 @@ async function probeLiveInstances(
[...seen.values()].map(async (c): Promise<{ inst: ProxyInstanceFile; health: HealthInfo } | undefined> => {
if (!isPidAlive(c.pid)) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): owner process gone`); return undefined; }
const health = await fetchHealthInfo(c.origin);
if (!health || !health.ok) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed`); return undefined; }
if (!health || !health.ok) {
// #3654: a refused connection and a saturated event loop both
// fail this probe — tell them apart in the log: connect-but-
// silent is BUSY (alive, loop inside a large fold), not dead.
// Still dropped either way (#1335 gate unchanged).
const busy = await isConnectable(c.origin);
diag?.(busy
? `native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed but TCP connect succeeded — instance is BUSY (event loop saturated), not dead (#3654)`
: `native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed`);
return undefined;
}
if (health.instanceId !== undefined && health.instanceId !== c.instanceId) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): instance id mismatch (${c.instanceId} vs ${health.instanceId})`); return undefined; }
return { inst: c, health };
}),
Expand Down Expand Up @@ -4593,6 +4613,12 @@ export async function ensureProxyRunning(
if (/^HTTP \d{3}$/.test(d)) {
return `port ${port} answered ${d} — the listener there is not reporting bili health`;
}
// #3654: a timeout with a live TCP listener underneath is the
// event-loop-saturation signature (connect succeeds, JS never runs)
// — name it instead of letting it read as a dead port.
if (/^timeout after \d+ms$/i.test(d) && (await (deps.isConnectable ?? tcpConnectable)(origin))) {
return `port ${port} accepts TCP connections but never answers /__bili/health within the probe budget — the instance there is BUSY (event loop saturated, typically a large compression), not dead (#3654)`;
}
return `last health probe failed with ${d}`;
};

Expand Down Expand Up @@ -4674,6 +4700,21 @@ export async function ensureProxyRunning(
// discovery+attach decision as the one-shot probe above, re-run on a
// short budget so a listener whose identity record published during
// our spawn attempt is still attachable instead of mistaken for air.
// #3654: re-run the identity criteria the child's findSameLaneTwin
// applied at bind time against the CURRENT registry, verify the holder
// still answers TCP (pid reuse), then ride it through the normal attach
// path (watcher + heartbeat included).
const adoptServingTwin = async (): Promise<ProxyHandle | undefined> => {
const candidates = (deps.listAdoptCandidates ?? discoverLiveInstances)()
.filter((inst) => isPidAlive(inst.pid) && instanceCompatible(inst, opts, codeFingerprint, foreignLanes) === undefined)
.sort((a, b) => Number(b.port === port) - Number(a.port === port) || b.startedAt - a.startedAt);
for (const inst of candidates) {
if (!(await (deps.isConnectable ?? tcpConnectable)(inst.origin))) continue;
attachDiag(`bili: adopting serving twin ${inst.origin} (pid ${inst.pid}) — the spawned child lost the #3044 duplicate race to it; requests ride the existing instance (#3654)`);
return attachTo(inst);
}
return undefined;
};
const rediscoverAfterChildExit = async (): Promise<ProxyHandle | undefined> => {
console.error(
`bili: spawned proxy exited before becoming healthy — re-checking for a late-publishing listener on port ${port} before failing`,
Expand All @@ -4697,6 +4738,16 @@ export async function ensureProxyRunning(
throw new Error(`bili: proxy spawn failed (${detail}) (runtime: ${runtime}) (log: ${logPath})`);
}
if (childExit) {
// #3654: exit code 3 is the child's twin-collision signal (#3044
// listen-policy): a live same-lane same-build instance already
// serves the port. Discovery above declared it unhealthy or missed
// it; the child's bind-time twin check proved it was serving.
// Adopt the holder instead of failing — the user's proxy is up.
if (childExit.code === TWIN_ADOPT_EXIT_CODE) {
const adopted = await adoptServingTwin();
if (adopted !== undefined) return adopted;
attachDiag("bili: serving-twin adoption failed verification — continuing with the standard post-exit re-discovery");
}
// #1903: the child died before becoming healthy. Under strictPort
// that death is almost certainly EADDRINUSE — proof the pinned port
// is held by a listener our one-shot discovery above missed because
Expand All @@ -4709,7 +4760,7 @@ export async function ensureProxyRunning(
const retried = await rediscoverAfterChildExit();
if (retried) return retried;
const detail = childExit.code !== null
? `code ${childExit.code}`
? `code ${childExit.code}${childExit.code === TWIN_ADOPT_EXIT_CODE ? " (twin-collision signal)" : ""}`
: childExit.signal ? `signal ${childExit.signal}` : "unknown reason";
throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${await diagnosePort(false)} (runtime: ${runtime}) (log: ${logPath}${logTail()})`);
}
Expand Down
18 changes: 18 additions & 0 deletions src/listen-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,24 @@
* (#3044): the marked process must bind-or-die on exactly the freed port. */
export const UPGRADE_REPLACEMENT_ENV = "BILI_UPGRADE_REPLACEMENT";

/** #3654: the exit code a twin-collision loser reports to its spawner. The
* duplicate still LOSES (#3044 — it cannot bind the port the serving twin
* holds), but the code says WHY it died: the spawner reads it as an adopt
* signal and attaches to the serving twin instead of failing the bring-up —
* closing the #3654 contradiction where discovery judged the holder dead
* (busy) while the twin rule proved it alive. Every other listen failure
* keeps exit 1. */
export const TWIN_ADOPT_EXIT_CODE = 3;

export type FailFastReason = "upgrade-replacement" | "strict-port" | "port-busy" | "twin-collision";

/** Pure exit-code mapping for a terminal EADDRINUSE decision — extracted so
* the server (which calls process.exit) and its tests share one definition
* without booting a listener (#3654). */
export function eaddrinuseExitCode(reason: FailFastReason | undefined): number {
return reason === "twin-collision" ? TWIN_ADOPT_EXIT_CODE : 1;
}

type EaddrinuseAction =
| { kind: "wait-predecessor"; port: number }
/** nextPort 0 = ask the OS for an ephemeral port (ladder exhausted). */
Expand Down
Loading
Loading