Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 187 additions & 25 deletions src/agent/omp-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,43 @@
// cancel and the prompt_cache_key stamp all reuse the launcher-mode
// code paths. omp never emits before_provider_headers, so its
// runtime-info report (#955) rides before_provider_request instead.
// Skipped when a proxy is already managed (BILLION_CONTEXT_PROXY set by a
// `bili` MITM launch, or BILI_PROVIDER_REWRITES set by a `bili` /bili/
// launch) or opted out (BILI_NATIVE_OMP=0).
// Posture (planNativeOmp — #886/#983 mechanism, ported to the omp lane for
// #1795): a preset BILLION_CONTEXT_PROXY (or explicit BILLION_CONTEXT_ATTACH)
// is an ATTACH target, not a stand-down — the pseudo-attach hole used to
// disarm routing entirely while tools still found the proxy through the env,
// sending every model request direct and uncompressed. Stand down only for a
// /bili/ rewrite launch (BILI_PROVIDER_REWRITES) or an opt-out
// (BILLION_CONTEXT_PLUGIN=0 / BILI_NATIVE_OMP=0).

import { ensureProxyRunning, LAUNCHER_DEFAULT_HOST } from "../launcher.js";
import { createBiliPlugin } from "./pi.js";
import { applyOmpFirstEventTimeout, markNativeHost, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, setNativeOriginWaiter, singleFlight } from "./native-bootstrap.js";
import { installNativeFetchIntercept, readyOrigin, type NativeInterceptState } from "./native-intercept.js";
import { applyOmpFirstEventTimeout, markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, setNativeOriginWaiter, singleFlight } from "./native-bootstrap.js";
import { installNativeFetchIntercept, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, type NativeInterceptState } from "./native-intercept.js";
import { fetchProxyVersion, waitForProxyVersion } from "./shared.js";

/** Decides whether the native bootstrap should run in this process. */
export function shouldBootstrapNativeOmp(env: NodeJS.ProcessEnv): boolean {
return nativeBootstrapGate(env, "BILI_NATIVE_OMP");
}

/** Decide this process's native posture (#886/#983 mechanism, ported to the
* omp lane in #1795). Precedence kill-switch > /bili/ rewrite launch >
* attach > spawn. A preset BILLION_CONTEXT_PROXY is an ATTACH target, not a
* stand-down: the pseudo-attach hole (preset env + bare omp) used to disarm
* routing entirely while tools still found the proxy through the env — model
* traffic went direct and uncompressed, the proxy never saw the session, and
* every tool forward 404'd. Attach keeps the user's intent ("route through
* THIS proxy"); explicit BILLION_CONTEXT_ATTACH wins when both are set. A
* /bili/ launch (BILI_PROVIDER_REWRITES) still stands us down — its URLs are
* already proxy-shaped. */
export function planNativeOmp(env: NodeJS.ProcessEnv): { mode: "off" | "attach" | "spawn"; attachOrigin?: string } {
if (env.BILLION_CONTEXT_PLUGIN === "0" || env.BILI_NATIVE_OMP === "0") return { mode: "off" };
if (env.BILI_PROVIDER_REWRITES !== undefined) return { mode: "off" };
const attachOrigin = nativeAttachOrigin(env) ?? proxyEnvOrigin(env);
if (attachOrigin !== undefined) return { mode: "attach", attachOrigin };
return { mode: "spawn" };
}

/** #1774: true when this OMP process routes model traffic through a bili proxy —
* self-bootstrap (gate passed), launcher/MITM launch (BILLION_CONTEXT_PROXY), or
* /bili/-rewrite launch (BILI_PROVIDER_REWRITES). Only then can a long preflight
Expand Down Expand Up @@ -59,8 +82,139 @@ async function bootstrap(): Promise<string | undefined> {
}
}

const nativeActive = shouldBootstrapNativeOmp(process.env);
if (nativeActive) markNativeHost(process.env, "omp");
let _spawnForTest: (() => Promise<string | undefined>) | undefined;

/** Test hook: replace the attach-fallback spawn (and the respawn self-heal's
* spawn) with a stub. Pass undefined to restore. */
export function _setSpawnForTest(fn?: () => Promise<string | undefined>): void {
_spawnForTest = fn;
}

/** #1135/#1365: the attached origin can go stale — at startup (the owning
* launcher exited before this process started) or at runtime (it exits while
* this session still rides the shared proxy, #1130). Probe before trusting
* it: healthy → keep attaching (a transient blip costs nothing — the session
* never migrates). DEAD + routed-channel evidence (#1365: /bili/-baked model
* traffic was observed at some origin) → the session's context lives at THAT
* origin, so wait for the pinned target to come back (bounded by
* BILI_ATTACH_HEALTH_DEADLINE_MS) instead of spawning — a second instance
* would serve tools while the model channel stays pinned elsewhere and every
* bili tool call 404s against it (unrecoverable split); the env is preserved
* so the user's target stays declared. DEAD with no evidence after the grace
* window (BILI_ATTACH_EVIDENCE_GRACE_MS) → unfreeze the preset env and fall
* back to spawning our own proxy, re-arming respawn as a pure spawn. Resolves
* to the origin the client should use — the (recovered) attach origin, the
* fallback origin, or undefined when nothing came up. */
export async function verifyAttachAndRecover(attachOrigin: string): Promise<string | undefined> {
// #1365: routed evidence outranks the planned origin — probe where the
// model channel actually points, not where the env says it should.
const home = state.routedOrigin ?? attachOrigin;
const version = await fetchProxyVersion(home).catch(() => undefined);
if (version !== undefined) {
// Restore the readyOrigin short-circuit — a runtime recovery clears
// state.origin before re-probing, and a transient blip must not leave
// it dangling.
state.origin = home;
process.env.BILLION_CONTEXT_PROXY = home;
return home;
}
const pinned = state.routedOrigin ?? (await observeRoutedOrigin(state));
if (pinned !== undefined) {
const back = await waitForProxyVersion(pinned);
if (back !== undefined) {
state.origin = back;
process.env.BILLION_CONTEXT_PROXY = back;
console.log(`bili-native(omp): attach target ${pinned} is healthy again — attached, no second instance spawned`);
return back;
}
console.error(`bili-native(omp): attach target ${pinned} is down and this process's model channel is pinned to it — refusing to spawn a second instance (bili tools would 404 against the other one). Start your proxy at ${pinned} or unset BILLION_CONTEXT_PROXY; bili keeps re-checking and self-heals when it comes back.`);
state.origin = undefined;
return undefined;
}
console.error(`bili-native(omp): attach target ${attachOrigin} is not healthy — falling back to a spawned proxy`);
delete process.env.BILLION_CONTEXT_PROXY;
state.attach = false;
state.origin = undefined;
const start = singleFlight(_spawnForTest ?? bootstrap);
state.respawn = start;
// Publish whatever origin lands on BOTH the shared state and the env —
// env readers (compaction reporter, /acp status) must never disagree with
// what the route resolves, whoever produced the origin.
const landed = start().then((o) => {
if (o !== undefined) {
state.origin = o;
process.env.BILLION_CONTEXT_PROXY = o;
}
return o;
});
state.ready = landed;
return landed;
}

const plan = planNativeOmp(process.env);

/** Wire the shared intercept state for the planned mode. Attach mode arms
* even under NODE_TEST_CONTEXT (it installs no global patches and spawns
* nothing while the target is healthy, so tests can drive it directly);
* spawn mode stays test-guarded because bootstrap forks a real proxy. */
export function armNativeOmp(p: typeof plan): void {
if (p.mode === "off") return;
markNativeHost(process.env, "omp");
if (p.mode === "attach") {
state.attach = true;
const attachOrigin = p.attachOrigin;
if (attachOrigin !== undefined) {
// #1135: arm the SAME probe+fallback for runtime death — the
// attached proxy is usually owned by ANOTHER launcher that can
// exit while this session rides it; the resolver then re-probes
// and falls back exactly like at startup. No synchronous
// state.origin freeze: the fetch intercept resolves live per
// request via readyOrigin(state).
process.env.BILLION_CONTEXT_PROXY = attachOrigin;
const start = singleFlight(() => verifyAttachAndRecover(attachOrigin));
state.respawn = start;
state.onGiveUp = () => {
delete process.env.BILLION_CONTEXT_PROXY;
};
// #1365: late routed evidence — if model traffic later arrives baked
// against a DIFFERENT origin than the one we attached to, rebind there
// (the context lives where the models go). Fire-and-forget with a
// liveness check: only converge on a target we can actually reach.
state.onRoutedOriginObserved = (origin) => {
if (state.origin === origin) return;
void fetchProxyVersion(origin).catch(() => undefined).then((version) => {
if (version === undefined || state.origin === origin) return;
state.origin = origin;
process.env.BILLION_CONTEXT_PROXY = origin;
console.warn(`bili-native(omp): model channel pinned to ${origin} — rebinding bili tools there`);
});
};
state.ready = start();
} else {
state.ready = Promise.resolve(undefined);
}
} else if (process.env.NODE_TEST_CONTEXT === undefined) {
const start = singleFlight(bootstrap);
state.respawn = start;
// #1531: pi-native's #1243 pattern — before_provider_request reads the
// proxy base from BILLION_CONTEXT_PROXY, which bootstrap() writes
// asynchronously. Publish the ready promise so the awaited runtime-info
// report can wait on the writer instead of racing it.
setNativeOriginWaiter({ wait: () => readyOrigin(state) });
state.onGiveUp = () => {
// We wrote BILLION_CONTEXT_PROXY at successful bootstrap. If the proxy
// dies mid-session and the respawn fails, traffic goes direct — clear
// the env so event-time ownership checks (session_before_compact
// cancel, prompt_cache_key stamping) stop claiming compression
// ownership and native compaction comes back with the direct traffic.
delete process.env.BILLION_CONTEXT_PROXY;
};
state.ready = start();
}
}

if (plan.mode !== "off") armNativeOmp(plan);

// #1774: widen OMP's first-parsed-event watchdog before any stream can start — a
// preflight over a large context holds the response for minutes while OMP only
// sees keep-alive comments, and its default 300s timer would abort mid-compression.
Expand All @@ -70,25 +224,33 @@ if (process.env.NODE_TEST_CONTEXT === undefined && ompTrafficRidesBili(process.e
}

// node:test imports this module for shouldBootstrapNativeOmp — never
// bootstrap a real proxy from inside a test run.
if (process.env.NODE_TEST_CONTEXT === undefined && nativeActive) {
const start = singleFlight(bootstrap);
state.respawn = start;
// #1531: pi-native's #1243 pattern — before_provider_request reads the
// proxy base from BILLION_CONTEXT_PROXY, which bootstrap() writes
// asynchronously. Publish the ready promise so the awaited runtime-info
// report can wait on the writer instead of racing it.
setNativeOriginWaiter({ wait: () => readyOrigin(state) });
state.onGiveUp = () => {
// We wrote BILLION_CONTEXT_PROXY at successful bootstrap. If the proxy
// dies mid-session and the respawn fails, traffic goes direct — clear
// the env so event-time ownership checks (session_before_compact
// cancel, prompt_cache_key stamping) stop claiming compression
// ownership and native compaction comes back with the direct traffic.
delete process.env.BILLION_CONTEXT_PROXY;
};
state.ready = start();
// patch globalThis.fetch or bootstrap a real proxy from inside a test run.
if (process.env.NODE_TEST_CONTEXT === undefined && plan.mode !== "off") {
installNativeFetchIntercept(state);
}

/** Test hook: expose the armed runtime-recovery seam (mirrors pi/opencode/dsh). */
export function _stateRespawnForTest(): (() => Promise<string | undefined>) | undefined {
return state.respawn;
}

/** Test hook (#1365): record routed-channel evidence without the fetch patch. */
export function _noteRoutedForTest(url: string): void {
noteRoutedOrigin(state, url);
}

/** Test hook: reset the module-level state in place (closures capture the
* object reference) so suites can drive armNativeOmp repeatedly. */
export function _resetNativeStateForTest(): void {
state.attach = undefined;
state.origin = undefined;
state.routedOrigin = undefined;
state.onRoutedOriginObserved = undefined;
state.respawn = undefined;
state.onGiveUp = undefined;
state.ready = Promise.resolve(undefined);
_spawnForTest = undefined;
delete process.env.BILLION_CONTEXT_PROXY;
}

export default createBiliPlugin("omp");
101 changes: 100 additions & 1 deletion tests/omp-native.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import factory, { ompTrafficRidesBili, shouldBootstrapNativeOmp } from "../src/agent/omp-native.ts";
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import factory, { ompTrafficRidesBili, shouldBootstrapNativeOmp, planNativeOmp, armNativeOmp, _resetNativeStateForTest, _setSpawnForTest, _stateRespawnForTest } from "../src/agent/omp-native.ts";
import { applyOmpFirstEventTimeout, nativeProxyScriptPath, ompFirstEventTimeoutValue, OMP_FIRST_EVENT_TIMEOUT_DEFAULT_MS, OMP_FIRST_EVENT_TIMEOUT_ENV } from "../src/agent/native-bootstrap.ts";
import { pluginInstall, pluginRemove, pluginStatusAll, selfPackageRoot, ompPluginLoadedFrom } from "../src/plugin-install.ts";
import { rmrf } from "./tmp-rm.ts";

// #1135 wiring tests drive verifyAttachAndRecover against a dead target — cap
// the routed-evidence grace so the probe fails fast instead of paying the 5s default.
process.env.BILI_ATTACH_EVIDENCE_GRACE_MS = "30";

// #957 coexistence: markNativeHost must be set synchronously during module
// evaluation so any in-process bili extension backs off in THIS process.
test("module evaluation marks the process as a native omp host", () => {
Expand Down Expand Up @@ -174,3 +180,96 @@ test("ompPluginLoadedFrom: launcher skips -e only for a loadable entry (either f
fs.writeFileSync(file, `extensions:\n - ${NATIVE_ENTRY}\n`);
if (fs.existsSync(NATIVE_ENTRY)) assert.equal(ompPluginLoadedFrom(home), true, "loadable native entry counts");
}));

// —— #1795: preset BILLION_CONTEXT_PROXY is an attach target, never a stand-down ——

test("planNativeOmp: default is spawn; opt-out and /bili/ launches are off", () => {
assert.deepEqual(planNativeOmp({}), { mode: "spawn" });
assert.deepEqual(planNativeOmp({ BILLION_CONTEXT_PLUGIN: "0" }), { mode: "off" });
assert.deepEqual(planNativeOmp({ BILI_NATIVE_OMP: "0" }), { mode: "off" });
assert.deepEqual(planNativeOmp({ BILI_PROVIDER_REWRITES: '{"vllm":"http://127.0.0.1:1/bili/http://x"}' }), { mode: "off" });
});

test("planNativeOmp: a preset BILLION_CONTEXT_PROXY is an attach target, not a stand-down", () => {
// #1795 regression: the pseudo-attach hole used to resolve a preset proxy
// to "off" (gate closed), disarming the fetch intercept entirely so every
// model request went direct and uncompressed.
assert.deepEqual(
planNativeOmp({ BILLION_CONTEXT_PROXY: "http://127.0.0.1:36485" }),
{ mode: "attach", attachOrigin: "http://127.0.0.1:36485" },
);
assert.deepEqual(
planNativeOmp({ BILLION_CONTEXT_PROXY: "http://127.0.0.1:36485/" }),
{ mode: "attach", attachOrigin: "http://127.0.0.1:36485" },
);
// kill switches and a /bili/ launch still win over the preset
assert.deepEqual(planNativeOmp({ BILLION_CONTEXT_PROXY: "http://127.0.0.1:36485", BILLION_CONTEXT_PLUGIN: "0" }), { mode: "off" });
assert.deepEqual(planNativeOmp({ BILLION_CONTEXT_PROXY: "http://127.0.0.1:36485", BILI_PROVIDER_REWRITES: "{}" }), { mode: "off" });
// a garbage / blank preset falls back to spawn (self-managed), never dead-off
assert.deepEqual(planNativeOmp({ BILLION_CONTEXT_PROXY: "garbage" }), { mode: "spawn" });
assert.deepEqual(planNativeOmp({ BILLION_CONTEXT_PROXY: " " }), { mode: "spawn" });
});

test("planNativeOmp: explicit BILLION_CONTEXT_ATTACH wins over the env preset", () => {
assert.deepEqual(
planNativeOmp({ BILLION_CONTEXT_PROXY: "http://127.0.0.1:36485", BILLION_CONTEXT_ATTACH: "http://10.0.0.5:9000" }),
{ mode: "attach", attachOrigin: "http://10.0.0.5:9000" },
);
});

test("#1135 wiring: a dead attach target falls back to a spawned proxy", async () => {
const savedProxy = process.env.BILLION_CONTEXT_PROXY;
try {
_resetNativeStateForTest();
let spawned = 0;
_setSpawnForTest(async () => {
spawned++;
return "http://127.0.0.1:7777";
});
armNativeOmp({ mode: "attach", attachOrigin: "http://127.0.0.1:9" });
assert.equal(typeof _stateRespawnForTest(), "function", "respawn seam armed at load");
// port 9 refuses connections instantly — the manifest probe fails fast,
// the fallback spawn lands and replaces the env origin
const landed = await _stateRespawnForTest()!();
assert.equal(landed, "http://127.0.0.1:7777");
assert.ok(spawned >= 1);
assert.equal(process.env.BILLION_CONTEXT_PROXY, "http://127.0.0.1:7777");
} finally {
if (savedProxy === undefined) delete process.env.BILLION_CONTEXT_PROXY;
else process.env.BILLION_CONTEXT_PROXY = savedProxy;
_resetNativeStateForTest();
}
});

test("#1135 wiring: a healthy attach target stays attached (no migration, no spawn)", async () => {
const savedProxy = process.env.BILLION_CONTEXT_PROXY;
const server = createServer((req, res) => {
if (req.url === "/__bili/plugin/manifest") {
res.setHeader("content-type", "application/json");
res.end(JSON.stringify({ version: "0.1.175", tools: [] }));
return;
}
res.statusCode = 404;
res.end();
});
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
const port = (server.address() as AddressInfo).port;
try {
_resetNativeStateForTest();
let spawned = 0;
_setSpawnForTest(async () => {
spawned++;
return "http://127.0.0.1:7778";
});
armNativeOmp({ mode: "attach", attachOrigin: `http://127.0.0.1:${port}` });
const landed = await _stateRespawnForTest()!();
assert.equal(landed, `http://127.0.0.1:${port}`, "healthy target resolves to itself");
assert.equal(spawned, 0, "no fallback spawn for a healthy target");
assert.equal(process.env.BILLION_CONTEXT_PROXY, `http://127.0.0.1:${port}`);
} finally {
server.close();
if (savedProxy === undefined) delete process.env.BILLION_CONTEXT_PROXY;
else process.env.BILLION_CONTEXT_PROXY = savedProxy;
_resetNativeStateForTest();
}
});
Loading