Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 70 additions & 4 deletions src/agent/dsh-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@ export const name = "bili-native";
export const inject = ["tools", "commands", "agents"];

const RETRY_INTERVAL_MS = 10000;
/** #2082: BILI_DSH_RETRY_INTERVAL_MS — test/ops knob to shrink the retry
* wall without wall-clock waits (same pattern as BILI_MODEL_INFO_RETRY_MS). */
function retryIntervalMs(): number {
const raw = process.env.BILI_DSH_RETRY_INTERVAL_MS;
const parsed = raw === undefined ? Number.NaN : Number(raw);
return Number.isFinite(parsed) && parsed > 0 ? parsed : RETRY_INTERVAL_MS;
}

type AgentLike = { session?: { id?: unknown } | undefined };
// #1677: DSH's command executor hands the invoking agent to the handler via the
Expand Down Expand Up @@ -465,7 +472,7 @@ async function registerTools(ctx: PluginContext): Promise<void> {
// apply chain unfroze a dead preset, maybeRetry healed elsewhere),
// an armed retryAt would gate the NEXT base's first heal behind a
// 10s wall — exactly the windows-22 CI deadlock (#1783).
if (register.base === base) register.retryAt = Date.now() + RETRY_INTERVAL_MS;
if (register.base === base) register.retryAt = Date.now() + retryIntervalMs();
console.error(`bili-native-dsh: manifest registration failed (${errMessage(err)}) — retrying; requests stay in wire mode until it succeeds`);
})
.finally(() => {
Expand All @@ -474,7 +481,37 @@ async function registerTools(ctx: PluginContext): Promise<void> {
return register.pending;
}

// #2082: recovery retries are traffic-driven (headersFor on model
// requests, statusOutcome on /acp). When the user goes idle — or the model
// already gave up on the missing tools, exactly the #2082 report — nothing
// re-arms them, so a proxy that recovered in the meantime sits unused until
// the next request. A detached repeating timer keeps the loop running while
// the tools are down and clears itself on the first healthy (or dead) tick.
// BILI_DSH_RECOVERY_INTERVAL_MS: test hook to shrink the cadence (also an
// ops knob); absent it tracks RETRY_INTERVAL_MS. Tests that do not opt in
// never start a timer at all (NODE_TEST_CONTEXT guard keeps suites quiet).
let recoveryTimer: NodeJS.Timeout | undefined;
function recoveryIntervalMs(): number {
const raw = process.env.BILI_DSH_RECOVERY_INTERVAL_MS;
const parsed = raw === undefined ? Number.NaN : Number(raw);
return Number.isFinite(parsed) && parsed > 0 ? parsed : RETRY_INTERVAL_MS;
}
function armRecoveryTimer(ctx: PluginContext): void {
if (recoveryTimer !== undefined) return;
if (process.env.NODE_TEST_CONTEXT !== undefined && process.env.BILI_DSH_RECOVERY_INTERVAL_MS === undefined) return;
recoveryTimer = setInterval(() => {
if (register.dead || register.toolsReady) {
if (recoveryTimer !== undefined) clearInterval(recoveryTimer);
recoveryTimer = undefined;
return;
}
maybeRetry(ctx);
}, recoveryIntervalMs());
recoveryTimer.unref?.();
}

function maybeRetry(ctx: PluginContext): void {
if (!register.dead && !register.toolsReady) armRecoveryTimer(ctx);
if (register.dead || register.toolsReady) return;
if (register.pending !== undefined) return;
if (Date.now() < register.retryAt) return;
Expand All @@ -483,9 +520,18 @@ function maybeRetry(ctx: PluginContext): void {
// without this branch the plugin never recovers and tools die for
// good. Self-heal: re-arm the respawn every retry interval until a
// proxy comes back (attach mode arms one since #1130).
const respawn = state.respawn;
if (respawn === undefined) return;
register.retryAt = Date.now() + RETRY_INTERVAL_MS;
let respawn = state.respawn;
if (respawn === undefined) {
// #2082: nothing armed the respawn hook — a plan edge (attach
// landing after a dead spawn chain) or an apply() that ran under
// the NODE_TEST_CONTEXT spawn guard. A base-less register with no
// respawn armed stays dead forever; arm the fallback here so the
// retry chain below has something to drive.
if (_spawnForTest === undefined && process.env.NODE_TEST_CONTEXT !== undefined) return;
respawn = singleFlight(_spawnForTest ?? bootstrap);
state.respawn = respawn;
}
register.retryAt = Date.now() + retryIntervalMs();
void trackChain(respawn())
.then((origin) => {
if (origin === undefined) return;
Expand Down Expand Up @@ -659,6 +705,15 @@ export function apply(ctx: PluginContext): void {
const plan = planNativeDsh(process.env);
if (plan.mode === "off") return;

// #2082: a fresh activation invalidates the "host tearing down" verdict.
// Without this reset, ONE registerTools rejection carrying "inactive
// context" (a cordis deactivate/reactivate cycle — dsh reloading the
// plugin after the desktop profile copy was swapped in place, a profile
// switch, a settings toggle) poisons register.dead for the whole process
// lifetime: every later maybeRetry() returns immediately and the bili
// tools never come back until dsh restarts.
register.dead = false;

// #1590: the dsh web-profile settings panel shows a "bili设置" entry
// (dsh-native-client.js) that opens this proxy's Web UI. The origin is
// only published while known — attach mode binds register.base
Expand Down Expand Up @@ -1006,6 +1061,10 @@ export function apply(ctx: PluginContext): void {
/** Test hook: reset the module-level registration lifecycle so suites can
* drive apply() repeatedly with a fresh mock ctx. */
export function _resetRegisterForTest(base: string | undefined): void {
if (recoveryTimer !== undefined) {
clearInterval(recoveryTimer);
recoveryTimer = undefined;
}
register.base = base;
register.toolsReady = false;
register.dead = false;
Expand All @@ -1016,6 +1075,13 @@ export function _resetRegisterForTest(base: string | undefined): void {
modelInfo.refreshing = false;
}

/** Test hook (#2082): drive the recovery loop directly — production callers
* are traffic-side (headersFor / statusOutcome); tests trigger the same
* path without fabricating model traffic. */
export function maybeRetryForTest(ctx: PluginContext): void {
maybeRetry(ctx);
}

/** Test hook (#1797): resolve once every in-flight attach/recovery chain has
* settled, plus one macrotask so their .then state-writes have run. */
export function _settleNativeForTest(): Promise<void> {
Expand Down
82 changes: 79 additions & 3 deletions src/update.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@
* cycle automatically.
*/
import { readFile, writeFile, mkdir, access, constants, rm, cp, unlink, lstat, rename } from "node:fs/promises";
import { execFile } from "node:child_process";
import { execFile, spawn } from "node:child_process";
import { realpathSync } from "node:fs";
import crypto from "node:crypto";
import * as tar from "tar";
import path from "node:path";
import os from "node:os";
import { fileURLToPath } from "node:url";
import { safeSuffix } from "./text-safe.js";
import { cacheDir } from "./paths.js";
import { log as loggerLog, type Logger } from "./logger.js";
import { refreshDshProfileBundles, isDshProfileCopy, dshProfileDirs, dshProfileDependsOnBili, dshProfileDepSpec, isRegistryDepSpec, DSH_PACKAGE, DSH_DESKTOP_PROFILE } from "./dsh-channel.js";
Expand Down Expand Up @@ -842,9 +843,9 @@ export async function refreshDshDesktopCopy(
log("warn", `[update] dsh ${DSH_DESKTOP_PROFILE}: in-place refresh to ${targetVersion} failed \u2014 no dist.tarball for that version on the registry; retrying next cycle`);
return;
}
const result = await installViaTarball(targetVersion, doc.tarball, flat, doc.integrity, doc.shasum, egressDispatcher({ resolveProxy }, doc.tarball), env);
const result = await installViaTarball(targetVersion, doc.tarball, flat, doc.integrity, doc.shasum, egressDispatcher({ resolveProxy }, doc.tarball), env, { bootSmoke: true });
if (result.ok) {
log("info", `[update] refreshed dsh ${DSH_DESKTOP_PROFILE} profile copy in place (${diskVersion ?? "?"} \u2192 ${targetVersion}) \u2014 restart dsh to load it`);
log("info", `[update] refreshed dsh ${DSH_DESKTOP_PROFILE} profile copy in place (${diskVersion ?? "?"} \u2192 ${targetVersion}) \u2014 restart dsh to load it (the running app keeps the old code in memory; across the handoff bili tools may fail once until dsh restarts, #2082)`);
} else {
log("warn", `[update] dsh ${DSH_DESKTOP_PROFILE}: in-place refresh to ${targetVersion} failed: ${result.error}; retrying next cycle`);
}
Expand Down Expand Up @@ -1177,6 +1178,63 @@ export function verifyTarballIntegrity(buf: Buffer, integrity?: string, shasum?:
* over the install directory. `dispatcher` (optional) routes the download
* through a proxy (#609); omitted = direct connection. */

const BOOT_SMOKE_TIMEOUT_MS = 20_000;

/** #2082: run the freshly installed copy's CLI entry with --version in a
* clean child process. Loading the module graph and exiting 0 is the smoke;
* any non-zero exit, timeout, or spawn failure returns a short reason (with
* the last stderr lines) for the caller to log before rolling back.
* Env scrub: the child must not inherit the parent's BILLION_CONTEXT_* /
* BILI_* knobs (a preset proxy origin would make --version spin up lanes)
* nor NODE_OPTIONS / NODE_TEST_CONTEXT interference. */
async function bootSmoke(installDir: string, env: NodeJS.ProcessEnv): Promise<string | null> {
let entry: string | undefined;
try {
const pkg = JSON.parse(await readFile(path.join(installDir, "package.json"), "utf8")) as { main?: string };
if (typeof pkg.main === "string" && pkg.main.length > 0) entry = pkg.main;
} catch (err) {
return `package.json unreadable: ${err instanceof Error ? err.message : String(err)}`;
}
if (entry === undefined) return "package.json declares no main entry to smoke";
const childEnv: NodeJS.ProcessEnv = {};
for (const [key, value] of Object.entries(env)) {
if (value === undefined) continue;
if (key.startsWith("BILI_") || key.startsWith("BILLION_CONTEXT_") || key.startsWith("ACP_")) continue;
if (key === "NODE_OPTIONS" || key === "NODE_TEST_CONTEXT" || key === "NODE_ENV") continue;
childEnv[key] = value;
}
return await new Promise<string | null>((resolve) => {
const child = spawn(process.execPath, [path.join(installDir, entry!), "--version"], {
cwd: installDir,
env: childEnv,
stdio: ["ignore", "ignore", "pipe"],
});
let stderr = "";
child.stderr?.on("data", (chunk: Buffer) => {
stderr += chunk.toString();
if (stderr.length > 8192) stderr = safeSuffix(stderr, 8192);
});
const timer = setTimeout(() => {
child.kill();
resolve(`boot smoke timed out after ${BOOT_SMOKE_TIMEOUT_MS}ms`);
}, BOOT_SMOKE_TIMEOUT_MS);
child.on("error", (err) => {
clearTimeout(timer);
resolve(`boot smoke could not spawn ${process.execPath}: ${err.message}`);
});
child.on("close", (code, signal) => {
clearTimeout(timer);
if (code === 0) {
resolve(null);
return;
}
const lines = stderr.split("\n").filter((l) => l.trim().length > 0 && !l.trim().startsWith("at ") && !/^Node\.js v/.test(l.trim()));
const tail = lines.length > 0 ? `: ${safeSuffix(lines.join(" | "), 600)}` : "";
resolve(`boot smoke exited ${code ?? signal}${tail}`);
});
});
}

export async function installViaTarball(
version: string,
tarballUrl: string,
Expand All @@ -1185,6 +1243,7 @@ export async function installViaTarball(
shasum?: string,
dispatcher?: object,
env: NodeJS.ProcessEnv = process.env,
opts: { bootSmoke?: boolean } = {},
): Promise<{ ok: boolean; error?: string }> {
if (!installDir) {
return { ok: false, error: "cannot determine install directory (package.json not found walking up from running binary)" };
Expand Down Expand Up @@ -1396,6 +1455,23 @@ export async function installViaTarball(
return { ok: false, error: rb ?? postEntryErr };
}

// #2082: optional boot smoke. verifyEntries proves the declared entries
// PARSE; it cannot prove the artifact BOOTS (a top-level throw, a broken
// import graph, a runtime dependency the flattened copy no longer
// resolves — all syntactically valid). The desktop lane swaps this copy
// in place under a RUNNING dsh whose plugin respawns from it on the next
// proxy death; an unbootable copy turns that respawn into an infinite
// give-up loop and the bili tools vanish until dsh restarts. Run the new
// copy's CLI entry with --version once — the module graph loads and exits
// immediately. Any failure rolls the working copy back to the backup.
if (opts.bootSmoke) {
const smokeErr = await bootSmoke(installDir, env);
if (smokeErr !== null) {
const rb = await restoreFromBackup();
return { ok: false, error: rb ?? `boot smoke failed (rolled back to the previous copy): ${smokeErr}` };
}
}

// Success: the backup is no longer needed, and neither is the displaced
// pnpm link (the store copy underneath it was left byte-identical).
await rm(backupDir, { recursive: true, force: true });
Expand Down
Loading
Loading