Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions scripts/check-unused-exports.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ const WAIVERS = new Map([
// structurally via Awaited<ReturnType<typeof getLastInstall>>); keep the
// export so the wire shape stays named at its definition site.
["src/update.ts", new Set(["LastInstallRecord"])],
// #5146 /__bili/upstreams read contract: the two report payload interfaces
// are referenced only inside src/upstream-alerts.ts (admin.ts consumes
// getUpstreamHealthReport() structurally via JSON.stringify); keep the
// exports so the wire shape stays named at its definition site.
["src/upstream-alerts.ts", new Set(["UpstreamHealthEntry", "UpstreamHealthReport"])],
// #3962 gate telemetry: the coverage record crosses the gate/dsh-native
// BUNDLE boundary on the globalThis channel — consumers use it
// structurally through dshCompactionGateTelemetry()'s return type; keep
Expand Down
13 changes: 12 additions & 1 deletion src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2855,7 +2855,14 @@ export class SessionStore {
this.drainFlushWaiters(id, this.lastSaveLanded.get(id) === true);
});
}, this.debounceMs);
timer.unref?.();
// #4788: the debounce timer is host-owned periodic upkeep and must not
// pin a process open on its own -- so it is unref'd. But `afterNextFlush`
// makes it a HOST: a caller that awaits a verdict has no other handle
// guaranteed to fire, and an unref'd timer is not a reason for Node to
// stay alive. Awaiting then exits the loop with the promise unresolved
// (`Promise resolution is still pending but the event loop has already
// resolved`). Keep the timer referenced exactly while someone waits.
if (!this.flushWaiters.has(id)) timer.unref?.();
this.saveTimers.set(id, timer);
}

Expand All @@ -2876,6 +2883,10 @@ export class SessionStore {
const ws = this.flushWaiters.get(sessionId) ?? [];
ws.push((ok: boolean): void => { resolve(ok); });
this.flushWaiters.set(sessionId, ws);
// #4788: the timer was armed unref'd (no waiter existed then). The
// waiter now holds the promise open, so the timer must hold the loop
// open too -- otherwise the process drains and the verdict is lost.
this.saveTimers.get(sessionId)?.ref?.();
});
}

Expand Down
16 changes: 15 additions & 1 deletion src/server/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { PCK_NOTICE_ID, dismissUpgradeNotice, pckNoticeActive } from "../pck-not
import { listSessions, memoryReport, peekSession, splitSessionWarnings, totalInFlight } from "../session.js";
import { BILI_TUNNEL_HEADER } from "../tunnel-guard.js";
import { isLoopbackAddress } from "../util.js";
import { clearUpstreamAlertsForHost, getUpstreamAlerts } from "../upstream-alerts.js";
import { clearUpstreamAlertsForHost, getUpstreamAlerts, getUpstreamHealthReport } from "../upstream-alerts.js";
import { pckAblations } from "../pck-probe.js";
import { formatUpstreamError, getUpstreamConnectionStatus, proxyDispatcher, recordUpstreamConnection, resetProxyCache, resolveProxyDecision } from "../upstream-proxy.js";
import { detectStaleInstall, getLastInstall } from "../update.js";
Expand Down Expand Up @@ -235,6 +235,20 @@ export async function handleAdminRoute(req: http.IncomingMessage, res: http.Serv
if (req.method === "GET" && req.url === "/__bili/status") return sendStatus(res, opts, typeof req.headers.host === "string" ? req.headers.host : undefined);
if (req.method === "GET" && req.url === "/__bili/resign") return sendResignStatus(res);
if (req.method === "GET" && req.url === "/__bili/overview") return sendOverview(res, opts, typeof req.headers.host === "string" ? req.headers.host : undefined);
// #5146: per-host upstream health (alerts + open circuits) for DSH's frame-level
// banner — loopback-gated like every other /__bili/* route above. The DSH
// renderer polls this CROSS-ORIGIN from its own web port, so echo the
// requesting origin as ACAO exactly like the /__bili/ shell and
// /__bili/plugin/status do: only origins the trusted-origin gate admitted
// reach this line, and the payload is a bounded in-memory snapshot.
if (req.method === "GET" && req.url === "/__bili/upstreams") {
const reqOrigin = req.headers.origin;
const headers: Record<string, string> = { "content-type": "application/json" };
if (typeof reqOrigin === "string" && reqOrigin.length > 0) headers["access-control-allow-origin"] = reqOrigin;
res.writeHead(200, headers);
res.end(JSON.stringify(getUpstreamHealthReport()));
return;
}
if (req.method === "GET" && (req.url === "/__bili/sessions" || req.url?.startsWith("/__bili/sessions?"))) return sendWebSessions(res, req);
if (req.method === "GET" && req.url?.startsWith("/__bili/logs")) return sendWebLogs(res, req);
if (req.method === "GET" && req.url?.startsWith("/__bili/diagnostics")) return sendDiagnostics(res, req.url);
Expand Down
4 changes: 2 additions & 2 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -329,12 +329,12 @@ export class UnifiedShadow {
: "; delete the file to reset the index";
loggerLog(
"warn",
`[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${tail}`,
`[storage] unified sidecar failed to open ${path.join(dir, "index.db")} (${errStr}) — staying legacy-only${tail}`,
);
recordSidecarOpenFailure(errStr);
return null;
}
const shadow = new UnifiedShadow(mode, store, legacyLoad ?? (() => null), undefined, `${dir}/index.db`);
const shadow = new UnifiedShadow(mode, store, legacyLoad ?? (() => null), undefined, path.join(dir, "index.db"));
recordSidecarInstance(shadow);
const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`;
const walTag = ` [wal-cap: ${walLimit}]`;
Expand Down
104 changes: 101 additions & 3 deletions src/upstream-alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ export interface UpstreamAlert {
* the full English string for JSON consumers). Redacted. */
skippedProxy?: string;
skippedSource?: string;
/** #5146: errno captured at failure time (e.g. ECONNREFUSED) — surfaces in the
* /__bili/upstreams report so the DSH banner can show WHY. */
code?: string;
}

/** v1 scope (#1682): only "fundamentally cannot connect" transport failures.
Expand Down Expand Up @@ -62,6 +65,13 @@ function keyFor(kind: UpstreamFailureKind, host: string): string {
return kind + "\u0000" + host;
}

/** #5146: pull the errno off a Node error when present (ECONNREFUSED, ENOTFOUND, …) so the
* /__bili/upstreams report can show WHY beyond the classified kind. undefined for non-errors. */
function errnoCode(error: unknown): string | undefined {
const c = typeof error === "object" && error !== null ? (error as { code?: unknown }).code : undefined;
return typeof c === "string" && c.length > 0 ? c : undefined;
}

/** Record one failed model request. Dedup key (kind, host): repeats refresh
* lastSeen/count only. Non-alert kinds are ignored entirely. `skipped` (#2809)
* rides along so the banner can name the proxy default-direct mode skipped. */
Expand All @@ -70,6 +80,7 @@ export function recordUpstreamAlert(url: string, error: unknown, viaProxy: boole
if (!ALERT_KINDS.has(kind)) return;
const host = hostOf(url);
const now = Date.now();
const code = errnoCode(error);
const key = keyFor(kind, host);
const skippedFields = skipped ? { skippedProxy: redactedSkippedProxy(skipped), skippedSource: skipped.source } : {};
const hint = hintWithSkippedProxy(kind, skipped);
Expand All @@ -78,6 +89,7 @@ export function recordUpstreamAlert(url: string, error: unknown, viaProxy: boole
existing.lastSeen = now;
existing.count += 1;
existing.hint = hint;
if (code !== undefined) existing.code = code;
Object.assign(existing, skippedFields);
return;
}
Expand All @@ -92,7 +104,7 @@ export function recordUpstreamAlert(url: string, error: unknown, viaProxy: boole
}
if (oldestKey !== undefined) alerts.delete(oldestKey);
}
alerts.set(key, { kind, host, hint, firstSeen: now, lastSeen: now, count: 1, ...skippedFields });
alerts.set(key, { kind, host, hint, firstSeen: now, lastSeen: now, count: 1, ...(code !== undefined ? { code } : {}), ...skippedFields });
}

/** Any resolved response to this host proves the path works again — clear ALL
Expand Down Expand Up @@ -133,6 +145,8 @@ interface UpstreamCircuitFailure {
kind: string;
detail: string;
at: number;
/** #5146: errno at failure time for the /__bili/upstreams report. */
code?: string;
}

interface UpstreamCircuit {
Expand Down Expand Up @@ -248,6 +262,7 @@ export function recordUpstreamCircuitFailure(url: string, error: unknown, viaPro
const kind = classifyUpstreamFailure(error, { viaProxy });
const host = hostOf(url);
const now = Date.now();
const code = errnoCode(error);
let entry = circuits.get(host);
if (!entry) {
if (!ALERT_KINDS.has(kind)) return;
Expand All @@ -271,14 +286,14 @@ export function recordUpstreamCircuitFailure(url: string, error: unknown, viaPro
if (c.openedAt !== null) {
// Open: the half-open probe just finished without a response — release
// the slot and re-arm the cooldown regardless of kind (see above).
c.lastFailure = { kind, detail: shortDetail, at: now };
c.lastFailure = { kind, detail: shortDetail, at: now, ...(code !== undefined ? { code } : {}) };
entry.probeInFlight = false;
c.nextProbeAt = now + circuitProbeCooldownMs;
loggerLog("warn", `[acp-proxy] upstream circuit probe FAILED for ${host} (${kind}) — re-armed until ${new Date(c.nextProbeAt).toISOString()} (#3148)`);
return;
}
if (!ALERT_KINDS.has(kind)) return;
c.lastFailure = { kind, detail: shortDetail, at: now };
c.lastFailure = { kind, detail: shortDetail, at: now, ...(code !== undefined ? { code } : {}) };
c.consecutiveFailures += 1;
if (c.consecutiveFailures >= CIRCUIT_OPEN_AFTER) {
c.openedAt = now;
Expand Down Expand Up @@ -306,6 +321,89 @@ export function getOpenUpstreamCircuits(): UpstreamCircuit[] {
.sort((a, b) => (b.openedAt ?? 0) - (a.openedAt ?? 0));
}

// #5146: the read contract behind GET /__bili/upstreams — the loopback-gated
// surface DSH's frame-level banner polls (contract settled in #4254; that PR's
// landing of this half was dropped when it closed unmerged, which is exactly
// why the shipped banner polled a route that never existed). bili is a
// transparent pipe, so this reports per HOST (the only identity it knows),
// never per conversation; DSH filters that down to the hosts its own
// conversations point at. Merges the two existing tables into one stable
// shape: the open circuit is the authoritative "currently down" signal
// (fail-fast stops minting new alerts once open), while the alert rows carry
// the actionable hint and cover the pre-circuit window.

export interface UpstreamHealthEntry {
/** Host[:port] the failing traffic targeted — raw (endpoint is loopback-gated). */
host: string;
port: number | null;
/** Primary failure kind: the circuit's last failure when open, else the most
* recently seen alert for the host. */
kind: string;
/** errno captured at failure time (ECONNREFUSED, …) when available. */
code?: string;
/** Actionable hint (most recent alert of the primary kind, else circuit detail). */
hint?: string;
/** Observed failure count — circuit strikes when open, else summed alert counts. */
count: number;
firstSeen: number;
lastSeen: number;
/** Present only while a circuit is OPEN; null in the alert-only window. */
circuit: { state: "open"; nextProbeInMs: number } | null;
}

export interface UpstreamHealthReport {
upstreams: UpstreamHealthEntry[];
}

export function getUpstreamHealthReport(now: number = Date.now()): UpstreamHealthReport {
interface Acc {
alerts: UpstreamAlert[];
circuit: UpstreamCircuit | null;
}
const byHost = new Map<string, Acc>();
const touch = (host: string): Acc => {
let acc = byHost.get(host);
if (!acc) {
acc = { alerts: [], circuit: null };
byHost.set(host, acc);
}
return acc;
};
for (const a of alerts.values()) touch(a.host).alerts.push(a);
for (const e of circuits.values()) {
if (e.state.openedAt !== null) touch(e.state.host).circuit = e.state;
}
const upstreams: UpstreamHealthEntry[] = [];
for (const [host, { alerts: hostAlerts, circuit }] of byHost.entries()) {
const latest = hostAlerts.length > 0 ? hostAlerts.reduce((m, a) => (a.lastSeen > m.lastSeen ? a : m)) : undefined;
const kind = circuit?.lastFailure?.kind ?? latest?.kind ?? "unknown";
const code = circuit?.lastFailure?.code ?? latest?.code;
const hint = latest?.hint ?? circuit?.lastFailure?.detail;
const count = circuit ? circuit.consecutiveFailures : hostAlerts.reduce((s, a) => s + a.count, 0);
const beginTimes = [...hostAlerts.map((a) => a.firstSeen), ...(circuit ? [circuit.openedAt ?? now] : [])];
const endTimes = [...hostAlerts.map((a) => a.lastSeen), ...(circuit ? [circuit.lastFailure?.at ?? now] : [])];
let port: number | null = null;
const colon = host.lastIndexOf(":");
if (colon >= 0) {
const p = Number(host.slice(colon + 1));
if (Number.isInteger(p) && p > 0 && p <= 65535) port = p;
}
upstreams.push({
host,
port,
kind,
...(code !== undefined ? { code } : {}),
...(hint !== undefined ? { hint } : {}),
count,
firstSeen: Math.min(...beginTimes),
lastSeen: Math.max(...endTimes),
circuit: circuit ? { state: "open", nextProbeInMs: Math.max(0, (circuit.nextProbeAt ?? now) - now) } : null,
});
}
upstreams.sort((a, b) => b.lastSeen - a.lastSeen);
return { upstreams };
}

export function _resetUpstreamCircuitsForTest(): void {
circuits = new Map();
circuitProbeCooldownMs = 60_000;
Expand Down
14 changes: 12 additions & 2 deletions src/web/diagnostics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,18 @@ function hostClientVersionLines(): string[] {
return lines;
}
for (const dir of dirs.slice(0, DSH_PROFILE_CAP)) {
const vers = DSH_REPORT_PACKAGES.map((p) => `${p.split("/")[1]} ${dshPackageVersion(dir, p) ?? "?"}`).join(", ");
lines.push(`dsh profile ${path.basename(dir)}: ${vers}`);
const versions = DSH_REPORT_PACKAGES.map((p) => dshPackageVersion(dir, p));
if (versions.every((v) => v === undefined)) {
// #5146: three bare "?"s told triage nothing about WHERE the version
// truth lives — on DSH Desktop bundle installs the @deepseek-ai core
// ships inside the app, not under the profile's node_modules (which
// holds third-party plugins like billion-context alone), so every
// lookup missed and reports lost the host-version dimension entirely.
lines.push(`dsh profile ${path.basename(dir)}: (no @deepseek-ai packages under this profile bundle — on desktop installs the core ships inside the app; include the app version)`);
} else {
const vers = DSH_REPORT_PACKAGES.map((p, i) => `${p.split("/")[1]} ${versions[i] ?? "?"}`).join(", ");
lines.push(`dsh profile ${path.basename(dir)}: ${vers}`);
}
}
if (dirs.length > DSH_PROFILE_CAP) lines.push(`… ${dirs.length - DSH_PROFILE_CAP} more profile(s)`);
return lines;
Expand Down
8 changes: 6 additions & 2 deletions tests/inc-canonical-pointers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,11 @@ test("cold canonical rebuild: identical resync keeps the shared prefix; delta as
// #5155: the wrap-up defers the durable flush (debounced) — let the
// rows settle before dropping the resident caches, or the cold
// rebuild reads a one-turn-stale table and honestly 409s.
await new Promise((resolve) => setTimeout(resolve, 20));
// Await the settle itself (afterNextFlush resolves once the armed save
// lands) instead of a fixed sleep: the settle is a real condition, and
// a magic millisecond budget is a deadline a loaded windows-latest
// runner can miss — the 409 flake this replaces.
assert.equal(await getStore().afterNextFlush("conv-ptrs"), true, "the turn's durable save must land before the cold reset");
__resetIncrementalLaneForTests();

const delta = await h.turn({ sinceSeq: 3, messages: [core("u4", "four")] });
Expand Down Expand Up @@ -135,7 +139,7 @@ test("cold canonical rebuild: edited-tail resync keeps the shared prefix, drops
await h.turn({ sinceSeq: 0, resync: true, messages: [core("u1", "one"), core("u2", "two-EDITED")] });
await __incrementalPtrChainForTests("conv-ptrs");
// #5155: settle the debounced flush before the cold reset (see above).
await new Promise((resolve) => setTimeout(resolve, 20));
assert.equal(await getStore().afterNextFlush("conv-ptrs"), true, "the resync's durable save must land before the cold reset");
__resetIncrementalLaneForTests();
const delta = await h.turn({ sinceSeq: 2, messages: [core("u3", "three")] });
assert.equal(delta.status, 200);
Expand Down
4 changes: 2 additions & 2 deletions tests/issue4739-bounded-session-load.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ test("full mode: viewFor decodes only pointer rows and matches the full-log deco
assert.deepEqual(via, reference, "bounded decode byte-equal to the full-log decode");
assert.equal(delta, expectedPointers, `viewFor read ${delta} message rows for ${expectedPointers} pointer seqs (full log = ${liveSeqs.size})`);
} finally {
store.cancelAll();
store.closeQuietly();
if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED;
else process.env.BILI_STORAGE_UNIFIED = prev;
rmrf(dir);
Expand Down Expand Up @@ -178,7 +178,7 @@ test("full mode: dropped metadata restores from the index alone (narrow read, no
assert.ok(again, "envelope rewritten by later save");
assert.deepEqual(Object.keys(JSON.parse(fs.readFileSync(again, "utf8")).payload.metadata as object).sort(), persistedKeys);
} finally {
store.cancelAll();
store.closeQuietly();
if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED;
else process.env.BILI_STORAGE_UNIFIED = prev;
rmrf(dir);
Expand Down
Loading
Loading