Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/fetch-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,19 @@ export function upstreamTimeoutMs(): number {
// explicit Agent per timeout value so the transport layer matches the watchdog
// instead of firing first.
const directDispatchers = new Map<string, Agent>();
let directCaIdentity: string | undefined;

function directDispatcher(timeoutMs: number): Agent {
// #3776: key on the CA identity too so a configured/rotated network.caFile
// rotates the cached Agent (and its connect.ca) instead of reusing a stale trust set.
const key = `${timeoutMs}\u0000${upstreamCaIdentity()}`;
const caIdentity = upstreamCaIdentity();
if (caIdentity !== directCaIdentity) {
// Retire the old identity's pools without aborting their in-flight responses.
// Keeping every rotated identity here otherwise retains Agents and CA material forever.
closeDirectDispatchers();
directCaIdentity = caIdentity;
}
const key = `${timeoutMs}\u0000${caIdentity}`;
let agent = directDispatchers.get(key);
if (!agent) {
const ca = upstreamExtraCa();
Expand All @@ -105,11 +113,16 @@ function directDispatcher(timeoutMs: number): Agent {
return agent;
}

export function _resetFetchUtilForTest(): void {
function closeDirectDispatchers(): void {
for (const agent of directDispatchers.values()) {
try { void agent.close().catch(() => undefined); } catch { /* already closed */ }
}
directDispatchers.clear();
directCaIdentity = undefined;
}

export function _resetFetchUtilForTest(): void {
closeDirectDispatchers();
_resetUpstreamCaForTest();
}

Expand Down
7 changes: 6 additions & 1 deletion src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2080,7 +2080,12 @@ export async function handlePluginStatus(conversationId: string, res: import("no
res.end(JSON.stringify({ ok: true, conversationId, phase: "pre-first-request", model: pre.model, contextLimit: pre.contextWindow ?? null, runtimeInfo: pre, panel: null, sessionId: null, sessionRevision: null, contextTokens: null, contextTokensSource: "unavailable", contextTokensAt: null, contextGeneration: null }));
return;
}
res.writeHead(404, { "content-type": "application/json" });
// #6642: the 404 fallthrough must carry the SAME cors-mode headers the
// 2xx branches above do — the dsh Context tab reads this route
// cross-origin, and a bare 404 without access-control-allow-origin is
// rejected by the browser as "Failed to fetch", masking the real
// "unknown plugin conversation" error the panel is trying to surface.
res.writeHead(404, jsonHeaders());
res.end(JSON.stringify({ ok: false, error: fallbackLatest ? "no session with activity since boot — issue a model request or pass the conversation id" : "unknown plugin conversation" }));
return;
}
Expand Down
17 changes: 16 additions & 1 deletion src/server/fastpath.ts
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ export function evaluateWatermarkFastpath(
for (let hi = 0; hi < seedHead.length; hi++) {
const a = headCores.msgs[hi];
const b = seedHead[hi];
if (a.id !== b.id || JSON.stringify(a) !== JSON.stringify(b)) return miss("head-prefix-drift", false, tailIds);
if (headCoreContentDiffers(a, b)) return miss("head-prefix-drift", false, tailIds);
}
}
const windowRaw = openaiToCore(windowFragment as OpenAIRequestBody);
Expand Down Expand Up @@ -407,6 +407,21 @@ function baseOfRawId(id: string): string {
return m === null ? id : id.slice(0, 18);
}

/** In-place head-edit check, IGNORING the `id` field. In a pinned session
* (pluginLaneServerHistory) the seed cores carry host/ledger msgids while a
* fresh openaiToCore parse mints content-hash ids, so the id namespace differs
* structurally on every turn (#7347) — the id is not a content signal here.
* Content/role/shape inequality is still caught via the full JSON minus the
* id; a remint with identical bytes is not a rewrite. */
function headCoreContentDiffers(a: CoreMessage, b: CoreMessage): boolean {
const strip = (c: CoreMessage): Record<string, unknown> => {
const rest: Record<string, unknown> = { ...c };
delete rest.id;
return rest;
};
return JSON.stringify(strip(a)) !== JSON.stringify(strip(b));
}

/** openaiToCore materializes user content-array images as distinct cores
* carrying an `imagePart`/`rawOpenaiContent` pair — used to detect
* multimodal tails from already-converted cores (legacy-side seeding). */
Expand Down
6 changes: 5 additions & 1 deletion src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2240,10 +2240,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
clearUpstreamTimer();
if (prepared?.resetAfterSuccess) {
log("warn", `[${prepared.session.id}] native compact response had no body; rebase NOT scheduled`);
} else {
} else if (req.method !== "HEAD") {
// #821: a 2xx with a null body (e.g. upstream answered 204/304) is a
// SILENT empty response — the client receives zero SSE chunks and
// reports an "empty stream" with no trace in the log. Name it.
// HEAD is excluded: a HEAD response carries no body by definition
// (RFC 9110 §9.3.2), so a 200 HEAD with a null body is normal, not
// the #821 anomaly — health probes (e.g. HEAD /api/hello) would
// otherwise log a warn on every request (#7072).
loggerLog("warn", `[${prepared?.session.id ?? "unknown"}] ← upstream ${upstream.status} returned a null body; responding empty to client`);
}
return;
Expand Down
40 changes: 31 additions & 9 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
/**
* Unified-storage sidecar (#2671).
*
* Two opt-in levels via BILI_STORAGE_UNIFIED:
* Selections via BILI_STORAGE_UNIFIED / storage.unified:
* - absent or `sqlite` (the DEFAULT since #6480, "pure-sqlite"): the
* drop-in replacement. Reads go unified-first with organic import (a
* session the index has not seen is imported from its legacy file on
* first load) and legacy fallback on any miss, and the JSON envelope
* WRITER defaults off — index.db is the only persistent copy (restore
* dual-write with storage.jsonWriter=on).
* - `full`: the historical #3818 posture — the same unified-first
* machinery, but the JSON envelope writer defaults ON (dual-write;
* rollback = unset the knob).
* - `1`/`true`/`shadow` (Phase 1): double-write dry-run. Every session
* record the legacy SessionStore commits is ALSO ingested into
* <sessionsDir>/index.db, and the reconstructed view is digest-checked
* against what the legacy loader (buildSession) would serve after a
* restart. Any drift is logged as an error — that is the entire point:
* the unified layer must replay legacy bytes exactly, and production
* traffic becomes the migration test corpus.
* - `full` (Phase 1.5): the drop-in replacement. Reads flip to
* unified-first with organic import (a session the index has not seen
* is imported from its legacy file on first load) and legacy fallback
* on any miss. Legacy files keep being written (rollback = unset the
* env), but the unified index is the load path.
* traffic becomes the migration test corpus. The JSON envelope stays
* the sole source of truth at this level.
* - `legacy`: the one-line escape hatch — no sidecar at all.
*
* Failure policy: sidecar problems must NEVER break the legacy write path.
* All work is try/catch'd. The sidecar NEVER gives up on index writes
Expand All @@ -28,7 +34,7 @@ import { log as loggerLog } from "../logger.js";
import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts";
import { splitMediaContent } from "./identity.js";
import type { BlockView } from "../session.js";
import { resolveStorageMode, resolveStorageWalJournalSizeLimit, effectiveJsonWriterOff, type StorageMode } from "../knobs.ts";
import { effectiveJsonWriterOff, resolveStorageMode, resolveStorageWalJournalSizeLimit, unifiedSelectionIsSqlite, type StorageMode } from "../knobs.ts";
import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts";
import path from "node:path";
import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts";
Expand Down Expand Up @@ -501,10 +507,26 @@ export class UnifiedShadow {
recordSidecarInstance(shadow);
const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`;
const walTag = ` [wal-cap: ${walLimit}]`;
// #7408: the FULL-mode line must state the ACTUAL writer state and
// selection — the pre-#6480 wording ("legacy files keep being
// written — unset the knob to roll back") is false under the
// pure-sqlite default (no knob set, envelopes NOT written) and
// contradicted the accurate jsonWriter-off line printed by persist.
// At this point mode is full, the backend is sqlite (mysql logs its
// own line in openAsync), and encryption-blocking never reaches here
// (persist gates the open), so effectiveJsonWriterOff equals the
// jsonWriterOff persist.ts computes for save decisions.
const sqliteSelection = unifiedSelectionIsSqlite(env, fileUnified);
const selection = sqliteSelection
? "pure-sqlite selection (storage.unified absent or =sqlite)"
: "storage.unified=full or BILI_STORAGE_UNIFIED=full";
const writerOff = effectiveJsonWriterOff(env, fileJsonWriter, fileUnified);
loggerLog(
"info",
mode === "full"
? `[storage] unified FULL mode ENABLED (storage.unified=full or BILI_STORAGE_UNIFIED=full): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — unset the knob to roll back${engineTag}${walTag}`
? writerOff
? `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on${engineTag}${walTag}`
: `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — ${sqliteSelection ? "remove the explicit storage.jsonWriter=on to return to the pure-sqlite default" : "unset the knob to roll back"}${engineTag}${walTag}`
: `[storage] unified shadow mode ENABLED (storage.unified=shadow or BILI_STORAGE_UNIFIED): every session write is double-ingested into index.db and digest-reconciled${engineTag}${walTag}`,
);
return shadow;
Expand Down
8 changes: 8 additions & 0 deletions src/tunnel-guard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,14 @@ export const workerResolveHost: WorkerProbe = (host) =>
if (settled) return;
settled = true;
clearTimeout(timer);
if (msg?.error === "ETIMEDOUT") {
// #7407: the inner deadline won — the worker's getaddrinfo may
// still be queued in its libuv pool, keeping the one-shot
// thread (and a pool slot) alive. Same termination as the
// outer deadline; other failure codes mean the lookup already
// settled, and the worker drains and exits on its own.
try { void worker?.terminate().catch(() => undefined); } catch { /* already gone */ }
}
if (msg && msg.ok === true && Array.isArray(msg.addrs)) resolve(msg.addrs);
else reject(Object.assign(new Error(`worker dns lookup failed (${String(msg?.error ?? "unknown")})`), { code: typeof msg?.error === "string" && msg.error ? msg.error : "EWORKERLOOKUP" }));
});
Expand Down
17 changes: 15 additions & 2 deletions src/upstream-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ type UpstreamConnectionStatus = {
};

const dispatcherCache = new Map<string, ProxyAgent>();
let dispatcherCaIdentity: string | undefined;
let lastConnection: UpstreamConnectionStatus = {};
let windowsProxyCache: { at: number; value: WindowsSystemProxy } | undefined;

Expand Down Expand Up @@ -401,7 +402,14 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number
const t = timeoutMs ?? upstreamTimeoutMs();
const ka = proxyKeepAliveMaxMs();
// #3776: include the CA identity in the key so a configured/rotated network.caFile rotates the cached agent.
const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${upstreamCaIdentity()}`;
const caIdentity = upstreamCaIdentity();
if (caIdentity !== dispatcherCaIdentity) {
// Gracefully drain the old trust identity's proxy and target pools; future
// requests use only the current identity without retaining every past Agent.
closeProxyDispatchers();
dispatcherCaIdentity = caIdentity;
}
const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${caIdentity}`;
let agent = dispatcherCache.get(key);
if (!agent) {
const withTimeouts = (options: object): object => {
Expand Down Expand Up @@ -432,11 +440,16 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number
return agent;
}

export function resetProxyCache(): void {
function closeProxyDispatchers(): void {
for (const agent of dispatcherCache.values()) {
try { void agent.close().catch(() => undefined); } catch { }
}
dispatcherCache.clear();
dispatcherCaIdentity = undefined;
}

export function resetProxyCache(): void {
closeProxyDispatchers();
lastConnection = {};
}

Expand Down
67 changes: 67 additions & 0 deletions tests/fastpath.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import http from "node:http";
import { openaiToCore, coreToOpenai } from "acp-kernel/wire";
import { evaluateWatermarkFastpath, finalizeWatermarkTurn, stampOnlyWatermarkTurn, clearWatermarkSeed, watermarkDebugState, PF_COUNT_HEADER, PF_SEQ_HEADER, PF_TAIL_IDS_HEADER, PF_SLOT_UNKNOWN } from "../src/server/fastpath.ts";
import { stripRawStage } from "../src/marker/engine.ts";
import { applyMsgIdPin } from "../src/msgid-pin.ts";
import { INCREMENTAL_TURN_MARKER_HEADER } from "../src/incremental-lane.ts";
import { _setStoreForTest, _resetStoreForTest, type SessionStore } from "../src/persist.ts";
import type { Session } from "../src/session.ts";
Expand Down Expand Up @@ -65,6 +66,33 @@ const runTurn = (
return { out, headers };
};

/** Drive one turn the way prepareOpenai does for a PINNED session
* (pluginLaneServerHistory on, #4491): on a miss the seed is re-keyed to the
* host msgids via applyMsgIdPin (the id namespace the stash then carries), on
* a hit the pin is skipped (the seed's prefix was already pinned when it was
* materialized). This is the 2x2 traffic the watermark fastpath exists to
* accelerate — the seed cores carry host msgids while a fresh openaiToCore
* parse mints content-hash ids, so the id namespaces differ structurally
* (#7347). */
const runPinnedTurn = (
session: Session,
reqHeaders: Record<string, string>,
bodyText: string,
hostIds: string[],
): { out: ReturnType<typeof evaluateWatermarkFastpath>; headers: Record<string, string> } => {
const model = JSON.parse(bodyText).model as string;
const out = evaluateWatermarkFastpath(bodyText, headersOf(reqHeaders), session, { pluginMode: true, titleGen: false, model }, log);
const oracle = openaiToCore(JSON.parse(bodyText));
const stripCounts = stripRawStage(JSON.parse(bodyText), "openai");
const headers = finalizeWatermarkTurn(session, out, {
model,
legacyCores: out.hit !== null ? out.hit.view : applyMsgIdPin(oracle.msgs, hostIds),
rawSystemText: out.hit !== null ? out.hit.systemText : oracle.systemText,
turnStripCounts: stripCounts,
});
return { out, headers };
};

test("gate off / title-gen / incremental-turn requests are untouched", () => {
_resetStoreForTest();
makeStubStore();
Expand Down Expand Up @@ -478,3 +506,42 @@ test("clearWatermarkSeed forces the next turn back through verification", () =>
const { out } = runTurn(session, { [PF_COUNT_HEADER]: "3" }, b2);
assert.equal(out.decision, "unseeded");
});

test("pinned seed (host msgids) + tail-only growth HITS; in-place head edit still drifts (#7347)", () => {
makeStubStore();
const session = freshSession("pf-pin");
// turn 1: seed. pluginLaneServerHistory pins the seed cores to host msgids
// (applyMsgIdPin), so the stash carries a DIFFERENT id namespace than the
// content-hash ids a fresh openaiToCore parse mints.
const b1 = bodyOf([SYS("s"), USR("a"), USR("b"), ASST("c"), USR("d")]);
const t1 = runPinnedTurn(session, {}, b1, ["host-a", "host-b", "host-c", "host-d"]);
assert.equal(t1.out.decision, "no-stamp", LOGS.join("\n"));
assert.equal(t1.out.measuredT, 5);
// turn 2: same prefix + one new tail element. The fresh head cores carry
// content-hash ids while the seed carries host msgids — the head check must
// ignore the id namespace and HIT on the unchanged content. Pre-fix this is
// a permanent head-prefix-drift miss (the field symptom: every turn of a
// 2x2 pinned session fell back to the full legacy parse).
const b2 = bodyOf([SYS("s"), USR("a"), USR("b"), ASST("c"), USR("d"), USR("e")]);
const t2 = runPinnedTurn(session, { [PF_COUNT_HEADER]: "5" }, b2, ["host-a", "host-b", "host-c", "host-d", "host-e"]);
assert.equal(t2.out.decision, "hit", LOGS.join("\n"));
assert.equal(t2.out.creditedMiss, false);
// the view must still be the legacy full-parse oracle on CONTENT. The
// prefix ids are host msgids by design in a pinned session, so assert
// text/role parity (not id parity — that is the namespace difference).
if (!t2.out.hit) throw new Error("expected hit");
const oracle2 = openaiToCore(JSON.parse(b2));
assert.equal(t2.out.hit.view.length, oracle2.msgs.length);
for (let i = 0; i < oracle2.msgs.length; i++) {
assert.equal(t2.out.hit.view[i]!.text, oracle2.msgs[i]!.text, `text @${i}`);
assert.equal(t2.out.hit.view[i]!.role, oracle2.msgs[i]!.role, `role @${i}`);
}
// turn 3: rewrite a HEAD element IN PLACE (prefix counts unchanged, system
// identical, stamp honest) + append one tail — the per-core content
// comparison (id excluded) must still catch the rewrite, so the fix does
// not mask a genuine in-place edit.
const b3 = bodyOf([SYS("s"), USR("a"), USR("b-EDITED"), ASST("c"), USR("d"), USR("e"), USR("f")]);
const t3 = runPinnedTurn(session, { [PF_COUNT_HEADER]: "6" }, b3, ["host-a", "host-b2", "host-c", "host-d", "host-e", "host-f"]);
assert.equal(t3.out.decision, "head-prefix-drift", LOGS.join("\n"));
assert.equal(t3.out.creditedMiss, false);
});
Loading
Loading