diff --git a/src/fetch-util.ts b/src/fetch-util.ts index 4242d2853a..005d4e50fc 100644 --- a/src/fetch-util.ts +++ b/src/fetch-util.ts @@ -91,11 +91,19 @@ export function upstreamTimeoutMs(): number { // explicit Agent per timeout value so the transport layer matches the watchdog // instead of firing first. const directDispatchers = new Map(); +let directCaIdentity: string | undefined; function directDispatcher(timeoutMs: number): Agent { // #3776: key on the CA identity too so a configured/rotated network.caFile // rotates the cached Agent (and its connect.ca) instead of reusing a stale trust set. - const key = `${timeoutMs}\u0000${upstreamCaIdentity()}`; + const caIdentity = upstreamCaIdentity(); + if (caIdentity !== directCaIdentity) { + // Retire the old identity's pools without aborting their in-flight responses. + // Keeping every rotated identity here otherwise retains Agents and CA material forever. + closeDirectDispatchers(); + directCaIdentity = caIdentity; + } + const key = `${timeoutMs}\u0000${caIdentity}`; let agent = directDispatchers.get(key); if (!agent) { const ca = upstreamExtraCa(); @@ -105,11 +113,16 @@ function directDispatcher(timeoutMs: number): Agent { return agent; } -export function _resetFetchUtilForTest(): void { +function closeDirectDispatchers(): void { for (const agent of directDispatchers.values()) { try { void agent.close().catch(() => undefined); } catch { /* already closed */ } } directDispatchers.clear(); + directCaIdentity = undefined; +} + +export function _resetFetchUtilForTest(): void { + closeDirectDispatchers(); _resetUpstreamCaForTest(); } diff --git a/src/plugin.ts b/src/plugin.ts index 867038aef8..d271347af5 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -2080,7 +2080,12 @@ export async function handlePluginStatus(conversationId: string, res: import("no res.end(JSON.stringify({ ok: true, conversationId, phase: "pre-first-request", model: pre.model, contextLimit: pre.contextWindow ?? null, runtimeInfo: pre, panel: null, sessionId: null, sessionRevision: null, contextTokens: null, contextTokensSource: "unavailable", contextTokensAt: null, contextGeneration: null })); return; } - res.writeHead(404, { "content-type": "application/json" }); + // #6642: the 404 fallthrough must carry the SAME cors-mode headers the + // 2xx branches above do — the dsh Context tab reads this route + // cross-origin, and a bare 404 without access-control-allow-origin is + // rejected by the browser as "Failed to fetch", masking the real + // "unknown plugin conversation" error the panel is trying to surface. + res.writeHead(404, jsonHeaders()); res.end(JSON.stringify({ ok: false, error: fallbackLatest ? "no session with activity since boot — issue a model request or pass the conversation id" : "unknown plugin conversation" })); return; } diff --git a/src/server/fastpath.ts b/src/server/fastpath.ts index 0773e16517..0871202b70 100644 --- a/src/server/fastpath.ts +++ b/src/server/fastpath.ts @@ -332,7 +332,7 @@ export function evaluateWatermarkFastpath( for (let hi = 0; hi < seedHead.length; hi++) { const a = headCores.msgs[hi]; const b = seedHead[hi]; - if (a.id !== b.id || JSON.stringify(a) !== JSON.stringify(b)) return miss("head-prefix-drift", false, tailIds); + if (headCoreContentDiffers(a, b)) return miss("head-prefix-drift", false, tailIds); } } const windowRaw = openaiToCore(windowFragment as OpenAIRequestBody); @@ -407,6 +407,21 @@ function baseOfRawId(id: string): string { return m === null ? id : id.slice(0, 18); } +/** In-place head-edit check, IGNORING the `id` field. In a pinned session + * (pluginLaneServerHistory) the seed cores carry host/ledger msgids while a + * fresh openaiToCore parse mints content-hash ids, so the id namespace differs + * structurally on every turn (#7347) — the id is not a content signal here. + * Content/role/shape inequality is still caught via the full JSON minus the + * id; a remint with identical bytes is not a rewrite. */ +function headCoreContentDiffers(a: CoreMessage, b: CoreMessage): boolean { + const strip = (c: CoreMessage): Record => { + const rest: Record = { ...c }; + delete rest.id; + return rest; + }; + return JSON.stringify(strip(a)) !== JSON.stringify(strip(b)); +} + /** openaiToCore materializes user content-array images as distinct cores * carrying an `imagePart`/`rawOpenaiContent` pair — used to detect * multimodal tails from already-converted cores (legacy-side seeding). */ diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..afc5f23b2a 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -2240,10 +2240,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare clearUpstreamTimer(); if (prepared?.resetAfterSuccess) { log("warn", `[${prepared.session.id}] native compact response had no body; rebase NOT scheduled`); - } else { + } else if (req.method !== "HEAD") { // #821: a 2xx with a null body (e.g. upstream answered 204/304) is a // SILENT empty response — the client receives zero SSE chunks and // reports an "empty stream" with no trace in the log. Name it. + // HEAD is excluded: a HEAD response carries no body by definition + // (RFC 9110 §9.3.2), so a 200 HEAD with a null body is normal, not + // the #821 anomaly — health probes (e.g. HEAD /api/hello) would + // otherwise log a warn on every request (#7072). loggerLog("warn", `[${prepared?.session.id ?? "unknown"}] ← upstream ${upstream.status} returned a null body; responding empty to client`); } return; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 0e306de770..8eaade1bbc 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1,19 +1,25 @@ /** * Unified-storage sidecar (#2671). * - * Two opt-in levels via BILI_STORAGE_UNIFIED: + * Selections via BILI_STORAGE_UNIFIED / storage.unified: + * - absent or `sqlite` (the DEFAULT since #6480, "pure-sqlite"): the + * drop-in replacement. Reads go unified-first with organic import (a + * session the index has not seen is imported from its legacy file on + * first load) and legacy fallback on any miss, and the JSON envelope + * WRITER defaults off — index.db is the only persistent copy (restore + * dual-write with storage.jsonWriter=on). + * - `full`: the historical #3818 posture — the same unified-first + * machinery, but the JSON envelope writer defaults ON (dual-write; + * rollback = unset the knob). * - `1`/`true`/`shadow` (Phase 1): double-write dry-run. Every session * record the legacy SessionStore commits is ALSO ingested into * /index.db, and the reconstructed view is digest-checked * against what the legacy loader (buildSession) would serve after a * restart. Any drift is logged as an error — that is the entire point: * the unified layer must replay legacy bytes exactly, and production - * traffic becomes the migration test corpus. - * - `full` (Phase 1.5): the drop-in replacement. Reads flip to - * unified-first with organic import (a session the index has not seen - * is imported from its legacy file on first load) and legacy fallback - * on any miss. Legacy files keep being written (rollback = unset the - * env), but the unified index is the load path. + * traffic becomes the migration test corpus. The JSON envelope stays + * the sole source of truth at this level. + * - `legacy`: the one-line escape hatch — no sidecar at all. * * Failure policy: sidecar problems must NEVER break the legacy write path. * All work is try/catch'd. The sidecar NEVER gives up on index writes @@ -28,7 +34,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import { splitMediaContent } from "./identity.js"; import type { BlockView } from "../session.js"; -import { resolveStorageMode, resolveStorageWalJournalSizeLimit, effectiveJsonWriterOff, type StorageMode } from "../knobs.ts"; +import { effectiveJsonWriterOff, resolveStorageMode, resolveStorageWalJournalSizeLimit, unifiedSelectionIsSqlite, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; @@ -501,10 +507,26 @@ export class UnifiedShadow { recordSidecarInstance(shadow); const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`; const walTag = ` [wal-cap: ${walLimit}]`; + // #7408: the FULL-mode line must state the ACTUAL writer state and + // selection — the pre-#6480 wording ("legacy files keep being + // written — unset the knob to roll back") is false under the + // pure-sqlite default (no knob set, envelopes NOT written) and + // contradicted the accurate jsonWriter-off line printed by persist. + // At this point mode is full, the backend is sqlite (mysql logs its + // own line in openAsync), and encryption-blocking never reaches here + // (persist gates the open), so effectiveJsonWriterOff equals the + // jsonWriterOff persist.ts computes for save decisions. + const sqliteSelection = unifiedSelectionIsSqlite(env, fileUnified); + const selection = sqliteSelection + ? "pure-sqlite selection (storage.unified absent or =sqlite)" + : "storage.unified=full or BILI_STORAGE_UNIFIED=full"; + const writerOff = effectiveJsonWriterOff(env, fileJsonWriter, fileUnified); loggerLog( "info", mode === "full" - ? `[storage] unified FULL mode ENABLED (storage.unified=full or BILI_STORAGE_UNIFIED=full): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — unset the knob to roll back${engineTag}${walTag}` + ? writerOff + ? `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on${engineTag}${walTag}` + : `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — ${sqliteSelection ? "remove the explicit storage.jsonWriter=on to return to the pure-sqlite default" : "unset the knob to roll back"}${engineTag}${walTag}` : `[storage] unified shadow mode ENABLED (storage.unified=shadow or BILI_STORAGE_UNIFIED): every session write is double-ingested into index.db and digest-reconciled${engineTag}${walTag}`, ); return shadow; diff --git a/src/tunnel-guard.ts b/src/tunnel-guard.ts index 3c9b79924d..c1a966b391 100644 --- a/src/tunnel-guard.ts +++ b/src/tunnel-guard.ts @@ -181,6 +181,14 @@ export const workerResolveHost: WorkerProbe = (host) => if (settled) return; settled = true; clearTimeout(timer); + if (msg?.error === "ETIMEDOUT") { + // #7407: the inner deadline won — the worker's getaddrinfo may + // still be queued in its libuv pool, keeping the one-shot + // thread (and a pool slot) alive. Same termination as the + // outer deadline; other failure codes mean the lookup already + // settled, and the worker drains and exits on its own. + try { void worker?.terminate().catch(() => undefined); } catch { /* already gone */ } + } if (msg && msg.ok === true && Array.isArray(msg.addrs)) resolve(msg.addrs); else reject(Object.assign(new Error(`worker dns lookup failed (${String(msg?.error ?? "unknown")})`), { code: typeof msg?.error === "string" && msg.error ? msg.error : "EWORKERLOOKUP" })); }); diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index c2f548a79f..40438d01f2 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -64,6 +64,7 @@ type UpstreamConnectionStatus = { }; const dispatcherCache = new Map(); +let dispatcherCaIdentity: string | undefined; let lastConnection: UpstreamConnectionStatus = {}; let windowsProxyCache: { at: number; value: WindowsSystemProxy } | undefined; @@ -401,7 +402,14 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number const t = timeoutMs ?? upstreamTimeoutMs(); const ka = proxyKeepAliveMaxMs(); // #3776: include the CA identity in the key so a configured/rotated network.caFile rotates the cached agent. - const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${upstreamCaIdentity()}`; + const caIdentity = upstreamCaIdentity(); + if (caIdentity !== dispatcherCaIdentity) { + // Gracefully drain the old trust identity's proxy and target pools; future + // requests use only the current identity without retaining every past Agent. + closeProxyDispatchers(); + dispatcherCaIdentity = caIdentity; + } + const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${caIdentity}`; let agent = dispatcherCache.get(key); if (!agent) { const withTimeouts = (options: object): object => { @@ -432,11 +440,16 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number return agent; } -export function resetProxyCache(): void { +function closeProxyDispatchers(): void { for (const agent of dispatcherCache.values()) { try { void agent.close().catch(() => undefined); } catch { } } dispatcherCache.clear(); + dispatcherCaIdentity = undefined; +} + +export function resetProxyCache(): void { + closeProxyDispatchers(); lastConnection = {}; } diff --git a/tests/fastpath.test.ts b/tests/fastpath.test.ts index 1ff6ca5348..3ab650dafe 100644 --- a/tests/fastpath.test.ts +++ b/tests/fastpath.test.ts @@ -9,6 +9,7 @@ import http from "node:http"; import { openaiToCore, coreToOpenai } from "acp-kernel/wire"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, stampOnlyWatermarkTurn, clearWatermarkSeed, watermarkDebugState, PF_COUNT_HEADER, PF_SEQ_HEADER, PF_TAIL_IDS_HEADER, PF_SLOT_UNKNOWN } from "../src/server/fastpath.ts"; import { stripRawStage } from "../src/marker/engine.ts"; +import { applyMsgIdPin } from "../src/msgid-pin.ts"; import { INCREMENTAL_TURN_MARKER_HEADER } from "../src/incremental-lane.ts"; import { _setStoreForTest, _resetStoreForTest, type SessionStore } from "../src/persist.ts"; import type { Session } from "../src/session.ts"; @@ -65,6 +66,33 @@ const runTurn = ( return { out, headers }; }; +/** Drive one turn the way prepareOpenai does for a PINNED session + * (pluginLaneServerHistory on, #4491): on a miss the seed is re-keyed to the + * host msgids via applyMsgIdPin (the id namespace the stash then carries), on + * a hit the pin is skipped (the seed's prefix was already pinned when it was + * materialized). This is the 2x2 traffic the watermark fastpath exists to + * accelerate — the seed cores carry host msgids while a fresh openaiToCore + * parse mints content-hash ids, so the id namespaces differ structurally + * (#7347). */ +const runPinnedTurn = ( + session: Session, + reqHeaders: Record, + bodyText: string, + hostIds: string[], +): { out: ReturnType; headers: Record } => { + const model = JSON.parse(bodyText).model as string; + const out = evaluateWatermarkFastpath(bodyText, headersOf(reqHeaders), session, { pluginMode: true, titleGen: false, model }, log); + const oracle = openaiToCore(JSON.parse(bodyText)); + const stripCounts = stripRawStage(JSON.parse(bodyText), "openai"); + const headers = finalizeWatermarkTurn(session, out, { + model, + legacyCores: out.hit !== null ? out.hit.view : applyMsgIdPin(oracle.msgs, hostIds), + rawSystemText: out.hit !== null ? out.hit.systemText : oracle.systemText, + turnStripCounts: stripCounts, + }); + return { out, headers }; +}; + test("gate off / title-gen / incremental-turn requests are untouched", () => { _resetStoreForTest(); makeStubStore(); @@ -478,3 +506,42 @@ test("clearWatermarkSeed forces the next turn back through verification", () => const { out } = runTurn(session, { [PF_COUNT_HEADER]: "3" }, b2); assert.equal(out.decision, "unseeded"); }); + +test("pinned seed (host msgids) + tail-only growth HITS; in-place head edit still drifts (#7347)", () => { + makeStubStore(); + const session = freshSession("pf-pin"); + // turn 1: seed. pluginLaneServerHistory pins the seed cores to host msgids + // (applyMsgIdPin), so the stash carries a DIFFERENT id namespace than the + // content-hash ids a fresh openaiToCore parse mints. + const b1 = bodyOf([SYS("s"), USR("a"), USR("b"), ASST("c"), USR("d")]); + const t1 = runPinnedTurn(session, {}, b1, ["host-a", "host-b", "host-c", "host-d"]); + assert.equal(t1.out.decision, "no-stamp", LOGS.join("\n")); + assert.equal(t1.out.measuredT, 5); + // turn 2: same prefix + one new tail element. The fresh head cores carry + // content-hash ids while the seed carries host msgids — the head check must + // ignore the id namespace and HIT on the unchanged content. Pre-fix this is + // a permanent head-prefix-drift miss (the field symptom: every turn of a + // 2x2 pinned session fell back to the full legacy parse). + const b2 = bodyOf([SYS("s"), USR("a"), USR("b"), ASST("c"), USR("d"), USR("e")]); + const t2 = runPinnedTurn(session, { [PF_COUNT_HEADER]: "5" }, b2, ["host-a", "host-b", "host-c", "host-d", "host-e"]); + assert.equal(t2.out.decision, "hit", LOGS.join("\n")); + assert.equal(t2.out.creditedMiss, false); + // the view must still be the legacy full-parse oracle on CONTENT. The + // prefix ids are host msgids by design in a pinned session, so assert + // text/role parity (not id parity — that is the namespace difference). + if (!t2.out.hit) throw new Error("expected hit"); + const oracle2 = openaiToCore(JSON.parse(b2)); + assert.equal(t2.out.hit.view.length, oracle2.msgs.length); + for (let i = 0; i < oracle2.msgs.length; i++) { + assert.equal(t2.out.hit.view[i]!.text, oracle2.msgs[i]!.text, `text @${i}`); + assert.equal(t2.out.hit.view[i]!.role, oracle2.msgs[i]!.role, `role @${i}`); + } + // turn 3: rewrite a HEAD element IN PLACE (prefix counts unchanged, system + // identical, stamp honest) + append one tail — the per-core content + // comparison (id excluded) must still catch the rewrite, so the fix does + // not mask a genuine in-place edit. + const b3 = bodyOf([SYS("s"), USR("a"), USR("b-EDITED"), ASST("c"), USR("d"), USR("e"), USR("f")]); + const t3 = runPinnedTurn(session, { [PF_COUNT_HEADER]: "6" }, b3, ["host-a", "host-b2", "host-c", "host-d", "host-e", "host-f"]); + assert.equal(t3.out.decision, "head-prefix-drift", LOGS.join("\n")); + assert.equal(t3.out.creditedMiss, false); +}); diff --git a/tests/issue6642-plugin-status-404-cors.test.ts b/tests/issue6642-plugin-status-404-cors.test.ts new file mode 100644 index 0000000000..2ace4a9e7a --- /dev/null +++ b/tests/issue6642-plugin-status-404-cors.test.ts @@ -0,0 +1,66 @@ +// #6642: the plugin-status 404 fallthrough must carry the SAME cors-mode +// headers the 2xx branches do. The dsh Context tab polls this route +// cross-origin (its own loopback port / dsh-app:// scheme); a bare 404 +// without access-control-allow-origin is rejected by the browser as +// "Failed to fetch", masking the real "unknown plugin conversation" error +// the panel is trying to surface — which is exactly how the DSH 0.2.2-rc.1 +// takeover-dead case hid behind a network error for 10 days. +// +// Both directions are pinned (gate discipline #3617): the header is present +// when a trusted origin is admitted, and ABSENT when the request is same-origin +// (no trustedOrigin) so we never leak an allow-origin to an untrusted caller. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { createCore, defaultConfig } from "acp-kernel"; +import { handlePluginStatus, _resetPluginStateForTest } from "../src/plugin.ts"; + +const ORIGIN = "http://127.0.0.1:51162"; + +function mockRes(): { res: http.ServerResponse; status: number; headers: Record; body: string } { + const out = { res: undefined as unknown as http.ServerResponse, status: 0, headers: {} as Record, body: "" }; + const res = { + writeHead(code: number, headers?: Record) { + out.status = code; + if (headers) out.headers = headers; + return res; + }, + end(chunk?: unknown) { + if (typeof chunk === "string") out.body = chunk; + return res; + }, + } as unknown as http.ServerResponse; + out.res = res; + return out; +} + +const deps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; + +test("plugin-status 404 carries access-control-allow-origin when a trusted origin is admitted (#6642)", async () => { + _resetPluginStateForTest(); + + // fallback=latest: no session with post-boot activity → the "since boot" 404. + const r1 = mockRes(); + await handlePluginStatus("never-seen", r1.res, deps, true, ORIGIN); + assert.equal(r1.status, 404, "no post-boot activity → 404"); + assert.ok(r1.body.includes("since boot"), "the fallback=latest 404 names the no-activity cause"); + assert.equal(r1.headers["access-control-allow-origin"], ORIGIN, "the 404 echoes the admitted origin so the cross-origin browser fetch can read it"); + assert.equal(r1.headers["content-type"], "application/json"); + + // bare (no fallback): the "unknown plugin conversation" 404. + const r2 = mockRes(); + await handlePluginStatus("never-seen", r2.res, deps, false, ORIGIN); + assert.equal(r2.status, 404); + assert.ok(r2.body.includes("unknown plugin conversation"), "the bare 404 names the unknown conversation"); + assert.equal(r2.headers["access-control-allow-origin"], ORIGIN, "the bare 404 also echoes the admitted origin"); +}); + +test("plugin-status 404 omits access-control-allow-origin for a same-origin (untrusted) caller (#6642 negative)", async () => { + _resetPluginStateForTest(); + const r = mockRes(); + await handlePluginStatus("never-seen", r.res, deps, false); + assert.equal(r.status, 404); + assert.ok(r.body.includes("unknown plugin conversation")); + assert.equal(r.headers["access-control-allow-origin"], undefined, "no trusted origin → no allow-origin header (never echoed to an untrusted caller)"); + assert.equal(r.headers["content-type"], "application/json", "content-type is still set"); +}); diff --git a/tests/issue7072-head-null-body.test.ts b/tests/issue7072-head-null-body.test.ts new file mode 100644 index 0000000000..d20d409258 --- /dev/null +++ b/tests/issue7072-head-null-body.test.ts @@ -0,0 +1,115 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { setLogCapture } from "../src/logger.ts"; + +/** #7072: a 2xx with a null body logs the #821 warn. For HEAD that warn is + * NOISE — a HEAD response has no body by definition (RFC 9110 §9.3.2), so a + * 200 HEAD with a null body is normal, not the #821 anomaly. Health probes + * (e.g. HEAD /api/hello) hit this on every request and flooded bili.log with + * `[unknown] ← upstream 200 returned a null body` lines. The relay must skip + * the warn for HEAD while still warning for a genuine 2xx null body (204/304 + * on a body-bearing method). */ + +function close(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +interface Captured { + level: string; + msg: string; +} + +async function startProxyWith( + upstream: http.Server, +): Promise<{ proxy: http.Server; proxyPort: number; upstreamPort: number; captured: Captured[] }> { + const captured: Captured[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const opts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { + [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 400_000 } } }, + }, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + return { proxy, proxyPort, upstreamPort, captured }; +} + +test("HEAD 200 with a null body does NOT log the #821 null-body warn (#7072)", async () => { + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + // HEAD: 200 with no body (correct per RFC 9110 §9.3.2). + res.writeHead(200, { "content-type": "text/plain" }); + res.end(); + }); + }); + const { proxy, proxyPort, upstreamPort, captured } = await startProxyWith(upstream); + try { + const resp = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/api/hello`, { method: "HEAD" }); + assert.equal(resp.status, 200); + assert.equal(await resp.text(), ""); + const nullBodyWarns = captured.filter((c) => c.level === "warn" && c.msg.includes("returned a null body")); + assert.equal(nullBodyWarns.length, 0, `expected no null-body warn for HEAD, got: ${nullBodyWarns.map((c) => c.msg).join(" | ")}`); + } finally { + setLogCapture(null); + await close(proxy); + await close(upstream); + } +}); + +test("GET 204 with a null body STILL logs the #821 null-body warn (#821 preserved)", async () => { + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + // GET: 204 → null body → the #821 anomaly the warn exists to name. + res.writeHead(204); + res.end(); + }); + }); + const { proxy, proxyPort, upstreamPort, captured } = await startProxyWith(upstream); + try { + const resp = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/api/hello`, { method: "GET" }); + assert.equal(resp.status, 204); + assert.equal(await resp.text(), ""); + const warn = captured.find((c) => c.level === "warn" && c.msg.includes("returned a null body")); + assert.ok(warn, `expected null-body warn for GET 204, got: ${captured.map((c) => c.msg).join(" | ")}`); + assert.match(warn.msg, /upstream 204 returned a null body/); + } finally { + setLogCapture(null); + await close(proxy); + await close(upstream); + } +}); diff --git a/tests/issue7408-storage-boot-log.test.ts b/tests/issue7408-storage-boot-log.test.ts new file mode 100644 index 0000000000..33d944778d --- /dev/null +++ b/tests/issue7408-storage-boot-log.test.ts @@ -0,0 +1,127 @@ +/** + * #7408 §3b — the full-mode storage boot line must state the ACTUAL + * selection and writer state. The pre-#6480 wording ("legacy files keep + * being written — unset the knob to roll back") survived the pure-sqlite + * default flip: under the default (no knob set, envelopes NOT written) it + * claimed a knob that was never set, a dual-write that does not happen, and + * a rollback that does not exist — contradicting the accurate jsonWriter-off + * line persist.ts prints in the same boot and steering field diagnosis into + * chasing a non-existent dual-write. Pinned for every reachable posture: + * default pure-sqlite, explicit full, and the two explicit jsonWriter + * overrides (env tier + file tier through the new fileJsonWriter parameter). + */ +import assert from "node:assert"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, before, test } from "node:test"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { _resetStorageRuntimeStatusForTest } from "../src/storage/runtime-status.ts"; + +let root: string; +let engineAvailable = true; + +function freshDir(prefix: string): string { + return mkdtempSync(path.join(root, prefix)); +} + +function withEnv(vars: Record, fn: () => void): void { + const prev: Record = {}; + for (const [k, v] of Object.entries(vars)) { + prev[k] = process.env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + fn(); + } finally { + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +before(() => { + root = mkdtempSync(path.join(tmpdir(), "bili-7408-")); + const probe = freshDir("probe-"); + try { + const s = UnifiedStore.open(path.join(probe, "index.db")); + s.close(); + } catch { + engineAvailable = false; + } +}); + +after(() => { + rmSync(root, { recursive: true, force: true }); + setLogCapture(null); +}); + +/** Open the sidecar under the given knob posture and return the exact + * full-mode boot line (both state homes kept out of the way so no test + * instance touches a real state dir). */ +function bootLine(env: NodeJS.ProcessEnv, fileUnified?: string, fileJsonWriter?: string): string { + const state = freshDir("state-"); + const sessions = freshDir("sessions-"); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + try { + withEnv({ XDG_STATE_HOME: state }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, env, (r) => r as LegacySessionLike, fileUnified, undefined, undefined, fileJsonWriter); + assert.ok(shadow, "sidecar must open"); + shadow.close(); + }); + const line = logs.find((m) => m.includes("unified FULL mode ENABLED")); + assert.ok(line, `FULL-mode boot line expected, got: ${logs.join(" | ")}`); + return line; + } finally { + setLogCapture(null); + } +} + +test("#7408 §3b: default (pure-sqlite) — the boot line says index.db is the only persistent copy and envelopes are NOT written", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({}); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on"), line); + assert.ok(!line.includes("legacy files keep being written"), "the pre-#6480 dual-write claim must be gone under the default"); + assert.ok(!line.includes("unset the knob"), "no knob is set under the default — the rollback hint must not claim otherwise"); +}); + +test("#7408 §3b: explicit full — the historical dual-write wording stays (knob set, envelopes written)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_UNIFIED: "full" }); + assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line); + assert.ok(line.includes("legacy files keep being written — unset the knob to roll back"), line); +}); + +test("#7408 §3b: pure-sqlite + explicit jsonWriter=on (env) — envelopes ARE written; the line names the explicit on, not a unified knob", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_JSON_WRITER: "on" }); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line); +}); + +test("#7408 §3b: explicit full + explicit jsonWriter=off — the writer-off wording wins over the selection wording", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_UNIFIED: "full", BILI_STORAGE_JSON_WRITER: "off" }); + assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line); + assert.ok(line.includes("JSON envelopes are not written"), line); + assert.ok(!line.includes("legacy files keep being written"), line); +}); + +test("#7408 §3b: file tier — storage.unified=sqlite + storage.jsonWriter=on reaches the line through the fileJsonWriter parameter (the persist.ts plumbing)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({}, "sqlite", "on"); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line); +}); diff --git a/tests/tunnel-guard-worker-lifecycle.test.ts b/tests/tunnel-guard-worker-lifecycle.test.ts new file mode 100644 index 0000000000..1e0e9682be --- /dev/null +++ b/tests/tunnel-guard-worker-lifecycle.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { syncBuiltinESMExports } from "node:module"; +import test, { type TestContext } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; +import workerThreads, { Worker, type WorkerOptions } from "node:worker_threads"; +import { workerResolveHost } from "../src/tunnel-guard.ts"; + +// #7407: the one-shot DNS fallback worker is throwaway, but on the inner +// deadline the parent used to clear its outer timer and reject WITHOUT +// stopping the worker — the expired probe (and its still-queued +// getaddrinfo) kept a whole thread plus a libuv pool slot alive until +// unrelated native work finished. These tests run the REAL Worker +// implementation (only the lookup boundary is stalled via a prelude the +// harness prepends to the worker source) and assert the worker thread is +// actually gone once the probe has settled. +// +// Gate discipline (#3617): the stalled-lookup case is the regression +// (fails on the unfixed base — the expired worker stays alive); the outer +// deadline and the successful-lookup cases are the "does not over-trigger" +// controls (termination still works there, success exits normally). + +const OBSERVE_EXIT_GRACE_MS = 1000; + +function observeWorker(t: TestContext, prelude = "") { + const workers: Array<{ worker: Worker; exited: Promise }> = []; + class ObservedWorker extends Worker { + constructor(source: string | URL, options?: WorkerOptions) { + assert.ok(typeof source === "string" && options?.eval, "the probe must use the real eval-worker plumbing"); + super(`${prelude}\n${source}`, { ...options, execArgv: [] }); + workers.push({ worker: this, exited: new Promise((resolve) => this.once("exit", resolve)) }); + } + } + const replacement = t.mock.method(workerThreads, "Worker", ObservedWorker); + syncBuiltinESMExports(); + t.after(async () => { + replacement.mock.restore(); + syncBuiltinESMExports(); + await Promise.all(workers.map(({ worker }) => worker.terminate())); + }); + return async () => { + assert.equal(workers.length, 1, "one-shot probe — exactly one worker"); + const { worker, exited } = workers[0]!; + const outcome = await Promise.race([ + exited.then((code) => ({ code })), + delay(OBSERVE_EXIT_GRACE_MS, undefined, { ref: false }).then(() => ({ code: null })), + ]); + assert.notEqual(outcome.code, null, "the settled one-shot probe must not retain a live worker"); + assert.equal(worker.threadId, -1); + return outcome.code; + }; +} + +test("#7407 regression: the inner lookup deadline stops its one-shot worker", async (t) => { + const exited = observeWorker(t, ` + require("node:dns").promises.lookup = () => { + setInterval(() => {}, 10000); // anchor: an unfixed worker provably outlives the observation window + return new Promise(() => {}); // stalled getaddrinfo boundary + }; + `); + await assert.rejects(workerResolveHost("stalled-lookup.invalid"), (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as NodeJS.ErrnoException).code, "ETIMEDOUT"); + assert.match(error.message, /worker dns lookup failed/); + return true; + }); + assert.equal(await exited(), 1, "termination, not natural exit"); +}); + +test("#7407 control: the outer deadline still stops a worker that cannot send its inner verdict", async (t) => { + const exited = observeWorker(t, "Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);"); + await assert.rejects(workerResolveHost("localhost"), { code: "ETIMEDOUT", message: "ETIMEDOUT" }); + assert.equal(await exited(), 1); +}); + +test("#7407 control: a successful native lookup exits normally and keeps its addresses", async (t) => { + const exited = observeWorker(t); + const addresses = await workerResolveHost("localhost"); + assert.ok(addresses.includes("127.0.0.1") || addresses.includes("::1")); + assert.equal(await exited(), 0, "success path must keep its natural exit"); +}); diff --git a/tests/upstream-ca-dispatcher-retirement.test.ts b/tests/upstream-ca-dispatcher-retirement.test.ts new file mode 100644 index 0000000000..b733970881 --- /dev/null +++ b/tests/upstream-ca-dispatcher-retirement.test.ts @@ -0,0 +1,192 @@ +import test, { type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import net from "node:net"; +import { once } from "node:events"; +import type { Duplex } from "node:stream"; +import { rootCertificates } from "node:tls"; +import { Agent, ProxyAgent } from "undici"; +import { _resetFetchUtilForTest, fetchWithTimeout, type FetchOptions } from "../src/fetch-util.js"; +import { proxyDispatcher, resetProxyCache } from "../src/upstream-proxy.js"; + +// Undici's runtime DispatcherBase exposes these lifecycle getters, but ProxyAgent's +// declaration omits them. Read and validate the real getters rather than fabricating state. +function poolState(agent: Agent | ProxyAgent): { closed: boolean; destroyed: boolean } { + const closed: unknown = Reflect.get(agent, "closed"); + const destroyed: unknown = Reflect.get(agent, "destroyed"); + assert.ok(typeof closed === "boolean" && typeof destroyed === "boolean"); + return { closed, destroyed }; +} + +function caFiles(t: TestContext): { a: string; b: string } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ca-pools-")); + const a = path.join(dir, "a.pem"); + const b = path.join(dir, "b.pem"); + fs.writeFileSync(a, rootCertificates[0]); + fs.writeFileSync(b, rootCertificates[0]); + const previous = process.env.BILI_CA_FILE; + _resetFetchUtilForTest(); + resetProxyCache(); + t.after(() => { + _resetFetchUtilForTest(); + resetProxyCache(); + if (previous === undefined) delete process.env.BILI_CA_FILE; + else process.env.BILI_CA_FILE = previous; + fs.rmSync(dir, { recursive: true, force: true }); + }); + return { a, b }; +} + +function captureDirectAgent(t: TestContext): (timeoutMs?: number) => Promise { + let captured: object | undefined; + t.mock.method(globalThis, "fetch", async (_url: unknown, opts: FetchOptions) => { + captured = opts.dispatcher; + return new Response("ok"); + }); + return async (timeoutMs = 10_000) => { + const result = await fetchWithTimeout("http://127.0.0.1/fixture", {}, timeoutMs); + try { + await result.response.text(); + assert.ok(captured instanceof Agent); + return captured; + } finally { + result.clearTimer(); + } + }; +} + +test("CA rotation retires every old direct pool while unchanged trust reuses each timeout pool", async (t) => { + const files = caFiles(t); + const dispatcher = captureDirectAgent(t); + process.env.BILI_CA_FILE = files.a; + const a = await dispatcher(); + const longer = await dispatcher(20_000); + assert.equal(await dispatcher(), a); + assert.notEqual(longer, a); + assert.equal(poolState(a).closed, false); + assert.equal(poolState(longer).closed, false); + + process.env.BILI_CA_FILE = files.b; + const b = await dispatcher(); + assert.notEqual(b, a); + assert.equal(poolState(a).closed, true, "retired direct Agent must release its origin pools and CA material"); + assert.equal(poolState(longer).closed, true, "all timeout variants of the retired identity must close"); + assert.equal(poolState(b).closed, false); + assert.equal(await dispatcher(), b); + + process.env.BILI_CA_FILE = files.a; + assert.notEqual(await dispatcher(), a, "returning to an old identity must not resurrect its retired Agent"); + assert.equal(poolState(b).closed, true); + delete process.env.BILI_CA_FILE; + const defaults = await dispatcher(); + assert.equal(poolState(defaults).closed, false); + assert.equal(await dispatcher(), defaults, "the unset trust identity is cached normally"); +}); + +test("CA rotation retires every old proxy pool without evicting other routes for unchanged trust", (t) => { + const files = caFiles(t); + process.env.BILI_CA_FILE = files.a; + const a = proxyDispatcher("http://127.0.0.1:1", 10_000) as ProxyAgent; + const otherRoute = proxyDispatcher("http://127.0.0.1:2", 20_000) as ProxyAgent; + assert.equal(proxyDispatcher("http://127.0.0.1:1", 10_000), a); + assert.equal(poolState(a).closed, false); + assert.equal(poolState(otherRoute).closed, false); + + process.env.BILI_CA_FILE = files.b; + const b = proxyDispatcher("http://127.0.0.1:1", 10_000) as ProxyAgent; + assert.notEqual(b, a); + assert.equal(poolState(a).closed, true, "retired ProxyAgent must release both proxy and target pools"); + assert.equal(poolState(otherRoute).closed, true, "all proxy routes using the retired identity must close"); + assert.equal(poolState(b).closed, false); + assert.equal(proxyDispatcher("http://127.0.0.1:1", 10_000), b); + + process.env.BILI_CA_FILE = files.a; + assert.notEqual(proxyDispatcher("http://127.0.0.1:1", 10_000), a); + assert.equal(poolState(b).closed, true); + delete process.env.BILI_CA_FILE; + const defaults = proxyDispatcher("http://127.0.0.1:1", 10_000) as ProxyAgent; + assert.equal(poolState(defaults).closed, false); + assert.equal(proxyDispatcher("http://127.0.0.1:1", 10_000), defaults); +}); + +for (const proxied of [false, true]) { + test(`retiring ${proxied ? "proxy" : "direct"} CA pools lets an in-flight response finish`, async (t) => { + const files = caFiles(t); + const sockets = new Set(); + const timers: Array<() => void> = []; + let streaming: http.ServerResponse | undefined; + const upstream = http.createServer((req, res) => { + if (req.url === "/stream") { + streaming = res; + res.writeHead(200, { "content-type": "text/plain" }); + res.write("start\n"); + } else { + res.end("new trust\n"); + } + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const port = (upstream.address() as net.AddressInfo).port; + const proxy = http.createServer(); + proxy.on("connect", (_req, client, head) => { + const target = net.connect(port, "127.0.0.1", () => { + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length) target.write(head); + client.pipe(target); + target.pipe(client); + }); + for (const socket of [client, target]) { + sockets.add(socket); + socket.on("error", () => socket.destroy()); + socket.on("close", () => sockets.delete(socket)); + } + }); + proxy.listen(0, "127.0.0.1"); + await once(proxy, "listening"); + const proxyUrl = `http://127.0.0.1:${(proxy.address() as net.AddressInfo).port}`; + t.after(async () => { + for (const clear of timers) clear(); + _resetFetchUtilForTest(); + resetProxyCache(); + for (const socket of sockets) socket.destroy(); + upstream.closeAllConnections(); + proxy.closeAllConnections(); + await Promise.all([upstream, proxy].map((server) => new Promise((resolve) => server.close(() => resolve())))); + }); + const nativeFetch = globalThis.fetch; + const used: Array = []; + t.mock.method(globalThis, "fetch", (url: string, opts: FetchOptions) => { + assert.ok(opts.dispatcher instanceof Agent || opts.dispatcher instanceof ProxyAgent); + used.push(opts.dispatcher); + return nativeFetch(url, opts as RequestInit); + }); + async function get(route: string) { + const result = await fetchWithTimeout(`http://127.0.0.1:${port}${route}`, { + dispatcher: proxied ? proxyDispatcher(proxyUrl, 10_000) : undefined, + }, 10_000); + timers.push(result.clearTimer); + return result; + } + + process.env.BILI_CA_FILE = files.a; + const first = await get("/stream"); + const reader = first.response.body!.getReader(); + assert.equal(new TextDecoder().decode((await reader.read()).value), "start\n"); + process.env.BILI_CA_FILE = files.b; + const second = await get("/fast"); + assert.equal(await second.response.text(), "new trust\n"); + assert.notEqual(used[0], used[1]); + assert.equal(poolState(used[0]).closed, true, "the old dispatcher stops admitting new requests"); + assert.equal(poolState(used[0]).destroyed, false, "retirement waits for the existing body instead of destroying it"); + assert.ok(streaming); + streaming.end("finish\n"); + assert.equal(new TextDecoder().decode((await reader.read()).value), "finish\n"); + assert.equal((await reader.read()).done, true); + await used[0].close(); + assert.equal(poolState(used[0]).destroyed, true, "the retired pools are released after their last response"); + assert.equal(poolState(used[1]).closed, false, "the current identity remains available"); + }); +}