From bb93a8e1ce218e5dd9e2bfbd15c4f7362f1eae3e Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 14 Nov 2026 13:18:48 +0800 Subject: [PATCH 1/6] fix: plugin-status 404 carries CORS headers so the DSH panel sees the real error (#6642) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 404 fallthrough in handlePluginStatus built headers by hand with only content-type, while the 2xx branches above it use jsonHeaders() (which adds access-control-allow-origin when a trusted origin is admitted). The dsh Context tab polls this route cross-origin, so a bare 404 is rejected by the browser as 'Failed to fetch', masking the real 'unknown plugin conversation' error the panel is trying to surface. Reuse jsonHeaders() in the 404 branch. Both 404 paths (fallback=latest 'since boot' and bare 'unknown plugin conversation') go through this one writeHead, so one fix covers both. Negative case pinned: no trusted origin → no allow-origin header (never echoed to an untrusted caller). (cherry picked from commit b154797f1e7a1014013fa93c7360305262b72fd2) --- src/plugin.ts | 7 +- .../issue6642-plugin-status-404-cors.test.ts | 66 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/issue6642-plugin-status-404-cors.test.ts diff --git a/src/plugin.ts b/src/plugin.ts index 867038aef8..d271347af5 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -2080,7 +2080,12 @@ export async function handlePluginStatus(conversationId: string, res: import("no res.end(JSON.stringify({ ok: true, conversationId, phase: "pre-first-request", model: pre.model, contextLimit: pre.contextWindow ?? null, runtimeInfo: pre, panel: null, sessionId: null, sessionRevision: null, contextTokens: null, contextTokensSource: "unavailable", contextTokensAt: null, contextGeneration: null })); return; } - res.writeHead(404, { "content-type": "application/json" }); + // #6642: the 404 fallthrough must carry the SAME cors-mode headers the + // 2xx branches above do — the dsh Context tab reads this route + // cross-origin, and a bare 404 without access-control-allow-origin is + // rejected by the browser as "Failed to fetch", masking the real + // "unknown plugin conversation" error the panel is trying to surface. + res.writeHead(404, jsonHeaders()); res.end(JSON.stringify({ ok: false, error: fallbackLatest ? "no session with activity since boot — issue a model request or pass the conversation id" : "unknown plugin conversation" })); return; } diff --git a/tests/issue6642-plugin-status-404-cors.test.ts b/tests/issue6642-plugin-status-404-cors.test.ts new file mode 100644 index 0000000000..2ace4a9e7a --- /dev/null +++ b/tests/issue6642-plugin-status-404-cors.test.ts @@ -0,0 +1,66 @@ +// #6642: the plugin-status 404 fallthrough must carry the SAME cors-mode +// headers the 2xx branches do. The dsh Context tab polls this route +// cross-origin (its own loopback port / dsh-app:// scheme); a bare 404 +// without access-control-allow-origin is rejected by the browser as +// "Failed to fetch", masking the real "unknown plugin conversation" error +// the panel is trying to surface — which is exactly how the DSH 0.2.2-rc.1 +// takeover-dead case hid behind a network error for 10 days. +// +// Both directions are pinned (gate discipline #3617): the header is present +// when a trusted origin is admitted, and ABSENT when the request is same-origin +// (no trustedOrigin) so we never leak an allow-origin to an untrusted caller. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { createCore, defaultConfig } from "acp-kernel"; +import { handlePluginStatus, _resetPluginStateForTest } from "../src/plugin.ts"; + +const ORIGIN = "http://127.0.0.1:51162"; + +function mockRes(): { res: http.ServerResponse; status: number; headers: Record; body: string } { + const out = { res: undefined as unknown as http.ServerResponse, status: 0, headers: {} as Record, body: "" }; + const res = { + writeHead(code: number, headers?: Record) { + out.status = code; + if (headers) out.headers = headers; + return res; + }, + end(chunk?: unknown) { + if (typeof chunk === "string") out.body = chunk; + return res; + }, + } as unknown as http.ServerResponse; + out.res = res; + return out; +} + +const deps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; + +test("plugin-status 404 carries access-control-allow-origin when a trusted origin is admitted (#6642)", async () => { + _resetPluginStateForTest(); + + // fallback=latest: no session with post-boot activity → the "since boot" 404. + const r1 = mockRes(); + await handlePluginStatus("never-seen", r1.res, deps, true, ORIGIN); + assert.equal(r1.status, 404, "no post-boot activity → 404"); + assert.ok(r1.body.includes("since boot"), "the fallback=latest 404 names the no-activity cause"); + assert.equal(r1.headers["access-control-allow-origin"], ORIGIN, "the 404 echoes the admitted origin so the cross-origin browser fetch can read it"); + assert.equal(r1.headers["content-type"], "application/json"); + + // bare (no fallback): the "unknown plugin conversation" 404. + const r2 = mockRes(); + await handlePluginStatus("never-seen", r2.res, deps, false, ORIGIN); + assert.equal(r2.status, 404); + assert.ok(r2.body.includes("unknown plugin conversation"), "the bare 404 names the unknown conversation"); + assert.equal(r2.headers["access-control-allow-origin"], ORIGIN, "the bare 404 also echoes the admitted origin"); +}); + +test("plugin-status 404 omits access-control-allow-origin for a same-origin (untrusted) caller (#6642 negative)", async () => { + _resetPluginStateForTest(); + const r = mockRes(); + await handlePluginStatus("never-seen", r.res, deps, false); + assert.equal(r.status, 404); + assert.ok(r.body.includes("unknown plugin conversation")); + assert.equal(r.headers["access-control-allow-origin"], undefined, "no trusted origin → no allow-origin header (never echoed to an untrusted caller)"); + assert.equal(r.headers["content-type"], "application/json", "content-type is still set"); +}); From 993b96f2332227bead0447b3ec576a9a74b60cf6 Mon Sep 17 00:00:00 2001 From: Afloat Date: Fri, 20 Nov 2026 14:18:22 +0800 Subject: [PATCH 2/6] fix: retire upstream dispatchers when CA identity changes (cherry picked from commit c11a7ef37063f2ad358cc78a3c86b1cf748453d9) --- src/fetch-util.ts | 17 +- src/upstream-proxy.ts | 17 +- .../upstream-ca-dispatcher-retirement.test.ts | 192 ++++++++++++++++++ 3 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 tests/upstream-ca-dispatcher-retirement.test.ts diff --git a/src/fetch-util.ts b/src/fetch-util.ts index 4242d2853a..005d4e50fc 100644 --- a/src/fetch-util.ts +++ b/src/fetch-util.ts @@ -91,11 +91,19 @@ export function upstreamTimeoutMs(): number { // explicit Agent per timeout value so the transport layer matches the watchdog // instead of firing first. const directDispatchers = new Map(); +let directCaIdentity: string | undefined; function directDispatcher(timeoutMs: number): Agent { // #3776: key on the CA identity too so a configured/rotated network.caFile // rotates the cached Agent (and its connect.ca) instead of reusing a stale trust set. - const key = `${timeoutMs}\u0000${upstreamCaIdentity()}`; + const caIdentity = upstreamCaIdentity(); + if (caIdentity !== directCaIdentity) { + // Retire the old identity's pools without aborting their in-flight responses. + // Keeping every rotated identity here otherwise retains Agents and CA material forever. + closeDirectDispatchers(); + directCaIdentity = caIdentity; + } + const key = `${timeoutMs}\u0000${caIdentity}`; let agent = directDispatchers.get(key); if (!agent) { const ca = upstreamExtraCa(); @@ -105,11 +113,16 @@ function directDispatcher(timeoutMs: number): Agent { return agent; } -export function _resetFetchUtilForTest(): void { +function closeDirectDispatchers(): void { for (const agent of directDispatchers.values()) { try { void agent.close().catch(() => undefined); } catch { /* already closed */ } } directDispatchers.clear(); + directCaIdentity = undefined; +} + +export function _resetFetchUtilForTest(): void { + closeDirectDispatchers(); _resetUpstreamCaForTest(); } diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index c2f548a79f..40438d01f2 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -64,6 +64,7 @@ type UpstreamConnectionStatus = { }; const dispatcherCache = new Map(); +let dispatcherCaIdentity: string | undefined; let lastConnection: UpstreamConnectionStatus = {}; let windowsProxyCache: { at: number; value: WindowsSystemProxy } | undefined; @@ -401,7 +402,14 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number const t = timeoutMs ?? upstreamTimeoutMs(); const ka = proxyKeepAliveMaxMs(); // #3776: include the CA identity in the key so a configured/rotated network.caFile rotates the cached agent. - const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${upstreamCaIdentity()}`; + const caIdentity = upstreamCaIdentity(); + if (caIdentity !== dispatcherCaIdentity) { + // Gracefully drain the old trust identity's proxy and target pools; future + // requests use only the current identity without retaining every past Agent. + closeProxyDispatchers(); + dispatcherCaIdentity = caIdentity; + } + const key = `${proxyUrl}\u0000${t}\u0000${ka}\u0000${caIdentity}`; let agent = dispatcherCache.get(key); if (!agent) { const withTimeouts = (options: object): object => { @@ -432,11 +440,16 @@ export function proxyDispatcher(proxyUrl: string | undefined, timeoutMs?: number return agent; } -export function resetProxyCache(): void { +function closeProxyDispatchers(): void { for (const agent of dispatcherCache.values()) { try { void agent.close().catch(() => undefined); } catch { } } dispatcherCache.clear(); + dispatcherCaIdentity = undefined; +} + +export function resetProxyCache(): void { + closeProxyDispatchers(); lastConnection = {}; } diff --git a/tests/upstream-ca-dispatcher-retirement.test.ts b/tests/upstream-ca-dispatcher-retirement.test.ts new file mode 100644 index 0000000000..b733970881 --- /dev/null +++ b/tests/upstream-ca-dispatcher-retirement.test.ts @@ -0,0 +1,192 @@ +import test, { type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import net from "node:net"; +import { once } from "node:events"; +import type { Duplex } from "node:stream"; +import { rootCertificates } from "node:tls"; +import { Agent, ProxyAgent } from "undici"; +import { _resetFetchUtilForTest, fetchWithTimeout, type FetchOptions } from "../src/fetch-util.js"; +import { proxyDispatcher, resetProxyCache } from "../src/upstream-proxy.js"; + +// Undici's runtime DispatcherBase exposes these lifecycle getters, but ProxyAgent's +// declaration omits them. Read and validate the real getters rather than fabricating state. +function poolState(agent: Agent | ProxyAgent): { closed: boolean; destroyed: boolean } { + const closed: unknown = Reflect.get(agent, "closed"); + const destroyed: unknown = Reflect.get(agent, "destroyed"); + assert.ok(typeof closed === "boolean" && typeof destroyed === "boolean"); + return { closed, destroyed }; +} + +function caFiles(t: TestContext): { a: string; b: string } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ca-pools-")); + const a = path.join(dir, "a.pem"); + const b = path.join(dir, "b.pem"); + fs.writeFileSync(a, rootCertificates[0]); + fs.writeFileSync(b, rootCertificates[0]); + const previous = process.env.BILI_CA_FILE; + _resetFetchUtilForTest(); + resetProxyCache(); + t.after(() => { + _resetFetchUtilForTest(); + resetProxyCache(); + if (previous === undefined) delete process.env.BILI_CA_FILE; + else process.env.BILI_CA_FILE = previous; + fs.rmSync(dir, { recursive: true, force: true }); + }); + return { a, b }; +} + +function captureDirectAgent(t: TestContext): (timeoutMs?: number) => Promise { + let captured: object | undefined; + t.mock.method(globalThis, "fetch", async (_url: unknown, opts: FetchOptions) => { + captured = opts.dispatcher; + return new Response("ok"); + }); + return async (timeoutMs = 10_000) => { + const result = await fetchWithTimeout("http://127.0.0.1/fixture", {}, timeoutMs); + try { + await result.response.text(); + assert.ok(captured instanceof Agent); + return captured; + } finally { + result.clearTimer(); + } + }; +} + +test("CA rotation retires every old direct pool while unchanged trust reuses each timeout pool", async (t) => { + const files = caFiles(t); + const dispatcher = captureDirectAgent(t); + process.env.BILI_CA_FILE = files.a; + const a = await dispatcher(); + const longer = await dispatcher(20_000); + assert.equal(await dispatcher(), a); + assert.notEqual(longer, a); + assert.equal(poolState(a).closed, false); + assert.equal(poolState(longer).closed, false); + + process.env.BILI_CA_FILE = files.b; + const b = await dispatcher(); + assert.notEqual(b, a); + assert.equal(poolState(a).closed, true, "retired direct Agent must release its origin pools and CA material"); + assert.equal(poolState(longer).closed, true, "all timeout variants of the retired identity must close"); + assert.equal(poolState(b).closed, false); + assert.equal(await dispatcher(), b); + + process.env.BILI_CA_FILE = files.a; + assert.notEqual(await dispatcher(), a, "returning to an old identity must not resurrect its retired Agent"); + assert.equal(poolState(b).closed, true); + delete process.env.BILI_CA_FILE; + const defaults = await dispatcher(); + assert.equal(poolState(defaults).closed, false); + assert.equal(await dispatcher(), defaults, "the unset trust identity is cached normally"); +}); + +test("CA rotation retires every old proxy pool without evicting other routes for unchanged trust", (t) => { + const files = caFiles(t); + process.env.BILI_CA_FILE = files.a; + const a = proxyDispatcher("http://127.0.0.1:1", 10_000) as ProxyAgent; + const otherRoute = proxyDispatcher("http://127.0.0.1:2", 20_000) as ProxyAgent; + assert.equal(proxyDispatcher("http://127.0.0.1:1", 10_000), a); + assert.equal(poolState(a).closed, false); + assert.equal(poolState(otherRoute).closed, false); + + process.env.BILI_CA_FILE = files.b; + const b = proxyDispatcher("http://127.0.0.1:1", 10_000) as ProxyAgent; + assert.notEqual(b, a); + assert.equal(poolState(a).closed, true, "retired ProxyAgent must release both proxy and target pools"); + assert.equal(poolState(otherRoute).closed, true, "all proxy routes using the retired identity must close"); + assert.equal(poolState(b).closed, false); + assert.equal(proxyDispatcher("http://127.0.0.1:1", 10_000), b); + + process.env.BILI_CA_FILE = files.a; + assert.notEqual(proxyDispatcher("http://127.0.0.1:1", 10_000), a); + assert.equal(poolState(b).closed, true); + delete process.env.BILI_CA_FILE; + const defaults = proxyDispatcher("http://127.0.0.1:1", 10_000) as ProxyAgent; + assert.equal(poolState(defaults).closed, false); + assert.equal(proxyDispatcher("http://127.0.0.1:1", 10_000), defaults); +}); + +for (const proxied of [false, true]) { + test(`retiring ${proxied ? "proxy" : "direct"} CA pools lets an in-flight response finish`, async (t) => { + const files = caFiles(t); + const sockets = new Set(); + const timers: Array<() => void> = []; + let streaming: http.ServerResponse | undefined; + const upstream = http.createServer((req, res) => { + if (req.url === "/stream") { + streaming = res; + res.writeHead(200, { "content-type": "text/plain" }); + res.write("start\n"); + } else { + res.end("new trust\n"); + } + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const port = (upstream.address() as net.AddressInfo).port; + const proxy = http.createServer(); + proxy.on("connect", (_req, client, head) => { + const target = net.connect(port, "127.0.0.1", () => { + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length) target.write(head); + client.pipe(target); + target.pipe(client); + }); + for (const socket of [client, target]) { + sockets.add(socket); + socket.on("error", () => socket.destroy()); + socket.on("close", () => sockets.delete(socket)); + } + }); + proxy.listen(0, "127.0.0.1"); + await once(proxy, "listening"); + const proxyUrl = `http://127.0.0.1:${(proxy.address() as net.AddressInfo).port}`; + t.after(async () => { + for (const clear of timers) clear(); + _resetFetchUtilForTest(); + resetProxyCache(); + for (const socket of sockets) socket.destroy(); + upstream.closeAllConnections(); + proxy.closeAllConnections(); + await Promise.all([upstream, proxy].map((server) => new Promise((resolve) => server.close(() => resolve())))); + }); + const nativeFetch = globalThis.fetch; + const used: Array = []; + t.mock.method(globalThis, "fetch", (url: string, opts: FetchOptions) => { + assert.ok(opts.dispatcher instanceof Agent || opts.dispatcher instanceof ProxyAgent); + used.push(opts.dispatcher); + return nativeFetch(url, opts as RequestInit); + }); + async function get(route: string) { + const result = await fetchWithTimeout(`http://127.0.0.1:${port}${route}`, { + dispatcher: proxied ? proxyDispatcher(proxyUrl, 10_000) : undefined, + }, 10_000); + timers.push(result.clearTimer); + return result; + } + + process.env.BILI_CA_FILE = files.a; + const first = await get("/stream"); + const reader = first.response.body!.getReader(); + assert.equal(new TextDecoder().decode((await reader.read()).value), "start\n"); + process.env.BILI_CA_FILE = files.b; + const second = await get("/fast"); + assert.equal(await second.response.text(), "new trust\n"); + assert.notEqual(used[0], used[1]); + assert.equal(poolState(used[0]).closed, true, "the old dispatcher stops admitting new requests"); + assert.equal(poolState(used[0]).destroyed, false, "retirement waits for the existing body instead of destroying it"); + assert.ok(streaming); + streaming.end("finish\n"); + assert.equal(new TextDecoder().decode((await reader.read()).value), "finish\n"); + assert.equal((await reader.read()).done, true); + await used[0].close(); + assert.equal(poolState(used[0]).destroyed, true, "the retired pools are released after their last response"); + assert.equal(poolState(used[1]).closed, false, "the current identity remains available"); + }); +} From eda38907daa9978bc2f4b72fafb4db84c671da66 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 20 Nov 2026 21:13:16 +0800 Subject: [PATCH 3/6] fix(storage): the FULL-mode boot line states the actual selection and writer state (#7408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-#6480 wording ("storage.unified=full or BILI_STORAGE_UNIFIED=full: ... legacy files keep being written — unset the knob to roll back") survived the pure-sqlite default flip. Under the default posture — no knob set, JSON envelopes NOT written — all three clauses were false, contradicting the accurate jsonWriter-off line persist.ts prints in the same boot and steering field diagnosis into chasing a non-existent dual-write (#7408 §3b). The line now branches on the raw selection (unifiedSelectionIsSqlite) and the effective writer state (effectiveJsonWriterOff, which at that point equals persist.ts's jsonWriterOff: mode is full, backend is sqlite, and an encryption-blocked open never reaches the line): - writer off → "index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on" - writer on + pure-sqlite selection → names the explicit storage.jsonWriter=on as the reason envelopes are written - writer on + explicit full → the historical dual-write wording, unchanged maybeOpen/openAsync take the file-tier jsonWriter value as a new optional parameter (persist.ts passes fileStorageJsonWriter() at both call sites); the module doc header is updated to the four current postures. Regression test: tests/issue7408-storage-boot-log.test.ts pins the exact boot line for all five reachable postures. Log wording only — no wire, config, or storage-semantics change. (cherry picked from commit 3e82f42a5c39a8645bff04f57b70a7cdb5908928) --- src/storage/shadow.ts | 40 +++++-- tests/issue7408-storage-boot-log.test.ts | 127 +++++++++++++++++++++++ 2 files changed, 158 insertions(+), 9 deletions(-) create mode 100644 tests/issue7408-storage-boot-log.test.ts diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 0e306de770..8eaade1bbc 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1,19 +1,25 @@ /** * Unified-storage sidecar (#2671). * - * Two opt-in levels via BILI_STORAGE_UNIFIED: + * Selections via BILI_STORAGE_UNIFIED / storage.unified: + * - absent or `sqlite` (the DEFAULT since #6480, "pure-sqlite"): the + * drop-in replacement. Reads go unified-first with organic import (a + * session the index has not seen is imported from its legacy file on + * first load) and legacy fallback on any miss, and the JSON envelope + * WRITER defaults off — index.db is the only persistent copy (restore + * dual-write with storage.jsonWriter=on). + * - `full`: the historical #3818 posture — the same unified-first + * machinery, but the JSON envelope writer defaults ON (dual-write; + * rollback = unset the knob). * - `1`/`true`/`shadow` (Phase 1): double-write dry-run. Every session * record the legacy SessionStore commits is ALSO ingested into * /index.db, and the reconstructed view is digest-checked * against what the legacy loader (buildSession) would serve after a * restart. Any drift is logged as an error — that is the entire point: * the unified layer must replay legacy bytes exactly, and production - * traffic becomes the migration test corpus. - * - `full` (Phase 1.5): the drop-in replacement. Reads flip to - * unified-first with organic import (a session the index has not seen - * is imported from its legacy file on first load) and legacy fallback - * on any miss. Legacy files keep being written (rollback = unset the - * env), but the unified index is the load path. + * traffic becomes the migration test corpus. The JSON envelope stays + * the sole source of truth at this level. + * - `legacy`: the one-line escape hatch — no sidecar at all. * * Failure policy: sidecar problems must NEVER break the legacy write path. * All work is try/catch'd. The sidecar NEVER gives up on index writes @@ -28,7 +34,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import { splitMediaContent } from "./identity.js"; import type { BlockView } from "../session.js"; -import { resolveStorageMode, resolveStorageWalJournalSizeLimit, effectiveJsonWriterOff, type StorageMode } from "../knobs.ts"; +import { effectiveJsonWriterOff, resolveStorageMode, resolveStorageWalJournalSizeLimit, unifiedSelectionIsSqlite, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; @@ -501,10 +507,26 @@ export class UnifiedShadow { recordSidecarInstance(shadow); const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`; const walTag = ` [wal-cap: ${walLimit}]`; + // #7408: the FULL-mode line must state the ACTUAL writer state and + // selection — the pre-#6480 wording ("legacy files keep being + // written — unset the knob to roll back") is false under the + // pure-sqlite default (no knob set, envelopes NOT written) and + // contradicted the accurate jsonWriter-off line printed by persist. + // At this point mode is full, the backend is sqlite (mysql logs its + // own line in openAsync), and encryption-blocking never reaches here + // (persist gates the open), so effectiveJsonWriterOff equals the + // jsonWriterOff persist.ts computes for save decisions. + const sqliteSelection = unifiedSelectionIsSqlite(env, fileUnified); + const selection = sqliteSelection + ? "pure-sqlite selection (storage.unified absent or =sqlite)" + : "storage.unified=full or BILI_STORAGE_UNIFIED=full"; + const writerOff = effectiveJsonWriterOff(env, fileJsonWriter, fileUnified); loggerLog( "info", mode === "full" - ? `[storage] unified FULL mode ENABLED (storage.unified=full or BILI_STORAGE_UNIFIED=full): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — unset the knob to roll back${engineTag}${walTag}` + ? writerOff + ? `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on${engineTag}${walTag}` + : `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — ${sqliteSelection ? "remove the explicit storage.jsonWriter=on to return to the pure-sqlite default" : "unset the knob to roll back"}${engineTag}${walTag}` : `[storage] unified shadow mode ENABLED (storage.unified=shadow or BILI_STORAGE_UNIFIED): every session write is double-ingested into index.db and digest-reconciled${engineTag}${walTag}`, ); return shadow; diff --git a/tests/issue7408-storage-boot-log.test.ts b/tests/issue7408-storage-boot-log.test.ts new file mode 100644 index 0000000000..33d944778d --- /dev/null +++ b/tests/issue7408-storage-boot-log.test.ts @@ -0,0 +1,127 @@ +/** + * #7408 §3b — the full-mode storage boot line must state the ACTUAL + * selection and writer state. The pre-#6480 wording ("legacy files keep + * being written — unset the knob to roll back") survived the pure-sqlite + * default flip: under the default (no knob set, envelopes NOT written) it + * claimed a knob that was never set, a dual-write that does not happen, and + * a rollback that does not exist — contradicting the accurate jsonWriter-off + * line persist.ts prints in the same boot and steering field diagnosis into + * chasing a non-existent dual-write. Pinned for every reachable posture: + * default pure-sqlite, explicit full, and the two explicit jsonWriter + * overrides (env tier + file tier through the new fileJsonWriter parameter). + */ +import assert from "node:assert"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, before, test } from "node:test"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { _resetStorageRuntimeStatusForTest } from "../src/storage/runtime-status.ts"; + +let root: string; +let engineAvailable = true; + +function freshDir(prefix: string): string { + return mkdtempSync(path.join(root, prefix)); +} + +function withEnv(vars: Record, fn: () => void): void { + const prev: Record = {}; + for (const [k, v] of Object.entries(vars)) { + prev[k] = process.env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + fn(); + } finally { + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +before(() => { + root = mkdtempSync(path.join(tmpdir(), "bili-7408-")); + const probe = freshDir("probe-"); + try { + const s = UnifiedStore.open(path.join(probe, "index.db")); + s.close(); + } catch { + engineAvailable = false; + } +}); + +after(() => { + rmSync(root, { recursive: true, force: true }); + setLogCapture(null); +}); + +/** Open the sidecar under the given knob posture and return the exact + * full-mode boot line (both state homes kept out of the way so no test + * instance touches a real state dir). */ +function bootLine(env: NodeJS.ProcessEnv, fileUnified?: string, fileJsonWriter?: string): string { + const state = freshDir("state-"); + const sessions = freshDir("sessions-"); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + try { + withEnv({ XDG_STATE_HOME: state }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, env, (r) => r as LegacySessionLike, fileUnified, undefined, undefined, fileJsonWriter); + assert.ok(shadow, "sidecar must open"); + shadow.close(); + }); + const line = logs.find((m) => m.includes("unified FULL mode ENABLED")); + assert.ok(line, `FULL-mode boot line expected, got: ${logs.join(" | ")}`); + return line; + } finally { + setLogCapture(null); + } +} + +test("#7408 §3b: default (pure-sqlite) — the boot line says index.db is the only persistent copy and envelopes are NOT written", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({}); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on"), line); + assert.ok(!line.includes("legacy files keep being written"), "the pre-#6480 dual-write claim must be gone under the default"); + assert.ok(!line.includes("unset the knob"), "no knob is set under the default — the rollback hint must not claim otherwise"); +}); + +test("#7408 §3b: explicit full — the historical dual-write wording stays (knob set, envelopes written)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_UNIFIED: "full" }); + assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line); + assert.ok(line.includes("legacy files keep being written — unset the knob to roll back"), line); +}); + +test("#7408 §3b: pure-sqlite + explicit jsonWriter=on (env) — envelopes ARE written; the line names the explicit on, not a unified knob", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_JSON_WRITER: "on" }); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line); +}); + +test("#7408 §3b: explicit full + explicit jsonWriter=off — the writer-off wording wins over the selection wording", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_UNIFIED: "full", BILI_STORAGE_JSON_WRITER: "off" }); + assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line); + assert.ok(line.includes("JSON envelopes are not written"), line); + assert.ok(!line.includes("legacy files keep being written"), line); +}); + +test("#7408 §3b: file tier — storage.unified=sqlite + storage.jsonWriter=on reaches the line through the fileJsonWriter parameter (the persist.ts plumbing)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({}, "sqlite", "on"); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line); +}); From 4038a248bb0ae582121b523f992b1441aefb6c2f Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 21 Nov 2026 06:51:45 +0800 Subject: [PATCH 4/6] fix(relay): skip the #821 null-body warn for HEAD responses (#7072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A HEAD response has no body by definition (RFC 9110 §9.3.2), so a 200 HEAD with a null body is normal, not the #821 anomaly. Health probes (e.g. HEAD /api/hello) hit the relay's 2xx null-body branch on every request and flooded bili.log with `[unknown] ← upstream 200 returned a null body` lines — the exact noise in the #7072 report. The relay now skips the warn for HEAD while still warning for a genuine 2xx null body (204/304 on a body-bearing method). No behavior change: the response is still relayed empty for HEAD (the client gets the 200 with no body, as before); only the spurious warn is suppressed. (cherry picked from commit 83c0e0444c43ad4d78e724e0d8a0cffb65ec213d) --- src/server/relay.ts | 6 +- tests/issue7072-head-null-body.test.ts | 115 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 tests/issue7072-head-null-body.test.ts diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..afc5f23b2a 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -2240,10 +2240,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare clearUpstreamTimer(); if (prepared?.resetAfterSuccess) { log("warn", `[${prepared.session.id}] native compact response had no body; rebase NOT scheduled`); - } else { + } else if (req.method !== "HEAD") { // #821: a 2xx with a null body (e.g. upstream answered 204/304) is a // SILENT empty response — the client receives zero SSE chunks and // reports an "empty stream" with no trace in the log. Name it. + // HEAD is excluded: a HEAD response carries no body by definition + // (RFC 9110 §9.3.2), so a 200 HEAD with a null body is normal, not + // the #821 anomaly — health probes (e.g. HEAD /api/hello) would + // otherwise log a warn on every request (#7072). loggerLog("warn", `[${prepared?.session.id ?? "unknown"}] ← upstream ${upstream.status} returned a null body; responding empty to client`); } return; diff --git a/tests/issue7072-head-null-body.test.ts b/tests/issue7072-head-null-body.test.ts new file mode 100644 index 0000000000..d20d409258 --- /dev/null +++ b/tests/issue7072-head-null-body.test.ts @@ -0,0 +1,115 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { setLogCapture } from "../src/logger.ts"; + +/** #7072: a 2xx with a null body logs the #821 warn. For HEAD that warn is + * NOISE — a HEAD response has no body by definition (RFC 9110 §9.3.2), so a + * 200 HEAD with a null body is normal, not the #821 anomaly. Health probes + * (e.g. HEAD /api/hello) hit this on every request and flooded bili.log with + * `[unknown] ← upstream 200 returned a null body` lines. The relay must skip + * the warn for HEAD while still warning for a genuine 2xx null body (204/304 + * on a body-bearing method). */ + +function close(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +interface Captured { + level: string; + msg: string; +} + +async function startProxyWith( + upstream: http.Server, +): Promise<{ proxy: http.Server; proxyPort: number; upstreamPort: number; captured: Captured[] }> { + const captured: Captured[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const opts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { + [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 400_000 } } }, + }, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + return { proxy, proxyPort, upstreamPort, captured }; +} + +test("HEAD 200 with a null body does NOT log the #821 null-body warn (#7072)", async () => { + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + // HEAD: 200 with no body (correct per RFC 9110 §9.3.2). + res.writeHead(200, { "content-type": "text/plain" }); + res.end(); + }); + }); + const { proxy, proxyPort, upstreamPort, captured } = await startProxyWith(upstream); + try { + const resp = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/api/hello`, { method: "HEAD" }); + assert.equal(resp.status, 200); + assert.equal(await resp.text(), ""); + const nullBodyWarns = captured.filter((c) => c.level === "warn" && c.msg.includes("returned a null body")); + assert.equal(nullBodyWarns.length, 0, `expected no null-body warn for HEAD, got: ${nullBodyWarns.map((c) => c.msg).join(" | ")}`); + } finally { + setLogCapture(null); + await close(proxy); + await close(upstream); + } +}); + +test("GET 204 with a null body STILL logs the #821 null-body warn (#821 preserved)", async () => { + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + // GET: 204 → null body → the #821 anomaly the warn exists to name. + res.writeHead(204); + res.end(); + }); + }); + const { proxy, proxyPort, upstreamPort, captured } = await startProxyWith(upstream); + try { + const resp = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/api/hello`, { method: "GET" }); + assert.equal(resp.status, 204); + assert.equal(await resp.text(), ""); + const warn = captured.find((c) => c.level === "warn" && c.msg.includes("returned a null body")); + assert.ok(warn, `expected null-body warn for GET 204, got: ${captured.map((c) => c.msg).join(" | ")}`); + assert.match(warn.msg, /upstream 204 returned a null body/); + } finally { + setLogCapture(null); + await close(proxy); + await close(upstream); + } +}); From 6b3553c88e295b84384be3b584f308555f7336e0 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 20 Nov 2026 14:27:46 +0800 Subject: [PATCH 5/6] fix(fastpath): exclude id from watermark head check for pinned sessions (#7345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watermark fastpath head check compared a.id !== b.id between the fresh head cores (content-hash ids minted by openaiToCore, h_…) and the stashed seed cores. In a pinned session (pluginLaneServerHistory + pluginMsgIdIncremental — the 2x2 traffic the fastpath exists to accelerate) prepare-openai rebinds originalMessages to the POST-PIN cores before finalizeWatermarkTurn, so the seed carries host/ledger msgids while a fresh openaiToCore parse mints content-hash ids. The two id namespaces differ structurally on every turn, so the comparison could never pass: every turn of a pinned session logged fastpath miss (head-prefix-drift) and fell back to the full legacy parse. The id is not a content signal: exclude it from the head comparison (content and role inequality are still caught, and a remint with identical bytes is not a rewrite). Non-pinned traffic keeps content-hash ids on both sides, so when a.id === b.id the full-JSON equality holds iff the id-stripped equality holds — existing behavior is unchanged. Regression test plays the field sequence: seed with a host-id list, then a tail-only growth turn must HIT (was head-prefix-drift) and an in-place head edit must still drift. Fixes #7345 (cherry picked from commit ba4d5a6646e4371a75ce7ab98c64ac1f2b78a810) --- src/server/fastpath.ts | 17 ++++++++++- tests/fastpath.test.ts | 67 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/server/fastpath.ts b/src/server/fastpath.ts index 0773e16517..0871202b70 100644 --- a/src/server/fastpath.ts +++ b/src/server/fastpath.ts @@ -332,7 +332,7 @@ export function evaluateWatermarkFastpath( for (let hi = 0; hi < seedHead.length; hi++) { const a = headCores.msgs[hi]; const b = seedHead[hi]; - if (a.id !== b.id || JSON.stringify(a) !== JSON.stringify(b)) return miss("head-prefix-drift", false, tailIds); + if (headCoreContentDiffers(a, b)) return miss("head-prefix-drift", false, tailIds); } } const windowRaw = openaiToCore(windowFragment as OpenAIRequestBody); @@ -407,6 +407,21 @@ function baseOfRawId(id: string): string { return m === null ? id : id.slice(0, 18); } +/** In-place head-edit check, IGNORING the `id` field. In a pinned session + * (pluginLaneServerHistory) the seed cores carry host/ledger msgids while a + * fresh openaiToCore parse mints content-hash ids, so the id namespace differs + * structurally on every turn (#7347) — the id is not a content signal here. + * Content/role/shape inequality is still caught via the full JSON minus the + * id; a remint with identical bytes is not a rewrite. */ +function headCoreContentDiffers(a: CoreMessage, b: CoreMessage): boolean { + const strip = (c: CoreMessage): Record => { + const rest: Record = { ...c }; + delete rest.id; + return rest; + }; + return JSON.stringify(strip(a)) !== JSON.stringify(strip(b)); +} + /** openaiToCore materializes user content-array images as distinct cores * carrying an `imagePart`/`rawOpenaiContent` pair — used to detect * multimodal tails from already-converted cores (legacy-side seeding). */ diff --git a/tests/fastpath.test.ts b/tests/fastpath.test.ts index 1ff6ca5348..3ab650dafe 100644 --- a/tests/fastpath.test.ts +++ b/tests/fastpath.test.ts @@ -9,6 +9,7 @@ import http from "node:http"; import { openaiToCore, coreToOpenai } from "acp-kernel/wire"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, stampOnlyWatermarkTurn, clearWatermarkSeed, watermarkDebugState, PF_COUNT_HEADER, PF_SEQ_HEADER, PF_TAIL_IDS_HEADER, PF_SLOT_UNKNOWN } from "../src/server/fastpath.ts"; import { stripRawStage } from "../src/marker/engine.ts"; +import { applyMsgIdPin } from "../src/msgid-pin.ts"; import { INCREMENTAL_TURN_MARKER_HEADER } from "../src/incremental-lane.ts"; import { _setStoreForTest, _resetStoreForTest, type SessionStore } from "../src/persist.ts"; import type { Session } from "../src/session.ts"; @@ -65,6 +66,33 @@ const runTurn = ( return { out, headers }; }; +/** Drive one turn the way prepareOpenai does for a PINNED session + * (pluginLaneServerHistory on, #4491): on a miss the seed is re-keyed to the + * host msgids via applyMsgIdPin (the id namespace the stash then carries), on + * a hit the pin is skipped (the seed's prefix was already pinned when it was + * materialized). This is the 2x2 traffic the watermark fastpath exists to + * accelerate — the seed cores carry host msgids while a fresh openaiToCore + * parse mints content-hash ids, so the id namespaces differ structurally + * (#7347). */ +const runPinnedTurn = ( + session: Session, + reqHeaders: Record, + bodyText: string, + hostIds: string[], +): { out: ReturnType; headers: Record } => { + const model = JSON.parse(bodyText).model as string; + const out = evaluateWatermarkFastpath(bodyText, headersOf(reqHeaders), session, { pluginMode: true, titleGen: false, model }, log); + const oracle = openaiToCore(JSON.parse(bodyText)); + const stripCounts = stripRawStage(JSON.parse(bodyText), "openai"); + const headers = finalizeWatermarkTurn(session, out, { + model, + legacyCores: out.hit !== null ? out.hit.view : applyMsgIdPin(oracle.msgs, hostIds), + rawSystemText: out.hit !== null ? out.hit.systemText : oracle.systemText, + turnStripCounts: stripCounts, + }); + return { out, headers }; +}; + test("gate off / title-gen / incremental-turn requests are untouched", () => { _resetStoreForTest(); makeStubStore(); @@ -478,3 +506,42 @@ test("clearWatermarkSeed forces the next turn back through verification", () => const { out } = runTurn(session, { [PF_COUNT_HEADER]: "3" }, b2); assert.equal(out.decision, "unseeded"); }); + +test("pinned seed (host msgids) + tail-only growth HITS; in-place head edit still drifts (#7347)", () => { + makeStubStore(); + const session = freshSession("pf-pin"); + // turn 1: seed. pluginLaneServerHistory pins the seed cores to host msgids + // (applyMsgIdPin), so the stash carries a DIFFERENT id namespace than the + // content-hash ids a fresh openaiToCore parse mints. + const b1 = bodyOf([SYS("s"), USR("a"), USR("b"), ASST("c"), USR("d")]); + const t1 = runPinnedTurn(session, {}, b1, ["host-a", "host-b", "host-c", "host-d"]); + assert.equal(t1.out.decision, "no-stamp", LOGS.join("\n")); + assert.equal(t1.out.measuredT, 5); + // turn 2: same prefix + one new tail element. The fresh head cores carry + // content-hash ids while the seed carries host msgids — the head check must + // ignore the id namespace and HIT on the unchanged content. Pre-fix this is + // a permanent head-prefix-drift miss (the field symptom: every turn of a + // 2x2 pinned session fell back to the full legacy parse). + const b2 = bodyOf([SYS("s"), USR("a"), USR("b"), ASST("c"), USR("d"), USR("e")]); + const t2 = runPinnedTurn(session, { [PF_COUNT_HEADER]: "5" }, b2, ["host-a", "host-b", "host-c", "host-d", "host-e"]); + assert.equal(t2.out.decision, "hit", LOGS.join("\n")); + assert.equal(t2.out.creditedMiss, false); + // the view must still be the legacy full-parse oracle on CONTENT. The + // prefix ids are host msgids by design in a pinned session, so assert + // text/role parity (not id parity — that is the namespace difference). + if (!t2.out.hit) throw new Error("expected hit"); + const oracle2 = openaiToCore(JSON.parse(b2)); + assert.equal(t2.out.hit.view.length, oracle2.msgs.length); + for (let i = 0; i < oracle2.msgs.length; i++) { + assert.equal(t2.out.hit.view[i]!.text, oracle2.msgs[i]!.text, `text @${i}`); + assert.equal(t2.out.hit.view[i]!.role, oracle2.msgs[i]!.role, `role @${i}`); + } + // turn 3: rewrite a HEAD element IN PLACE (prefix counts unchanged, system + // identical, stamp honest) + append one tail — the per-core content + // comparison (id excluded) must still catch the rewrite, so the fix does + // not mask a genuine in-place edit. + const b3 = bodyOf([SYS("s"), USR("a"), USR("b-EDITED"), ASST("c"), USR("d"), USR("e"), USR("f")]); + const t3 = runPinnedTurn(session, { [PF_COUNT_HEADER]: "6" }, b3, ["host-a", "host-b2", "host-c", "host-d", "host-e", "host-f"]); + assert.equal(t3.out.decision, "head-prefix-drift", LOGS.join("\n")); + assert.equal(t3.out.creditedMiss, false); +}); From 904d4e0de03c9aff4c0de4bf0eece4021d5a6029 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 21 Nov 2026 15:18:43 +0800 Subject: [PATCH 6/6] fix(tunnel): terminate the one-shot DNS fallback worker when its inner deadline settles the probe (#7407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tunnel-admission worker probe (the post-main-pool-exhaustion fallback from #5222) runs in a throwaway worker thread with an inner 2.5s lookup deadline and an outer 3s worker deadline. When the inner timer posted ETIMEDOUT, the parent rejected the probe and cleared the outer timer but never stopped the worker: the expired one-shot worker (and its still- queued getaddrinfo, holding a libuv pool slot) stayed alive until unrelated native work completed. Repeated failing admissions could each retain a worker thread. The message handler now requests worker.terminate() for the inner-deadline ETIMEDOUT verdict — the same termination the outer deadline already used — handling terminate's async rejection before the existing timeout error is returned. Success and settled-failure verdicts are untouched (the lookup has already completed there and the worker drains and exits on its own). Admission classification, retries, and configured behavior are preserved; no wire, config, or message-shape change. Regression: tests/tunnel-guard-worker-lifecycle.test.ts runs the REAL Worker implementation with a controlled stalled-lookup boundary (real eval-worker plumbing; only dns.promises.lookup is replaced with a never-settling boundary plus a liveness anchor). It fails on the unfixed base (expired worker still alive after the ETIMEDOUT verdict) and passes with the fix (terminated, exit code 1). Controls: the outer deadline still stops a worker that cannot send its inner verdict, and a successful localhost lookup exits normally with its addresses. --- src/tunnel-guard.ts | 8 +++ tests/tunnel-guard-worker-lifecycle.test.ts | 80 +++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 tests/tunnel-guard-worker-lifecycle.test.ts diff --git a/src/tunnel-guard.ts b/src/tunnel-guard.ts index 3c9b79924d..c1a966b391 100644 --- a/src/tunnel-guard.ts +++ b/src/tunnel-guard.ts @@ -181,6 +181,14 @@ export const workerResolveHost: WorkerProbe = (host) => if (settled) return; settled = true; clearTimeout(timer); + if (msg?.error === "ETIMEDOUT") { + // #7407: the inner deadline won — the worker's getaddrinfo may + // still be queued in its libuv pool, keeping the one-shot + // thread (and a pool slot) alive. Same termination as the + // outer deadline; other failure codes mean the lookup already + // settled, and the worker drains and exits on its own. + try { void worker?.terminate().catch(() => undefined); } catch { /* already gone */ } + } if (msg && msg.ok === true && Array.isArray(msg.addrs)) resolve(msg.addrs); else reject(Object.assign(new Error(`worker dns lookup failed (${String(msg?.error ?? "unknown")})`), { code: typeof msg?.error === "string" && msg.error ? msg.error : "EWORKERLOOKUP" })); }); diff --git a/tests/tunnel-guard-worker-lifecycle.test.ts b/tests/tunnel-guard-worker-lifecycle.test.ts new file mode 100644 index 0000000000..1e0e9682be --- /dev/null +++ b/tests/tunnel-guard-worker-lifecycle.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { syncBuiltinESMExports } from "node:module"; +import test, { type TestContext } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; +import workerThreads, { Worker, type WorkerOptions } from "node:worker_threads"; +import { workerResolveHost } from "../src/tunnel-guard.ts"; + +// #7407: the one-shot DNS fallback worker is throwaway, but on the inner +// deadline the parent used to clear its outer timer and reject WITHOUT +// stopping the worker — the expired probe (and its still-queued +// getaddrinfo) kept a whole thread plus a libuv pool slot alive until +// unrelated native work finished. These tests run the REAL Worker +// implementation (only the lookup boundary is stalled via a prelude the +// harness prepends to the worker source) and assert the worker thread is +// actually gone once the probe has settled. +// +// Gate discipline (#3617): the stalled-lookup case is the regression +// (fails on the unfixed base — the expired worker stays alive); the outer +// deadline and the successful-lookup cases are the "does not over-trigger" +// controls (termination still works there, success exits normally). + +const OBSERVE_EXIT_GRACE_MS = 1000; + +function observeWorker(t: TestContext, prelude = "") { + const workers: Array<{ worker: Worker; exited: Promise }> = []; + class ObservedWorker extends Worker { + constructor(source: string | URL, options?: WorkerOptions) { + assert.ok(typeof source === "string" && options?.eval, "the probe must use the real eval-worker plumbing"); + super(`${prelude}\n${source}`, { ...options, execArgv: [] }); + workers.push({ worker: this, exited: new Promise((resolve) => this.once("exit", resolve)) }); + } + } + const replacement = t.mock.method(workerThreads, "Worker", ObservedWorker); + syncBuiltinESMExports(); + t.after(async () => { + replacement.mock.restore(); + syncBuiltinESMExports(); + await Promise.all(workers.map(({ worker }) => worker.terminate())); + }); + return async () => { + assert.equal(workers.length, 1, "one-shot probe — exactly one worker"); + const { worker, exited } = workers[0]!; + const outcome = await Promise.race([ + exited.then((code) => ({ code })), + delay(OBSERVE_EXIT_GRACE_MS, undefined, { ref: false }).then(() => ({ code: null })), + ]); + assert.notEqual(outcome.code, null, "the settled one-shot probe must not retain a live worker"); + assert.equal(worker.threadId, -1); + return outcome.code; + }; +} + +test("#7407 regression: the inner lookup deadline stops its one-shot worker", async (t) => { + const exited = observeWorker(t, ` + require("node:dns").promises.lookup = () => { + setInterval(() => {}, 10000); // anchor: an unfixed worker provably outlives the observation window + return new Promise(() => {}); // stalled getaddrinfo boundary + }; + `); + await assert.rejects(workerResolveHost("stalled-lookup.invalid"), (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as NodeJS.ErrnoException).code, "ETIMEDOUT"); + assert.match(error.message, /worker dns lookup failed/); + return true; + }); + assert.equal(await exited(), 1, "termination, not natural exit"); +}); + +test("#7407 control: the outer deadline still stops a worker that cannot send its inner verdict", async (t) => { + const exited = observeWorker(t, "Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);"); + await assert.rejects(workerResolveHost("localhost"), { code: "ETIMEDOUT", message: "ETIMEDOUT" }); + assert.equal(await exited(), 1); +}); + +test("#7407 control: a successful native lookup exits normally and keeps its addresses", async (t) => { + const exited = observeWorker(t); + const addresses = await workerResolveHost("localhost"); + assert.ok(addresses.includes("127.0.0.1") || addresses.includes("::1")); + assert.equal(await exited(), 0, "success path must keep its natural exit"); +});