From 6b7f66f731c2d80c63fe15ccce4fc92eb43b9f9e Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 20 Nov 2026 16:25:47 +0800 Subject: [PATCH] fix: cap x-bili-pf-tail-ids at 160 ids and decouple the pf-seq stamp from the fastpath gate (#7367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With pluginFullResendWatermark gated off, the watermark fastpath never evaluated, so the x-bili-pf-seq response stamp (the client's watermark advance) was never emitted. The client's k2 froze while its captured msgid list kept growing, so x-bili-pf-tail-ids grew unbounded and eventually overflowed Node's 16 KB header limit — the proxy's HTTP parser dropped the connection before any handler ran, and the session was wedged on every retry. Two-part fix: 1. Client safety net (src/agent/pf-channel.ts): cap the tail-ids list at the newest 160 ids. The capped tail still ends at the true window origin; a delta wider than 160 degrades to a conservative server-side miss (the width check fails) instead of a header overflow. 160 ids ≈ 5.8 KB for UUIDs, well under the limit. 2. Server root fix (src/server/fastpath.ts + prepare-openai.ts): emit the x-bili-pf-seq stamp via a structural scan even when the fastpath gate is off. The stamp is the client's watermark advance, not fastpath-only state — decoupling it from the gate keeps k2 advancing, which is what bounds the tail list at its natural width. Behavior change (disclosed per owner policy 2026-09-22): with pluginFullResendWatermark OFF, the proxy now emits x-bili-pf-seq on plugin-lane non-title-gen openai responses (old: never emitted). The client-visible x-bili-pf-tail-ids is now capped at the newest 160 ids (old: unbounded from a frozen k2). Both changes are the fix. Fixes #7367 --- src/agent/pf-channel.ts | 16 +++++++++++++++- src/server/fastpath.ts | 15 +++++++++++++++ src/server/prepare-openai.ts | 9 ++++++++- tests/fastpath.test.ts | 16 +++++++++++++++- tests/pf-channel.test.ts | 22 ++++++++++++++++++++++ 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/agent/pf-channel.ts b/src/agent/pf-channel.ts index 104c33635f..a1f3e5b287 100644 --- a/src/agent/pf-channel.ts +++ b/src/agent/pf-channel.ts @@ -43,6 +43,15 @@ const RESERVED_PREFIX = "acp_summary_"; const BAD_ID_CHARS = /[\s\p{C}]/u; const MAX_IDS = 2048; // long-conversation ceiling; the sliced head stays small const MAX_TRACKED = 1024; // sid table cap — at capacity exactly ONE oldest session is evicted (#6044) +// #7367: the tail list is [id(K-1)] + new ids since the last stamp. When the +// stamp freezes (the flag flips off, a restart, a scan miss) K stops moving and +// this slice grows with every turn — unbounded it crosses Node's 16 KB header +// limit and the proxy drops the connection before any handler runs, wedging +// the session on every retry. A tail this large would miss the fold anyway +// (the server's width check requires len === T - stampT + 1), so keep only the +// newest MAX_TAIL_IDS ids: the window origin the fastpath needs is the LAST +// element, and a delta wider than this degrades to a (correct) miss. +const MAX_TAIL_IDS = 160; type Tracked = { t?: number; ids?: string[] }; @@ -130,7 +139,12 @@ export function pfRequestHeaders(incoming: Record): Record = { [COUNT_HEADER]: String(k) }; const ids = slot.ids; if (ids !== undefined && ids.length >= k) { - out[TAIL_IDS_HEADER] = ids.slice(k - 1).map(encodeURIComponent).join(","); + // #7367: cap the tail to the newest MAX_TAIL_IDS ids. When K is frozen + // the full [K-1..] slice would grow past the 16 KB header limit; a + // capped tail still ends at the true window origin (ids.length-1) and + // degrades a wide delta to a conservative server-side miss. + const start = Math.max(k - 1, ids.length - MAX_TAIL_IDS); + out[TAIL_IDS_HEADER] = ids.slice(start).map(encodeURIComponent).join(","); } return out; } diff --git a/src/server/fastpath.ts b/src/server/fastpath.ts index 72970d4984..0773e16517 100644 --- a/src/server/fastpath.ts +++ b/src/server/fastpath.ts @@ -512,6 +512,21 @@ export function finalizeWatermarkTurn( return headers; } +/** Stamp-only turn (flag off, #7367): the x-bili-pf-seq heartbeat is the + * client's watermark advance, NOT fastpath-only state. When + * pluginFullResendWatermark is gated off the fastpath never evaluates, so + * finalizeWatermarkTurn never runs and the stamp stops — the client's k2 + * freezes and its x-bili-pf-tail-ids grows one id per turn until it crosses + * Node's 16 KB header limit and wedges the session. Emit the stamp whenever + * T is measurable (the structural scan is the same zero-allocation pass the + * fastpath uses) so the client's tail stays bounded. No seed, no saved + * state: the fastpath re-establishes both on its first evaluated turn. */ +export function stampOnlyWatermarkTurn(bodyText: string): Record { + const scan = scanMessagesSpans(bodyText, "all"); + if (scan === null) return {}; + return { [PF_SEQ_HEADER]: String(scan.total) }; +} + /** Drop the in-process seed (e.g. an announced compaction was detected while * the fastpath was silent on later turns). Cheap and idempotent; the next * turn simply re-verifies via the legacy path. */ diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index b69e92d319..aa01fb9c46 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -48,7 +48,7 @@ import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-respon import { injectOpenaiTool, injectTool } from "./inject.js"; import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js"; import { featureEnabled } from "../rollout.js"; -import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; +import { evaluateWatermarkFastpath, finalizeWatermarkTurn, stampOnlyWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; export async function prepareOpenai( @@ -677,6 +677,13 @@ export async function prepareOpenai( turnStripCounts: wpTurnStrip, }); if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; + } else if (bodyText !== undefined && pluginMode && !isTitleGen && !featureEnabled("pluginFullResendWatermark")) { + // #7367: flag off — the fastpath never evaluates, but the x-bili-pf-seq + // heartbeat still must flow or the client's watermark freezes and its + // tail-ids grow past the 16 KB header limit. Stamp-only: no seed, no + // saved state (the fastpath re-establishes both when it next runs). + wpResponseHeaders = stampOnlyWatermarkTurn(bodyText); + if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; } if (nudgeOutcome === "injected") recordNudgeInject(session, nudge); return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, attachedRetrievals, attachedRetrievalNoteIds, processedMessages, originalMessages, judgeBaseline, protocol: "openai", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, nudgeStampsBefore: nudgeOutcome === "injected" ? nudgeStampsBefore : undefined, prompts, surface, openaiSystemText, systemNotes: sysNotes, renderTags: renderStrategy, responseExtraHeaders: wpResponseHeaders, dropReasoning: (msgs: BiliMessage[]) => applyReasoningDrop(msgs, reasoningPlan, log, sessionId) } as Prepared; diff --git a/tests/fastpath.test.ts b/tests/fastpath.test.ts index 8965a7b9f6..1ff6ca5348 100644 --- a/tests/fastpath.test.ts +++ b/tests/fastpath.test.ts @@ -7,7 +7,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import http from "node:http"; import { openaiToCore, coreToOpenai } from "acp-kernel/wire"; -import { evaluateWatermarkFastpath, finalizeWatermarkTurn, clearWatermarkSeed, watermarkDebugState, PF_COUNT_HEADER, PF_SEQ_HEADER, PF_TAIL_IDS_HEADER, PF_SLOT_UNKNOWN } from "../src/server/fastpath.ts"; +import { evaluateWatermarkFastpath, finalizeWatermarkTurn, stampOnlyWatermarkTurn, clearWatermarkSeed, watermarkDebugState, PF_COUNT_HEADER, PF_SEQ_HEADER, PF_TAIL_IDS_HEADER, PF_SLOT_UNKNOWN } from "../src/server/fastpath.ts"; import { stripRawStage } from "../src/marker/engine.ts"; import { INCREMENTAL_TURN_MARKER_HEADER } from "../src/incremental-lane.ts"; import { _setStoreForTest, _resetStoreForTest, type SessionStore } from "../src/persist.ts"; @@ -86,6 +86,20 @@ test("gate off / title-gen / incremental-turn requests are untouched", () => { _resetStoreForTest(); }); +test("stampOnlyWatermarkTurn: flag off still emits the seq heartbeat (#7367)", () => { + // The x-bili-pf-seq stamp is the client's watermark advance, NOT + // fastpath-only state. With pluginFullResendWatermark gated off the + // fastpath never evaluates, so finalizeWatermarkTurn never runs — the + // stamp must still flow (via the structural scan) or the client's k2 + // freezes and its tail-ids grow past the 16 KB header limit. + const body = bodyOf([SYS("s"), USR("hi"), USR("yo"), ASST("ok"), USR("tail")]); + const headers = stampOnlyWatermarkTurn(body); + assert.equal(headers[PF_SEQ_HEADER], "5", "stamp = measured element count T"); + // an unscannable body degrades to no stamp (the client's cap is the backstop) + assert.deepEqual(stampOnlyWatermarkTurn("not-json-at-all"), {}); + assert.deepEqual(stampOnlyWatermarkTurn(""), {}); +}); + test("first turn: unseeded miss, measured T, response re-stamps", () => { makeStubStore(); const session = freshSession("pf-first"); diff --git a/tests/pf-channel.test.ts b/tests/pf-channel.test.ts index 95a3ea3679..36a1c5f858 100644 --- a/tests/pf-channel.test.ts +++ b/tests/pf-channel.test.ts @@ -43,6 +43,28 @@ test("pf-channel: count mode; tail-ids ride only a validated list; encoding", () assert.deepEqual(pfChannel.pfRequestHeaders({ [CONVO]: sidShort }), { "x-bili-pf-count": "2" }); }); +test("pf-channel: tail-ids capped at 160 when the stamp freezes (#7367)", () => { + const sid = "sid-cap-1"; + const inH = { [CONVO]: sid }; + // k2 frozen at 2 (the flag flipped off, so the stamp never advances again) + learn(inH, "2"); + // a long session: 200 ids captured; the full [K-1..] slice would be 199 ids + pfChannel.pfCaptureIds(sid, Array.from({ length: 200 }, (_, i) => `id${i}`)); + const tail = pfChannel.pfRequestHeaders(inH)["x-bili-pf-tail-ids"]; + assert.ok(tail !== undefined, "tail-ids present"); + const segs = tail!.split(","); + // capped to the NEWEST 160 ids (the window origin ids.length-1 is kept) + assert.equal(segs.length, 160, "tail capped at 160 ids"); + assert.equal(segs[0], "id40", "cap keeps the newest ids, drops the oldest"); + assert.equal(segs[159], "id199", "cap ends at the true window origin"); + // a delta NARROWER than the cap is sent whole (no truncation) + const sidNarrow = "sid-cap-narrow"; + learn({ [CONVO]: sidNarrow }, "2"); + pfChannel.pfCaptureIds(sidNarrow, Array.from({ length: 10 }, (_, i) => `n${i}`)); + const narrowTail = pfChannel.pfRequestHeaders({ [CONVO]: sidNarrow })["x-bili-pf-tail-ids"]; + assert.equal(narrowTail!.split(",").length, 9, "narrow delta sent whole (ids.slice(k-1))"); +}); + test("pf-channel: failed whole-resolution CLEARS the stored list (no stale slice)", () => { const sid = "sid-clear-1"; const inH = { [CONVO]: sid };