diff --git a/src/agent/incremental-client.ts b/src/agent/incremental-client.ts index 929b3f9fa8..57f5c50881 100644 --- a/src/agent/incremental-client.ts +++ b/src/agent/incremental-client.ts @@ -16,7 +16,7 @@ // messages ({role, content: string | blocks[]}) — the lane derives stable // content ids server-side, so hosts without their own message ids work. -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import type { DirectSendPrep } from "./native-intercept.js"; /** Marker header the lane's divert recognizes. Client-side copy of @@ -218,6 +218,21 @@ interface ConvState { const convs = new Map(); +/** #4868: per-process identity stamped on EVERY incremental envelope. Two + * hosts sharing one conversation id flip each other's canonical with their + * resyncs until both burn REJECTION_CIRCUIT strikes into the sticky full- + * resend degradation — without a source fingerprint on the wire that race is + * indistinguishable in server logs from ordinary drift. One value per process + * lifetime (a restart is exactly when the stickiness clears); older servers + * ignore unknown envelope fields, older clients omit it (the lane then cannot + * fingerprint — no false alarm). */ +const INSTANCE_ID = randomUUID(); + +/** Test seam for the stamped per-process fingerprint (stable within a process). */ +export function __incrementalInstanceIdForTests(): string { + return INSTANCE_ID; +} + // #4491 host msgids: ids this client has already put on an envelope for the // conversation — the client-side mirror of the lane's canonical id set. The // lane HARD-FAILS a non-resync turn (400) whose msgIds reuse a live canonical @@ -428,6 +443,7 @@ function buildBody(args: PlanArgs, sinceSeq: number, messages: IncrementalWireMe path: args.path ?? "/v1/messages", request, messages, + instanceId: INSTANCE_ID, }; if (resync) body.resync = true; if (includeSystem) body.system = args.system === undefined ? null : args.system; diff --git a/src/incremental-lane.ts b/src/incremental-lane.ts index a233e6f5b9..985a704e25 100644 --- a/src/incremental-lane.ts +++ b/src/incremental-lane.ts @@ -9,6 +9,7 @@ * "sinceSeq": 0, // canonical length the client has acked * "resync": false, // true → messages[] is a full replacement * "system": "..." | [blocks], // latest system text (sticky when omitted) + * "instanceId": "", // OPTIONAL per-process client fingerprint (#4868 dual-writer alert) * "path": "/v1/messages", // optional forward path: bare relative (default, rides the lane's default upstream) or /bili//… embedded * "request": { model, max_tokens, tools, stream, ... }, // anthropic params (allowlist) * "messages": [ ...anthropic wire messages... ] @@ -165,6 +166,42 @@ function warnDroppedRequestParams(conversationId: string, request: Record(); +const INSTANCE_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/; + +function noteResyncSource(conversationId: string, raw: unknown, log: (level: string, msg: string) => void): void { + const now = Date.now(); + const instanceId = typeof raw === "string" && INSTANCE_ID_PATTERN.test(raw) ? raw : ""; + const prev = resyncSources.get(conversationId); + // Same bounded discipline as droppedParamWarned: past the budget, drop the + // oldest epoch rather than grow without bound. + if (resyncSources.size >= RESYNC_SOURCE_BUDGET && prev === undefined) resyncSources.clear(); + const entry: ResyncSource = { instanceId, at: now, warnedAt: prev?.warnedAt ?? 0 }; + resyncSources.set(conversationId, entry); + const distinct = prev !== undefined && instanceId !== "" && prev.instanceId !== "" && prev.instanceId !== instanceId; + if (!distinct || now - prev.at > DUAL_WRITER_WINDOW_MS || now - entry.warnedAt < DUAL_WRITER_WARN_COOLDOWN_MS) return; + entry.warnedAt = now; + log("warn", `[incremental] ${conversationId}: suspected dual writers — distinct sources "${prev.instanceId}" and "${instanceId}" replaced this canonical ${Math.round((now - prev.at) / 1000)}s apart; each resync invalidates the other's acks, so both hosts will burn 3 consecutive 409s into the sticky full-resend fallback unless they stop sharing the conversation id (#4868)`); +} + /** Canonical prefix per conversation, in memory. Bounded LRU: cold entries * rebuild from the unified messages table on demand (see loadCanonical). */ const CANONICAL_LIMIT = 16; @@ -579,7 +616,11 @@ function applyTurn( const serverSeq = entry.msgs.length; if (resync) { // Full replacement: the client is the source of truth for the whole - // history (edit / branch reset / recovery). Marks and system survive. + // history (edit / branch reset / recovery). System survives (sticky + // state, never positional); marks are pruned below to the live set — + // ids are unique so a surviving mark can only attach to a live + // message, but a replaced branch's marks would otherwise linger in + // the map (and its ext key) forever (#4868). } else if (sinceSeq !== serverSeq) { // Strict ack contract — including sinceSeq 0 against a populated // canonical: a client that believes it is seeding a fresh server must @@ -661,6 +702,11 @@ function applyTurn( let marksTouched = flat.cacheControls.size > 0; if (resync) { for (const m of deltaMsgs) if (entry.marks.delete(m.id)) marksTouched = true; + // #4868: also prune every mark whose id left the served history — + // the loop above only covers this turn's messages, so dead-branch + // marks would linger in the map forever otherwise. + const live = new Set(entry.msgs.map((m) => m.id)); + for (const id of entry.marks.keys()) if (!live.has(id) && entry.marks.delete(id)) marksTouched = true; } for (const r of retouchMarks) { // Opaque blocks render verbatim, so their validated decoration must @@ -705,6 +751,7 @@ export function __resetIncrementalLaneForTests(): void { pendingPtrWork.clear(); droppedParamWarned.clear(); droppedParamWarnPairs = 0; + resyncSources.clear(); } /** Marker header native extensions stamp on an ordinary model request to @@ -884,7 +931,10 @@ export async function handleIncrementalTurn( } touchCache(conversationId, entry); // Record the epoch BEFORE the forward lands the new branch's rows. - if (resync) await recordResyncBaseSeq(sessionId, log); + if (resync) { + await recordResyncBaseSeq(sessionId, log); + noteResyncSource(conversationId, payload.instanceId, log); + } persistLaneState(sessionId, entry.system, entry.systemSet, entry.marks, log); const body: Record = { ...forwarded }; diff --git a/tests/incremental-client-recovery.test.ts b/tests/incremental-client-recovery.test.ts index c92363460d..8ae1da8c1d 100644 --- a/tests/incremental-client-recovery.test.ts +++ b/tests/incremental-client-recovery.test.ts @@ -8,7 +8,7 @@ import assert from "node:assert/strict"; process.env.NODE_TEST_CONTEXT = "1"; -const { __resetIncrementalClientForTests, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn } = await import("../src/agent/incremental-client.ts"); +const { __incrementalInstanceIdForTests, __resetIncrementalClientForTests, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn } = await import("../src/agent/incremental-client.ts"); const SID = "inc-recovery-1"; @@ -63,3 +63,20 @@ test("#3369: rejection with no prior state is a no-op reset (a seed/resync canno const next = planIncrementalTurn({ conversationId: SID, messages: msgs(2), system: undefined }); assert.equal(next.mode, "seed"); }); + +test("#4868: every envelope carries the per-process instanceId fingerprint, stable across plans", () => { + __resetIncrementalClientForTests(); + const expected = __incrementalInstanceIdForTests(); + assert.ok(typeof expected === "string" && expected.length > 0, "process fingerprint is minted at module load"); + + const seed = planIncrementalTurn({ conversationId: SID, messages: msgs(2), system: undefined }); + assert.equal(seed.mode, "seed"); + const seedBody = JSON.parse(seed.body) as Record; + assert.equal(seedBody.instanceId, expected, "seed envelope stamps the fingerprint"); + + noteIncrementalAck(SID, 2, msgs(2), undefined); + const delta = planIncrementalTurn({ conversationId: SID, messages: msgs(4), system: undefined }); + assert.equal(delta.mode, "delta"); + const deltaBody = JSON.parse(delta.body) as Record; + assert.equal(deltaBody.instanceId, expected, "delta envelope stamps the SAME fingerprint (one per process lifetime)"); +}); diff --git a/tests/incremental-lane.test.ts b/tests/incremental-lane.test.ts index 2408a2831d..2d493f8613 100644 --- a/tests/incremental-lane.test.ts +++ b/tests/incremental-lane.test.ts @@ -14,6 +14,7 @@ import { _resetSessionsForTest, peekSession } from "../src/session.ts"; import { _setForTest } from "../src/registry.ts"; import { resetToolRingForTest } from "../src/tool-ring.ts"; import { __resetIncrementalLaneForTests } from "../src/incremental-lane.ts"; +import { setLogCapture } from "../src/logger.ts"; process.env.NODE_ENV = "test"; const testRoot = mkdtempSync(join(tmpdir(), "bili-incr-")); @@ -25,6 +26,10 @@ interface Turn { resync?: boolean; system?: string; request?: Record; + /** #4868: client source fingerprint stamped on the envelope (dual-writer alert). */ + instanceId?: string; + /** Defaults to "conv-inc-1" (header and body must agree). */ + conversationId?: string; } interface Harness { @@ -36,11 +41,12 @@ interface Harness { markerRaw: (reqPath: string, anthropicBody: Record) => Promise<{ status: number; body: unknown }>; forwarded: Record[]; dir: string; + store: SessionStore; settle: () => Promise; close: () => Promise; } -async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: number; upstreamStatus?: number; debounceMs?: number } = {}): Promise { +async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: number; upstreamStatus?: number; debounceMs?: number; logEnabled?: boolean } = {}): Promise { const dir = mkdtempSync(join(testRoot, "run-")); for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = dir; process.env.BILI_STORAGE_UNIFIED = storageMode === "full" ? "full" : "legacy"; @@ -70,15 +76,16 @@ async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: const addr = upstream.address(); assert(addr && typeof addr === "object"); const upstreamUrl = `http://127.0.0.1:${addr.port}`; - const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: upstreamUrl, routes: { [upstreamUrl]: { models: { "claude-test": { context: 400000 } } } }, modelContextLimit: 400000, kernelConfig: defaultConfig(400000), compress: { injectTool: true, injectNudge: false }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: false, debug: false, passthrough: false, autoUpdate: false, mitm: { enabled: false, domains: [] } } as ProxyOptions); + const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: upstreamUrl, routes: { [upstreamUrl]: { models: { "claude-test": { context: 400000 } } } }, modelContextLimit: 400000, kernelConfig: defaultConfig(400000), compress: { injectTool: true, injectNudge: false }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: opts.logEnabled ?? false, debug: false, passthrough: false, autoUpdate: false, mitm: { enabled: false, domains: [] } } as ProxyOptions); if (!(proxy as unknown as { listening?: boolean }).listening) await once(proxy, "listening"); const proxyUrl = `http://127.0.0.1:${(proxy.address() as { port: number }).port}`; const path = "/v1/messages"; const turn = async (t: Turn) => { + const cid = t.conversationId ?? "conv-inc-1"; const response = await fetch(`${proxyUrl}/__bili/plugin/turn`, { method: "POST", - headers: { "content-type": "application/json", "x-bili-plugin": "dsh", "x-bili-plugin-conversation": "conv-inc-1" }, - body: JSON.stringify({ conversationId: "conv-inc-1", sinceSeq: t.sinceSeq, ...(t.resync ? { resync: true } : {}), ...(t.system !== undefined ? { system: t.system } : {}), path, request: { model: "claude-test", max_tokens: 1024, ...(t.request ?? {}) }, messages: t.messages }), + headers: { "content-type": "application/json", "x-bili-plugin": "dsh", "x-bili-plugin-conversation": cid }, + body: JSON.stringify({ conversationId: cid, sinceSeq: t.sinceSeq, ...(t.resync ? { resync: true } : {}), ...(t.system !== undefined ? { system: t.system } : {}), path, request: { model: "claude-test", max_tokens: 1024, ...(t.request ?? {}) }, messages: t.messages, ...(t.instanceId !== undefined ? { instanceId: t.instanceId } : {}) }), }); const body: unknown = await response.json().catch(() => null); return { status: response.status, headers: response.headers, body }; @@ -107,7 +114,7 @@ async function harness(storageMode: "full" | "legacy", opts: { upstreamDelayMs?: const body: unknown = await response.json().catch(() => null); return { status: response.status, body }; }; - return { proxyUrl, path, turn, probe, markerTurn, markerRaw, forwarded, dir, settle: async () => { await new Promise((resolve) => setTimeout(resolve, (opts.debounceMs ?? 60000) + 40)); }, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; + return { proxyUrl, path, turn, probe, markerTurn, markerRaw, forwarded, dir, store, settle: async () => { await new Promise((resolve) => setTimeout(resolve, (opts.debounceMs ?? 60000) + 40)); }, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); store.closeQuietly(); await new Promise((resolve) => upstream.close(() => resolve())); } }; } const userTurn = (n: number, text = "hello") => ({ id: `u${n}`, role: "user", contentType: "text", content: text.repeat(400) }); @@ -591,3 +598,98 @@ test("#3433: lane rejects non-anthropic message shapes with 400, never half-conv await h.close(); } }); + +// #4868: two DISTINCT client sources replacing the same canonical in quick +// succession = two hosts sharing one conversation id. Each resync invalidates +// the other's acks, so both burn consecutive 409s into the client-side +// REJECTION_CIRCUIT=3 sticky full-resend degradation — without the source +// fingerprint nothing in the log distinguishes that race from ordinary drift. +// The lane must name the race (once, per cooldown); same-source resyncs are +// normal recovery and must NOT alarm (gate discipline: both directions). +test("#4868: alternating resync sources on one conversation warn once; same source never alarms", async () => { + const h = await harness("full", { logEnabled: true }); + const warnings: string[] = []; + setLogCapture((level, msg) => { if (level === "warn") warnings.push(msg); }); + try { + const seed = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(1)], instanceId: "host-a" }); + assert.equal(seed.status, 200); + // host-b believes it owns the conversation too: its stale delta 409s + // against host-a's canonical... + const clash = await h.turn({ sinceSeq: 0, messages: [userTurn(9)], instanceId: "host-b" }); + assert.equal(clash.status, 409); + // ...and its resync replaces the canonical from a DISTINCT source → + // exactly one clear warn naming both sides. + const resync = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(1), userTurn(9)], instanceId: "host-b" }); + assert.equal(resync.status, 200); + const alarms = warnings.filter((w) => w.includes("dual writers")); + assert.equal(alarms.length, 1, `expected exactly one dual-writer warn, got: ${JSON.stringify(warnings)}`); + for (const needle of ["conv-inc-1", "host-a", "host-b"]) assert.ok(alarms[0].includes(needle), `warn must name ${needle}: ${alarms[0]}`); + // Same source resyncing again is ordinary recovery — no second alarm. + const again = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(1), userTurn(9), assistantTurn(1)], instanceId: "host-b" }); + assert.equal(again.status, 200); + assert.equal(warnings.filter((w) => w.includes("dual writers")).length, 1); + } finally { + setLogCapture(null); + await h.close(); + } +}); + +test("#4868: missing or malformed instanceId cannot trigger the dual-writer alarm", async () => { + const h = await harness("full", { logEnabled: true }); + const warnings: string[] = []; + setLogCapture((level, msg) => { if (level === "warn") warnings.push(msg); }); + try { + // Older clients stamp nothing: two anonymous resyncs must stay silent. + const seed = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(1)] }); + assert.equal(seed.status, 200); + const clash = await h.turn({ sinceSeq: 0, messages: [userTurn(9)] }); + assert.equal(clash.status, 409); + const resync = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(1), userTurn(9)] }); + assert.equal(resync.status, 200); + // A malformed fingerprint degrades to anonymous for that side only — + // still no alarm against a valid other side. + const seed2 = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(2)], conversationId: "conv-inc-2", instanceId: "bad id!" }); + assert.equal(seed2.status, 200); + const clash2 = await h.turn({ sinceSeq: 0, messages: [userTurn(8)], conversationId: "conv-inc-2", instanceId: "host-b" }); + assert.equal(clash2.status, 409); + const resync2 = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(2), userTurn(8)], conversationId: "conv-inc-2", instanceId: "host-b" }); + assert.equal(resync2.status, 200); + assert.equal(warnings.filter((w) => w.includes("dual writers")).length, 0, JSON.stringify(warnings)); + } finally { + setLogCapture(null); + await h.close(); + } +}); + +// #4868: resync replaces the canonical — marks keyed by ids that no longer +// exist in the new branch must be pruned (they would linger in the map and be +// rewritten into the inc:cache-marks ext key forever, O(turns across branches)). +// Live marks survive; new marks land as before. +test("#4868: resync prunes cache marks of the replaced branch; live and new marks survive", async () => { + const h = await harness("full"); + try { + const marked = { role: "user", content: [{ type: "text", text: "hello".repeat(400), cache_control: { type: "ephemeral" } }] }; + const seed = await h.turn({ sinceSeq: 0, resync: true, messages: [marked] }); + assert.equal(seed.status, 200); + let ext = h.store.getSessionExt("conv-inc-1", "inc:cache-marks"); + assert.ok(Array.isArray(ext) && ext.length === 1, `seed mark persisted: ${JSON.stringify(ext)}`); + + // Resync onto a DIFFERENT branch (distinct text → distinct derived id): + // the old mark's id is gone from the canonical → pruned; unmarked + // history adds nothing. Same-content messages share their id and would + // legitimately keep the mark. + const resync = await h.turn({ sinceSeq: 0, resync: true, messages: [userTurn(7, "branch seven")] }); + assert.equal(resync.status, 200); + ext = h.store.getSessionExt("conv-inc-1", "inc:cache-marks"); + assert.ok(Array.isArray(ext) && ext.length === 0, `dead-branch mark pruned: ${JSON.stringify(ext)}`); + + // A delta carrying a fresh mark appends normally. + const marked2 = { role: "user", content: [{ type: "text", text: "world".repeat(400), cache_control: { type: "ephemeral" } }] }; + const delta = await h.turn({ sinceSeq: 1, messages: [marked2] }); + assert.equal(delta.status, 200); + ext = h.store.getSessionExt("conv-inc-1", "inc:cache-marks"); + assert.ok(Array.isArray(ext) && ext.length === 1, `live mark retained: ${JSON.stringify(ext)}`); + } finally { + await h.close(); + } +});