Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/agent/incremental-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
// messages ({role, content: string | blocks[]}) — the lane derives stable
// content ids server-side, so hosts without their own message ids work.

import { createHash } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import type { DirectSendPrep } from "./native-intercept.js";

/** Marker header the lane's divert recognizes. Client-side copy of
Expand Down Expand Up @@ -218,6 +218,21 @@ interface ConvState {

const convs = new Map<string, ConvState>();

/** #4868: per-process identity stamped on EVERY incremental envelope. Two
* hosts sharing one conversation id flip each other's canonical with their
* resyncs until both burn REJECTION_CIRCUIT strikes into the sticky full-
* resend degradation — without a source fingerprint on the wire that race is
* indistinguishable in server logs from ordinary drift. One value per process
* lifetime (a restart is exactly when the stickiness clears); older servers
* ignore unknown envelope fields, older clients omit it (the lane then cannot
* fingerprint — no false alarm). */
const INSTANCE_ID = randomUUID();

/** Test seam for the stamped per-process fingerprint (stable within a process). */
export function __incrementalInstanceIdForTests(): string {
return INSTANCE_ID;
}

// #4491 host msgids: ids this client has already put on an envelope for the
// conversation — the client-side mirror of the lane's canonical id set. The
// lane HARD-FAILS a non-resync turn (400) whose msgIds reuse a live canonical
Expand Down Expand Up @@ -428,6 +443,7 @@ function buildBody(args: PlanArgs, sinceSeq: number, messages: IncrementalWireMe
path: args.path ?? "/v1/messages",
request,
messages,
instanceId: INSTANCE_ID,
};
if (resync) body.resync = true;
if (includeSystem) body.system = args.system === undefined ? null : args.system;
Expand Down
54 changes: 52 additions & 2 deletions src/incremental-lane.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* "sinceSeq": 0, // canonical length the client has acked
* "resync": false, // true → messages[] is a full replacement
* "system": "..." | [blocks], // latest system text (sticky when omitted)
* "instanceId": "<uuid>", // OPTIONAL per-process client fingerprint (#4868 dual-writer alert)
* "path": "/v1/messages", // optional forward path: bare relative (default, rides the lane's default upstream) or /bili/<upstream>/… embedded
* "request": { model, max_tokens, tools, stream, ... }, // anthropic params (allowlist)
* "messages": [ ...anthropic wire messages... ]
Expand Down Expand Up @@ -165,6 +166,42 @@ function warnDroppedRequestParams(conversationId: string, request: Record<string
log("warn", `[incremental] ${conversationId}: request parameter(s) not in the forward allowlist were dropped: ${unwarned.join(", ")} — extend REQUEST_PARAM_KEYS (src/incremental-lane.ts) if they must reach the upstream (#3406)`);
}

// #4868: last canonical REPLACEMENT per conversation — the client-stamped
// source fingerprint plus when it landed. Two DISTINCT sources replacing the
// same canonical inside DUAL_WRITER_WINDOW_MS means two hosts share one
// conversation id: each resync invalidates the other's acks, so both burn
// consecutive 409s into the client-side REJECTION_CIRCUIT=3 sticky full-
// resend degradation — and nothing else on the wire distinguishes that race
// from ordinary drift. Warn once per cooldown so the log names the actual
// cause instead of three mystery 409s per host. An empty instanceId (older
// client / malformed value) cannot be fingerprinted: it records the event but
// never triggers the alarm (no false alarms by design).
const DUAL_WRITER_WINDOW_MS = 60_000;
const DUAL_WRITER_WARN_COOLDOWN_MS = 5 * 60_000;
const RESYNC_SOURCE_BUDGET = 4096;
interface ResyncSource {
instanceId: string;
at: number;
warnedAt: number;
}
const resyncSources = new Map<string, ResyncSource>();
const INSTANCE_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/;

function noteResyncSource(conversationId: string, raw: unknown, log: (level: string, msg: string) => void): void {
const now = Date.now();
const instanceId = typeof raw === "string" && INSTANCE_ID_PATTERN.test(raw) ? raw : "";
const prev = resyncSources.get(conversationId);
// Same bounded discipline as droppedParamWarned: past the budget, drop the
// oldest epoch rather than grow without bound.
if (resyncSources.size >= RESYNC_SOURCE_BUDGET && prev === undefined) resyncSources.clear();
const entry: ResyncSource = { instanceId, at: now, warnedAt: prev?.warnedAt ?? 0 };
resyncSources.set(conversationId, entry);
const distinct = prev !== undefined && instanceId !== "" && prev.instanceId !== "" && prev.instanceId !== instanceId;
if (!distinct || now - prev.at > DUAL_WRITER_WINDOW_MS || now - entry.warnedAt < DUAL_WRITER_WARN_COOLDOWN_MS) return;
entry.warnedAt = now;
log("warn", `[incremental] ${conversationId}: suspected dual writers — distinct sources "${prev.instanceId}" and "${instanceId}" replaced this canonical ${Math.round((now - prev.at) / 1000)}s apart; each resync invalidates the other's acks, so both hosts will burn 3 consecutive 409s into the sticky full-resend fallback unless they stop sharing the conversation id (#4868)`);
}

/** Canonical prefix per conversation, in memory. Bounded LRU: cold entries
* rebuild from the unified messages table on demand (see loadCanonical). */
const CANONICAL_LIMIT = 16;
Expand Down Expand Up @@ -579,7 +616,11 @@ function applyTurn(
const serverSeq = entry.msgs.length;
if (resync) {
// Full replacement: the client is the source of truth for the whole
// history (edit / branch reset / recovery). Marks and system survive.
// history (edit / branch reset / recovery). System survives (sticky
// state, never positional); marks are pruned below to the live set —
// ids are unique so a surviving mark can only attach to a live
// message, but a replaced branch's marks would otherwise linger in
// the map (and its ext key) forever (#4868).
} else if (sinceSeq !== serverSeq) {
// Strict ack contract — including sinceSeq 0 against a populated
// canonical: a client that believes it is seeding a fresh server must
Expand Down Expand Up @@ -661,6 +702,11 @@ function applyTurn(
let marksTouched = flat.cacheControls.size > 0;
if (resync) {
for (const m of deltaMsgs) if (entry.marks.delete(m.id)) marksTouched = true;
// #4868: also prune every mark whose id left the served history —
// the loop above only covers this turn's messages, so dead-branch
// marks would linger in the map forever otherwise.
const live = new Set(entry.msgs.map((m) => m.id));
for (const id of entry.marks.keys()) if (!live.has(id) && entry.marks.delete(id)) marksTouched = true;
}
for (const r of retouchMarks) {
// Opaque blocks render verbatim, so their validated decoration must
Expand Down Expand Up @@ -705,6 +751,7 @@ export function __resetIncrementalLaneForTests(): void {
pendingPtrWork.clear();
droppedParamWarned.clear();
droppedParamWarnPairs = 0;
resyncSources.clear();
}

/** Marker header native extensions stamp on an ordinary model request to
Expand Down Expand Up @@ -884,7 +931,10 @@ export async function handleIncrementalTurn(
}
touchCache(conversationId, entry);
// Record the epoch BEFORE the forward lands the new branch's rows.
if (resync) await recordResyncBaseSeq(sessionId, log);
if (resync) {
await recordResyncBaseSeq(sessionId, log);
noteResyncSource(conversationId, payload.instanceId, log);
}
persistLaneState(sessionId, entry.system, entry.systemSet, entry.marks, log);

const body: Record<string, unknown> = { ...forwarded };
Expand Down
19 changes: 18 additions & 1 deletion tests/incremental-client-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import assert from "node:assert/strict";

process.env.NODE_TEST_CONTEXT = "1";

const { __resetIncrementalClientForTests, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn } = await import("../src/agent/incremental-client.ts");
const { __incrementalInstanceIdForTests, __resetIncrementalClientForTests, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn } = await import("../src/agent/incremental-client.ts");

const SID = "inc-recovery-1";

Expand Down Expand Up @@ -63,3 +63,20 @@ test("#3369: rejection with no prior state is a no-op reset (a seed/resync canno
const next = planIncrementalTurn({ conversationId: SID, messages: msgs(2), system: undefined });
assert.equal(next.mode, "seed");
});

test("#4868: every envelope carries the per-process instanceId fingerprint, stable across plans", () => {
__resetIncrementalClientForTests();
const expected = __incrementalInstanceIdForTests();
assert.ok(typeof expected === "string" && expected.length > 0, "process fingerprint is minted at module load");

const seed = planIncrementalTurn({ conversationId: SID, messages: msgs(2), system: undefined });
assert.equal(seed.mode, "seed");
const seedBody = JSON.parse(seed.body) as Record<string, unknown>;
assert.equal(seedBody.instanceId, expected, "seed envelope stamps the fingerprint");

noteIncrementalAck(SID, 2, msgs(2), undefined);
const delta = planIncrementalTurn({ conversationId: SID, messages: msgs(4), system: undefined });
assert.equal(delta.mode, "delta");
const deltaBody = JSON.parse(delta.body) as Record<string, unknown>;
assert.equal(deltaBody.instanceId, expected, "delta envelope stamps the SAME fingerprint (one per process lifetime)");
});
Loading
Loading