Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/server/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ import { adminGateDenial, handleAdminRoute, isManagementPath } from "./admin.js"
import { handleIncrementalTurn, handleIncrementalProbe, incrementalTurnBodyShape, incrementalTurnsEnabled, INCREMENTAL_TURN_MARKER_HEADER, INCREMENTAL_TURN_PATH } from "../incremental-lane.js";
import { ccrPluginWireOk, storeEffectiveCcr, storeEffectiveImageRescale, type CcrSettings } from "../store.js";
import { DEFAULT_DECIDE_MAX_TOKENS } from "../nudge-decide.js";
import { BodyTooLargeError, forward, forwardUpstreamUrl, googleModelFromPath, googlePathKind, headroomEffectiveLogged, headroomFallbackLogged, headerValue, imageBillingFor, imageReserveFor, imageTokenCapFor, isCountTokensRequest, isPreflightFailFast, logRequestCost, NON_CONVERSATION_RELAY_WARN_CAP, nonConversationRelayWarned, prepareCountTokens, preflightCompressIfNeeded, readBody, resolveKnownOutputCeiling, resolveUpstream, resignSettingsFor, selfAddressedLocalPath, scrubAnthropicPck, scrubCompatDrop, vertexAnthropicModelFromPath, warnRouteMissIfNew, type Prepared } from "../server.js";
import { BodyTooLargeError, buildForwardTarget, forward, forwardUpstreamUrl, googleModelFromPath, googlePathKind, headroomEffectiveLogged, headroomFallbackLogged, headerValue, imageBillingFor, imageReserveFor, imageTokenCapFor, isCountTokensRequest, isPreflightFailFast, logRequestCost, NON_CONVERSATION_RELAY_WARN_CAP, nonConversationRelayWarned, prepareCountTokens, preflightCompressIfNeeded, readBody, resolveKnownOutputCeiling, resolveUpstream, resignSettingsFor, selfAddressedLocalPath, scrubAnthropicPck, scrubCompatDrop, vertexAnthropicModelFromPath, warnRouteMissIfNew, type Prepared } from "../server.js";
import { normalizeVertexModelId } from "../vertex.js";

// #2986: the derivedFromSessionId link is what keeps a kind child session
Expand Down Expand Up @@ -2320,11 +2320,17 @@ async function handleRequest(
// `body` keeps the original continue and the forward proceeds.
if (prepared.threadsReforge !== undefined) {
const route0 = route;
// #3569: the side-call must hit the SAME endpoint as the
// main forward — full URL incl. path/query plus the same
// proxy decision. The bare origin 404s on real providers,
// so the fold never completed.
const reforgeTarget = buildForwardTarget(req, opts, route0);
const outcome = await executeThreadsReforge(
prepared.threadsReforge,
session,
req,
(route0?.upstream ?? opts.upstream ?? "") || "",
reforgeTarget.upstreamUrl,
reforgeTarget.proxyUrl,
parsed as { model?: unknown; stream?: unknown; max_tokens?: unknown; temperature?: unknown; top_p?: unknown; messages?: unknown },
prepared.anthropicBillingBlock,
log,
Expand Down
81 changes: 69 additions & 12 deletions src/server/threads-lane.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,17 @@
//
// Failure posture is fail-open everywhere: legacy-only storage (no unified
// sidecar) captures nothing and never reforges; a failed summary side-call
// forwards the continue untouched. The lane must never be the reason a
// threads client breaks.
// forwards the continue untouched (repeat failures cool down per-session —
// a broken side-call must not re-upload the transcript every turn, #3569).
// The lane must never be the reason a threads client breaks.

import { SessionStore } from "../persist.js";
import { featureEnabled } from "../rollout.js";
import { estimateRawBodyTokens } from "../preflight.js";
import { defaultCountTokens } from "acp-kernel";
import type { Session } from "../session.js";
import { fetchWithTimeout } from "../fetch-util.js";
import { proxyDispatcher } from "../upstream-proxy.js";

/** Raw wire message as the client sent it. Stored verbatim so the reforge
* can replay tail entries byte-faithfully (shape drift between anthropic
Expand Down Expand Up @@ -98,6 +101,10 @@ const SUMMARY_TIMEOUT_MS = 120_000;
const SUMMARY_MAX_TOKENS = 1024;
/** Transcript handed to the summarizer, capped to bound the side-call. */
const TRANSCRIPT_BYTE_CAP = 400_000;
/** A session whose side-call keeps failing (dead auth, rate limit…) stops
* re-uploading its transcript on every continue: the next attempt lands
* after this window instead (#3569). */
const SUMMARY_RETRY_COOLDOWN_MS = 10 * 60_000;

const SUMMARIZER_SYSTEM = [
"You are compressing an older slice of a coding-agent conversation so it can be replaced by your summary.",
Expand All @@ -118,6 +125,8 @@ function freshMeta(): ThreadsMirrorMeta {

let mirrorStore: SessionStore | undefined;
const warnedSessions = new Set<string>();
/** sessionId → epoch-ms until which the summary side-call is suppressed. */
const reforgeCooldownUntil = new Map<string, number>();

function laneStore(): SessionStore {
if (!mirrorStore) mirrorStore = new SessionStore({ enabled: true });
Expand All @@ -128,6 +137,7 @@ export function __resetThreadsLaneForTests(): void {
mirrorStore?.closeQuietly();
mirrorStore = undefined;
warnedSessions.clear();
reforgeCooldownUntil.clear();
}

export function isThreadsBody(parsed: unknown): boolean {
Expand Down Expand Up @@ -342,21 +352,62 @@ interface ThreadsReforgeOutcome {
body?: string;
}

/** Execute an armed reforge: one summarizer side-call (client's auth, same
* upstream), then swap the continue for a fresh create. Fail-open — any
* error returns ok:false and the caller forwards the original continue. */
function urlPathOf(url: string): string {
try {
const u = new URL(url);
return u.pathname + u.search;
} catch {
return "<bad-url>";
}
}

/** Pull `error.type`/`error.message` out of a failed side-call body so the
* warn log says WHY (e.g. `not_found_error`), not just the status (#3569). */
async function sideCallErrorDetail(resp: Response): Promise<string> {
try {
const text = await resp.text();
if (!text || text.length > 8192) return "";
const json = JSON.parse(text) as { type?: unknown; message?: unknown; error?: { type?: unknown; message?: unknown } };
const t = typeof json.error?.type === "string" ? json.error.type : typeof json.type === "string" ? json.type : undefined;
const m = typeof json.error?.message === "string" ? json.error.message : typeof json.message === "string" ? json.message : undefined;
return [t !== undefined ? `error.type=${t}` : "", m !== undefined ? `error.message=${m.slice(0, 200)}` : ""].filter(Boolean).join(" ");
} catch {
return "";
}
}

function armReforgeCooldown(sessionId: string): void {
const now = Date.now();
reforgeCooldownUntil.set(sessionId, now + SUMMARY_RETRY_COOLDOWN_MS);
const staleBefore = now - SUMMARY_RETRY_COOLDOWN_MS;
for (const [id, until] of reforgeCooldownUntil) if (id !== sessionId && until < staleBefore) reforgeCooldownUntil.delete(id);
}

/** Execute an armed reforge: one summarizer side-call (client's auth, aimed
* at the SAME endpoint + proxy decision as the main forward — #3569), then
* swap the continue for a fresh create. Fail-open — any error returns
* ok:false and the caller forwards the original continue; repeat failures
* arm a per-session cooldown so a broken session stops re-uploading its
* transcript on every turn. */
export async function executeThreadsReforge(
plan: ThreadsReforgePlan,
session: Session,
req: { headers: Record<string, string | string[] | undefined> },
upstreamUrl: string,
proxyUrl: string | undefined,
parsed: { model?: unknown; stream?: unknown; max_tokens?: unknown; temperature?: unknown; top_p?: unknown; messages?: unknown },
billingBlock: { type: "text"; text: string } | undefined,
log: (level: string, msg: string) => void,
): Promise<ThreadsReforgeOutcome> {
const { sessionId, entries, meta, windowTokens } = plan;
const cut = ringCutIndex(meta, entries);
if (cut === undefined) return { ok: false };
const now = Date.now();
const coolUntil = reforgeCooldownUntil.get(sessionId);
if (coolUntil !== undefined && now < coolUntil) {
log("debug", `[${sessionId}] [threads] summary side-call cooling down (${Math.ceil((coolUntil - now) / 1000)}s left) — forwarding continue unchanged (#3569)`);
return { ok: false };
}
try {
const summaryReq = buildSummaryRequestBody(meta.shape, typeof parsed.model === "string" ? parsed.model : undefined, meta, entries, cut, billingBlock);
const headers: Record<string, string> = { "content-type": "application/json" };
Expand All @@ -368,23 +419,27 @@ export async function executeThreadsReforge(
passThrough("authorization");
passThrough("anthropic-version");
passThrough("anthropic-beta");
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), SUMMARY_TIMEOUT_MS);
// Same outbound-fetch contract as every other bili call (#551 idle
// watchdog, direct-dispatcher fallback, manual redirects); the proxy
// decision rides the dispatcher exactly like the main forward does.
let summaryText: string | undefined;
const { response: resp, clearTimer } = await fetchWithTimeout(upstreamUrl, { method: "POST", headers, body: JSON.stringify(summaryReq), ...(proxyUrl ? { dispatcher: proxyDispatcher(proxyUrl) } : {}) }, SUMMARY_TIMEOUT_MS);
try {
const resp = await fetch(upstreamUrl, { method: "POST", headers, body: JSON.stringify(summaryReq), signal: controller.signal });
if (!resp.ok) {
log("warn", `[${sessionId}] [threads] summary side-call failed (HTTP ${resp.status}) — forwarding continue unchanged (#2926)`);
const detail = await sideCallErrorDetail(resp);
armReforgeCooldown(sessionId);
log("warn", `[${sessionId}] [threads] summary side-call failed (HTTP ${resp.status}${detail ? ` ${detail}` : ""}) → ${urlPathOf(upstreamUrl)} — forwarding continue unchanged (#2926/#3569)`);
return { ok: false };
}
const json = (await resp.json()) as { content?: { type?: string; text?: unknown }[] };
const textBlock = json.content?.find((b) => b.type === "text");
summaryText = typeof textBlock?.text === "string" ? textBlock.text : undefined;
} finally {
clearTimeout(timer);
clearTimer();
}
if (summaryText === undefined || summaryText.trim() === "") {
log("warn", `[${sessionId}] [threads] summary side-call returned no text — forwarding continue unchanged (#2926)`);
armReforgeCooldown(sessionId);
log("warn", `[${sessionId}] [threads] summary side-call returned no text (${urlPathOf(upstreamUrl)}) — forwarding continue unchanged (#2926/#3569)`);
return { ok: false };
}
const shape = meta.shape ?? {};
Expand All @@ -408,11 +463,13 @@ export async function executeThreadsReforge(
// only the summary + tail ring from now on.
const newMeta: ThreadsMirrorMeta = { ...meta, foldedBefore: cut, foldedSummary: summaryText, count: entries.length };
persistMirror(laneStore(), sessionId, newMeta, entries, log);
reforgeCooldownUntil.delete(sessionId);
const est = estimateThreadsTokens(newMeta, entries);
log("info", `[${sessionId}] [threads] reforge: continue swapped for a fresh create (folded ${cut} entries, mirror est ${est} tok vs window ${windowTokens}) (#2926)`);
return { ok: true, body: JSON.stringify(createBody) };
} catch (err) {
log("warn", `[${sessionId}] [threads] reforge aborted (${String(err)}) — forwarding continue unchanged (#2926)`);
armReforgeCooldown(sessionId);
log("warn", `[${sessionId}] [threads] reforge aborted (${String(err)}; ${urlPathOf(upstreamUrl)}) — forwarding continue unchanged (#2926/#3569)`);
return { ok: false };
}
}
35 changes: 35 additions & 0 deletions tests/threads-lane.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ interface Harness {
proxyUrl: string;
post: (body: Record<string, unknown>, sid?: string) => Promise<{ status: number; body: unknown }>;
forwarded: Record<string, unknown>[];
forwardedPaths: string[];
readMirror: (sid: string) => { meta?: ThreadsMirrorMeta; error?: string };
close: () => Promise<void>;
}
Expand All @@ -63,13 +64,15 @@ async function harness(storageMode: "full" | "legacy", opts: HarnessOpts = {}):
const store = new SessionStore({ debounceMs: 60000, enabled: true });
_setStoreForTest(store);
const forwarded: Record<string, unknown>[] = [];
const forwardedPaths: string[] = [];
let sideCalls = 0;
const upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => chunks.push(chunk));
req.on("end", () => {
const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
forwarded.push(body);
forwardedPaths.push(req.url ?? "/");
const stream = body.stream === true;
const isSideCall = !stream && body.thread === undefined;
if (isSideCall) sideCalls++;
Expand Down Expand Up @@ -116,6 +119,7 @@ async function harness(storageMode: "full" | "legacy", opts: HarnessOpts = {}):
proxyUrl,
post,
forwarded,
forwardedPaths,
readMirror,
close: async () => {
await new Promise<void>((resolve) => proxy.close(() => resolve()));
Expand Down Expand Up @@ -212,6 +216,7 @@ test("threads lane: over-window continue is reforged into a fresh create (summar
const sideCall = h.forwarded[1] as { stream?: boolean; thread?: unknown; messages?: { role: string; content: unknown }[]; system?: { type: string; text: string }[] };
assert.equal(sideCall.stream, false, "summary side-call is non-stream");
assert.equal(sideCall.thread, undefined, "side-call is a plain message request, no thread field");
assert.equal(h.forwardedPaths[1], "/v1/messages", "#3569: the side-call must land on the chat endpoint, not the bare origin");
const reforged = h.forwarded[2] as { thread?: { type?: string }; system?: unknown; tools?: unknown; messages?: { role: string; content: unknown }[] };
assert.equal(reforged.thread?.type, "create", "continue was swapped for a fresh create");
assert.deepEqual(reforged.system, CLIENT_SYSTEM, "client system copied byte-faithfully");
Expand Down Expand Up @@ -327,6 +332,36 @@ test("threads lane: summary side-call failure fails open (continue forwarded unc
}
});

test("threads lane: repeated side-call failures cool down instead of re-uploading every continue (#3569)", async () => {
process.env.BILI_ROLLOUT_ANTHROPIC_THREADS_REFORGE = "1";
const h = await harness("full", { window: 5000, failFirstSideCall: true });
try {
await h.post(createBody([
{ role: "user", content: turnText(1, 400) }, { role: "assistant", content: "a1" },
{ role: "user", content: turnText(2, 400) }, { role: "assistant", content: "a2" },
{ role: "user", content: turnText(3, 400) }, { role: "assistant", content: "a3" },
]));
// First over-window continue: side-call fails (500), continue forwards.
const before = h.forwarded.length;
const r1 = await h.post(continueBody([{ role: "user", content: turnText(4, 400) }], "msg_1"));
assert.equal(r1.status, 200);
assert.equal(h.forwarded.length, before + 2, "side-call attempted, then the original continue forwarded");
// The mirror is still over the window (no fold landed), so this
// continue would arm the reforge again — the per-session cooldown
// must suppress the doomed retry instead of re-uploading everything.
const before2 = h.forwarded.length;
const r2 = await h.post(continueBody([{ role: "user", content: turnText(5, 400) }], "msg_3"));
assert.equal(r2.status, 200);
assert.equal(h.forwarded.length, before2 + 1, "cooldown active: no second side-call within the window");
const next = h.forwarded.at(-1) as { thread?: { type?: string }; messages?: unknown[] };
assert.equal(next.thread?.type, "continue", "fail-open still holds while cooling down");
assert.equal(next.messages?.length, 1);
} finally {
delete process.env.BILI_ROLLOUT_ANTHROPIC_THREADS_REFORGE;
await h.close();
}
});

test("threads lane: a re-create under the same session id resets bili's own fold tracking", async () => {
process.env.BILI_ROLLOUT_ANTHROPIC_THREADS_REFORGE = "1";
const h = await harness("full", { window: 5000 });
Expand Down
Loading