diff --git a/src/server/handle.ts b/src/server/handle.ts index de34135e3f..0985046893 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -66,7 +66,7 @@ import { adminGateDenial, handleAdminRoute, isManagementPath } from "./admin.js" import { handleIncrementalTurn, handleIncrementalProbe, incrementalTurnBodyShape, incrementalTurnsEnabled, INCREMENTAL_TURN_MARKER_HEADER, INCREMENTAL_TURN_PATH } from "../incremental-lane.js"; import { ccrPluginWireOk, storeEffectiveCcr, storeEffectiveImageRescale, type CcrSettings } from "../store.js"; import { DEFAULT_DECIDE_MAX_TOKENS } from "../nudge-decide.js"; -import { BodyTooLargeError, forward, forwardUpstreamUrl, googleModelFromPath, googlePathKind, headroomEffectiveLogged, headroomFallbackLogged, headerValue, imageBillingFor, imageReserveFor, imageTokenCapFor, isCountTokensRequest, isPreflightFailFast, logRequestCost, NON_CONVERSATION_RELAY_WARN_CAP, nonConversationRelayWarned, prepareCountTokens, preflightCompressIfNeeded, readBody, resolveKnownOutputCeiling, resolveUpstream, resignSettingsFor, selfAddressedLocalPath, scrubAnthropicPck, scrubCompatDrop, vertexAnthropicModelFromPath, warnRouteMissIfNew, type Prepared } from "../server.js"; +import { BodyTooLargeError, buildForwardTarget, forward, forwardUpstreamUrl, googleModelFromPath, googlePathKind, headroomEffectiveLogged, headroomFallbackLogged, headerValue, imageBillingFor, imageReserveFor, imageTokenCapFor, isCountTokensRequest, isPreflightFailFast, logRequestCost, NON_CONVERSATION_RELAY_WARN_CAP, nonConversationRelayWarned, prepareCountTokens, preflightCompressIfNeeded, readBody, resolveKnownOutputCeiling, resolveUpstream, resignSettingsFor, selfAddressedLocalPath, scrubAnthropicPck, scrubCompatDrop, vertexAnthropicModelFromPath, warnRouteMissIfNew, type Prepared } from "../server.js"; import { normalizeVertexModelId } from "../vertex.js"; // #2986: the derivedFromSessionId link is what keeps a kind child session @@ -2320,11 +2320,17 @@ async function handleRequest( // `body` keeps the original continue and the forward proceeds. if (prepared.threadsReforge !== undefined) { const route0 = route; + // #3569: the side-call must hit the SAME endpoint as the + // main forward — full URL incl. path/query plus the same + // proxy decision. The bare origin 404s on real providers, + // so the fold never completed. + const reforgeTarget = buildForwardTarget(req, opts, route0); const outcome = await executeThreadsReforge( prepared.threadsReforge, session, req, - (route0?.upstream ?? opts.upstream ?? "") || "", + reforgeTarget.upstreamUrl, + reforgeTarget.proxyUrl, parsed as { model?: unknown; stream?: unknown; max_tokens?: unknown; temperature?: unknown; top_p?: unknown; messages?: unknown }, prepared.anthropicBillingBlock, log, diff --git a/src/server/threads-lane.ts b/src/server/threads-lane.ts index bd592401b3..0218e2557e 100644 --- a/src/server/threads-lane.ts +++ b/src/server/threads-lane.ts @@ -33,14 +33,17 @@ // // Failure posture is fail-open everywhere: legacy-only storage (no unified // sidecar) captures nothing and never reforges; a failed summary side-call -// forwards the continue untouched. The lane must never be the reason a -// threads client breaks. +// forwards the continue untouched (repeat failures cool down per-session — +// a broken side-call must not re-upload the transcript every turn, #3569). +// The lane must never be the reason a threads client breaks. import { SessionStore } from "../persist.js"; import { featureEnabled } from "../rollout.js"; import { estimateRawBodyTokens } from "../preflight.js"; import { defaultCountTokens } from "acp-kernel"; import type { Session } from "../session.js"; +import { fetchWithTimeout } from "../fetch-util.js"; +import { proxyDispatcher } from "../upstream-proxy.js"; /** Raw wire message as the client sent it. Stored verbatim so the reforge * can replay tail entries byte-faithfully (shape drift between anthropic @@ -98,6 +101,10 @@ const SUMMARY_TIMEOUT_MS = 120_000; const SUMMARY_MAX_TOKENS = 1024; /** Transcript handed to the summarizer, capped to bound the side-call. */ const TRANSCRIPT_BYTE_CAP = 400_000; +/** A session whose side-call keeps failing (dead auth, rate limit…) stops + * re-uploading its transcript on every continue: the next attempt lands + * after this window instead (#3569). */ +const SUMMARY_RETRY_COOLDOWN_MS = 10 * 60_000; const SUMMARIZER_SYSTEM = [ "You are compressing an older slice of a coding-agent conversation so it can be replaced by your summary.", @@ -118,6 +125,8 @@ function freshMeta(): ThreadsMirrorMeta { let mirrorStore: SessionStore | undefined; const warnedSessions = new Set(); +/** sessionId → epoch-ms until which the summary side-call is suppressed. */ +const reforgeCooldownUntil = new Map(); function laneStore(): SessionStore { if (!mirrorStore) mirrorStore = new SessionStore({ enabled: true }); @@ -128,6 +137,7 @@ export function __resetThreadsLaneForTests(): void { mirrorStore?.closeQuietly(); mirrorStore = undefined; warnedSessions.clear(); + reforgeCooldownUntil.clear(); } export function isThreadsBody(parsed: unknown): boolean { @@ -342,14 +352,49 @@ interface ThreadsReforgeOutcome { body?: string; } -/** Execute an armed reforge: one summarizer side-call (client's auth, same - * upstream), then swap the continue for a fresh create. Fail-open — any - * error returns ok:false and the caller forwards the original continue. */ +function urlPathOf(url: string): string { + try { + const u = new URL(url); + return u.pathname + u.search; + } catch { + return ""; + } +} + +/** Pull `error.type`/`error.message` out of a failed side-call body so the + * warn log says WHY (e.g. `not_found_error`), not just the status (#3569). */ +async function sideCallErrorDetail(resp: Response): Promise { + try { + const text = await resp.text(); + if (!text || text.length > 8192) return ""; + const json = JSON.parse(text) as { type?: unknown; message?: unknown; error?: { type?: unknown; message?: unknown } }; + const t = typeof json.error?.type === "string" ? json.error.type : typeof json.type === "string" ? json.type : undefined; + const m = typeof json.error?.message === "string" ? json.error.message : typeof json.message === "string" ? json.message : undefined; + return [t !== undefined ? `error.type=${t}` : "", m !== undefined ? `error.message=${m.slice(0, 200)}` : ""].filter(Boolean).join(" "); + } catch { + return ""; + } +} + +function armReforgeCooldown(sessionId: string): void { + const now = Date.now(); + reforgeCooldownUntil.set(sessionId, now + SUMMARY_RETRY_COOLDOWN_MS); + const staleBefore = now - SUMMARY_RETRY_COOLDOWN_MS; + for (const [id, until] of reforgeCooldownUntil) if (id !== sessionId && until < staleBefore) reforgeCooldownUntil.delete(id); +} + +/** Execute an armed reforge: one summarizer side-call (client's auth, aimed + * at the SAME endpoint + proxy decision as the main forward — #3569), then + * swap the continue for a fresh create. Fail-open — any error returns + * ok:false and the caller forwards the original continue; repeat failures + * arm a per-session cooldown so a broken session stops re-uploading its + * transcript on every turn. */ export async function executeThreadsReforge( plan: ThreadsReforgePlan, session: Session, req: { headers: Record }, upstreamUrl: string, + proxyUrl: string | undefined, parsed: { model?: unknown; stream?: unknown; max_tokens?: unknown; temperature?: unknown; top_p?: unknown; messages?: unknown }, billingBlock: { type: "text"; text: string } | undefined, log: (level: string, msg: string) => void, @@ -357,6 +402,12 @@ export async function executeThreadsReforge( const { sessionId, entries, meta, windowTokens } = plan; const cut = ringCutIndex(meta, entries); if (cut === undefined) return { ok: false }; + const now = Date.now(); + const coolUntil = reforgeCooldownUntil.get(sessionId); + if (coolUntil !== undefined && now < coolUntil) { + log("debug", `[${sessionId}] [threads] summary side-call cooling down (${Math.ceil((coolUntil - now) / 1000)}s left) — forwarding continue unchanged (#3569)`); + return { ok: false }; + } try { const summaryReq = buildSummaryRequestBody(meta.shape, typeof parsed.model === "string" ? parsed.model : undefined, meta, entries, cut, billingBlock); const headers: Record = { "content-type": "application/json" }; @@ -368,23 +419,27 @@ export async function executeThreadsReforge( passThrough("authorization"); passThrough("anthropic-version"); passThrough("anthropic-beta"); - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), SUMMARY_TIMEOUT_MS); + // Same outbound-fetch contract as every other bili call (#551 idle + // watchdog, direct-dispatcher fallback, manual redirects); the proxy + // decision rides the dispatcher exactly like the main forward does. let summaryText: string | undefined; + const { response: resp, clearTimer } = await fetchWithTimeout(upstreamUrl, { method: "POST", headers, body: JSON.stringify(summaryReq), ...(proxyUrl ? { dispatcher: proxyDispatcher(proxyUrl) } : {}) }, SUMMARY_TIMEOUT_MS); try { - const resp = await fetch(upstreamUrl, { method: "POST", headers, body: JSON.stringify(summaryReq), signal: controller.signal }); if (!resp.ok) { - log("warn", `[${sessionId}] [threads] summary side-call failed (HTTP ${resp.status}) — forwarding continue unchanged (#2926)`); + const detail = await sideCallErrorDetail(resp); + armReforgeCooldown(sessionId); + log("warn", `[${sessionId}] [threads] summary side-call failed (HTTP ${resp.status}${detail ? ` ${detail}` : ""}) → ${urlPathOf(upstreamUrl)} — forwarding continue unchanged (#2926/#3569)`); return { ok: false }; } const json = (await resp.json()) as { content?: { type?: string; text?: unknown }[] }; const textBlock = json.content?.find((b) => b.type === "text"); summaryText = typeof textBlock?.text === "string" ? textBlock.text : undefined; } finally { - clearTimeout(timer); + clearTimer(); } if (summaryText === undefined || summaryText.trim() === "") { - log("warn", `[${sessionId}] [threads] summary side-call returned no text — forwarding continue unchanged (#2926)`); + armReforgeCooldown(sessionId); + log("warn", `[${sessionId}] [threads] summary side-call returned no text (${urlPathOf(upstreamUrl)}) — forwarding continue unchanged (#2926/#3569)`); return { ok: false }; } const shape = meta.shape ?? {}; @@ -408,11 +463,13 @@ export async function executeThreadsReforge( // only the summary + tail ring from now on. const newMeta: ThreadsMirrorMeta = { ...meta, foldedBefore: cut, foldedSummary: summaryText, count: entries.length }; persistMirror(laneStore(), sessionId, newMeta, entries, log); + reforgeCooldownUntil.delete(sessionId); const est = estimateThreadsTokens(newMeta, entries); log("info", `[${sessionId}] [threads] reforge: continue swapped for a fresh create (folded ${cut} entries, mirror est ${est} tok vs window ${windowTokens}) (#2926)`); return { ok: true, body: JSON.stringify(createBody) }; } catch (err) { - log("warn", `[${sessionId}] [threads] reforge aborted (${String(err)}) — forwarding continue unchanged (#2926)`); + armReforgeCooldown(sessionId); + log("warn", `[${sessionId}] [threads] reforge aborted (${String(err)}; ${urlPathOf(upstreamUrl)}) — forwarding continue unchanged (#2926/#3569)`); return { ok: false }; } } diff --git a/tests/threads-lane.test.ts b/tests/threads-lane.test.ts index bda495aa83..39b22df8c9 100644 --- a/tests/threads-lane.test.ts +++ b/tests/threads-lane.test.ts @@ -48,6 +48,7 @@ interface Harness { proxyUrl: string; post: (body: Record, sid?: string) => Promise<{ status: number; body: unknown }>; forwarded: Record[]; + forwardedPaths: string[]; readMirror: (sid: string) => { meta?: ThreadsMirrorMeta; error?: string }; close: () => Promise; } @@ -63,6 +64,7 @@ async function harness(storageMode: "full" | "legacy", opts: HarnessOpts = {}): const store = new SessionStore({ debounceMs: 60000, enabled: true }); _setStoreForTest(store); const forwarded: Record[] = []; + const forwardedPaths: string[] = []; let sideCalls = 0; const upstream = http.createServer((req, res) => { const chunks: Buffer[] = []; @@ -70,6 +72,7 @@ async function harness(storageMode: "full" | "legacy", opts: HarnessOpts = {}): req.on("end", () => { const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; forwarded.push(body); + forwardedPaths.push(req.url ?? "/"); const stream = body.stream === true; const isSideCall = !stream && body.thread === undefined; if (isSideCall) sideCalls++; @@ -116,6 +119,7 @@ async function harness(storageMode: "full" | "legacy", opts: HarnessOpts = {}): proxyUrl, post, forwarded, + forwardedPaths, readMirror, close: async () => { await new Promise((resolve) => proxy.close(() => resolve())); @@ -212,6 +216,7 @@ test("threads lane: over-window continue is reforged into a fresh create (summar const sideCall = h.forwarded[1] as { stream?: boolean; thread?: unknown; messages?: { role: string; content: unknown }[]; system?: { type: string; text: string }[] }; assert.equal(sideCall.stream, false, "summary side-call is non-stream"); assert.equal(sideCall.thread, undefined, "side-call is a plain message request, no thread field"); + assert.equal(h.forwardedPaths[1], "/v1/messages", "#3569: the side-call must land on the chat endpoint, not the bare origin"); const reforged = h.forwarded[2] as { thread?: { type?: string }; system?: unknown; tools?: unknown; messages?: { role: string; content: unknown }[] }; assert.equal(reforged.thread?.type, "create", "continue was swapped for a fresh create"); assert.deepEqual(reforged.system, CLIENT_SYSTEM, "client system copied byte-faithfully"); @@ -327,6 +332,36 @@ test("threads lane: summary side-call failure fails open (continue forwarded unc } }); +test("threads lane: repeated side-call failures cool down instead of re-uploading every continue (#3569)", async () => { + process.env.BILI_ROLLOUT_ANTHROPIC_THREADS_REFORGE = "1"; + const h = await harness("full", { window: 5000, failFirstSideCall: true }); + try { + await h.post(createBody([ + { role: "user", content: turnText(1, 400) }, { role: "assistant", content: "a1" }, + { role: "user", content: turnText(2, 400) }, { role: "assistant", content: "a2" }, + { role: "user", content: turnText(3, 400) }, { role: "assistant", content: "a3" }, + ])); + // First over-window continue: side-call fails (500), continue forwards. + const before = h.forwarded.length; + const r1 = await h.post(continueBody([{ role: "user", content: turnText(4, 400) }], "msg_1")); + assert.equal(r1.status, 200); + assert.equal(h.forwarded.length, before + 2, "side-call attempted, then the original continue forwarded"); + // The mirror is still over the window (no fold landed), so this + // continue would arm the reforge again — the per-session cooldown + // must suppress the doomed retry instead of re-uploading everything. + const before2 = h.forwarded.length; + const r2 = await h.post(continueBody([{ role: "user", content: turnText(5, 400) }], "msg_3")); + assert.equal(r2.status, 200); + assert.equal(h.forwarded.length, before2 + 1, "cooldown active: no second side-call within the window"); + const next = h.forwarded.at(-1) as { thread?: { type?: string }; messages?: unknown[] }; + assert.equal(next.thread?.type, "continue", "fail-open still holds while cooling down"); + assert.equal(next.messages?.length, 1); + } finally { + delete process.env.BILI_ROLLOUT_ANTHROPIC_THREADS_REFORGE; + await h.close(); + } +}); + test("threads lane: a re-create under the same session id resets bili's own fold tracking", async () => { process.env.BILI_ROLLOUT_ANTHROPIC_THREADS_REFORGE = "1"; const h = await harness("full", { window: 5000 });