diff --git a/reference/architecture.md b/reference/architecture.md index f02e4afefa..477af01d9e 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -304,6 +304,7 @@ billion-context/ │ ├── remote-notices.ts # Remote announcement feed watcher: fail-open polling, last-known-good snapshot (#5548) │ ├── request-membership.ts # Settle-side membership probe: is this usage report ours (#4805) │ ├── resolver-wedge.ts # Persistent local DNS resolver wedge detector + alarm (#5221) +│ ├── responses-system-sections.ts # Single source of truth for the Responses-wire developer-section join: steady prepare + round-2 rebuild share one separator (#6922) │ ├── responses-tool-output.ts # Responses tool-output image parts: projection + patch helpers │ ├── rollout.ts # Feature rollout: per-install uuid, basis-point buckets, nested cohorts (#3062) │ ├── runaway-guard.ts # Intrinsic stream terminator for runaway tag/codepoint enumeration (#2346) diff --git a/src/config.ts b/src/config.ts index ef473e7a98..4911799b59 100644 --- a/src/config.ts +++ b/src/config.ts @@ -697,6 +697,12 @@ const CONTEXT_LIMIT_TABLE: Array<{ match: RegExp; limit: number }> = [ { match: /^gpt-4o/i, limit: 128_000 }, { match: /^gpt-4-turbo/i, limit: 128_000 }, { match: /^o[13]-/i, limit: 200_000 }, + // Antigravity (CloudCode) model line: gemini-4-* plus the + // orchestrator/agent/worker/latest/lite ids served on cloudcode-pa.googleapis.com + // all carry the 1M window (#6963) — the 200K fallback clamped + // maxOutputTokens to the 4K floor at ~168K input and stranded turns at ~227K. + { match: /^gemini-4/i, limit: 1_048_576 }, + { match: /^gemini-(?:pro|flash)-(?:orchestrator|worker|agent|latest|lite)/i, limit: 1_048_576 }, { match: /^gemini-3/i, limit: 1_048_576 }, { match: /^gemini-2\.5/i, limit: 1_000_000 }, { match: /^gemini-1\.5/i, limit: 1_000_000 }, diff --git a/src/server/handle.ts b/src/server/handle.ts index 11dbc5d5b5..17018b55da 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -67,7 +67,7 @@ import { DecompressedTooLargeError, decodeRequestBody, IngestBodyTooLargeError } import { matchModelEndpoint } from "../model-endpoints.js"; import { autoModeForgeEnabled, dshAuthoredCompactionEnabled, imageRescaleEnabled, limitsDecodedBytes as knobLimitsDecodedBytes, limitsIngestBytes as knobLimitsIngestBytes, noInjectTool as knobNoInjectTool, pluginBypassOverride, rawDumpDir as knobRawDumpDir } from "../knobs.js"; import { buildForgeResponse, classifierForgeVariant, estimateCjkTokens, isAutoModeClassifier, requestText, summarizeAction } from "../auto-mode-forge.js"; -import { extractConversationId, isV1BackgroundSideRequest, isV1InternalRequest, unwrapV1Internal } from "../v1internal.js"; +import { extractConversationId, findV1Continuation, isV1BackgroundSideRequest, isV1InternalRequest, spliceV1ContinuationInner, unwrapV1Internal } from "../v1internal.js"; import { anthropicBetaContextWindow, anthropicBetaMessageThreads, BILI_HOP_HEADER, capRegistryWindowByStandard, expandedContextSuffixWindow, launcherContextWindow, launcherMaxOutput, noteUnresolvedModelSession, UNRESOLVED_MODEL_SESSION_THRESHOLD, windowSourceLogged } from "./context-window.js"; import { NO_IDENTITY_MESSAGE, safeSessionId } from "./headers.js"; import { demoteGate, hasLeakedBiliToolsOnly, hasSingleShotBudget, isAudioOutputCall, isClaudeCodeTitleRequest, isDshAutoReviewRequest, isDshSingleShotSidecar, isDshTitleRequest, isResponsesServerToolSingleShot, isServerToolUtilityCall, isSideRequest, isStructuredOutputCall, resolveSideLane, restoreOutputBudget, sideRequestGuard, stripLeakedBiliTools } from "./side-request.js"; @@ -3028,6 +3028,47 @@ async function handleRequest( respondFailFast: boolean, overflowWindow?: number, ) => Promise<{ body: string | Buffer; prepared: Prepared | null } | null>) | undefined = async (respondFailFast, overflowWindow) => { + // #6963: a v1internal continuation leg (continuationToken present) + // must reach upstream BYTE-IDENTICAL to the input the token was + // minted for — bili's own leg-1 egress. Re-running the stateful + // pipeline (nudge / status line / ACP tag re-render / output + // clamp) mutates the body and upstream 400s the leg ("does not + // match the continuation token"). Replay the cached egress inner, + // splicing back only the client's fresh maxOutputTokens and the + // token itself. Cache miss (restart / first leg) falls through to + // the full pipeline — pre-fix behavior. + const v1ParsedInner = (typeof parsed === "object" && parsed !== null ? parsed : null) as Record | null; + const v1Cont = v1Meta !== undefined && protocol === "google" && !countTokens && v1ParsedInner !== null + ? findV1Continuation(v1Meta, v1ParsedInner) + : undefined; + if (v1Cont !== undefined) { + const v1Replay = session.transient?.v1ContinuationReplay; + let replayInner: Record | undefined; + if (v1Replay !== undefined && v1ParsedInner !== null) { + try { + replayInner = spliceV1ContinuationInner(JSON.parse(v1Replay.body) as Record, v1ParsedInner, v1Cont); + } catch { + log("warn", `[${session.id}] v1internal continuation replay: cached egress inner is not valid JSON — running full pipeline`); + } + } + if (replayInner !== undefined && v1Replay !== undefined) { + const outbound = JSON.stringify(replayInner); + log("info", `[${session.id}] v1internal continuation leg: replaying cached egress inner (${v1Replay.body.length} bytes, age ${Date.now() - v1Replay.at}ms), pipeline skipped`); + const replayPrepared: Prepared = { + body: outbound, + parsedBody: replayInner, + session, + processedMessages: [], + originalMessages: [], + protocol: "google", + stream: googlePathKind(urlPath) === "stream-generate", + compressInjected: false, + model: v1Model ?? googleModel, + }; + return { body: outbound, prepared: replayPrepared }; + } + log("warn", `[${session.id}] v1internal continuation leg without usable cached egress inner (restart?) — running full pipeline; upstream may reject the leg`); + } const runPrepareInner = async (): Promise => { // #1820: this prepare consumes one unit of post-rebuild anchor // validity (the last one deletes it — see session.ts). diff --git a/src/server/relay.ts b/src/server/relay.ts index 0bc270a470..b5acf45384 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -426,10 +426,24 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // Antigravity inside the CloudCode envelope it arrived in. String-level for // the direct fetch sites; object-level via wireTransform below for loop re-sends. const v1Active = v1Meta !== undefined && prepared !== null; + // #6963: a continuationToken is bound to the EXACT inner bytes upstream + // last saw, so every v1 egress records its inner on the session (latest- + // wins, in-memory only) for the continuation leg to replay. countTokens + // egress never mints continuation tokens and must not clobber the cache. + const v1CountTokensEgress = typeof req.url === "string" && req.url.split("?")[0]?.endsWith("/v1internal:countTokens"); + const noteV1EgressInner = (innerJson: string): void => { + if (v1CountTokensEgress) return; + const s = prepared; + if (s === null) return; + const t = s.session.transient ??= {}; + t.v1ContinuationReplay = { body: innerJson, at: Date.now() }; + }; const v1Wrap = (b: string | Buffer): string | Buffer => { if (!v1Active || typeof b !== "string") return b; try { - return JSON.stringify(wrapV1Internal(JSON.parse(b) as Record, v1Meta)); + const inner = JSON.parse(b) as Record; + noteV1EgressInner(b); + return JSON.stringify(wrapV1Internal(inner, v1Meta)); } catch { return b; } @@ -785,7 +799,10 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare if (steerCfg && steerCfg.enabled && steerProtocol) applyOutputSteeringJson(out, steerProtocol, steerCfg); if (compatDropPaths.length > 0) dropCompatFieldsJson(out, compatDropPaths); // #3129: re-nest the rebuilt inner request into the CloudCode envelope. - out = v1Active ? wrapV1Internal(out, v1Meta) : out; + if (v1Active) { + noteV1EgressInner(JSON.stringify(out)); + out = wrapV1Internal(out, v1Meta); + } if (ccMetaForResend !== undefined) out = rewrapCommandcodeBody(out, ccMetaForResend); return out; } diff --git a/src/session.ts b/src/session.ts index f518aa9ca0..8395722c5b 100644 --- a/src/session.ts +++ b/src/session.ts @@ -366,6 +366,14 @@ export type Session = { * the TTL lapses. In-memory only — a restart replants nothing, and a * restart also drops the truncated stream it described. */ truncatedUpstream?: { at: number; protocol: string; cause: "eof" | "read-error" }; + /** #6963: the exact inner request bytes the last v1internal egress + * carried — what upstream minted any continuationToken for. A + * continuation leg replays them byte-identical (splicing back only + * the fresh maxOutputTokens and the token): re-running the stateful + * pipeline would mutate the body and upstream 400s the leg ("does + * not match the continuation token"). Latest-wins, in-memory only — + * a restart lapses to the pre-fix pipeline behavior (cache miss). */ + v1ContinuationReplay?: { body: string; at: number }; }; id: string; /** Identity / descriptive metadata. Populated on first request. */ diff --git a/src/v1internal.ts b/src/v1internal.ts index 1199e62497..b82794009e 100644 --- a/src/v1internal.ts +++ b/src/v1internal.ts @@ -100,6 +100,76 @@ export function isV1BackgroundSideRequest(meta: Record, inner: return model.includes("flash-lite"); } +// #6963: a MAX_TOKENS-cut turn is continued by resending the SAME input plus a +// `continuationToken` — upstream binds the token to the exact bytes it saw on +// the minting leg and 400s any deviation ("pass the exact same input and +// conversation history as the initial request"). The token may ride the +// envelope meta, the inner request root, or inner.generationConfig, in +// camelCase or snake_case. +type V1Continuation = { location: "meta" | "inner" | "generationConfig"; key: string; value: string }; + +const CONTINUATION_TOKEN_KEYS = ["continuationToken", "continuation_token"] as const; + +function readContinuationToken(obj: Record): { key: string; value: string } | undefined { + for (const key of CONTINUATION_TOKEN_KEYS) { + const v = obj[key]; + if (typeof v === "string" && v.length > 0) return { key, value: v }; + } + return undefined; +} + +export function findV1Continuation(meta: Record, inner: Record): V1Continuation | undefined { + const atMeta = readContinuationToken(meta); + if (atMeta !== undefined) return { location: "meta", ...atMeta }; + const atInner = readContinuationToken(inner); + if (atInner !== undefined) return { location: "inner", ...atInner }; + const gc = inner.generationConfig; + if (gc && typeof gc === "object" && !Array.isArray(gc)) { + const atGc = readContinuationToken(gc as Record); + if (atGc !== undefined) return { location: "generationConfig", ...atGc }; + } + return undefined; +} + +// Rebuild the continuation leg's inner from the cached minting-leg egress inner: +// contents / systemInstruction / tools / sessionId / the rest of +// generationConfig stay byte-identical to what upstream minted the token for; +// only the client's fresh maxOutputTokens (accumulated across legs — upstream +// tells the client to pass a HIGHER value) and the token itself are spliced +// back. A token located in the envelope meta is not spliced here: the relay +// re-wraps with the incoming leg's own meta, which already carries it. +export function spliceV1ContinuationInner( + cachedInner: Record, + incomingInner: Record, + cont: V1Continuation, +): Record { + const out: Record = { ...cachedInner }; + const incomingGc = incomingInner.generationConfig; + const incomingGen = incomingGc && typeof incomingGc === "object" && !Array.isArray(incomingGc) + ? (incomingGc as Record) + : {}; + const cachedGc = out.generationConfig; + const hasCachedGen = cachedGc !== undefined && typeof cachedGc === "object" && !Array.isArray(cachedGc); + const mergedGen: Record = hasCachedGen ? { ...(cachedGc as Record) } : {}; + // A stale token from a previous leg may sit at ANY location in the cached + // inner — strip every spelling at both spots before splicing the fresh one + // back, so the replayed inner carries exactly one token (or none, when it + // rides the envelope meta). A leftover token at another location is itself + // a deviation upstream would 400. + for (const key of CONTINUATION_TOKEN_KEYS) { + delete out[key]; + delete mergedGen[key]; + } + for (const key of ["maxOutputTokens", "max_output_tokens"] as const) { + const v = incomingGen[key]; + if (typeof v === "number" && Number.isFinite(v) && v > 0) mergedGen[key] = v; + } + if (cont.location === "generationConfig") mergedGen[cont.key] = cont.value; + if (hasCachedGen || Object.keys(mergedGen).length > 0) out.generationConfig = mergedGen; + if (cont.location === "inner") out[cont.key] = cont.value; + return out; +} + // Inbound unwrap: pull the GenerateContentResponse out of `response`. No-op when // not enveloped, so repeated application and non-v1 traffic pass through intact. export function unwrapV1ResponseJson(json: string): string { diff --git a/tests/issue2929-codex-fold-double-miss.test.ts b/tests/issue2929-codex-fold-double-miss.test.ts index 81b61ca727..7419629dc1 100644 --- a/tests/issue2929-codex-fold-double-miss.test.ts +++ b/tests/issue2929-codex-fold-double-miss.test.ts @@ -34,16 +34,17 @@ type Item = Record; const SUMMARY_MARKER = "[Compressed conversation section]"; const NUDGE_MARKER = "Compressible ranges ("; -// Indices verified against the kernel's nudge gate with nudgeGrowthTokens=2000 -// and ~9KiB filler turns. Pre-#4521 (master) the injections landed on -// requests {7,8,9}; since #4521 the armed-but-silent status line rides an -// ephemeral carrier on the gated turns and its shown-marker feeds the nudge -// gate back, so the schedule shifts to {4,6,8,9} — the fold/proxy-tool-round -// mechanics under test are unchanged, only this probe's anchor re-calibrates -// to the next nudge-carrying request pair (6 → tool round, 7 → fold round). -const PROXY_TOOL_AT = 6; // upstream request index whose response = acp_status call (round 1) -const FOLD_AT = 7; // upstream request index whose response = compress call (round 2) -// FOLD_AT+1 = post-fold refreshed re-request (round 3); FOLD_AT+2 = next steady client turn. +// The fold turn is materialized SCHEDULE-INDEPENDENTLY: the fake upstream +// arms on the first request carrying the ephemeral nudge (with >=11 tagged +// refs to fold) and folds on the re-request that follows the acp_status +// round. A hardcoded request-index anchor broke every time the nudge gate's +// token schedule shifted — #4521's status-line feedback ({7,8,9} → {4,6,8,9}), +// #6928's status-line removal, and #4313's timestamp-rendering default flip +// (the suite preload pin BILI_MESSAGE_TIMESTAMPS="0" changes the wire shape +// and hence the schedule: OFF {4,13} vs ON {3,5,11,14} — direct local runs +// without the preload diverge from CI). The probe also pins +// compress.messageTimestamps:false so its wire shape is hermetic regardless +// of how the suite is launched. const TOTAL_TURNS = 13; const FILLER = (seed: number, kb: number): string => { @@ -120,6 +121,13 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl try { const captured: Array<{ url: string; body: string }> = []; + // Object (not bare lets) so the outer-scope reads aren't narrowed to + // the initial "idle" — the mutations happen inside the request callback. + const fold = { + state: "idle" as "idle" | "armed" | "folded", + armIdx: -1, // upstream request index that carried the nudge (acp_status round) + foldIdx: -1, // upstream request index that carried the compress call + }; upstream = http.createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c: Buffer) => chunks.push(c)); @@ -127,15 +135,21 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl const idx = captured.length; const body = Buffer.concat(chunks).toString("utf8"); captured.push({ url: req.url ?? "", body }); - if (idx === PROXY_TOOL_AT) { - // Round 1 of the fold turn: model asks for proxy status — the - // loop executes it locally and re-requests (round 2). + const input = (JSON.parse(body) as { input?: Item[] }).input ?? []; + if (fold.state === "idle" && hasNudge(input) && refsOf(input).length >= 11) { + // Round 1 of the fold turn: the first nudge-carrying request + // with enough refs — model asks for proxy status, the loop + // executes it locally and re-requests (round 2). + fold.state = "armed"; + fold.armIdx = idx; functionCallSse(res, `resp_${idx}`, "acp_status", "{}", `call_st_${idx}`); return; } - if (idx === FOLD_AT) { - const prev = (JSON.parse(captured[idx - 1]!.body) as { input?: Item[] }).input ?? []; - const uniq = refsOf(prev); + if (fold.state === "armed") { + // Round 2: the re-request after the acp_status round — fold. + fold.state = "folded"; + fold.foldIdx = idx; + const uniq = refsOf(input); assert.ok(uniq.length >= 11, `fold round: expected >=11 tagged refs, saw ${uniq.length}`); const args = JSON.stringify({ content: [ { startId: uniq[3]!, endId: uniq[5]!, topic: "F1A", summary: `F1A summary: ${FILLER(1, 1)}` }, @@ -144,10 +158,9 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl functionCallSse(res, `resp_${idx}`, "compress", args, `call_cmp_${idx}`); return; } - const parsed = JSON.parse(body) as { input?: Item[] }; let label = "0"; - for (let i = (parsed.input ?? []).length - 1; i >= 0; i--) { - const it = parsed.input![i]!; + for (let i = input.length - 1; i >= 0; i--) { + const it = input[i]!; if (it.type === "message" && it.role === "user" && typeof it.content === "string") { const m = (it.content as string).match(/Turn (\d+):/); if (m) { label = m[1]!; break; } @@ -169,7 +182,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 200_000 } } } }, modelContextLimit: 200_000, kernelConfig: defaultConfig(200_000), - compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000 }, + compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000, messageTimestamps: false }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: true, @@ -223,17 +236,18 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl } const inputs = captured.map((c) => (JSON.parse(c.body) as { input?: Item[] }).input ?? []); - assert.ok(inputs.length >= FOLD_AT + 3, `expected >= ${FOLD_AT + 3} upstream requests, got ${inputs.length}`); + assert.ok(fold.state === "folded", `scenario miss: the nudge never fired on a request with >=11 refs — probe cannot materialize the #2929 trigger; adjust nudgeGrowthTokens/filler`); + assert.ok(inputs.length >= fold.foldIdx + 3, `expected >= ${fold.foldIdx + 3} upstream requests, got ${inputs.length}`); // (0) Scenario materialization: the nudge must have been injected into - // the PRE-FOLD original request (production shape: nudge armed -> model + // the arming (pre-fold) request (production shape: nudge armed -> model // calls proxy tool -> compress). Without it this probe does not cover // the #2929 trigger. - assert.ok(hasNudge(inputs[PROXY_TOOL_AT]!), `scenario miss: no ephemeral nudge in the pre-fold request (inputs[${PROXY_TOOL_AT}]) — probe does not cover the production trigger; adjust nudgeGrowthTokens/filler`); - console.log(`probe: nudge confirmed in pre-fold request; total upstream requests=${inputs.length}`); + assert.ok(hasNudge(inputs[fold.armIdx]!), `scenario miss: no ephemeral nudge in the arming request (inputs[${fold.armIdx}]) — probe does not cover the production trigger`); + console.log(`probe: nudge at inputs[${fold.armIdx}], fold at inputs[${fold.foldIdx}]; total upstream requests=${inputs.length}`); // (1) The post-fold re-request carries the in-place summary(s). - const rPost = inputs[FOLD_AT + 1]!; + const rPost = inputs[fold.foldIdx + 1]!; const postSums = summaryIndices(rPost); assert.ok(postSums.length >= 1, "post-fold re-request must carry the in-place summary"); @@ -241,7 +255,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl // the post-fold re-request up to and including the summary anchor. In // production this is where the chain breaks (next turn hits only the // instructions baseline). - const next = inputs[FOLD_AT + 2]!; + const next = inputs[fold.foldIdx + 2]!; const nextSums = summaryIndices(next); assert.ok(nextSums.length >= 1, `next steady turn lost the summary entirely (summaries: ${nextSums.length}) — fold did not survive to the client turn`); assert.equal(nextSums[0], postSums[0], `summary anchor drifted: re-request item[${postSums[0]}] vs next turn item[${nextSums[0]}]`); diff --git a/tests/issue6963-v1internal-continuation.test.ts b/tests/issue6963-v1internal-continuation.test.ts new file mode 100644 index 0000000000..74a7778c51 --- /dev/null +++ b/tests/issue6963-v1internal-continuation.test.ts @@ -0,0 +1,425 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { findV1Continuation, spliceV1ContinuationInner } from "../src/v1internal.ts"; + +// #6963: Antigravity (CloudCode) continues a MAX_TOKENS-cut turn by resending +// the SAME input plus a continuationToken. Upstream binds the token to the +// exact inner bytes it saw on the minting leg and 400s any deviation — and +// bili's stateful pipeline (nudge / status line / ACP tag re-render / output +// clamp) mutated leg 2's outbound vs leg 1's. The fix replays the cached +// minting-leg egress inner, splicing back only the client's fresh +// maxOutputTokens and the token itself. These tests pin the splice helpers +// hermetically and the replay contract through the real proxy. + +const CONV_ID = "3f9c1a6e-4b2d-4e8a-9c1f-2d5e8b7a6c4d"; +const MODEL = "gemini-4-argon-high"; + +type Call = { path: string; raw: string }; + +function sse(data: unknown): string { + return `data: ${JSON.stringify(data)}\n\n`; +} + +function gcr(text: string): Record { + return { + candidates: [{ index: 0, content: { role: "model", parts: [{ text }] }, finishReason: "STOP" }], + usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 5, totalTokenCount: 105 }, + }; +} + +function envelope(resp: Record): Record { + return { response: resp, traceId: "trace-xyz", metadata: {} }; +} + +function contents(tag: string): { role: string; parts: { text: string }[] }[] { + const out: { role: string; parts: { text: string }[] }[] = []; + for (let i = 0; i < 6; i++) { + out.push({ role: i % 2 === 0 ? "user" : "model", parts: [{ text: `${tag} Turn ${i}: ` + "marker".repeat(40) }] }); + } + return out; +} + +function systemInstruction(convId?: string): Record { + const head = "\nCurrent Date: 2026-10-14\nOperating System: Windows 11\n"; + const tail = convId ? `\nConversation ID: ${convId}\n` : "\n"; + return { parts: [{ text: head + tail }] }; +} + +type V1Opts = { + convId?: string; + requestType?: string; + model?: string; + tools?: boolean; + contents?: { role: string; parts: { text: string }[] }[]; + maxOutputTokens?: number; + requestId?: string; + // Where the continuation token rides: envelope meta, inner root, or + // inner.generationConfig — camelCase or snake_case. + token?: { location: "meta" | "inner" | "generationConfig"; value: string; snake?: boolean }; +}; + +function v1Request(opts: V1Opts): Record { + const inner: Record = { + contents: opts.contents ?? contents("A"), + systemInstruction: systemInstruction(opts.convId), + generationConfig: { maxOutputTokens: opts.maxOutputTokens ?? 4096 }, + sessionId: -3750763034362895579, + }; + if (opts.tools) inner.tools = []; + const key = opts.token?.snake ? "continuation_token" : "continuationToken"; + if (opts.token) { + if (opts.token.location === "inner") inner[key] = opts.token.value; + else if (opts.token.location === "generationConfig") (inner.generationConfig as Record)[key] = opts.token.value; + } + const env: Record = { + project: "aicode-consumers", + requestId: opts.requestId ?? `agent/${CONV_ID}`, + model: opts.model ?? MODEL, + userAgent: "antigravity", + requestType: opts.requestType ?? "agent", + request: inner, + }; + if (opts.token?.location === "meta") env[key] = opts.token.value; + return env; +} + +function makeUpstream(calls: Call[]): http.Server { + return http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + const path = req.url ?? ""; + calls.push({ path, raw }); + if (path.includes(":streamGenerateContent") || path.includes(":generateContent")) { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(sse(envelope(gcr("hello from cloudcode")))); + res.end(); + } else if (path.includes(":countTokens")) { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(envelope({ totalTokens: 42 }))); + } else { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ userInfo: { uid: 1 } })); + } + }); + }); +} + +function startProxy(upstreamPort: number): Promise { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + return startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { [MODEL]: { context: 10_000 } } } }, + modelContextLimit: 10_000, + kernelConfig: defaultConfig(10_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); +} + +async function drive(proxyPort: number, upstreamPort: number, path: string, body: unknown): Promise<{ status: number; text: string }> { + const resp = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { status: resp.status, text: await resp.text() }; +} + +function ssePayloads(text: string): unknown[] { + const out: unknown[] = []; + for (const chunk of text.split("\n\n")) { + const line = chunk.split("\n").find((l) => l.startsWith("data:")); + if (!line) continue; + const payload = line.slice(5).trim(); + if (payload === "" || payload === "[DONE]") continue; + try { out.push(JSON.parse(payload)); } catch { /* non-JSON keepalive */ } + } + return out; +} + +// Strip exactly the two fields the replay is allowed to splice (the token and +// the client's fresh maxOutputTokens, either spelling) so the rest of the +// inner can be compared byte-for-byte. +function normalizeInner(inner: Record): Record { + const c: Record = { ...inner }; + const gc = c.generationConfig; + if (gc && typeof gc === "object" && !Array.isArray(gc)) { + const g = { ...(gc as Record) }; + delete g.maxOutputTokens; + delete g.max_output_tokens; + delete g.continuationToken; + delete g.continuation_token; + c.generationConfig = Object.keys(g).length > 0 ? g : undefined; + if (c.generationConfig === undefined) delete c.generationConfig; + } + delete c.continuationToken; + delete c.continuation_token; + return c; +} + +// --------------------------------------------------------------------------- +// Splice helpers (hermetic) +// --------------------------------------------------------------------------- + +test("findV1Continuation locates the token at meta / inner / generationConfig, camelCase + snake_case", () => { + const meta = { project: "p", continuationToken: "t-meta" }; + const inner = { contents: [], generationConfig: { continuationToken: "t-gc" } }; + const found = findV1Continuation(meta, inner); + assert.deepEqual(found, { location: "meta", key: "continuationToken", value: "t-meta" }, "meta wins over inner"); + + const atInner = findV1Continuation({}, { continuation_token: "t-inner", generationConfig: { continuationToken: "t-gc" } }); + assert.deepEqual(atInner, { location: "inner", key: "continuation_token", value: "t-inner" }, "inner root wins over generationConfig, snake_case kept"); + + const atGc = findV1Continuation({}, { generationConfig: { continuation_token: "t-gc" } }); + assert.deepEqual(atGc, { location: "generationConfig", key: "continuation_token", value: "t-gc" }); + + assert.equal(findV1Continuation({}, { generationConfig: {} }), undefined, "no token anywhere"); + assert.equal(findV1Continuation({}, { generationConfig: { continuationToken: "" } }), undefined, "empty string is not a token"); + assert.equal(findV1Continuation({}, { generationConfig: { continuationToken: 42 } }), undefined, "non-string is not a token"); +}); + +test("spliceV1ContinuationInner: generationConfig location splices token + raised maxOutputTokens, rest stays byte-identical", () => { + const cached = { + contents: [{ role: "user", parts: [{ text: "x" }] }], + systemInstruction: { parts: [{ text: "sys" }] }, + tools: [{ name: "compress" }], + sessionId: -3750763034362895579, + generationConfig: { maxOutputTokens: 4096, temperature: 0.7 }, + }; + const incoming = { + contents: [{ role: "user", parts: [{ text: "CHANGED" }] }], + generationConfig: { maxOutputTokens: 8192, continuationToken: "tok-1" }, + }; + const out = spliceV1ContinuationInner(cached, incoming, { location: "generationConfig", key: "continuationToken", value: "tok-1" }); + const gc = out.generationConfig as Record; + assert.equal(gc.continuationToken, "tok-1"); + assert.equal(gc.maxOutputTokens, 8192, "client's raised budget wins"); + assert.equal(gc.temperature, 0.7, "other generationConfig fields preserved"); + assert.deepEqual(out.contents, cached.contents, "contents stay byte-identical to the minting leg"); + assert.deepEqual(out.tools, cached.tools, "tools stay byte-identical to the minting leg"); + assert.equal(out.sessionId, cached.sessionId); +}); + +test("spliceV1ContinuationInner: inner-root location splices at the root, not into generationConfig", () => { + const cached = { contents: [], generationConfig: { maxOutputTokens: 4096 } }; + const out = spliceV1ContinuationInner(cached, { generationConfig: { maxOutputTokens: 9999, continuationToken: "t" } }, { location: "inner", key: "continuationToken", value: "t" }); + const gc = out.generationConfig as Record; + assert.equal(out.continuationToken, "t"); + assert.equal(gc.continuationToken, undefined, "token must not leak into generationConfig"); + assert.equal(gc.maxOutputTokens, 9999); +}); + +test("spliceV1ContinuationInner: meta location leaves the inner untouched (relay re-wraps with the incoming meta)", () => { + const cached = { contents: [{ role: "user", parts: [{ text: "x" }] }], generationConfig: { maxOutputTokens: 4096 } }; + const out = spliceV1ContinuationInner(cached, { generationConfig: { maxOutputTokens: 2048 } }, { location: "meta", key: "continuationToken", value: "t" }); + const gc = out.generationConfig as Record; + assert.equal(out.continuationToken, undefined); + assert.equal(gc.continuationToken, undefined); + assert.equal(gc.max_output_tokens, undefined); + assert.equal(gc.maxOutputTokens, 2048, "raised budget still spliced"); + assert.deepEqual(out.contents, cached.contents); +}); + +test("spliceV1ContinuationInner: non-finite maxOutputTokens keeps the cached value", () => { + const cached = { generationConfig: { maxOutputTokens: 4096 } }; + const out = spliceV1ContinuationInner(cached, { generationConfig: { maxOutputTokens: "huge", continuationToken: "t" } }, { location: "generationConfig", key: "continuationToken", value: "t" }); + assert.equal((out.generationConfig as Record).maxOutputTokens, 4096, "string budget is ignored, cached value kept"); + const out2 = spliceV1ContinuationInner(cached, { generationConfig: { continuationToken: "t" } }, { location: "generationConfig", key: "continuationToken", value: "t" }); + assert.equal((out2.generationConfig as Record).maxOutputTokens, 4096, "absent budget keeps the cached value"); +}); + +// --------------------------------------------------------------------------- +// Replay contract through the real proxy +// --------------------------------------------------------------------------- + +test("#6963 A: a continuation leg replays the minting leg's egress inner byte-identical", async () => { + const calls: Call[] = []; + const upstream = makeUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + // Leg 1: the minting turn (MAX_TOKENS-cut in production; a plain turn + // here is enough to mint the cache). + const leg1 = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", v1Request({ convId: CONV_ID, tools: true })); + assert.equal(leg1.status, 200, `leg 1 must succeed, got HTTP ${leg1.status}: ${leg1.text.slice(0, 300)}`); + + // Leg 2: the client resends the SAME inbound plus the token, with a + // raised maxOutputTokens (accumulated across legs) and a fresh + // requestId (the envelope meta differs per leg by design). + const leg2 = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", + v1Request({ convId: CONV_ID, tools: true, requestId: `agent/${CONV_ID}/2`, maxOutputTokens: 8192, token: { location: "generationConfig", value: "tok-abc" } })); + assert.equal(leg2.status, 200, `leg 2 must succeed, got HTTP ${leg2.status}: ${leg2.text.slice(0, 300)}`); + assert.match(leg2.text, /hello from cloudcode/); + + const env1 = JSON.parse(calls[0].raw) as Record; + const env2 = JSON.parse(calls[1].raw) as Record; + const inner1 = env1.request as Record; + const inner2 = env2.request as Record; + + // The spliced fields carry leg 2's fresh values. + const gc2 = inner2.generationConfig as Record; + assert.equal(gc2.continuationToken, "tok-abc", "token spliced back into generationConfig"); + assert.equal(gc2.maxOutputTokens, 8192, "client's raised budget reaches upstream"); + + // Everything else is byte-identical to the minting leg's egress inner + // — re-running the stateful pipeline (nudge / status line / ACP tag + // re-render / clamp) would have mutated it and upstream 400s the leg. + assert.deepEqual(normalizeInner(inner2), normalizeInner(inner1), "egress inner must equal the minting leg's egress inner modulo the spliced fields"); + + // The envelope is re-wrapped from leg 2's OWN meta (fresh requestId), + // not the cached leg-1 meta. + assert.equal(env2.requestId, `agent/${CONV_ID}/2`, "egress envelope carries the incoming leg's meta"); + assert.equal(env2.project, "aicode-consumers"); + } finally { + proxy.close(); upstream.close(); + } +}); + +test("#6963 B: a continuation leg without a prior egress falls back to the full pipeline", async () => { + const calls: Call[] = []; + const upstream = makeUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + // Fresh conversation: the token arrives before any minting leg was + // egressed (client-side restart / first leg after a bili restart). + // The pipeline must still run — no crash, upstream gets a processed + // body (compress tool injected). + const leg = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", + v1Request({ convId: "aaaa1111-2222-3333-4444-555566667777", tools: true, token: { location: "generationConfig", value: "tok-orphan" } })); + assert.equal(leg.status, 200, `orphan continuation leg must not crash the pipeline, got HTTP ${leg.status}: ${leg.text.slice(0, 300)}`); + const inner = (JSON.parse(calls[0].raw) as Record).request as Record; + assert.ok(Array.isArray(inner.tools) && (inner.tools as unknown[]).length >= 1, "fallback leg enters the pipeline (compress tool injected)"); + assert.equal((inner.generationConfig as Record).continuationToken, "tok-orphan", "token forwarded on the fallback path"); + } finally { + proxy.close(); upstream.close(); + } +}); + +test("#6963 C: countTokens egress does not clobber the continuation cache", async () => { + const calls: Call[] = []; + const upstream = makeUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + // Leg 1 mints the cache with contents tagged "A". + await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", v1Request({ convId: CONV_ID, tools: true, contents: contents("A") })); + // A countTokens side request with DIFFERENT contents ("B") must not + // overwrite the cache — it never mints continuation tokens. + const ct = await drive(proxyPort, upstreamPort, "/v1internal:countTokens", v1Request({ convId: CONV_ID, tools: true, contents: contents("B") })); + assert.equal(ct.status, 200, `countTokens must succeed, got HTTP ${ct.status}: ${ct.text.slice(0, 300)}`); + // Leg 2 continues: the replay must be leg 1's inner (contents "A"), + // not the countTokens inner (contents "B"). + const leg2 = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", + v1Request({ convId: CONV_ID, tools: true, contents: contents("A"), maxOutputTokens: 8192, token: { location: "generationConfig", value: "tok-c" } })); + assert.equal(leg2.status, 200); + const inner2 = (JSON.parse(calls[2].raw) as Record).request as Record; + const inner1 = (JSON.parse(calls[0].raw) as Record).request as Record; + assert.deepEqual(inner2.contents, inner1.contents, "replay must be the minting leg's inner, not the countTokens inner"); + assert.match(JSON.stringify(inner2.contents), /A Turn 0/, "contents are the minting leg's, not the countTokens leg's"); + } finally { + proxy.close(); upstream.close(); + } +}); + +test("#6963 D: a token at the inner root or the envelope meta is honored on replay", async () => { + const calls: Call[] = []; + const upstream = makeUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + // Mint with a plain turn. + await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", v1Request({ convId: CONV_ID, tools: true })); + // Leg 2a: token at the inner root. + const leg2a = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", + v1Request({ convId: CONV_ID, tools: true, requestId: `agent/${CONV_ID}/2a`, token: { location: "inner", value: "tok-root" } })); + assert.equal(leg2a.status, 200); + const env2a = JSON.parse(calls[1].raw) as Record; + const inner2a = env2a.request as Record; + assert.equal(inner2a.continuationToken, "tok-root", "token spliced back at the inner root"); + assert.equal((inner2a.generationConfig as Record).continuationToken, undefined, "token must not leak into generationConfig"); + assert.deepEqual(normalizeInner(inner2a), normalizeInner((JSON.parse(calls[0].raw) as Record).request as Record)); + + // Leg 2b: token in the envelope meta — the inner stays token-free and + // the relay re-wraps with leg 2b's own meta, which carries it. + const leg2b = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", + v1Request({ convId: CONV_ID, tools: true, requestId: `agent/${CONV_ID}/2b`, token: { location: "meta", value: "tok-meta" } })); + assert.equal(leg2b.status, 200); + const env2b = JSON.parse(calls[2].raw) as Record; + const inner2b = env2b.request as Record; + assert.equal(env2b.continuationToken, "tok-meta", "envelope meta carries the incoming leg's token"); + assert.equal(inner2b.continuationToken, undefined, "inner root stays token-free"); + assert.equal((inner2b.generationConfig as Record).continuationToken, undefined, "generationConfig stays token-free"); + assert.deepEqual(normalizeInner(inner2b), normalizeInner((JSON.parse(calls[0].raw) as Record).request as Record)); + } finally { + proxy.close(); upstream.close(); + } +}); + +test("#6963 E: the replayed leg streams its response back enveloped", async () => { + const calls: Call[] = []; + const upstream = makeUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", v1Request({ convId: CONV_ID, tools: true })); + const leg2 = await drive(proxyPort, upstreamPort, "/v1internal:streamGenerateContent?alt=sse", + v1Request({ convId: CONV_ID, tools: true, maxOutputTokens: 8192, token: { location: "generationConfig", value: "tok-e" } })); + assert.equal(leg2.status, 200); + const frames = ssePayloads(leg2.text); + assert.ok(frames.length >= 1, `expected at least one SSE frame, got: ${leg2.text.slice(0, 300)}`); + for (const f of frames) { + assert.ok((f as Record).response !== undefined, "client frame re-wrapped in {response}"); + } + assert.match(leg2.text, /hello from cloudcode/); + } finally { + proxy.close(); upstream.close(); + } +}); diff --git a/tests/routing.test.ts b/tests/routing.test.ts index f6bae556b5..1c36a5cee4 100644 --- a/tests/routing.test.ts +++ b/tests/routing.test.ts @@ -84,6 +84,27 @@ test("lookupContextLimit returns known windows", () => { assert.equal(lookupContextLimit("kimi-k2"), 200_000); }); +test("lookupContextLimit covers the Antigravity (CloudCode) Gemini line at 1M (#6963)", () => { + // The Antigravity CLI (agy) serves the Gemini 4 line plus the role-suffixed + // subagent models; none of these matched a table entry before #6963, so + // they fell to the anonymous 200K default and 502'd at ~227K input. + assert.equal(lookupContextLimit("gemini-4-carbon-low"), 1_048_576); + assert.equal(lookupContextLimit("gemini-4-argon-medium"), 1_048_576); + assert.equal(lookupContextLimit("gemini-4-hydrogen-high"), 1_048_576); + assert.equal(lookupContextLimit("gemini-4-carbon-xhigh"), 1_048_576); + assert.equal(lookupContextLimit("gemini-pro-orchestrator"), 1_048_576); + assert.equal(lookupContextLimit("gemini-pro-orchestrator-argon"), 1_048_576); + assert.equal(lookupContextLimit("gemini-pro-agent"), 1_048_576); + assert.equal(lookupContextLimit("gemini-flash-worker-agy"), 1_048_576); + assert.equal(lookupContextLimit("gemini-pro-latest"), 1_048_576); + assert.equal(lookupContextLimit("gemini-flash-latest"), 1_048_576); + assert.equal(lookupContextLimit("gemini-flash-lite-latest"), 1_048_576); + // Controls — the pre-existing Gemini entries are untouched. + assert.equal(lookupContextLimit("gemini-3.8-flash"), 1_048_576); + assert.equal(lookupContextLimit("gemini-2.5-pro"), 1_000_000); + assert.equal(lookupContextLimit("gemini-1.5-pro"), 1_000_000); +}); + test("lookupContextLimit keeps DeepSeek flagship at 1M and legacy r1/v3/ocr at 128k (#852)", () => { assert.equal(lookupContextLimit("deepseek-flash"), 1_000_000); assert.equal(lookupContextLimit("deepseek-v4-flash"), 1_000_000);