Skip to content
1 change: 1 addition & 0 deletions reference/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ billion-context/
│ ├── remote-notices.ts # Remote announcement feed watcher: fail-open polling, last-known-good snapshot (#5548)
│ ├── request-membership.ts # Settle-side membership probe: is this usage report ours (#4805)
│ ├── resolver-wedge.ts # Persistent local DNS resolver wedge detector + alarm (#5221)
│ ├── responses-system-sections.ts # Single source of truth for the Responses-wire developer-section join: steady prepare + round-2 rebuild share one separator (#6922)
│ ├── responses-tool-output.ts # Responses tool-output image parts: projection + patch helpers
│ ├── rollout.ts # Feature rollout: per-install uuid, basis-point buckets, nested cohorts (#3062)
│ ├── runaway-guard.ts # Intrinsic stream terminator for runaway tag/codepoint enumeration (#2346)
Expand Down
6 changes: 6 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,12 @@ const CONTEXT_LIMIT_TABLE: Array<{ match: RegExp; limit: number }> = [
{ match: /^gpt-4o/i, limit: 128_000 },
{ match: /^gpt-4-turbo/i, limit: 128_000 },
{ match: /^o[13]-/i, limit: 200_000 },
// Antigravity (CloudCode) model line: gemini-4-* plus the
// orchestrator/agent/worker/latest/lite ids served on cloudcode-pa.googleapis.com
// all carry the 1M window (#6963) — the 200K fallback clamped
// maxOutputTokens to the 4K floor at ~168K input and stranded turns at ~227K.
{ match: /^gemini-4/i, limit: 1_048_576 },
{ match: /^gemini-(?:pro|flash)-(?:orchestrator|worker|agent|latest|lite)/i, limit: 1_048_576 },
{ match: /^gemini-3/i, limit: 1_048_576 },
{ match: /^gemini-2\.5/i, limit: 1_000_000 },
{ match: /^gemini-1\.5/i, limit: 1_000_000 },
Expand Down
43 changes: 42 additions & 1 deletion src/server/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ import { DecompressedTooLargeError, decodeRequestBody, IngestBodyTooLargeError }
import { matchModelEndpoint } from "../model-endpoints.js";
import { autoModeForgeEnabled, dshAuthoredCompactionEnabled, imageRescaleEnabled, limitsDecodedBytes as knobLimitsDecodedBytes, limitsIngestBytes as knobLimitsIngestBytes, noInjectTool as knobNoInjectTool, pluginBypassOverride, rawDumpDir as knobRawDumpDir } from "../knobs.js";
import { buildForgeResponse, classifierForgeVariant, estimateCjkTokens, isAutoModeClassifier, requestText, summarizeAction } from "../auto-mode-forge.js";
import { extractConversationId, isV1BackgroundSideRequest, isV1InternalRequest, unwrapV1Internal } from "../v1internal.js";
import { extractConversationId, findV1Continuation, isV1BackgroundSideRequest, isV1InternalRequest, spliceV1ContinuationInner, unwrapV1Internal } from "../v1internal.js";
import { anthropicBetaContextWindow, anthropicBetaMessageThreads, BILI_HOP_HEADER, capRegistryWindowByStandard, expandedContextSuffixWindow, launcherContextWindow, launcherMaxOutput, noteUnresolvedModelSession, UNRESOLVED_MODEL_SESSION_THRESHOLD, windowSourceLogged } from "./context-window.js";
import { NO_IDENTITY_MESSAGE, safeSessionId } from "./headers.js";
import { demoteGate, hasLeakedBiliToolsOnly, hasSingleShotBudget, isAudioOutputCall, isClaudeCodeTitleRequest, isDshAutoReviewRequest, isDshSingleShotSidecar, isDshTitleRequest, isResponsesServerToolSingleShot, isServerToolUtilityCall, isSideRequest, isStructuredOutputCall, resolveSideLane, restoreOutputBudget, sideRequestGuard, stripLeakedBiliTools } from "./side-request.js";
Expand Down Expand Up @@ -3028,6 +3028,47 @@ async function handleRequest(
respondFailFast: boolean,
overflowWindow?: number,
) => Promise<{ body: string | Buffer; prepared: Prepared | null } | null>) | undefined = async (respondFailFast, overflowWindow) => {
// #6963: a v1internal continuation leg (continuationToken present)
// must reach upstream BYTE-IDENTICAL to the input the token was
// minted for — bili's own leg-1 egress. Re-running the stateful
// pipeline (nudge / status line / ACP tag re-render / output
// clamp) mutates the body and upstream 400s the leg ("does not
// match the continuation token"). Replay the cached egress inner,
// splicing back only the client's fresh maxOutputTokens and the
// token itself. Cache miss (restart / first leg) falls through to
// the full pipeline — pre-fix behavior.
const v1ParsedInner = (typeof parsed === "object" && parsed !== null ? parsed : null) as Record<string, unknown> | null;
const v1Cont = v1Meta !== undefined && protocol === "google" && !countTokens && v1ParsedInner !== null
? findV1Continuation(v1Meta, v1ParsedInner)
: undefined;
if (v1Cont !== undefined) {
const v1Replay = session.transient?.v1ContinuationReplay;
let replayInner: Record<string, unknown> | undefined;
if (v1Replay !== undefined && v1ParsedInner !== null) {
try {
replayInner = spliceV1ContinuationInner(JSON.parse(v1Replay.body) as Record<string, unknown>, v1ParsedInner, v1Cont);
} catch {
log("warn", `[${session.id}] v1internal continuation replay: cached egress inner is not valid JSON — running full pipeline`);
}
}
if (replayInner !== undefined && v1Replay !== undefined) {
const outbound = JSON.stringify(replayInner);
log("info", `[${session.id}] v1internal continuation leg: replaying cached egress inner (${v1Replay.body.length} bytes, age ${Date.now() - v1Replay.at}ms), pipeline skipped`);
const replayPrepared: Prepared = {
body: outbound,
parsedBody: replayInner,
session,
processedMessages: [],
originalMessages: [],
protocol: "google",
stream: googlePathKind(urlPath) === "stream-generate",
compressInjected: false,
model: v1Model ?? googleModel,
};
return { body: outbound, prepared: replayPrepared };
}
log("warn", `[${session.id}] v1internal continuation leg without usable cached egress inner (restart?) — running full pipeline; upstream may reject the leg`);
}
const runPrepareInner = async (): Promise<Prepared> => {
// #1820: this prepare consumes one unit of post-rebuild anchor
// validity (the last one deletes it — see session.ts).
Expand Down
21 changes: 19 additions & 2 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -426,10 +426,24 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
// Antigravity inside the CloudCode envelope it arrived in. String-level for
// the direct fetch sites; object-level via wireTransform below for loop re-sends.
const v1Active = v1Meta !== undefined && prepared !== null;
// #6963: a continuationToken is bound to the EXACT inner bytes upstream
// last saw, so every v1 egress records its inner on the session (latest-
// wins, in-memory only) for the continuation leg to replay. countTokens
// egress never mints continuation tokens and must not clobber the cache.
const v1CountTokensEgress = typeof req.url === "string" && req.url.split("?")[0]?.endsWith("/v1internal:countTokens");
const noteV1EgressInner = (innerJson: string): void => {
if (v1CountTokensEgress) return;
const s = prepared;
if (s === null) return;
const t = s.session.transient ??= {};
t.v1ContinuationReplay = { body: innerJson, at: Date.now() };
};
const v1Wrap = (b: string | Buffer): string | Buffer => {
if (!v1Active || typeof b !== "string") return b;
try {
return JSON.stringify(wrapV1Internal(JSON.parse(b) as Record<string, unknown>, v1Meta));
const inner = JSON.parse(b) as Record<string, unknown>;
noteV1EgressInner(b);
return JSON.stringify(wrapV1Internal(inner, v1Meta));
} catch {
return b;
}
Expand Down Expand Up @@ -785,7 +799,10 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
if (steerCfg && steerCfg.enabled && steerProtocol) applyOutputSteeringJson(out, steerProtocol, steerCfg);
if (compatDropPaths.length > 0) dropCompatFieldsJson(out, compatDropPaths);
// #3129: re-nest the rebuilt inner request into the CloudCode envelope.
out = v1Active ? wrapV1Internal(out, v1Meta) : out;
if (v1Active) {
noteV1EgressInner(JSON.stringify(out));
out = wrapV1Internal(out, v1Meta);
}
if (ccMetaForResend !== undefined) out = rewrapCommandcodeBody(out, ccMetaForResend);
return out;
}
Expand Down
8 changes: 8 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,14 @@ export type Session = {
* the TTL lapses. In-memory only — a restart replants nothing, and a
* restart also drops the truncated stream it described. */
truncatedUpstream?: { at: number; protocol: string; cause: "eof" | "read-error" };
/** #6963: the exact inner request bytes the last v1internal egress
* carried — what upstream minted any continuationToken for. A
* continuation leg replays them byte-identical (splicing back only
* the fresh maxOutputTokens and the token): re-running the stateful
* pipeline would mutate the body and upstream 400s the leg ("does
* not match the continuation token"). Latest-wins, in-memory only —
* a restart lapses to the pre-fix pipeline behavior (cache miss). */
v1ContinuationReplay?: { body: string; at: number };
};
id: string;
/** Identity / descriptive metadata. Populated on first request. */
Expand Down
70 changes: 70 additions & 0 deletions src/v1internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,76 @@ export function isV1BackgroundSideRequest(meta: Record<string, unknown>, inner:
return model.includes("flash-lite");
}

// #6963: a MAX_TOKENS-cut turn is continued by resending the SAME input plus a
// `continuationToken` — upstream binds the token to the exact bytes it saw on
// the minting leg and 400s any deviation ("pass the exact same input and
// conversation history as the initial request"). The token may ride the
// envelope meta, the inner request root, or inner.generationConfig, in
// camelCase or snake_case.
type V1Continuation = { location: "meta" | "inner" | "generationConfig"; key: string; value: string };

const CONTINUATION_TOKEN_KEYS = ["continuationToken", "continuation_token"] as const;

function readContinuationToken(obj: Record<string, unknown>): { key: string; value: string } | undefined {
for (const key of CONTINUATION_TOKEN_KEYS) {
const v = obj[key];
if (typeof v === "string" && v.length > 0) return { key, value: v };
}
return undefined;
}

export function findV1Continuation(meta: Record<string, unknown>, inner: Record<string, unknown>): V1Continuation | undefined {
const atMeta = readContinuationToken(meta);
if (atMeta !== undefined) return { location: "meta", ...atMeta };
const atInner = readContinuationToken(inner);
if (atInner !== undefined) return { location: "inner", ...atInner };
const gc = inner.generationConfig;
if (gc && typeof gc === "object" && !Array.isArray(gc)) {
const atGc = readContinuationToken(gc as Record<string, unknown>);
if (atGc !== undefined) return { location: "generationConfig", ...atGc };
}
return undefined;
}

// Rebuild the continuation leg's inner from the cached minting-leg egress inner:
// contents / systemInstruction / tools / sessionId / the rest of
// generationConfig stay byte-identical to what upstream minted the token for;
// only the client's fresh maxOutputTokens (accumulated across legs — upstream
// tells the client to pass a HIGHER value) and the token itself are spliced
// back. A token located in the envelope meta is not spliced here: the relay
// re-wraps with the incoming leg's own meta, which already carries it.
export function spliceV1ContinuationInner(
cachedInner: Record<string, unknown>,
incomingInner: Record<string, unknown>,
cont: V1Continuation,
): Record<string, unknown> {
const out: Record<string, unknown> = { ...cachedInner };
const incomingGc = incomingInner.generationConfig;
const incomingGen = incomingGc && typeof incomingGc === "object" && !Array.isArray(incomingGc)
? (incomingGc as Record<string, unknown>)
: {};
const cachedGc = out.generationConfig;
const hasCachedGen = cachedGc !== undefined && typeof cachedGc === "object" && !Array.isArray(cachedGc);
const mergedGen: Record<string, unknown> = hasCachedGen ? { ...(cachedGc as Record<string, unknown>) } : {};
// A stale token from a previous leg may sit at ANY location in the cached
// inner — strip every spelling at both spots before splicing the fresh one
// back, so the replayed inner carries exactly one token (or none, when it
// rides the envelope meta). A leftover token at another location is itself
// a deviation upstream would 400.
for (const key of CONTINUATION_TOKEN_KEYS) {
delete out[key];
delete mergedGen[key];
}
for (const key of ["maxOutputTokens", "max_output_tokens"] as const) {
const v = incomingGen[key];
if (typeof v === "number" && Number.isFinite(v) && v > 0) mergedGen[key] = v;
}
if (cont.location === "generationConfig") mergedGen[cont.key] = cont.value;
if (hasCachedGen || Object.keys(mergedGen).length > 0) out.generationConfig = mergedGen;
if (cont.location === "inner") out[cont.key] = cont.value;
return out;
}

// Inbound unwrap: pull the GenerateContentResponse out of `response`. No-op when
// not enveloped, so repeated application and non-v1 traffic pass through intact.
export function unwrapV1ResponseJson(json: string): string {
Expand Down
Loading
Loading