diff --git a/CONFIGURATION.md b/CONFIGURATION.md index d7817ce3f0..e97f71e485 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -325,6 +325,25 @@ For each request, the proxy resolves the settings by longest-URL-prefix match (t ``` - `threshold: number` — character gate; runs **strictly greater** than this are dropped (`0` = drop any non-empty run). Invalid values fall back to the default instead of throwing. +#### `reasoningGuard` + +- **Type:** `object` (`{ enabled?, maxContinue?, maxTierN?, markerText?, base?, offset?, debugLog? }`) +- **Default:** *(disabled — off unless you set `enabled: true` at some level)* +- **Status:** ACTIVE +- **Description:** Opt-in guard against gpt-5.x/gpt-6.x **"lattice" reasoning truncation** (issue #739; upstream [openai/codex#30364](https://github.com/openai/codex/issues/30364)). These models intermittently stop at exactly `base*n + offset` reasoning tokens (default `518n−2` → 516, 1034, 1552, …) mid-thought, then answer from a half-finished thought. When an in-scope terminal round hits the lattice **and** carries an `encrypted_content` blob, bili buffers the response, re-sends it replaying its own reasoning plus a continue nudge (up to `maxContinue` continuation rounds), and folds everything into ONE response whose usage is the true summed total. Reasoning streams live to the client during the fold (no full buffering); only the final clean round's non-reasoning output is passed through. Applies to Responses/SSE streaming requests only (bili is SSE-only); compress-injected turns are exempt (the loop owns those). Sub-fields (merged deepest-wins like every other CompressSettings field): + - `enabled: boolean` — master switch; anything other than `true` keeps the guard fully off. Scope is set by **where** this block sits in the three-level tree (global / provider / model) — there is no separate model list. The strict signature (exact lattice hit + `encrypted_content` + no tool calls) limits which rounds actually trigger recovery. + - `maxContinue: number` — max continuation rounds after the initial round (default `3`). + - `maxTierN: number` — highest lattice tier `n` allowed to continue (default `6`); `0` = unlimited. Raise for rare deep-tier truncations (e.g. an `n=11` hit observed on gpt-6-astra). + - `markerText: string` — nudge text appended as a commentary message each continued round (default `"Continue thinking..."`). + - `base: number` / `offset: number` — the lattice signature `tokens == base*n + offset` (defaults `518` / `-2`). Override if another model family truncates on a different lattice. + - `debugLog: boolean` — verbose per-round logging (default `false`). + ```jsonc + // enable globally + { "compress": { "reasoningGuard": { "enabled": true } } } + // tune per provider (placement scopes it to that provider's traffic) + { "providers": { "https://your-relay.example": { "compress": { "reasoningGuard": { "enabled": true, "maxContinue": 2 } } } } } + ``` + #### `stripImages` - **Type:** `boolean` diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index d19522867f..09ff1987a0 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -323,6 +323,25 @@ ``` - `threshold: number` — 字符门槛;**严格大于**该值的段才被剥离(`0` = 只要非空就剥)。非法值回退默认而不是报错。 +#### `reasoningGuard` + +- **类型:** `object`(`{ enabled?, maxContinue?, maxTierN?, markerText?, base?, offset?, debugLog? }`) +- **默认值:** *(禁用 —— 除非在某一层设置 `enabled: true`)* +- **状态:** ACTIVE +- **说明:** gpt-5.x/gpt-6.x **"晶格"(lattice)推理截断**守卫(issue #739;上游 [openai/codex#30364](https://github.com/openai/codex/issues/30364))。这些模型会间歇性地在恰好 `base*n + offset` 个 reasoning token 处(默认 `518n−2` → 516、1034、1552 …)思考到一半就停下,然后基于未完成的思路作答。当作用域内的终止回合落在晶格上**且**携带 `encrypted_content` 块时,bili 会缓冲该响应、带着自己的 reasoning 加一条继续提示重发(最多 `maxContinue` 个续写回合),再把全部折叠成**一个**响应,其 usage 为真实求和值。折叠期间 reasoning 实时流式发给客户端(不做整段缓冲),只有最后一轮干净回合的非 reasoning 输出被透传。仅作用于 Responses/SSE 流式请求(bili 只支持 SSE);压缩注入的回合被豁免(由循环负责)。子字段与其他 CompressSettings 字段一样按“深层覆盖”合并: + - `enabled: boolean` — 总开关;非 `true` 时守卫完全关闭。作用域由该块在三级树(全局 / provider / model)**放在哪一层**决定——没有单独模型列表。严格签名(精确晶格命中 + `encrypted_content` + 无工具调用)限制哪些回合真正触发续写。 + - `maxContinue: number` — 首轮之后最多续写的回合数(默认 `3`)。 + - `maxTierN: number` — 允许续写的最高晶格层级 `n`(默认 `6`);`0` = 不限制。遇到罕见的深层截断时调高(例如在 gpt-6-astra 上观察到一次 `n=11`)。 + - `markerText: string` — 每个续写回合追加的 commentary 提示文本(默认 `"Continue thinking..."`)。 + - `base: number` / `offset: number` — 晶格签名 `tokens == base*n + offset`(默认 `518` / `-2`)。若其他模型家族在不同晶格上截断则覆盖。 + - `debugLog: boolean` — 逐回合详细日志(默认 `false`)。 + ```jsonc + // 全局开启 + { "compress": { "reasoningGuard": { "enabled": true } } } + // 按 provider 调参(放在哪一层就作用于哪一层的流量) + { "providers": { "https://your-relay.example": { "compress": { "reasoningGuard": { "enabled": true, "maxContinue": 2 } } } } } + ``` + #### `stripImages` - **类型:** `boolean` diff --git a/src/compress-settings.ts b/src/compress-settings.ts index dc1361d2ad..9dc197329f 100644 --- a/src/compress-settings.ts +++ b/src/compress-settings.ts @@ -54,6 +54,7 @@ export function mergeCompress( // provider-level excludeTools. const absorbLevels = [global?.absorb, provider?.absorb, model?.absorb].filter(Boolean) as NonNullable[]; const reasoningLevels = [global?.reasoning, provider?.reasoning, model?.reasoning].filter(Boolean) as NonNullable[]; + const reasoningGuardLevels = [global?.reasoningGuard, provider?.reasoningGuard, model?.reasoningGuard].filter(Boolean) as NonNullable[]; return { modelContextLimit: pick("modelContextLimit"), maxContextLimit: pick("maxContextLimit"), @@ -73,6 +74,7 @@ stripImages: pick("stripImages"), // exactly like `absorb`/`prompts`: a model-level `threshold` must not // discard a provider-level `drop: false`. reasoning: reasoningLevels.length > 0 ? Object.assign({}, ...reasoningLevels) : undefined, + reasoningGuard: reasoningGuardLevels.length > 0 ? Object.assign({}, ...reasoningGuardLevels) : undefined, promptPack: pick("promptPack"), }; } diff --git a/src/config.ts b/src/config.ts index 8c66441016..1ca57af66e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,6 +6,7 @@ import { log as loggerLog } from "./logger.js"; import { validateHttpProxy, type ProxyFallbackOptions } from "./upstream-proxy.js"; import { parseCompatRoles } from "./compat-roles.js"; +import type { ReasoningGuardConfig } from "./reasoning-guard.js"; export function safeReadJson(path: string): unknown { try { @@ -187,6 +188,15 @@ export type CompressSettings = { * dropped (default 2048). */ threshold?: number; }; + /** [#739] Opt-in guard against gpt-5.x/gpt-6.x "lattice" reasoning truncation + * (reasoning stops at exactly base*n+offset tokens, default 518n-2 -> 516, + * 1034, ..., mid-thought). When engaged on a matched-model terminal round that + * hits the lattice AND carries an encrypted_content blob, bili buffers the + * response, replays its own reasoning plus a continue nudge (up to maxContinue + * rounds), and folds to ONE response with true summed usage. Merged sub-field-wise + * across the three levels like `absorb`/`reasoning`; off unless enabled at some + * level. See src/reasoning-guard.ts. */ + reasoningGuard?: ReasoningGuardConfig; }; export type PromptCacheRouting = "auto" | "enabled" | "disabled"; export type UpstreamProxyMode = "auto" | "manual" | "direct"; @@ -699,6 +709,33 @@ export function parseCompressSettings(v: unknown): (CompressSettings & { injectT if (typeof obj.promptPack !== "string" || obj.promptPack.trim().length === 0) ok = false; else out.promptPack = obj.promptPack.trim(); } + if ("reasoningGuard" in obj && obj.reasoningGuard !== undefined) { + const rg = obj.reasoningGuard; + if (!rg || typeof rg !== "object" || Array.isArray(rg)) { + ok = false; + } else { + const rgo = rg as Record; + const cleaned: ReasoningGuardConfig = {}; + for (const key of ["enabled", "maxContinue", "maxTierN", "markerText", "base", "offset", "debugLog"] as const) { + if (!(key in rgo)) continue; + const v = rgo[key]; + if (key === "enabled") { + if (typeof v !== "boolean") { ok = false; continue; } + cleaned.enabled = v; + } else if (key === "debugLog") { + if (typeof v !== "boolean") { ok = false; continue; } + cleaned.debugLog = v; + } else if (key === "markerText") { + if (typeof v !== "string" || v.trim().length === 0) { ok = false; continue; } + cleaned.markerText = v.trim(); + } else { + if (typeof v !== "number" || !Number.isFinite(v)) { ok = false; continue; } + (cleaned as Record)[key] = v; + } + } + if (ok) out.reasoningGuard = cleaned; + } + } if (!ok) return undefined; return out; } diff --git a/src/reasoning-guard.ts b/src/reasoning-guard.ts new file mode 100644 index 0000000000..62291f6462 --- /dev/null +++ b/src/reasoning-guard.ts @@ -0,0 +1,560 @@ +import { fetchWithTimeout } from "./fetch-util.js"; +import { normalizeSseLineEndings } from "./sse-util.js"; + +/** Minimal server-response surface the guard writes SSE frames to. Kept apart + * from node:http so the live fold can be unit-tested against a recording sink. */ +export interface SseSink { + headersSent: boolean; + writableEnded: boolean; + writableNeedDrain?: boolean; + writeHead(status: number, headers: Record): void; + write(chunk: string | Uint8Array): unknown; + end(): void; + once(event: "drain", cb: () => void): unknown; +} + +/** Guard against OpenAI gpt-5.x/gpt-6.x "lattice" reasoning truncation (#739): these + * models intermittently stop at exactly base*n+offset reasoning tokens (default + * 518n-2 -> 516,1034,...) mid-thought. On a matched-model terminal round hitting + * the lattice AND carrying an encrypted_content blob, re-send replaying its own + * reasoning plus a nudge (up to N rounds), then fold into ONE response with true + * summed usage. Detection is data-driven (config); recovery is generic. Scope is + * expressed by WHERE the config sits in the three-level tree (global/provider/model); + * the strict signature (exact lattice hit + encrypted_content + no tool calls) limits + * actual action. Default off. */ + +export interface ReasoningGuardConfig { + enabled?: boolean; + /** Max continuation rounds after the initial round (default 3). */ + maxContinue?: number; + /** Max lattice tier n allowed to continue (default 6); 0 = unlimited. */ + maxTierN?: number; + /** Nudge text appended as a commentary message (default "Continue thinking..."). */ + markerText?: string; + /** Lattice base (default 518); truncated tokens == base*n + offset. */ + base?: number; + /** Lattice offset (default -2); gpt-5.x/gpt-6.x truncate at 518n-2. */ + offset?: number; + debugLog?: boolean; +} + +const MIN_N = 1; +const DEFAULT_MARKER = "Continue thinking..."; +const ENC_INCLUDE = "reasoning.encrypted_content"; +const TERMINAL_TYPES = new Set(["response.completed", "response.failed", "response.incomplete"]); + +interface ResolvedGuard { + maxContinue: number; + maxTierN: number; + markerText: string; + base: number; + offset: number; + debugLog: boolean; +} + +type Usage = Record; + +export function resolveGuardConfig(cfg: ReasoningGuardConfig | undefined): ResolvedGuard { + const c = cfg ?? {}; + return { + maxContinue: numOr(c.maxContinue, 3), + maxTierN: numOr(c.maxTierN, 6), + markerText: typeof c.markerText === "string" && c.markerText.trim() ? c.markerText.trim() : DEFAULT_MARKER, + base: numOr(c.base, 518), + offset: numOr(c.offset, -2), + debugLog: c.debugLog === true, + }; +} + +function numOr(v: unknown, dflt: number): number { + return typeof v === "number" && Number.isFinite(v) ? v : dflt; +} + +export function reasoningGuardEngages(cfg: ReasoningGuardConfig | undefined): boolean { + return cfg?.enabled === true; +} + +export function reasoningTokens(usage: Usage | null | undefined): number | null { + if (!usage) return null; + const details = usage["output_tokens_details"] as Usage | undefined; + const rt = details?.["reasoning_tokens"]; + return typeof rt === "number" && Number.isFinite(rt) ? rt : null; +} + +/** n when tokens == base*n + offset (e.g. 518n-2 -> 516,1034,...), else null. */ +export function tierN(tokens: number | null, base: number, offset: number): number | null { + if (tokens === null || base <= 0) return null; + const t = tokens - offset; + if (t < base * MIN_N || t % base !== 0) return null; + const n = t / base; + return Number.isInteger(n) && n >= MIN_N ? n : null; +} + +export function inContinueWindow(n: number | null, maxTierN: number): boolean { + if (n === null || n < MIN_N) return false; + if (maxTierN !== 0 && n > maxTierN) return false; + return true; +} + +export function hasEncryptedContent(reasoningItems: Array>): boolean { + if (reasoningItems.length === 0) return false; + const ec = reasoningItems[reasoningItems.length - 1]?.["encrypted_content"]; + return typeof ec === "string" && ec.length > 0; +} + +/** "truncation" when the terminal reasoning hits the lattice within window, else "". */ +export function continueReason(usage: Usage | null, base: number, offset: number, maxTierN: number): "truncation" | "" { + const n = tierN(reasoningTokens(usage), base, offset); + return inContinueWindow(n, maxTierN) ? "truncation" : ""; +} + +function num(v: unknown): number { + return typeof v === "number" && Number.isFinite(v) ? v : 0; +} + +export function sumUsage(acc: Usage, usage: Usage | null): void { + if (!usage) return; + for (const key of ["input_tokens", "output_tokens", "total_tokens"] as const) { + const v = usage[key]; + if (typeof v === "number") acc[key] = num(acc[key]) + v; + } + const inDetails = usage["input_tokens_details"] as Usage | undefined; + if (inDetails && typeof inDetails.cached_tokens === "number") { + const accIn = (acc["input_tokens_details"] as Usage) ?? {}; + accIn.cached_tokens = num(accIn.cached_tokens) + inDetails.cached_tokens; + acc["input_tokens_details"] = accIn; + } + const rt = reasoningTokens(usage); + if (rt !== null) { + const accOut = (acc["output_tokens_details"] as Usage) ?? {}; + accOut.reasoning_tokens = num(accOut.reasoning_tokens) + rt; + acc["output_tokens_details"] = accOut; + } +} + +/** Reconstruct the usage reported on the folded response: input from the first + * round (prompt sent once), output = summed reasoning + final non-reasoning + * part, total = input + output. Preserves first-round cached_tokens. */ +export function agentUsage(first: Usage | null, summed: Usage, finalRound: Usage | null, flushedFinal: boolean): Usage { + const f = first ?? {}; + const inTok = num(f.input_tokens); + const fDetails = f["input_tokens_details"] as Usage | undefined; + const cached = fDetails && typeof fDetails.cached_tokens === "number" ? fDetails.cached_tokens : null; + const sOut = summed["output_tokens_details"] as Usage | undefined; + const reason = sOut && typeof sOut.reasoning_tokens === "number" ? sOut.reasoning_tokens : 0; + let finalPart = 0; + if (flushedFinal && finalRound) { + const out = num(finalRound.output_tokens); + const rt = reasoningTokens(finalRound) ?? 0; + finalPart = Math.max(0, out - rt); + } + const usage: Usage = { + input_tokens: inTok, + output_tokens: reason + finalPart, + total_tokens: inTok + reason + finalPart, + output_tokens_details: { reasoning_tokens: reason }, + }; + if (cached !== null) usage["input_tokens_details"] = { cached_tokens: cached }; + return usage; +} + +export function buildTerminalEvent(opts: { + upstreamTerminal: Record | null; + baseResponse: Record | null; + output: Array>; + usage: Usage; + rounds: Array>; + billed: Usage; + stoppedReason: string; + incompleteReason: string; +}): Record { + const tresp = (opts.upstreamTerminal?.["response"] as Usage) ?? {}; + const resp: Record = { ...(opts.baseResponse ?? tresp) }; + resp.output = opts.output; + resp.usage = opts.usage; + const metadata: Record = { ...((resp.metadata as Record) ?? {}) }; + metadata.proxy_rounds = opts.rounds; + metadata.proxy_billed_usage = opts.billed; + if (opts.stoppedReason) metadata.proxy_stopped_reason = opts.stoppedReason; + resp.metadata = metadata; + if (opts.incompleteReason) { + resp.status = "incomplete"; + resp.incomplete_details = { reason: opts.incompleteReason }; + return { type: "response.incomplete", response: resp }; + } + const status = typeof tresp.status === "string" ? tresp.status : "completed"; + resp.status = status; + if (tresp.incomplete_details !== undefined) resp.incomplete_details = tresp.incomplete_details; + const evType = typeof opts.upstreamTerminal?.["type"] === "string" ? opts.upstreamTerminal["type"] : "response.completed"; + return { type: evType, response: resp }; +} + +export function commentaryNudge(markerText: string): Record { + return { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: markerText }], + phase: "commentary", + }; +} + +export function nextRoundBody(baseBody: Record, inputItems: unknown[]): Record { + const body: Record = { ...baseBody }; + body.stream = true; + body.input = inputItems; + const includeRaw = Array.isArray(body.include) ? (body.include as unknown[]) : []; + const include = includeRaw.filter((x): x is string => typeof x === "string"); + if (!include.includes(ENC_INCLUDE)) include.push(ENC_INCLUDE); + body.include = include; + delete body.previous_response_id; + return body; +} + +export interface GuardParams { + firstResponse: Response; + clearFirstTimer: () => void; + upstreamUrl: string; + reqHeaders: Record; + dispatcher?: object; + originalBody: Buffer | string; + signal: AbortSignal; + res: SseSink; + config: ReasoningGuardConfig; + log: (msg: string) => void; +} + +interface FoldState { + cfg: ResolvedGuard; + baseBody: Record; + origInput: unknown[]; + replayTail: unknown[]; + summedUsage: Usage; + firstUsage: Usage | null; + finalOutput: Array>; + roundsInfo: Array>; + dsOI: number; + seq: number; + baseResponse: Record | null; + roundNo: number; + roundReasoning: Array>; + kind: Map; + oiToDS: Map; + buffered: Array<{ oi: number; item: Record; events: Array> }>; + terminal: Record | null; + usage: Usage | null; +} + +const DONE_SENTINEL = Symbol("done"); + +function parseFrame(frame: string): Record | typeof DONE_SENTINEL | null { + let payload = ""; + for (const line of frame.split("\n")) { + if (line.startsWith("data:")) payload += (payload ? "\n" : "") + line.slice(5).replace(/^ /, ""); + } + const trimmed = payload.trim(); + if (trimmed === "") return null; + if (trimmed === "[DONE]") return DONE_SENTINEL; + try { + return JSON.parse(trimmed) as Record; + } catch { + return null; + } +} + +function writeEvent(res: SseSink, ev: Record): void { + res.write(`data: ${JSON.stringify(ev)}\n\n`); +} + +function stamp(st: FoldState, ev: Record): void { + ev.sequence_number = st.seq++; +} + +function processEvent(ev: Record, st: FoldState, res: SseSink): Record | null { + const etype = typeof ev.type === "string" ? ev.type : ""; + if (etype === "response.created" || etype === "response.in_progress") { + if (st.roundNo === 1) { + if (etype === "response.created" && typeof ev.response === "object" && ev.response !== null) { + st.baseResponse = ev.response as Record; + } + stamp(st, ev); + writeEvent(res, ev); + } + return null; + } + if (TERMINAL_TYPES.has(etype)) { + st.terminal = ev; + const resp = ev.response as Usage | undefined; + st.usage = resp && typeof resp === "object" ? ((resp.usage as Usage) ?? null) : null; + return ev; + } + const oiRaw = ev.output_index; + const oi = typeof oiRaw === "number" ? oiRaw : -1; + if (etype === "response.output_item.added") { + const item = (typeof ev.item === "object" && ev.item !== null) ? (ev.item as Record) : {}; + const itemType = typeof item.type === "string" ? item.type : ""; + if (itemType === "reasoning") { + st.kind.set(oi, "reasoning"); + st.oiToDS.set(oi, st.dsOI); + ev.output_index = st.dsOI; + st.dsOI++; + stamp(st, ev); + writeEvent(res, ev); + } else { + st.kind.set(oi, "buffered"); + st.buffered.push({ oi, item, events: [ev] }); + } + return null; + } + const k = st.kind.get(oi); + if (k === "reasoning") { + const ds = st.oiToDS.get(oi); + if (ds !== undefined) ev.output_index = ds; + if (etype === "response.output_item.done" && typeof ev.item === "object" && ev.item !== null) { + const item = ev.item as Record; + st.roundReasoning.push(item); + st.finalOutput.push(item); + } + stamp(st, ev); + writeEvent(res, ev); + } else if (k === "buffered") { + const entry = st.buffered.find((b) => b.oi === oi); + if (entry) { + entry.events.push(ev); + if (etype === "response.output_item.done" && typeof ev.item === "object" && ev.item !== null) { + entry.item = ev.item as Record; + } + } + } else { + stamp(st, ev); + writeEvent(res, ev); + } + return null; +} + +async function consumeRound( + stream: ReadableStream, + st: FoldState, + res: SseSink, +): Promise<{ terminal: Record } | { error: string }> { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let buf = ""; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + buf = normalizeSseLineEndings(buf); + let idx: number; + while ((idx = buf.indexOf("\n\n")) !== -1) { + const frame = buf.slice(0, idx); + buf = buf.slice(idx + 2); + const ev = parseFrame(frame); + if (ev === null || ev === DONE_SENTINEL) continue; + const term = processEvent(ev, st, res); + if (term) return { terminal: term }; + } + } + return { error: "upstream_eof" }; + } catch (e) { + return { error: e instanceof Error ? e.message : String(e) }; + } finally { + reader.releaseLock(); + } +} + +function endRound(st: FoldState): void { + sumUsage(st.summedUsage, st.usage); + if (st.roundNo === 1) st.firstUsage = cloneUsage(st.usage); + const rt = reasoningTokens(st.usage); + const n = tierN(rt, st.cfg.base, st.cfg.offset); + st.roundsInfo.push({ round: st.roundNo, reasoning_tokens: rt ?? 0, n: n ?? 0 }); +} + +function shouldContinue(st: FoldState): "truncation" | "" { + if (!st.terminal) return ""; + const etype = typeof st.terminal.type === "string" ? st.terminal.type : ""; + if (etype !== "response.completed") return ""; + const reason = continueReason(st.usage, st.cfg.base, st.cfg.offset, st.cfg.maxTierN); + if (reason === "") return ""; + if (!hasEncryptedContent(st.roundReasoning)) return ""; + if (st.roundNo > st.cfg.maxContinue) return ""; + st.roundsInfo[st.roundsInfo.length - 1]!.continue_reason = reason; + return reason; +} + +function prepareNextRound(st: FoldState): void { + st.replayTail.push(...st.roundReasoning, commentaryNudge(st.cfg.markerText)); + st.roundReasoning = []; + st.kind = new Map(); + st.oiToDS = new Map(); + st.buffered = []; + st.terminal = null; + st.usage = null; +} + +function flushCleanStop(st: FoldState, res: SseSink): void { + for (const entry of st.buffered) { + for (const ev of entry.events) { + if ("output_index" in ev) ev.output_index = st.dsOI; + stamp(st, ev); + writeEvent(res, ev); + } + st.dsOI++; + st.finalOutput.push(entry.item); + } +} + +function stoppedReason(st: FoldState): string { + if (st.roundNo <= 1) return ""; + const etype = typeof st.terminal?.type === "string" ? st.terminal.type : ""; + if (etype !== "response.completed") return ""; + if (continueReason(st.usage, st.cfg.base, st.cfg.offset, st.cfg.maxTierN) === "") return ""; + if (!hasEncryptedContent(st.roundReasoning)) return "no_encrypted_content"; + if (st.roundNo > st.cfg.maxContinue) return "max_continue"; + const n = tierN(reasoningTokens(st.usage), st.cfg.base, st.cfg.offset); + if (!inContinueWindow(n, st.cfg.maxTierN)) return "tier_out_of_window"; + return ""; +} + +function buildIncomplete(st: FoldState, reason: string): Record { + const usage = agentUsage(st.firstUsage, st.summedUsage, st.usage, false); + return buildTerminalEvent({ + upstreamTerminal: null, + baseResponse: st.baseResponse, + output: st.finalOutput, + usage, + rounds: st.roundsInfo, + billed: st.summedUsage, + stoppedReason: "", + incompleteReason: reason, + }); +} + +function cloneUsage(u: Usage | null): Usage | null { + if (!u) return null; + return JSON.parse(JSON.stringify(u)) as Usage; +} + +async function pipeThroughRaw(stream: ReadableStream, res: SseSink): Promise { + const reader = stream.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + res.write(value); + if (res.writableNeedDrain) await new Promise((r) => res.once("drain", r)); + } + } finally { + reader.releaseLock(); + if (!res.writableEnded) res.end(); + } +} + +export async function runReasoningGuard(p: GuardParams): Promise { + const cfg = resolveGuardConfig(p.config); + let bodyStr: string; + try { + bodyStr = typeof p.originalBody === "string" ? p.originalBody : p.originalBody.toString("utf8"); + } catch { + bodyStr = ""; + } + let baseBody: Record; + try { + baseBody = JSON.parse(bodyStr) as Record; + } catch { + p.log("reasoning-guard: failed to parse request body; passing round 1 through verbatim"); + const rawBody = p.firstResponse.body; + if (!rawBody) { + if (!p.res.headersSent) p.res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + if (!p.res.writableEnded) p.res.end(); + p.clearFirstTimer(); + return; + } + await pipeThroughRaw(rawBody, p.res); + p.clearFirstTimer(); + return; + } + const origInput = Array.isArray(baseBody.input) ? (baseBody.input as unknown[]) : []; + const st: FoldState = { + cfg, + baseBody, + origInput, + replayTail: [], + summedUsage: {}, + firstUsage: null, + finalOutput: [], + roundsInfo: [], + dsOI: 0, + seq: 0, + baseResponse: null, + roundNo: 1, + roundReasoning: [], + kind: new Map(), + oiToDS: new Map(), + buffered: [], + terminal: null, + usage: null, + }; + + const finish = (ev: Record): void => { + if (!p.res.headersSent) p.res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + if (!p.res.writableEnded) { + writeEvent(p.res, ev); + p.res.end(); + } + }; + + const firstBody = p.firstResponse.body; + if (!firstBody) { + finish(buildIncomplete(st, "no_body")); + p.clearFirstTimer(); + return; + } + let result = await consumeRound(firstBody, st, p.res); + p.clearFirstTimer(); + + for (;;) { + if ("error" in result) { + finish(buildIncomplete(st, result.error)); + return; + } + endRound(st); + const cont = shouldContinue(st); + if (cfg.debugLog) p.log(`reasoning-guard round=${st.roundNo} rt=${reasoningTokens(st.usage)} continue=${cont || "-"}`); + if (cont) { + prepareNextRound(st); + st.roundNo++; + const nextBody = nextRoundBody(st.baseBody, [...st.origInput, ...st.replayTail]); + let roundRes: Awaited>; + try { + roundRes = await fetchWithTimeout(p.upstreamUrl, { + method: "POST", + headers: p.reqHeaders, + body: JSON.stringify(nextBody), + dispatcher: p.dispatcher, + }, undefined, p.signal); + } catch { + finish(buildIncomplete(st, "upstream_error")); + return; + } + const contBody = roundRes.response.body; + roundRes.clearTimer(); + result = contBody ? await consumeRound(contBody, st, p.res) : { error: "no_body" }; + continue; + } + flushCleanStop(st, p.res); + const usage = agentUsage(st.firstUsage, st.summedUsage, st.usage, true); + finish(buildTerminalEvent({ + upstreamTerminal: st.terminal, + baseResponse: st.baseResponse, + output: st.finalOutput, + usage, + rounds: st.roundsInfo, + billed: st.summedUsage, + stoppedReason: stoppedReason(st), + incompleteReason: "", + })); + return; + } +} diff --git a/src/server.ts b/src/server.ts index 8d82ababc5..9f723ddc80 100644 --- a/src/server.ts +++ b/src/server.ts @@ -60,6 +60,7 @@ import { compressLoopResponsesJson } from "./compress-loop-responses.js"; import { runCompressLoop, pickAdapter } from "./loop/index.js"; import { containsToolCallXmlFragment } from "./loop/tag-echo-filter.js"; import { isFakeCompletion, injectFakeCompletionHint, maxFakeCompletionRetries, fakeBufCap } from "./fake-completion.js"; +import { reasoningGuardEngages, runReasoningGuard } from "./reasoning-guard.js"; import { sanitizeResponsesInputIds, dropWhitespaceResponsesMessages, normalizeResponsesMessageItems } from "./loop/adapter-responses.js"; import { codexCompactMode, isCodexClient, hasCompactionTrigger, stripBiliCompactionItems, replaceBiliCompactionItems, codexCompactGate, codexCompactGatePre, buildTriggerForgeBody, mergeForgedSummaries } from "./codex-compact.js"; import { stripAcpPanelMessages, stripAcpPanelResponsesInput } from "./acp-panel.js"; @@ -3802,6 +3803,34 @@ async function forward( } return; } + if (prepared && prepared.protocol === "responses" && prepared.stream && !prepared.sidePassthrough && !prepared.compressInjected) { + const sse = (upstream.headers.get("content-type") ?? "").includes("text/event-stream"); + if (sse) { + let reqModel: string | undefined; + try { + const wb = typeof wireBody === "string" ? wireBody : wireBody.toString("utf8"); + const parsed = JSON.parse(wb) as Record; + if (typeof parsed.model === "string") reqModel = parsed.model; + } catch { /* non-JSON body: guard stays off */ } + const rg = resolveCompress(opts.routes, upstreamUrl, reqModel, opts.compress).reasoningGuard; + if (rg && reasoningGuardEngages(rg)) { + log("info", `[reasoning-guard] engaged model=${reqModel ?? "?"} session=${prepared.session?.id ?? "-"}`); + await runReasoningGuard({ + firstResponse: upstream, + clearFirstTimer: clearUpstreamTimer, + upstreamUrl, + reqHeaders: buildForwardHeaders(headers), + dispatcher, + originalBody: wireBody, + signal: clientAbort.signal, + res, + config: rg, + log: (msg) => log("info", msg), + }); + return; + } + } + } // #371: detect + retry a fake completion for every non-plugin streaming // response (any turn, not just compress-injected). Buffering is required: // the retry re-requests before the client sees the fake completion. diff --git a/tests/compress-settings.test.ts b/tests/compress-settings.test.ts index 1a85f90416..d795d539e8 100644 --- a/tests/compress-settings.test.ts +++ b/tests/compress-settings.test.ts @@ -257,3 +257,39 @@ test("parseCompressSettings: parses stripImages (bool) + stripImagesKeepRecent ( assert.equal(parseCompressSettings({ stripImages: "yes" }), undefined); assert.equal(parseCompressSettings({ stripImagesKeepRecent: "many" }), undefined); }); + +test("mergeCompress: reasoningGuard merges sub-field-wise deepest-wins (#739)", () => { + const merged = mergeCompress( + { reasoningGuard: { enabled: true, maxContinue: 3, base: 518 } }, + { reasoningGuard: { offset: -4, maxContinue: 2 } }, + { reasoningGuard: { maxTierN: 4 } }, + ); + assert.equal(merged.reasoningGuard?.enabled, true); + assert.equal(merged.reasoningGuard?.maxContinue, 2); + assert.equal(merged.reasoningGuard?.maxTierN, 4); + assert.equal(merged.reasoningGuard?.base, 518); + assert.equal(merged.reasoningGuard?.offset, -4); +}); + +test("mergeCompress: reasoningGuard absent at all levels stays undefined", () => { + const merged = mergeCompress({ nudgeGrowthTokens: 50000 }, { tiers: false }, undefined); + assert.equal(merged.reasoningGuard, undefined); +}); + +test("parseCompressSettings: parses reasoningGuard sub-fields and rejects malformed (#739)", () => { + const ok = parseCompressSettings({ + reasoningGuard: { enabled: true, maxContinue: 3, maxTierN: 6, markerText: " go ", base: 518, offset: -2, debugLog: true }, + }); + assert.equal(ok?.reasoningGuard?.enabled, true); + assert.equal(ok?.reasoningGuard?.maxContinue, 3); + assert.equal(ok?.reasoningGuard?.maxTierN, 6); + assert.equal(ok?.reasoningGuard?.markerText, "go"); + assert.equal(ok?.reasoningGuard?.base, 518); + assert.equal(ok?.reasoningGuard?.offset, -2); + assert.equal(ok?.reasoningGuard?.debugLog, true); + assert.equal(parseCompressSettings({})?.reasoningGuard, undefined); + assert.equal(parseCompressSettings({ reasoningGuard: { enabled: "yes" } }), undefined); + assert.equal(parseCompressSettings({ reasoningGuard: { enabled: true, models: ["gpt-5"] } })?.reasoningGuard?.enabled, true); + assert.equal(parseCompressSettings({ reasoningGuard: { base: "518" } }), undefined); + assert.equal(parseCompressSettings({ reasoningGuard: [] }), undefined); +}); diff --git a/tests/reasoning-guard.test.ts b/tests/reasoning-guard.test.ts new file mode 100644 index 0000000000..0975a75d26 --- /dev/null +++ b/tests/reasoning-guard.test.ts @@ -0,0 +1,298 @@ +import assert from "node:assert"; +import http from "node:http"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { + runReasoningGuard, + tierN, + inContinueWindow, + reasoningGuardEngages, + hasEncryptedContent, + continueReason, + sumUsage, + agentUsage, + nextRoundBody, + commentaryNudge, + buildTerminalEvent, + type SseSink, +} from "../src/reasoning-guard.ts"; + +test("tierN maps 518n-2 lattice to tier index", () => { + assert.strictEqual(tierN(516, 518, -2), 1); + assert.strictEqual(tierN(1034, 518, -2), 2); + assert.strictEqual(tierN(1552, 518, -2), 3); + assert.strictEqual(tierN(517, 518, -2), null); + assert.strictEqual(tierN(500, 518, -2), null); + assert.strictEqual(tierN(518, 518, -2), null); + assert.strictEqual(tierN(null, 518, -2), null); +}); + +test("inContinueWindow bounds the lattice tier", () => { + assert.ok(inContinueWindow(1, 6)); + assert.ok(inContinueWindow(6, 6)); + assert.ok(!inContinueWindow(7, 6)); + assert.ok(inContinueWindow(99, 0)); + assert.ok(!inContinueWindow(null, 6)); + assert.ok(!inContinueWindow(0, 6)); +}); + +test("reasoningGuardEngages gates on enabled only", () => { + assert.ok(!reasoningGuardEngages(undefined)); + assert.ok(!reasoningGuardEngages({})); + assert.ok(!reasoningGuardEngages({ enabled: false })); + assert.ok(reasoningGuardEngages({ enabled: true })); +}); + +test("hasEncryptedContent inspects the last reasoning item", () => { + assert.ok(!hasEncryptedContent([])); + assert.ok(hasEncryptedContent([{ encrypted_content: "abc" }])); + assert.ok(!hasEncryptedContent([{ encrypted_content: "" }])); + assert.ok(!hasEncryptedContent([{}])); + assert.ok(hasEncryptedContent([{ a: 1 }, { encrypted_content: "x" }])); +}); + +test("continueReason fires only within the lattice window", () => { + assert.strictEqual(continueReason({ output_tokens_details: { reasoning_tokens: 516 } }, 518, -2, 6), "truncation"); + assert.strictEqual(continueReason({ output_tokens_details: { reasoning_tokens: 500 } }, 518, -2, 6), ""); + assert.strictEqual(continueReason(null, 518, -2, 6), ""); +}); + +test("sumUsage accumulates token counters across rounds", () => { + const acc: Record = {}; + sumUsage(acc, { input_tokens: 100, output_tokens: 516, total_tokens: 616, output_tokens_details: { reasoning_tokens: 516 }, input_tokens_details: { cached_tokens: 40 } }); + sumUsage(acc, { input_tokens: 100, output_tokens: 1034, total_tokens: 1134, output_tokens_details: { reasoning_tokens: 1034 } }); + assert.strictEqual(acc.input_tokens, 200); + assert.strictEqual(acc.output_tokens, 516 + 1034); + assert.strictEqual(acc.total_tokens, 616 + 1134); + assert.strictEqual((acc["output_tokens_details"] as Record)["reasoning_tokens"], 516 + 1034); + assert.strictEqual((acc["input_tokens_details"] as Record)["cached_tokens"], 40); +}); + +test("agentUsage reports summed reasoning plus final non-reasoning part", () => { + const first = { input_tokens: 100, output_tokens: 516, total_tokens: 616, input_tokens_details: { cached_tokens: 40 }, output_tokens_details: { reasoning_tokens: 516 } }; + const summed = { input_tokens: 200, output_tokens: 1550, total_tokens: 1650, output_tokens_details: { reasoning_tokens: 1550 } }; + const finalRound = { output_tokens: 1560, output_tokens_details: { reasoning_tokens: 1550 } }; + const u = agentUsage(first, summed, finalRound, true); + assert.strictEqual(u.input_tokens, 100); + assert.strictEqual(u.output_tokens, 1560); + assert.strictEqual(u.total_tokens, 1660); + assert.strictEqual((u["output_tokens_details"] as Record)["reasoning_tokens"], 1550); + assert.deepStrictEqual(u["input_tokens_details"], { cached_tokens: 40 }); +}); + +test("nextRoundBody replays input, forces stream, adds encrypted include", () => { + const base = { model: "gpt-5.5", stream: false, input: ["a"], include: [], previous_response_id: "prev" }; + const nb = nextRoundBody(base, ["a", "b"]); + assert.strictEqual(nb.stream, true); + assert.deepStrictEqual(nb.input, ["a", "b"]); + assert.deepStrictEqual(nb.include, ["reasoning.encrypted_content"]); + assert.strictEqual(nb.previous_response_id, undefined); + const nb2 = nextRoundBody({ include: ["reasoning.encrypted_content", "other"] }, []); + assert.deepStrictEqual(nb2.include, ["reasoning.encrypted_content", "other"]); +}); + +test("commentaryNudge builds a phase:commentary message", () => { + const n = commentaryNudge("Keep going"); + assert.strictEqual(n.type, "message"); + assert.strictEqual(n.role, "assistant"); + assert.strictEqual(n.phase, "commentary"); + assert.deepStrictEqual(n.content, [{ type: "output_text", text: "Keep going" }]); +}); + +test("buildTerminalEvent stamps proxy metadata and honors incomplete", () => { + const ok = buildTerminalEvent({ + upstreamTerminal: { type: "response.completed", response: { status: "completed", id: "r" } }, + baseResponse: { id: "r", model: "gpt-5.5" }, + output: [{ type: "message" }], + usage: { input_tokens: 1 }, + rounds: [{ round: 1 }], + billed: { input_tokens: 1 }, + stoppedReason: "", + incompleteReason: "", + }); + assert.strictEqual(ok.type, "response.completed"); + const r = ok.response as Record; + assert.strictEqual(r.status, "completed"); + assert.strictEqual(((r.metadata as Record)["proxy_rounds"] as unknown[]).length, 1); + + const bad = buildTerminalEvent({ + upstreamTerminal: null, + baseResponse: null, + output: [], + usage: {}, + rounds: [], + billed: {}, + stoppedReason: "", + incompleteReason: "upstream_eof", + }); + assert.strictEqual(bad.type, "response.incomplete"); + assert.strictEqual((bad.response as Record).status, "incomplete"); +}); + +function sseRound(o: { reasoningTokens: number; encrypted?: string; messageText?: string; inputTokens?: number }): string { + const inputTokens = o.inputTokens ?? 100; + const nonReasoning = o.messageText ? 10 : 0; + const outputTokens = o.reasoningTokens + nonReasoning; + const events: Array> = []; + events.push({ type: "response.created", response: { id: "resp_x", status: "in_progress", model: "gpt-5.5" } }); + events.push({ type: "response.output_item.added", output_index: 0, item: { id: "rs_1", type: "reasoning", status: "in_progress" } }); + const reasonDone: Record = { id: "rs_1", type: "reasoning", status: "completed" }; + if (o.encrypted !== undefined) reasonDone.encrypted_content = o.encrypted; + events.push({ type: "response.output_item.done", output_index: 0, item: reasonDone }); + let oi = 1; + if (o.messageText) { + events.push({ type: "response.output_item.added", output_index: oi, item: { id: "msg_1", type: "message", role: "assistant", status: "in_progress", content: [] } }); + events.push({ type: "response.output_item.done", output_index: oi, item: { id: "msg_1", type: "message", role: "assistant", status: "completed", content: [{ type: "output_text", text: o.messageText }] } }); + oi++; + } + events.push({ + type: "response.completed", + response: { + id: "resp_x", + status: "completed", + model: "gpt-5.5", + output: [], + usage: { input_tokens: inputTokens, output_tokens: outputTokens, total_tokens: inputTokens + outputTokens, output_tokens_details: { reasoning_tokens: o.reasoningTokens } }, + }, + }); + let out = ""; + for (const e of events) out += `data: ${JSON.stringify(e)}\n\n`; + out += "data: [DONE]\n\n"; + return out; +} + +function streamFrom(text: string): ReadableStream { + const enc = new TextEncoder(); + return new ReadableStream({ + start(controller) { + controller.enqueue(enc.encode(text)); + controller.close(); + }, + }); +} + +function makeSink(): { sink: SseSink; frames: string[] } { + const frames: string[] = []; + const sink: SseSink = { + headersSent: false, + writableEnded: false, + writeHead() { + sink.headersSent = true; + }, + write(chunk) { + frames.push(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8")); + return true; + }, + end() { + sink.writableEnded = true; + }, + once() { + return undefined; + }, + }; + return { sink, frames }; +} + +function parseFrames(frames: string[]): Array> { + const lines = frames.map((f) => f.trim()).filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim()).filter(Boolean); + const out: Array> = []; + for (const l of lines) { + if (l === "[DONE]") continue; + out.push(JSON.parse(l) as Record); + } + return out; +} + +test("runReasoningGuard folds truncated gpt-5.5 rounds into one completed response", async () => { + let contCount = 0; + const bodies: string[] = []; + const server = http.createServer((req, res) => { + let data = ""; + req.on("data", (c) => (data += c)); + req.on("end", () => { + bodies.push(data); + contCount++; + const payload = contCount === 1 + ? sseRound({ reasoningTokens: 1034, encrypted: "enc2" }) + : sseRound({ reasoningTokens: 900, encrypted: "enc3", messageText: "The answer." }); + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(payload); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const addr = server.address(); + const port = typeof addr === "object" && addr !== null ? addr.port : 0; + const url = `http://127.0.0.1:${port}/v1/responses`; + + try { + const firstResponse = new Response(streamFrom(sseRound({ reasoningTokens: 516, encrypted: "enc1" })), { headers: { "content-type": "text/event-stream" } }); + const { sink, frames } = makeSink(); + let cleared = 0; + await runReasoningGuard({ + firstResponse, + clearFirstTimer: () => { + cleared++; + }, + upstreamUrl: url, + reqHeaders: { "content-type": "application/json" }, + originalBody: JSON.stringify({ model: "gpt-5.5", stream: true, input: [{ type: "message", role: "user", content: "hi" }] }), + signal: new AbortController().signal, + res: sink, + config: { enabled: true }, + log: () => {}, + }); + + assert.strictEqual(contCount, 2, "exactly two continuation rounds"); + assert.ok(cleared >= 1, "upstream idle timer cleared"); + const events = parseFrames(frames); + const terminals = events.filter((e) => e.type === "response.completed" || e.type === "response.incomplete"); + assert.strictEqual(terminals.length, 1, "exactly one terminal event"); + const terminal = terminals[0]; + assert.strictEqual(terminal.type, "response.completed"); + const resp = terminal.response as Record; + assert.strictEqual(resp.status, "completed"); + const usage = resp.usage as Record; + assert.strictEqual((usage["output_tokens_details"] as Record)["reasoning_tokens"], 516 + 1034 + 900); + assert.strictEqual(usage.input_tokens, 100); + assert.strictEqual(usage.output_tokens, 2450 + 10); + assert.strictEqual(usage.total_tokens, 100 + 2460); + const meta = resp.metadata as Record; + assert.strictEqual((meta["proxy_rounds"] as unknown[]).length, 3); + const outputs = (resp.output ?? []) as Array>; + assert.ok(outputs.some((it) => it.type === "message" && ((it.content ?? []) as Array>).some((c) => c.text === "The answer."))); + + assert.ok(bodies[0].includes("Continue thinking..."), "first continuation carries the nudge"); + assert.ok(bodies[0].includes("enc1"), "first continuation replays round-1 encrypted reasoning"); + assert.ok(bodies[1].includes("enc2"), "second continuation replays round-2 encrypted reasoning"); + assert.ok(bodies[0].includes("reasoning.encrypted_content"), "include carries encrypted_content"); + } finally { + server.close(); + } +}); + +test("runReasoningGuard passes a clean single round through without continuation", async () => { + const { sink, frames } = makeSink(); + const firstResponse = new Response(streamFrom(sseRound({ reasoningTokens: 800, encrypted: "enc1", messageText: "done" })), { headers: { "content-type": "text/event-stream" } }); + await runReasoningGuard({ + firstResponse, + clearFirstTimer: () => {}, + upstreamUrl: "http://127.0.0.1:1/v1/responses", + reqHeaders: { "content-type": "application/json" }, + originalBody: JSON.stringify({ model: "gpt-5.5", stream: true, input: [] }), + signal: new AbortController().signal, + res: sink, + config: { enabled: true }, + log: () => {}, + }); + const events = parseFrames(frames); + const terminals = events.filter((e) => e.type === "response.completed" || e.type === "response.incomplete"); + assert.strictEqual(terminals.length, 1); + assert.strictEqual(terminals[0].type, "response.completed"); + const resp = terminals[0].response as Record; + const usage = resp.usage as Record; + assert.strictEqual((usage["output_tokens_details"] as Record)["reasoning_tokens"], 800); + const outputs = (resp.output ?? []) as Array>; + assert.ok(outputs.some((it) => it.type === "message")); +});