diff --git a/docs/configuration.md b/docs/configuration.md index 58d24deea1..f5872b8f7c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1402,7 +1402,7 @@ Interaction with the budget: the achievable fold rate of a batch is bounded by h Deadline semantics (#4833): `targetTimeoutMs` caps every single dispatch — one slow call cannot spend the whole `totalTimeoutMs`, even on a one-target chain (each target in the fall-through order gets its own fresh `targetTimeoutMs` window, all drawn from the shared total). When a deadline expires the in-flight attempt is aborted and **not retried**: the bounded retry loop (`attempt 1/3`) re-dispatches only transient provider verdicts (429/5xx/risk-control body markers) and fail-fast network deaths; a deadline abort ends the attempt immediately. Failure logging names which clock expired: the per-attempt warn line carries the source — `target-timeout`, `batch-deadline`, or `client-abort` for a genuine downstream cancellation — plus safe diagnostics (elapsed time, the budgets in force, wire protocol/stream mode, payload byte size, an input token estimate, and the output character cap), and the `completed N/M range(s)` line records each attempt's detail (`attempts=[t0:timeout (target-timeout 25000ms (elapsed 25002ms))]`); the same diagnostics ride the tool result when a `compress` call fails wholesale. Sizing for large-context workloads: a non-streaming summary pays its full generation latency before any byte arrives, so `targetTimeoutMs` must cover input processing plus complete summary generation at your largest expected input — several hundred thousand tokens routinely exceeds the 25 s default on slower endpoints. If you see `target-timeout` on big folds, raise `targetTimeoutMs` (and keep `totalTimeoutMs` large enough for the fall-through depth you expect) rather than disabling the chain. -Two optional knobs turn the chain into a **proxy-driven auto-folder** (`autoFold`): with `"autoFold": true` the proxy stops waiting for the model to call the compression tool at all — the same zero-model folding engine that guards the overflow window also folds the conversation down to `autoFoldTargetTokens` (default: half the model window) *before* forwarding, using the external chain and kernel-chosen ranges. The model never sees a nudge and never spends thinking tokens drafting compression plans. The knob rides whole-chain replace semantics like the rest of the block; it only takes effect while the chain itself is enabled — setting either knob on a disabled (or omitted-`enabled`) chain has no effect and is logged once as a warning rather than failing silently. `autoFoldTargetTokens` must be an integer in `[8192, 10000000]`. Failure is fail-open: if the chain errors at fold time, the request forwards as-is, a 10-minute backoff suppresses auto-folding, and classic advisory nudges return for that window. One exception (#2662): a zero-progress fold caused purely by queue drops — batches submitted but no summary call ever dispatched because the shared pool is saturated — does not arm the backoff, since that is scheduler congestion rather than a chain failure; auto-fold simply retries on the next turn. +Two optional knobs turn the chain into a **proxy-driven auto-folder** (`autoFold`): with `"autoFold": true` the proxy stops waiting for the model to call the compression tool at all — the same zero-model folding engine that guards the overflow window also folds the conversation down to `autoFoldTargetTokens` (default: half the model window) *before* forwarding, using the external chain and kernel-chosen ranges. The model never sees a nudge and never spends thinking tokens drafting compression plans. The knob rides whole-chain replace semantics like the rest of the block; it only takes effect while the chain itself is enabled — setting either knob on a disabled (or omitted-`enabled`) chain has no effect and is logged once as a warning rather than failing silently. `autoFoldTargetTokens` must be an integer in `[8192, 10000000]`. Failure is fail-open: if the chain errors at fold time, the request forwards as-is, a 10-minute backoff suppresses auto-folding, and classic advisory nudges return for that window. One exception (#2662): a zero-progress fold caused purely by queue drops — batches submitted but no summary call ever dispatched because the shared pool is saturated — does not arm the backoff, since that is scheduler congestion rather than a chain failure; auto-fold simply retries on the next turn. Per-route arming caveat: the target is clamped to each route's effective overflow bar (window minus any proven completion reservation), and growth folding arms only while the clamped target stays strictly below that bar — an explicit `autoFoldTargetTokens` at or above a route's bar therefore leaves auto-fold inert on that route (only overflow folds run). The proxy logs a one-time warning naming both values whenever that state is reached (#4543); if one global target must serve wide- and narrow-window routes together, set it below the narrowest route you want armed. Operators running CJK-heavy long sessions should also consider this rail as a mitigation for fold-loss amplification across mid-session model switches (#1921/#3301/#3849): with auto-fold armed, the pre-forward fold happens before a single-pass history re-send can push the payload past the window, instead of only after the first upstream refusal. Auto-fold growth triggers are benefit-gated (#3444), not bare threshold crossings: a growth fold arms only when the text channel exceeds the effective target budget by at least **4096 tokens** — payloads sitting at (or barely past) the target forward as-is instead of burning a full external-summary round trip to save less than its fixed cost. The same gate applies post-fold: a growth fold that ends up saving fewer than 4096 tokens arms the 10-minute backoff exactly like a zero-progress pass. Image-dominant payloads are exempt (images are not foldable, so their overage is never "expected benefit"), and classic overflow folding is untouched — there, firing is mandatory. Additionally, the effective target has a **measured floor**: when the session's irreducible baseline (system prompt + tool definitions + live tail) is larger than the configured `autoFoldTargetTokens`, the target is raised to that measured size and a warning names the real number — an unreachable target would otherwise re-arm a futile fold on every request. The floor never lifts the target above the model window. diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index 4d9adc78fd..4ee1f97ad0 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -1399,7 +1399,7 @@ URL 键通道(或带 `bind` 的命名条目)可以向上游发送**自己的 deadline 语义(#4833):`targetTimeoutMs` 限制的是**每一次单独派发**——即使链上只有一个目标,一次慢调用也不能吃掉整个 `totalTimeoutMs`(回退顺序里的每个目标都拿到自己全新的 `targetTimeoutMs` 窗口,全部从共享的总预算里扣)。某个 deadline 到期时,在途的这次尝试会被中止且**不会重试**:有界重试循环(`attempt 1/3`)只对瞬时 provider 判定(429/5xx/风控 body 标记)和 fail-fast 网络死亡重新派发;deadline 中止会立即终结该次尝试。失败日志会标明是哪只时钟到期:每次尝试的 warn 行携带来源——`target-timeout`、`batch-deadline`,或真正的下游取消时的 `client-abort`——外加安全诊断(已耗时、实际生效的预算、协议/流式模式、payload 字节数、输入 token 估计与输出字符上限),`completed N/M range(s)` 行记录每个尝试的明细(`attempts=[t0:timeout (target-timeout 25000ms (elapsed 25002ms))]`);当一次 `compress` 整体失败时,同样的诊断也会进入工具结果。大上下文场景取值建议:非流式摘要在收到任何字节之前就要付完整生成延迟,所以 `targetTimeoutMs` 必须覆盖你最大预期输入下的处理 + 完整生成时间——几十万 token 在较慢端点上经常超过默认的 25 秒。若大折叠出现 `target-timeout`,应调高 `targetTimeoutMs`(并保持 `totalTimeoutMs` 足以容纳你预期的回退深度),而不是直接禁用整条链。 -两个可选旋钮可把这条链升级为**代理驱动的自动折叠器**(`autoFold`):`"autoFold": true` 后代理不再等模型调压缩工具 —— 原本只守溢出窗口的零模型折叠引擎同样会在转发前把对话折到 `autoFoldTargetTokens`(默认:模型窗口的一半)以下,范围由内核选定、摘要由外部链完成。主模型完全看不到 nudge,也不会把思考预算花在起草压缩方案上。该旋钮随整链替换语义生效,且只在链本身启用时才起作用——在链未启用(或省略 `enabled`)时设置任一旋钮均无效,并会记录一次警告而非静默失效;`autoFoldTargetTokens` 必须是 `[8192, 10000000]` 内的整数。失败时 fail-open:链在折叠时出错则照常转发请求,退避 10 分钟内暂停自动折叠并恢复经典 advisory nudge。一个例外(#2662):若零进展完全由队列丢弃造成——批次已提交但因共享池饱和、没有任何一次摘要调用真正派发——则不触发退避(那是调度拥塞而非链故障),auto-fold 在下一轮直接重试。 +两个可选旋钮可把这条链升级为**代理驱动的自动折叠器**(`autoFold`):`"autoFold": true` 后代理不再等模型调压缩工具 —— 原本只守溢出窗口的零模型折叠引擎同样会在转发前把对话折到 `autoFoldTargetTokens`(默认:模型窗口的一半)以下,范围由内核选定、摘要由外部链完成。主模型完全看不到 nudge,也不会把思考预算花在起草压缩方案上。该旋钮随整链替换语义生效,且只在链本身启用时才起作用——在链未启用(或省略 `enabled`)时设置任一旋钮均无效,并会记录一次警告而非静默失效;`autoFoldTargetTokens` 必须是 `[8192, 10000000]` 内的整数。失败时 fail-open:链在折叠时出错则照常转发请求,退避 10 分钟内暂停自动折叠并恢复经典 advisory nudge。一个例外(#2662):若零进展完全由队列丢弃造成——批次已提交但因共享池饱和、没有任何一次摘要调用真正派发——则不触发退避(那是调度拥塞而非链故障),auto-fold 在下一轮直接重试。按路由武装的注意项:目标值会被 clamp 到每条路由各自的有效溢出线(窗口减去已证实的完成量预留),且只有 clamp 后的目标严格低于该线时 growth folding 才武装——显式设置的 `autoFoldTargetTokens` 若 ≥ 某条路由的溢出线,auto-fold 在该路由上就是惰性的(只在 overflow 时折叠)。代理到达该状态时会记录一条一次性警告并写明两个数值(#4543);若同一个全局目标要同时服务宽窗口与窄窗口路由,请把它设到你想武装的最窄路由之下。运行以中文为主的长会话的运维还应把这条链作为「会话中途换模型引发的折叠损失放大」(#1921/#3301/#3849)的缓解手段:auto-fold 武装后,pre-forward 折叠发生在单轮历史重放把载荷推过窗口之前,而不是第一次上游拒绝之后。 auto-fold 的增长触发是**收益门槛制**(#3444),不是裸阈值:只有文本通道超出有效目标预算至少 **4096 token** 才会武装增长折叠——已坐在目标上(或刚刚过线)的载荷会直接照常转发,不会为了省不到固定成本的量去烧一整轮外部摘要往返。折叠后同一道门槛再算一次:一场增长折叠实际省下不足 4096 token,就按零进展同样处理,武装 10 分钟退避。图片主导的载荷豁免(图片不可折叠,其超出量永远算不上“预期收益”);经典溢出折叠不受影响——那是必须触发的。另外,有效目标有一个**实测下限**:当会话不可压缩的基线(系统提示词 + 工具定义 + 存活尾巴)大于配置的 `autoFoldTargetTokens` 时,目标会被抬到实测值并告警提示真实数字——否则一个永远够不着的目标会在每轮请求上重复武装注定徒劳的折叠。下限永远不会把目标抬过模型窗口。 diff --git a/src/server.ts b/src/server.ts index f06ee81a2b..9524a0c5b5 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3860,6 +3860,7 @@ export async function preflightCompressIfNeeded( // floor under the documented minimum. Clamping to the minimum disarms // auto-fold on sub-8K windows outright (target >= overflow ⇒ growthArmed // false), which matches the documented feature scope. + const configuredAutoFoldTarget = (config as ResolvedKernelConfig).externalSummary?.autoFoldTargetTokens; // [#3444] A configured target below what this session can physically reach // (system prompt + tool schemas + live tail) is UNREACHABLE: every fold // lands above it and the trigger re-fires on every request (fold storm). @@ -3871,10 +3872,19 @@ export async function preflightCompressIfNeeded( // releases via growthFoldingArmed's measuredFloor arg). const measuredFloor = autoFoldOn ? autoFoldFloor(session) : undefined; const autoFoldTarget = autoFoldOn - ? Math.min(Math.max(Math.max(AUTO_FOLD_TARGET_MIN, Math.min((config as ResolvedKernelConfig).externalSummary?.autoFoldTargetTokens ?? (overflowTarget > 0 ? Math.round(overflowTarget / 2) : 0), overflowTarget)), measuredFloor ?? 0), overflowTarget) + ? Math.min(Math.max(Math.max(AUTO_FOLD_TARGET_MIN, Math.min(configuredAutoFoldTarget ?? (overflowTarget > 0 ? Math.round(overflowTarget / 2) : 0), overflowTarget)), measuredFloor ?? 0), overflowTarget) : undefined; const growthArmed = autoFoldOn && autoFoldTarget !== undefined && autoFoldTarget > 0 && autoFoldTarget < overflowTarget; // narrowing form of growthFoldingArmed(config, overflowTarget, measuredFloor) — keep in sync (PR #2581 review) const compressionTarget = growthArmed ? autoFoldTarget : overflowTarget; + // #4543: an explicit autoFoldTargetTokens ≥ this route's overflow target clamps + // to it and leaves growth folding SILENTLY inert (noteAutoFoldIgnored only + // covers the disabled-chain no-op). Warn once per session per distinct route + // bar — logging only, no behavior change. + if (autoFoldOn && typeof configuredAutoFoldTarget === "number" && overflowTarget > 0 && configuredAutoFoldTarget >= overflowTarget + && session.metadata.autoFoldInertBarWarned !== overflowTarget) { + session.metadata.autoFoldInertBarWarned = overflowTarget; + log("warn", `[external-summary] auto-fold INERT on this route: autoFoldTargetTokens=${configuredAutoFoldTarget} is at or above this route's overflow target ${overflowTarget}; growth folding arms only while the target is below the overflow bar, so no pre-forward growth fold will run here — folding happens only at overflow. Set autoFoldTargetTokens below ${overflowTarget} (or use a wider-window route) to arm it (#4543).`); + } // A fresh session (id rotated, e.g. after a model switch) has // lastInputTokens = 0 while still carrying a full raw history; size the // trigger on the real post-fold payload too. outboundPayloadBreakdown is the diff --git a/tests/external-summary-autofold.test.ts b/tests/external-summary-autofold.test.ts index 85ce582b5f..194edd87df 100644 --- a/tests/external-summary-autofold.test.ts +++ b/tests/external-summary-autofold.test.ts @@ -9,6 +9,7 @@ import { startServer, type ProxyOptions } from "../src/server.ts"; import { defaultConfig } from "acp-kernel"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { setLogCapture } from "../src/logger.ts"; /** Anthropomorphic main upstream: records request bodies, answers SSE ok. */ function startMainUpstream(): Promise<{ server: http.Server; url: string; bodies: unknown[] }> { @@ -730,3 +731,114 @@ test("explicit autoFoldTargetTokens clamped to the window: classic nudges still const text = JSON.stringify(main.bodies.at(-1)); assert.ok(text.includes("Context breakdown:"), "classic nudge must fire when the explicit target clamps to the window"); }); + +test("explicit autoFoldTargetTokens above the route bar warns once that auto-fold is inert (#4543)", async () => { + // The operator trap: a configured target at or above the route's overflow + // bar clamps to it and silently disarms the pre-forward growth fold. The + // warn must fire exactly once per session per route bar and name both values. + // Target deliberately far ABOVE the route window: the env may floor the + // judge window upward (fallback floor for unproven models), so a value + // above every plausible bar keeps the inert condition deterministic. + const main = await startMainUpstream(); trackClose(main.server); + const sum = await startSummaryUpstream("ok"); trackClose(sum.server); + const server = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: main.url, + routes: { [main.url]: { models: { "test-model": { context: 50_000 } } } }, + modelContextLimit: 50_000, + kernelConfig: defaultConfig(50_000, { preserveRecentMessages: 0, preserveRecentTokens: 0, compress: { minCompressRange: 100, maxSummaryLength: 20000, minSummaryLength: 50 } }), + compress: { + injectTool: true, + injectNudge: true, + externalSummary: { enabled: true, targets: ["sum/sm"], autoFold: true, autoFoldTargetTokens: 500_000 }, + }, + namedProviders: { sum: { baseUrl: sum.url, api: "openai", apiKeyEnv: "E2E_SUM_KEY", models: { sm: {} } } }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: false, + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + trackClose(server); + await once(server, "listening"); + const port = (server.address() as AddressInfo).port; + process.env.E2E_SUM_KEY = "k"; + + const warns: string[] = []; + setLogCapture((level, msg) => { if (level === "warn") warns.push(msg); }); + try { + const first = await postTurn(port, main.url, "inert-bar", 2); + assert.equal(first.status, 200, "turn 1 forwards cleanly despite the inert config"); + let inertLines = warns.filter((m) => m.includes("auto-fold INERT")); + assert.equal(inertLines.length, 1, JSON.stringify(warns)); + assert.ok(inertLines[0].includes("autoFoldTargetTokens=500000"), inertLines[0]); + assert.ok(/overflow target \d+/.test(inertLines[0]), inertLines[0]); + assert.ok(inertLines[0].includes("#4543"), inertLines[0]); + + const second = await postTurn(port, main.url, "inert-bar", 2); + assert.equal(second.status, 200, "turn 2 forwards cleanly"); + inertLines = warns.filter((m) => m.includes("auto-fold INERT")); + assert.equal(inertLines.length, 1, `throttled to one warn per session per route bar: ${JSON.stringify(warns)}`); + } finally { + setLogCapture(null); + } +}); + +test("no inert warning while growth folding is armed (target below the window)", async () => { + // Negative arm of the same gate: an explicit target that stays below the + // overflow bar arms growth folding and must not produce the inert warning. + const main = await startMainUpstream(); trackClose(main.server); + const sum = await startSummaryUpstream("ok"); trackClose(sum.server); + const server = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: main.url, + routes: { [main.url]: { models: { "test-model": { context: 50_000 } } } }, + modelContextLimit: 50_000, + kernelConfig: defaultConfig(50_000, { preserveRecentMessages: 0, preserveRecentTokens: 0, compress: { minCompressRange: 100, maxSummaryLength: 20000, minSummaryLength: 50 } }), + compress: { + injectTool: true, + injectNudge: true, + externalSummary: { enabled: true, targets: ["sum/sm"], autoFold: true, autoFoldTargetTokens: 8192 }, + }, + namedProviders: { sum: { baseUrl: sum.url, api: "openai", apiKeyEnv: "E2E_SUM_KEY", models: { sm: {} } } }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: false, + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + trackClose(server); + await once(server, "listening"); + const port = (server.address() as AddressInfo).port; + process.env.E2E_SUM_KEY = "k"; + + const warns: string[] = []; + setLogCapture((level, msg) => { if (level === "warn") warns.push(msg); }); + try { + const turn = await postTurn(port, main.url, "armed-bar", 2); + assert.equal(turn.status, 200, "turn forwards cleanly with an armed target"); + assert.equal(warns.filter((m) => m.includes("auto-fold INERT")).length, 0, JSON.stringify(warns)); + } finally { + setLogCapture(null); + } +});