diff --git a/src/affinity-persist.ts b/src/affinity-persist.ts new file mode 100644 index 0000000000..02957199e9 --- /dev/null +++ b/src/affinity-persist.ts @@ -0,0 +1,72 @@ +import fs from "node:fs"; +import path from "node:path"; +import { prefixAffinity } from "./prefix-affinity.js"; +import { stateDir } from "./paths.js"; +import { log } from "./logger.js"; + +/** + * #499 P1a: prefix-affinity persistence. The anonymous affinity chains were + * pure in-memory (#309), so a proxy restart orphaned every anonymous session: + * the next replay forked a fresh session with zero compression state and + * resent the raw history (#351 — 458K tokens, 0% cache). The chains are + * small (≤256 sessions × ≤128 hashes); persist them to the state dir with a + * debounced atomic write and hydrate on boot. + */ + +const PERSIST_DEBOUNCE_MS = 5_000; + +function affinityFile(): string { + return path.join(stateDir(), "prefix-affinity.json"); +} + +let timer: NodeJS.Timeout | null = null; +let writing = false; + +function writeSnapshot(): void { + if (writing) return; + writing = true; + try { + const file = affinityFile(); + const snapshot = { version: 1, entries: prefixAffinity.exportSnapshot() }; + const tmp = `${file}.tmp`; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(tmp, JSON.stringify(snapshot)); + fs.renameSync(tmp, file); + } catch (e) { + log("warn", `[prefix-affinity] persist failed (${e instanceof Error ? e.message : String(e)}); affinity survives in memory`); + } finally { + writing = false; + } +} + +/** Debounced snapshot write — call after every affinity mutation. */ +export function scheduleAffinityPersist(): void { + if (timer) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + writeSnapshot(); + }, PERSIST_DEBOUNCE_MS); + timer.unref?.(); +} + +/** Immediate snapshot write — shutdown path. */ +export function flushPrefixAffinity(): void { + if (timer) { + clearTimeout(timer); + timer = null; + } + writeSnapshot(); +} + +/** Load the snapshot a previous process left behind. Call once on boot. */ +export function hydratePrefixAffinity(): void { + try { + const file = affinityFile(); + if (!fs.existsSync(file)) return; + const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as Record; + const imported = prefixAffinity.importSnapshot(parsed.entries); + if (imported > 0) log("info", `[prefix-affinity] hydrated ${imported} chain(s) from ${path.basename(file)} — anonymous sessions reattach across restarts`); + } catch (e) { + log("warn", `[prefix-affinity] hydrate failed (${e instanceof Error ? e.message : String(e)}); starting with empty affinity`); + } +} diff --git a/src/cache-warn.ts b/src/cache-warn.ts new file mode 100644 index 0000000000..763bb67a65 --- /dev/null +++ b/src/cache-warn.ts @@ -0,0 +1,56 @@ +import type { Session } from "./session.js"; +import { log } from "./logger.js"; + +/** + * #499 P1b: upstream prompt-cache collapse warning. A warm session normally + * rides the provider's prefix cache (90%+ hit). When a restart or a session + * fork makes every request cold, the raw resend is expensive but INVISIBLE: + * requests succeed, so no error path fires — the only trace is a hit ratio + * that fell off a cliff. This watches each usage sample; once a session has + * demonstrated a healthy cache (≥HIGH_HIT on a large input) and then logs + * LOW_RUN consecutive large inputs with ~zero hits, it warns ONCE with the + * numbers and the likely causes. Diagnostics only — no state is changed. + */ + +const MIN_INPUT = 8000; +const HIGH_HIT = 0.5; +const LOW_HIT = 0.1; +const LOW_RUN = 5; +const MAX_TRACKED_SESSIONS = 1024; + +interface CacheWatch { + sawHigh: boolean; + lowRun: number; + warned: boolean; +} + +const watches = new Map(); + +/** Feed one usage sample (per successful upstream turn). */ +export function warnCacheCollapse(session: Session, input: number, cached: number): void { + if (input < MIN_INPUT) return; + if (watches.size > MAX_TRACKED_SESSIONS) { + const oldest = watches.keys().next().value; + if (oldest !== undefined) watches.delete(oldest); + } + const watch = watches.get(session.id) ?? { sawHigh: false, lowRun: 0, warned: false }; + watches.set(session.id, watch); + const hit = cached / input; + if (hit >= HIGH_HIT) { + watch.sawHigh = true; + watch.lowRun = 0; + return; + } + if (!watch.sawHigh || watch.warned) return; + if (hit > LOW_HIT) { + watch.lowRun = 0; + return; + } + watch.lowRun++; + if (watch.lowRun < LOW_RUN) return; + watch.warned = true; + log( + "warn", + `[${session.id}] upstream prompt-cache collapse: ${LOW_RUN} consecutive large requests at ~0% hit (last: input=${input}, cached=${cached}) after earlier ≥${Math.round(HIGH_HIT * 100)}% hits — the context is being resent cold every turn. Likely causes: proxy/session restart re-forked the session (#499), or the upstream KV cache was dropped (window/deployment change). Check the session id upstream sees and compress.modelContextLimit.`, + ); +} diff --git a/src/loop/core.ts b/src/loop/core.ts index e0dafed800..1c3178c150 100644 --- a/src/loop/core.ts +++ b/src/loop/core.ts @@ -19,6 +19,8 @@ import { resolveDecompress } from "../decompress-shared.js"; import { buildVisibilityMarker } from "../compress-loop.js"; import { fetchWithRetry, UpstreamHttpError } from "../fetch-util.js"; import { proxyDispatcher } from "../upstream-proxy.js"; +import { noteWeakOverflow } from "../weak-overflow.js"; +import { warnCacheCollapse } from "../cache-warn.js"; import { log as loggerLog } from "../logger.js"; import type { WireProtocol } from "../util.js"; @@ -199,6 +201,7 @@ function recordUsage( if (typeof out === "number") ctx.session.stats.outputTokens += out; const hitPct = typeof cached === "number" && total > 0 ? Math.round((cached / total) * 100) : 0; + warnCacheCollapse(ctx.session, total, cached ?? 0); ctx.log( `[acp-usage] round ${round} input=${total} cached=${cached ?? 0} (cache hit ${hitPct}%)`, ); @@ -521,6 +524,18 @@ export async function* runCompressLoop( // blind to why it failed. MAX_LOOP_ROUNDS bounds runaway loops. const reRequest = proxyResults.length > 0 && realCalls === 0; if (!reRequest) { + // #498: a stream that cut without a completion event is a weak + // overflow signal (sglang-style backends accept an oversized + // prompt then die mid-stream). Both shapes: !sawDone (chat / + // anthropic) and truncatedDone (responses synthetic failed done). + if (!sawDone || truncatedDone) { + const reqModel = typeof requestBody["model"] === "string" ? requestBody["model"] : undefined; + noteWeakOverflow(ctx.session, { + inputTokens: usage.inputTokens, + model: reqModel, + reason: `round ${round}: upstream stream truncated without a completion event`, + }); + } if (!sawDone) { const partialText = assistantText.length; const partialReasoning = assistantReasoning.length; @@ -589,6 +604,12 @@ export async function* runCompressLoop( const suffix = e.attempts > 1 ? ` after ${e.attempts} attempt(s)` : ""; ctx.log(`[acp-proxy: compress loop upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}]`); loggerLog("error", `[acp-loop] upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}`); + const reqModel = typeof requestBody["model"] === "string" ? requestBody["model"] : undefined; + noteWeakOverflow(ctx.session, { + inputTokens: usage.inputTokens, + model: reqModel, + reason: `round ${round}: upstream error ${e.status}${suffix}`, + }); yield adapter.emitError(`upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}`); return; } diff --git a/src/plugin.ts b/src/plugin.ts index 1bd5933206..af083789a4 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -9,6 +9,8 @@ import { executeProxyTool } from "./loop/core.js"; import { normalizeSseLineEndings } from "./sse-util.js"; import { containsRenderTagText, createTagEchoFilter, mayStartRenderTag, stripAnthropicText, stripOpenaiChatText, stripResponsesText, type TagEchoFilter } from "./loop/tag-echo-filter.js"; import { log as loggerLog } from "./logger.js"; +import { noteWeakOverflow } from "./weak-overflow.js"; +import { warnCacheCollapse } from "./cache-warn.js"; import type { WireProtocol } from "./util.js"; import { stateDir } from "./paths.js"; @@ -549,6 +551,7 @@ function applyUsageSample(session: Session, sample: UsageSample, protocol?: Wire // Net out pending compress savings (see stream.ts applyRanges): plugin // compress tool results shrink the next request, not this report. session.stats.lastInputTokens = Math.max(0, total - (session.stats.compressCreditTokens ?? 0)); + warnCacheCollapse(session, total, sample.cachedTokens ?? 0); } if (sample.outputTokens !== undefined) session.stats.outputTokens += sample.outputTokens; } @@ -640,6 +643,17 @@ export async function pipePluginChatWithStrip( markDirty(session); } }; + // #498: whether a terminal event ([DONE] / message_stop) was seen. A + // stream that ends without one was cut mid-flight — a weak overflow + // signal on high-usage sessions. + let sawTerminal = false; + const maybeNoteTruncated = () => { + if (sawTerminal || !session || res.destroyed || res.writableEnded) return; + noteWeakOverflow(session, { + inputTokens: acc.inputTokens, + reason: "plugin chat passthrough stream ended without a completion event", + }); + }; const pushField = (field: string, index: number, text: string): [string, boolean] => { const s = filterFor(field, index); const clean = s.filter.push(text); @@ -729,6 +743,7 @@ export async function pipePluginChatWithStrip( const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim(); if (!jsonStr) continue; if (jsonStr === "[DONE]") { + sawTerminal = true; await write(flushTails() + rawEvent + "\n\n"); continue; } @@ -739,6 +754,7 @@ export async function pipePluginChatWithStrip( await write(rawEvent + "\n\n"); continue; } + if (ev["type"] === "message_stop") sawTerminal = true; const sample = usageFromSseEvent(ev); if (sample) mergeUsageSample(acc, sample); const out = protocol === "anthropic" ? processAnthropic(ev, rawEvent) : processOpenai(ev, rawEvent); @@ -750,8 +766,10 @@ export async function pipePluginChatWithStrip( const rest = flushTails(); if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(rest); settleUsage(); + maybeNoteTruncated(); } catch (e) { settleUsage(); + maybeNoteTruncated(); if (res.destroyed || res.writableEnded) { log?.("client aborted mid-stream"); return; @@ -820,6 +838,17 @@ export async function pipePluginResponsesWithStrip( markDirty(session); } }; + // #498: whether a terminal event (done-family / [DONE]) was seen. A + // stream that ends without one was cut mid-flight — a weak overflow + // signal on high-usage sessions. + let sawTerminal = false; + const maybeNoteTruncated = () => { + if (sawTerminal || !session || res.destroyed || res.writableEnded) return; + noteWeakOverflow(session, { + inputTokens: acc.inputTokens, + reason: "plugin responses passthrough stream ended without a completion event", + }); + }; let lastDeltaMeta: { item_id?: unknown; output_index?: unknown } | null = null; const flushTail = (after: string) => { const tail = tagFilter.flush(); @@ -843,6 +872,7 @@ export async function pipePluginResponsesWithStrip( if (dataLines.length === 0) continue; const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim(); if (!jsonStr || jsonStr === "[DONE]") { + if (jsonStr === "[DONE]") sawTerminal = true; await write(rawEvent + "\n\n"); continue; } @@ -864,6 +894,7 @@ export async function pipePluginResponsesWithStrip( type === "response.failed" || type === "response.incomplete" ) { + if (type === "response.completed" || type === "response.failed" || type === "response.incomplete") sawTerminal = true; // done-family events also carry full text payloads — strip those too. const out = containsRenderTagText(jsonStr) ? rebuildEvent(rawEvent, stripResponsesText(ev)) : rawEvent + "\n\n"; await write(flushTail(out)); @@ -901,8 +932,10 @@ export async function pipePluginResponsesWithStrip( if (rest.length > 0) await write(rest); } settleUsage(); + maybeNoteTruncated(); } catch (e) { settleUsage(); + maybeNoteTruncated(); if (res.destroyed || res.writableEnded) { log?.("client aborted mid-stream"); return; diff --git a/src/prefix-affinity.ts b/src/prefix-affinity.ts index 22a8dbed97..f0caaf3695 100644 --- a/src/prefix-affinity.ts +++ b/src/prefix-affinity.ts @@ -100,6 +100,18 @@ interface ChainEntry { itemHashes: string[]; } +/** On-disk snapshot entry (#499 P1a): the tracked chains persisted across + * restarts so an anonymous replay reattaches its session instead of + * forking a fresh one with zero compression state (the #351 failure mode: + * a 458K-token history resent raw because the affinity was in-memory). */ +export interface AffinitySnapshotEntry { + sessionId: string; + depth: number; + tailHash: string; + itemHashes: string[]; + lastSeen: number; +} + /** Deterministic JSON with recursively sorted object keys, so two replays * of the same logical message hash identically regardless of key order. */ export function stableStringify(value: unknown): string { @@ -290,6 +302,50 @@ export class PrefixAffinityResolver { return [...this.trackedChains.keys()]; } + /** Serializable copy of the tracked chains (for #499 P1a persistence). */ + exportSnapshot(): AffinitySnapshotEntry[] { + return [...this.trackedChains.values()].map((e) => ({ + sessionId: e.sessionId, + depth: e.depth, + tailHash: e.tailHash, + itemHashes: e.itemHashes, + lastSeen: e.lastSeen, + })); + } + + /** Load chains persisted by a previous process. Entries older than the + * TTL are dropped (they would not match anyway). Returns the count + * actually imported. Defensive: a corrupt/hand-edited file must never + * crash the proxy — malformed entries are skipped. */ + importSnapshot(entries: unknown): number { + if (!Array.isArray(entries)) return 0; + const now = Date.now(); + let imported = 0; + for (const raw of entries) { + if (!raw || typeof raw !== "object") continue; + const e = raw as Record; + if (typeof e.sessionId !== "string" || typeof e.depth !== "number" || typeof e.tailHash !== "string") continue; + if (!Array.isArray(e.itemHashes) || e.itemHashes.some((h) => typeof h !== "string")) continue; + if (typeof e.lastSeen !== "number" || now - e.lastSeen > TTL_MS) continue; + const entry: ChainEntry = { + sessionId: e.sessionId, + depth: e.depth, + tailHash: e.tailHash, + itemHashes: (e.itemHashes as string[]).slice(-MAX_STORED_ITEMS), + lastSeen: e.lastSeen, + }; + this.trackedChains.delete(entry.sessionId); + this.trackedChains.set(entry.sessionId, entry); + imported++; + } + while (this.trackedChains.size > MAX_TRACKED_SESSIONS) { + const oldest = [...this.trackedChains.values()].sort((a, b) => a.lastSeen - b.lastSeen)[0]; + if (!oldest) break; + this.trackedChains.delete(oldest.sessionId); + } + return imported; + } + private expire(tracked: Map): void { if (tracked.size === 0) return; const now = Date.now(); diff --git a/src/server.ts b/src/server.ts index ca7914271d..f867e8b52c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -65,6 +65,7 @@ import { observeResponsesTerminalState } from "./stream-terminal.js"; import { emitStreamError } from "./stream-error.js"; import { affinityToken, clientConversationHeader, codexTurnIdentity, preferPromptCacheKeyIdentity, type ConversationIdentity } from "./session-id.js"; import { prefixAffinity, type AnonymousAffinity } from "./prefix-affinity.js"; +import { flushPrefixAffinity, hydratePrefixAffinity, scheduleAffinityPersist } from "./affinity-persist.js"; import { consumePluginRegisterFor, flushConversations, handlePluginCompact, handlePluginManifest, handlePluginRegister, handlePluginStatus, handlePluginTool, loadConversations, pipePluginChatWithStrip, pipePluginJson, pipePluginResponsesWithStrip, pluginAgentHeader, pluginConversationHeader, pluginReportedContextWindow, recordPluginSession, rememberPluginMessages, takePendingPluginRegister } from "./plugin.js"; import { setupMitm, readMitmUpstream } from "./mitm.js"; import type { BiliMessage } from "acp-kernel/wire"; @@ -386,6 +387,7 @@ export async function startServer(opts: ProxyOptions): Promise { const origin = `http://${originHost}:${actualPort}`; try { fs.mkdirSync(stateDir(), { recursive: true }); + hydratePrefixAffinity(); atomicWriteInstanceFile({ origin, instanceId, @@ -471,6 +473,7 @@ export async function startServer(opts: ProxyOptions): Promise { // debounced writes keep disk within ~500ms of in-memory state. let shuttingDown = false; const finishShutdown = (): void => { + flushPrefixAffinity(); clearProxyInstanceFile(instanceId); unregisterInstance(instanceId); closeLogger(); @@ -1150,6 +1153,7 @@ async function handle( const session = getSession(sessionId, { protocol, upstreamOrigin, label: clientLabel ?? (anonAffinity ? "prefix-affinity" : undefined) }); if (anonAffinity) { prefixAffinity.note(sessionId, anonAffinity.incomingDepth, anonAffinity.tailHash, anonAffinity.itemHashes); + scheduleAffinityPersist(); session.metadata.anonymousPrefixAffinity = { depth: anonAffinity.incomingDepth, tailHash: anonAffinity.tailHash, diff --git a/src/weak-overflow.ts b/src/weak-overflow.ts new file mode 100644 index 0000000000..4f9d0f6c38 --- /dev/null +++ b/src/weak-overflow.ts @@ -0,0 +1,97 @@ +import { markDirty, type Session } from "./session.js"; +import { log as loggerLog } from "./logger.js"; + +/** + * #498: weak overflow signals. A 400 with a parseable window is a STRONG + * overflow signal (server.ts learns from it directly). But an upstream that + * dies mid-stream instead — truncation, timeout — fails NON-400, and on + * sglang-style backends an oversized input manifests exactly this way: the + * prompt is accepted, then the stream cuts with no completion event. Those + * failures carry no window number, so they can never teach the proxy + * anything on their own. What we CAN observe: the request was at high usage + * AND it kept failing. A single truncation is indistinguishable from network + * noise (that is why the loop retries it once, #413); three high-usage + * truncations inside a quarter hour are a pattern. + * + * When the pattern fires we learn the failing input size as a conservative + * window (shrink-only, mirroring the 400-without-window path: the payload + * was above the real window, so its size is an upper bound) and arm the + * emergency shrink so the next turn compresses below it. This unblocks the + * #351/#499 failure family: oversized requests never succeed, never cache, + * and never self-heal — without this, the session loops on truncated + * streams until the client gives up. + */ + +const MIN_USAGE = 0.9; +const WINDOW_MS = 15 * 60 * 1000; +const MIN_EVENTS = 3; +const MAX_TRACKED_SESSIONS = 512; + +interface WeakOverflowState { + events: number[]; +} + +const states = new Map(); + +function resolvedWindow(session: Session): number { + const md = (session.metadata ?? {}) as Record; + const learned = md.learnedContextLimit as number | undefined; + const effective = md.effectiveContextLimit as number | undefined; + const candidates = [learned, effective].filter((v): v is number => typeof v === "number" && v > 0); + if (candidates.length === 0) return 0; + return Math.min(...candidates); +} + +/** + * Record a non-400 stream failure (truncation / timeout) for this session. + * Only counts when usage was already high; arms the emergency shrink after + * MIN_EVENTS repeats inside WINDOW_MS. `inputTokens` is the failing request's + * input size when known (usage already sniffed), else the last known input. + */ +export function noteWeakOverflow( + session: Session, + opts: { inputTokens?: number; model?: string; reason: string }, +): void { + const window = resolvedWindow(session); + if (window <= 0) return; + const input = opts.inputTokens && opts.inputTokens > 0 ? opts.inputTokens : session.stats?.lastInputTokens ?? 0; + if (input <= 0) return; + if (input / window < MIN_USAGE) return; + + if (states.size > MAX_TRACKED_SESSIONS) { + const oldest = states.keys().next().value; + if (oldest !== undefined) states.delete(oldest); + } + const state = states.get(session.id) ?? { events: [] }; + const now = Date.now(); + state.events = state.events.filter((t) => now - t < WINDOW_MS); + state.events.push(now); + states.set(session.id, state); + if (state.events.length < MIN_EVENTS) { + loggerLog("warn", `[${session.id}] weak overflow signal ${state.events.length}/${MIN_EVENTS} (usage ${Math.round((input / window) * 100)}%, ${opts.reason})`); + return; + } + states.delete(session.id); + + const md = ((session.metadata ?? {}) as Record); + if (md.learnedContextLimits === undefined) md.learnedContextLimits = {}; + const learnedMap = md.learnedContextLimits as Record; + const reqModel = opts.model; + const prev = (reqModel ? learnedMap[reqModel] : undefined) ?? (md.learnedContextLimit as number | undefined); + // Shrink-only: a previously learned (smaller) value is the tighter bound. + if (prev === undefined || input < prev) { + if (reqModel) learnedMap[reqModel] = input; + else md.learnedContextLimit = input; + session.metadata = md; + loggerLog("warn", `[${session.id}] weak overflow confirmed (${MIN_EVENTS}× high-usage failures, ${opts.reason}) — learned conservative window ${input} for ${reqModel ?? "(unknown model)"} (was ${prev ?? "unset"}); arming emergency shrink`); + } else { + loggerLog("warn", `[${session.id}] weak overflow confirmed (${MIN_EVENTS}× high-usage failures, ${opts.reason}) — conservative window ${input} not below learned ${prev}; arming emergency shrink only`); + } + if (!session.stats) session.stats = { lastInputTokens: input } as Session["stats"]; + else session.stats.lastInputTokens = Math.max(session.stats.lastInputTokens, input); + markDirty(session); +} + +export function resetWeakOverflow(sessionId: string): void { + states.delete(sessionId); +} diff --git a/tests/affinity-persist.test.ts b/tests/affinity-persist.test.ts new file mode 100644 index 0000000000..ae9638832e --- /dev/null +++ b/tests/affinity-persist.test.ts @@ -0,0 +1,71 @@ +import { test, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { PrefixAffinityResolver, prefixAffinity } from "../src/prefix-affinity.ts"; +import { flushPrefixAffinity, hydratePrefixAffinity } from "../src/affinity-persist.ts"; + +let tmp: string; + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "affinity-persist-")); + process.env.XDG_STATE_HOME = tmp; +}); + +function messages(n: number): unknown[] { + const out: unknown[] = [{ role: "system", content: "sys" }]; + for (let i = 0; i < n; i++) out.push({ role: "user", content: `msg ${i}` }); + return out; +} + +test("exportSnapshot/importSnapshot roundtrip preserves resolution", () => { + const a = new PrefixAffinityResolver(); + const first = a.resolve(messages(10)); + assert.ok(first); + a.note(first.sessionId, first.incomingDepth, first.tailHash, first.itemHashes); + const snapshot = a.exportSnapshot(); + assert.ok(snapshot.length === 1); + + const b = new PrefixAffinityResolver(); + assert.equal(b.importSnapshot(snapshot), 1); + const replay = b.resolve(messages(10)); + assert.ok(replay); + assert.equal(replay.via, "prefix"); + assert.equal(replay.sessionId, first.sessionId, "rehydrated chain still resolves to the same session"); +}); + +test("importSnapshot drops malformed and expired entries", () => { + const r = new PrefixAffinityResolver(); + assert.equal(r.importSnapshot("nope"), 0); + assert.equal(r.importSnapshot([{ sessionId: 1 }, { sessionId: "x", depth: "3" }, null, { sessionId: "ok", depth: 3, tailHash: "t", itemHashes: ["h1", "h2"], lastSeen: Date.now() - 8 * 24 * 60 * 60 * 1000 }]), 0, "malformed + expired are skipped"); +}); + +test("flush writes the snapshot file; hydrate reattaches after a restart", () => { + const m = messages(6); + const aff = prefixAffinity.resolve(m); + assert.ok(aff && aff.via === "new"); + prefixAffinity.note(aff.sessionId, aff.incomingDepth, aff.tailHash, aff.itemHashes); + flushPrefixAffinity(); + + const file = path.join(tmp, "billion-context", "prefix-affinity.json"); + assert.ok(fs.existsSync(file)); + const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as { version: number; entries: unknown[] }; + assert.equal(parsed.version, 1); + assert.ok(parsed.entries.length >= 1); + + prefixAffinity.forget(aff.sessionId); + assert.ok(!prefixAffinity.trackedSessionIds().includes(aff.sessionId)); + hydratePrefixAffinity(); + assert.ok(prefixAffinity.trackedSessionIds().includes(aff.sessionId), "chain restored from disk"); + const replay = prefixAffinity.resolve(m); + assert.ok(replay && replay.via === "prefix" && replay.sessionId === aff.sessionId); +}); + +test("hydrate on a missing or corrupt file is a no-op", () => { + hydratePrefixAffinity(); + const file = path.join(tmp, "billion-context", "prefix-affinity.json"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, "{not json"); + hydratePrefixAffinity(); +}); diff --git a/tests/cache-warn.test.ts b/tests/cache-warn.test.ts new file mode 100644 index 0000000000..24fe6012bc --- /dev/null +++ b/tests/cache-warn.test.ts @@ -0,0 +1,62 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { warnCacheCollapse } from "../src/cache-warn.ts"; +import { setLogCapture } from "../src/logger.ts"; +import type { Session } from "../src/session.ts"; + +const warnings: string[] = []; + +beforeEach(() => { + warnings.length = 0; + setLogCapture((level, msg) => { + if (level === "warn") warnings.push(msg); + }); +}); + +afterEach(() => { + setLogCapture(null); +}); + +function makeSession(): Session { + return { + id: `cw-${Math.random().toString(36).slice(2, 8)}`, + metadata: {}, + stats: { lastInputTokens: 0 }, + } as unknown as Session; +} + +test("collapse after a healthy cache warns exactly once", () => { + const session = makeSession(); + warnCacheCollapse(session, 50000, 45000); + for (let i = 0; i < 10; i++) warnCacheCollapse(session, 50000, 100); + const hits = warnings.filter((w) => w.includes("prompt-cache collapse")); + assert.equal(hits.length, 1, "warns once"); + assert.match(hits[0]!, /input=50000, cached=100/); +}); + +test("no warning without a prior high-hit sample", () => { + const session = makeSession(); + for (let i = 0; i < 10; i++) warnCacheCollapse(session, 50000, 0); + assert.equal(warnings.filter((w) => w.includes("prompt-cache collapse")).length, 0); +}); + +test("a mid-run partial hit resets the low streak", () => { + const session = makeSession(); + warnCacheCollapse(session, 50000, 45000); + warnCacheCollapse(session, 50000, 0); + warnCacheCollapse(session, 50000, 0); + warnCacheCollapse(session, 50000, 0); + warnCacheCollapse(session, 50000, 0); + warnCacheCollapse(session, 50000, 10000, ); + for (let i = 0; i < 4; i++) warnCacheCollapse(session, 50000, 0); + assert.equal(warnings.filter((w) => w.includes("prompt-cache collapse")).length, 0, "streak broken at 4 < 5"); + warnCacheCollapse(session, 50000, 0); + assert.equal(warnings.filter((w) => w.includes("prompt-cache collapse")).length, 1, "a full second streak warns"); +}); + +test("small inputs are never counted", () => { + const session = makeSession(); + warnCacheCollapse(session, 50000, 45000); + for (let i = 0; i < 10; i++) warnCacheCollapse(session, 500, 0); + assert.equal(warnings.filter((w) => w.includes("prompt-cache collapse")).length, 0); +}); diff --git a/tests/plugin-truncation-weak-overflow.test.ts b/tests/plugin-truncation-weak-overflow.test.ts new file mode 100644 index 0000000000..5e469411cd --- /dev/null +++ b/tests/plugin-truncation-weak-overflow.test.ts @@ -0,0 +1,120 @@ +import { test, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { pipePluginChatWithStrip, pipePluginResponsesWithStrip } from "../src/plugin.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { resetWeakOverflow } from "../src/weak-overflow.ts"; +import type { Session } from "../src/session.ts"; + +const SID = "plug-trunc-test"; + +function makeSession(): Session { + return { + id: SID, + metadata: { effectiveContextLimit: 100000 }, + stats: { lastInputTokens: 95000 }, + } as unknown as Session; +} + +function makeRes() { + const chunks: string[] = []; + return { + res: { + write(b: Buffer | string) { + chunks.push(typeof b === "string" ? b : b.toString("utf8")); + return true; + }, + end(b?: Buffer | string) { + if (b !== undefined) chunks.push(typeof b === "string" ? b : b.toString("utf8")); + }, + once() {}, + destroyed: false, + writableEnded: false, + } as unknown as import("node:http").ServerResponse, + chunks, + }; +} + +function streamOf(chunks: string[]): ReadableStream { + const enc = new TextEncoder(); + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < chunks.length) { + controller.enqueue(enc.encode(chunks[i])); + i += 1; + } else { + controller.close(); + } + }, + }); +} + +function chatChunk(delta: Record, extra: Record = {}): string { + return `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "qwen", choices: [{ index: 0, delta, finish_reason: null }], ...extra })}\n\n`; +} + +const DONE = "data: [DONE]\n\n"; + +beforeEach(() => { + _setStoreForTest(new SessionStore({ enabled: false })); + resetWeakOverflow(SID); +}); + +function learnedOf(session: Session): number | undefined { + return (session.metadata as { learnedContextLimit?: number }).learnedContextLimit; +} + +test("chat pipe: 3 truncated streams at high usage learn a conservative window", async () => { + let session = makeSession(); + for (let i = 0; i < 3; i++) { + const { res } = makeRes(); + session = makeSession(); + await pipePluginChatWithStrip(streamOf([chatChunk({ content: "hi" })]), res, "openai", session); + } + assert.equal(learnedOf(session), 95000, "learned the failing input size after the 3rd truncation"); +}); + +test("chat pipe: a [DONE]-terminated stream never arms the signal", async () => { + let session = makeSession(); + for (let i = 0; i < 5; i++) { + const { res } = makeRes(); + session = makeSession(); + await pipePluginChatWithStrip(streamOf([chatChunk({ content: "hi" }), DONE]), res, "openai", session); + } + assert.equal(learnedOf(session), undefined); +}); + +test("chat pipe (anthropic): message_stop terminates, no signal", async () => { + let session = makeSession(); + const start = `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { usage: { input_tokens: 95000 } } })}\n\n`; + const stop = `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`; + for (let i = 0; i < 5; i++) { + const { res } = makeRes(); + session = makeSession(); + await pipePluginChatWithStrip(streamOf([start, stop]), res, "anthropic", session); + } + assert.equal(learnedOf(session), undefined); +}); + +test("responses pipe: stream without a done-family event arms the signal", async () => { + const delta = `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "hello" })}\n\n`; + let session = makeSession(); + for (let i = 0; i < 3; i++) { + const { res } = makeRes(); + session = makeSession(); + await pipePluginResponsesWithStrip(streamOf([delta]), res, session); + } + assert.equal(learnedOf(session), 95000); +}); + +test("responses pipe: response.completed terminates, no signal", async () => { + const delta = `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "hello" })}\n\n`; + const completed = `data: ${JSON.stringify({ type: "response.completed", response: { usage: { input_tokens: 95000 } } })}\n\n`; + let session = makeSession(); + for (let i = 0; i < 5; i++) { + const { res } = makeRes(); + session = makeSession(); + await pipePluginResponsesWithStrip(streamOf([delta, completed]), res, session); + } + assert.equal(learnedOf(session), undefined); +}); diff --git a/tests/weak-overflow.test.ts b/tests/weak-overflow.test.ts new file mode 100644 index 0000000000..72aca4986a --- /dev/null +++ b/tests/weak-overflow.test.ts @@ -0,0 +1,91 @@ +import { test, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { noteWeakOverflow, resetWeakOverflow } from "../src/weak-overflow.ts"; +import type { Session } from "../src/session.ts"; + +const ids: string[] = []; + +function makeSession(window: number, learned?: number): Session { + const metadata: Record = { effectiveContextLimit: window }; + if (learned !== undefined) metadata.learnedContextLimit = learned; + const session = { + id: `weak-${Math.random().toString(36).slice(2, 8)}`, + metadata, + stats: { lastInputTokens: 0 }, + } as unknown as Session; + ids.push(session.id); + return session; +} + +beforeEach(() => { + for (const id of ids) resetWeakOverflow(id); + ids.length = 0; +}); + +test("low usage is ignored entirely", () => { + const session = makeSession(100000); + for (let i = 0; i < 10; i++) noteWeakOverflow(session, { inputTokens: 50000, reason: "test" }); + assert.equal(session.metadata.learnedContextLimit, undefined); + assert.equal((session.stats as { lastInputTokens: number }).lastInputTokens, 0); +}); + +test("three high-usage events learn a conservative window and arm emergency", () => { + const session = makeSession(100000); + noteWeakOverflow(session, { inputTokens: 95000, reason: "r1" }); + noteWeakOverflow(session, { inputTokens: 96000, reason: "r2" }); + assert.equal(session.metadata.learnedContextLimit, undefined, "not before the 3rd event"); + noteWeakOverflow(session, { inputTokens: 97000, reason: "r3" }); + assert.equal(session.metadata.learnedContextLimit, 97000, "learns the LAST failing input size"); + assert.equal((session.stats as { lastInputTokens: number }).lastInputTokens, 97000, "emergency shrink armed"); +}); + +test("model-scoped learning lands in learnedContextLimits", () => { + const session = makeSession(100000); + noteWeakOverflow(session, { inputTokens: 95000, model: "qwen", reason: "r1" }); + noteWeakOverflow(session, { inputTokens: 95000, model: "qwen", reason: "r2" }); + noteWeakOverflow(session, { inputTokens: 95000, model: "qwen", reason: "r3" }); + assert.deepEqual(session.metadata.learnedContextLimits, { qwen: 95000 }); + assert.equal(session.metadata.learnedContextLimit, undefined); +}); + +test("shrink-only: never grows a previously learned smaller window", () => { + const session = makeSession(100000, 50000); + noteWeakOverflow(session, { inputTokens: 95000, reason: "r1" }); + noteWeakOverflow(session, { inputTokens: 95000, reason: "r2" }); + noteWeakOverflow(session, { inputTokens: 95000, reason: "r3" }); + assert.equal(session.metadata.learnedContextLimit, 50000, "smaller learned value wins"); + assert.equal((session.stats as { lastInputTokens: number }).lastInputTokens, 95000, "emergency shrink still armed"); +}); + +test("events older than the window do not accumulate", () => { + const session = makeSession(100000); + const realNow = Date.now; + let t = realNow(); + Date.now = () => t; + try { + noteWeakOverflow(session, { inputTokens: 95000, reason: "r1" }); + t += 16 * 60 * 1000; + noteWeakOverflow(session, { inputTokens: 95000, reason: "r2" }); + t += 1000; + noteWeakOverflow(session, { inputTokens: 95000, reason: "r3" }); + assert.equal(session.metadata.learnedContextLimit, undefined, "only r2+r3 are in the window — 2 < 3"); + } finally { + Date.now = realNow; + } +}); + +test("unknown window disables the signal", () => { + const session = makeSession(0); + noteWeakOverflow(session, { inputTokens: 95000, reason: "r" }); + assert.equal(session.metadata.learnedContextLimit, undefined); + assert.equal((session.stats as { lastInputTokens: number }).lastInputTokens, 0); +}); + +test("falls back to lastInputTokens when inputTokens is absent", () => { + const session = makeSession(100000); + (session.stats as { lastInputTokens: number }).lastInputTokens = 92000; + noteWeakOverflow(session, { reason: "r1" }); + noteWeakOverflow(session, { reason: "r2" }); + noteWeakOverflow(session, { reason: "r3" }); + assert.equal(session.metadata.learnedContextLimit, 92000); +});