Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions src/affinity-persist.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import fs from "node:fs";
import path from "node:path";
import { prefixAffinity } from "./prefix-affinity.js";
import { stateDir } from "./paths.js";
import { log } from "./logger.js";

/**
* #499 P1a: prefix-affinity persistence. The anonymous affinity chains were
* pure in-memory (#309), so a proxy restart orphaned every anonymous session:
* the next replay forked a fresh session with zero compression state and
* resent the raw history (#351 — 458K tokens, 0% cache). The chains are
* small (≤256 sessions × ≤128 hashes); persist them to the state dir with a
* debounced atomic write and hydrate on boot.
*/

const PERSIST_DEBOUNCE_MS = 5_000;

function affinityFile(): string {
return path.join(stateDir(), "prefix-affinity.json");
}

let timer: NodeJS.Timeout | null = null;
let writing = false;

function writeSnapshot(): void {
if (writing) return;
writing = true;
try {
const file = affinityFile();
const snapshot = { version: 1, entries: prefixAffinity.exportSnapshot() };
const tmp = `${file}.tmp`;
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(tmp, JSON.stringify(snapshot));
fs.renameSync(tmp, file);
} catch (e) {
log("warn", `[prefix-affinity] persist failed (${e instanceof Error ? e.message : String(e)}); affinity survives in memory`);
} finally {
writing = false;
}
}

/** Debounced snapshot write — call after every affinity mutation. */
export function scheduleAffinityPersist(): void {
if (timer) clearTimeout(timer);
timer = setTimeout(() => {
timer = null;
writeSnapshot();
}, PERSIST_DEBOUNCE_MS);
timer.unref?.();
}

/** Immediate snapshot write — shutdown path. */
export function flushPrefixAffinity(): void {
if (timer) {
clearTimeout(timer);
timer = null;
}
writeSnapshot();
}

/** Load the snapshot a previous process left behind. Call once on boot. */
export function hydratePrefixAffinity(): void {
try {
const file = affinityFile();
if (!fs.existsSync(file)) return;
const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as Record<string, unknown>;
const imported = prefixAffinity.importSnapshot(parsed.entries);
if (imported > 0) log("info", `[prefix-affinity] hydrated ${imported} chain(s) from ${path.basename(file)} — anonymous sessions reattach across restarts`);
} catch (e) {
log("warn", `[prefix-affinity] hydrate failed (${e instanceof Error ? e.message : String(e)}); starting with empty affinity`);
}
}
56 changes: 56 additions & 0 deletions src/cache-warn.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import type { Session } from "./session.js";
import { log } from "./logger.js";

/**
* #499 P1b: upstream prompt-cache collapse warning. A warm session normally
* rides the provider's prefix cache (90%+ hit). When a restart or a session
* fork makes every request cold, the raw resend is expensive but INVISIBLE:
* requests succeed, so no error path fires — the only trace is a hit ratio
* that fell off a cliff. This watches each usage sample; once a session has
* demonstrated a healthy cache (≥HIGH_HIT on a large input) and then logs
* LOW_RUN consecutive large inputs with ~zero hits, it warns ONCE with the
* numbers and the likely causes. Diagnostics only — no state is changed.
*/

const MIN_INPUT = 8000;
const HIGH_HIT = 0.5;
const LOW_HIT = 0.1;
const LOW_RUN = 5;
const MAX_TRACKED_SESSIONS = 1024;

interface CacheWatch {
sawHigh: boolean;
lowRun: number;
warned: boolean;
}

const watches = new Map<string, CacheWatch>();

/** Feed one usage sample (per successful upstream turn). */
export function warnCacheCollapse(session: Session, input: number, cached: number): void {
if (input < MIN_INPUT) return;
if (watches.size > MAX_TRACKED_SESSIONS) {
const oldest = watches.keys().next().value;
if (oldest !== undefined) watches.delete(oldest);
}
const watch = watches.get(session.id) ?? { sawHigh: false, lowRun: 0, warned: false };
watches.set(session.id, watch);
const hit = cached / input;
if (hit >= HIGH_HIT) {
watch.sawHigh = true;
watch.lowRun = 0;
return;
}
if (!watch.sawHigh || watch.warned) return;
if (hit > LOW_HIT) {
watch.lowRun = 0;
return;
}
watch.lowRun++;
if (watch.lowRun < LOW_RUN) return;
watch.warned = true;
log(
"warn",
`[${session.id}] upstream prompt-cache collapse: ${LOW_RUN} consecutive large requests at ~0% hit (last: input=${input}, cached=${cached}) after earlier ≥${Math.round(HIGH_HIT * 100)}% hits — the context is being resent cold every turn. Likely causes: proxy/session restart re-forked the session (#499), or the upstream KV cache was dropped (window/deployment change). Check the session id upstream sees and compress.modelContextLimit.`,
);
}
21 changes: 21 additions & 0 deletions src/loop/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import { resolveDecompress } from "../decompress-shared.js";
import { buildVisibilityMarker } from "../compress-loop.js";
import { fetchWithRetry, UpstreamHttpError } from "../fetch-util.js";
import { proxyDispatcher } from "../upstream-proxy.js";
import { noteWeakOverflow } from "../weak-overflow.js";
import { warnCacheCollapse } from "../cache-warn.js";
import { log as loggerLog } from "../logger.js";
import type { WireProtocol } from "../util.js";

Expand Down Expand Up @@ -199,6 +201,7 @@ function recordUsage(
if (typeof out === "number") ctx.session.stats.outputTokens += out;
const hitPct =
typeof cached === "number" && total > 0 ? Math.round((cached / total) * 100) : 0;
warnCacheCollapse(ctx.session, total, cached ?? 0);
ctx.log(
`[acp-usage] round ${round} input=${total} cached=${cached ?? 0} (cache hit ${hitPct}%)`,
);
Expand Down Expand Up @@ -521,6 +524,18 @@ export async function* runCompressLoop(
// blind to why it failed. MAX_LOOP_ROUNDS bounds runaway loops.
const reRequest = proxyResults.length > 0 && realCalls === 0;
if (!reRequest) {
// #498: a stream that cut without a completion event is a weak
// overflow signal (sglang-style backends accept an oversized
// prompt then die mid-stream). Both shapes: !sawDone (chat /
// anthropic) and truncatedDone (responses synthetic failed done).
if (!sawDone || truncatedDone) {
const reqModel = typeof requestBody["model"] === "string" ? requestBody["model"] : undefined;
noteWeakOverflow(ctx.session, {
inputTokens: usage.inputTokens,
model: reqModel,
reason: `round ${round}: upstream stream truncated without a completion event`,
});
}
if (!sawDone) {
const partialText = assistantText.length;
const partialReasoning = assistantReasoning.length;
Expand Down Expand Up @@ -589,6 +604,12 @@ export async function* runCompressLoop(
const suffix = e.attempts > 1 ? ` after ${e.attempts} attempt(s)` : "";
ctx.log(`[acp-proxy: compress loop upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}]`);
loggerLog("error", `[acp-loop] upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}`);
const reqModel = typeof requestBody["model"] === "string" ? requestBody["model"] : undefined;
noteWeakOverflow(ctx.session, {
inputTokens: usage.inputTokens,
model: reqModel,
reason: `round ${round}: upstream error ${e.status}${suffix}`,
});
yield adapter.emitError(`upstream error ${e.status}${suffix}: ${e.body.slice(0, 200)}`);
return;
}
Expand Down
33 changes: 33 additions & 0 deletions src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { executeProxyTool } from "./loop/core.js";
import { normalizeSseLineEndings } from "./sse-util.js";
import { containsRenderTagText, createTagEchoFilter, mayStartRenderTag, stripAnthropicText, stripOpenaiChatText, stripResponsesText, type TagEchoFilter } from "./loop/tag-echo-filter.js";
import { log as loggerLog } from "./logger.js";
import { noteWeakOverflow } from "./weak-overflow.js";
import { warnCacheCollapse } from "./cache-warn.js";
import type { WireProtocol } from "./util.js";
import { stateDir } from "./paths.js";

Expand Down Expand Up @@ -549,6 +551,7 @@ function applyUsageSample(session: Session, sample: UsageSample, protocol?: Wire
// Net out pending compress savings (see stream.ts applyRanges): plugin
// compress tool results shrink the next request, not this report.
session.stats.lastInputTokens = Math.max(0, total - (session.stats.compressCreditTokens ?? 0));
warnCacheCollapse(session, total, sample.cachedTokens ?? 0);
}
if (sample.outputTokens !== undefined) session.stats.outputTokens += sample.outputTokens;
}
Expand Down Expand Up @@ -640,6 +643,17 @@ export async function pipePluginChatWithStrip(
markDirty(session);
}
};
// #498: whether a terminal event ([DONE] / message_stop) was seen. A
// stream that ends without one was cut mid-flight — a weak overflow
// signal on high-usage sessions.
let sawTerminal = false;
const maybeNoteTruncated = () => {
if (sawTerminal || !session || res.destroyed || res.writableEnded) return;
noteWeakOverflow(session, {
inputTokens: acc.inputTokens,
reason: "plugin chat passthrough stream ended without a completion event",
});
};
const pushField = (field: string, index: number, text: string): [string, boolean] => {
const s = filterFor(field, index);
const clean = s.filter.push(text);
Expand Down Expand Up @@ -729,6 +743,7 @@ export async function pipePluginChatWithStrip(
const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim();
if (!jsonStr) continue;
if (jsonStr === "[DONE]") {
sawTerminal = true;
await write(flushTails() + rawEvent + "\n\n");
continue;
}
Expand All @@ -739,6 +754,7 @@ export async function pipePluginChatWithStrip(
await write(rawEvent + "\n\n");
continue;
}
if (ev["type"] === "message_stop") sawTerminal = true;
const sample = usageFromSseEvent(ev);
if (sample) mergeUsageSample(acc, sample);
const out = protocol === "anthropic" ? processAnthropic(ev, rawEvent) : processOpenai(ev, rawEvent);
Expand All @@ -750,8 +766,10 @@ export async function pipePluginChatWithStrip(
const rest = flushTails();
if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(rest);
settleUsage();
maybeNoteTruncated();
} catch (e) {
settleUsage();
maybeNoteTruncated();
if (res.destroyed || res.writableEnded) {
log?.("client aborted mid-stream");
return;
Expand Down Expand Up @@ -820,6 +838,17 @@ export async function pipePluginResponsesWithStrip(
markDirty(session);
}
};
// #498: whether a terminal event (done-family / [DONE]) was seen. A
// stream that ends without one was cut mid-flight — a weak overflow
// signal on high-usage sessions.
let sawTerminal = false;
const maybeNoteTruncated = () => {
if (sawTerminal || !session || res.destroyed || res.writableEnded) return;
noteWeakOverflow(session, {
inputTokens: acc.inputTokens,
reason: "plugin responses passthrough stream ended without a completion event",
});
};
let lastDeltaMeta: { item_id?: unknown; output_index?: unknown } | null = null;
const flushTail = (after: string) => {
const tail = tagFilter.flush();
Expand All @@ -843,6 +872,7 @@ export async function pipePluginResponsesWithStrip(
if (dataLines.length === 0) continue;
const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim();
if (!jsonStr || jsonStr === "[DONE]") {
if (jsonStr === "[DONE]") sawTerminal = true;
await write(rawEvent + "\n\n");
continue;
}
Expand All @@ -864,6 +894,7 @@ export async function pipePluginResponsesWithStrip(
type === "response.failed" ||
type === "response.incomplete"
) {
if (type === "response.completed" || type === "response.failed" || type === "response.incomplete") sawTerminal = true;
// done-family events also carry full text payloads — strip those too.
const out = containsRenderTagText(jsonStr) ? rebuildEvent(rawEvent, stripResponsesText(ev)) : rawEvent + "\n\n";
await write(flushTail(out));
Expand Down Expand Up @@ -901,8 +932,10 @@ export async function pipePluginResponsesWithStrip(
if (rest.length > 0) await write(rest);
}
settleUsage();
maybeNoteTruncated();
} catch (e) {
settleUsage();
maybeNoteTruncated();
if (res.destroyed || res.writableEnded) {
log?.("client aborted mid-stream");
return;
Expand Down
56 changes: 56 additions & 0 deletions src/prefix-affinity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,18 @@ interface ChainEntry {
itemHashes: string[];
}

/** On-disk snapshot entry (#499 P1a): the tracked chains persisted across
* restarts so an anonymous replay reattaches its session instead of
* forking a fresh one with zero compression state (the #351 failure mode:
* a 458K-token history resent raw because the affinity was in-memory). */
export interface AffinitySnapshotEntry {
sessionId: string;
depth: number;
tailHash: string;
itemHashes: string[];
lastSeen: number;
}

/** Deterministic JSON with recursively sorted object keys, so two replays
* of the same logical message hash identically regardless of key order. */
export function stableStringify(value: unknown): string {
Expand Down Expand Up @@ -290,6 +302,50 @@ export class PrefixAffinityResolver {
return [...this.trackedChains.keys()];
}

/** Serializable copy of the tracked chains (for #499 P1a persistence). */
exportSnapshot(): AffinitySnapshotEntry[] {
return [...this.trackedChains.values()].map((e) => ({
sessionId: e.sessionId,
depth: e.depth,
tailHash: e.tailHash,
itemHashes: e.itemHashes,
lastSeen: e.lastSeen,
}));
}

/** Load chains persisted by a previous process. Entries older than the
* TTL are dropped (they would not match anyway). Returns the count
* actually imported. Defensive: a corrupt/hand-edited file must never
* crash the proxy — malformed entries are skipped. */
importSnapshot(entries: unknown): number {
if (!Array.isArray(entries)) return 0;
const now = Date.now();
let imported = 0;
for (const raw of entries) {
if (!raw || typeof raw !== "object") continue;
const e = raw as Record<string, unknown>;
if (typeof e.sessionId !== "string" || typeof e.depth !== "number" || typeof e.tailHash !== "string") continue;
if (!Array.isArray(e.itemHashes) || e.itemHashes.some((h) => typeof h !== "string")) continue;
if (typeof e.lastSeen !== "number" || now - e.lastSeen > TTL_MS) continue;
const entry: ChainEntry = {
sessionId: e.sessionId,
depth: e.depth,
tailHash: e.tailHash,
itemHashes: (e.itemHashes as string[]).slice(-MAX_STORED_ITEMS),
lastSeen: e.lastSeen,
};
this.trackedChains.delete(entry.sessionId);
this.trackedChains.set(entry.sessionId, entry);
imported++;
}
while (this.trackedChains.size > MAX_TRACKED_SESSIONS) {
const oldest = [...this.trackedChains.values()].sort((a, b) => a.lastSeen - b.lastSeen)[0];
if (!oldest) break;
this.trackedChains.delete(oldest.sessionId);
}
return imported;
}

private expire(tracked: Map<string, ChainEntry>): void {
if (tracked.size === 0) return;
const now = Date.now();
Expand Down
4 changes: 4 additions & 0 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ import { observeResponsesTerminalState } from "./stream-terminal.js";
import { emitStreamError } from "./stream-error.js";
import { affinityToken, clientConversationHeader, codexTurnIdentity, preferPromptCacheKeyIdentity, type ConversationIdentity } from "./session-id.js";
import { prefixAffinity, type AnonymousAffinity } from "./prefix-affinity.js";
import { flushPrefixAffinity, hydratePrefixAffinity, scheduleAffinityPersist } from "./affinity-persist.js";
import { consumePluginRegisterFor, flushConversations, handlePluginCompact, handlePluginManifest, handlePluginRegister, handlePluginStatus, handlePluginTool, loadConversations, pipePluginChatWithStrip, pipePluginJson, pipePluginResponsesWithStrip, pluginAgentHeader, pluginConversationHeader, pluginReportedContextWindow, recordPluginSession, rememberPluginMessages, takePendingPluginRegister } from "./plugin.js";
import { setupMitm, readMitmUpstream } from "./mitm.js";
import type { BiliMessage } from "acp-kernel/wire";
Expand Down Expand Up @@ -386,6 +387,7 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
const origin = `http://${originHost}:${actualPort}`;
try {
fs.mkdirSync(stateDir(), { recursive: true });
hydratePrefixAffinity();
atomicWriteInstanceFile({
origin,
instanceId,
Expand Down Expand Up @@ -471,6 +473,7 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
// debounced writes keep disk within ~500ms of in-memory state.
let shuttingDown = false;
const finishShutdown = (): void => {
flushPrefixAffinity();
clearProxyInstanceFile(instanceId);
unregisterInstance(instanceId);
closeLogger();
Expand Down Expand Up @@ -1150,6 +1153,7 @@ async function handle(
const session = getSession(sessionId, { protocol, upstreamOrigin, label: clientLabel ?? (anonAffinity ? "prefix-affinity" : undefined) });
if (anonAffinity) {
prefixAffinity.note(sessionId, anonAffinity.incomingDepth, anonAffinity.tailHash, anonAffinity.itemHashes);
scheduleAffinityPersist();
session.metadata.anonymousPrefixAffinity = {
depth: anonAffinity.incomingDepth,
tailHash: anonAffinity.tailHash,
Expand Down
Loading
Loading