Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
248221b
perf: scan cache body boundaries in linear time
Afloat16 Nov 15, 2026
7f3c8c5
fix: bound retained image encode cache payloads
Afloat16 Nov 15, 2026
a6006e5
test: isolate rescale knob from host config in cache-budget suite
Nov 15, 2026
466fdaa
perf: frame loop SSE chunks in linear time
Afloat16 Nov 15, 2026
fb7047c
fix: recover each Responses image from its own ref
Afloat16 Nov 15, 2026
ee1d278
fix: retain active lanes during conversation recall eviction
Afloat16 Nov 17, 2026
aa4a23d
refactor(recall): drop the now-redundant leading sweep in learnConver…
Nov 17, 2026
16db722
perf: consume repeated canonical pointers in linear time
Afloat16 Nov 17, 2026
7f432cb
test: pin repeated-pointer fixtures to unified storage
Afloat16 Nov 17, 2026
4dd000d
fix: search ancestor originals in derived sessions
Afloat16 Nov 17, 2026
6ff8939
fix: preserve long histories when replacing the compression view
Afloat16 Nov 17, 2026
4f90659
test: cover large histories in the thinking-strip retry
Afloat16 Nov 17, 2026
cf083aa
fix: retain the longest active upstream throttle deadline
Afloat16 Nov 17, 2026
f8f58d3
fix: close pending WebSocket upgrades during server shutdown
Afloat16 Nov 17, 2026
116b42d
fix: keep catalog invalidation effective across overlapping requests
Afloat16 Nov 17, 2026
122e6df
fix: retain prior summaries across Threads reforges
Afloat16 Nov 17, 2026
3868cc0
fix: relay root JSON arrays without conversation preparation
Afloat16 Nov 17, 2026
a706d6f
fix: preserve image exports while session twins survive
Afloat16 Nov 17, 2026
059aaf7
fix: expire writer conflict verdicts with session retention
Afloat16 Nov 17, 2026
22cc56d
fix: scope restored-image shedding exemptions to their tool result
Afloat16 Nov 17, 2026
8dcc5cd
fix: verify image spill bytes before advertising restore refs
Afloat16 Nov 17, 2026
c32deee
fix: release completed reasoning-round response bodies
Afloat16 Nov 17, 2026
7fd2974
fix: isolate egress module cache generations across reloads
Afloat16 Nov 17, 2026
23a9108
fix: isolate binary body drafts from failed egress plugins
Afloat16 Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions scripts/bench-cache-array-scan.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { performance } from "node:perf_hooks";
import { getCacheLedger, noteForwardedBody, settleUsageReport } from "../src/cache-ledger.ts";

const counts = process.argv.slice(2).map(Number);
for (const count of counts.length ? counts : [1000, 4000, 8000]) {
const messages = Array.from({ length: count }, (_, i) => ({
role: i % 2 === 0 ? "user" : "assistant",
content: `Turn ${i}: ${"Conversation text with ordinary words. ".repeat(10)}`,
}));
const first = JSON.stringify({ model: "m", messages, max_tokens: 16384, stop: ["end-of-turn", "stop-marker-one", "stop-marker-two", "presence-padding-field"] });
const next = JSON.stringify({ model: "m", messages: [...messages, { role: "user", content: "Continue the conversation." }], max_tokens: 16384, stop: ["end-of-turn", "stop-marker-one", "stop-marker-two", "presence-padding-field"] });
const times = [];
for (let run = 0; run < 4; run++) {
const session = { id: `array-scan-${count}-${run}`, metadata: {}, stats: {}, state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} } };
const report = { total: 100_000, reportedCached: 99_000, output: 0, protocol: "openai", upstream: "http://fixture" };
noteForwardedBody(session, first, count);
settleUsageReport(session, report);
const start = performance.now();
noteForwardedBody(session, next, count + 1);
settleUsageReport(session, report);
const elapsed = performance.now() - start;
const bodyClass = getCacheLedger(session).lines.at(-1)?.bs;
if (bodyClass !== "tail") throw new Error(`unexpected body class: ${bodyClass}`);
if (run > 0) times.push(elapsed);
}
times.sort((a, b) => a - b);
console.log(JSON.stringify({ count, bytes: Buffer.byteLength(first), medianMs: times[1], samplesMs: times }));
}
29 changes: 29 additions & 0 deletions scripts/bench-canonical-repeated-pointers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/** node --import ./tests/preload-test-env.mjs --import tsx scripts/bench-canonical-repeated-pointers.ts */
import assert from "node:assert/strict";
import { performance } from "node:perf_hooks";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { BiliMessage } from "acp-kernel/wire";
import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persist.ts";
import { recordCanonicalPointers } from "../src/incremental-lane.ts";
const dir = mkdtempSync(join(tmpdir(), "bili-pointer-bench-"));
const store = new SessionStore({ dir, enabled: true });
_setStoreForTest(store);
try {
for (const count of [10_000, 50_000, 100_000]) {
const messages: BiliMessage[] = Array.from({ length: count }, () => ({ id: "h_0123456789abcdef", role: "assistant", contentType: "text", text: "ok" }));
const rows = messages.map((message, i) => ({ seq: i, content: JSON.stringify({ ...message, id: `${message.id}_${i + 1}` }), ts: null }));
store.unifiedRawMessages = () => rows;
const samples: number[] = [];
for (let run = 0; run < 3; run++) {
const start = performance.now();
recordCanonicalPointers(`bench-${count}-${run}`, messages, () => {});
samples.push(performance.now() - start);
assert.equal((store.getSessionExt(`bench-${count}-${run}`, "inc:canonical-ptrs") as number[]).length, count);
}
console.log(JSON.stringify({ count, milliseconds: samples, medianMs: [...samples].sort((a,b)=>a-b)[1] }));
}
} finally {
_resetStoreForTest(); rmSync(dir, { recursive: true, force: true });
}
42 changes: 42 additions & 0 deletions scripts/bench-image-encode-cache.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { Jimp } from "jimp";
import { applyImageRescalePass, clearRescaleEncodeCache } from "../src/image-rescale.ts";

if (typeof globalThis.gc !== "function") throw new Error("Run with node --expose-gc");
const count = Number(process.argv[2] ?? 20);
const image = new Jimp({ width: 2400, height: 1600 });
let seed = 42;
for (let i = 0; i < image.bitmap.data.length; i += 4) {
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
image.bitmap.data[i] = seed & 255;
image.bitmap.data[i + 1] = (seed >>> 8) & 255;
image.bitmap.data[i + 2] = (seed >>> 16) & 255;
image.bitmap.data[i + 3] = 255;
}
const ctx = { session: { state: { messageRefs: { byRaw: {} } } }, sessionId: "bench-image-cache", maxDim: 1600, toolArmed: false, log: () => {} };
async function one(index) {
image.bitmap.data[0] = index;
const input = (await image.getBuffer("image/png")).toString("base64");
const [out] = await applyImageRescalePass([{ id: `image-${index}`, role: "user", contentType: "text", imageBase64: input, imageMediaType: "image/png" }], ctx);
if (!out || out.imageBase64 === input) throw new Error("Expected a real rescale");
return { inputBytes: input.length, outputBytes: out.imageBase64.length };
}
async function memoryAfterGc() {
// V8's external-memory counters can lag one collection. Give reclaimed
// codec temporaries several collections and event-loop turns to settle.
for (let i = 0; i < 3; i++) {
globalThis.gc();
await new Promise(resolve => setImmediate(resolve));
}
return process.memoryUsage();
}
await one(255); // load the encoder before measuring retained cache bytes
clearRescaleEncodeCache();
const before = await memoryAfterGc();
let sizes;
for (let i = 0; i < count; i++) sizes = await one(i);
const retained = await memoryAfterGc();
clearRescaleEncodeCache();
const cleared = await memoryAfterGc();
const mb = bytes => Math.round(bytes / 1024 / 1024 * 10) / 10;
const memory = usage => ({ heapMiB: mb(usage.heapUsed), externalMiB: mb(usage.external), rssMiB: mb(usage.rss) });
console.log(JSON.stringify({ count, sizes, before: memory(before), retained: memory(retained), cleared: memory(cleared), cacheReleaseMiB: mb(retained.heapUsed + retained.external - cleared.heapUsed - cleared.external) }, null, 2));
30 changes: 30 additions & 0 deletions scripts/bench-stream-frame-scan.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { performance } from "node:perf_hooks";
import { createGoogleAdapter } from "../src/loop/adapter-google.ts";

const chunkSize = 16 * 1024;
const sizes = process.argv.slice(2).map(Number);
for (const size of sizes.length ? sizes : [1, 4, 8]) {
const payload = "A".repeat(size * 1024 * 1024);
const wire = Buffer.from(`data: ${JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: "Generated image." }, { inlineData: { mimeType: "image/png", data: payload } }] } }] })}\n\ndata: ${JSON.stringify({ candidates: [{ finishReason: "STOP" }] })}\n\n`);
const times = [];
for (let pass = 0; pass < 4; pass++) {
let at = 0;
const stream = new ReadableStream({ pull(controller) {
if (at >= wire.length) return controller.close();
controller.enqueue(wire.subarray(at, at + chunkSize));
at += chunkSize;
} });
const adapter = createGoogleAdapter({}, undefined, undefined, "gemini-test");
let outputBytes = 0;
const start = performance.now();
for await (const event of adapter.parseStream(stream, 1)) {
if (event.kind === "meta") outputBytes += event.chunk.length;
else if ("raw" in event && event.raw) outputBytes += event.raw.length;
}
const elapsed = performance.now() - start;
if (outputBytes < payload.length) throw new Error("Image frame was not forwarded");
if (pass > 0) times.push(elapsed);
}
times.sort((a, b) => a - b);
console.log(JSON.stringify({ payloadMiB: size, wireBytes: wire.length, networkChunkBytes: chunkSize, medianMs: Number(times[1].toFixed(2)) }));
}
14 changes: 7 additions & 7 deletions src/agent/conversation-recall.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,13 @@ function sweep(now: number): void {
let oldestKey: string | undefined;
let oldestAt = Infinity;
for (const [laneKey, perLane] of lanes) {
let latestAt = -Infinity;
for (const entry of perLane.values()) {
if (entry.at < oldestAt) {
oldestAt = entry.at;
oldestKey = laneKey;
}
latestAt = Math.max(latestAt, entry.at);
}
if (latestAt < oldestAt) {
oldestAt = latestAt;
oldestKey = laneKey;
}
}
if (oldestKey === undefined) break;
Expand Down Expand Up @@ -178,7 +180,6 @@ export function recallBoundaryOfMessages(messages: unknown): string | undefined
/** Learn (attribution ok): this session's latest dispatch boundary on this lane. */
export function learnConversation(laneKey: string, sid: string, boundaryFp: string): void {
const now = Date.now();
sweep(now);
let perLane = lanes.get(laneKey);
if (perLane === undefined) {
perLane = new Map();
Expand All @@ -200,8 +201,7 @@ export function learnConversation(laneKey: string, sid: string, boundaryFp: stri
}
perLane.set(sid, { sid, boundaryFp, at: now });
stats.learned++;
stats.lanes = lanes.size;
stats.sessions = [...lanes.values()].reduce((n, l) => n + l.size, 0);
sweep(now);
}

/**
Expand Down
37 changes: 15 additions & 22 deletions src/cache-ledger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1040,14 +1040,9 @@ function messagesArrayCloseOffset(body: Buffer): number | null {
i++;
continue;
}
// String content: hop to the next quote or backslash (native
// scans) instead of branching per byte — the bulk of any body.
let m = n;
const q = body.indexOf(34, i + 1);
if (q !== -1 && q < m) m = q;
const b = body.indexOf(92, i + 1);
if (b !== -1 && b < m) m = b;
i = m;
// Advance once per byte. Separate delimiter searches would each
// rescan the whole remaining body when a delimiter is absent.
i++;
continue;
}
if (c === 34) { inStr = true; keyAt = depth === 1 ? i : -1; i++; continue; }
Expand All @@ -1058,14 +1053,8 @@ function messagesArrayCloseOffset(body: Buffer): number | null {
i++;
continue;
}
// Inter-token filler (whitespace/commas/colons/literals): hop to the
// next structural byte, again natively.
let m = n;
for (const code of [34, 123, 125, 91, 93]) {
const p = body.indexOf(code, i + 1);
if (p !== -1 && p < m) m = p;
}
i = m;
// Whitespace, commas, colons and literals need no separate search.
i++;
}
return null;
}
Expand Down Expand Up @@ -2082,12 +2071,16 @@ export function settleUsageReport(
// side would misread the pair as mid. No array locatable
// (unparseable body) keeps the conservative pre-fix class.
const shorterFull = prevSlot.full.length <= curSlot.full.length ? prevSlot.full : curSlot.full;
const arrClose = messagesArrayCloseOffset(shorterFull);
// #3476: a growing logical unit (single giant message whose interior
// container extends in place) is byte-wise a mid-body insertion but
// semantically an append — recognize it in the would-be-mid slot.
const midInsert = isMidInsertionAppend(prevSlot.full, curSlot.full, lcp);
curLine.bs = lcp >= Math.min(prevSlot.full.length, curSlot.full.length) - 64 ? "append" : lcp < 1024 ? "head" : arrClose !== null && lcp >= arrClose ? "tail" : midInsert ? "append" : "mid";
if (lcp >= shorterFull.length - 64) curLine.bs = "append";
else if (lcp < 1024) curLine.bs = "head";
else {
const arrClose = messagesArrayCloseOffset(shorterFull);
// #3476: a growing logical unit (single giant message whose interior
// container extends in place) is byte-wise a mid-body insertion but
// semantically an append — recognize it in the would-be-mid slot.
curLine.bs = arrClose !== null && lcp >= arrClose ? "tail"
: isMidInsertionAppend(prevSlot.full, curSlot.full, lcp) ? "append" : "mid";
}
} else {
curLine.bs = "unknown";
}
Expand Down
4 changes: 2 additions & 2 deletions src/decompress-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -997,8 +997,8 @@ export function executeSearchContextTarget(
const self2 = peekSession(sessionId);
if (self2) {
for (const anc of derivedAncestorSessions(self2)) {
if (!searchBlocks(blockDocs(anc.state), String(args.query ?? ""), { limit: anc.state.blocks.length, algorithm: "substring", minScore: 1e-9 }).length) continue;
return executeSearchContext(args, core, anc.state, anc.id, undefined, originalTextsOf(anc));
const inherited = executeSearchContext(args, core, anc.state, anc.id, undefined, originalTextsOf(anc));
if (inherited.text.startsWith("Found ")) return inherited;
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/egress-shed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,6 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
messages.forEach((m, entryIndex) => {
if (!isObj(m)) return;
if (protocol === "openai" && m.role === "tool" && isExemptOutput(protocol, m, names)) return;
if (protocol === "anthropic" && Array.isArray(m.content) && m.content.some((b) => isObj(b) && b.type === "tool_result" && isExemptOutput(protocol, b, names))) return;
if (!Array.isArray(m.content)) return;
const content = m.content as unknown[];
if (protocol === "anthropic") addAnthropicImages(content, entryIndex, false);
Expand All @@ -242,6 +241,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
applyShrink: (b64) => { p.image_url = typeof iu === "string" ? dataUrl(d.mediaType, b64) : { ...(iu ?? {}), url: dataUrl(d.mediaType, b64) }; },
});
} else if (p.type === "tool_result" && Array.isArray(p.content)) {
if (isExemptOutput(protocol, p, names)) return;
addAnthropicImages(p.content, entryIndex, true);
}
});
Expand Down
10 changes: 7 additions & 3 deletions src/egress-transforms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ export type EgressTransformEntry = {
/** Context handed to each plugin's `transform`. `headers` is a writable draft:
* successful direct mutations commit to the outgoing map before returned
* headers merge; failed or timed-out drafts are discarded. `body` is the
* current stage's outbound copy (string JSON text or Buffer). */
* current stage's string or a private Buffer draft. Buffer edits take effect
* only when the plugin successfully returns that draft as `body`. */
export type EgressTransformCtx = {
sessionId: string | undefined;
upstreamUrl: string;
Expand Down Expand Up @@ -138,12 +139,14 @@ type LoadedModule =
| { ok: false; err: string };

const moduleCache = new Map<string, LoadedModule>();
let moduleGeneration = 0;

/** Clear the loaded-module cache. Called on `POST /__bili/config/reload` so a
* plugin's code (and any newly-added/removed entry) takes effect on the next
* request without a restart. Modules stay stable WITHIN a generation so a
* running session never swaps transform behavior mid-flight. */
export function resetEgressModuleCache(): void {
moduleGeneration++;
moduleCache.clear();
}

Expand All @@ -160,6 +163,7 @@ function coerceModule(def: unknown): EgressTransformModule | null {
async function loadModule(absPath: string): Promise<LoadedModule> {
const cached = moduleCache.get(absPath);
if (cached) return cached;
const generation = moduleGeneration;
let result: LoadedModule;
try {
if (!existsSync(absPath)) throw new Error(`module not found: ${absPath}`);
Expand All @@ -175,7 +179,7 @@ async function loadModule(absPath: string): Promise<LoadedModule> {
} catch (e) {
result = { ok: false, err: String(e instanceof Error ? e.message : e) };
}
moduleCache.set(absPath, result);
if (generation === moduleGeneration) moduleCache.set(absPath, result);
return result;
}

Expand Down Expand Up @@ -261,7 +265,7 @@ export async function applyEgressTransforms(
for (const t of resolved) {
if (!matchUpstream(t.match, base.upstreamUrl)) continue;
const headers = { ...base.headers };
const ctx: EgressTransformCtx = { ...base, headers, body: current };
const ctx: EgressTransformCtx = { ...base, headers, body: Buffer.isBuffer(current) ? Buffer.from(current) : current };
const inBytes = byteLen(current);
let result: EgressTransformResult;
try {
Expand Down
Loading
Loading