Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,14 @@ export async function prepareOpenai(
// stashed prefix lives OUTSIDE the Session by design; the fold machinery
// applies to the pin exactly as it already does for folded sessions.
snapshotMessages(session, wpHit !== undefined ? wpHit.windowCores : originalMessages);
// #7346: on a HIT the pin is this turn's window (the new tail), NOT the
// complete raw log. snapshotMessages stamps it raw (folded=false), so the
// save presents the window as the full log and the #3285 authoritative-set
// sweep retires every verified prefix row the window omits — a 100k-session
// ledger drained to the ~2-row window each turn. Mark the pin non-raw so
// the save ingests the window through the bounded envelope view and does
// NOT sweep the prefix.
if (wpHit !== undefined) session.lastMessagesFolded = true;
// #3397 durability-before-render (see prepare-anthropic.ts): rows durable NOW, before the model call.
getStore().ingestRowsNow(session);
markDirty(session);
Expand Down
212 changes: 212 additions & 0 deletions tests/issue7346-pf-window-ledger-fullpath.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
// #7346 full-path regression: a watermark-HIT turn must not retire the
// verified prefix.
//
// The sibling test (issue7346-pf-window-ledger.test.ts) pins the #3285
// sweep/no-sweep contract by calling shadow.record directly with the two
// rawTail shapes (undefined = the fix, window = pre-fix). It simulates the
// fix's downstream effect (rawTail=undefined => no sweep) but never drives
// prepareOpenai, so it would not fail if the fix line
// `if (wpHit !== undefined) session.lastMessagesFolded = true`
// (src/server/prepare-openai.ts) were removed.
//
// This test drives a REAL watermark-HIT turn through the full proxy in
// shadow mode:
// turn 0: sidecar (4K budget) -> rides the side lane, never stamped;
// turn 1: full resend -> the watermark is stamped (x-bili-pf-seq) and the
// full log is ingested (the verified prefix);
// turn 2: the client echoes x-bili-pf-count -> a HIT. prepareOpenai pins
// only this turn's ~2-row window; the fix marks it non-raw so
// persist's durability ingest (ingestRowsNow) omits the raw tail
// and the #3285 sweep never fires.
// Asserts the verified prefix survives in the ledger after the HIT turn.
// Fails if the fix line is removed (the window save would then sweep the
// prefix, leaving only the ~2-row window).

import test from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import http from "node:http";
import { once } from "node:events";
import os from "node:os";
import path from "node:path";
import { defaultConfig } from "acp-kernel";
import { startServer } from "../src/server.ts";
import type { ProxyOptions } from "../src/config.ts";
import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persist.ts";
import { UnifiedStore } from "../src/storage/store.ts";
import { _resetPluginStateForTest } from "../src/plugin.ts";
import { _resetSessionsForTest } from "../src/session.ts";
import { closeServers } from "./server-close-guard.ts";

const ROLLOUT_ENV = "BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK";
const UNIFIED_ENV = "BILI_STORAGE_UNIFIED";

interface Capture { raw: string; headerKeys: string[] }

function makeUpstream(sink: Capture[]) {
return http.createServer((req, res) => {
let data = "";
req.on("data", (c) => (data += c));
req.on("end", () => {
sink.push({ raw: data, headerKeys: Object.keys(req.headers).map((k) => k.toLowerCase()) });
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({
id: "chatcmpl-test", object: "chat.completion", created: 1, model: "model-test",
choices: [{ index: 0, message: { role: "assistant", content: "answer" }, finish_reason: "stop" }],
usage: { prompt_tokens: 100, completion_tokens: 4, total_tokens: 104 },
}));
});
});
}

interface Harness { root: string; captured: Capture[]; upstream: http.Server; store: SessionStore }

function makeHarness(tag: string): Harness {
_resetStoreForTest();
_resetPluginStateForTest();
_resetSessionsForTest();
const root = mkdtempSync(path.join(os.tmpdir(), `pfwp-${tag}-`));
for (const k of ["HOME", "APPHOME", "HERMES_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[k] = root;
// enabled:true so the store opens its shadow (the constructor gates the
// shadow on this.enabled && !unifiedBlockedByEncryption).
const store = new SessionStore({ dir: path.join(root, "sessions"), enabled: true });
_setStoreForTest(store);
const captured: Capture[] = [];
return { root, store, captured, upstream: makeUpstream(captured) };
}

async function bringUp(h: Harness) {
h.upstream.listen(0, "127.0.0.1");
await once(h.upstream, "listening");
const uaddr = h.upstream.address();
assert(uaddr && typeof uaddr === "object");
const upstreamUrl = `http://127.0.0.1:${uaddr.port}`;
const proxy = await startServer({
port: 0, host: "127.0.0.1", upstream: upstreamUrl,
routes: { [upstreamUrl]: { models: { "model-test": { context: 64_000 } } } },
modelContextLimit: 200_000,
kernelConfig: defaultConfig(200_000),
compress: { injectTool: false, injectNudge: false },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: true, debug: true, passthrough: false, autoUpdate: false,
mitm: { enabled: false, domains: [] },
} as ProxyOptions);
await once(proxy, "listening");
const paddr = proxy.address();
assert(paddr && typeof paddr === "object");
return { proxy, origin: `http://127.0.0.1:${paddr.port}`, upstreamUrl };
}

// Three turns: turn 0 is the sidecar (4K budget, unstamped), turn 1 is the
// first full resend (stamped), turn 2 is the HIT (the client echoes the
// stamp). The history grows by one user + one assistant per turn.
function scriptTurns(): Array<{ messages: unknown[]; max_tokens: number; stream: boolean }> {
const lines = ["alpha", "beta", "gamma"];
const turns: Array<{ messages: unknown[]; max_tokens: number; stream: boolean }> = [];
for (let t = 0; t < lines.length; t++) {
const history: unknown[] = [{ role: "system", content: "You are TestBuddy v2." }];
for (let i = 0; i <= t; i++) {
history.push({ role: "user", content: lines[i] });
if (i < t) history.push({ role: "assistant", content: `ack ${i}` });
}
turns.push({ messages: history, max_tokens: t === 0 ? 4_096 : 64_000, stream: false });
}
return turns;
}

function idOf(el: unknown): string {
const s = JSON.stringify(el);
let h = 0x811c9dc5;
for (let k = 0; k < s.length; k++) {
h ^= s.charCodeAt(k);
h = Math.imul(h, 0x01000193) >>> 0;
}
return `eid-${h.toString(16)}`;
}

async function runClient(origin: string, upstreamUrl: string, sid: string, turns: Array<{ messages: unknown[]; max_tokens: number; stream: boolean }>): Promise<(number | null)[]> {
const stamps: (number | null)[] = [];
for (let i = 0; i < turns.length; i++) {
const headers: Record<string, string> = {
"content-type": "application/json",
"x-acp-session": sid,
"x-bili-plugin": "dsh",
};
const prev = stamps[stamps.length - 1];
if (prev !== null && prev !== undefined) headers["x-bili-pf-count"] = String(prev);
const ids = (turns[i].messages as unknown[]).map(idOf);
if (prev !== null && prev !== undefined && ids.length >= prev) {
headers["x-bili-pf-tail-ids"] = ids.slice(prev - 1).map(encodeURIComponent).join(",");
}
const res = await fetch(`${origin}/bili/${upstreamUrl}/v1/chat/completions`, {
method: "POST", headers, body: JSON.stringify(turns[i]),
});
assert.equal(res.status, 200, `turn ${i}: proxy answers 200`);
const s = res.headers.get("x-bili-pf-seq");
stamps.push(s !== null && /^\d+$/.test(s) ? Number(s) : null);
await res.arrayBuffer();
}
return stamps;
}

/** Windows cannot unlink a directory while index.db is open (the shadow
* UnifiedStore holds a file lock); rmSync with bounded retries, swallowed on
* persistent EBUSY (same pattern as #3397/#4986/#6780). */
function rmBestEffort(p: string): void {
try {
rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
} catch {
/* leaked temp dir on a Windows runner is harmless */
}
}

test("#7346 full-path: a watermark-HIT window save keeps the verified prefix in the ledger", async () => {
const savedRollout = process.env[ROLLOUT_ENV];
const savedUnified = process.env[UNIFIED_ENV];
const roots: string[] = [];
const sid = "7346-fullpath-sid";
let h0: Harness | undefined;
try {
process.env[ROLLOUT_ENV] = "1";
process.env[UNIFIED_ENV] = "shadow";
h0 = makeHarness("fullpath");
roots.push(h0.root);
const h = await bringUp(h0);
const stamps = await runClient(h.origin, h.upstreamUrl, sid, scriptTurns());
// Turn 0 is the sidecar (unstamped); turn 1 must stamp the watermark.
assert.equal(stamps[0], null, "turn 0: side lane must not stamp");
assert.ok(stamps[1] !== null, "turn 1: full resend must stamp the watermark");
// Turn 2 echoes the stamp -> a HIT.
assert.ok(stamps[1]! > 0, "turn 1 stamp is a positive wire total");
// Let the debounced save settle (the durability ingest already ran
// synchronously in prepareOpenai; the debounced save is a belt-and-
// braces second ingest of the same state).
await new Promise((r) => setTimeout(r, 50));
h0.store.cancelAll();
// Query the ledger in a separate reader connection (SQLite WAL allows
// concurrent readers while the proxy's writer is open).
const db = UnifiedStore.open(path.join(h0.root, "sessions", "index.db"));
try {
const rows = db.getMessages(sid);
// With the fix: the verified prefix (turn 1's full log) survives
// the HIT window save -> 6 live rows (system + 2 user + 2
// assistant + the HIT turn's 2 new). Without the fix: the window
// save sweeps the prefix -> only the ~2-row window remains.
assert.ok(rows.length > 2,
`the verified prefix must survive the HIT window save (got ${rows.length} live rows)`);
} finally {
db.close();
}
await closeServers([h.proxy, h0.upstream], "7346 fullpath");
} finally {
if (savedRollout === undefined) delete process.env[ROLLOUT_ENV];
else process.env[ROLLOUT_ENV] = savedRollout;
if (savedUnified === undefined) delete process.env[UNIFIED_ENV];
else process.env[UNIFIED_ENV] = savedUnified;
// Release the shadow's index.db handle before cleanup — Windows cannot
// unlink a directory while the SQLite file is locked (#3397 class).
h0?.store.closeQuietly();
for (const root of roots) rmBestEffort(root);
}
});
144 changes: 144 additions & 0 deletions tests/issue7346-pf-window-ledger.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/**
* #7346: a watermark-HIT window save must not retire the verified prefix.
*
* On a fastpath HIT the server pins only this turn's ~2-row window
* (snapshotMessages(session, wpHit.windowCores)). snapshotMessages stamps the
* pin raw (lastMessagesFolded=false), so the save presents the window as the
* complete raw log and the #3285 authoritative-set sweep retires every ledger
* row the window omits — a 100k-session prefix drained to the ~2-row window
* each turn. The fix marks the pin non-raw on HIT turns
* (lastMessagesFolded=true), so persist.ts stops passing the raw tail and the
* sweep never fires.
*
* Pinned here (the sweep only runs on the FULL path, so the watermark is
* cleared to force it):
* - a non-raw window save (rawTail omitted, the fix) appends the window and
* keeps the verified prefix — no sweep;
* - a raw window save (rawTail = the window, the pre-fix shape) still sweeps
* the prefix away — a genuine truncation is NOT masked.
*/

import { test } from "node:test";
import assert from "node:assert/strict";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { UnifiedStore } from "../src/storage/store.ts";
import { UnifiedShadow } from "../src/storage/shadow.ts";
import type { LegacySessionLike } from "../src/storage/ingest.ts";
import type { CoreMessage } from "acp-kernel";

process.env.BILI_PERSIST_ZSTD = "0";
const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-7346-hermetic-"));
process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json");
process.env.BILI_STORAGE_UNIFIED = "legacy";

function tmpdir(prefix: string): string {
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
}

function rmBestEffort(p: string): void {
try {
fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
} catch {
/* leaked temp dir on a Windows runner is harmless */
}
}

function msg(i: number, text = `hello ${i}`): CoreMessage {
return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text } as CoreMessage;
}

function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): LegacySessionLike {
return {
id,
createdAt: 1_000,
lastSeen: 2_000,
meta: { protocol: "anthropic" },
stats: { requests: 1 },
metadata: {},
// Key order matters: loadLegacyView rebuilds state in MERGE_STATE_FIELDS
// order and the shadow digest is JSON.stringify-based (order-sensitive).
state: {
blocks: [],
messageRefs: { byRaw: {}, byRef: {} },
nextBlockId: 1,
nextRunId: 1,
},
blockContents: new Map(),
lastMessages,
lastMessagesFolded: false,
};
}

function openShadow(dir: string): { shadow: UnifiedShadow; store: UnifiedStore } {
const store = UnifiedStore.open(path.join(dir, "index.db"));
const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike);
assert.ok(shadow, "shadow mode sidecar must open");
return { shadow, store };
}

function liveRows(store: UnifiedStore, id: string): number {
return store.getMessages(id).length;
}

test("a non-raw window save (the fix) appends the window and keeps the verified prefix", () => {
const dir = tmpdir("bili-7346-fix-");
try {
const { shadow, store } = openShadow(dir);
try {
const id = "pf-window-fix";
// Establish a 100-row verified prefix (a 100k-session ledger, scaled).
const prefix = Array.from({ length: 100 }, (_, i) => msg(i));
shadow.record(mkLegacy(id, prefix), { lastMessages: prefix });
assert.equal(liveRows(store, id), 100, "prefix ingested");

// The watermark-HIT turn pins only the ~2-row window. The fix marks
// it non-raw, so persist.ts omits the raw tail — shadow.record sees
// rawTail=undefined and the #3285 sweep never fires. Force the full
// path (the sweep's only home) by clearing the watermark.
const window = [msg(100), msg(101)];
store.clearIngestWatermark(id);
shadow.record(mkLegacy(id, window), undefined);

assert.equal(liveRows(store, id), 102, "prefix retained + window appended — no sweep on a HIT window save");
// The verified head row is still live (not retired by the sweep).
const head = store.getMessages(id).find((r) => r.msgId === "h_msg0");
assert.ok(head, "the verified head row survived");
} finally {
shadow.close();
store.close();
}
} finally {
rmBestEffort(dir);
}
});

test("a raw window save (the pre-fix shape) still sweeps the prefix — genuine truncation is not masked", () => {
const dir = tmpdir("bili-7346-raw-");
try {
const { shadow, store } = openShadow(dir);
try {
const id = "pf-window-raw";
const prefix = Array.from({ length: 100 }, (_, i) => msg(i));
shadow.record(mkLegacy(id, prefix), { lastMessages: prefix });
assert.equal(liveRows(store, id), 100, "prefix ingested");

// The pre-fix shape: the window is pinned raw (folded=false), so
// persist.ts passes it as the raw tail. The full path treats it as
// the complete log and the #3285 sweep retires every prefix row the
// window omits. This is the genuine-truncation path the fix must
// NOT mask.
const window = [msg(100), msg(101)];
store.clearIngestWatermark(id);
shadow.record(mkLegacy(id, window), { lastMessages: window, lastMessagesFolded: false });

assert.equal(liveRows(store, id), 2, "the raw window save sweeps the verified prefix away (the bug, still reachable on genuine truncation)");
} finally {
shadow.close();
store.close();
}
} finally {
rmBestEffort(dir);
}
});
Loading