Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions scripts/bench-plan-aware-search.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Run with --import ./tests/preload-test-env.mjs --import tsx.
// Optional argument: module URL relative to this script (for a saved base).
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { performance } from "node:perf_hooks";
import { createCore, defaultConfig, type Config, type CoreMessage } from "acp-kernel";
import { applyCompressSettings } from "../src/compress-settings.ts";
import { createSession } from "../src/session.ts";
import { applyRanges } from "../src/stream.ts";
import { parseCompressInput } from "../src/compress-tool.ts";

const { executeSearchContext, extractPlanState } = await import(
new URL(process.argv[2] ?? "../src/decompress-shared.ts", import.meta.url).href
) as typeof import("../src/decompress-shared.ts");
const core = createCore();
const config = applyCompressSettings(defaultConfig(200_000), 200_000, {
absorb: { enabled: true, minToolTokens: 50 },
ccr: { enabled: true, minToolTokens: 50 },
}) as Config;
const session = createSession("plan-normalization-benchmark");
const messages: CoreMessage[] = Array.from({ length: 40 }, (_, i) => ({
id: `history_${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text",
text: `Historical detail ${i}. ${"retained context evidence. ".repeat(80)}`,
timestamp: 1_700_000_000_000 + i,
}));
const turn = core.processTurn({ messages, state: session.state, config, tokenCount: 9999, renderTags: "text-only" });
session.state = turn.state;
const topics = ["auth", "storage", "validation", "refresh", "ratelimit 缓存路由会话", "metrics"];
applyRanges(parseCompressInput({ content: topics.map((topic, i) => ({
startId: `m${String(i * 5 + 1).padStart(5, "0")}`,
endId: `m${String(i * 5 + 5).padStart(5, "0")}`,
topic: `token ${topic}`, summary: `Historical token ${topic} implementation and verification.`,
})) }), { core, config, messages: turn.messages, session, log: () => {} });
assert.equal(session.state.blocks.filter(b => b.active).length, 6);

function measure(work: () => string) {
const expected = work();
for (let i = 0; i < 3; i++) assert.equal(work(), expected);
const elapsed: number[] = [];
for (let i = 0; i < 15; i++) {
const start = performance.now();
const actual = work();
elapsed.push(performance.now() - start);
assert.equal(actual, expected);
}
elapsed.sort((a, b) => a - b);
return {
medianMs: elapsed[7], minMs: elapsed[0], maxMs: elapsed[14],
digest: createHash("sha256").update(expected).digest("hex"),
};
}

const languages = {
latin: "Implement Ratelimit Middleware And Preserve Session Identity With Tests. ",
cjk: "实现缓存路由并保留会话身份验证和回归测试。",
mixed: "实现Cache路由API和Session身份Verification测试。",
};
for (const [language, sentence] of Object.entries(languages)) {
for (const characters of [1000, 2000, 4000]) {
const text = sentence.repeat(Math.ceil(characters / sentence.length)).slice(0, characters);
const plan: CoreMessage[] = [
{ id: "user", role: "user", contentType: "text", text },
{ id: "plan", role: "assistant", contentType: "tool-call", toolName: "TodoWrite",
text: JSON.stringify({ todos: [{ content: text, status: "in_progress", activeForm: "Implementing the plan" }] }) },
];
const opts = { messages: plan, session, log: () => {}, protectedPatterns: config.protectedLatestTools };
const search = () => executeSearchContext({ query: "token", limit: 5 }, core, session.state, undefined, opts).text;
assert.match(search(), /\[plan-aware\]/);
console.log(JSON.stringify({ language, characters, sources: plan.length, workload: "extract", ...measure(
() => JSON.stringify([...extractPlanState(plan)!.terms]),
) }));
console.log(JSON.stringify({ language, characters, sources: plan.length, workload: "search", ...measure(search) }));
}
}
console.log(JSON.stringify({ workload: "plain-search", ...measure(
() => executeSearchContext({ query: "token", limit: 5 }, core, session.state).text,
) }));
4 changes: 2 additions & 2 deletions src/decompress-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -556,10 +556,10 @@ type PlanTerms = Map<string, number>;
function planTokenize(text: string): string[] {
const out: string[] = [];
// Rendered ref-tag markup () is proxy metadata, not plan content.
const clean = text.replace(/\x3cacp\b[\s\S]*?\x3c\/acp\x3e/g, " ");
const clean = text.replace(/\x3cacp\b[\s\S]*?\x3c\/acp\x3e/g, " ").toLowerCase();
const re = /[a-z0-9]+|[\u4e00-\u9fff]/g;
let m: RegExpExecArray | null;
while ((m = re.exec(clean.toLowerCase())) !== null) {
while ((m = re.exec(clean)) !== null) {
const t = m[0];
const cjk = /[\u4e00-\u9fff]/.test(t);
// Latin tokens need >= 3 chars to carry signal; CJK single chars count.
Expand Down
Loading