Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 101 additions & 3 deletions src/decompress-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,96 @@ type SearchPlanOpts = {
protectedPatterns?: string[];
};

// #2751: preview windowing. The old fixed head-clamp (safePrefix 200) hid a hit
// whose only occurrence sat past char 200, and "Found N" was indistinguishable
// from "N is all that exists". Center the preview on the pool-rarest matched
// term (marking it) so unique-match evidence survives truncation; budget stays
// at the old 200-char total so output never grows unbounded.
const SNIPPET_TOTAL = 200;
const SNIPPET_AHEAD = 80;

// Code-unit-safe middle slice — never leaves a lone surrogate at an edge
// (#816/#828/#1615: hand-sliced model-visible text must reuse this discipline).
function safeMidSlice(text: string, start: number, end: number): string {
let s = start;
let e = end;
if (s > 0 && s < text.length) {
const c = text.charCodeAt(s);
if (c >= 0xdc00 && c <= 0xdfff) s += 1;
}
if (e > 0 && e < text.length) {
const c = text.charCodeAt(e - 1);
if (c >= 0xd800 && c <= 0xdbff) e -= 1;
}
return text.slice(s, e);
}

function poolTermFrequency(pool: CompressionBlock[], terms: string[]): Map<string, number> {
const freq = new Map<string, number>();
const lower = terms.map((t) => t.toLowerCase()).filter(Boolean);
for (const t of lower) if (!freq.has(t)) freq.set(t, 0);
for (const b of pool) {
const hay = `${b.topic ?? ""}\n${b.summary}`.toLowerCase();
for (const t of lower) if (hay.includes(t)) freq.set(t, (freq.get(t) ?? 0) + 1);
}
return freq;
}

// Most distinctive query term present in this block's summary = fewest
// containing blocks, tie → earliest occurrence. Null when no term is in the
// summary body (hit then lives in the topic, already shown in the header line).
function pickAnchorTerm(summary: string, terms: string[], freq: Map<string, number>): { term: string; index: number } | null {
const lower = summary.toLowerCase();
let best: { term: string; index: number; freq: number } | null = null;
for (const t of terms) {
const tl = t.toLowerCase();
if (!tl) continue;
const i = lower.indexOf(tl);
if (i === -1) continue;
const f = freq.get(tl) ?? 0;
if (!best || f < best.freq || (f === best.freq && i < best.index)) best = { term: tl, index: i, freq: f };
}
return best ? { term: best.term, index: best.index } : null;
}

function markFirstMatch(text: string, needle: string): string {
const idx = text.toLowerCase().indexOf(needle);
if (idx === -1) return text;
return text.slice(0, idx) + "\u3010" + text.slice(idx, idx + needle.length) + "\u3011" + text.slice(idx + needle.length);
}

function evidenceSnippet(summary: string, terms: string[], freq: Map<string, number>): string {
const anchor = pickAnchorTerm(summary, terms, freq);
if (!anchor) return summary.length > SNIPPET_TOTAL ? safePrefix(summary, SNIPPET_TOTAL) + "..." : summary;
const start = Math.max(0, anchor.index - SNIPPET_AHEAD);
const end = Math.min(summary.length, start + SNIPPET_TOTAL);
const win = markFirstMatch(safeMidSlice(summary, start, end), anchor.term);
return (start > 0 ? "... " : "") + win + (end < summary.length ? " ..." : "");
}

// #2751: for a long multi-term query, if a term is absent from every returned
// block yet present in lower-ranked candidates, steer the model to narrow to it
// rather than reading "absent from top-N" as "absent from history". At most once.
function narrowingHint(terms: string[], returned: CompressionBlock[], ranked: CompressionBlock[]): string {
if (terms.length < 3) return "";
const returnedSet = new Set(returned);
const lowerCands = ranked.filter((b) => !returnedSet.has(b));
if (lowerCands.length === 0) return "";
const contains = (b: CompressionBlock, tl: string): boolean =>
(b.topic ?? "").toLowerCase().includes(tl) || b.summary.toLowerCase().includes(tl);
let best: { term: string; count: number } | null = null;
for (const t of terms) {
const tl = t.toLowerCase();
if (!tl) continue;
if (returned.some((b) => contains(b, tl))) continue;
const count = lowerCands.filter((b) => contains(b, tl)).length;
if (count === 0) continue;
if (!best || count < best.count || (count === best.count && t.length > best.term.length)) best = { term: t, count };
}
if (!best) return "";
return `[tip: none of the top ${returned.length} results contain "${best.term}" but ${best.count} lower-ranked candidate(s) do — try searching "${best.term}" alone to surface its block directly]`;
}

export function executeSearchContext(
args: Record<string, unknown>,
core: CompressionCore,
Expand All @@ -701,7 +791,9 @@ export function executeSearchContext(
if (query.length === 0) return toolFail("[search_context FAILED: query is required]");
const scope = foreignSessionId ? ` in session ${foreignSessionId}` : "";
const limit = typeof args.limit === "number" && args.limit > 0 ? Math.floor(args.limit) : 5;
const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
const pool = core.search(query, state);
const freq = poolTermFrequency(pool, terms);
let ranked = pool;
let scored: Array<{ block: CompressionBlock; score: number }> | null = null;
if (plan && !foreignSessionId && pool.length > limit) {
Expand All @@ -718,11 +810,12 @@ export function executeSearchContext(
if (blocks.length === 0) {
// #1875: an empty result is a valid answer to a query, not a failure.
if (!state.blocks.some((b) => b.active)) return toolOk(`[No compressed blocks exist yet${scope} — nothing to search.]`);
return toolOk(`[No blocks matched "${query}"${scope}]`);
const activeCount = state.blocks.filter((b) => b.active).length;
return toolOk(`[No blocks matched "${query}" among ${activeCount} active block(s)${scope}. Scope = active compressed topics/summaries only — NOT visible messages, inactive/distilled blocks, or other sessions; absence here ≠ never-discussed.]`);
}
const lines = blocks.map((b) => {
const topic = b.topic ?? "(no topic)";
const preview = b.summary.length > 200 ? safePrefix(b.summary, 200) + "..." : b.summary;
const preview = evidenceSnippet(b.summary, terms, freq);
// #1179 CCR v2: covered message-ref span(s) — connect block space to
// store space so the model can target acp_retrieve / range decompress
// at individual messages instead of whole blocks.
Expand All @@ -735,10 +828,15 @@ export function executeSearchContext(
const returned = new Set(blocks);
steering = buildSteering(scored.filter((s) => returned.has(s.block)), state, plan.session);
}
const truncNote = ranked.length > blocks.length
? `\n\n[truncated: showing ${blocks.length} of ${ranked.length} candidates by relevance — pass limit:${ranked.length} to see the rest]`
: "";
const tip = narrowingHint(terms, blocks, ranked);
const tipNote = tip ? `\n\n${tip}` : "";
const note = foreignSessionId
? `\n\n[Read-only search of historical session ${foreignSessionId}. Block ids are per-session namespaces — decompress acts on the current session only. For bulk content use bili export ${foreignSessionId} [--full].]`
: "";
return toolOk(`Found ${blocks.length} block(s) for "${query}"${scope}:\n\n${lines.join("\n\n")}${steering}${note}`);
return toolOk(`Found ${blocks.length} block(s) for "${query}"${scope}:\n\n${lines.join("\n\n")}${truncNote}${steering}${tipNote}${note}`);
}

// #841: resolve a requested session id to its compression state without
Expand Down
2 changes: 1 addition & 1 deletion tests/plugin-protocol.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ test("plugin tool API executes compress under the session lock; next request fol
assert.equal(searchResp.status, 200);
const searchJson = (await searchResp.json()) as { ok: boolean; result: string };
assert.equal(searchJson.ok, true);
assert.match(searchJson.result, /plugin-side compress of the two early turns/, "block summary must be retrievable via search_context");
assert.match(searchJson.result, /\u3010plugin\u3011-side compress of the two early turns/, "block summary must be retrievable via search_context");
} finally {
await h.close();
}
Expand Down
77 changes: 77 additions & 0 deletions tests/search-context-2751.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// #2751: search_context receipt semantics — candidate/truncation disclosure,
// evidence windowing around the distinctive match, and the narrowing hint.
import { test } from "node:test";
import assert from "node:assert/strict";
import { createCore, createInitialState, type CompressionBlock, type CompressionState } from "acp-kernel";
import { executeSearchContext } from "../src/decompress-shared.ts";

function mkBlock(id: string, summary: string, topic?: string, active = true): CompressionBlock {
return {
blockId: id, runId: "run-test", tier: 1, topic, summary,
directMessageIds: [], effectiveMessageIds: [], directBlockIds: [],
compressedTokens: 0, createdAt: 0, survivedCount: 0, generation: "young", active,
};
}

function stateWith(...blocks: CompressionBlock[]): CompressionState {
const s = createInitialState();
s.blocks.push(...blocks);
return s;
}

const core = createCore();

test("#2751 S1: over-limit result discloses candidate count + truncation", () => {
const blocks = Array.from({ length: 8 }, (_, i) => mkBlock(`b${i}`, `release notes audit log entry ${i}`));
const out = executeSearchContext({ query: "audit" }, core, stateWith(...blocks)).text;
assert.match(out, /^Found 5 block\(s\) for "audit":/, "default limit still 5");
assert.ok(out.includes("[truncated: showing 5 of 8 candidates by relevance"), "truncation disclosed");
assert.ok(out.includes("pass limit:8 to see the rest"), "recovery path named");
});

test("#2751 S1: exactly-limit result reports no truncation", () => {
const blocks = Array.from({ length: 5 }, (_, i) => mkBlock(`b${i}`, `release notes audit log entry ${i}`));
const out = executeSearchContext({ query: "audit" }, core, stateWith(...blocks)).text;
assert.match(out, /^Found 5 block\(s\) for "audit":/);
assert.ok(!out.includes("[truncated:"), "no truncation note when nothing was cut");
});

test("#2751 S2: distinctive term past char 200 is surfaced + marked in the window", () => {
// Filler pushes the unique identifier well beyond the old 200-char head clamp.
const filler = "audit deploy audit deploy ".repeat(9); // 225 chars of generic
const target = mkBlock("b-target", filler + "vaultwarden-password-audit v2.3 deployed and audited after review", "Vaultwarden research");
const crowd1 = mkBlock("b-c1", "audit deploy audit deploy audit deploy release notes");
const crowd2 = mkBlock("b-c2", "audit deploy rollout checklist and deploy audit log");
const out = executeSearchContext({ query: "audit deploy vaultwarden" }, core, stateWith(target, crowd1, crowd2)).text;
assert.match(out, /^Found 3 block\(s\)/);
assert.ok(out.includes("\u3010vaultwarden\u3011"), "unique term marked inside its evidence window");
// Old behavior (head-200) would show only the filler and hide "vaultwarden".
const targetLine = out.split("\n\n").find((l) => l.startsWith("b-target")) ?? "";
assert.ok(targetLine.includes("vaultwarden"), "target block's own line carries the match evidence");
});

test("#2751 S1+S3: unique term crowded out below the cutoff → truncation note + narrowing hint", () => {
// Six generic-heavy crowders out-score the single target, pushing it to rank 7.
const crowders = Array.from({ length: 6 }, (_, i) => mkBlock(`b-crowd${i}`, "audit deploy ".repeat(20) + `entry ${i}`));
const target = mkBlock("b-target", "audit deploy vaultwarden final status only");
const out = executeSearchContext({ query: "audit deploy vaultwarden" }, core, stateWith(target, ...crowders)).text;
assert.match(out, /^Found 5 block\(s\)/, "only 5 of 7 returned");
assert.ok(out.includes("[truncated: showing 5 of 7 candidates"), "truncation disclosed");
assert.ok(!out.includes("b-target"), "low-ranked target is genuinely cut, not silently dropped as 'absent'");
assert.ok(out.includes('try searching "vaultwarden" alone'), "narrowing hint names the distinctive term");
assert.ok(out.includes("none of the top 5 results contain \"vaultwarden\""), "hint scopes the negative to the returned set");
});

test("#2751 S2: CJK distinctive term past char 200 is surfaced without breaking surrogates", () => {
const filler = "部署审计进展记录 ".repeat(30); // 270 CJK chars of generic
const target = mkBlock("b-cjk", filler + "密码管理器项目最终归档完成", "归档");
const crowd = mkBlock("b-cjk2", "部署审计进展记录 " + "日常运维审计日志汇总");
const out = executeSearchContext({ query: "密码管理器 部署 审计" }, core, stateWith(target, crowd)).text;
assert.match(out, /^Found 2 block\(s\)/);
assert.ok(out.includes("\u3010密码管理器\u3011"), "distinctive CJK term marked in its window");
for (let i = 0; i < out.length; i++) {
const c = out.charCodeAt(i);
if (c >= 0xd800 && c <= 0xdbff) assert.ok(out.charCodeAt(i + 1) >= 0xdc00 && out.charCodeAt(i + 1) <= 0xdfff, "no lone high surrogate");
else if (c >= 0xdc00 && c <= 0xdfff) assert.fail("lone low surrogate leaked into the receipt");
}
});
4 changes: 3 additions & 1 deletion tests/search-context-cross-session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,9 @@ test("#841 unknown conversation_id → FAILED with id echoed", () => {
test("#841 foreign no-match and empty-state strings carry session scope", () => {
const s1 = makeSession("s1");
const core = compressInto(s1);
assert.match(executeSearchContext({ query: "zzz-no-such-topic" }, core, s1.state, "pfa-old").text, /^\[No blocks matched "zzz-no-such-topic" in session pfa-old\]$/);
// #2751: negative result is scoped to what was actually searched (active
// topics/summaries), never phrased as "history doesn't contain it".
assert.match(executeSearchContext({ query: "zzz-no-such-topic" }, core, s1.state, "pfa-old").text, /^\[No blocks matched "zzz-no-such-topic" among 1 active block\(s\) in session pfa-old\. Scope = active compressed topics\/summaries only/);
assert.equal(executeSearchContext({ query: "anything" }, createCore(), createInitialState(), "pfa-old").text, "[No compressed blocks exist yet in session pfa-old — nothing to search.]");
});

Expand Down
10 changes: 7 additions & 3 deletions tests/search-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,17 +95,21 @@ test("executeSearchContext: zero active blocks → explicit empty-state message
assert.equal(executeSearchContext({ query: "anything" }, core, state).text, "[No compressed blocks exist yet — nothing to search.]");
});

test("executeSearchContext: active block exists but none match → no-match string", () => {
test("executeSearchContext: active block exists but none match → scope-limited no-match (#2751)", () => {
const { core, state } = makeSessionWithBlock();
assert.match(executeSearchContext({ query: "zzz-no-such-topic" }, core, state).text, /^\[No blocks matched "zzz-no-such-topic"\]$/);
// Negative result states what was searched (active topics/summaries) and never
// reads as "history doesn't contain it".
assert.match(executeSearchContext({ query: "zzz-no-such-topic" }, core, state).text, /^\[No blocks matched "zzz-no-such-topic" among 1 active block\(s\)\. Scope = active compressed topics\/summaries only/);
});

test("executeSearchContext: matching block → Found listing with id/topic/preview", () => {
const { core, state } = makeSessionWithBlock();
const out = executeSearchContext({ query: "auth token" }, core, state).text;
assert.match(out, /^Found \d+ block\(s\) for "auth token":/);
assert.ok(out.includes("(T"), "tier present");
assert.ok(out.includes("auth token exchange"), "summary preview present");
// #2751: the distinctive matched term is surfaced + marked inside the window.
assert.ok(out.includes("\u3010auth\u3011"), "distinctive term marked");
assert.ok(out.includes("token exchange and refresh design decisions"), "summary preview present");
});

test("executeSearchContext: clamp cut straddling an astral char leaves no lone surrogate (#816)", () => {
Expand Down
Loading