Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions src/decompress-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,24 @@ function poolTermFrequency(pool: CompressionBlock[], terms: string[]): Map<strin
return freq;
}

// Search uses the whole-string lowercase form (including contextual final
// sigma), but its offsets cannot slice the original: e.g. İ becomes i + ◌̇.
// Walk code point widths to map a matched span back without splitting either
// an expanded lowercase character or an original surrogate pair.
function originalLowercaseSpan(text: string, lowerStart: number, lowerEnd: number): { start: number; end: number } {
let lowerOffset = 0;
let start = 0;
let end = 0;
for (const char of text) {
const nextLower = lowerOffset + char.toLowerCase().length;
if (nextLower <= lowerStart) start += char.length;
end += char.length;
if (nextLower >= lowerEnd) break;
lowerOffset = nextLower;
}
return { start, end };
}

// Most distinctive query term present in this block's summary = fewest
// containing blocks, tie → earliest occurrence. Null when no term is in the
// summary body (hit then lives in the topic, already shown in the header line).
Expand All @@ -748,13 +766,16 @@ function pickAnchorTerm(summary: string, terms: string[], freq: Map<string, numb
const f = freq.get(tl) ?? 0;
if (!best || f < best.freq || (f === best.freq && i < best.index)) best = { term: tl, index: i, freq: f };
}
return best ? { term: best.term, index: best.index } : null;
if (!best) return null;
const span = originalLowercaseSpan(summary, best.index, best.index + best.term.length);
return { term: best.term, index: span.start };
}

function markFirstMatch(text: string, needle: string): string {
const idx = text.toLowerCase().indexOf(needle);
if (idx === -1) return text;
return text.slice(0, idx) + "\u3010" + text.slice(idx, idx + needle.length) + "\u3011" + text.slice(idx + needle.length);
const { start, end } = originalLowercaseSpan(text, idx, idx + needle.length);
return text.slice(0, start) + "\u3010" + text.slice(start, end) + "\u3011" + text.slice(end);
}

function evidenceSnippet(summary: string, terms: string[], freq: Map<string, number>): string {
Expand Down
5 changes: 5 additions & 0 deletions tests/e2e/e2e-update-state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,11 @@ test("self-update keeps live compression state: folds survive the OLD→NEW hop
envBase.BILI_UPDATE_POLICY_URL = "http://127.0.0.1:1/no-policy.json";
envBase.BILI_PERSIST_DEBOUNCE_MS = "200";
envBase.BILI_SESSIONS_DIR = sessionsDir;
// Pure-sqlite default (5a80e2b8e, #6480) turns the JSON envelope
// writer OFF, but this lane asserts on the on-disk .json session file
// (state survival across the update). Pin the writer ON so the
// envelope this test was written for is still produced.
envBase.BILI_STORAGE_JSON_WRITER = "on";

const port = await freePort();
proxy = spawnProxy(installDir, port, envBase);
Expand Down
48 changes: 48 additions & 0 deletions tests/search-context-unicode-offsets.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { createCore, createInitialState, type CompressionBlock } from "acp-kernel";
import { executeSearchContext } from "../src/decompress-shared.ts";

function search(summary: string, query: string): string {
const state = createInitialState();
const block: CompressionBlock = {
blockId: "b-unicode", runId: "test", tier: 1, topic: "Evidence", summary,
directMessageIds: [], effectiveMessageIds: [], directBlockIds: [],
compressedTokens: 0, createdAt: 0, survivedCount: 0, generation: "young", active: true,
};
state.blocks.push(block);
const out = executeSearchContext({ query }, createCore(), state);
assert.match(out.text, /^Found 1 block\(s\)/);
return out.text.split("\n").find((line) => line.startsWith(" ")) ?? "";
}

test("positions a distant evidence window in the original summary after expanding lowercase characters", () => {
const summary = "İ".repeat(400) + " needle-target original evidence";
const preview = search(summary, "needle-target");
assert.ok(preview.includes("【needle-target】 original evidence"), preview);
assert.ok(preview.startsWith(" ... "));
});

test("highlights the original match after an expanding lowercase prefix in a short summary", () => {
assert.equal(search("İİİİİ needle-target remains verbatim", "needle-target"), " İİİİİ 【needle-target】 remains verbatim");
});

test("does not include following text when the matched term itself expands on lowercasing", () => {
assert.equal(search("Visit İstanbul tomorrow", "İstanbul"), " Visit 【İstanbul】 tomorrow");
});

test("maps an expanded code point back to one original character", () => {
assert.equal(search("İNDIGO", "i"), " 【İ】NDIGO");
});

test("retains ordinary case-insensitive evidence highlighting", () => {
assert.equal(search("before NeEdLe after", "needle"), " before 【NeEdLe】 after");
});

test("retains whole-string lowercase matching for contextual Greek final sigma", () => {
assert.equal(search("before ΟΣ after", "ος"), " before 【ΟΣ】 after");
});

test("keeps astral prefix and match characters intact", () => {
assert.equal(search("İ😀 𐐀𐐁 remains", "𐐨𐐩"), " İ😀 【𐐀𐐁】 remains");
});
Loading