diff --git a/kernel/package.json b/kernel/package.json index 859ed61896..78e3186ef7 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.153", + "version": "0.0.154", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/tokenize.ts b/kernel/src/tokenize.ts index 5ced7d278d..25267bdc16 100644 --- a/kernel/src/tokenize.ts +++ b/kernel/src/tokenize.ts @@ -2,14 +2,24 @@ import { createRequire } from "node:module"; const require = createRequire(import.meta.url); +// Every Han ideograph — basic block, extensions A/B–H, compatibility ideographs +// and their supplement — via the Unicode script property (stays current as new +// extension blocks land), plus the kana and Hangul-syllable ranges the estimator +// has always billed at ~1 token per char (#4315). +const CJK_TOKEN_RE = /[\p{Script=Han}\u3040-\u30ff\uac00-\ud7af]/gu; + export function defaultCountTokens(text: string): number { if (!text) return 0; - // CJK chars tokenize ~1:1 (chars/4 badly underestimates them). Count them - // directly, then estimate the non-CJK remainder with chars/4 so digits, - // punctuation, and symbols in code/JSON are not dropped to zero. - const cjk = text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g); + // CJK chars tokenize ~1:1 (some providers split rare glyphs further, but + // 1:1 is the right default bias). Count them directly, then estimate the + // non-CJK remainder with chars/4. + const cjk = text.match(CJK_TOKEN_RE); const cjkCount = cjk?.length ?? 0; - return cjkCount + Math.ceil((text.length - cjkCount) / 4); + // Supplementary-plane matches occupy TWO UTF-16 units each in text.length; + // subtract their full unit footprint (not the char count) or their second + // unit leaks into the chars/4 remainder (#4315). + const cjkUnits = cjk?.join("").length ?? 0; + return cjkCount + Math.ceil((text.length - cjkUnits) / 4); } export function estimateMessageTokens(text: string | undefined): number { diff --git a/kernel/tests/tokenizer.test.ts b/kernel/tests/tokenizer.test.ts index d8cb981844..7db8536ee2 100644 --- a/kernel/tests/tokenizer.test.ts +++ b/kernel/tests/tokenizer.test.ts @@ -60,6 +60,47 @@ test("defaultCountTokens: mixed ASCII + CJK", () => { assert.equal(count, 8); }); +// #4315: every Han ideograph bills 1 token per CHARACTER (code point), in every +// extension block, and the chars/4 remainder is computed on CODE UNITS so a +// supplementary-plane match (2 UTF-16 units) is never double-counted. +test("defaultCountTokens: Han coverage across extension blocks (#4315)", () => { + const cases: Array<[number, string]> = [ + [0x4e00, "basic block U+4E00"], + [0x9fff, "basic block tail U+9FFF"], + [0x3400, "extension A U+3400"], + [0x4dbf, "extension A tail U+4DBF"], + [0xf900, "compatibility ideographs U+F900"], + // U+FA6D is the LAST ASSIGNED compatibility ideograph; the block tail + // (U+FA6E–U+FAFF) is unassigned and must NOT match. + [0xfa6d, "compatibility ideographs tail U+FA6D"], + [0x2f800, "compatibility supplement U+2F800"], + [0x20000, "extension B U+20000"], + [0x2a700, "extension C U+2A700"], + [0x30000, "extension G U+30000"], + ]; + for (const [cp, label] of cases) { + const text = String.fromCodePoint(cp).repeat(100); + assert.equal( + defaultCountTokens(text), + 100, + `${label}: 100 chars must bill 100 tokens`, + ); + } +}); + +test("defaultCountTokens: supplementary-plane matches do not leak into the /4 remainder (#4315)", () => { + // 50 basic + 50 supplementary-plane chars = 150 UTF-16 units. Old code counted + // 50 matched chars and left 100 units in the remainder → 50 + 25 = 75. + const text = "\u4e00".repeat(50) + String.fromCodePoint(0x20000).repeat(50); + assert.equal(defaultCountTokens(text), 100); +}); + +test("defaultCountTokens: ASCII behavior unchanged by the Han widening (#4315)", () => { + // 50 ASCII (ceil(50/4) = 13) + 50 basic Han (50) → 63, same as before the fix. + assert.equal(defaultCountTokens("a".repeat(50) + "\u4e00".repeat(50)), 63); + assert.equal(defaultCountTokens("a".repeat(100)), 25); +}); + test("TokenCountFn type is compatible with createCore ports", () => { const tokenizer: (text: string) => number = createBpeTokenizer(); const fast: (text: string) => number = estimateTokensFast; diff --git a/package-lock.json b/package-lock.json index 2c15e33725..2c3f9e24df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.153", + "version": "0.0.154", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/preflight-fork-estimate.test.ts b/tests/preflight-fork-estimate.test.ts index d9f64581c3..e62b133f8e 100644 --- a/tests/preflight-fork-estimate.test.ts +++ b/tests/preflight-fork-estimate.test.ts @@ -5,7 +5,7 @@ import test from "node:test"; process.env.NODE_ENV = "test"; -import { defaultConfig } from "acp-kernel"; +import { defaultConfig, defaultCountTokens } from "acp-kernel"; import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; @@ -52,13 +52,10 @@ function okSse(inputTokens: number): string { ); } -// Realistic tokenizer model matching bili's own CJK-aware estimator (kernel -// defaultCountTokens): CJK ideographs/syllables bill ~1 token each, everything -// else ~1 token per 4 characters. -function realisticTokens(text: string): number { - const cjk = (text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g) ?? []).length; - return cjk + Math.ceil((text.length - cjk) / 4); -} +// Realistic tokenizer model: bili's own CJK-aware estimator, imported directly +// so the mock upstream's window enforcement stays on the kernel's caliber +// instead of drifting from a copied formula (#4315). +const realisticTokens = defaultCountTokens; function textBlockTokens(b: unknown): number { if (b && typeof b === "object") { diff --git a/tests/side-request-isolation.test.ts b/tests/side-request-isolation.test.ts index d27a087328..334e85ae67 100644 --- a/tests/side-request-isolation.test.ts +++ b/tests/side-request-isolation.test.ts @@ -547,7 +547,10 @@ test("sideRequestGuard: raw-body fit against declared ∩ armed window minus out assert.equal(sideRequestGuard(body, "anthropic", 0, undefined, undefined, 1).blocked, false, "unknown window → forward as before"); assert.equal(sideRequestGuard(body, "anthropic", est + 1, undefined, undefined, 1).blocked, false, "fits"); assert.equal(sideRequestGuard(body, "anthropic", Math.floor(est / 1.15), undefined, undefined, 1).blocked, true, "boundary: estimate == limit x 1.15 blocks"); - assert.equal(sideRequestGuard(body, "anthropic", Math.floor(est / 1.10), undefined, undefined, 1).blocked, false, "within the 15% estimator tolerance → forward"); + // #3900 reserves output headroom on every wire (anthropic included), so the + // declared window must also carry the max_tokens reservation for the payload + // to forward — same shape as the OpenAI case below. + assert.equal(sideRequestGuard(body, "anthropic", Math.floor(est / 1.10) + 100, undefined, undefined, 1).blocked, false, "within the 15% estimator tolerance (after the #3900 reservation) → forward"); assert.equal(sideRequestGuard(body, "anthropic", 1_000_000, undefined, undefined, 1, Math.floor(est / 1.15)).blocked, true, "armed smaller (beyond tolerance) → blocks"); assert.equal(sideRequestGuard(body, "anthropic", est + 1, undefined, undefined, 1, 1_000_000).blocked, false, "armed larger than declared is ignored"); // OpenAI wire: the output budget counts against the window → headroom reserved.