Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.153",
"version": "0.0.154",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
20 changes: 15 additions & 5 deletions kernel/src/tokenize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,24 @@ import { createRequire } from "node:module";

const require = createRequire(import.meta.url);

// Every Han ideograph — basic block, extensions A/B–H, compatibility ideographs
// and their supplement — via the Unicode script property (stays current as new
// extension blocks land), plus the kana and Hangul-syllable ranges the estimator
// has always billed at ~1 token per char (#4315).
const CJK_TOKEN_RE = /[\p{Script=Han}\u3040-\u30ff\uac00-\ud7af]/gu;

export function defaultCountTokens(text: string): number {
if (!text) return 0;
// CJK chars tokenize ~1:1 (chars/4 badly underestimates them). Count them
// directly, then estimate the non-CJK remainder with chars/4 so digits,
// punctuation, and symbols in code/JSON are not dropped to zero.
const cjk = text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g);
// CJK chars tokenize ~1:1 (some providers split rare glyphs further, but
// 1:1 is the right default bias). Count them directly, then estimate the
// non-CJK remainder with chars/4.
const cjk = text.match(CJK_TOKEN_RE);
const cjkCount = cjk?.length ?? 0;
return cjkCount + Math.ceil((text.length - cjkCount) / 4);
// Supplementary-plane matches occupy TWO UTF-16 units each in text.length;
// subtract their full unit footprint (not the char count) or their second
// unit leaks into the chars/4 remainder (#4315).
const cjkUnits = cjk?.join("").length ?? 0;
return cjkCount + Math.ceil((text.length - cjkUnits) / 4);
}

export function estimateMessageTokens(text: string | undefined): number {
Expand Down
41 changes: 41 additions & 0 deletions kernel/tests/tokenizer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,47 @@ test("defaultCountTokens: mixed ASCII + CJK", () => {
assert.equal(count, 8);
});

// #4315: every Han ideograph bills 1 token per CHARACTER (code point), in every
// extension block, and the chars/4 remainder is computed on CODE UNITS so a
// supplementary-plane match (2 UTF-16 units) is never double-counted.
test("defaultCountTokens: Han coverage across extension blocks (#4315)", () => {
const cases: Array<[number, string]> = [
[0x4e00, "basic block U+4E00"],
[0x9fff, "basic block tail U+9FFF"],
[0x3400, "extension A U+3400"],
[0x4dbf, "extension A tail U+4DBF"],
[0xf900, "compatibility ideographs U+F900"],
// U+FA6D is the LAST ASSIGNED compatibility ideograph; the block tail
// (U+FA6E–U+FAFF) is unassigned and must NOT match.
[0xfa6d, "compatibility ideographs tail U+FA6D"],
[0x2f800, "compatibility supplement U+2F800"],
[0x20000, "extension B U+20000"],
[0x2a700, "extension C U+2A700"],
[0x30000, "extension G U+30000"],
];
for (const [cp, label] of cases) {
const text = String.fromCodePoint(cp).repeat(100);
assert.equal(
defaultCountTokens(text),
100,
`${label}: 100 chars must bill 100 tokens`,
);
}
});

test("defaultCountTokens: supplementary-plane matches do not leak into the /4 remainder (#4315)", () => {
// 50 basic + 50 supplementary-plane chars = 150 UTF-16 units. Old code counted
// 50 matched chars and left 100 units in the remainder → 50 + 25 = 75.
const text = "\u4e00".repeat(50) + String.fromCodePoint(0x20000).repeat(50);
assert.equal(defaultCountTokens(text), 100);
});

test("defaultCountTokens: ASCII behavior unchanged by the Han widening (#4315)", () => {
// 50 ASCII (ceil(50/4) = 13) + 50 basic Han (50) → 63, same as before the fix.
assert.equal(defaultCountTokens("a".repeat(50) + "\u4e00".repeat(50)), 63);
assert.equal(defaultCountTokens("a".repeat(100)), 25);
});

test("TokenCountFn type is compatible with createCore ports", () => {
const tokenizer: (text: string) => number = createBpeTokenizer();
const fast: (text: string) => number = estimateTokensFast;
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 5 additions & 8 deletions tests/preflight-fork-estimate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import test from "node:test";

process.env.NODE_ENV = "test";

import { defaultConfig } from "acp-kernel";
import { defaultConfig, defaultCountTokens } from "acp-kernel";
import { startServer } from "../src/server.ts";
import type { ProxyOptions } from "../src/config.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
Expand Down Expand Up @@ -52,13 +52,10 @@ function okSse(inputTokens: number): string {
);
}

// Realistic tokenizer model matching bili's own CJK-aware estimator (kernel
// defaultCountTokens): CJK ideographs/syllables bill ~1 token each, everything
// else ~1 token per 4 characters.
function realisticTokens(text: string): number {
const cjk = (text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g) ?? []).length;
return cjk + Math.ceil((text.length - cjk) / 4);
}
// Realistic tokenizer model: bili's own CJK-aware estimator, imported directly
// so the mock upstream's window enforcement stays on the kernel's caliber
// instead of drifting from a copied formula (#4315).
const realisticTokens = defaultCountTokens;

function textBlockTokens(b: unknown): number {
if (b && typeof b === "object") {
Expand Down
5 changes: 4 additions & 1 deletion tests/side-request-isolation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,10 @@ test("sideRequestGuard: raw-body fit against declared ∩ armed window minus out
assert.equal(sideRequestGuard(body, "anthropic", 0, undefined, undefined, 1).blocked, false, "unknown window → forward as before");
assert.equal(sideRequestGuard(body, "anthropic", est + 1, undefined, undefined, 1).blocked, false, "fits");
assert.equal(sideRequestGuard(body, "anthropic", Math.floor(est / 1.15), undefined, undefined, 1).blocked, true, "boundary: estimate == limit x 1.15 blocks");
assert.equal(sideRequestGuard(body, "anthropic", Math.floor(est / 1.10), undefined, undefined, 1).blocked, false, "within the 15% estimator tolerance → forward");
// #3900 reserves output headroom on every wire (anthropic included), so the
// declared window must also carry the max_tokens reservation for the payload
// to forward — same shape as the OpenAI case below.
assert.equal(sideRequestGuard(body, "anthropic", Math.floor(est / 1.10) + 100, undefined, undefined, 1).blocked, false, "within the 15% estimator tolerance (after the #3900 reservation) → forward");
assert.equal(sideRequestGuard(body, "anthropic", 1_000_000, undefined, undefined, 1, Math.floor(est / 1.15)).blocked, true, "armed smaller (beyond tolerance) → blocks");
assert.equal(sideRequestGuard(body, "anthropic", est + 1, undefined, undefined, 1, 1_000_000).blocked, false, "armed larger than declared is ignored");
// OpenAI wire: the output budget counts against the window → headroom reserved.
Expand Down
Loading