diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml new file mode 100644 index 0000000000..08704ca356 --- /dev/null +++ b/.github/workflows/ci-e2e.yml @@ -0,0 +1,44 @@ +name: E2E (real codex) + +on: + workflow_dispatch: + inputs: + forge: + description: "run forge phase (needs BILI_CODEX_COMPACT support in master)" + type: boolean + default: false + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Install codex CLI + run: npm install -g @openai/codex@latest + - run: npm ci + - run: npm run build + - name: Preflight + run: E2E_CHECK=1 node --import tsx --test tests/e2e/e2e-codex.test.ts + env: + E2E_UPSTREAM_URL: ${{ secrets.E2E_UPSTREAM_URL }} + E2E_UPSTREAM_KEY: ${{ secrets.E2E_UPSTREAM_KEY }} + - name: E2E suite + run: ACP_TEST_E2E=1 node --import tsx --test tests/e2e/e2e-codex.test.ts + env: + E2E_UPSTREAM_URL: ${{ secrets.E2E_UPSTREAM_URL }} + E2E_UPSTREAM_KEY: ${{ secrets.E2E_UPSTREAM_KEY }} + E2E_FORGE: ${{ inputs.forge && '1' || '' }} + - name: Collect artifacts + if: always() + run: | + mkdir -p artifacts && cp -r tmp/e2e-codex-* artifacts/ 2>/dev/null || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-logs + path: artifacts/ + retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index 9a97407a22..a81f004949 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,6 +106,41 @@ because `package.json` has proper `bin` + `files` fields. Either approach is fine; the registry install is preferred for testing the real published artifact. +### E2E Regression (real client through bili) + +`tests/e2e/e2e-codex.test.ts` drives the **real `codex` CLI** through a built +proxy against a real Responses-compatible upstream and asserts the full +context lifecycle end-to-end: warmup → load growth → ACP compress → purity → +native-compact interception (last phase gated by `E2E_FORGE=1`). Full phase +details, env vars, and mechanics: `tests/e2e/README.md`. + +```bash +# zero-token preflight (codex binary + dist + upstream reachable) +E2E_CHECK=1 node --import tsx --test tests/e2e/e2e-codex.test.ts + +# full run (defaults to local sglang at http://127.0.0.1:8199/v1, zero cost) +npm run build +ACP_TEST_E2E=1 node --import tsx --test tests/e2e/e2e-codex.test.ts + +# + native-compact interception phase +ACP_TEST_E2E=1 E2E_FORGE=1 node --import tsx --test tests/e2e/e2e-codex.test.ts +``` + +Rules: + +- The suite **skips by default** so `npm test` stays free; never remove the + `ACP_TEST_E2E` gate. +- Run it (at least the 4-phase core) before merging changes to the request + pipeline — `server.ts`, `src/loop/`, adapters, preflight/compact paths. + It is the only coverage that exercises real client behavior (codex UA, + wire quirks, retry loops). +- Any Responses-compatible upstream works via `E2E_UPSTREAM_URL` / + `E2E_UPSTREAM_KEY`; the provider is configured as `name = "OpenAI"` so codex + stays on the remote compaction (V2) path — do not "fix" this. +- CI (`.github/workflows/ci-e2e.yml`) is manual-dispatch only and needs repo + secrets `E2E_UPSTREAM_URL` / `E2E_UPSTREAM_KEY`; a hosted runner cannot + reach `127.0.0.1` upstreams. + ### Code Quality - **No `as any`**, **No `@ts-ignore`** diff --git a/package.json b/package.json index 19f4101e73..9754168dcc 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "registry:snapshot": "node scripts/update-registry-snapshot.mjs", "typecheck": "tsc --noEmit --project tsconfig.build.json", "test": "node --import tsx --test tests/*.test.ts", - "start": "node dist/index.js" + "start": "node dist/index.js", + "test:e2e": "node --import tsx --test tests/e2e/e2e-codex.test.ts" }, "keywords": [ "context", diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000000..99f681be50 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,71 @@ +# E2E: real codex client through bili against a real upstream + +This suite runs the **real `codex` CLI** against a **real Responses-compatible +upstream** through the bili proxy, and asserts the full context-management +lifecycle end-to-end: + +1. **warmup** — one turn; plants a known contamination value (`1400`) that the + purity guard must later defeat. +2. **load growth** — `seq` tool outputs grow the payload deterministically + (client-side; independent of model cooperation). Asserts `[acp-usage]` + input grows. +3. **ACP compress** — under a small configured window the proxy must compress + (model-cooperative `compress requested` **or** autonomous `preflight + compressed`). Asserts usage drops after compression. +4. **purity** — post-compress recall of pre-compress facts (row counts of the + `seq` runs) must be correct and must **not** echo the planted `1400`. +5. **forge** (gated by `E2E_FORGE=1`) — establishes an ACP block under an 8k + window, then flips to a 16k window with a codex-side auto-compact limit + below the ACP trigger so codex emits a **native compaction request**; + asserts `codex compact intercepted … upstream not contacted`, a + `fc_bili_` compaction item in the codex rollout, correct echo-stripping + on the next turn, and intact recall after the forged compact. + +## Running + +```bash +npm run build # suite spawns dist/index.js +ACP_TEST_E2E=1 node --import tsx --test tests/e2e/e2e-codex.test.ts +``` + +By default the suite **skips** (`set ACP_TEST_E2E=1`) so `npm test` stays free. + +## Environment + +| Variable | Default | Purpose | +|---|---|---| +| `ACP_TEST_E2E` | – | `1` enables the suite | +| `E2E_UPSTREAM_URL` | `http://127.0.0.1:8199/v1` | Responses-compatible upstream (no `/bili/` prefix) | +| `E2E_UPSTREAM_KEY` | `bili-local-test` | API key passed via codex `env_key` | +| `E2E_MODEL` | `qwen3.8-27b` | model name | +| `E2E_BILI_DIST` | repo `dist/index.js` | proxy entry under test — point at any build for capability-matrix runs | +| `E2E_CODEX_BIN` | `codex` | codex binary | +| `E2E_FORGE` | – | `1` runs the forge phase (needs a dist with `BILI_CODEX_COMPACT` support, i.e. #325+) | +| `E2E_TMO` | `300000` | per-turn timeout ms | + +Preflight (no tokens): `E2E_CHECK=1 node --import tsx --test tests/e2e/e2e-codex.test.ts` +prints codex version, dist path, and an upstream `/models` probe. + +## Mechanics + +- Full isolation: `CODEX_HOME`, `XDG_{CONFIG,CACHE,STATE}_HOME` and the work + dir (`tmp/e2e-codex-*` in the repo — **not** `/tmp`, codex refuses TMPDIR + homes) are throwaway. +- The bili window is forced via the `BILI_LAUNCHER_MODEL_WINDOWS` env + (per-model override; wins over registry peek) so compression triggers + deterministically on every run, independent of the upstream's advertised + window and of which window-alignment PRs are merged. +- Provider `name = "OpenAI"` in `config.toml` keeps codex on the remote + compaction path (V2) so the forge phase exercises the real interception. +- Assertions scrape bili's own log lines (`[acp-usage]`, `preflight + compressed`, `codex compact intercepted`, `stripped … bili compaction + item(s)`) plus codex exit codes, `--output-last-message` answers, and the + rollout JSONL. + +## CI + +`.github/workflows/ci-e2e.yml` runs the suite on `workflow_dispatch` with +`E2E_UPSTREAM_URL` / `E2E_UPSTREAM_KEY` from repository secrets, against the +repo's own `dist` (i.e. whatever is on master at dispatch time). The forge +phase is enabled via a repository variable `E2E_FORGE` so it can be turned on +once interception ships. diff --git a/tests/e2e/e2e-codex.test.ts b/tests/e2e/e2e-codex.test.ts new file mode 100644 index 0000000000..19a1a10509 --- /dev/null +++ b/tests/e2e/e2e-codex.test.ts @@ -0,0 +1,312 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +const run = process.env.ACP_TEST_E2E === "1"; +const skipReason = !run ? "set ACP_TEST_E2E=1 (real codex + real upstream; costs tokens)" : undefined; + +const UPSTREAM_URL = process.env.E2E_UPSTREAM_URL ?? "http://127.0.0.1:8199/v1"; +const UPSTREAM_KEY = process.env.E2E_UPSTREAM_KEY ?? "bili-local-test"; +const CODEX_BIN = process.env.E2E_CODEX_BIN ?? "codex"; +const DIST = process.env.E2E_BILI_DIST ?? path.resolve(import.meta.dirname, "../../dist/index.js"); +const MODEL = process.env.E2E_MODEL ?? "qwen3.8-27b"; +const TMO = Number(process.env.E2E_TMO ?? 420_000); +const FORGE = process.env.E2E_FORGE === "1"; +const WORK_ROOT = path.join(process.cwd(), "tmp"); +fs.mkdirSync(WORK_ROOT, { recursive: true }); +const WORK = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-")); + +/** Deterministic filler for payload turns: unique per index. + * Carried verbatim in user messages so context growth is fully deterministic + * (models routinely shrink tool output like `seq 1 1500 | tail -n 1`). */ +function filler(i: number, lines: number): string { + const out: string[] = []; + for (let n = 0; n < lines; n += 1) { + out.push(`doc#${String(i).padStart(2, "0")} line${String(n).padStart(4, "0")} checksum ${(n * 7919 + i * 104729) % 999983}`); + } + return out.join("\n"); +} +const SENTINELS = [4781, 2903, 6577, 1249, 8461, 30217]; + +type Ctx = { + proxy?: { pid: number; logPath: string }; + codexHome: string; + xdg: { config: string; cache: string; state: string }; + port: number; + resumed: boolean; + turnCount: number; +}; + +function freePort(): Promise { + return new Promise((resolve) => { + const s = net.createServer(); + s.listen(0, "127.0.0.1", () => { + const p = (s.address() as net.AddressInfo).port; + s.close(() => resolve(p)); + }); + }); +} + +function windowEnv(contextWindow: number): Record { + return { BILI_LAUNCHER_MODEL_WINDOWS: JSON.stringify({ [MODEL]: contextWindow }) }; +} + +function writeCodexConfig(ctx: Ctx): void { + fs.mkdirSync(ctx.codexHome, { recursive: true }); + fs.writeFileSync(path.join(ctx.codexHome, "config.toml"), [ + `model = "${MODEL}"`, + 'model_provider = "e2e"', + "model_context_window = 60000", + "", + "[model_providers.e2e]", + 'name = "OpenAI"', + `base_url = "http://127.0.0.1:${ctx.port}/bili/${UPSTREAM_URL}"`, + 'wire_api = "responses"', + 'env_key = "E2E_UPSTREAM_KEY"', + "", + ].join("\n")); +} + +function startProxy(ctx: Ctx, env: Record = {}): Promise { + return new Promise((resolve, reject) => { + const logPath = path.join(WORK, `bili-${Date.now()}.log`); + const child = spawn(process.execPath, [DIST, "start", "--port", String(ctx.port), "--no-auto-update"], { + env: { + ...process.env, + XDG_CONFIG_HOME: ctx.xdg.config, + XDG_CACHE_HOME: ctx.xdg.cache, + XDG_STATE_HOME: ctx.xdg.state, + BILLION_CONTEXT_NO_AUTO_UPDATE: "1", + ...env, + }, + stdio: ["ignore", "ignore", "pipe"], + }); + child.stderr!.on("data", (c: Buffer) => { try { fs.appendFileSync(logPath, c); } catch { /* noop */ } }); + ctx.proxy = { pid: child.pid!, logPath }; + const started = Date.now(); + const poll = (): void => { + fetch(`http://127.0.0.1:${ctx.port}/__bili/health`).then((r) => { + if (r.ok) resolve(); + else retry(); + }).catch(retry); + }; + const retry = (): void => { + if (Date.now() - started > 30_000) { killProxy(ctx); reject(new Error("proxy did not start in 30s")); return; } + setTimeout(poll, 300); + }; + poll(); + child.on("exit", (code) => { + if (Date.now() - started < 30_000) reject(new Error(`proxy exited early (code ${code}); log: ${logPath}`)); + }); + }); +} + +function killProxy(ctx: Ctx): void { + if (ctx.proxy && ctx.proxy.pid) { + try { process.kill(ctx.proxy.pid, "SIGTERM"); } catch { /* already gone */ } + } + ctx.proxy = undefined; +} + +function logs(ctx: Ctx): string { + const parts: string[] = []; + const stateLog = path.join(ctx.xdg.state, "billion-context", "bili.log"); + for (const f of fs.existsSync(stateLog) ? [stateLog] : []) parts.push(fs.readFileSync(f, "utf8")); + for (const f of fs.readdirSync(WORK).filter((x) => x.startsWith("bili-")).sort()) { + try { parts.push(fs.readFileSync(path.join(WORK, f), "utf8")); } catch { /* mid-rotation */ } + } + return parts.join(""); +} + +function turn(ctx: Ctx, prompt: string, extra: string[] = []): Promise<{ code: number; out: string; last: string }> { + ctx.turnCount += 1; + const label = `t${ctx.turnCount}`; + const outFile = path.join(WORK, `${label}.out`); + const lastFile = path.join(WORK, `${label}.last`); + const args = ["exec", "--skip-git-repo-check", "--output-last-message", lastFile, ...extra]; + if (ctx.resumed) args.push("resume", "--last"); + args.push(prompt); + return new Promise((resolve, reject) => { + const child = spawn(CODEX_BIN, args, { + cwd: WORK, + env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: UPSTREAM_KEY, RUST_LOG: "error" }, + stdio: ["ignore", "ignore", "pipe"], + }); + let err = ""; + child.stderr.on("data", (c: Buffer) => { err += c.toString(); }); + const timer = setTimeout(() => { + try { child.kill("SIGKILL"); } catch { /* noop */ } + reject(new Error(`turn ${label} timed out after ${TMO}ms`)); + }, TMO); + child.on("exit", (code) => { + clearTimeout(timer); + const last = fs.existsSync(lastFile) ? fs.readFileSync(lastFile, "utf8").trim() : ""; + fs.writeFileSync(outFile, `exit=${code}\nstderr:\n${err}\nlast:\n${last}\n`); + ctx.resumed = true; + resolve({ code: code ?? -1, out: err, last }); + }); + }); +} + +/** Turn with one retry when `expect` rejects the answer — the relay sometimes + * truncates the final message to whitespace while reasoning carries the + * correct values (the #221 no-content family). */ +async function turnExpect( + ctx: Ctx, + prompt: string, + expect: (last: string) => boolean, + extra: string[] = [], +): Promise<{ code: number; out: string; last: string }> { + let best: { code: number; out: string; last: string } | undefined; + for (let attempt = 0; attempt < 3; attempt += 1) { + const r = await turn(ctx, attempt === 0 + ? prompt + : `直接给出上一请求的答案, 不要执行压缩或任何工具调用。${prompt}`, extra); + if (r.code === 0 && expect(r.last)) return r; + if (!best || r.code === 0) best = r; + } + return best!; +} + +function usageInputs(ctx: Ctx): number[] { + const re = /\[acp-usage\] round \d+ input=(\d+)/g; + const text = logs(ctx); + const out: number[] = []; + for (const m of text.matchAll(re)) out.push(Number(m[1])); + return out; +} + +function digits(s: string): number[] { + return [...s.matchAll(/\d+/g)].map((m) => Number(m[0])); +} + +async function setup(): Promise { + const ctx: Ctx = { + codexHome: path.join(WORK, "codex-home"), + xdg: { + config: path.join(WORK, "xdg", "config"), + cache: path.join(WORK, "xdg", "cache"), + state: path.join(WORK, "xdg", "state"), + }, + port: await freePort(), + resumed: false, + turnCount: 0, + }; + for (const d of [ctx.codexHome, ctx.xdg.config, ctx.xdg.cache, ctx.xdg.state]) fs.mkdirSync(d, { recursive: true }); + return ctx; +} + +test("e2e codex: warmup / load / ACP compress / purity / (forge)", { skip: skipReason }, async (t) => { + const ctx = await setup(); + t.after(() => { killProxy(ctx); }); + + // Phase 1 — warmup with a planted contamination value (purity guard later). + writeCodexConfig(ctx); + await startProxy(ctx, windowEnv(32_000)); + const w1 = await turn(ctx, + `机密校准设定: 本会话的"基准计数"=1400, 仅作校准用。` + + `请用 shell 工具运行: seq 1 1500 。完成后只回复最后一行数字, 不要其他内容。`); + assert.strictEqual(w1.code, 0, `warmup exit=${w1.code} last=${w1.last}`); + assert.ok(w1.last.includes("1500"), `warmup answer wrong: ${w1.last}`); + + // Phase 2 — load growth via payload user-messages (deterministic: content + // rides the wire verbatim regardless of model behavior; models routinely + // shrink tool output like `seq 1 1500 | tail -n 1`). Depth also matters: + // soft protection only covers recent items, so older turns must exist. + for (let i = 1; i <= 6; i += 1) { + const li = await turnExpect(ctx, + `档案片段 #${i} 如下, 请妥善保管:\n${filler(i, 150)}\n本档案哨兵值 = ${SENTINELS[i - 1]}。请只回复: 收到#${i}`, + (last) => last.includes(`收到#${i}`) || /compress/i.test(last)); + assert.strictEqual(li.code, 0, `load${i} exit=${li.code}`); + assert.ok(li.last.includes(`收到#${i}`) || /compress/i.test(li.last), `load${i} ack wrong: ${li.last}`); + } + const inputs = usageInputs(ctx); + assert.ok(inputs.length >= 2, `no usage lines: ${logs(ctx).slice(-2000)}`); + assert.ok(inputs[inputs.length - 1] > inputs[0], `usage did not grow: ${inputs.join(",")}`); + + // Phase 3 — ACP compression (evidence usually appears during the loads — + // the window makes T1 nudges fire mid-load; one opportunistic extra turn + // if not). Purity is asked IMMEDIATELY after: at a calm post-fold usage + // level, or the model narrates another compression instead of answering. + let compressed = /preflight compressed|compress requested|\[Compressed m/.test(logs(ctx)); + if (!compressed) { + const c = await turn(ctx, + `补充档案片段 #7 如下:\n${filler(7, 300)}\n哨兵值 = 9000。请只回复: 收到#7`); + assert.strictEqual(c.code, 0, `compress-turn exit=${c.code}`); + compressed = /preflight compressed|compress requested|\[Compressed m/.test(logs(ctx)); + } + assert.ok(compressed, "no ACP compression observed within 3 turns"); + const post = usageInputs(ctx); + assert.ok(Math.min(...post.slice(-3)) < Math.max(...post), `usage never dropped after compress: ${post.join(",")}`); + + // Phase 4 — purity: recall must beat the planted 1400 and survive compression. + // One retry allowed: relays sometimes truncate the final message to + // whitespace while the reasoning channel carries the correct values. + const p1 = await turnExpect(ctx, + "只回答两个数字, 用逗号分隔, 不要其他内容: 第一次 seq 打印了多少行? 档案#6 的哨兵值是多少?", + (last) => digits(last).includes(SENTINELS[5]) && digits(last).includes(1500)); + assert.strictEqual(p1.code, 0, `purity exit=${p1.code}`); + const got = digits(p1.last); + assert.ok(got.includes(1500) && got.includes(SENTINELS[5]), `recall wrong (contaminated?): ${p1.last}`); + assert.ok(!got.includes(1400), `contamination echo detected (planted 1400): ${p1.last}`); + + // Phase 5 — forge (capability-gated). + if (!FORGE) { t.diagnostic("E2E_FORGE!=1 — forge phase skipped"); return; } + // Restart under a big window: persist reloads the SAME session (blocks + // survive — verified), and the gate's steady-vs-90% margin becomes + // trivial. The small phase-1 window stays behind for compression phases. + killProxy(ctx); + await new Promise((r) => setTimeout(r, 800)); + await startProxy(ctx, { ...windowEnv(128_000), BILI_CODEX_COMPACT: "intercept" }); + const f0 = await turnExpect(ctx, `补充档案片段 #9 如下:\n${filler(9, 300)}\n哨兵值 = 9500。请只回复: 收到#9`, + (last) => last.includes("收到#9") || /compress/i.test(last), + ["-c", "model_auto_compact_token_limit=999999"]); + assert.strictEqual(f0.code, 0, `forge-warm exit=${f0.code}`); + assert.ok(/preflight compressed|compress requested/.test(logs(ctx)), "forge phase: no ACP block established"); + // Settle turn right after the restart's first fold. + const f0b = await turnExpect(ctx, "请只回复: OK", (last) => /ok/i.test(last)); + assert.strictEqual(f0b.code, 0, `forge-settle exit=${f0b.code}`); + const f1 = await turnExpect(ctx, `补充档案片段 #10 如下:\n${filler(10, 200)}\n哨兵值 = 9600。请只回复: 收到#10`, + (last) => last.includes("收到#10") || /compress/i.test(last), + ["-c", "model_auto_compact_token_limit=3000"]); + assert.strictEqual(f1.code, 0, `forge-turn exit=${f1.code}`); + assert.ok(/codex compact intercepted/.test(logs(ctx)), + `forge not observed (BILI_CODEX_COMPACT support missing in dist, or gate passed through): last 3000 chars: ${logs(ctx).slice(-3000)}`); + const f2 = await turnExpect(ctx, "只回答一个数字: 档案#10 的哨兵值是多少?", + (last) => last.includes("9600"), + ["-c", "model_auto_compact_token_limit=3000"]); + assert.strictEqual(f2.code, 0, `post-forge exit=${f2.code}`); + assert.ok(f2.last.includes("9600"), `post-forge recall wrong: ${f2.last}`); + const sessionsDir = path.join(ctx.codexHome, "sessions"); + let rolloutHasForgeItem = false; + if (fs.existsSync(sessionsDir)) { + const walk = (d: string): void => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) walk(p); + else if (e.name.endsWith(".jsonl") && fs.readFileSync(p, "utf8").includes("fc_bili_")) rolloutHasForgeItem = true; + } + }; + walk(sessionsDir); + } + assert.ok(rolloutHasForgeItem, "codex rollout has no fc_bili_ compaction item"); +}); + +if (!run && process.env.E2E_CHECK === "1") { + test("e2e preflight check", async () => { + fs.accessSync(DIST); + const v = await new Promise((resolve, reject) => { + const c = spawn(CODEX_BIN, ["--version"], { stdio: ["ignore", "pipe", "pipe"] }); + let o = ""; c.stdout.on("data", (x: Buffer) => { o += x; }); + c.on("exit", () => resolve(o.trim())); + c.on("error", reject); + }); + console.log(`codex: ${v}; dist: ${DIST}; upstream: ${UPSTREAM_URL}`); + const probe = await fetch(`${UPSTREAM_URL}/models`).then((r) => r.status).catch(() => "unreachable"); + console.log(`upstream /models probe: ${probe}`); + }); +} +