From 319c4c47011a9aae88a82388213aec44ede2c8ae Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 16 Sep 2026 12:19:56 +0800 Subject: [PATCH] fix: decouple e2e suites from repo AGENTS.md; pin codex; realistic fake usage - e2e-fake + e2e-codex: spawn codex with a cwd outside the repo tree ($TMPDIR/billion-context-e2e-*). codex discovers AGENTS.md by walking UP from its spawn cwd and concatenates every level into the system prompt, so any repo-doc growth inflated every request and broke the deterministic overflow phase (#815). State dirs stay in-repo: codex refuses TMPDIR homes and CI collects artifacts from tmp/e2e-codex-*. - overflow window 12k -> 10k: the hermetic baseline measures ~14k reported tokens, so the window must sit below it for preflight to engage early enough to keep the forwarded payload bounded (same role as the old 12k against the AGENTS.md-inflated baseline). - ci-e2e*.yml: pin @openai/codex@0.147.0 and print the version at runtime; unpinned @latest let any codex release shift the baseline and break the suite the same way. - fake-upstream.mjs: the stream path now reports estIn(raw) instead of a hardcoded 1 input token (matching the non-stream path); bili's gate consumed the reported usage, which was stuck at 1. --- .github/workflows/ci-e2e-fake.yml | 8 +++++++- .github/workflows/ci-e2e.yml | 8 +++++++- tests/e2e/README.md | 6 +++++- tests/e2e/e2e-codex-fake.test.ts | 16 ++++++++++++++-- tests/e2e/e2e-codex.test.ts | 6 +++++- tests/e2e/fake-upstream.mjs | 6 +++--- 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-e2e-fake.yml b/.github/workflows/ci-e2e-fake.yml index 7fb668d1f1..3f142c62f8 100644 --- a/.github/workflows/ci-e2e-fake.yml +++ b/.github/workflows/ci-e2e-fake.yml @@ -21,8 +21,14 @@ jobs: node-version: 22 cache: npm + # Pinned (#815): an unpinned @latest let a codex release change the + # baseline system-prompt size and break the deterministic overflow phase. + # Bump deliberately after re-running this suite locally on the new version. - name: Install codex CLI - run: npm i -g @openai/codex@latest + run: npm i -g @openai/codex@0.147.0 + + - name: Print codex version + run: codex --version - name: Install deps run: npm ci diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 08704ca356..0ecd3531f7 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -17,8 +17,14 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + # Pinned (#815): an unpinned @latest let a codex release change the + # baseline system-prompt size and break the deterministic overflow phase. + # Bump deliberately after re-running this suite locally on the new version. - name: Install codex CLI - run: npm install -g @openai/codex@latest + run: npm install -g @openai/codex@0.147.0 + + - name: Print codex version + run: codex --version - run: npm ci - run: npm run build - name: Preflight diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 99f681be50..eab3859dc5 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -50,7 +50,11 @@ prints codex version, dist path, and an upstream `/models` probe. - Full isolation: `CODEX_HOME`, `XDG_{CONFIG,CACHE,STATE}_HOME` and the work dir (`tmp/e2e-codex-*` in the repo — **not** `/tmp`, codex refuses TMPDIR - homes) are throwaway. + homes) are throwaway. Codex's **spawn cwd** is deliberately OUTSIDE the repo + tree (`$TMPDIR/billion-context-e2e*`): codex discovers `AGENTS.md` by walking + up from its cwd, so an in-repo cwd folds the entire repo doc into every + request payload and couples CI to doc size (#815). A stub `AGENTS.md` does + NOT help — codex concatenates every level on the way up. - The bili window is forced via the `BILI_LAUNCHER_MODEL_WINDOWS` env (per-model override; wins over registry peek) so compression triggers deterministically on every run, independent of the upstream's advertised diff --git a/tests/e2e/e2e-codex-fake.test.ts b/tests/e2e/e2e-codex-fake.test.ts index b2856e9b7e..4896f10c18 100644 --- a/tests/e2e/e2e-codex-fake.test.ts +++ b/tests/e2e/e2e-codex-fake.test.ts @@ -1,6 +1,7 @@ import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import net from "node:net"; +import os from "node:os"; import path from "node:path"; import { strict as assert } from "node:assert"; import { test } from "node:test"; @@ -12,6 +13,10 @@ const TMO = Number(process.env.E2E_TMO ?? 120_000); const FAKE_UPSTREAM = path.join(import.meta.dirname, "fake-upstream.mjs"); const WORK_ROOT = path.join(process.cwd(), "tmp"); fs.mkdirSync(WORK_ROOT, { recursive: true }); +// codex discovers AGENTS.md by walking UP from its spawn cwd (#815): keep the +// cwd outside the repo tree or the whole repo doc leaks into every request. +const CWD_ROOT = path.join(os.tmpdir(), "billion-context-e2e"); +fs.mkdirSync(CWD_ROOT, { recursive: true }); function codexAvailable(): boolean { try { return spawnSync(CODEX_BIN, ["--version"], { timeout: 15_000 }).status === 0; } catch { return false; } @@ -61,6 +66,7 @@ function windowEnv(contextWindow: number): Record { type Ctx = { work: string; + codexCwd: string; codexHome: string; xdg: { config: string; cache: string; state: string }; port: number; @@ -76,6 +82,7 @@ async function startCtx(contextWindow: number): Promise { const work = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-fake-")); const ctx: Ctx = { work, + codexCwd: fs.mkdtempSync(path.join(CWD_ROOT, "cwd-")), codexHome: path.join(work, "codex-home"), xdg: { config: path.join(work, "xdg-config"), cache: path.join(work, "xdg-cache"), state: path.join(work, "xdg-state") }, port: await freePort(), @@ -161,7 +168,7 @@ function turn(ctx: Ctx, prompt: string): Promise<{ code: number; last: string }> args.push(prompt); return new Promise((resolve, reject) => { const child = spawn(CODEX_BIN, args, { - cwd: ctx.work, + cwd: ctx.codexCwd, env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: "fake", RUST_LOG: "error" }, stdio: ["ignore", "ignore", "pipe"], }); @@ -179,7 +186,12 @@ function turn(ctx: Ctx, prompt: string): Promise<{ code: number; last: string }> } test("overflow: compression really happens in codex; bulk folded, sentinels retained", { skip: skipReason }, async (t) => { - const ctx = await startCtx(12_000); + // 10k: hermetic cwd removed the repo AGENTS.md from the payload (#815), + // dropping the baseline to ~14k reported tokens; 10k puts the window below + // the baseline so preflight engages from the second load turn and keeps the + // forwarded payload bounded (assertion below), like the original 12k did + // against the AGENTS.md-inflated baseline. + const ctx = await startCtx(10_000); t.after(() => teardown(ctx)); const planted = [4781, 2903, 6577]; diff --git a/tests/e2e/e2e-codex.test.ts b/tests/e2e/e2e-codex.test.ts index d0f253a892..b918f34ba0 100644 --- a/tests/e2e/e2e-codex.test.ts +++ b/tests/e2e/e2e-codex.test.ts @@ -1,6 +1,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import net from "node:net"; +import os from "node:os"; import path from "node:path"; import { strict as assert } from "node:assert"; import { test } from "node:test"; @@ -18,6 +19,9 @@ const FORGE = process.env.E2E_FORGE === "1"; const WORK_ROOT = path.join(process.cwd(), "tmp"); fs.mkdirSync(WORK_ROOT, { recursive: true }); const WORK = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-")); +// codex discovers AGENTS.md by walking UP from its spawn cwd (#815): keep the +// cwd outside the repo tree or the whole repo doc leaks into every request. +const CODEX_CWD = fs.mkdtempSync(path.join(os.tmpdir(), "billion-context-e2e-")); /** Deterministic filler for payload turns: unique per index. * Carried verbatim in user messages so context growth is fully deterministic @@ -131,7 +135,7 @@ function turn(ctx: Ctx, prompt: string, extra: string[] = []): Promise<{ code: n args.push(prompt); return new Promise((resolve, reject) => { const child = spawn(CODEX_BIN, args, { - cwd: WORK, + cwd: CODEX_CWD, env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: UPSTREAM_KEY, RUST_LOG: "error" }, stdio: ["ignore", "ignore", "pipe"], }); diff --git a/tests/e2e/fake-upstream.mjs b/tests/e2e/fake-upstream.mjs index 3b3c9d21bb..3bb47549d7 100644 --- a/tests/e2e/fake-upstream.mjs +++ b/tests/e2e/fake-upstream.mjs @@ -66,11 +66,11 @@ function isSummaryRequest(parsed) { return /must be compressed because the session context exceeds|Write a \*?\*?(tier-1|\d+-tier) compression summary/.test(ins); } -function messageEvents(text) { +function messageEvents(text, inTok) { const id = uid(); const msgId = `msg_${id}`; const msg = { type: "message", id: msgId, role: "assistant", content: [{ type: "output_text", text }] }; - const inTok = 1, outTok = estOut(text); + const outTok = estOut(text); return [ ["response.created", { type: "response.created", response: { id, status: "in_progress", output: [] } }], ["response.output_item.added", { type: "response.output_item.added", output_index: 0, item: { type: "message", id: msgId, role: "assistant", content: [] } }], @@ -119,7 +119,7 @@ const server = http.createServer((req, res) => { fs.appendFileSync(REQLOG, JSON.stringify({ t: Date.now(), model: parsed.model, stream: !!parsed.stream, isSummary: summary, inputLen: raw.length, input: parsed.input }) + "\n"); } catch { /* noop */ } const inTok = estIn(raw); - if (parsed.stream) sse(res, messageEvents(text)); + if (parsed.stream) sse(res, messageEvents(text, inTok)); else jsonResponse(res, text, inTok); }); return;