diff --git a/.github/workflows/ci-e2e-fake.yml b/.github/workflows/ci-e2e-fake.yml index 7fb668d1f1..3f142c62f8 100644 --- a/.github/workflows/ci-e2e-fake.yml +++ b/.github/workflows/ci-e2e-fake.yml @@ -21,8 +21,14 @@ jobs: node-version: 22 cache: npm + # Pinned (#815): an unpinned @latest let a codex release change the + # baseline system-prompt size and break the deterministic overflow phase. + # Bump deliberately after re-running this suite locally on the new version. - name: Install codex CLI - run: npm i -g @openai/codex@latest + run: npm i -g @openai/codex@0.147.0 + + - name: Print codex version + run: codex --version - name: Install deps run: npm ci diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 08704ca356..0ecd3531f7 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -17,8 +17,14 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + # Pinned (#815): an unpinned @latest let a codex release change the + # baseline system-prompt size and break the deterministic overflow phase. + # Bump deliberately after re-running this suite locally on the new version. - name: Install codex CLI - run: npm install -g @openai/codex@latest + run: npm install -g @openai/codex@0.147.0 + + - name: Print codex version + run: codex --version - run: npm ci - run: npm run build - name: Preflight diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 99f681be50..eab3859dc5 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -50,7 +50,11 @@ prints codex version, dist path, and an upstream `/models` probe. - Full isolation: `CODEX_HOME`, `XDG_{CONFIG,CACHE,STATE}_HOME` and the work dir (`tmp/e2e-codex-*` in the repo — **not** `/tmp`, codex refuses TMPDIR - homes) are throwaway. + homes) are throwaway. Codex's **spawn cwd** is deliberately OUTSIDE the repo + tree (`$TMPDIR/billion-context-e2e*`): codex discovers `AGENTS.md` by walking + up from its cwd, so an in-repo cwd folds the entire repo doc into every + request payload and couples CI to doc size (#815). A stub `AGENTS.md` does + NOT help — codex concatenates every level on the way up. - The bili window is forced via the `BILI_LAUNCHER_MODEL_WINDOWS` env (per-model override; wins over registry peek) so compression triggers deterministically on every run, independent of the upstream's advertised diff --git a/tests/e2e/e2e-codex-fake.test.ts b/tests/e2e/e2e-codex-fake.test.ts index b2856e9b7e..4896f10c18 100644 --- a/tests/e2e/e2e-codex-fake.test.ts +++ b/tests/e2e/e2e-codex-fake.test.ts @@ -1,6 +1,7 @@ import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import net from "node:net"; +import os from "node:os"; import path from "node:path"; import { strict as assert } from "node:assert"; import { test } from "node:test"; @@ -12,6 +13,10 @@ const TMO = Number(process.env.E2E_TMO ?? 120_000); const FAKE_UPSTREAM = path.join(import.meta.dirname, "fake-upstream.mjs"); const WORK_ROOT = path.join(process.cwd(), "tmp"); fs.mkdirSync(WORK_ROOT, { recursive: true }); +// codex discovers AGENTS.md by walking UP from its spawn cwd (#815): keep the +// cwd outside the repo tree or the whole repo doc leaks into every request. +const CWD_ROOT = path.join(os.tmpdir(), "billion-context-e2e"); +fs.mkdirSync(CWD_ROOT, { recursive: true }); function codexAvailable(): boolean { try { return spawnSync(CODEX_BIN, ["--version"], { timeout: 15_000 }).status === 0; } catch { return false; } @@ -61,6 +66,7 @@ function windowEnv(contextWindow: number): Record { type Ctx = { work: string; + codexCwd: string; codexHome: string; xdg: { config: string; cache: string; state: string }; port: number; @@ -76,6 +82,7 @@ async function startCtx(contextWindow: number): Promise { const work = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-fake-")); const ctx: Ctx = { work, + codexCwd: fs.mkdtempSync(path.join(CWD_ROOT, "cwd-")), codexHome: path.join(work, "codex-home"), xdg: { config: path.join(work, "xdg-config"), cache: path.join(work, "xdg-cache"), state: path.join(work, "xdg-state") }, port: await freePort(), @@ -161,7 +168,7 @@ function turn(ctx: Ctx, prompt: string): Promise<{ code: number; last: string }> args.push(prompt); return new Promise((resolve, reject) => { const child = spawn(CODEX_BIN, args, { - cwd: ctx.work, + cwd: ctx.codexCwd, env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: "fake", RUST_LOG: "error" }, stdio: ["ignore", "ignore", "pipe"], }); @@ -179,7 +186,12 @@ function turn(ctx: Ctx, prompt: string): Promise<{ code: number; last: string }> } test("overflow: compression really happens in codex; bulk folded, sentinels retained", { skip: skipReason }, async (t) => { - const ctx = await startCtx(12_000); + // 10k: hermetic cwd removed the repo AGENTS.md from the payload (#815), + // dropping the baseline to ~14k reported tokens; 10k puts the window below + // the baseline so preflight engages from the second load turn and keeps the + // forwarded payload bounded (assertion below), like the original 12k did + // against the AGENTS.md-inflated baseline. + const ctx = await startCtx(10_000); t.after(() => teardown(ctx)); const planted = [4781, 2903, 6577]; diff --git a/tests/e2e/e2e-codex.test.ts b/tests/e2e/e2e-codex.test.ts index d0f253a892..b918f34ba0 100644 --- a/tests/e2e/e2e-codex.test.ts +++ b/tests/e2e/e2e-codex.test.ts @@ -1,6 +1,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import net from "node:net"; +import os from "node:os"; import path from "node:path"; import { strict as assert } from "node:assert"; import { test } from "node:test"; @@ -18,6 +19,9 @@ const FORGE = process.env.E2E_FORGE === "1"; const WORK_ROOT = path.join(process.cwd(), "tmp"); fs.mkdirSync(WORK_ROOT, { recursive: true }); const WORK = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-")); +// codex discovers AGENTS.md by walking UP from its spawn cwd (#815): keep the +// cwd outside the repo tree or the whole repo doc leaks into every request. +const CODEX_CWD = fs.mkdtempSync(path.join(os.tmpdir(), "billion-context-e2e-")); /** Deterministic filler for payload turns: unique per index. * Carried verbatim in user messages so context growth is fully deterministic @@ -131,7 +135,7 @@ function turn(ctx: Ctx, prompt: string, extra: string[] = []): Promise<{ code: n args.push(prompt); return new Promise((resolve, reject) => { const child = spawn(CODEX_BIN, args, { - cwd: WORK, + cwd: CODEX_CWD, env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: UPSTREAM_KEY, RUST_LOG: "error" }, stdio: ["ignore", "ignore", "pipe"], }); diff --git a/tests/e2e/fake-upstream.mjs b/tests/e2e/fake-upstream.mjs index 3b3c9d21bb..3bb47549d7 100644 --- a/tests/e2e/fake-upstream.mjs +++ b/tests/e2e/fake-upstream.mjs @@ -66,11 +66,11 @@ function isSummaryRequest(parsed) { return /must be compressed because the session context exceeds|Write a \*?\*?(tier-1|\d+-tier) compression summary/.test(ins); } -function messageEvents(text) { +function messageEvents(text, inTok) { const id = uid(); const msgId = `msg_${id}`; const msg = { type: "message", id: msgId, role: "assistant", content: [{ type: "output_text", text }] }; - const inTok = 1, outTok = estOut(text); + const outTok = estOut(text); return [ ["response.created", { type: "response.created", response: { id, status: "in_progress", output: [] } }], ["response.output_item.added", { type: "response.output_item.added", output_index: 0, item: { type: "message", id: msgId, role: "assistant", content: [] } }], @@ -119,7 +119,7 @@ const server = http.createServer((req, res) => { fs.appendFileSync(REQLOG, JSON.stringify({ t: Date.now(), model: parsed.model, stream: !!parsed.stream, isSummary: summary, inputLen: raw.length, input: parsed.input }) + "\n"); } catch { /* noop */ } const inTok = estIn(raw); - if (parsed.stream) sse(res, messageEvents(text)); + if (parsed.stream) sse(res, messageEvents(text, inTok)); else jsonResponse(res, text, inTok); }); return;