Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .github/workflows/ci-e2e-fake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,14 @@ jobs:
node-version: 22
cache: npm

# Pinned (#815): an unpinned @latest let a codex release change the
# baseline system-prompt size and break the deterministic overflow phase.
# Bump deliberately after re-running this suite locally on the new version.
- name: Install codex CLI
run: npm i -g @openai/codex@latest
run: npm i -g @openai/codex@0.147.0

- name: Print codex version
run: codex --version

- name: Install deps
run: npm ci
Expand Down
8 changes: 7 additions & 1 deletion .github/workflows/ci-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,14 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 22
# Pinned (#815): an unpinned @latest let a codex release change the
# baseline system-prompt size and break the deterministic overflow phase.
# Bump deliberately after re-running this suite locally on the new version.
- name: Install codex CLI
run: npm install -g @openai/codex@latest
run: npm install -g @openai/codex@0.147.0

- name: Print codex version
run: codex --version
- run: npm ci
- run: npm run build
- name: Preflight
Expand Down
6 changes: 5 additions & 1 deletion tests/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,11 @@ prints codex version, dist path, and an upstream `/models` probe.

- Full isolation: `CODEX_HOME`, `XDG_{CONFIG,CACHE,STATE}_HOME` and the work
dir (`tmp/e2e-codex-*` in the repo — **not** `/tmp`, codex refuses TMPDIR
homes) are throwaway.
homes) are throwaway. Codex's **spawn cwd** is deliberately OUTSIDE the repo
tree (`$TMPDIR/billion-context-e2e*`): codex discovers `AGENTS.md` by walking
up from its cwd, so an in-repo cwd folds the entire repo doc into every
request payload and couples CI to doc size (#815). A stub `AGENTS.md` does
NOT help — codex concatenates every level on the way up.
- The bili window is forced via the `BILI_LAUNCHER_MODEL_WINDOWS` env
(per-model override; wins over registry peek) so compression triggers
deterministically on every run, independent of the upstream's advertised
Expand Down
16 changes: 14 additions & 2 deletions tests/e2e/e2e-codex-fake.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { strict as assert } from "node:assert";
import { test } from "node:test";
Expand All @@ -12,6 +13,10 @@ const TMO = Number(process.env.E2E_TMO ?? 120_000);
const FAKE_UPSTREAM = path.join(import.meta.dirname, "fake-upstream.mjs");
const WORK_ROOT = path.join(process.cwd(), "tmp");
fs.mkdirSync(WORK_ROOT, { recursive: true });
// codex discovers AGENTS.md by walking UP from its spawn cwd (#815): keep the
// cwd outside the repo tree or the whole repo doc leaks into every request.
const CWD_ROOT = path.join(os.tmpdir(), "billion-context-e2e");
fs.mkdirSync(CWD_ROOT, { recursive: true });

function codexAvailable(): boolean {
try { return spawnSync(CODEX_BIN, ["--version"], { timeout: 15_000 }).status === 0; } catch { return false; }
Expand Down Expand Up @@ -61,6 +66,7 @@ function windowEnv(contextWindow: number): Record<string, string> {

type Ctx = {
work: string;
codexCwd: string;
codexHome: string;
xdg: { config: string; cache: string; state: string };
port: number;
Expand All @@ -76,6 +82,7 @@ async function startCtx(contextWindow: number): Promise<Ctx> {
const work = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-fake-"));
const ctx: Ctx = {
work,
codexCwd: fs.mkdtempSync(path.join(CWD_ROOT, "cwd-")),
codexHome: path.join(work, "codex-home"),
xdg: { config: path.join(work, "xdg-config"), cache: path.join(work, "xdg-cache"), state: path.join(work, "xdg-state") },
port: await freePort(),
Expand Down Expand Up @@ -161,7 +168,7 @@ function turn(ctx: Ctx, prompt: string): Promise<{ code: number; last: string }>
args.push(prompt);
return new Promise((resolve, reject) => {
const child = spawn(CODEX_BIN, args, {
cwd: ctx.work,
cwd: ctx.codexCwd,
env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: "fake", RUST_LOG: "error" },
stdio: ["ignore", "ignore", "pipe"],
});
Expand All @@ -179,7 +186,12 @@ function turn(ctx: Ctx, prompt: string): Promise<{ code: number; last: string }>
}

test("overflow: compression really happens in codex; bulk folded, sentinels retained", { skip: skipReason }, async (t) => {
const ctx = await startCtx(12_000);
// 10k: hermetic cwd removed the repo AGENTS.md from the payload (#815),
// dropping the baseline to ~14k reported tokens; 10k puts the window below
// the baseline so preflight engages from the second load turn and keeps the
// forwarded payload bounded (assertion below), like the original 12k did
// against the AGENTS.md-inflated baseline.
const ctx = await startCtx(10_000);
t.after(() => teardown(ctx));

const planted = [4781, 2903, 6577];
Expand Down
6 changes: 5 additions & 1 deletion tests/e2e/e2e-codex.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { spawn } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { strict as assert } from "node:assert";
import { test } from "node:test";
Expand All @@ -18,6 +19,9 @@ const FORGE = process.env.E2E_FORGE === "1";
const WORK_ROOT = path.join(process.cwd(), "tmp");
fs.mkdirSync(WORK_ROOT, { recursive: true });
const WORK = fs.mkdtempSync(path.join(WORK_ROOT, "e2e-codex-"));
// codex discovers AGENTS.md by walking UP from its spawn cwd (#815): keep the
// cwd outside the repo tree or the whole repo doc leaks into every request.
const CODEX_CWD = fs.mkdtempSync(path.join(os.tmpdir(), "billion-context-e2e-"));

/** Deterministic filler for payload turns: unique per index.
* Carried verbatim in user messages so context growth is fully deterministic
Expand Down Expand Up @@ -131,7 +135,7 @@ function turn(ctx: Ctx, prompt: string, extra: string[] = []): Promise<{ code: n
args.push(prompt);
return new Promise((resolve, reject) => {
const child = spawn(CODEX_BIN, args, {
cwd: WORK,
cwd: CODEX_CWD,
env: { ...process.env, CODEX_HOME: ctx.codexHome, E2E_UPSTREAM_KEY: UPSTREAM_KEY, RUST_LOG: "error" },
stdio: ["ignore", "ignore", "pipe"],
});
Expand Down
6 changes: 3 additions & 3 deletions tests/e2e/fake-upstream.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,11 @@ function isSummaryRequest(parsed) {
return /must be compressed because the session context exceeds|Write a \*?\*?(tier-1|\d+-tier) compression summary/.test(ins);
}

function messageEvents(text) {
function messageEvents(text, inTok) {
const id = uid();
const msgId = `msg_${id}`;
const msg = { type: "message", id: msgId, role: "assistant", content: [{ type: "output_text", text }] };
const inTok = 1, outTok = estOut(text);
const outTok = estOut(text);
return [
["response.created", { type: "response.created", response: { id, status: "in_progress", output: [] } }],
["response.output_item.added", { type: "response.output_item.added", output_index: 0, item: { type: "message", id: msgId, role: "assistant", content: [] } }],
Expand Down Expand Up @@ -119,7 +119,7 @@ const server = http.createServer((req, res) => {
fs.appendFileSync(REQLOG, JSON.stringify({ t: Date.now(), model: parsed.model, stream: !!parsed.stream, isSummary: summary, inputLen: raw.length, input: parsed.input }) + "\n");
} catch { /* noop */ }
const inTok = estIn(raw);
if (parsed.stream) sse(res, messageEvents(text));
if (parsed.stream) sse(res, messageEvents(text, inTok));
else jsonResponse(res, text, inTok);
});
return;
Expand Down
Loading