Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions bunfig.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# so a bare `bun test` — or `bun test tests/` (a substring filter that also matches
# devlog/opencode-cursor/tests/) — drags them in and reports hundreds of spurious failures.
# `root` pins discovery to ./tests so every invocation stays on the real suite.
# File-level `--parallel` has no bunfig key; `scripts/test.ts` passes it for `bun run test`.
# The npm script already uses `bun test ./tests/`; this makes a bare `bun test` behave the same.
[test]
root = "tests"
Expand Down
118 changes: 116 additions & 2 deletions scripts/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,120 @@ export function createIsolatedTestEnvironment(
};
}

function hasCliFlag(requested: string[], name: string): boolean {
const delimiterIndex = requested.indexOf("--");
const wrapperArgs = delimiterIndex === -1 ? requested : requested.slice(0, delimiterIndex);
return wrapperArgs.some(arg => arg === name || arg.startsWith(`${name}=`));
}

// Bun 1.4.0 builds `bun test` options from its test, runtime, transpiler, and base tables.
// Only required values consume the next argument. Optional values such as `--parallel=2`
// must stay attached so a bare option cannot hide the positional filter that follows it.
const BUN_TEST_OPTIONS_REQUIRING_VALUES = new Set([
// Test options.
"--timeout",
"--rerun-each",
"--retry",
"--seed",
"--coverage-reporter",
"--coverage-dir",
"-t",
"--test-name-pattern",
"--grep",
"--reporter",
"--reporter-outfile",
"--max-concurrency",
"--path-ignore-patterns",
"--parallel-delay",
"--shard",
"--timings",
// Runtime options accepted by `bun test`.
"--watch-kill-signal",
"-r",
"--preload",
"--require",
"--import",
"--cpu-prof-name",
"--cpu-prof-dir",
"--cpu-prof-interval",
"--heap-prof-name",
"--heap-prof-dir",
"--heap-prof-interval",
"--install",
"-e",
"--eval",
"-p",
"--print",
"--port",
"--origin",
"--conditions",
"--fetch-preconnect",
"--max-http-header-size",
"--dns-result-order",
"--redirect-warnings",
"--disable-warning",
"--title",
"--unhandled-rejections",
"--console-depth",
"--user-agent",
"--cron-title",
"--cron-period",
"--trace-event-categories",
"--trace-event-file-pattern",
"--stack-trace-limit",
// Transpiler and base options accepted by `bun test`.
"--main-fields",
"--extension-order",
"--tsconfig-override",
"-d",
"--define",
"--drop",
"--feature",
"-l",
"--loader",
"--jsx-factory",
"--jsx-fragment",
"--jsx-import-source",
"--jsx-runtime",
"--env-file",
"--cwd",
"-c",
"--config",
]);
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/** True for a filter-less `bun run test`. `--timeout` / `--dots` / `--parallel=N` still count. */
function isFullSuiteRun(requested: string[]): boolean {
const delimiterIndex = requested.indexOf("--");
const wrapperArgs = delimiterIndex === -1 ? requested : requested.slice(0, delimiterIndex);
const passedThrough = delimiterIndex === -1 ? [] : requested.slice(delimiterIndex + 1);
if (passedThrough.length > 0) return false;

for (let index = 0; index < wrapperArgs.length; index++) {
const arg = wrapperArgs[index];
if (arg === "-" || !arg.startsWith("-")) return false;
if (!arg.includes("=") && BUN_TEST_OPTIONS_REQUIRING_VALUES.has(arg)) index++;
}
return true;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/**
* Default `bun test` argv for this repo.
*
* `--isolate` keeps a fresh global per file, and is the substring the exclusive-run pgrep
* matches. `--parallel` is what makes the suite finishable: with isolate alone Bun re-evaluates
* the module graph once per file on a single core, so past ~900 files the run stops looking slow
* and starts looking hung — measured here at 1 h 29 m with zero output, ~57 % CPU and 8.5 MB RSS,
* against ~110-190 s for the identical suite with `--parallel`. A caller-supplied `--parallel=N`
* is left alone.
*/
export function resolveBunTestArgs(requested: string[]): string[] {
const args = ["--isolate"];
if (!hasCliFlag(requested, "--parallel")) args.push("--parallel");
args.push(...requested);
if (isFullSuiteRun(requested)) args.push("./tests/");
return args;
}

/**
* Other `bun test` runners already on this machine.
*
Expand Down Expand Up @@ -141,7 +255,7 @@ if (import.meta.main) {
await waitForExclusiveRun(process.pid);
const startedAt = Date.now();
const child = Bun.spawnSync(
[process.execPath, "test", "--isolate", ...(requestedTests.length > 0 ? requestedTests : ["./tests/"])],
[process.execPath, "test", ...resolveBunTestArgs(requestedTests)],
{
env: isolated.env,
stdin: "inherit",
Expand All @@ -152,7 +266,7 @@ if (import.meta.main) {
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
if (requestedTests.length === 0 && elapsedSeconds > 600) {
console.warn(
`[test] the suite took ${elapsedSeconds}s; it normally runs in about 210s on an idle machine. `
`[test] the suite took ${elapsedSeconds}s; with --parallel it should finish in a few minutes on an idle machine. `
+ "Check for another test runner, a busy CPU, or a test that started polling something real.",
);
}
Expand Down
98 changes: 96 additions & 2 deletions tests/test-runner.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { describe, expect, test } from "bun:test";
import { existsSync } from "node:fs";
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { createIsolatedTestEnvironment } from "../scripts/test";
import { createIsolatedTestEnvironment, resolveBunTestArgs } from "../scripts/test";
import {
decodeWindowsIdentityPowerShellOutputForTests,
windowsIdentityPowerShellCommandForTests,
Expand Down Expand Up @@ -68,3 +69,96 @@ describe("test runner isolation", () => {
},
);
});

/**
* Without `--parallel`, `--isolate` re-evaluates the module graph once per file on a single
* core. Past ~900 files that stops reading as slow and starts reading as hung: measured at
* 1 h 29 m with zero output, ~57 % CPU and 8.5 MB RSS, against ~110-190 s for the identical
* suite with the flag. These pin the argv so the flag cannot be dropped again silently.
*/
describe("bun test argv", () => {
test("a filter-less run gets isolate, parallel and the suite path", () => {
expect(resolveBunTestArgs([])).toEqual(["--isolate", "--parallel", "./tests/"]);
});

test("a file filter keeps isolate and parallel but no suite path", () => {
expect(resolveBunTestArgs(["tests/foo.test.ts"]))
.toEqual(["--isolate", "--parallel", "tests/foo.test.ts"]);
expect(resolveBunTestArgs(["-"]))
.toEqual(["--isolate", "--parallel", "-"]);
});

test("a caller-supplied concurrency is left alone", () => {
expect(resolveBunTestArgs(["--parallel=2"]))
.toEqual(["--isolate", "--parallel=2", "./tests/"]);
expect(resolveBunTestArgs(["--parallel"]))
.toEqual(["--isolate", "--parallel", "./tests/"]);
expect(resolveBunTestArgs(["--parallel", "tests/foo.test.ts"]))
.toEqual(["--isolate", "--parallel", "tests/foo.test.ts"]);
expect(resolveBunTestArgs(["--parallel=2", "tests/foo.test.ts"]))
.toEqual(["--isolate", "--parallel=2", "tests/foo.test.ts"]);
});

test("option-only arguments still count as a full suite run", () => {
expect(resolveBunTestArgs(["--timeout=30000"]))
.toEqual(["--isolate", "--parallel", "--timeout=30000", "./tests/"]);
expect(resolveBunTestArgs(["--timeout", "30000"]))
.toEqual(["--isolate", "--parallel", "--timeout", "30000", "./tests/"]);
expect(resolveBunTestArgs(["--timeout", "30000", "tests/foo.test.ts"]))
.toEqual(["--isolate", "--parallel", "--timeout", "30000", "tests/foo.test.ts"]);
expect(resolveBunTestArgs(["--timings", ".bun-test-timings/current.json"]))
.toEqual([
"--isolate",
"--parallel",
"--timings",
".bun-test-timings/current.json",
"./tests/",
]);
for (const configFlag of ["-c", "--config"]) {
expect(resolveBunTestArgs([configFlag, "ci.bunfig.toml"]))
.toEqual(["--isolate", "--parallel", configFlag, "ci.bunfig.toml", "./tests/"]);
}
expect(resolveBunTestArgs(["-t", "serial test"])).toEqual([
"--isolate",
"--parallel",
"-t",
"serial test",
"./tests/",
]);
});

test("arguments after the delimiter are passed through instead of parsed as wrapper flags", () => {
expect(resolveBunTestArgs(["--", "--parallel=2"]))
.toEqual(["--isolate", "--parallel", "--", "--parallel=2"]);
});

test("the wrapper passes parallel execution through to bun", () => {
const fixtureRoot = mkdtempSync(join(tmpdir(), "opencodex-test-runner-"));
const fixturePath = join(fixtureRoot, "parallel-smoke.test.ts");
const markerPath = join(fixtureRoot, "executed.marker");
writeFileSync(
fixturePath,
`import { test } from "bun:test"; import { writeFileSync } from "node:fs"; test("smoke", () => writeFileSync(${JSON.stringify(markerPath)}, "executed"));\n`,
);
try {
const result = Bun.spawnSync([
process.execPath,
join(import.meta.dir, "../scripts/test.ts"),
fixturePath,
], {
cwd: join(import.meta.dir, ".."),
env: { ...process.env, OCX_TEST_NO_QUEUE: "1" },
stdout: "pipe",
stderr: "pipe",
});

const output = new TextDecoder().decode(result.stdout)
+ new TextDecoder().decode(result.stderr);
expect(result.exitCode).toBe(0);
expect(output).toContain("PARALLEL");
expect(existsSync(markerPath)).toBe(true);
} finally {
rmSync(fixtureRoot, { recursive: true, force: true });
}
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});
Loading