From 1823e821336dfe94defcaf598def28438ba50f62 Mon Sep 17 00:00:00 2001 From: Monikon Date: Sun, 30 Aug 2026 17:37:49 +0300 Subject: [PATCH 01/13] fix(runner): read the console log from a cursor instead of clearing it session.consoleLog.clear() wiped the whole shared buffer at the start of every test. Fine when only one test runs at a time, but it means two tests running concurrently would race to wipe each other's messages out from under them. ServerWrapper now captures its own startIndex at construction (and can resetCursor() to "now"), and the toHaveReceivedMessage matcher defaults to reading from that instead of index 0 when since isn't given. Same observable behavior for a solo test, but the log itself is never destroyed, so nothing racing to read it can lose messages. --- runner-package/lib/matchers.ts | 6 ++++-- runner-package/lib/server.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/runner-package/lib/matchers.ts b/runner-package/lib/matchers.ts index 6bab347..dd5c522 100644 --- a/runner-package/lib/matchers.ts +++ b/runner-package/lib/matchers.ts @@ -87,11 +87,13 @@ export class RunnerMatchers extends Matchers { // A player's messages are its own (see `PlayerWrapper.messageBuffer`) so one bot's chat // never satisfies an assertion made against another; the server log has no such split, - // it's one console shared by the whole session. + // it's one console shared by the whole session — and never cleared, so a test that + // doesn't pass `since` defaults to its own `ServerWrapper.startIndex` instead of 0. const buffer = this.actual instanceof PlayerWrapper ? this.actual.messageBuffer : session.consoleLog; - const view = (): string[] => buffer.slice(since); + const effectiveSince = since ?? (this.actual instanceof PlayerWrapper ? undefined : this.actual.startIndex); + const view = (): string[] => buffer.slice(effectiveSince); await this.pollAssertion( () => view().some(isMatch), diff --git a/runner-package/lib/server.ts b/runner-package/lib/server.ts index 876423a..bfa32f6 100644 --- a/runner-package/lib/server.ts +++ b/runner-package/lib/server.ts @@ -2,9 +2,21 @@ import type { Session } from './session.js'; export class ServerWrapper { readonly session: Session; + /** Default read cursor for `toHaveReceivedMessage` when no `since` is given — the log index + * at construction time, so a fresh test only sees lines from its own start. Non-destructive + * replacement for the old `session.consoleLog.clear()`: the log itself is never wiped, so + * concurrent tests reading it don't race. */ + startIndex: number; constructor(session: Session) { this.session = session; + this.startIndex = session.consoleLog.length; + } + + /** Moves the default read cursor to "now". Used by a `describe.serial` block between its + * tests, which share one `ServerWrapper` — the block-level equivalent of a fresh one. */ + resetCursor(): void { + this.startIndex = this.session.consoleLog.length; } execute(cmd: string): void { From b71620d88113262a7e0a24317055df317d071b1d Mon Sep 17 00:00:00 2001 From: Monikon Date: Sun, 30 Aug 2026 17:38:01 +0300 Subject: [PATCH 02/13] feat(runner): add concurrency option to test() and describe.serial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test(name, { concurrency: N }, fn) and describe.serial(name, { concurrency: N }, fn) fan out into N independent instances running at once, each with its own bot leased from the account pool. One failing instance fails the whole result — this is for races between real players, not a pass-rate to average. - Whole-session preflight: every spec file is loaded (imported once, registrations snapshotted) before any test runs, and every concurrency value is checked against the account pool's capacity() up front. A misconfigured concurrency aborts immediately instead of the Nth lease() hanging mid-run. An environment with no pool (LocalMode mints a throwaway account per bot) has nothing to check. - createBotScope.close() used to call session.disconnectAllBots() with no arguments, tearing down every bot in the session rather than just its own scope's. Harmless when nothing ran concurrently; with concurrency it would mean one finishing instance kicking every still-running sibling's bot. Fixed to keep every bot outside its own scope. - The report still has one row per test/test-in-block. Its durationMs is the slowest instance, and a new instances array carries every instance's own outcome (bot username, pass/fail, duration) so a failure names which bot lost the race. Wired into the console summary and the JSON report; JUnit keeps the single aggregated pass/fail, no per-instance breakdown. - A concurrent describe.serial block runs N full copies of the block; instances can diverge mid-block (one loses its race and stops early while another keeps going), so a test position only counts as skipped if every instance skipped it. - Console log lines (Test:, Serial block:, PASSED/FAILED, bot creation) are tagged with [i/N] — N instances logging the same test name at the same time is unreadable without it. --- runner-package/lib/reporter.ts | 32 +++++++ runner-package/lib/test-registry.ts | 19 ++++ runner-package/lib/test-runner.ts | 135 ++++++++++++++++++++++------ runner-package/lib/types.ts | 15 ++++ runner-package/runner.ts | 115 ++++++++++++++++++------ 5 files changed, 261 insertions(+), 55 deletions(-) diff --git a/runner-package/lib/reporter.ts b/runner-package/lib/reporter.ts index 8211386..116a8ed 100644 --- a/runner-package/lib/reporter.ts +++ b/runner-package/lib/reporter.ts @@ -15,6 +15,16 @@ function statusOf(result: TestResult): 'PASS' | 'FAIL' | 'SKIP' { return result.passed ? 'PASS' : 'FAIL'; } +/** min/avg/max duration across a `concurrency > 1` result's instances. */ +function instanceStats(instances: NonNullable): { min: number; avg: number; max: number } { + const durations = instances.map(i => i.durationMs); + return { + min: Math.min(...durations), + avg: Math.round(durations.reduce((sum, d) => sum + d, 0) / durations.length), + max: Math.max(...durations), + }; +} + export function printTestSummary(testResults: TestResult[]): number { console.log(`\n${pc.bold("=".repeat(40))}`); console.log(pc.bold(' Test Summary')); @@ -83,6 +93,17 @@ export function printTestSummary(testResults: TestResult[]): number { } } + if (result.instances) { + const { min, avg, max } = instanceStats(result.instances); + console.log(` ${pc.dim(`${result.instances.length} instances: min ${formatDuration(min)} / avg ${formatDuration(avg)} / max ${formatDuration(max)}`)}`); + for (const instance of result.instances) { + const label = instance.botUsername ?? '?'; + const status = instance.passed ? pc.green('OK') : pc.red('FAIL'); + const detail = instance.error ? pc.red(` ${instance.error.message}`) : ''; + console.log(` ${pc.dim(`- ${label}:`)} ${status} ${pc.dim(`(${formatDuration(instance.durationMs)})`)}${detail}`); + } + } + console.log(''); } @@ -126,6 +147,17 @@ export function writeJsonReport(path: string, environmentName: string, testResul error: r.error ? r.error.message : null, skipReason: r.skipReason ?? null, plugin: r.plugin ?? null, + botUsername: r.botUsername ?? null, + // Present when this row aggregates a `concurrency > 1` test/block: every instance's + // own outcome, so a failure names which bot lost the race instead of just that one did. + instances: r.instances + ? r.instances.map(i => ({ + botUsername: i.botUsername ?? null, + passed: i.passed, + durationMs: i.durationMs, + error: i.error ? i.error.message : null, + })) + : null, })), }; diff --git a/runner-package/lib/test-registry.ts b/runner-package/lib/test-registry.ts index 6ec04ec..89a5e95 100644 --- a/runner-package/lib/test-registry.ts +++ b/runner-package/lib/test-registry.ts @@ -14,6 +14,11 @@ type TestFn = (context: TestContext) => Promise; export interface TestOptions { requires?: string[]; environments?: string[]; + /** Runs this many independent instances of the test concurrently, each with its own bot + * leased from the account pool, to exercise races between players hitting the same feature + * at once. One failing instance fails the whole test. Defaults to 1 (sequential, no pool + * requirement). Validated against the pool's capacity before any test runs. */ + concurrency?: number; } interface DescribeScope { @@ -32,6 +37,7 @@ export interface TestCase { afterHooks: Hook[]; requires: string[]; environments: string[] | null; + concurrency: number; } /** What a `describe.serial` block accepts beyond the usual filters. */ @@ -50,6 +56,7 @@ export interface SerialBlock { tests: TestCase[]; requires: string[]; environments: string[] | null; + concurrency: number; } export type RegistryItem = @@ -82,9 +89,20 @@ function scopedEntry(name: string, options: TestOptions) { afterHooks: [...scopeStack].reverse().flatMap(s => s.afterHooks), requires: options.requires ?? [], environments: options.environments ?? null, + concurrency: normalizeConcurrency(options.concurrency), }; } +/** `concurrency` must be a whole number of at least 1 — anything else can't be turned into a + * bot count. Checked at registration time so a typo fails on import, not mid-run. */ +function normalizeConcurrency(concurrency: number | undefined): number { + if (concurrency === undefined) return 1; + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new Error(`concurrency must be a whole number >= 1, got ${concurrency}`); + } + return concurrency; +} + function registerTest(name: string, options: TestOptions, fn: TestFn): void { const testCase = { ...scopedEntry(name, options), fn }; if (currentBlock) { @@ -156,6 +174,7 @@ function serialImpl(label: string, optionsOrFn: SerialOptions | (() => void), ma tests: [], requires: options.requires ?? [], environments: options.environments ?? null, + concurrency: normalizeConcurrency(options.concurrency), }; currentBlock = block; diff --git a/runner-package/lib/test-runner.ts b/runner-package/lib/test-runner.ts index a492ca2..c3ff851 100644 --- a/runner-package/lib/test-runner.ts +++ b/runner-package/lib/test-runner.ts @@ -10,6 +10,17 @@ import type { BotConnectionOptions } from './environment.js'; import type { SerialBlock, TestCase } from './test-registry.js'; import type { TestContext, TestResult } from './types.js'; +/** Which of a `concurrency > 1` run's N instances this is, for console log labeling — without + * it, several instances logging the same test name at the same time is unreadable. */ +export interface InstanceTag { + index: number; + total: number; +} + +function formatInstanceTag(instance?: InstanceTag): string { + return instance ? pc.dim(` [${instance.index}/${instance.total}]`) : ''; +} + export interface RunTestCaseParams { file: string; testCase: TestCase; @@ -19,6 +30,8 @@ export interface RunTestCaseParams { timeoutMs: number; /** Set when this test came from a plugin's inherited `tests`, for report labeling. */ pluginName?: string | null; + /** Set by `runConcurrentTestCase` on each fanned-out instance, for console log labeling. */ + instance?: InstanceTag; } export interface RunSerialBlockParams { @@ -29,6 +42,8 @@ export interface RunSerialBlockParams { connOpts: BotConnectionOptions; timeoutMs: number; pluginName?: string | null; + /** Set by `runConcurrentSerialBlock` on each fanned-out instance, for console log labeling. */ + instance?: InstanceTag; } /** Bots created while one test, or one `describe.serial` block, is running: who leased what, @@ -44,7 +59,7 @@ interface BotScope { close(): Promise; } -function createBotScope(session: Session, server: ServerWrapper, connOpts: BotConnectionOptions): BotScope { +function createBotScope(session: Session, server: ServerWrapper, connOpts: BotConnectionOptions, instance?: InstanceTag): BotScope { const leased: Array<{ account: Account; pool: AccountPool }> = []; const named = new Map(); const connected: PlayerWrapper[] = []; @@ -63,7 +78,7 @@ function createBotScope(session: Session, server: ServerWrapper, connOpts: BotCo try { const botUsername = account.username; - console.log(`${pc.cyan('[Bot]')} Creating bot: ${pc.bold(botUsername)}`); + console.log(`${pc.cyan('[Bot]')} Creating bot: ${pc.bold(botUsername)}${formatInstanceTag(instance)}`); await session.env.beforeJoin?.(); @@ -103,7 +118,10 @@ function createBotScope(session: Session, server: ServerWrapper, connOpts: BotCo }, players: () => [...connected], async close(): Promise { - await session.disconnectAllBots(); + // Only this scope's own bots — a concurrent sibling instance's bots are still + // running and must not be torn down by this one finishing first. + const ownBots = new Set(connected.map(p => p.bot)); + await session.disconnectAllBots(session.bots.filter(b => !ownBots.has(b))); for (const { account, pool } of leased) pool.release(account); leased.length = 0; named.clear(); @@ -175,12 +193,12 @@ async function executeTest(params: ExecuteParams): Promise { await Promise.race([body().finally(() => clearTimeout(timeoutHandle)), timeoutPromise]); } -function reportPassed(durationMs: number): void { - console.log(` ${pc.green(pc.bold('PASSED'))} ${pc.dim(`(${formatDuration(durationMs)})`)}\n`); +function reportPassed(durationMs: number, instance?: InstanceTag): void { + console.log(` ${pc.green(pc.bold('PASSED'))}${formatInstanceTag(instance)} ${pc.dim(`(${formatDuration(durationMs)})`)}\n`); } -function reportFailed(durationMs: number, error: Error): void { - console.log(` ${pc.red(pc.bold('FAILED'))} ${pc.dim(`(${formatDuration(durationMs)})`)}: ${pc.red(error.message)}\n`); +function reportFailed(durationMs: number, error: Error, instance?: InstanceTag): void { + console.log(` ${pc.red(pc.bold('FAILED'))}${formatInstanceTag(instance)} ${pc.dim(`(${formatDuration(durationMs)})`)}: ${pc.red(error.message)}\n`); } /** @@ -188,13 +206,12 @@ function reportFailed(durationMs: number, error: Error): void { * body, and disconnects everything it created on the way out. */ export async function runTestCase(params: RunTestCaseParams): Promise { - const { file, testCase, session, plugins, connOpts, timeoutMs, pluginName = null } = params; + const { file, testCase, session, plugins, connOpts, timeoutMs, pluginName = null, instance } = params; - console.log(` ${pc.bold(`Test: ${testCase.name}`)}`); - session.consoleLog.clear(); + console.log(` ${pc.bold(`Test: ${testCase.name}`)}${formatInstanceTag(instance)}`); const server = new ServerWrapper(session); - const bots = createBotScope(session, server, connOpts); + const bots = createBotScope(session, server, connOpts, instance); const finalizers: Array<() => void | Promise> = []; const startedAt = Date.now(); @@ -203,7 +220,7 @@ export async function runTestCase(params: RunTestCaseParams): Promise { + const { concurrency, ...rest } = params; + if (concurrency <= 1) return runTestCase(rest); + + const instanceResults = await Promise.all( + Array.from({ length: concurrency }, (_, i) => runTestCase({ ...rest, instance: { index: i + 1, total: concurrency } })) + ); + return aggregateInstances(instanceResults); +} + /** * Runs a `describe.serial` block: one player, one connection, its tests in declaration order. * @@ -245,13 +277,12 @@ export async function runTestCase(params: RunTestCaseParams): Promise { - const { file, block, session, plugins, connOpts, timeoutMs, pluginName = null } = params; + const { file, block, session, plugins, connOpts, timeoutMs, pluginName = null, instance } = params; - console.log(` ${pc.bold(`Serial block: ${block.name}`)}${block.account ? pc.dim(` (account ${block.account})`) : ''}`); - session.consoleLog.clear(); + console.log(` ${pc.bold(`Serial block: ${block.name}`)}${block.account ? pc.dim(` (account ${block.account})`) : ''}${formatInstanceTag(instance)}`); const server = new ServerWrapper(session); - const bots = createBotScope(session, server, connOpts); + const bots = createBotScope(session, server, connOpts, instance); const results: TestResult[] = []; let player: PlayerWrapper; @@ -260,7 +291,7 @@ export async function runSerialBlock(params: RunSerialBlockParams): Promise index === 0 ? { file, testName: testCase.name, passed: false, durationMs: 0, error: error as Error, plugin: pluginName } @@ -279,7 +310,7 @@ export async function runSerialBlock(params: RunSerialBlockParams): Promise void | Promise> = []; @@ -320,12 +351,12 @@ export async function runSerialBlock(params: RunSerialBlockParams): Promise { + const { concurrency, ...rest } = params; + if (concurrency <= 1) return runSerialBlock(rest); + + const instanceRuns = await Promise.all( + Array.from({ length: concurrency }, (_, i) => runSerialBlock({ ...rest, instance: { index: i + 1, total: concurrency } })) + ); + return instanceRuns[0].map((_, index) => aggregateInstances(instanceRuns.map(run => run[index]))); +} + +/** + * Rolls up N concurrent runs of the same test/test-position into one `TestResult`. `durationMs` + * is the slowest instance (roughly the wall-clock cost of the `Promise.all`). + * + * Only meaningful for a serial block: its instances can diverge mid-block (one instance's race + * loses and it stops early, skipping the rest, while another keeps going) so a position isn't + * uniformly pass/fail/skip the way a plain concurrent test's instances are. An actual failure in + * any instance fails the whole result; short of that, a position only counts as skipped if every + * instance skipped it — one instance actually exercising it is enough to call it run. + */ +function aggregateInstances(results: TestResult[]): TestResult { + const first = results[0]; + const failed = results.find(r => !r.skipped && !r.passed); + const allSkipped = results.every(r => r.skipped); + return { + file: first.file, + testName: first.testName, + plugin: first.plugin, + passed: !failed, + durationMs: Math.max(...results.map(r => r.durationMs)), + error: failed?.error, + skipped: !failed && allSkipped, + skipReason: !failed && allSkipped ? first.skipReason : undefined, + instances: results.map(r => ({ + botUsername: r.botUsername, + passed: r.passed, + durationMs: r.durationMs, + error: r.error, + })), + }; +} diff --git a/runner-package/lib/types.ts b/runner-package/lib/types.ts index 6c979f4..6474d2c 100644 --- a/runner-package/lib/types.ts +++ b/runner-package/lib/types.ts @@ -18,6 +18,15 @@ export interface TestContext { cleanup: (fn: () => void | Promise) => void; } +/** One concurrent instance's own outcome, rolled up into the `instances` array of the + * aggregate `TestResult` for a `concurrency > 1` test/block. */ +export interface TestInstanceResult { + botUsername?: string; + passed: boolean; + durationMs: number; + error?: Error; +} + export interface TestResult { file: string; testName: string; @@ -30,4 +39,10 @@ export interface TestResult { skipReason?: string; /** Name of the plugin this test was inherited from, or null for a user spec. */ plugin?: string | null; + /** The bot that ran this test, when one connected. Absent for a skip, or a test that failed + * before it got as far as leasing a bot. */ + botUsername?: string; + /** Set when this result aggregates `concurrency > 1` concurrent instances: `passed` is AND + * across all of them, `durationMs` is the slowest, `error` is the first failure. */ + instances?: TestInstanceResult[]; } diff --git a/runner-package/runner.ts b/runner-package/runner.ts index 07f14c7..d8b570a 100644 --- a/runner-package/runner.ts +++ b/runner-package/runner.ts @@ -7,7 +7,7 @@ import { ItemWrapper, GuiWrapper, LiveGuiHandle, GuiItemLocator } from './lib/wr import { testRegistry, resetRegistry } from './lib/test-registry.js'; import { Session } from './lib/session.js'; import { PluginHost } from './lib/plugin-host.js'; -import { runSerialBlock, runTestCase } from './lib/test-runner.js'; +import { runSerialBlock, runTestCase, runConcurrentSerialBlock, runConcurrentTestCase } from './lib/test-runner.js'; import { skipReasonForOptions } from './lib/skip-reason.js'; import { LocalEnvironment } from './lib/environments/local.js'; import { externalEnvironment } from './lib/environments/external.js'; @@ -19,7 +19,7 @@ import type { Environment } from './lib/environment.js'; import type { EnvironmentConfig, LocalEnvironmentConfig, RunnerConfig } from './lib/config.js'; import type { ExternalEnvironmentConfig } from './lib/environments/external.js'; import type { TestResult } from './lib/types.js'; -import type { SerialBlock, TestCase } from './lib/test-registry.js'; +import type { SerialBlock, TestCase, RegistryItem } from './lib/test-registry.js'; import type { Account, AccountPool } from './lib/account.js'; // Enable source map support for accurate TypeScript stack traces @@ -147,14 +147,54 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): return null; } - /** Imports one compiled spec file (a fresh `testRegistry`) and runs everything it - * registered, appending results to `testResults`. Shared by user specs and every - * plugin-inherited test file. */ - async function runFile(file: string, pluginName: string | null): Promise { + /** One imported spec file's registered tests, snapshotted right after import so its + * concurrency values can be validated before any file's tests run — re-importing later + * to re-check wouldn't work anyway: ESM caches the module, so a second `import()` of + * the same file wouldn't re-run its top-level `test()`/`describe()` calls. */ + interface LoadedFile { + file: string; + pluginName: string | null; + items: RegistryItem[]; + } + + async function loadFile(file: string, pluginName: string | null): Promise { resetRegistry(); await import(pathToFileURL(file).href); + return { file, pluginName, items: [...testRegistry] }; + } + + /** Fails fast, before any test in the session runs, on a `concurrency` the account pool + * here can't satisfy — rather than the test itself blocking on its Nth `pool.lease()`. An + * environment with no pool (e.g. `LocalMode`) mints a synthetic throwaway account per + * connection instead of leasing one, so there's no pool capacity to check against; its + * ceiling is the server's own `max-players`, which is on the operator, not this check. */ + function validateConcurrency(loaded: LoadedFile[]): void { + const pool = env.accounts?.() ?? null; + if (!pool) return; + const capacity = pool.capacity(); + + for (const { file, items } of loaded) { + for (const item of items) { + const [kind, name, concurrency] = item.kind === 'serial' + ? ['describe.serial', item.block.name, item.block.concurrency] as const + : ['test', item.testCase.name, item.testCase.concurrency] as const; + if (concurrency <= 1) continue; + if (concurrency > capacity) { + throw new Error( + `${kind} "${name}" (${file}) declares concurrency: ${concurrency}, exceeding the ` + + `account pool's capacity (${capacity}). Reduce concurrency or grow the pool.` + ); + } + } + } + } + + /** Runs everything one loaded file registered, appending results to `testResults`. + * Shared by user specs and every plugin-inherited test file. */ + async function runLoadedFile(loaded: LoadedFile): Promise { + const { file, pluginName, items } = loaded; - for (const item of testRegistry) { + for (const item of items) { if (item.kind === 'serial') { const { block } = item; const skipReason = blockSkipReason(block); @@ -166,7 +206,10 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): continue; } - testResults.push(...await runSerialBlock({ file, block, session, plugins, connOpts, timeoutMs, pluginName })); + const results = block.concurrency > 1 + ? await runConcurrentSerialBlock({ file, block, session, plugins, connOpts, timeoutMs, pluginName, concurrency: block.concurrency }) + : await runSerialBlock({ file, block, session, plugins, connOpts, timeoutMs, pluginName }); + testResults.push(...results); continue; } @@ -178,22 +221,16 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): continue; } - const result = await runTestCase({ file, testCase, session, plugins, connOpts, timeoutMs, pluginName }); + const result = testCase.concurrency > 1 + ? await runConcurrentTestCase({ file, testCase, session, plugins, connOpts, timeoutMs, pluginName, concurrency: testCase.concurrency }) + : await runTestCase({ file, testCase, session, plugins, connOpts, timeoutMs, pluginName }); testResults.push(result); } } - // Preflight: plugin auth/setup tests, run before anything else. A failure aborts the - // whole session. - for (const { file, pluginName } of plugins.testFiles('preflight')) { - console.log(`\n${pc.blue(pc.bold(`Running preflight tests from: ${file} ${pc.dim(`(plugin ${pluginName})`)}`))}`); - const before = testResults.length; - await runFile(file, pluginName); - const failed = testResults.slice(before).find(r => !r.skipped && !r.passed); - if (failed) { - throw new Error(`Preflight test "${failed.testName}" failed (plugin ${pluginName}): ${failed.error?.message ?? 'unknown error'}`); - } - } + const preflightEntries = [...plugins.testFiles('preflight')]; + const loadedPreflight: LoadedFile[] = []; + for (const { file, pluginName } of preflightEntries) loadedPreflight.push(await loadFile(file, pluginName)); let testFiles = await findSpecFiles(config.tests.dir || process.cwd()); if (testFileFilters) { @@ -208,20 +245,44 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): }) ); } + const loadedMain: LoadedFile[] = []; + for (const file of testFiles) loadedMain.push(await loadFile(file, null)); - console.log(`${pc.bold(`Found ${testFiles.length} test file(s)${testFileFilters ? ` matching filter: ${testFileFilters.join(',')}` : ''}`)}\n`); + const suiteEntries = [...plugins.testFiles('suite')]; + const loadedSuite: LoadedFile[] = []; + for (const { file, pluginName } of suiteEntries) loadedSuite.push(await loadFile(file, pluginName)); - for (const file of testFiles) { - console.log(`\n${pc.blue(pc.bold(`Running tests from: ${file}`))}`); - await runFile(file, null); + // Whole-session preflight: every file is loaded (imported once, registrations + // snapshotted) before any of them runs, so a misconfigured `concurrency` aborts here + // instead of after burning time on earlier tests. + validateConcurrency([...loadedPreflight, ...loadedMain, ...loadedSuite]); + + // Preflight: plugin auth/setup tests, run before anything else. A failure aborts the + // whole session. + for (const loaded of loadedPreflight) { + console.log(`\n${pc.blue(pc.bold(`Running preflight tests from: ${loaded.file} ${pc.dim(`(plugin ${loaded.pluginName})`)}`))}`); + const before = testResults.length; + await runLoadedFile(loaded); + const failed = testResults.slice(before).find(r => !r.skipped && !r.passed); + if (failed) { + throw new Error(`Preflight test "${failed.testName}" failed (plugin ${loaded.pluginName}): ${failed.error?.message ?? 'unknown error'}`); + } + } + + console.log(`${pc.bold(`Found ${loadedMain.length} test file(s)${testFileFilters ? ` matching filter: ${testFileFilters.join(',')}` : ''}`)}\n`); + + for (const loaded of loadedMain) { + console.log(`\n${pc.blue(pc.bold(`Running tests from: ${loaded.file}`))}`); + await runLoadedFile(loaded); } // Suite: plugin tests that run alongside user specs, tagged with the plugin's name. - for (const { file, pluginName } of plugins.testFiles('suite')) { - console.log(`\n${pc.blue(pc.bold(`Running tests from: ${file} ${pc.dim(`(plugin ${pluginName})`)}`))}`); - await runFile(file, pluginName); + for (const loaded of loadedSuite) { + console.log(`\n${pc.blue(pc.bold(`Running tests from: ${loaded.file} ${pc.dim(`(plugin ${loaded.pluginName})`)}`))}`); + await runLoadedFile(loaded); } + } finally { await plugins.runCleanup(session, 'session'); await plugins.teardown(); From de4e9cf424bb01f0f524cdb1d91c96052d71079e Mon Sep 17 00:00:00 2001 From: Monikon Date: Sun, 30 Aug 2026 17:38:08 +0300 Subject: [PATCH 03/13] test(example): add e2e coverage for concurrent test execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers both shapes: a plain test with concurrency, and a concurrent describe.serial block. Each instance checks its own marker against the shared server log and confirms it's still connected afterward — the two bugs concurrency exposed (destructive log clear, scope-wide bot teardown) would show up here as flaky or crashed instances. --- .../src/test/e2e/tests/concurrency.spec.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 example_plugin/src/test/e2e/tests/concurrency.spec.ts diff --git a/example_plugin/src/test/e2e/tests/concurrency.spec.ts b/example_plugin/src/test/e2e/tests/concurrency.spec.ts new file mode 100644 index 0000000..3f0b84c --- /dev/null +++ b/example_plugin/src/test/e2e/tests/concurrency.spec.ts @@ -0,0 +1,38 @@ +/** + * `concurrency` fans a test (or a `describe.serial` block) out into N independent bots running + * at once — the shape a race between real players needs. These also stand in as regression + * coverage for the two bugs that concurrency exposed in the previously-sequential-only runner: + * - `session.consoleLog` used to be wiped by `clear()` at the start of every test; N bots + * writing into it at once would have raced. Each instance now reads from its own + * `ServerWrapper.startIndex` cursor instead, so its own marker is never missed no matter what + * the other instances are doing to the same shared log. + * - `createBotScope.close()` used to disconnect every bot in the session, not just its own — + * the first instance to finish would have kicked every other still-running instance's bot. + */ + +import { describe, expect, test } from '@plugwright/runner'; + +test('concurrent bots each see their own marker and stay connected', { concurrency: 3 }, async ({ player, server }) => { + const marker = `concurrency-marker-${player.username}`; + player.chat(`/say ${marker}`); + await expect(server).toHaveReceivedMessage(marker, { timeout: 10000 }); + + // Still connected: an earlier-finishing sibling instance's teardown must not have + // disconnected this one. + await player.teleport(50, 100, 50); + await expect(player).toBeNear(50, 100, 50, { tolerance: 2, timeout: 10000 }); +}); + +describe.serial('concurrent kit lifecycle', { concurrency: 2 }, () => { + test('claims the starter kit', async ({ player }) => { + player.chat('/kit starter'); + + await expect(player).toHaveReceivedMessage('Received starter kit'); + await expect(player).toContainItem('diamond_sword'); + }); + + test('is on cooldown right after', async ({ player }) => { + player.chat('/kit starter'); + await expect(player).toHaveReceivedMessage('cooldown'); + }); +}); From 405caacb31fd65fd1e2c22612f8f4e60732bb101 Mon Sep 17 00:00:00 2001 From: Monikon Date: Sun, 30 Aug 2026 17:38:15 +0300 Subject: [PATCH 04/13] docs: document the concurrency option and its report shape Writing Tests gets a new section covering test()/describe.serial with concurrency: what it's for, what you get back, how many instances you can ask for, and why the server log stays shared and unfiltered across instances. Reports gets the aggregated JSON shape (instances array, botUsername) and a note that JUnit only ever sees the one aggregate. --- docs/reports.mdx | 26 +++++++++++++++++++++++++- docs/writing-tests.mdx | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/reports.mdx b/docs/reports.mdx index 49b7659..9367166 100644 --- a/docs/reports.mdx +++ b/docs/reports.mdx @@ -40,12 +40,36 @@ build/reports/plugwright/.log per-environment output, matrix runs o } ``` -`status` is `pass`, `fail` or `skip`. `plugin` names the plugin a test came from when it was inherited rather than found in your test directory. +`status` is `pass`, `fail` or `skip`. `plugin` names the plugin a test came from when it was inherited rather than found in your test directory. `botUsername` is the bot that ran it, when one connected. Every skip carries its reason: excluded by name, wrong environment, a capability the environment doesn't have, or an earlier test in the same [`describe.serial`](/writing-tests) block that stopped the chain. A skipped test that doesn't say why is worse than a failing one, because it reads as coverage. Tests from a serial block appear as ordinary entries, in the order they ran, under their full `describe` path. +## Concurrent tests + +A test (or block) run with [`concurrency`](/writing-tests) still gets one entry, not N. `durationMs` is the slowest instance, and `instances` carries every instance's own outcome: + +```json +{ + "file": "…/dist/claim.spec.js", + "name": "only one player can claim the chest", + "status": "fail", + "durationMs": 812, + "error": "Expected message matching \"Claimed\" not received", + "skipReason": null, + "plugin": null, + "botUsername": null, + "instances": [ + { "botUsername": "pw_a1", "passed": true, "durationMs": 640, "error": null }, + { "botUsername": "pw_b2", "passed": false, "durationMs": 812, "error": "Expected message matching \"Claimed\" not received" }, + { "botUsername": "pw_c3", "passed": true, "durationMs": 701, "error": null } + ] +} +``` + +`instances` is `null` for an ordinary, non-concurrent test — `botUsername` on the row itself is where its bot lives instead. The JUnit report doesn't carry this breakdown; it only ever sees the one aggregated pass/fail/duration, so read the JSON report when a concurrent test fails. + ## JUnit XML ```xml diff --git a/docs/writing-tests.mdx b/docs/writing-tests.mdx index 9319cc2..53225d8 100644 --- a/docs/writing-tests.mdx +++ b/docs/writing-tests.mdx @@ -199,7 +199,7 @@ describe.serial('vip shop', { account: 'pw_0001' }, async () => { }); ``` -The account must exist in the environment's `accounts { }` pool and be free. If it is already leased, or the environment has no pool at all (`local` invents a name per bot and has none), the block fails with that message rather than quietly running as somebody else. +The account must exist in the environment's `accounts { }` pool and be free. If it is already leased, or the environment has no pool at all (`LocalMode` invents a name per bot and has none), the block fails with that message rather than quietly running as somebody else. ### A second bot for the block @@ -221,6 +221,40 @@ describe.serial('trading', () => { }); ``` +## Racing bots against each other: `concurrency` + +One bot can't produce a race. Two players opening the same chest at once, three players buying the last item in stock — bugs like that only exist when several bots hit the same feature at the same time, on the same server. `concurrency` runs a test as N independent instances at once, each with its own bot: + +```typescript +test('only one player can claim the chest', { concurrency: 3 }, async ({ player }) => { + player.chat('/claim'); + await expect(player).toHaveReceivedMessage(/Claimed|already claimed/); +}); +``` + +Each instance leases its own account and runs the full test body on its own bot. The test only passes if every instance does — one instance losing the race it wasn't supposed to lose is the bug you're trying to catch, not noise to average away. + +`describe.serial` blocks take the same option, running N independent copies of the whole ordered chain at once: + +```typescript +describe.serial('kit lifecycle', { concurrency: 5 }, () => { + test('claims the starter kit', async ({ player }) => { /* ... */ }); + test('is on cooldown right after', async ({ player }) => { /* ... */ }); +}); +``` + +### The report still has one row per test + +`concurrency` multiplies how many bots run a test, not how many rows it produces in the summary. That row's duration is the slowest instance, and it carries an `instances` array with every instance's own outcome — bot username, pass or fail, duration — so a failure tells you which bot lost, not just that somebody did. + +### How many you can ask for + +`concurrency: N` needs N free accounts. On an environment with an account pool, this is checked before any test in the run starts: ask for `concurrency: 10` against a 4-account pool and it fails immediately with a clear error, instead of the 5th bot hanging on a lease nobody's going to release. `LocalMode` has no pool — it mints a throwaway account per bot — so there's nothing to check there; its only real ceiling is the server's own `max-players`. + +### The server log is still one shared log + +`expect(server)` reads the console output the whole session shares, so one instance's commands sit in that log right next to every other instance's. Nothing filters that for you, and that's deliberate — asserting the server log never produced something no bot should have caused (`expect(server).not.toHaveReceivedMessage('NullPointerException')`) is exactly what a concurrency test is for. If you need one bot's output specifically, either put its username in the pattern or check `expect(player)` instead, since a player's own messages never mix with another bot's. + ## Best Practices 1. **Keep tests isolated** - Each test gets a fresh bot unless it is in a `describe.serial` block @@ -232,7 +266,7 @@ describe.serial('trading', () => { ## Tips -- Tests run sequentially, not in parallel +- Tests run sequentially by default — a test that needs bots racing each other can opt into `concurrency` - Server starts fresh for each test run - Bot automatically connects to the server - Server logs are visible in the console output From 647d3d8e48df29033ffbca23ea8cd8785fab20e4 Mon Sep 17 00:00:00 2001 From: Monikon Date: Sun, 30 Aug 2026 17:57:02 +0300 Subject: [PATCH 05/13] fix(example): skip the server-log concurrency test where console output isn't full MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit expect(server).toHaveReceivedMessage() needs consoleOutput: 'full'. The stand environment's RCON console only offers 'responses', same limitation simple-ts.spec.ts's 'server logs command execution' already declares — this test needed the same requires and didn't have it, so it failed test-example-plugin-stand in CI instead of skipping. --- example_plugin/src/test/e2e/tests/concurrency.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_plugin/src/test/e2e/tests/concurrency.spec.ts b/example_plugin/src/test/e2e/tests/concurrency.spec.ts index 3f0b84c..df45bca 100644 --- a/example_plugin/src/test/e2e/tests/concurrency.spec.ts +++ b/example_plugin/src/test/e2e/tests/concurrency.spec.ts @@ -12,7 +12,7 @@ import { describe, expect, test } from '@plugwright/runner'; -test('concurrent bots each see their own marker and stay connected', { concurrency: 3 }, async ({ player, server }) => { +test('concurrent bots each see their own marker and stay connected', { concurrency: 3, requires: ['consoleOutput:full'] }, async ({ player, server }) => { const marker = `concurrency-marker-${player.username}`; player.chat(`/say ${marker}`); await expect(server).toHaveReceivedMessage(marker, { timeout: 10000 }); From 1ef323225436c004784ff12fec42926f7282710b Mon Sep 17 00:00:00 2001 From: Monikon Date: Sun, 30 Aug 2026 18:11:25 +0300 Subject: [PATCH 06/13] feat(runner): show instance pass counts in the report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additions, both surfacing data the aggregate result already had: - TestInstanceResult gets index (1-based, matching the [i/N] console log tag for that same run) — was missing from both the console detail breakdown and the JSON report. - The summary table's own row for a concurrent test now tags itself with [passed/total] next to the duration, instead of that count only showing up in the Failed Tests detail section below. A passing concurrent test previously gave no indication in the table that it was even concurrent. --- docs/reports.mdx | 6 +++--- runner-package/lib/reporter.ts | 11 +++++++++-- runner-package/lib/test-runner.ts | 3 ++- runner-package/lib/types.ts | 3 +++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/reports.mdx b/docs/reports.mdx index 9367166..54d7b4c 100644 --- a/docs/reports.mdx +++ b/docs/reports.mdx @@ -61,9 +61,9 @@ A test (or block) run with [`concurrency`](/writing-tests) still gets one entry, "plugin": null, "botUsername": null, "instances": [ - { "botUsername": "pw_a1", "passed": true, "durationMs": 640, "error": null }, - { "botUsername": "pw_b2", "passed": false, "durationMs": 812, "error": "Expected message matching \"Claimed\" not received" }, - { "botUsername": "pw_c3", "passed": true, "durationMs": 701, "error": null } + { "index": 1, "botUsername": "pw_a1", "passed": true, "durationMs": 640, "error": null }, + { "index": 2, "botUsername": "pw_b2", "passed": false, "durationMs": 812, "error": "Expected message matching \"Claimed\" not received" }, + { "index": 3, "botUsername": "pw_c3", "passed": true, "durationMs": 701, "error": null } ] } ``` diff --git a/runner-package/lib/reporter.ts b/runner-package/lib/reporter.ts index 116a8ed..dfa8a8c 100644 --- a/runner-package/lib/reporter.ts +++ b/runner-package/lib/reporter.ts @@ -65,7 +65,12 @@ export function printTestSummary(testResults: TestResult[]): number { ? pc.yellow(pc.bold(statusPadded)) : pc.red(pc.bold(statusPadded)); const duration = formatDuration(result.durationMs); - console.log(` ${coloredStatus} ${result.testName.padEnd(testWidth)} ${pc.dim(duration.padStart(durationWidth))}`); + // A concurrent test/block's row is one aggregate over N instances — say how many + // passed right in the table, not just in the failed-tests detail below. + const instanceTag = result.instances + ? pc.dim(` [${result.instances.filter(i => i.passed).length}/${result.instances.length}]`) + : ''; + console.log(` ${coloredStatus} ${result.testName.padEnd(testWidth)} ${pc.dim(duration.padStart(durationWidth))}${instanceTag}`); } console.log(separator); @@ -97,10 +102,11 @@ export function printTestSummary(testResults: TestResult[]): number { const { min, avg, max } = instanceStats(result.instances); console.log(` ${pc.dim(`${result.instances.length} instances: min ${formatDuration(min)} / avg ${formatDuration(avg)} / max ${formatDuration(max)}`)}`); for (const instance of result.instances) { + const tag = `[${instance.index}/${result.instances.length}]`; const label = instance.botUsername ?? '?'; const status = instance.passed ? pc.green('OK') : pc.red('FAIL'); const detail = instance.error ? pc.red(` ${instance.error.message}`) : ''; - console.log(` ${pc.dim(`- ${label}:`)} ${status} ${pc.dim(`(${formatDuration(instance.durationMs)})`)}${detail}`); + console.log(` ${pc.dim(`- ${tag} ${label}:`)} ${status} ${pc.dim(`(${formatDuration(instance.durationMs)})`)}${detail}`); } } @@ -152,6 +158,7 @@ export function writeJsonReport(path: string, environmentName: string, testResul // own outcome, so a failure names which bot lost the race instead of just that one did. instances: r.instances ? r.instances.map(i => ({ + index: i.index, botUsername: i.botUsername ?? null, passed: i.passed, durationMs: i.durationMs, diff --git a/runner-package/lib/test-runner.ts b/runner-package/lib/test-runner.ts index c3ff851..1487deb 100644 --- a/runner-package/lib/test-runner.ts +++ b/runner-package/lib/test-runner.ts @@ -415,7 +415,8 @@ function aggregateInstances(results: TestResult[]): TestResult { error: failed?.error, skipped: !failed && allSkipped, skipReason: !failed && allSkipped ? first.skipReason : undefined, - instances: results.map(r => ({ + instances: results.map((r, i) => ({ + index: i + 1, botUsername: r.botUsername, passed: r.passed, durationMs: r.durationMs, diff --git a/runner-package/lib/types.ts b/runner-package/lib/types.ts index 6474d2c..896a17a 100644 --- a/runner-package/lib/types.ts +++ b/runner-package/lib/types.ts @@ -21,6 +21,9 @@ export interface TestContext { /** One concurrent instance's own outcome, rolled up into the `instances` array of the * aggregate `TestResult` for a `concurrency > 1` test/block. */ export interface TestInstanceResult { + /** 1-based position among the N concurrent instances — matches the `[i/N]` tag in the + * console log for this same run. */ + index: number; botUsername?: string; passed: boolean; durationMs: number; From dcc5ef858347cfc7e21271c65762d277e247ca76 Mon Sep 17 00:00:00 2001 From: Monikon Date: Thu, 3 Sep 2026 05:25:16 +0300 Subject: [PATCH 07/13] fix(session): stop disconnectAllBots from losing bots added mid-teardown disconnectAllBots recomputed "remaining" from this.bots after its await, using a keepSet fixed at call start. A bot connected by a concurrent test while the await was in flight was neither in keepSet nor among the ones just disconnected, so the post-await reset silently dropped it from tracking without ever disconnecting it. Now it snapshots the bots to disconnect before awaiting and removes exactly those afterward, leaving anything added mid-await untouched. --- runner-package/lib/session.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/runner-package/lib/session.ts b/runner-package/lib/session.ts index 8e1525d..31d2087 100644 --- a/runner-package/lib/session.ts +++ b/runner-package/lib/session.ts @@ -168,19 +168,25 @@ export class Session { * * Each bot goes through `disconnectBot`, which is also what strips its listeners: a kept * bot is still connected and still listening, so tearing the others down must not be a - * second implementation that forgets to. */ + * second implementation that forgets to. + * + * Snapshots which bots to disconnect before the `await`, then removes exactly those from + * `this.bots` afterward — not "whatever isn't in `keep`" recomputed after the fact. A + * concurrent caller can push a new bot onto `this.bots` while this call is awaiting; that + * bot is in neither snapshot, so it survives here untouched instead of being silently + * dropped from tracking without ever being disconnected. */ async disconnectAllBots(keep: Bot[] = []): Promise { const keepSet = new Set(keep); + const toDisconnect = this.bots.filter(b => !keepSet.has(b)); await Promise.all( - this.bots - .filter(b => !keepSet.has(b)) - .map((b, i) => this.disconnectBot(b, b.username ?? `bot-${i}`, 2000)) + toDisconnect.map((b, i) => this.disconnectBot(b, b.username ?? `bot-${i}`, 2000)) ); - const remaining = this.bots.filter(b => keepSet.has(b)); - this.bots.length = 0; - this.bots.push(...remaining); + const disconnectedSet = new Set(toDisconnect); + for (let i = this.bots.length - 1; i >= 0; i--) { + if (disconnectedSet.has(this.bots[i])) this.bots.splice(i, 1); + } } /** Feeds raw environment output (e.g. Minecraft server stdout/stderr) into the console log buffer. */ From 7109b6b298f8b900910937a860d5b4deceb7605c Mon Sep 17 00:00:00 2001 From: Monikon Date: Thu, 3 Sep 2026 05:25:41 +0300 Subject: [PATCH 08/13] fix(test-runner): disconnect bots whose join() failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createBotScope.close() tracked "own" bots as connected.map(p => p.bot) — populated only after a successful join(). A bot whose join() failed never landed in connected, so close() treated it as belonging to another scope and left it connected, leaking the connection. Track every bot the scope creates in ownBots instead, regardless of join outcome, and tear down from that. --- runner-package/lib/test-runner.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/runner-package/lib/test-runner.ts b/runner-package/lib/test-runner.ts index 1487deb..8011249 100644 --- a/runner-package/lib/test-runner.ts +++ b/runner-package/lib/test-runner.ts @@ -1,4 +1,5 @@ import pc from 'picocolors'; +import type { Bot } from 'mineflayer'; import { PlayerWrapper } from './player.js'; import { ServerWrapper } from './server.js'; import { formatDuration } from './reporter.js'; @@ -63,6 +64,10 @@ function createBotScope(session: Session, server: ServerWrapper, connOpts: BotCo const leased: Array<{ account: Account; pool: AccountPool }> = []; const named = new Map(); const connected: PlayerWrapper[] = []; + // Every bot this scope created, whether or not `player.join()` went on to succeed — unlike + // `connected`, which only gains an entry after a successful join. `close()` needs this one: + // a bot whose join failed still opened a real connection and must still be torn down. + const ownBots: Bot[] = []; const connect = async (options?: { username?: string; account?: string }): Promise => { const pool = options?.username ? null : session.env.accounts?.() ?? null; @@ -88,6 +93,7 @@ function createBotScope(session: Session, server: ServerWrapper, connOpts: BotCo profilesFolder: account.microsoftCacheDir, }; const bot = session.createBot({ ...botOptions, username: botUsername }); + ownBots.push(bot); const player = new PlayerWrapper(bot, session); player._captureSpawnPromise(); player.setServerWrapper(server); @@ -119,13 +125,17 @@ function createBotScope(session: Session, server: ServerWrapper, connOpts: BotCo players: () => [...connected], async close(): Promise { // Only this scope's own bots — a concurrent sibling instance's bots are still - // running and must not be torn down by this one finishing first. - const ownBots = new Set(connected.map(p => p.bot)); - await session.disconnectAllBots(session.bots.filter(b => !ownBots.has(b))); + // running and must not be torn down by this one finishing first. Uses `ownBots` + // (everything this scope ever created), not `connected`, so a bot whose join() + // failed and never made it into `connected` still gets torn down here instead of + // leaking an open connection forever. + const ownBotsSet = new Set(ownBots); + await session.disconnectAllBots(session.bots.filter(b => !ownBotsSet.has(b))); for (const { account, pool } of leased) pool.release(account); leased.length = 0; named.clear(); connected.length = 0; + ownBots.length = 0; }, }; } From a529bb560f3e2265666386f9e8230df2431a3cb4 Mon Sep 17 00:00:00 2001 From: Monikon Date: Thu, 3 Sep 2026 05:25:45 +0300 Subject: [PATCH 09/13] fix(test-registry): reject concurrency on tests inside describe.serial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A serial block always runs its tests sequentially on one player, so a per-test concurrency set inside one was silently ignored by the block runner — nothing warned, the test just ran once. Now it throws at registration time, same as an invalid concurrency value does. --- runner-package/lib/test-registry.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/runner-package/lib/test-registry.ts b/runner-package/lib/test-registry.ts index 89a5e95..40a4819 100644 --- a/runner-package/lib/test-registry.ts +++ b/runner-package/lib/test-registry.ts @@ -106,6 +106,16 @@ function normalizeConcurrency(concurrency: number | undefined): number { function registerTest(name: string, options: TestOptions, fn: TestFn): void { const testCase = { ...scopedEntry(name, options), fn }; if (currentBlock) { + // A serial block always runs its tests one after another on the same player — fanning + // one of them out into concurrent instances makes no sense and would otherwise be + // silently ignored (the block runner never reads a per-test concurrency), leaving + // whoever set it wondering why nothing ran concurrently. + if (testCase.concurrency > 1) { + throw new Error( + `test "${name}": concurrency is not supported on tests inside describe.serial ` + + `(block "${currentBlock.name}") — set concurrency on the describe.serial block itself instead.` + ); + } currentBlock.tests.push(testCase); } else { testRegistry.push({ kind: 'test', testCase }); From 37fc1046a420f4e03acac931a928eebd465948f7 Mon Sep 17 00:00:00 2001 From: Monikon Date: Thu, 3 Sep 2026 05:25:50 +0300 Subject: [PATCH 10/13] fix(runner): load main/suite spec files after preflight runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateConcurrency needed every file loaded (imported) up front to check concurrency before any test ran, which meant main/suite spec files were now imported before preflight ran — importing against whatever state preflight was about to set up, rather than what it left behind. Preflight files are now loaded, validated, and run on their own first; main and suite files are loaded and validated afterward, restoring the original import order while keeping the fail-fast concurrency check. --- runner-package/runner.ts | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/runner-package/runner.ts b/runner-package/runner.ts index d8b570a..255c381 100644 --- a/runner-package/runner.ts +++ b/runner-package/runner.ts @@ -232,6 +232,23 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): const loadedPreflight: LoadedFile[] = []; for (const { file, pluginName } of preflightEntries) loadedPreflight.push(await loadFile(file, pluginName)); + // Preflight files are loaded and validated on their own, before any main/suite spec + // file is imported — importing those here would run their top-level code ahead of + // preflight, against whatever state preflight was going to set up during execution. + validateConcurrency(loadedPreflight); + + // Preflight: plugin auth/setup tests, run before anything else. A failure aborts the + // whole session. + for (const loaded of loadedPreflight) { + console.log(`\n${pc.blue(pc.bold(`Running preflight tests from: ${loaded.file} ${pc.dim(`(plugin ${loaded.pluginName})`)}`))}`); + const before = testResults.length; + await runLoadedFile(loaded); + const failed = testResults.slice(before).find(r => !r.skipped && !r.passed); + if (failed) { + throw new Error(`Preflight test "${failed.testName}" failed (plugin ${loaded.pluginName}): ${failed.error?.message ?? 'unknown error'}`); + } + } + let testFiles = await findSpecFiles(config.tests.dir || process.cwd()); if (testFileFilters) { const patterns = testFileFilters; @@ -252,22 +269,11 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): const loadedSuite: LoadedFile[] = []; for (const { file, pluginName } of suiteEntries) loadedSuite.push(await loadFile(file, pluginName)); - // Whole-session preflight: every file is loaded (imported once, registrations - // snapshotted) before any of them runs, so a misconfigured `concurrency` aborts here - // instead of after burning time on earlier tests. - validateConcurrency([...loadedPreflight, ...loadedMain, ...loadedSuite]); - - // Preflight: plugin auth/setup tests, run before anything else. A failure aborts the - // whole session. - for (const loaded of loadedPreflight) { - console.log(`\n${pc.blue(pc.bold(`Running preflight tests from: ${loaded.file} ${pc.dim(`(plugin ${loaded.pluginName})`)}`))}`); - const before = testResults.length; - await runLoadedFile(loaded); - const failed = testResults.slice(before).find(r => !r.skipped && !r.passed); - if (failed) { - throw new Error(`Preflight test "${failed.testName}" failed (plugin ${loaded.pluginName}): ${failed.error?.message ?? 'unknown error'}`); - } - } + // Main and suite files are loaded (imported once, registrations snapshotted) before any + // of them runs, so a misconfigured `concurrency` aborts here instead of after burning + // time on earlier tests. Preflight has already run by this point, so this no longer + // imports them ahead of the state preflight sets up. + validateConcurrency([...loadedMain, ...loadedSuite]); console.log(`${pc.bold(`Found ${loadedMain.length} test file(s)${testFileFilters ? ` matching filter: ${testFileFilters.join(',')}` : ''}`)}\n`); @@ -282,7 +288,6 @@ export async function runTestSession(config: RunnerConfig = loadRunnerConfig()): await runLoadedFile(loaded); } - } finally { await plugins.runCleanup(session, 'session'); await plugins.teardown(); From f7b7cd859b4ad812669c78ac00de0ff4b97ed199 Mon Sep 17 00:00:00 2001 From: Monikon Date: Thu, 3 Sep 2026 05:34:50 +0300 Subject: [PATCH 11/13] fix(test-runner): recognize a rejoined bot as the scope's own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix tracked "own" bots in ownBots, snapshotted once per connect() call. player.rejoin() swaps a player's .bot for a freshly created connection under the same username but never touches ownBots, so close() no longer recognized the new bot as its own and left it connected — a permanent "already playing" kick for every later test that leased the same account. close() now also reads connected.map(p => p.bot) fresh, which reflects whatever rejoin() last swapped in. --- runner-package/lib/test-runner.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/runner-package/lib/test-runner.ts b/runner-package/lib/test-runner.ts index 8011249..cb083fb 100644 --- a/runner-package/lib/test-runner.ts +++ b/runner-package/lib/test-runner.ts @@ -125,11 +125,12 @@ function createBotScope(session: Session, server: ServerWrapper, connOpts: BotCo players: () => [...connected], async close(): Promise { // Only this scope's own bots — a concurrent sibling instance's bots are still - // running and must not be torn down by this one finishing first. Uses `ownBots` - // (everything this scope ever created), not `connected`, so a bot whose join() - // failed and never made it into `connected` still gets torn down here instead of - // leaking an open connection forever. - const ownBotsSet = new Set(ownBots); + // running and must not be torn down by this one finishing first. Union of two + // sources: `ownBots` catches a bot whose join() failed and never made it into + // `connected`; reading `p.bot` fresh off `connected` catches the opposite case — + // `player.rejoin()` swaps a player's `.bot` for a new connection under the same + // scope, and that swapped-in bot isn't the one `ownBots` captured at connect time. + const ownBotsSet = new Set([...ownBots, ...connected.map(p => p.bot)]); await session.disconnectAllBots(session.bots.filter(b => !ownBotsSet.has(b))); for (const { account, pool } of leased) pool.release(account); leased.length = 0; From 9e423efad3f257e2de7c086e75206664ae0e98f0 Mon Sep 17 00:00:00 2001 From: Drownek Date: Thu, 3 Sep 2026 15:44:40 +0200 Subject: [PATCH 12/13] fix(authme): wait for authentication confirmation on session resume --- auth-authme-package/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/auth-authme-package/index.ts b/auth-authme-package/index.ts index c43618e..5a78e4f 100644 --- a/auth-authme-package/index.ts +++ b/auth-authme-package/index.ts @@ -121,7 +121,14 @@ export default definePlugin({ message: `authme: never saw a login/register prompt or session-resume message for "${account.username}"`, }, ); - if (promptResult === 'resumed') return; + if (promptResult === 'resumed') { + const authenticated = new RegExp(resolved.authenticatedPattern, 'i'); + await poll(() => since(joinIndex, authenticated), { + timeout: resolved.timeoutMs, + message: `authme: "${account.username}" session resumed, but never confirmed as authenticated`, + }); + return; + } const isRegistration = promptResult === 'register'; // Everything below only looks at messages newer than the command. A server's greeting From ce619d53c0094658c56f510578e741aa9a28b3a3 Mon Sep 17 00:00:00 2001 From: Drownek Date: Thu, 3 Sep 2026 16:28:18 +0200 Subject: [PATCH 13/13] test(example): format concurrency test and remove /say --- .../src/test/e2e/tests/concurrency.spec.ts | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/example_plugin/src/test/e2e/tests/concurrency.spec.ts b/example_plugin/src/test/e2e/tests/concurrency.spec.ts index df45bca..12776d0 100644 --- a/example_plugin/src/test/e2e/tests/concurrency.spec.ts +++ b/example_plugin/src/test/e2e/tests/concurrency.spec.ts @@ -12,16 +12,20 @@ import { describe, expect, test } from '@plugwright/runner'; -test('concurrent bots each see their own marker and stay connected', { concurrency: 3, requires: ['consoleOutput:full'] }, async ({ player, server }) => { - const marker = `concurrency-marker-${player.username}`; - player.chat(`/say ${marker}`); - await expect(server).toHaveReceivedMessage(marker, { timeout: 10000 }); +test( + 'concurrent bots each see their own marker and stay connected', + { concurrency: 3, requires: ['consoleOutput:full'] }, + async ({ player, server }) => { + const marker = `concurrency-marker-${player.username}`; + player.chat(marker); + await expect(server).toHaveReceivedMessage(marker, { timeout: 10000 }); - // Still connected: an earlier-finishing sibling instance's teardown must not have - // disconnected this one. - await player.teleport(50, 100, 50); - await expect(player).toBeNear(50, 100, 50, { tolerance: 2, timeout: 10000 }); -}); + // Still connected: an earlier-finishing sibling instance's teardown must not have + // disconnected this one. + await player.teleport(50, 100, 50); + await expect(player).toBeNear(50, 100, 50, { tolerance: 2, timeout: 10000 }); + } +); describe.serial('concurrent kit lifecycle', { concurrency: 2 }, () => { test('claims the starter kit', async ({ player }) => {