From 6fae92952df21305c8818ee635e6f187c334fff9 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:51:10 +1100 Subject: [PATCH 01/15] test: run the serial backend group beside the other suites The backend files that take Setup coordination ports are about two thirds of the portable Linux step and mostly wait. Run them in one lane next to a second lane holding both typechecks, the parallel backend group and the browser-model and automation tests, so the run takes as long as the longer lane. The serial group stays a single --test-concurrency=1 invocation that nothing joins. Add test:fast and test:all scripts that share the same plan, print each step's duration, and stop every lane at the first failure. --- CONTRIBUTING.md | 25 +++++++++++++++ package.json | 2 ++ scripts/ci-backend.ts | 10 ++++++ scripts/ci.ts | 16 +++++----- scripts/test-lanes.ts | 62 +++++++++++++++++++++++++++++++++++++ scripts/test-plan.ts | 42 +++++++++++++++++++++++++ scripts/test.ts | 23 ++++++++++++++ tests/automation/ci.test.ts | 53 +++++++++++++++++++++++++++++-- tsconfig.json | 2 +- 9 files changed, 223 insertions(+), 12 deletions(-) create mode 100644 scripts/test-lanes.ts create mode 100644 scripts/test-plan.ts create mode 100644 scripts/test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38a891f0..431ebc92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,6 +40,20 @@ Released-writer delegation fixtures require the local `v1.3.0` tag. Run skip with a prerequisite diagnostic when the tag is unavailable. Full portable CI requires full history and tags, so these cases run there. +Two scripts run the portable groups side by side after building every entry, and print each step's duration at the end: + +```sh +# Edit-and-check loop: both typechecks, the parallel backend group, browser-model +# and automation tests. It skips the serial Setup-port backend group (below), +# which holds most of the full run's time, so it is not what CI runs. +bun run test:fast + +# Everything portable CI tests, minus packaging and the rebuild comparison +bun run test:all +``` + +Both leave out the five native-only backend tests (see [Portable CI](#portable-ci--fresh-disposable-checkout-only)) and say so. Run `bun run test:all` or the commands below before a pull request when you touched backend, setup or service code; `test:fast` can pass where the serial group would fail. They are not replacements for `node --experimental-strip-types scripts/ci.ts` in a fresh checkout. + Run the checks relevant to your change: ```sh @@ -89,6 +103,17 @@ concurrent writer. Keep new fenced HTTP/worker test files in the list in This isolates tests, not product requests; no port override or relaxed refusal is used. +That serial group is the longest part of the run and mostly waits on deadlines +and worker start-up, and no other group acquires Setup ports. `scripts/ci.ts` +and `bun run test:all` therefore run it in one lane beside a second lane that +holds both typechecks, the parallel backend group and the browser-model and +automation tests (`scripts/test-plan.ts`, `scripts/test-lanes.ts`). The second +lane is capped at the machine's cores minus two so the timing assertions in the +serial group keep their headroom. Each step's output is printed whole under its +own heading, the first failure stops both lanes, and the run ends with a +per-step duration list. A file that takes a Setup port belongs in the serial +group; never add one to the other lane. + This is **not native GUI/OpenCode or host-installer certification**. The external host-installer gate is separate, as described below. Full `bun run check` retains the native backend prerequisites; it is not the portable CI command. The [workflow](.github/workflows/ci.yml) runs on pushes to `main` and `dev`, on every pull request whatever branch it targets, weekly on Tuesday at 04:17 UTC, and on manual dispatch. A PR stacked on another feature branch gets the same selection as one targeting `dev`. It retains the Linux portable checks on exactly Node 24.19.0, repeats them on the newest Node 24 release, and adds a targeted Windows native-filesystem suite. A newer push to a pull request cancels that pull request's earlier run. Versioned publication also waits for the full Windows job on its `main` push. Neither job substitutes for actual OpenCode/OpenChamber integration acceptance. diff --git a/package.json b/package.json index 56f46f99..59223562 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,8 @@ "scripts": { "typecheck": "tsc --noEmit", "test:backend": "bun run build:all && node --experimental-strip-types --test tests/backend/*.test.ts", + "test:all": "bun run build:all && node --experimental-strip-types scripts/test.ts", + "test:fast": "bun run build:all && node --experimental-strip-types scripts/test.ts --fast", "build:backend": "bun scripts/build.ts", "build:all": "node --experimental-strip-types scripts/build-all.ts", "build": "bun run build:all", diff --git a/scripts/ci-backend.ts b/scripts/ci-backend.ts index f3d52ff5..eda1d209 100644 --- a/scripts/ci-backend.ts +++ b/scripts/ci-backend.ts @@ -16,6 +16,16 @@ const fenceTests = new Set([ 'windows-service.test.ts', ]); +/** Backend files that need an independent OpenCode binary and Linux isolation + * tools. Portable CI and the local runners leave them out and say so. */ +export const nativeOnlyTests = [ + 'production-runtime.test.ts', + 'plugin-failure-runtime.test.ts', + 'progress-activation-runtime.test.ts', + 'subagent-activity-runtime.test.ts', + 'subagent-upgrade-runtime.test.ts', +]; + export function backendTestCommands(files: string[]): string[][] { const fenced = (file: string) => fenceTests.has(file.split('/').at(-1)!); return [ diff --git a/scripts/ci.ts b/scripts/ci.ts index 128c41c0..36045d1a 100644 --- a/scripts/ci.ts +++ b/scripts/ci.ts @@ -4,7 +4,9 @@ import { readFileSync, readdirSync, mkdirSync, existsSync, copyFileSync, writeFi import { createHash } from 'node:crypto'; import { eligibility, assetNames, validateReleaseNotes } from './release.ts'; import { verifyCompanionEntry } from './companion-entry.ts'; -import { backendTestCommands } from './ci-backend.ts'; +import { nativeOnlyTests } from './ci-backend.ts'; +import { runLanes } from './test-lanes.ts'; +import { portableLanes } from './test-plan.ts'; import { buildAll, outputHashes, compareOutputs } from './build-all.ts'; import { revisionKind, trackedOutputs } from './revision-kind.ts'; @@ -14,7 +16,6 @@ if (existsSync('build/artifacts') && readdirSync('build/artifacts').length > 0) process.exit(1); } -const excluded = ['production-runtime.test.ts', 'plugin-failure-runtime.test.ts', 'progress-activation-runtime.test.ts', 'subagent-activity-runtime.test.ts', 'subagent-upgrade-runtime.test.ts']; function run(command:string, args:string[]) { if (spawnSync(command,args,{stdio:'inherit'}).status !== 0) throw new Error('Portable check failed'); } @@ -33,12 +34,11 @@ if (kind === 'source') buildAll(); const before = outputHashes(); verifyCompanionEntry(process.cwd()); mkdirSync('/tmp/opencode',{recursive:true}); -console.log(`Native-only tests NOT RUN: ${excluded.join(', ')}`); -run('bun',['run','typecheck']); -run('./node_modules/.bin/tsc',['-p','ui/tsconfig.json','--noEmit']); -const tests = (dir:string) => readdirSync(dir).filter(f=>f.endsWith('.test.ts')).sort().map(f=>`${dir}/${f}`); -for (const args of backendTestCommands(tests('tests/backend').filter(f=>!excluded.includes(f.split('/').at(-1)!)))) run('node',args); -run('node',['--experimental-strip-types','--test',...tests('tests/browser'),...tests('tests/automation')]); +console.log(`Native-only tests NOT RUN: ${nativeOnlyTests.join(', ')}`); +// Root and UI typechecks, the parallel backend group, and browser-model and +// automation tests run beside the serial backend group that binds Setup ports. +// Output is grouped per step and the first failure stops every lane. +await runLanes(portableLanes({ fenced: true })); buildAll(); compareOutputs(before); // The SDK's own readiness check: manifest schema, page scripts and service entry. diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts new file mode 100644 index 00000000..54b159b3 --- /dev/null +++ b/scripts/test-lanes.ts @@ -0,0 +1,62 @@ +/** Runs independent groups of checks side by side. A lane is an ordered list of + * steps that stop at the first failure; lanes share nothing but the checkout. + * Output is captured per step and written whole when the step ends, so lines + * from different lanes never interleave. The first failure stops every lane. */ +import { spawn } from 'node:child_process'; + +export type Step = { label: string; command: string; args: string[] }; +export type Lane = { name: string; steps: Step[] }; +export type StepResult = { lane: string; label: string; seconds: number; status: 'passed' | 'failed' | 'stopped' }; + +const github = process.env.GITHUB_ACTIONS === 'true'; +const stamp = (seconds: number) => `${seconds.toFixed(1)}s`; + +function print(label: string, text: string, ok: boolean) { + // One write per block keeps another lane's output out of the middle of it. + const body = text.endsWith('\n') || text === '' ? text : `${text}\n`; + process.stdout.write(github ? `::group::${label}\n${body}::endgroup::\n` : `\n=== ${label} ${ok ? '' : '(failed) '}===\n${body}`); +} + +export async function runLanes(lanes: Lane[]): Promise { + const results: StepResult[] = []; + const children = new Set(); + let failed = false; + const stopAll = () => { + for (const pid of children) { try { process.kill(-pid, 'SIGTERM'); } catch { /* already gone */ } } + // A step that ignores SIGTERM must not outlive the run. + setTimeout(() => { for (const pid of children) { try { process.kill(-pid, 'SIGKILL'); } catch { /* already gone */ } } }, 10_000).unref(); + }; + const runStep = (lane: Lane, step: Step) => new Promise(resolve => { + const started = performance.now(); + const label = `${lane.name}: ${step.label}`; + console.log(`start ${label}`); + const chunks: Buffer[] = []; + // Own process group, so stopping a step also stops the test files it started. + const child = spawn(step.command, step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true }); + if (child.pid) children.add(child.pid); + child.stdout.on('data', chunk => chunks.push(chunk)); + child.stderr.on('data', chunk => chunks.push(chunk)); + const finish = (ok: boolean, note = '') => { + if (child.pid) children.delete(child.pid); + const seconds = (performance.now() - started) / 1000; + const status = ok ? 'passed' : failed ? 'stopped' : 'failed'; + if (!ok && !failed) { failed = true; stopAll(); } + if (status !== 'stopped') print(label, Buffer.concat(chunks).toString('utf8') + note, ok); + console.log(`${ok ? 'passed' : status} ${label} (${stamp(seconds)})`); + results.push({ lane: lane.name, label: step.label, seconds, status }); + resolve(ok); + }; + child.on('error', error => finish(false, `Could not start: ${error.message}\n`)); + child.on('close', (code, signal) => finish(code === 0, code === 0 ? '' : `Exited with ${signal ?? `code ${code}`}\n`)); + }); + await Promise.all(lanes.map(async lane => { + for (const step of lane.steps) { + if (failed) { results.push({ lane: lane.name, label: step.label, seconds: 0, status: 'stopped' }); continue; } + if (!await runStep(lane, step)) break; + } + })); + console.log('\nStep timings'); + for (const r of results) console.log(` ${r.status.padEnd(7)} ${stamp(r.seconds).padStart(7)} ${r.lane}: ${r.label}`); + if (failed) throw new Error('Portable check failed'); + return results; +} diff --git a/scripts/test-plan.ts b/scripts/test-plan.ts new file mode 100644 index 00000000..7406bd41 --- /dev/null +++ b/scripts/test-plan.ts @@ -0,0 +1,42 @@ +/** The portable check groups, shared by CI (`scripts/ci.ts`) and the local + * runner (`scripts/test.ts`) so they cannot drift apart. Nothing here weakens a + * check: it only decides which groups may run side by side. + * + * The backend files that acquire target-derived Setup coordination ports stay in + * one serial `--test-concurrency=1` invocation (see `scripts/ci-backend.ts`). They + * spend most of their time waiting on deadlines and HTTP replies, and no other + * group acquires those ports, so the other groups run beside them rather than + * before them. */ +import { readdirSync } from 'node:fs'; +import { availableParallelism } from 'node:os'; +import { backendTestCommands, nativeOnlyTests } from './ci-backend.ts'; +import type { Lane } from './test-lanes.ts'; + +export const testFiles = (dir: string) => readdirSync(dir).filter(f => f.endsWith('.test.ts')).sort().map(f => `${dir}/${f}`); + +export function portableLanes(options: { fenced: boolean }): Lane[] { + const backend = testFiles('tests/backend').filter(f => !nativeOnlyTests.includes(f.split('/').at(-1)!)); + const [parallel, serial] = backendTestCommands(backend); + // Leave two cores for the serial lane and the type checker; the lanes are + // short of waiting time, not of cores, and a starved timing test is a false failure. + const workers = `--test-concurrency=${Math.max(1, availableParallelism() - 2)}`; + const withWorkers = (args: string[]) => [...args.slice(0, 2), workers, ...args.slice(2)]; + const node = (label: string, args: string[]) => ({ label, command: 'node', args }); + const types = [ + { label: 'root typecheck', command: 'bun', args: ['run', 'typecheck'] }, + { label: 'UI typecheck', command: './node_modules/.bin/tsc', args: ['-p', 'ui/tsconfig.json', '--noEmit'] }, + ]; + const rest = [ + node('backend tests (parallel group)', withWorkers(parallel)), + node('browser-model and automation tests', withWorkers(['--experimental-strip-types', '--test', ...testFiles('tests/browser'), ...testFiles('tests/automation')])), + ]; + return options.fenced + ? [ + { name: 'port-bound backend', steps: [node('backend tests (serial group, Setup ports)', serial)] }, + { name: 'everything else', steps: [...types, ...rest] }, + ] + : [ + { name: 'types', steps: types }, + { name: 'tests', steps: rest }, + ]; +} diff --git a/scripts/test.ts b/scripts/test.ts new file mode 100644 index 00000000..3199f713 --- /dev/null +++ b/scripts/test.ts @@ -0,0 +1,23 @@ +/** Local portable test runner. Build the generated entries first: the package + * scripts `test:all` and `test:fast` do. It never packages or starts a native + * runtime, and it does not replace `scripts/ci.ts` for a pull request. + * + * node --experimental-strip-types scripts/test.ts every portable group + * node --experimental-strip-types scripts/test.ts --fast the same without the serial Setup-port group */ +import { mkdirSync } from 'node:fs'; +import { nativeOnlyTests } from './ci-backend.ts'; +import { runLanes } from './test-lanes.ts'; +import { portableLanes } from './test-plan.ts'; + +const fast = process.argv.includes('--fast'); +mkdirSync('/tmp/opencode', { recursive: true }); +console.log(`Native-only tests NOT RUN: ${nativeOnlyTests.join(', ')}`); +if (fast) console.log('Fast loop: the serial Setup-port backend group is skipped. Run `bun run test:all` or CI before a pull request.'); +const started = performance.now(); +try { + await runLanes(portableLanes({ fenced: !fast })); + console.log(`\nAll portable groups passed in ${((performance.now() - started) / 1000).toFixed(1)}s${fast ? ' (fast loop, not the full set)' : ''}.`); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} diff --git a/tests/automation/ci.test.ts b/tests/automation/ci.test.ts index ad0266cd..5493f9ff 100644 --- a/tests/automation/ci.test.ts +++ b/tests/automation/ci.test.ts @@ -3,13 +3,15 @@ import assert from 'node:assert/strict'; import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { spawnSync } from 'node:child_process'; -import { backendTestCommands } from '../../scripts/ci-backend.ts'; +import { backendTestCommands, nativeOnlyTests } from '../../scripts/ci-backend.ts'; +import { portableLanes } from '../../scripts/test-plan.ts'; import { outputs } from '../../scripts/build-all.ts'; test('package scripts that consume runtime entries build before invoking their readers',()=>{ const {scripts}=JSON.parse(readFileSync('package.json','utf8')); - for(const name of ['test:backend','package','verify:package','prepare:probe','setup','check']) + for(const name of ['test:backend','test:all','test:fast','package','verify:package','prepare:probe','setup','check']) assert.equal(scripts[name].split('&&')[0].trim(),'bun run build:all',`${name} must build before its readers`); }); @@ -55,7 +57,52 @@ test('portable backend groups isolate coordination ports without omitting or rep 'setup-suggestions.test.ts','windows-service.test.ts', ]); const source=readFileSync('scripts/ci.ts','utf8'); - assert.match(source,/for \(const args of backendTestCommands\([^\n]+\)\) run\('node',args\)/); + assert.match(source,/await runLanes\(portableLanes\(\{ fenced: true \}\)\)/); +}); + +test('portable lanes run every portable file once, keep the port-bound group serial and alone, and the fast loop drops only that group',()=>{ + const names=(dir:string)=>readdirSync(dir).filter(f=>f.endsWith('.test.ts')).map(f=>`${dir}/${f}`).sort(); + const native=new Set(nativeOnlyTests); + const backend=names('tests/backend').filter(f=>!native.has(f.split('/').at(-1)!)); + const every=(lanes:ReturnType)=>lanes.flatMap(l=>l.steps).filter(s=>s.command==='node'); + const files=(steps:{args:string[]}[])=>steps.flatMap(s=>s.args.filter(a=>a.endsWith('.test.ts'))).sort(); + const full=portableLanes({fenced:true}),fast=portableLanes({fenced:false}); + assert.deepEqual(files(every(full)),[...backend,...names('tests/browser'),...names('tests/automation')].sort()); + assert.equal(new Set(files(every(full))).size,files(every(full)).length,'no file runs twice'); + const serial=every(full).filter(s=>s.args.includes('--test-concurrency=1')); + assert.equal(serial.length,1); + assert.deepEqual(files(serial).map(f=>f.split('/').at(-1)),backendTestCommands(backend)[1].slice(3).map(f=>f.split('/').at(-1)).sort()); + const alone=full.filter(l=>l.steps.includes(serial[0])); + assert.equal(alone.length,1);assert.equal(alone[0].steps.length,1,'the port-bound group shares its lane with nothing that could take a port'); + // Every other test step is bounded by an explicit worker count rather than the host default. + for(const step of every(full).filter(s=>s!==serial[0]))assert.ok(step.args.some(a=>/^--test-concurrency=[1-9]\d*$/.test(a)),step.label); + assert.deepEqual(files(every(fast)),files(every(full)).filter(f=>!files(serial).includes(f))); + for(const lanes of [full,fast]){ + const labels=lanes.flatMap(l=>l.steps).map(s=>s.label); + assert.ok(labels.includes('root typecheck')&&labels.includes('UI typecheck'),'both typechecks stay in every run'); + } +}); + +test('lane runner stops every lane at the first failure and reports it without interleaving',async()=>{ + const root=mkdtempSync(join(tmpdir(),'portable-lanes-')); + try { + const script=join(root,'lanes.ts'); + writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; +const node=(label:string,code:string)=>({label,command:process.execPath,args:['-e',code]}); +try{await runLanes([ + {name:'slow',steps:[node('hangs',"setTimeout(()=>{},60000)"),node('never starts',"console.log('LATE-STEP')")]}, + {name:'fails',steps:[node('breaks',"console.log('FAIL-OUTPUT');process.exit(3)")]}, + {name:'ok',steps:[node('passes',"console.log('PASS-OUTPUT')"),node('skipped after failure',"setTimeout(()=>console.log('LATE-STEP'),5000)")]}, +]);}catch(error){console.log('REJECTED '+(error as Error).message);process.exit(1);}`); + const started=performance.now(); + const r=spawnSync(process.execPath,['--experimental-strip-types',script],{encoding:'utf8',timeout:20_000}); + assert.equal(r.status,1); + assert.ok(performance.now()-started<15_000,'the hanging lane is stopped, not waited for'); + assert.match(r.stdout,/FAIL-OUTPUT/);assert.match(r.stdout,/Exited with code 3/);assert.match(r.stdout,/REJECTED Portable check failed/); + assert.doesNotMatch(r.stdout,/LATE-STEP/); + assert.match(r.stdout,/stopped\s+[\d.]+s\s+slow: hangs/); + assert.match(r.stdout,/failed\s+[\d.]+s\s+fails: breaks/); + } finally {rmSync(root,{recursive:true,force:true});} }); test('actual portable CLI refuses nonfresh artifacts before entry reads or checks, preserving archives',()=>{ diff --git a/tsconfig.json b/tsconfig.json index a1645eef..8e75031b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,5 +10,5 @@ "types": ["node"], "lib": ["ES2023", "DOM"] }, - "include": ["shared/**/*.ts", "plugin/**/*.ts", "service/**/*.ts", "setup/**/*.ts", "tests/backend/**/*.ts", "tests/fixtures/**/*.ts", "scripts/cli-probe.ts", "scripts/backend-evidence.ts", "scripts/package*.ts", "scripts/verify*package.ts", "scripts/ci.ts", "scripts/release.ts", "tests/automation/**/*.ts"] + "include": ["shared/**/*.ts", "plugin/**/*.ts", "service/**/*.ts", "setup/**/*.ts", "tests/backend/**/*.ts", "tests/fixtures/**/*.ts", "scripts/cli-probe.ts", "scripts/backend-evidence.ts", "scripts/package*.ts", "scripts/verify*package.ts", "scripts/ci.ts", "scripts/test*.ts", "scripts/release.ts", "tests/automation/**/*.ts"] } From fed0c86781d3d27f38a46137f99b4895995c4952 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:51:10 +1100 Subject: [PATCH 02/15] ci: install the ZIP tools only when the runner lacks them --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71ed3ea7..c9b230e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,8 @@ jobs: with: bun-version: 1.3.14 - name: Install ZIP tools - run: sudo apt-get update && sudo apt-get install -y zip unzip + # Hosted images already carry both tools; install only if one is missing. + run: command -v zip >/dev/null && command -v unzip >/dev/null || (sudo apt-get update && sudo apt-get install -y zip unzip) - name: Install project dependencies run: bun install --frozen-lockfile --ignore-scripts - name: Run Linux tests, builds and package checks @@ -172,7 +173,8 @@ jobs: with: bun-version: 1.3.14 - name: Install ZIP tools - run: sudo apt-get update && sudo apt-get install -y zip unzip + # Hosted images already carry both tools; install only if one is missing. + run: command -v zip >/dev/null && command -v unzip >/dev/null || (sudo apt-get update && sudo apt-get install -y zip unzip) - name: Install project dependencies run: bun install --frozen-lockfile --ignore-scripts - name: Run Linux tests, builds and package checks From 68b055b7b8938380bd09587efd1aa350abc5618f Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:51:10 +1100 Subject: [PATCH 03/15] docs(openspec): record the faster test suites work in the 1.5.2 tasks --- openspec/changes/release-1-5-2/tasks.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index 6257201b..457158e2 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -64,6 +64,14 @@ Each numbered group is one pull request into the development branch. It carries - [ ] 7.5 In the same pull request, reconcile the task lists of the 1.5.0 change, the 1.5.1 change and this one: tick each task whose work shipped, and leave a task that did not happen unticked with a note saying whether it is deferred, superseded or out of scope. - [ ] 7.6 In the same pull request, archive the 1.5.0 change, then the 1.5.1 change, then this one, in that order, because each later change modifies capabilities the earlier ones introduced. Never use `--skip-specs` or `--no-validate`. A rehearsal showed what this pull request must correct in the 1.5.1 change's deltas before archiving that change. First, restore the two scenario names that the 1.5.0 specs hold and two of its modified requirements dropped, either by putting the old name on the new content or by keeping the old scenario beside the renamed one. Second, shorten or split the four requirement texts that would exceed 500 characters once in the specs, keeping their meaning and their scenarios. That includes the result requirement at 628 characters, although this change replaces it in the third archive: strict validation runs after the second archive too. After each archive, write the purpose of every capability it created, ten in all, and validate the specs strictly. That validation reports nothing after each of the three archives. If an archive refuses any other delta, correct the delta. +## 9. Faster test suites + +- [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. +- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure, and keep the second build comparison and the package checks after all tests. Show with a probe on the acquisition call that the parallel backend group and the browser and automation tests take no Setup port. +- [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. +- [x] 9.4 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan, the second build and the Windows coverage unchanged. +- [x] 9.5 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. + ## Deferred acceptance and out-of-scope follow-up - [ ] Promotion to the distribution branch and its host gates, as the contributor guide describes. It follows the release pull request, and so the archive of this change. The nine generated outputs are built and committed by the promotion only. From de6b50c00ef1e4de94f90ec9e96aa1378a49a2cc Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:02:07 +1100 Subject: [PATCH 04/15] test: run the lane beside the serial backend group at lower priority --- CONTRIBUTING.md | 4 ++-- scripts/test-lanes.ts | 7 +++++-- scripts/test-plan.ts | 4 +++- tests/automation/ci.test.ts | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 431ebc92..be993857 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,8 +108,8 @@ and worker start-up, and no other group acquires Setup ports. `scripts/ci.ts` and `bun run test:all` therefore run it in one lane beside a second lane that holds both typechecks, the parallel backend group and the browser-model and automation tests (`scripts/test-plan.ts`, `scripts/test-lanes.ts`). The second -lane is capped at the machine's cores minus two so the timing assertions in the -serial group keep their headroom. Each step's output is printed whole under its +lane is capped at the machine's cores minus two and runs under `nice`, so the +timing assertions in the serial group keep their headroom. Each step's output is printed whole under its own heading, the first failure stops both lanes, and the run ends with a per-step duration list. A file that takes a Setup port belongs in the serial group; never add one to the other lane. diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts index 54b159b3..7f31c912 100644 --- a/scripts/test-lanes.ts +++ b/scripts/test-lanes.ts @@ -4,7 +4,9 @@ * from different lanes never interleave. The first failure stops every lane. */ import { spawn } from 'node:child_process'; -export type Step = { label: string; command: string; args: string[] }; +/** `lowPriority` runs the step under `nice`, so a lane that must keep its timing + * headroom wins the cores when the machine is short of them. */ +export type Step = { label: string; command: string; args: string[]; lowPriority?: boolean }; export type Lane = { name: string; steps: Step[] }; export type StepResult = { lane: string; label: string; seconds: number; status: 'passed' | 'failed' | 'stopped' }; @@ -32,7 +34,8 @@ export async function runLanes(lanes: Lane[]): Promise { console.log(`start ${label}`); const chunks: Buffer[] = []; // Own process group, so stopping a step also stops the test files it started. - const child = spawn(step.command, step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true }); + const niced = step.lowPriority && process.platform !== 'win32'; + const child = spawn(niced ? 'nice' : step.command, niced ? ['-n', '10', step.command, ...step.args] : step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true }); if (child.pid) children.add(child.pid); child.stdout.on('data', chunk => chunks.push(chunk)); child.stderr.on('data', chunk => chunks.push(chunk)); diff --git a/scripts/test-plan.ts b/scripts/test-plan.ts index 7406bd41..e84641e1 100644 --- a/scripts/test-plan.ts +++ b/scripts/test-plan.ts @@ -30,10 +30,12 @@ export function portableLanes(options: { fenced: boolean }): Lane[] { node('backend tests (parallel group)', withWorkers(parallel)), node('browser-model and automation tests', withWorkers(['--experimental-strip-types', '--test', ...testFiles('tests/browser'), ...testFiles('tests/automation')])), ]; + // Beside the serial group, the other lane yields cores to it. + const yielding = [...types, ...rest].map(step => ({ ...step, lowPriority: true })); return options.fenced ? [ { name: 'port-bound backend', steps: [node('backend tests (serial group, Setup ports)', serial)] }, - { name: 'everything else', steps: [...types, ...rest] }, + { name: 'everything else', steps: yielding }, ] : [ { name: 'types', steps: types }, diff --git a/tests/automation/ci.test.ts b/tests/automation/ci.test.ts index 5493f9ff..6651b194 100644 --- a/tests/automation/ci.test.ts +++ b/tests/automation/ci.test.ts @@ -74,6 +74,8 @@ test('portable lanes run every portable file once, keep the port-bound group ser assert.deepEqual(files(serial).map(f=>f.split('/').at(-1)),backendTestCommands(backend)[1].slice(3).map(f=>f.split('/').at(-1)).sort()); const alone=full.filter(l=>l.steps.includes(serial[0])); assert.equal(alone.length,1);assert.equal(alone[0].steps.length,1,'the port-bound group shares its lane with nothing that could take a port'); + assert.ok(!serial[0].lowPriority&&full.find(l=>l.steps.includes(serial[0]))!.steps.length===1,'the serial group never yields cores'); + assert.ok(full.flatMap(l=>l.steps).filter(s=>s!==serial[0]).every(s=>s.lowPriority),'the lane beside it yields cores to it'); // Every other test step is bounded by an explicit worker count rather than the host default. for(const step of every(full).filter(s=>s!==serial[0]))assert.ok(step.args.some(a=>/^--test-concurrency=[1-9]\d*$/.test(a)),step.label); assert.deepEqual(files(every(fast)),files(every(full)).filter(f=>!files(serial).includes(f))); @@ -83,6 +85,20 @@ test('portable lanes run every portable file once, keep the port-bound group ser } }); +test('lane runner starts a yielding step at lower priority and an ordinary one at the usual priority',async()=>{ + const root=mkdtempSync(join(tmpdir(),'portable-lanes-priority-')); + try { + const script=join(root,'lanes.ts'); + writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; +const show=(label:string,lowPriority?:boolean)=>({label,command:process.execPath,args:['-e',"console.log('PRIORITY '+require('node:os').getPriority())"],lowPriority}); +await runLanes([{name:'a',steps:[show('usual')]},{name:'b',steps:[show('yielding',true)]}]);`); + const r=spawnSync(process.execPath,['--experimental-strip-types',script],{encoding:'utf8',timeout:20_000}); + assert.equal(r.status,0,r.stderr); + const seen=(label:string)=>Number(new RegExp(`=== ${label} ===\\nPRIORITY (-?\\d+)`).exec(r.stdout)?.[1]); + assert.ok(seen('b: yielding')>seen('a: usual'),'the yielding step has the larger niceness'); + } finally {rmSync(root,{recursive:true,force:true});} +}); + test('lane runner stops every lane at the first failure and reports it without interleaving',async()=>{ const root=mkdtempSync(join(tmpdir(),'portable-lanes-')); try { From d4ea6b08d09ea6d595c08332e4d783a96b372529 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:09:09 +1100 Subject: [PATCH 05/15] test: keep the priority check independent of the Actions log format --- tests/automation/ci.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/automation/ci.test.ts b/tests/automation/ci.test.ts index 6651b194..3216a284 100644 --- a/tests/automation/ci.test.ts +++ b/tests/automation/ci.test.ts @@ -92,7 +92,8 @@ test('lane runner starts a yielding step at lower priority and an ordinary one a writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; const show=(label:string,lowPriority?:boolean)=>({label,command:process.execPath,args:['-e',"console.log('PRIORITY '+require('node:os').getPriority())"],lowPriority}); await runLanes([{name:'a',steps:[show('usual')]},{name:'b',steps:[show('yielding',true)]}]);`); - const r=spawnSync(process.execPath,['--experimental-strip-types',script],{encoding:'utf8',timeout:20_000}); + // Plain headings: the runner writes `::group::` blocks when it sees GitHub Actions. + const r=spawnSync(process.execPath,['--experimental-strip-types',script],{encoding:'utf8',timeout:20_000,env:{...process.env,GITHUB_ACTIONS:''}}); assert.equal(r.status,0,r.stderr); const seen=(label:string)=>Number(new RegExp(`=== ${label} ===\\nPRIORITY (-?\\d+)`).exec(r.stdout)?.[1]); assert.ok(seen('b: yielding')>seen('a: usual'),'the yielding step has the larger niceness'); From 6bc4230d66a6fbc08b506808dd55dc97a5f4b81a Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:09:09 +1100 Subject: [PATCH 06/15] ci: run the two native Windows test files side by side --- .github/workflows/ci.yml | 45 +++++++++++++++------- tests/automation/release.test.ts | 4 +- tests/automation/workflow-triggers.test.ts | 6 ++- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9b230e5..fe7bf68a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,9 +218,19 @@ jobs: # ownership semantics without relaxing the product's SID policy. $name = 'progress' + [Guid]::NewGuid().ToString('N').Substring(0, 10) $password = ConvertTo-SecureString ('Aa1!' + [Guid]::NewGuid().ToString('N')) -AsPlainText -Force - $output = Join-Path $env:RUNNER_TEMP ($name + '.stdout') - $errors = Join-Path $env:RUNNER_TEMP ($name + '.stderr') $trace = Join-Path $env:RUNNER_TEMP 'windows-migration-trace.jsonl' + # The two native files run side by side, each in its own serial test + # runner. Only the service file starts a server and request workers, + # which are what take Setup coordination ports; the platform file + # drives the setup controller directly. + $runs = @('windows-platform', 'windows-service') | ForEach-Object { + [pscustomobject]@{ + File = $_ + Output = Join-Path $env:RUNNER_TEMP ($name + '.' + $_ + '.stdout') + Errors = Join-Path $env:RUNNER_TEMP ($name + '.' + $_ + '.stderr') + Process = $null + } + } # Leave a fixed diagnostic even if principal/process creation fails. '{"request":"runner-start","completed":false}' | Set-Content -LiteralPath $trace -Encoding utf8 $created = $false @@ -229,23 +239,32 @@ jobs: $created = $true Add-LocalGroupMember -SID 'S-1-5-32-545' -Member $user $credential = [PSCredential]::new(($env:COMPUTERNAME + '\' + $name), $password) - $process = Start-Process -FilePath (Get-Command node).Source -ArgumentList '--experimental-strip-types --test --test-concurrency=1 tests/backend/windows-platform.test.ts tests/backend/windows-service.test.ts' -WorkingDirectory $env:GITHUB_WORKSPACE -Credential $credential -LoadUserProfile -WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput $output -RedirectStandardError $errors - if (Test-Path $output) { - Get-Content -LiteralPath $output + foreach ($run in $runs) { + $run.Process = Start-Process -FilePath (Get-Command node).Source -ArgumentList "--experimental-strip-types --test --test-concurrency=1 tests/backend/$($run.File).test.ts" -WorkingDirectory $env:GITHUB_WORKSPACE -Credential $credential -LoadUserProfile -WindowStyle Hidden -PassThru -RedirectStandardOutput $run.Output -RedirectStandardError $run.Errors + # Reading the handle keeps the exit code available after the wait. + $null = $run.Process.Handle + } + foreach ($run in $runs) { $run.Process.WaitForExit() } + foreach ($run in $runs) { + if (Test-Path $run.Output) { + Get-Content -LiteralPath $run.Output + } + if (Test-Path $run.Errors) { Get-Content -LiteralPath $run.Errors } } - if (Test-Path $errors) { Get-Content -LiteralPath $errors } - if ($process.ExitCode -ne 0) { throw 'Native Windows tests failed.' } + if (@($runs | Where-Object { $_.Process.ExitCode -ne 0 }).Count -gt 0) { throw 'Native Windows tests failed.' } } finally { # Preserve per-request evidence even on an assertion/process failure. # Never upload raw stdout/stderr, local principal names or paths. - if (Test-Path $output) { - $traces = @(Get-Content -LiteralPath $output | ForEach-Object { - if ($_ -match '^\s*(?:#\s*)?WINDOWS_MIGRATION_TRACE (.+)$') { $Matches[1] } - }) - if ($traces.Count -gt 0) { $traces | Add-Content -LiteralPath $trace -Encoding utf8 } + foreach ($run in $runs) { + if (Test-Path $run.Output) { + $traces = @(Get-Content -LiteralPath $run.Output | ForEach-Object { + if ($_ -match '^\s*(?:#\s*)?WINDOWS_MIGRATION_TRACE (.+)$') { $Matches[1] } + }) + if ($traces.Count -gt 0) { $traces | Add-Content -LiteralPath $trace -Encoding utf8 } + } } if ($created) { Remove-LocalUser -Name $name } - Remove-Item -LiteralPath $output, $errors -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath @($runs | ForEach-Object { $_.Output; $_.Errors }) -Force -ErrorAction SilentlyContinue } $requests = @(Get-Content -LiteralPath $trace | ForEach-Object { $_ | ConvertFrom-Json } | Where-Object { $_.request -and $_.request -notin @('runner-start','summary') }) if ($requests.Count -eq 0) { throw 'Native Windows request trace is missing.' } diff --git a/tests/automation/release.test.ts b/tests/automation/release.test.ts index 3b9d706a..4a1223a0 100644 --- a/tests/automation/release.test.ts +++ b/tests/automation/release.test.ts @@ -314,7 +314,9 @@ test('workflow keeps PR read-only, exact same-run artifact join and version seri assert(!text.includes('pull_request_target'));assert(!text.includes('secrets.')); assert.match(text,/permissions:\n contents: read/);assert.match(text,/publish:\n(?: name: [^\n]+\n)? needs: \[portable, windows-native\]/); assert.match(text,/windows-native:\n(?: name: [^\n]+\n)? runs-on: windows-2025/); - assert.match(text,/--experimental-strip-types --test --test-concurrency=1 tests\/backend\/windows-platform\.test\.ts tests\/backend\/windows-service\.test\.ts/); + // Both native files still run, each in its own serial runner, as the same unprivileged principal. + assert.match(text,/\$runs = @\('windows-platform', 'windows-service'\)/); + assert.match(text,/--experimental-strip-types --test --test-concurrency=1 tests\/backend\/\$\(\$run\.File\)\.test\.ts/); assert.match(text,/git fetch --no-tags origin tag v1\.1\.1/); assert.match(text,/name: Upload sanitized Windows migration trace\n if: always\(\)/); assert.match(text,/name: windows-migration-trace\n path: \$\{\{ runner.temp \}\}\/windows-migration-trace\.jsonl/); diff --git a/tests/automation/workflow-triggers.test.ts b/tests/automation/workflow-triggers.test.ts index ca4fd916..3b8d1b39 100644 --- a/tests/automation/workflow-triggers.test.ts +++ b/tests/automation/workflow-triggers.test.ts @@ -197,7 +197,11 @@ test('Windows selects fast PR/push checks and full release, weekly and manual jo const windows = source.split('\n windows-native:\n')[1].split('\n publish:\n')[0]; assert.ok(windows.includes("AP_WINDOWS_FULL: ${{ needs.gate.outputs.windows_full == 'true' && '1' || '0' }}")); assert.match(windows, /name: Fetch released Windows migration fixture\n if: env.AP_WINDOWS_FULL == '1'\n run: git fetch --no-tags origin tag v1\.1\.1/); - assert.match(windows, /--test-concurrency=1 tests\/backend\/windows-platform\.test\.ts tests\/backend\/windows-service\.test\.ts/); + assert.match(windows, /\$runs = @\('windows-platform', 'windows-service'\)/); + assert.match(windows, /--test-concurrency=1 tests\/backend\/\$\(\$run\.File\)\.test\.ts/); + assert.ok(!windows.includes(' -Wait '), 'both native files are waited for before either result is read'); + assert.match(windows, /foreach \(\$run in \$runs\) \{ \$run\.Process\.WaitForExit\(\) \}/); + assert.match(windows, /Where-Object \{ \$_\.Process\.ExitCode -ne 0 \}/); const root = mkdtempSync(join(tmpdir(), 'workflow-windows-')); const git = (...args: string[]) => { From fa38feb57c1b17d3997ab87eb8db37ef1096ed99 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:24:45 +1100 Subject: [PATCH 07/15] docs: describe the side-by-side Windows test files --- CONTRIBUTING.md | 6 +++++- docs/compatibility.md | 2 +- openspec/changes/release-1-5-2/tasks.md | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index be993857..bcfe11a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,7 +139,11 @@ dispatch run the full set, even for UI/docs-only changes. The workflow sets `AP_WINDOWS_FULL=1` only for those full runs and fetches `v1.1.1` only then. Without that opt-in, the three journeys explicitly skip; no per-request assertion, budget or deadline is relaxed. Local full native runs must also set -`AP_WINDOWS_FULL=1`. Weekly and manual CI runs target Windows only; the Linux +`AP_WINDOWS_FULL=1`. The Windows job starts the platform file and the service file +as two processes at once, each in its own serial test runner and as the same +unprivileged principal, and waits for both before reading either exit code; only the +service file starts a server and request workers, which take the Setup coordination +ports. Weekly and manual CI runs target Windows only; the Linux jobs retain their PR/push coverage. See [native Windows verification](docs/compatibility.md#native-windows-verification). Only `main` pushes can publish releases. A `dev` push runs portable checks but is never release-eligible; its publish job may appear as skipped. Accept the source on `dev` first, then prepare a promotion commit containing that source and its built outputs. Run the required host gates on the promotion checkout before its PR into `main` merges. diff --git a/docs/compatibility.md b/docs/compatibility.md index cf86bc51..5742c470 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -493,7 +493,7 @@ permits cleanup/permission repair. See The `Windows setup, storage and capture` CI job runs the native platform suite and `tests/backend/windows-service.test.ts` in the same provisioned non-admin -account. Ordinary PRs run it when backend/build/test inputs change (unknown +account, as two processes at once, each in its own serial test runner. Ordinary PRs run it when backend/build/test inputs change (unknown paths conservatively run it); UI, documentation, OpenSpec and browser-test-only PRs skip it, as do duplicate same-repository main/dev heads outside promotion. Every `dev` push runs the fast set. This retains every native test except three diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index 457158e2..81f3ce42 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -69,8 +69,9 @@ Each numbered group is one pull request into the development branch. It carries - [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. - [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure, and keep the second build comparison and the package checks after all tests. Show with a probe on the acquisition call that the parallel backend group and the browser and automation tests take no Setup port. - [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. -- [x] 9.4 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan, the second build and the Windows coverage unchanged. -- [x] 9.5 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. +- [x] 9.4 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. +- [x] 9.5 Start the two native Windows test files as two processes at once in the Windows job, each in its own `--test-concurrency=1` runner and as the same unprivileged principal, and read both exit codes and both request traces after both end. Only the service file starts a server and request workers, so only it takes Setup coordination ports. The same 26 tests run, with the same pass and skip counts as before. +- [x] 9.6 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. ## Deferred acceptance and out-of-scope follow-up From 81953cef579ca314aec2af04892ac9e063c7da9f Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:36:28 +1100 Subject: [PATCH 08/15] test: share Node's compile cache across the test steps Each test file and each managed-step request worker compiles the same modules again. The compile cache keys compiled code by source content, so it changes no test input; it roughly halves the largest serial-group file and shortens the typechecks. --- CONTRIBUTING.md | 7 +++++++ openspec/changes/release-1-5-2/tasks.md | 7 ++++--- scripts/test-lanes.ts | 4 ++-- scripts/test-plan.ts | 22 ++++++++++++++++++---- tests/automation/ci.test.ts | 5 +++++ 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcfe11a8..d1376255 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,6 +114,13 @@ own heading, the first failure stops both lanes, and the run ends with a per-step duration list. A file that takes a Setup port belongs in the serial group; never add one to the other lane. +Both runners also turn on Node's compile cache for every step they start +(`NODE_COMPILE_CACHE`, a folder under the ignored `node_modules/.cache/`). It keeps +V8's compiled code per source, keyed by the source's content, so each test process and each +request worker the managed-step tests start compiles a shared module once. It changes +no test input and is safe to delete; a standalone `node --test` run does not use it +unless you export the variable yourself. + This is **not native GUI/OpenCode or host-installer certification**. The external host-installer gate is separate, as described below. Full `bun run check` retains the native backend prerequisites; it is not the portable CI command. The [workflow](.github/workflows/ci.yml) runs on pushes to `main` and `dev`, on every pull request whatever branch it targets, weekly on Tuesday at 04:17 UTC, and on manual dispatch. A PR stacked on another feature branch gets the same selection as one targeting `dev`. It retains the Linux portable checks on exactly Node 24.19.0, repeats them on the newest Node 24 release, and adds a targeted Windows native-filesystem suite. A newer push to a pull request cancels that pull request's earlier run. Versioned publication also waits for the full Windows job on its `main` push. Neither job substitutes for actual OpenCode/OpenChamber integration acceptance. diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index 81f3ce42..b1a68e93 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -69,9 +69,10 @@ Each numbered group is one pull request into the development branch. It carries - [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. - [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure, and keep the second build comparison and the package checks after all tests. Show with a probe on the acquisition call that the parallel backend group and the browser and automation tests take no Setup port. - [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. -- [x] 9.4 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. -- [x] 9.5 Start the two native Windows test files as two processes at once in the Windows job, each in its own `--test-concurrency=1` runner and as the same unprivileged principal, and read both exit codes and both request traces after both end. Only the service file starts a server and request workers, so only it takes Setup coordination ports. The same 26 tests run, with the same pass and skip counts as before. -- [x] 9.6 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. +- [x] 9.4 Turn on Node's compile cache, in a folder under the ignored dependency folder, for every step of the CI and local runners. It is keyed by source content, so no test input changes, and it roughly halves the wall time of the largest serial-group file and shortens both typechecks. +- [x] 9.5 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. +- [x] 9.6 Start the two native Windows test files as two processes at once in the Windows job, each in its own `--test-concurrency=1` runner and as the same unprivileged principal, and read both exit codes and both request traces after both end. Only the service file starts a server and request workers, so only it takes Setup coordination ports. The same 26 tests run, with the same pass and skip counts as before. +- [x] 9.7 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. ## Deferred acceptance and out-of-scope follow-up diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts index 7f31c912..3713387c 100644 --- a/scripts/test-lanes.ts +++ b/scripts/test-lanes.ts @@ -6,7 +6,7 @@ import { spawn } from 'node:child_process'; /** `lowPriority` runs the step under `nice`, so a lane that must keep its timing * headroom wins the cores when the machine is short of them. */ -export type Step = { label: string; command: string; args: string[]; lowPriority?: boolean }; +export type Step = { label: string; command: string; args: string[]; lowPriority?: boolean; env?: Record }; export type Lane = { name: string; steps: Step[] }; export type StepResult = { lane: string; label: string; seconds: number; status: 'passed' | 'failed' | 'stopped' }; @@ -35,7 +35,7 @@ export async function runLanes(lanes: Lane[]): Promise { const chunks: Buffer[] = []; // Own process group, so stopping a step also stops the test files it started. const niced = step.lowPriority && process.platform !== 'win32'; - const child = spawn(niced ? 'nice' : step.command, niced ? ['-n', '10', step.command, ...step.args] : step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true }); + const child = spawn(niced ? 'nice' : step.command, niced ? ['-n', '10', step.command, ...step.args] : step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true, env: { ...process.env, ...step.env } }); if (child.pid) children.add(child.pid); child.stdout.on('data', chunk => chunks.push(chunk)); child.stderr.on('data', chunk => chunks.push(chunk)); diff --git a/scripts/test-plan.ts b/scripts/test-plan.ts index e84641e1..4723ae0f 100644 --- a/scripts/test-plan.ts +++ b/scripts/test-plan.ts @@ -7,24 +7,38 @@ * spend most of their time waiting on deadlines and HTTP replies, and no other * group acquires those ports, so the other groups run beside them rather than * before them. */ -import { readdirSync } from 'node:fs'; +import { mkdirSync, readdirSync } from 'node:fs'; +import { resolve } from 'node:path'; import { availableParallelism } from 'node:os'; import { backendTestCommands, nativeOnlyTests } from './ci-backend.ts'; import type { Lane } from './test-lanes.ts'; export const testFiles = (dir: string) => readdirSync(dir).filter(f => f.endsWith('.test.ts')).sort().map(f => `${dir}/${f}`); +/** Node's compile cache keeps V8's compiled code for each source it loads, keyed by + * the source's content and the Node version, so a changed file is never served + * stale. Every test file runs in its own process and the managed-step tests start a + * worker per request, all loading the same modules: with the cache they compile + * each module once. It sits in the ignored dependency folder, never in tracked or + * packaged paths. A missing or unwritable folder only turns the cache off. */ +export function compileCacheEnv(root = process.cwd()): Record { + const directory = resolve(root, 'node_modules/.cache/agent-progress-compile'); + try { mkdirSync(directory, { recursive: true }); } catch { return {}; } + return { NODE_COMPILE_CACHE: directory }; +} + export function portableLanes(options: { fenced: boolean }): Lane[] { + const env = compileCacheEnv(); const backend = testFiles('tests/backend').filter(f => !nativeOnlyTests.includes(f.split('/').at(-1)!)); const [parallel, serial] = backendTestCommands(backend); // Leave two cores for the serial lane and the type checker; the lanes are // short of waiting time, not of cores, and a starved timing test is a false failure. const workers = `--test-concurrency=${Math.max(1, availableParallelism() - 2)}`; const withWorkers = (args: string[]) => [...args.slice(0, 2), workers, ...args.slice(2)]; - const node = (label: string, args: string[]) => ({ label, command: 'node', args }); + const node = (label: string, args: string[]) => ({ label, command: 'node', args, env }); const types = [ - { label: 'root typecheck', command: 'bun', args: ['run', 'typecheck'] }, - { label: 'UI typecheck', command: './node_modules/.bin/tsc', args: ['-p', 'ui/tsconfig.json', '--noEmit'] }, + { label: 'root typecheck', command: 'bun', args: ['run', 'typecheck'], env }, + { label: 'UI typecheck', command: './node_modules/.bin/tsc', args: ['-p', 'ui/tsconfig.json', '--noEmit'], env }, ]; const rest = [ node('backend tests (parallel group)', withWorkers(parallel)), diff --git a/tests/automation/ci.test.ts b/tests/automation/ci.test.ts index 3216a284..9522f162 100644 --- a/tests/automation/ci.test.ts +++ b/tests/automation/ci.test.ts @@ -76,6 +76,11 @@ test('portable lanes run every portable file once, keep the port-bound group ser assert.equal(alone.length,1);assert.equal(alone[0].steps.length,1,'the port-bound group shares its lane with nothing that could take a port'); assert.ok(!serial[0].lowPriority&&full.find(l=>l.steps.includes(serial[0]))!.steps.length===1,'the serial group never yields cores'); assert.ok(full.flatMap(l=>l.steps).filter(s=>s!==serial[0]).every(s=>s.lowPriority),'the lane beside it yields cores to it'); + // Every step shares one compile cache folder inside the ignored dependency folder, never a tracked path. + const caches=new Set(full.flatMap(l=>l.steps).map(s=>s.env?.NODE_COMPILE_CACHE)); + assert.equal(caches.size,1); + assert.ok([...caches][0]?.endsWith(join('node_modules','.cache','agent-progress-compile')),'the cache lives under node_modules/.cache'); + assert.equal(spawnSync('git',['check-ignore','-q',join('node_modules','.cache','agent-progress-compile','x')]).status,0,'git ignores the cache folder'); // Every other test step is bounded by an explicit worker count rather than the host default. for(const step of every(full).filter(s=>s!==serial[0]))assert.ok(step.args.some(a=>/^--test-concurrency=[1-9]\d*$/.test(a)),step.label); assert.deepEqual(files(every(fast)),files(every(full)).filter(f=>!files(serial).includes(f))); From a14b472efc009cd737967dc1662ded22402c3c47 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:06:30 +1100 Subject: [PATCH 09/15] test: stop the lanes on a signal and start each compile cache empty A signal to the runner did not reach the steps, which run in their own process groups, so Ctrl-C left them holding Setup ports. Stop every step on SIGINT or SIGTERM and print what the running steps had written. Compile cache entries are keyed by file name and the fixtures run from fresh temporary folders, so a kept cache grew by about 30 MB a run. Empty it at the start of each run; the speed-up comes from sharing within a run, which is unchanged. The earlier note that the cache shortens the typechecks was wrong: their times did not move. Describe the serial group as latency-bound rather than as waiting. --- CONTRIBUTING.md | 19 +++++--- openspec/changes/release-1-5-2/tasks.md | 4 +- scripts/ci.ts | 4 +- scripts/test-lanes.ts | 26 +++++++++-- scripts/test-plan.ts | 42 ++++++++++-------- scripts/test.ts | 4 +- tests/automation/ci.test.ts | 58 +++++++++++++++++++++---- 7 files changed, 116 insertions(+), 41 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1376255..6da6855f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,23 +103,28 @@ concurrent writer. Keep new fenced HTTP/worker test files in the list in This isolates tests, not product requests; no port override or relaxed refusal is used. -That serial group is the longest part of the run and mostly waits on deadlines -and worker start-up, and no other group acquires Setup ports. `scripts/ci.ts` +That serial group is the longest part of the run. It is one request after another, +each starting a worker thread, plus a few deliberate deadline waits, so it is bound by +latency rather than by cores; no other group acquires Setup ports (checked with a probe +on the acquisition call). `scripts/ci.ts` and `bun run test:all` therefore run it in one lane beside a second lane that holds both typechecks, the parallel backend group and the browser-model and automation tests (`scripts/test-plan.ts`, `scripts/test-lanes.ts`). The second lane is capped at the machine's cores minus two and runs under `nice`, so the timing assertions in the serial group keep their headroom. Each step's output is printed whole under its -own heading, the first failure stops both lanes, and the run ends with a +own heading, the first failure (or Ctrl-C or a termination signal) stops both lanes and +prints what the running steps had written, and the run ends with a per-step duration list. A file that takes a Setup port belongs in the serial group; never add one to the other lane. Both runners also turn on Node's compile cache for every step they start (`NODE_COMPILE_CACHE`, a folder under the ignored `node_modules/.cache/`). It keeps -V8's compiled code per source, keyed by the source's content, so each test process and each -request worker the managed-step tests start compiles a shared module once. It changes -no test input and is safe to delete; a standalone `node --test` run does not use it -unless you export the variable yourself. +V8's compiled code per source file and checks it against the source's hash, so a changed +file is recompiled rather than served stale, and each test process and each request +worker the managed-step tests start compiles a shared module once. It changes no test +input and is emptied at the start of every run (fixtures run from fresh temporary folders, so a +kept cache only grows); a standalone `node --test` run does not use it unless you export the +variable yourself. This is **not native GUI/OpenCode or host-installer certification**. The external host-installer gate is separate, as described below. Full `bun run check` retains the native backend prerequisites; it is not the portable CI command. diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index b1a68e93..82fa80c8 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -67,9 +67,9 @@ Each numbered group is one pull request into the development branch. It carries ## 9. Faster test suites - [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. -- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure, and keep the second build comparison and the package checks after all tests. Show with a probe on the acquisition call that the parallel backend group and the browser and automation tests take no Setup port. +- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT or SIGTERM, keeping the output of steps still running, and keep the second build comparison and the package checks after all tests. Show with a probe on the acquisition call that the parallel backend group and the browser and automation tests take no Setup port. - [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. -- [x] 9.4 Turn on Node's compile cache, in a folder under the ignored dependency folder, for every step of the CI and local runners. It is keyed by source content, so no test input changes, and it roughly halves the wall time of the largest serial-group file and shortens both typechecks. +- [x] 9.4 Turn on Node's compile cache, in a folder under the ignored dependency folder, for every step of the CI and local runners. It checks each cached file against the source's hash, so no test input changes. Record its effect from CI step timings and from the heaviest managed-step test run alone, and say that the typecheck times did not move. - [x] 9.5 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. - [x] 9.6 Start the two native Windows test files as two processes at once in the Windows job, each in its own `--test-concurrency=1` runner and as the same unprivileged principal, and read both exit codes and both request traces after both end. Only the service file starts a server and request workers, so only it takes Setup coordination ports. The same 26 tests run, with the same pass and skip counts as before. - [x] 9.7 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. diff --git a/scripts/ci.ts b/scripts/ci.ts index 36045d1a..6f300399 100644 --- a/scripts/ci.ts +++ b/scripts/ci.ts @@ -6,7 +6,7 @@ import { eligibility, assetNames, validateReleaseNotes } from './release.ts'; import { verifyCompanionEntry } from './companion-entry.ts'; import { nativeOnlyTests } from './ci-backend.ts'; import { runLanes } from './test-lanes.ts'; -import { portableLanes } from './test-plan.ts'; +import { freshCompileCache, portableLanes } from './test-plan.ts'; import { buildAll, outputHashes, compareOutputs } from './build-all.ts'; import { revisionKind, trackedOutputs } from './revision-kind.ts'; @@ -38,7 +38,7 @@ console.log(`Native-only tests NOT RUN: ${nativeOnlyTests.join(', ')}`); // Root and UI typechecks, the parallel backend group, and browser-model and // automation tests run beside the serial backend group that binds Setup ports. // Output is grouped per step and the first failure stops every lane. -await runLanes(portableLanes({ fenced: true })); +await runLanes(portableLanes({ fenced: true, env: freshCompileCache() })); buildAll(); compareOutputs(before); // The SDK's own readiness check: manifest schema, page scripts and service entry. diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts index 3713387c..7753f469 100644 --- a/scripts/test-lanes.ts +++ b/scripts/test-lanes.ts @@ -1,7 +1,11 @@ /** Runs independent groups of checks side by side. A lane is an ordered list of * steps that stop at the first failure; lanes share nothing but the checkout. * Output is captured per step and written whole when the step ends, so lines - * from different lanes never interleave. The first failure stops every lane. */ + * from different lanes never interleave. The first failure stops every lane, and + * so does SIGINT or SIGTERM: each step runs in its own process group, which a + * signal to this process would not otherwise reach, and a step left running + * would keep holding the Setup ports the next run needs. The output of steps + * still running is written before the runner exits. */ import { spawn } from 'node:child_process'; /** `lowPriority` runs the step under `nice`, so a lane that must keep its timing @@ -22,7 +26,17 @@ function print(label: string, text: string, ok: boolean) { export async function runLanes(lanes: Lane[]): Promise { const results: StepResult[] = []; const children = new Set(); + const running = new Map void>(); let failed = false; + let interrupted: NodeJS.Signals | undefined; + const onSignal = (signal: NodeJS.Signals) => { + if (interrupted) return; + interrupted = signal; failed = true; + for (const dump of running.values()) dump(); + stopAll(); + }; + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); const stopAll = () => { for (const pid of children) { try { process.kill(-pid, 'SIGTERM'); } catch { /* already gone */ } } // A step that ignores SIGTERM must not outlive the run. @@ -36,11 +50,14 @@ export async function runLanes(lanes: Lane[]): Promise { // Own process group, so stopping a step also stops the test files it started. const niced = step.lowPriority && process.platform !== 'win32'; const child = spawn(niced ? 'nice' : step.command, niced ? ['-n', '10', step.command, ...step.args] : step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true, env: { ...process.env, ...step.env } }); - if (child.pid) children.add(child.pid); + if (child.pid) { + children.add(child.pid); + running.set(child.pid, () => print(label, `${Buffer.concat(chunks).toString('utf8')}Interrupted before it ended.\n`, false)); + } child.stdout.on('data', chunk => chunks.push(chunk)); child.stderr.on('data', chunk => chunks.push(chunk)); const finish = (ok: boolean, note = '') => { - if (child.pid) children.delete(child.pid); + if (child.pid) { children.delete(child.pid); running.delete(child.pid); } const seconds = (performance.now() - started) / 1000; const status = ok ? 'passed' : failed ? 'stopped' : 'failed'; if (!ok && !failed) { failed = true; stopAll(); } @@ -58,8 +75,11 @@ export async function runLanes(lanes: Lane[]): Promise { if (!await runStep(lane, step)) break; } })); + process.off('SIGINT', onSignal); + process.off('SIGTERM', onSignal); console.log('\nStep timings'); for (const r of results) console.log(` ${r.status.padEnd(7)} ${stamp(r.seconds).padStart(7)} ${r.lane}: ${r.label}`); + if (interrupted) throw new Error(`Portable check interrupted by ${interrupted}`); if (failed) throw new Error('Portable check failed'); return results; } diff --git a/scripts/test-plan.ts b/scripts/test-plan.ts index 4723ae0f..e64d89ee 100644 --- a/scripts/test-plan.ts +++ b/scripts/test-plan.ts @@ -3,11 +3,12 @@ * check: it only decides which groups may run side by side. * * The backend files that acquire target-derived Setup coordination ports stay in - * one serial `--test-concurrency=1` invocation (see `scripts/ci-backend.ts`). They - * spend most of their time waiting on deadlines and HTTP replies, and no other - * group acquires those ports, so the other groups run beside them rather than - * before them. */ -import { mkdirSync, readdirSync } from 'node:fs'; + * one serial `--test-concurrency=1` invocation (see `scripts/ci-backend.ts`). It is + * a chain of requests, each starting a worker thread, with a few deliberate + * deadline waits, so it is bound by latency rather than by the number of cores. No + * other group acquires those ports (checked with a probe on the acquisition call), + * so the other groups run beside it rather than before it. */ +import { mkdirSync, readdirSync, rmSync } from 'node:fs'; import { resolve } from 'node:path'; import { availableParallelism } from 'node:os'; import { backendTestCommands, nativeOnlyTests } from './ci-backend.ts'; @@ -15,24 +16,31 @@ import type { Lane } from './test-lanes.ts'; export const testFiles = (dir: string) => readdirSync(dir).filter(f => f.endsWith('.test.ts')).sort().map(f => `${dir}/${f}`); -/** Node's compile cache keeps V8's compiled code for each source it loads, keyed by - * the source's content and the Node version, so a changed file is never served - * stale. Every test file runs in its own process and the managed-step tests start a - * worker per request, all loading the same modules: with the cache they compile - * each module once. It sits in the ignored dependency folder, never in tracked or - * packaged paths. A missing or unwritable folder only turns the cache off. */ -export function compileCacheEnv(root = process.cwd()): Record { +/** Node's compile cache keeps V8's compiled code for each source file it loads and + * checks it against the source's hash, so a changed file is recompiled, never + * served stale. Every test file runs in its own process and the managed-step tests + * start a worker per request, all loading the same modules: with the cache they + * compile each module once. Entries are keyed by file name and the fixtures run + * from fresh temporary folders, so a cache kept across runs only grows (about + * 30 MB a run); each run starts from an empty one instead. It sits in the ignored + * dependency folder, never in tracked or packaged paths. A missing or unwritable + * folder only turns the cache off. */ +export function freshCompileCache(root = process.cwd()): Record { const directory = resolve(root, 'node_modules/.cache/agent-progress-compile'); - try { mkdirSync(directory, { recursive: true }); } catch { return {}; } + try { + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { recursive: true }); + } catch { return {}; } return { NODE_COMPILE_CACHE: directory }; } -export function portableLanes(options: { fenced: boolean }): Lane[] { - const env = compileCacheEnv(); +export function portableLanes(options: { fenced: boolean; env?: Record }): Lane[] { + const env = options.env ?? {}; const backend = testFiles('tests/backend').filter(f => !nativeOnlyTests.includes(f.split('/').at(-1)!)); const [parallel, serial] = backendTestCommands(backend); - // Leave two cores for the serial lane and the type checker; the lanes are - // short of waiting time, not of cores, and a starved timing test is a false failure. + // The serial group's chain keeps about two cores busy (its main thread and the + // workers it starts). Leave them free so its timing assertions keep their + // headroom, and run this lane under `nice` as well. const workers = `--test-concurrency=${Math.max(1, availableParallelism() - 2)}`; const withWorkers = (args: string[]) => [...args.slice(0, 2), workers, ...args.slice(2)]; const node = (label: string, args: string[]) => ({ label, command: 'node', args, env }); diff --git a/scripts/test.ts b/scripts/test.ts index 3199f713..f9da108f 100644 --- a/scripts/test.ts +++ b/scripts/test.ts @@ -7,7 +7,7 @@ import { mkdirSync } from 'node:fs'; import { nativeOnlyTests } from './ci-backend.ts'; import { runLanes } from './test-lanes.ts'; -import { portableLanes } from './test-plan.ts'; +import { freshCompileCache, portableLanes } from './test-plan.ts'; const fast = process.argv.includes('--fast'); mkdirSync('/tmp/opencode', { recursive: true }); @@ -15,7 +15,7 @@ console.log(`Native-only tests NOT RUN: ${nativeOnlyTests.join(', ')}`); if (fast) console.log('Fast loop: the serial Setup-port backend group is skipped. Run `bun run test:all` or CI before a pull request.'); const started = performance.now(); try { - await runLanes(portableLanes({ fenced: !fast })); + await runLanes(portableLanes({ fenced: !fast, env: freshCompileCache() })); console.log(`\nAll portable groups passed in ${((performance.now() - started) / 1000).toFixed(1)}s${fast ? ' (fast loop, not the full set)' : ''}.`); } catch (error) { console.error(error instanceof Error ? error.message : error); diff --git a/tests/automation/ci.test.ts b/tests/automation/ci.test.ts index 9522f162..f678d9e0 100644 --- a/tests/automation/ci.test.ts +++ b/tests/automation/ci.test.ts @@ -4,9 +4,9 @@ import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSyn import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { spawnSync } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { backendTestCommands, nativeOnlyTests } from '../../scripts/ci-backend.ts'; -import { portableLanes } from '../../scripts/test-plan.ts'; +import { freshCompileCache, portableLanes } from '../../scripts/test-plan.ts'; import { outputs } from '../../scripts/build-all.ts'; test('package scripts that consume runtime entries build before invoking their readers',()=>{ @@ -57,7 +57,7 @@ test('portable backend groups isolate coordination ports without omitting or rep 'setup-suggestions.test.ts','windows-service.test.ts', ]); const source=readFileSync('scripts/ci.ts','utf8'); - assert.match(source,/await runLanes\(portableLanes\(\{ fenced: true \}\)\)/); + assert.match(source,/await runLanes\(portableLanes\(\{ fenced: true, env: freshCompileCache\(\) \}\)\)/); }); test('portable lanes run every portable file once, keep the port-bound group serial and alone, and the fast loop drops only that group',()=>{ @@ -76,11 +76,6 @@ test('portable lanes run every portable file once, keep the port-bound group ser assert.equal(alone.length,1);assert.equal(alone[0].steps.length,1,'the port-bound group shares its lane with nothing that could take a port'); assert.ok(!serial[0].lowPriority&&full.find(l=>l.steps.includes(serial[0]))!.steps.length===1,'the serial group never yields cores'); assert.ok(full.flatMap(l=>l.steps).filter(s=>s!==serial[0]).every(s=>s.lowPriority),'the lane beside it yields cores to it'); - // Every step shares one compile cache folder inside the ignored dependency folder, never a tracked path. - const caches=new Set(full.flatMap(l=>l.steps).map(s=>s.env?.NODE_COMPILE_CACHE)); - assert.equal(caches.size,1); - assert.ok([...caches][0]?.endsWith(join('node_modules','.cache','agent-progress-compile')),'the cache lives under node_modules/.cache'); - assert.equal(spawnSync('git',['check-ignore','-q',join('node_modules','.cache','agent-progress-compile','x')]).status,0,'git ignores the cache folder'); // Every other test step is bounded by an explicit worker count rather than the host default. for(const step of every(full).filter(s=>s!==serial[0]))assert.ok(step.args.some(a=>/^--test-concurrency=[1-9]\d*$/.test(a)),step.label); assert.deepEqual(files(every(fast)),files(every(full)).filter(f=>!files(serial).includes(f))); @@ -90,6 +85,25 @@ test('portable lanes run every portable file once, keep the port-bound group ser } }); +test('every step shares one compile cache folder that starts each run empty and stays out of tracked paths',()=>{ + const root=mkdtempSync(join(tmpdir(),'portable-cache-')); + try { + const stale=join(root,'node_modules','.cache','agent-progress-compile','stale-entry'); + mkdirSync(join(root,'node_modules','.cache','agent-progress-compile'),{recursive:true});writeFileSync(stale,'left by an earlier run'); + const env=freshCompileCache(root); + assert.equal(env.NODE_COMPILE_CACHE,join(root,'node_modules','.cache','agent-progress-compile')); + assert.deepEqual(readdirSync(env.NODE_COMPILE_CACHE),[],'an earlier run leaves nothing behind'); + for(const lanes of [portableLanes({fenced:true,env}),portableLanes({fenced:false,env})]) + assert.deepEqual(new Set(lanes.flatMap(l=>l.steps).map(s=>s.env?.NODE_COMPILE_CACHE)),new Set([env.NODE_COMPILE_CACHE]),'every step, typechecks included, gets the folder'); + // Without a cache folder, plans carry none: the plan tests above never touch a live cache. + assert.ok(portableLanes({fenced:true}).flatMap(l=>l.steps).every(s=>!s.env?.NODE_COMPILE_CACHE)); + assert.equal(spawnSync('git',['check-ignore','-q',join('node_modules','.cache','agent-progress-compile','x')]).status,0,'git ignores the folder'); + // An unusable location turns the cache off instead of failing the run. + writeFileSync(join(root,'blocked'),'a file where a folder is needed'); + assert.deepEqual(freshCompileCache(join(root,'blocked')),{}); + } finally {rmSync(root,{recursive:true,force:true});} +}); + test('lane runner starts a yielding step at lower priority and an ordinary one at the usual priority',async()=>{ const root=mkdtempSync(join(tmpdir(),'portable-lanes-priority-')); try { @@ -105,6 +119,34 @@ await runLanes([{name:'a',steps:[show('usual')]},{name:'b',steps:[show('yielding } finally {rmSync(root,{recursive:true,force:true});} }); +test('lane runner stops every step when it is signalled, keeping their output and leaving no process behind',async()=>{ + const root=mkdtempSync(join(tmpdir(),'portable-lanes-signal-')); + try { + const script=join(root,'lanes.ts'); + writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; +const hang=(label:string)=>({label,command:process.execPath,args:['-e',"console.log('CHILD-PID '+process.pid);setInterval(()=>{},1000)"]}); +try{await runLanes([{name:'a',steps:[hang('one')]},{name:'b',steps:[hang('two')]}]);}catch(error){console.log('REJECTED '+(error as Error).message);process.exit(1);}`); + const child=spawn(process.execPath,['--experimental-strip-types',script],{env:{...process.env,GITHUB_ACTIONS:''}}); + let out='';child.stdout.on('data',c=>{out+=c;}); + const exited=new Promise(done=>child.on('close',code=>done(code))); + for(const end=Date.now()+10_000;!(/start b: two/.test(out)&&/start a: one/.test(out))&&Date.now()setTimeout(r,25)); + await new Promise(r=>setTimeout(r,500)); + child.kill('SIGTERM'); + const code=await Promise.race([exited,new Promise(r=>setTimeout(()=>r('hung'),15_000))]); + assert.notEqual(code,'hung','the runner exits after the signal'); + assert.equal(code,1); + assert.match(out,/REJECTED Portable check interrupted by SIGTERM/); + const pids=[...out.matchAll(/CHILD-PID (\d+)/g)].map(m=>Number(m[1])); + assert.equal(pids.length,2,'the output of both running steps is kept'); + assert.match(out,/Interrupted before it ended/); + for(const pid of pids){ + let alive=true; + for(const end=Date.now()+5000;alive&&Date.now()setTimeout(r,50));}catch{alive=false;}} + assert.equal(alive,false,'a step outlives the runner'); + } + } finally {rmSync(root,{recursive:true,force:true});} +}); + test('lane runner stops every lane at the first failure and reports it without interleaving',async()=>{ const root=mkdtempSync(join(tmpdir(),'portable-lanes-')); try { From 11a9a55420ebfb91ff6b23c36a2b09bf926ddeb1 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:14:56 +1100 Subject: [PATCH 10/15] test: harden the lane runner and fence the built-service cleanup test Runner: record every process group and kill them again once the steps settle and from an exit hook, so a crash or a closed pipe leaves none behind; handle SIGHUP and a closed output pipe like SIGINT and SIGTERM; treat a repeated signal, or a step that has not stopped within ten seconds, as a kill and abandon it even if a leftover process still holds its pipes; exit 128 plus the signal number when interrupted; do not detach steps on Windows and stop them with taskkill /T /F; settle each step once; report a step the runner did not stop as failed with its output; name the running steps about every minute; keep only the tail of a step's output; print one line when the compile cache is off. Plan: take a core count, so one to three cores are tested, and find the serial step by its lane. Fence plan-cleanup-service.test.ts when CP_BUILT_SERVICE=1: its built service takes Setup writer slots then, as a listen probe on the bundled service showed. Windows job: start each native file under Start-Process -Wait inside a thread job, so it is waited for with everything it started. --- .github/workflows/ci.yml | 20 +- CONTRIBUTING.md | 31 ++- docs/compatibility.md | 3 +- openspec/changes/release-1-5-2/tasks.md | 4 +- scripts/ci-backend.ts | 14 +- scripts/ci.ts | 5 +- scripts/test-lanes.ts | 213 +++++++++++++---- scripts/test-plan.ts | 11 +- scripts/test.ts | 4 +- tests/automation/ci.test.ts | 132 ++++------- tests/automation/test-lanes.test.ts | 259 +++++++++++++++++++++ tests/automation/workflow-triggers.test.ts | 10 +- 12 files changed, 546 insertions(+), 160 deletions(-) create mode 100644 tests/automation/test-lanes.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe7bf68a..004cb4fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,13 +222,16 @@ jobs: # The two native files run side by side, each in its own serial test # runner. Only the service file starts a server and request workers, # which are what take Setup coordination ports; the platform file - # drives the setup controller directly. + # drives the setup controller directly. Each runs under Start-Process + # -Wait inside a thread job, so it is waited for with everything it + # started, exactly as when they ran one after the other. $runs = @('windows-platform', 'windows-service') | ForEach-Object { [pscustomobject]@{ File = $_ Output = Join-Path $env:RUNNER_TEMP ($name + '.' + $_ + '.stdout') Errors = Join-Path $env:RUNNER_TEMP ($name + '.' + $_ + '.stderr') - Process = $null + Job = $null + ExitCode = $null } } # Leave a fixed diagnostic even if principal/process creation fails. @@ -240,18 +243,21 @@ jobs: Add-LocalGroupMember -SID 'S-1-5-32-545' -Member $user $credential = [PSCredential]::new(($env:COMPUTERNAME + '\' + $name), $password) foreach ($run in $runs) { - $run.Process = Start-Process -FilePath (Get-Command node).Source -ArgumentList "--experimental-strip-types --test --test-concurrency=1 tests/backend/$($run.File).test.ts" -WorkingDirectory $env:GITHUB_WORKSPACE -Credential $credential -LoadUserProfile -WindowStyle Hidden -PassThru -RedirectStandardOutput $run.Output -RedirectStandardError $run.Errors - # Reading the handle keeps the exit code available after the wait. - $null = $run.Process.Handle + $run.Job = Start-ThreadJob -ArgumentList $run, $credential, $env:GITHUB_WORKSPACE, (Get-Command node).Source -ScriptBlock { + param($run, $credential, $workspace, $node) + $process = Start-Process -FilePath $node -ArgumentList "--experimental-strip-types --test --test-concurrency=1 tests/backend/$($run.File).test.ts" -WorkingDirectory $workspace -Credential $credential -LoadUserProfile -WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput $run.Output -RedirectStandardError $run.Errors + $process.ExitCode + } } - foreach ($run in $runs) { $run.Process.WaitForExit() } + $null = $runs.Job | Wait-Job + foreach ($run in $runs) { $run.ExitCode = @(Receive-Job -Job $run.Job -ErrorAction Stop)[-1] } foreach ($run in $runs) { if (Test-Path $run.Output) { Get-Content -LiteralPath $run.Output } if (Test-Path $run.Errors) { Get-Content -LiteralPath $run.Errors } } - if (@($runs | Where-Object { $_.Process.ExitCode -ne 0 }).Count -gt 0) { throw 'Native Windows tests failed.' } + if (@($runs | Where-Object { $_.ExitCode -ne 0 }).Count -gt 0) { throw 'Native Windows tests failed.' } } finally { # Preserve per-request evidence even on an assertion/process failure. # Never upload raw stdout/stderr, local principal names or paths. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6da6855f..793692ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,11 +111,29 @@ and `bun run test:all` therefore run it in one lane beside a second lane that holds both typechecks, the parallel backend group and the browser-model and automation tests (`scripts/test-plan.ts`, `scripts/test-lanes.ts`). The second lane is capped at the machine's cores minus two and runs under `nice`, so the -timing assertions in the serial group keep their headroom. Each step's output is printed whole under its -own heading, the first failure (or Ctrl-C or a termination signal) stops both lanes and -prints what the running steps had written, and the run ends with a -per-step duration list. A file that takes a Setup port belongs in the serial -group; never add one to the other lane. +timing assertions in the serial group keep their headroom (the cap never goes below +one worker, so on one to three cores the lane runs one worker at a time). A file that takes +a Setup port belongs in the serial group; never add one to the other lane. A listen probe +(a preload on `net.Server.prototype.listen`, covering worker threads, child processes and the +bundled `build/service/main.js`) found that no file in the parallel group or the browser and +automation tests takes one in the CI configuration. One file does in one configuration CI does +not use: `plan-cleanup-service.test.ts` runs the built service when `CP_BUILT_SERVICE=1`, and the +plan then moves it into the serial group (`fencedWhenSet` in `scripts/ci-backend.ts`). + +The runner (`scripts/test-lanes.ts`) is POSIX-first: it prints each step's output whole under its +own heading (the last 2 MB if a step is noisier, saying what was dropped), names the steps still +running about every minute, and ends with a per-step duration list. The first failure, SIGINT, +SIGTERM, SIGHUP or a closed output pipe (`test:all | head`) stops both lanes and prints what the +running steps had written. Each step runs in its own process group, so the runner stops the groups +itself: a repeated signal, or a step that has not stopped ten seconds after the first, is killed +outright and abandoned even if a leftover process still holds its output pipes, and every group +the runner started is killed once the steps settle and again if the runner itself crashes. An +interrupted run exits 128 plus the signal number (130, 143, 129, 141), a failed one 1. A step +the runner stopped is reported as stopped; one that fails on its own is reported as failed with +its output, whenever it does so. On Windows steps are not detached and are stopped with +`taskkill /T /F`; that path is covered by a unit test of the stop command but has not been run on +Windows, and the typecheck step's `./node_modules/.bin/tsc` is a POSIX path, so on Windows run the +individual commands listed under Local development by hand. Both runners also turn on Node's compile cache for every step they start (`NODE_COMPILE_CACHE`, a folder under the ignored `node_modules/.cache/`). It keeps @@ -153,7 +171,8 @@ Without that opt-in, the three journeys explicitly skip; no per-request assertion, budget or deadline is relaxed. Local full native runs must also set `AP_WINDOWS_FULL=1`. The Windows job starts the platform file and the service file as two processes at once, each in its own serial test runner and as the same -unprivileged principal, and waits for both before reading either exit code; only the +unprivileged principal, each started with `Start-Process -Wait` inside a PowerShell thread job so +it is waited for together with everything it started, and reads both exit codes after both end; only the service file starts a server and request workers, which take the Setup coordination ports. Weekly and manual CI runs target Windows only; the Linux jobs retain their PR/push coverage. See [native Windows verification](docs/compatibility.md#native-windows-verification). diff --git a/docs/compatibility.md b/docs/compatibility.md index 5742c470..1f8554f8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -493,7 +493,8 @@ permits cleanup/permission repair. See The `Windows setup, storage and capture` CI job runs the native platform suite and `tests/backend/windows-service.test.ts` in the same provisioned non-admin -account, as two processes at once, each in its own serial test runner. Ordinary PRs run it when backend/build/test inputs change (unknown +account, as two processes at once, each in its own serial test runner and each waited for +with everything it started (`Start-Process -Wait` inside a thread job). Ordinary PRs run it when backend/build/test inputs change (unknown paths conservatively run it); UI, documentation, OpenSpec and browser-test-only PRs skip it, as do duplicate same-repository main/dev heads outside promotion. Every `dev` push runs the fast set. This retains every native test except three diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index 099d6ca3..dc4256cc 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -67,11 +67,11 @@ Each numbered group is one pull request into the development branch. It carries ## 9. Faster test suites - [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. -- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT or SIGTERM, keeping the output of steps still running, and keep the second build comparison and the package checks after all tests. Show with a probe on the acquisition call that the parallel backend group and the browser and automation tests take no Setup port. +- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT, SIGTERM, SIGHUP or a closed output pipe, keeping the output of steps still running and killing every group the runner started (including after a crash or a repeated signal), exit with 128 plus the signal number when interrupted, tail-limit and heartbeat the step output, mark a step the runner stopped as stopped and one that fails by itself as failed, and keep the second build comparison and the package checks after all tests. Show with a preload probe on the listen call, covering worker threads, child processes and the bundled service, that the parallel backend group and the browser and automation tests take no Setup port, and fence the one file that does when `CP_BUILT_SERVICE=1` is set. Give the plan a core count so one to three cores are tested. - [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. - [x] 9.4 Turn on Node's compile cache, in a folder under the ignored dependency folder, for every step of the CI and local runners. It checks each cached file against the source's hash, so no test input changes. Record its effect from CI step timings and from the heaviest managed-step test run alone, and say that the typecheck times did not move. - [x] 9.5 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. -- [x] 9.6 Start the two native Windows test files as two processes at once in the Windows job, each in its own `--test-concurrency=1` runner and as the same unprivileged principal, and read both exit codes and both request traces after both end. Only the service file starts a server and request workers, so only it takes Setup coordination ports. The same 26 tests run, with the same pass and skip counts as before. +- [x] 9.6 Start the two native Windows test files as two processes at once in the Windows job, each in its own `--test-concurrency=1` runner and as the same unprivileged principal, each waited for with everything it started, and read both exit codes and both request traces after both end. Only the service file starts a server and request workers, so only it takes Setup coordination ports. The same 26 tests run, with the same pass and skip counts as before. - [x] 9.7 Remove or merge no test. Record what was considered and rejected as unsafe or unproven in the pull request. ## Deferred acceptance and out-of-scope follow-up diff --git a/scripts/ci-backend.ts b/scripts/ci-backend.ts index eda1d209..d9c74e07 100644 --- a/scripts/ci-backend.ts +++ b/scripts/ci-backend.ts @@ -26,8 +26,18 @@ export const nativeOnlyTests = [ 'subagent-upgrade-runtime.test.ts', ]; -export function backendTestCommands(files: string[]): string[][] { - const fenced = (file: string) => fenceTests.has(file.split('/').at(-1)!); +/** Files that take Setup ports only when an environment variable opts in, mapped to + * that variable. `plan-cleanup-service.test.ts` runs the checked-in built service, + * whose readiness probe takes writer slots, when `CP_BUILT_SERVICE=1` (found with a + * listen probe, bundled service included). Where the opt-in is set the file is + * fenced; where it is not, the file takes no port and stays in the parallel group. */ +export const fencedWhenSet: Record = { 'plan-cleanup-service.test.ts': 'CP_BUILT_SERVICE' }; + +export function backendTestCommands(files: string[], environment: NodeJS.ProcessEnv = process.env): string[][] { + const fenced = (file: string) => { + const name = file.split('/').at(-1)!; + return fenceTests.has(name) || (name in fencedWhenSet && environment[fencedWhenSet[name]] === '1'); + }; return [ ['--experimental-strip-types', '--test', ...files.filter(file => !fenced(file))], ['--experimental-strip-types', '--test', '--test-concurrency=1', ...files.filter(fenced)], diff --git a/scripts/ci.ts b/scripts/ci.ts index 6f300399..02dcc82e 100644 --- a/scripts/ci.ts +++ b/scripts/ci.ts @@ -5,7 +5,7 @@ import { createHash } from 'node:crypto'; import { eligibility, assetNames, validateReleaseNotes } from './release.ts'; import { verifyCompanionEntry } from './companion-entry.ts'; import { nativeOnlyTests } from './ci-backend.ts'; -import { runLanes } from './test-lanes.ts'; +import { LanesError, runLanes } from './test-lanes.ts'; import { freshCompileCache, portableLanes } from './test-plan.ts'; import { buildAll, outputHashes, compareOutputs } from './build-all.ts'; import { revisionKind, trackedOutputs } from './revision-kind.ts'; @@ -38,7 +38,8 @@ console.log(`Native-only tests NOT RUN: ${nativeOnlyTests.join(', ')}`); // Root and UI typechecks, the parallel backend group, and browser-model and // automation tests run beside the serial backend group that binds Setup ports. // Output is grouped per step and the first failure stops every lane. -await runLanes(portableLanes({ fenced: true, env: freshCompileCache() })); +try { await runLanes(portableLanes({ fenced: true, env: freshCompileCache() })); } +catch (error) { console.error(error instanceof Error ? error.message : error); process.exit(error instanceof LanesError ? error.exitCode : 1); } buildAll(); compareOutputs(before); // The SDK's own readiness check: manifest schema, page scripts and service entry. diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts index 7753f469..ace9d735 100644 --- a/scripts/test-lanes.ts +++ b/scripts/test-lanes.ts @@ -1,18 +1,86 @@ /** Runs independent groups of checks side by side. A lane is an ordered list of * steps that stop at the first failure; lanes share nothing but the checkout. * Output is captured per step and written whole when the step ends, so lines - * from different lanes never interleave. The first failure stops every lane, and - * so does SIGINT or SIGTERM: each step runs in its own process group, which a - * signal to this process would not otherwise reach, and a step left running - * would keep holding the Setup ports the next run needs. The output of steps - * still running is written before the runner exits. */ -import { spawn } from 'node:child_process'; + * from different lanes never interleave. + * + * POSIX first. Each step runs in its own process group, so a signal to this + * process would not reach it; the runner therefore stops every step itself, on the + * first failure and on SIGINT, SIGTERM, SIGHUP or a closed output pipe, and a step + * left running would keep holding the Setup ports the next run needs. A repeated + * signal, or a step that ignores the first one for too long, is killed outright + * and abandoned even if its output pipes are still held open; every group the + * runner started is killed again once the steps settle and when the process + * exits, however it exits. On Windows steps are not detached and are stopped with + * `taskkill /T /F`; that path is untested on Windows. */ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; /** `lowPriority` runs the step under `nice`, so a lane that must keep its timing * headroom wins the cores when the machine is short of them. */ export type Step = { label: string; command: string; args: string[]; lowPriority?: boolean; env?: Record }; export type Lane = { name: string; steps: Step[] }; export type StepResult = { lane: string; label: string; seconds: number; status: 'passed' | 'failed' | 'stopped' }; +export type RunOptions = { + /** How often to name the steps still running. */ + heartbeatMs?: number; + /** How long a step may take to stop before it is killed and abandoned. */ + killAfterMs?: number; + /** How long a step that has exited may keep its output pipes open. */ + graceMs?: number; + /** The most output kept per step; the earliest is dropped first. */ + maxOutputBytes?: number; +}; + +const SIGNAL_NUMBER: Record = { SIGHUP: 1, SIGINT: 2, SIGPIPE: 13, SIGTERM: 15 }; + +/** `exitCode` is 1 for a failed run and 128 plus the signal number for an interrupted one. */ +export class LanesError extends Error { + readonly exitCode: number; + constructor(message: string, exitCode: number) { super(message); this.exitCode = exitCode; } +} + +export type StopPlan = + | { kind: 'group'; group: number; signal: 'SIGTERM' | 'SIGKILL' } + | { kind: 'command'; command: string; args: string[] }; + +/** How to stop a step and everything it started on a platform. */ +export function stopPlan(platform: NodeJS.Platform, pid: number, force: boolean): StopPlan { + if (platform === 'win32') return { kind: 'command', command: 'taskkill', args: ['/T', '/F', '/PID', String(pid)] }; + return { kind: 'group', group: -pid, signal: force ? 'SIGKILL' : 'SIGTERM' }; +} + +/** Steps get their own process group everywhere but Windows, which has none to use. */ +export const detachesSteps = (platform: NodeJS.Platform) => platform !== 'win32'; + +function stop(pid: number, force: boolean) { + const plan = stopPlan(process.platform, pid, force); + try { + if (plan.kind === 'group') process.kill(plan.group, plan.signal); + else spawnSync(plan.command, plan.args, { stdio: 'ignore', windowsHide: true }); + } catch { /* already gone */ } +} + +/** The last `max` bytes written, with a note of what was dropped. */ +class Tail { + private chunks: Buffer[] = []; + private size = 0; + private dropped = 0; + private readonly max: number; + constructor(max: number) { this.max = max; } + push(chunk: Buffer) { + this.chunks.push(chunk); this.size += chunk.length; + while (this.size > this.max && this.chunks.length > 1) { + const first = this.chunks.shift()!; this.size -= first.length; this.dropped += first.length; + } + if (this.size > this.max) { + const cut = this.size - this.max; + this.chunks[0] = this.chunks[0].subarray(cut); this.size -= cut; this.dropped += cut; + } + } + text() { + const text = Buffer.concat(this.chunks).toString('utf8'); + return this.dropped ? `[${this.dropped} bytes of earlier output dropped]\n${text}` : text; + } +} const github = process.env.GITHUB_ACTIONS === 'true'; const stamp = (seconds: number) => `${seconds.toFixed(1)}s`; @@ -23,63 +91,118 @@ function print(label: string, text: string, ok: boolean) { process.stdout.write(github ? `::group::${label}\n${body}::endgroup::\n` : `\n=== ${label} ${ok ? '' : '(failed) '}===\n${body}`); } -export async function runLanes(lanes: Lane[]): Promise { +type Live = { label: string; started: number; child: ChildProcess; signalled: boolean; printed: boolean; dump: () => void; force: (note: string) => void }; + +export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise { + const { heartbeatMs = 60_000, killAfterMs = 10_000, graceMs = 3_000, maxOutputBytes = 2 * 1024 * 1024 } = options; const results: StepResult[] = []; - const children = new Set(); - const running = new Map void>(); + const groups = new Set(); + const live = new Map(); let failed = false; - let interrupted: NodeJS.Signals | undefined; - const onSignal = (signal: NodeJS.Signals) => { - if (interrupted) return; - interrupted = signal; failed = true; - for (const dump of running.values()) dump(); - stopAll(); + let interrupted: { signal: string; code: number } | undefined; + let killTimer: NodeJS.Timeout | undefined; + + const escalate = () => { + clearTimeout(killTimer); killTimer = undefined; + for (const [pid, step] of live) { step.signalled = true; stop(pid, true); } + for (const step of [...live.values()]) step.force('Did not stop in time; killed and abandoned.\n'); + }; + const beginStop = () => { + for (const [pid, step] of live) { + // A step that has already exited on its own is not ours to call stopped. + if (step.child.exitCode === null && step.child.signalCode === null) { step.signalled = true; stop(pid, false); } + } + // Not unref'd: steps are still running, and the run must outlast them. + killTimer ??= setTimeout(escalate, killAfterMs); }; - process.on('SIGINT', onSignal); - process.on('SIGTERM', onSignal); - const stopAll = () => { - for (const pid of children) { try { process.kill(-pid, 'SIGTERM'); } catch { /* already gone */ } } - // A step that ignores SIGTERM must not outlive the run. - setTimeout(() => { for (const pid of children) { try { process.kill(-pid, 'SIGKILL'); } catch { /* already gone */ } } }, 10_000).unref(); + const onSignal = (signal: string) => { + if (interrupted) { escalate(); return; } + interrupted = { signal, code: 128 + (SIGNAL_NUMBER[signal] ?? 0) }; failed = true; + for (const step of live.values()) step.dump(); + beginStop(); }; + const handlers = (['SIGINT', 'SIGTERM', 'SIGHUP'] as const).map(signal => [signal, () => onSignal(signal)] as const); + for (const [signal, handler] of handlers) process.on(signal, handler); + // `test:all | head` closes our output: stop rather than die with groups running. + const onStdoutError = (error: NodeJS.ErrnoException) => { if (error.code === 'EPIPE') onSignal('SIGPIPE'); }; + process.stdout.on('error', onStdoutError); + // A crash or process.exit must not orphan a group either. + const onExit = () => { for (const pid of groups) stop(pid, true); }; + process.on('exit', onExit); + const heartbeat = setInterval(() => { + const names = [...live.values()].map(step => `${step.label} (${stamp((performance.now() - step.started) / 1000)})`); + if (names.length) console.log(`still running: ${names.join('; ')}`); + }, heartbeatMs); + const runStep = (lane: Lane, step: Step) => new Promise(resolve => { const started = performance.now(); const label = `${lane.name}: ${step.label}`; console.log(`start ${label}`); - const chunks: Buffer[] = []; - // Own process group, so stopping a step also stops the test files it started. + const tail = new Tail(maxOutputBytes); const niced = step.lowPriority && process.platform !== 'win32'; - const child = spawn(niced ? 'nice' : step.command, niced ? ['-n', '10', step.command, ...step.args] : step.args, { stdio: ['ignore', 'pipe', 'pipe'], detached: true, env: { ...process.env, ...step.env } }); - if (child.pid) { - children.add(child.pid); - running.set(child.pid, () => print(label, `${Buffer.concat(chunks).toString('utf8')}Interrupted before it ended.\n`, false)); - } - child.stdout.on('data', chunk => chunks.push(chunk)); - child.stderr.on('data', chunk => chunks.push(chunk)); + const child = spawn(niced ? 'nice' : step.command, niced ? ['-n', '10', step.command, ...step.args] : step.args, { + stdio: ['ignore', 'pipe', 'pipe'], detached: detachesSteps(process.platform), windowsHide: true, env: { ...process.env, ...step.env }, + }); + const pid = child.pid; + let settled = false; + let exit: { code: number | null; signal: NodeJS.Signals | null } | undefined; + let grace: NodeJS.Timeout | undefined; + const entry: Live = { + label, started, child, signalled: false, printed: false, + dump: () => { entry.printed = true; print(label, `${tail.text()}Interrupted before it ended.\n`, false); }, + force: note => { child.stdout?.destroy(); child.stderr?.destroy(); finish(exit?.code === 0, note); }, + }; + if (pid) { groups.add(pid); live.set(pid, entry); } + child.stdout?.on('data', chunk => tail.push(chunk)); + child.stderr?.on('data', chunk => tail.push(chunk)); + child.stdout?.on('error', () => {}); + child.stderr?.on('error', () => {}); + // ENOENT emits error and then close; the first word wins. const finish = (ok: boolean, note = '') => { - if (child.pid) { children.delete(child.pid); running.delete(child.pid); } + if (settled) return; + settled = true; clearTimeout(grace); + if (pid) { + live.delete(pid); + if (ok) groups.delete(pid); // a clean pass leaves nothing to sweep + if (!live.size) { clearTimeout(killTimer); killTimer = undefined; } + } const seconds = (performance.now() - started) / 1000; - const status = ok ? 'passed' : failed ? 'stopped' : 'failed'; - if (!ok && !failed) { failed = true; stopAll(); } - if (status !== 'stopped') print(label, Buffer.concat(chunks).toString('utf8') + note, ok); - console.log(`${ok ? 'passed' : status} ${label} (${stamp(seconds)})`); + // Only a step the runner stopped is "stopped"; one that fails on its own is a failure, whenever. + const status = ok ? 'passed' : entry.signalled ? 'stopped' : 'failed'; + if (status === 'failed' && !failed) { failed = true; beginStop(); } + if ((status === 'passed' || status === 'failed') && !entry.printed) print(label, tail.text() + note, ok); + console.log(`${status} ${label} (${stamp(seconds)})`); results.push({ lane: lane.name, label: step.label, seconds, status }); resolve(ok); }; child.on('error', error => finish(false, `Could not start: ${error.message}\n`)); + child.on('exit', (code, signal) => { + exit = { code, signal }; + // A leftover process holding our pipes must not keep the step open for ever. + grace = setTimeout(() => entry.force('A leftover process held its output open; abandoned.\n'), graceMs); + }); child.on('close', (code, signal) => finish(code === 0, code === 0 ? '' : `Exited with ${signal ?? `code ${code}`}\n`)); }); - await Promise.all(lanes.map(async lane => { - for (const step of lane.steps) { - if (failed) { results.push({ lane: lane.name, label: step.label, seconds: 0, status: 'stopped' }); continue; } - if (!await runStep(lane, step)) break; - } - })); - process.off('SIGINT', onSignal); - process.off('SIGTERM', onSignal); + + try { + await Promise.all(lanes.map(async lane => { + for (const step of lane.steps) { + if (failed) { results.push({ lane: lane.name, label: step.label, seconds: 0, status: 'stopped' }); continue; } + if (!await runStep(lane, step)) break; + } + })); + } finally { + clearInterval(heartbeat); clearTimeout(killTimer); + if (failed || interrupted) for (const pid of groups) stop(pid, true); + groups.clear(); + for (const [signal, handler] of handlers) process.off(signal, handler); + process.stdout.off('error', onStdoutError); + process.stdout.on('error', () => {}); // a late EPIPE while exiting is not a crash + process.off('exit', onExit); + } console.log('\nStep timings'); for (const r of results) console.log(` ${r.status.padEnd(7)} ${stamp(r.seconds).padStart(7)} ${r.lane}: ${r.label}`); - if (interrupted) throw new Error(`Portable check interrupted by ${interrupted}`); - if (failed) throw new Error('Portable check failed'); + if (interrupted) throw new LanesError(`Portable check interrupted by ${interrupted.signal}`, interrupted.code); + if (failed) throw new LanesError('Portable check failed', 1); return results; } diff --git a/scripts/test-plan.ts b/scripts/test-plan.ts index e64d89ee..040e1b4e 100644 --- a/scripts/test-plan.ts +++ b/scripts/test-plan.ts @@ -30,18 +30,21 @@ export function freshCompileCache(root = process.cwd()): Record try { rmSync(directory, { recursive: true, force: true }); mkdirSync(directory, { recursive: true }); - } catch { return {}; } + } catch { + console.log('Compile cache disabled: node_modules/.cache/agent-progress-compile could not be prepared; tests run without it.'); + return {}; + } return { NODE_COMPILE_CACHE: directory }; } -export function portableLanes(options: { fenced: boolean; env?: Record }): Lane[] { +export function portableLanes(options: { fenced: boolean; env?: Record; cpus?: number; environment?: NodeJS.ProcessEnv }): Lane[] { const env = options.env ?? {}; const backend = testFiles('tests/backend').filter(f => !nativeOnlyTests.includes(f.split('/').at(-1)!)); - const [parallel, serial] = backendTestCommands(backend); + const [parallel, serial] = backendTestCommands(backend, options.environment ?? process.env); // The serial group's chain keeps about two cores busy (its main thread and the // workers it starts). Leave them free so its timing assertions keep their // headroom, and run this lane under `nice` as well. - const workers = `--test-concurrency=${Math.max(1, availableParallelism() - 2)}`; + const workers = `--test-concurrency=${Math.max(1, (options.cpus ?? availableParallelism()) - 2)}`; const withWorkers = (args: string[]) => [...args.slice(0, 2), workers, ...args.slice(2)]; const node = (label: string, args: string[]) => ({ label, command: 'node', args, env }); const types = [ diff --git a/scripts/test.ts b/scripts/test.ts index f9da108f..3e1e2eac 100644 --- a/scripts/test.ts +++ b/scripts/test.ts @@ -6,7 +6,7 @@ * node --experimental-strip-types scripts/test.ts --fast the same without the serial Setup-port group */ import { mkdirSync } from 'node:fs'; import { nativeOnlyTests } from './ci-backend.ts'; -import { runLanes } from './test-lanes.ts'; +import { LanesError, runLanes } from './test-lanes.ts'; import { freshCompileCache, portableLanes } from './test-plan.ts'; const fast = process.argv.includes('--fast'); @@ -19,5 +19,5 @@ try { console.log(`\nAll portable groups passed in ${((performance.now() - started) / 1000).toFixed(1)}s${fast ? ' (fast loop, not the full set)' : ''}.`); } catch (error) { console.error(error instanceof Error ? error.message : error); - process.exit(1); + process.exit(error instanceof LanesError ? error.exitCode : 1); } diff --git a/tests/automation/ci.test.ts b/tests/automation/ci.test.ts index f678d9e0..dc719e75 100644 --- a/tests/automation/ci.test.ts +++ b/tests/automation/ci.test.ts @@ -1,10 +1,9 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { tmpdir, availableParallelism } from 'node:os'; import { join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { spawn, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import { backendTestCommands, nativeOnlyTests } from '../../scripts/ci-backend.ts'; import { freshCompileCache, portableLanes } from '../../scripts/test-plan.ts'; import { outputs } from '../../scripts/build-all.ts'; @@ -46,7 +45,7 @@ test('Windows service tests have an explicit same-job backend build prerequisite test('portable backend groups isolate coordination ports without omitting or repeating tests',()=>{ const files=readdirSync('tests/backend').filter(f=>f.endsWith('.test.ts')).map(f=>`tests/backend/${f}`); - const [parallel,serial]=backendTestCommands(files); + const [parallel,serial]=backendTestCommands(files,{}); assert.deepEqual(parallel.slice(0,2),['--experimental-strip-types','--test']); assert.deepEqual(serial.slice(0,3),['--experimental-strip-types','--test','--test-concurrency=1']); assert.deepEqual([...parallel.slice(2),...serial.slice(3)].sort(),files.sort()); @@ -60,29 +59,55 @@ test('portable backend groups isolate coordination ports without omitting or rep assert.match(source,/await runLanes\(portableLanes\(\{ fenced: true, env: freshCompileCache\(\) \}\)\)/); }); -test('portable lanes run every portable file once, keep the port-bound group serial and alone, and the fast loop drops only that group',()=>{ +test('portable lanes run every portable file once, keep the port-bound group serial and alone, and the fast loop drops only that group, on any core count',()=>{ const names=(dir:string)=>readdirSync(dir).filter(f=>f.endsWith('.test.ts')).map(f=>`${dir}/${f}`).sort(); const native=new Set(nativeOnlyTests); const backend=names('tests/backend').filter(f=>!native.has(f.split('/').at(-1)!)); const every=(lanes:ReturnType)=>lanes.flatMap(l=>l.steps).filter(s=>s.command==='node'); const files=(steps:{args:string[]}[])=>steps.flatMap(s=>s.args.filter(a=>a.endsWith('.test.ts'))).sort(); - const full=portableLanes({fenced:true}),fast=portableLanes({fenced:false}); - assert.deepEqual(files(every(full)),[...backend,...names('tests/browser'),...names('tests/automation')].sort()); - assert.equal(new Set(files(every(full))).size,files(every(full)).length,'no file runs twice'); - const serial=every(full).filter(s=>s.args.includes('--test-concurrency=1')); - assert.equal(serial.length,1); - assert.deepEqual(files(serial).map(f=>f.split('/').at(-1)),backendTestCommands(backend)[1].slice(3).map(f=>f.split('/').at(-1)).sort()); - const alone=full.filter(l=>l.steps.includes(serial[0])); - assert.equal(alone.length,1);assert.equal(alone[0].steps.length,1,'the port-bound group shares its lane with nothing that could take a port'); - assert.ok(!serial[0].lowPriority&&full.find(l=>l.steps.includes(serial[0]))!.steps.length===1,'the serial group never yields cores'); - assert.ok(full.flatMap(l=>l.steps).filter(s=>s!==serial[0]).every(s=>s.lowPriority),'the lane beside it yields cores to it'); - // Every other test step is bounded by an explicit worker count rather than the host default. - for(const step of every(full).filter(s=>s!==serial[0]))assert.ok(step.args.some(a=>/^--test-concurrency=[1-9]\d*$/.test(a)),step.label); - assert.deepEqual(files(every(fast)),files(every(full)).filter(f=>!files(serial).includes(f))); - for(const lanes of [full,fast]){ - const labels=lanes.flatMap(l=>l.steps).map(s=>s.label); - assert.ok(labels.includes('root typecheck')&&labels.includes('UI typecheck'),'both typechecks stay in every run'); + // On one to three cores the other lane's worker count is also 1, so the serial group + // cannot be told apart by its --test-concurrency=1; it is the sole step of its own lane. + for(const cpus of [1,2,3,4,24]){ + const at=`on ${cpus} cpus`; + const full=portableLanes({fenced:true,cpus,environment:{}}),fast=portableLanes({fenced:false,cpus,environment:{}}); + assert.deepEqual(files(every(full)),[...backend,...names('tests/browser'),...names('tests/automation')].sort(),at); + assert.equal(new Set(files(every(full))).size,files(every(full)).length,`no file runs twice ${at}`); + const serialLanes=full.filter(l=>l.name==='port-bound backend'); + assert.equal(serialLanes.length,1,at);assert.equal(serialLanes[0].steps.length,1,`the port-bound group shares its lane with nothing that could take a port ${at}`); + const serial=serialLanes[0].steps[0]; + assert.ok(serial.args.includes('--test-concurrency=1'),at); + assert.deepEqual(serial.args.slice(0,3),backendTestCommands(backend,{})[1].slice(0,3),at); + assert.deepEqual(files([serial]).map(f=>f.split('/').at(-1)),backendTestCommands(backend,{})[1].slice(3).map(f=>f.split('/').at(-1)).sort(),at); + assert.ok(!serial.lowPriority,`the serial group never yields cores ${at}`); + const others=full.filter(l=>l!==serialLanes[0]).flatMap(l=>l.steps); + assert.ok(others.every(s=>s.lowPriority),`the lane beside it yields cores to it ${at}`); + // Every other test step is bounded by an explicit worker count: the cores minus two, never below one. + const workers=Math.max(1,cpus-2); + for(const step of others.filter(s=>s.command==='node'))assert.ok(step.args.includes(`--test-concurrency=${workers}`),`${step.label} ${at}`); + assert.deepEqual(files(every(fast)),files(every(full)).filter(f=>!files([serial]).includes(f)),at); + assert.equal(fast.some(l=>l.steps.some(s=>s.args.includes('--test-concurrency=1')&&files([s]).some(f=>files([serial]).includes(f)))),false,`the fast loop has no serial group ${at}`); + for(const lanes of [full,fast]){ + const labels=lanes.flatMap(l=>l.steps).map(s=>s.label); + assert.ok(labels.includes('root typecheck')&&labels.includes('UI typecheck'),`both typechecks stay in every run ${at}`); + } } + // Without a core count the host's is used. + assert.deepEqual(portableLanes({fenced:true,environment:{}}),portableLanes({fenced:true,cpus:availableParallelism(),environment:{}})); +}); + +test('a file that takes Setup ports only when CP_BUILT_SERVICE=1 runs in the serial group in that configuration and only there',()=>{ + const names=readdirSync('tests/backend').filter(f=>f.endsWith('.test.ts')).map(f=>`tests/backend/${f}`); + const opted='tests/backend/plan-cleanup-service.test.ts'; + const [parallelOff,serialOff]=backendTestCommands(names,{}); + assert.ok(parallelOff.includes(opted)&&!serialOff.includes(opted),'without the opt-in it stays parallel: it takes no port'); + const [parallelOn,serialOn]=backendTestCommands(names,{CP_BUILT_SERVICE:'1'}); + assert.ok(serialOn.includes(opted)&&!parallelOn.includes(opted),'with the opt-in its built service takes writer slots'); + assert.deepEqual([...parallelOn.slice(2),...serialOn.slice(3)].sort(),names.sort(),'still no file omitted or repeated'); + assert.ok(!backendTestCommands(names,{CP_BUILT_SERVICE:'0'})[1].includes(opted),'only 1 opts in'); + const plan=(environment:NodeJS.ProcessEnv)=>portableLanes({fenced:true,cpus:4,environment}).find(l=>l.name==='port-bound backend')!.steps[0].args; + assert.ok(plan({CP_BUILT_SERVICE:'1'}).includes(opted)&&!plan({}).includes(opted)); + const rest=(environment:NodeJS.ProcessEnv)=>portableLanes({fenced:true,cpus:4,environment}).filter(l=>l.name!=='port-bound backend').flatMap(l=>l.steps).flatMap(s=>s.args); + assert.ok(!rest({CP_BUILT_SERVICE:'1'}).includes(opted),'the lane beside the serial group never holds it'); }); test('every step shares one compile cache folder that starts each run empty and stays out of tracked paths',()=>{ @@ -104,71 +129,6 @@ test('every step shares one compile cache folder that starts each run empty and } finally {rmSync(root,{recursive:true,force:true});} }); -test('lane runner starts a yielding step at lower priority and an ordinary one at the usual priority',async()=>{ - const root=mkdtempSync(join(tmpdir(),'portable-lanes-priority-')); - try { - const script=join(root,'lanes.ts'); - writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; -const show=(label:string,lowPriority?:boolean)=>({label,command:process.execPath,args:['-e',"console.log('PRIORITY '+require('node:os').getPriority())"],lowPriority}); -await runLanes([{name:'a',steps:[show('usual')]},{name:'b',steps:[show('yielding',true)]}]);`); - // Plain headings: the runner writes `::group::` blocks when it sees GitHub Actions. - const r=spawnSync(process.execPath,['--experimental-strip-types',script],{encoding:'utf8',timeout:20_000,env:{...process.env,GITHUB_ACTIONS:''}}); - assert.equal(r.status,0,r.stderr); - const seen=(label:string)=>Number(new RegExp(`=== ${label} ===\\nPRIORITY (-?\\d+)`).exec(r.stdout)?.[1]); - assert.ok(seen('b: yielding')>seen('a: usual'),'the yielding step has the larger niceness'); - } finally {rmSync(root,{recursive:true,force:true});} -}); - -test('lane runner stops every step when it is signalled, keeping their output and leaving no process behind',async()=>{ - const root=mkdtempSync(join(tmpdir(),'portable-lanes-signal-')); - try { - const script=join(root,'lanes.ts'); - writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; -const hang=(label:string)=>({label,command:process.execPath,args:['-e',"console.log('CHILD-PID '+process.pid);setInterval(()=>{},1000)"]}); -try{await runLanes([{name:'a',steps:[hang('one')]},{name:'b',steps:[hang('two')]}]);}catch(error){console.log('REJECTED '+(error as Error).message);process.exit(1);}`); - const child=spawn(process.execPath,['--experimental-strip-types',script],{env:{...process.env,GITHUB_ACTIONS:''}}); - let out='';child.stdout.on('data',c=>{out+=c;}); - const exited=new Promise(done=>child.on('close',code=>done(code))); - for(const end=Date.now()+10_000;!(/start b: two/.test(out)&&/start a: one/.test(out))&&Date.now()setTimeout(r,25)); - await new Promise(r=>setTimeout(r,500)); - child.kill('SIGTERM'); - const code=await Promise.race([exited,new Promise(r=>setTimeout(()=>r('hung'),15_000))]); - assert.notEqual(code,'hung','the runner exits after the signal'); - assert.equal(code,1); - assert.match(out,/REJECTED Portable check interrupted by SIGTERM/); - const pids=[...out.matchAll(/CHILD-PID (\d+)/g)].map(m=>Number(m[1])); - assert.equal(pids.length,2,'the output of both running steps is kept'); - assert.match(out,/Interrupted before it ended/); - for(const pid of pids){ - let alive=true; - for(const end=Date.now()+5000;alive&&Date.now()setTimeout(r,50));}catch{alive=false;}} - assert.equal(alive,false,'a step outlives the runner'); - } - } finally {rmSync(root,{recursive:true,force:true});} -}); - -test('lane runner stops every lane at the first failure and reports it without interleaving',async()=>{ - const root=mkdtempSync(join(tmpdir(),'portable-lanes-')); - try { - const script=join(root,'lanes.ts'); - writeFileSync(script,`import {runLanes} from ${JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href)}; -const node=(label:string,code:string)=>({label,command:process.execPath,args:['-e',code]}); -try{await runLanes([ - {name:'slow',steps:[node('hangs',"setTimeout(()=>{},60000)"),node('never starts',"console.log('LATE-STEP')")]}, - {name:'fails',steps:[node('breaks',"console.log('FAIL-OUTPUT');process.exit(3)")]}, - {name:'ok',steps:[node('passes',"console.log('PASS-OUTPUT')"),node('skipped after failure',"setTimeout(()=>console.log('LATE-STEP'),5000)")]}, -]);}catch(error){console.log('REJECTED '+(error as Error).message);process.exit(1);}`); - const started=performance.now(); - const r=spawnSync(process.execPath,['--experimental-strip-types',script],{encoding:'utf8',timeout:20_000}); - assert.equal(r.status,1); - assert.ok(performance.now()-started<15_000,'the hanging lane is stopped, not waited for'); - assert.match(r.stdout,/FAIL-OUTPUT/);assert.match(r.stdout,/Exited with code 3/);assert.match(r.stdout,/REJECTED Portable check failed/); - assert.doesNotMatch(r.stdout,/LATE-STEP/); - assert.match(r.stdout,/stopped\s+[\d.]+s\s+slow: hangs/); - assert.match(r.stdout,/failed\s+[\d.]+s\s+fails: breaks/); - } finally {rmSync(root,{recursive:true,force:true});} -}); - test('actual portable CLI refuses nonfresh artifacts before entry reads or checks, preserving archives',()=>{ const root=mkdtempSync(join(tmpdir(),'portable-ci-archives-')); try { diff --git a/tests/automation/test-lanes.test.ts b/tests/automation/test-lanes.test.ts new file mode 100644 index 00000000..6f5daea4 --- /dev/null +++ b/tests/automation/test-lanes.test.ts @@ -0,0 +1,259 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import * as lanes from '../../scripts/test-lanes.ts'; + +const moduleUrl = JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href); +const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); +const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; +async function until(condition: () => boolean, ms = 10_000) { + for (const end = Date.now() + ms; Date.now() < end;) { if (condition()) return true; await sleep(25); } + return condition(); +} + +/** One disposable folder holding a step program whose behaviour comes from its environment: + * `ignore` ignores SIGTERM, `hang` runs for ever, `life=N` runs N ms, `code=N` exits with N, + * `grandchild` starts a same-group child that ignores SIGTERM and hangs, `session=...` starts a + * descendant in its own session (outside the step's group) that holds the step's pipes with the + * given modes, `print` and `flood` write output. Each process records its pid under ROOT. */ +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'portable-lanes-')); + writeFileSync(join(root, 'child.cjs'), ` +const fs = require('node:fs'), cp = require('node:child_process'); +const { ROOT, NAME, MODE = '' } = process.env; +const modes = MODE.split(','); +const value = key => (modes.find(m => m.startsWith(key + '=')) ?? '').split('=').slice(1).join('=') || undefined; +if (modes.includes('ignore')) process.on('SIGTERM', () => {}); +fs.writeFileSync(ROOT + '/' + NAME + '.pid', String(process.pid)); +const again = (suffix, mode, options = {}) => cp.spawn(process.execPath, [__filename], { env: { ...process.env, NAME: NAME + '-' + suffix, MODE: mode }, stdio: ['ignore', 'inherit', 'inherit'], ...options }); +if (modes.includes('grandchild')) again('grandchild', 'ignore,hang'); +if (value('session')) again('session', value('session').replaceAll('+', ','), { detached: true }); +if (modes.includes('print')) console.log('PRINTED-BY-' + NAME); +if (modes.includes('flood')) console.log('x'.repeat(5000) + 'END'); +if (modes.includes('hang')) setInterval(() => {}, 1000); +if (value('life')) setTimeout(() => process.exit(Number(value('code') ?? 0)), Number(value('life'))); +else if (!modes.includes('hang')) process.exit(Number(value('code') ?? 0)); +`); + const step = (label: string, mode: string) => + `{label:${JSON.stringify(label)},command:process.execPath,args:[${JSON.stringify(join(root, 'child.cjs'))}],env:{ROOT:${JSON.stringify(root)},NAME:${JSON.stringify(label)},MODE:${JSON.stringify(mode)}}}`; + const pid = (name: string) => (existsSync(join(root, name + '.pid')) ? Number(readFileSync(join(root, name + '.pid'), 'utf8')) : undefined); + return { root, step, pid, dispose: () => rmSync(root, { recursive: true, force: true }) }; +} + +function launch(script: string, env: Record = {}) { + const child = spawn(process.execPath, ['--experimental-strip-types', script], { env: { ...process.env, GITHUB_ACTIONS: '', ...env }, stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; + child.stdout.on('data', c => { out += c; }); + child.stderr.on('data', c => { out += c; }); + const exited = new Promise(done => child.on('close', code => done(code))); + return { child, output: () => out, exited }; +} + +/** A runner script. `${steps}` is written as real code so each test lists its lanes plainly. */ +function program(f: ReturnType, lanesCode: (s: (label: string, mode: string) => string) => string, options = '{}') { + const file = join(f.root, 'run.ts'); + writeFileSync(file, `import * as lanes from ${moduleUrl}; +try { await lanes.runLanes(${lanesCode(f.step)}, ${options}); console.log('RUN-OK'); } +catch (error) { console.log('REJECTED ' + (error as Error).message); process.exit((error as { exitCode?: number }).exitCode ?? 1); }`); + return file; +} +const bounded = async (promise: Promise, ms: number) => Promise.race([promise, sleep(ms).then(() => 'hung' as const)]); + +for (const [signal, code] of [['SIGINT', 130], ['SIGTERM', 143], ['SIGHUP', 129]] as const) { + test(`${signal} stops every step, keeps what they wrote and exits ${code}`, async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'print,hang')}]},{name:'b',steps:[${s('two', 'print,hang')}]}]`)); + assert.ok(await until(() => f.pid('one') !== undefined && f.pid('two') !== undefined), 'both steps started'); + await sleep(300); + const pids = [f.pid('one')!, f.pid('two')!]; + run.child.kill(signal); + assert.equal(await bounded(run.exited, 15_000), code, run.output()); + assert.match(run.output(), new RegExp(`REJECTED Portable check interrupted by ${signal}`)); + assert.match(run.output(), /PRINTED-BY-one/); assert.match(run.output(), /PRINTED-BY-two/); + assert.match(run.output(), /Interrupted before it ended/); + for (const pid of pids) assert.ok(await until(() => !alive(pid), 5000), `step ${pid} outlives the runner`); + } finally { f.dispose(); } + }); +} + +test('a grandchild that ignores SIGTERM and a descendant that holds the pipes from its own session do not hold the runner', async () => { + const f = fixture(); + let session: number | undefined; + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'ignore,hang,grandchild,session=hang')}]}]`, '{killAfterMs:1500}')); + assert.ok(await until(() => ['one', 'one-grandchild', 'one-session'].every(n => f.pid(n) !== undefined)), 'processes started'); + await sleep(300); + session = f.pid('one-session'); + const group = [f.pid('one')!, f.pid('one-grandchild')!]; + const started = Date.now(); + run.child.kill('SIGTERM'); + assert.equal(await bounded(run.exited, 15_000), 143, run.output()); + assert.ok(Date.now() - started < 12_000, 'bounded exit'); + for (const pid of group) assert.ok(await until(() => !alive(pid), 5000), `${pid} outlives the runner`); + } finally { if (session && alive(session)) process.kill(session, 'SIGKILL'); f.dispose(); } +}); + +test('a repeated SIGINT is not swallowed: it kills steps that ignore the first one', async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'ignore,hang,grandchild')}]}]`, '{killAfterMs:60000}')); + assert.ok(await until(() => f.pid('one') !== undefined && f.pid('one-grandchild') !== undefined)); + await sleep(300); + const pids = [f.pid('one')!, f.pid('one-grandchild')!]; + run.child.kill('SIGINT'); + await sleep(400); + assert.ok(alive(pids[0]), 'the first signal alone does not stop a step that ignores it'); + run.child.kill('SIGINT'); + assert.equal(await bounded(run.exited, 10_000), 130, run.output()); + for (const pid of pids) assert.ok(await until(() => !alive(pid), 5000), `${pid} outlives the runner`); + } finally { f.dispose(); } +}); + +test('a closed output pipe stops the steps instead of orphaning them', async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'ignore,hang')}]}]`, '{heartbeatMs:100,killAfterMs:1000}')); + assert.ok(await until(() => f.pid('one') !== undefined)); + const pid = f.pid('one')!; + run.child.stdout!.destroy(); + assert.notEqual(await bounded(run.exited, 15_000), 'hung', 'the runner exits'); + assert.ok(await until(() => !alive(pid), 5000), 'the step outlives the runner'); + } finally { f.dispose(); } +}); + +test('a crash of the runner still kills every group it started', async () => { + const f = fixture(); + try { + const file = join(f.root, 'crash.ts'); + writeFileSync(file, `import {runLanes} from ${moduleUrl}; +const pending = runLanes([{name:'a',steps:[${f.step('one', 'ignore,hang')}]}]).catch(() => {}); +setTimeout(() => { throw new Error('boom'); }, 600); +await pending;`); + const run = launch(file); + assert.ok(await until(() => f.pid('one') !== undefined)); + const pid = f.pid('one')!; + assert.notEqual(await bounded(run.exited, 15_000), 0); + assert.ok(await until(() => !alive(pid), 5000), 'the step outlives the crashed runner'); + } finally { f.dispose(); } +}); + +test('a step that cannot start is reported once', async () => { + const f = fixture(); + try { + const run = launch(program(f, () => `[{name:'a',steps:[{label:'missing',command:'definitely-not-a-command-xyz',args:[]}]}]`)); + assert.equal(await bounded(run.exited, 15_000), 1, run.output()); + const rows = run.output().split('\n').filter(l => /^(passed|failed|stopped) a: missing/.test(l)); + assert.equal(rows.length, 1, run.output()); + assert.match(rows[0], /^failed/); + assert.equal(run.output().split('\n').filter(l => /^\s+(passed|failed|stopped)\s+[\d.]+s\s+a: missing/.test(l)).length, 1); + assert.match(run.output(), /Could not start/); + } finally { f.dispose(); } +}); + +test('a step that fails on its own after the first failure is a failure with its output, not a stopped step', async () => { + const f = fixture(); + try { + // "early" has exited with 4 but a descendant in another session holds its pipes for 1.5 s, + // so its close arrives after "first" has failed. The runner never signalled it. + const run = launch(program(f, s => `[{name:'a',steps:[${s('first', 'life=700,code=3')}]},{name:'b',steps:[${s('early', 'print,code=4,session=life=1500')}]}]`)); + assert.equal(await bounded(run.exited, 15_000), 1, run.output()); + assert.match(run.output(), /^failed a: first/m); + assert.match(run.output(), /^failed b: early/m); + assert.match(run.output(), /PRINTED-BY-early/); + assert.match(run.output(), /^\s+failed\s+[\d.]+s\s+b: early/m); + } finally { f.dispose(); } +}); + +test('a step the runner stopped is "stopped" and prints nothing, even if it exits non-zero in reply', async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('first', 'life=500,code=3')}]},{name:'b',steps:[${s('victim', 'print,hang')}]}]`)); + assert.equal(await bounded(run.exited, 15_000), 1, run.output()); + assert.match(run.output(), /^failed a: first/m); + assert.match(run.output(), /^stopped b: victim/m); + assert.doesNotMatch(run.output(), /PRINTED-BY-victim/); + } finally { f.dispose(); } +}); + +test('the runner names the steps still running about every heartbeat', async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('slow', 'life=900')}]}]`, '{heartbeatMs:150}')); + assert.equal(await bounded(run.exited, 15_000), 0, run.output()); + assert.match(run.output(), /still running: a: slow \(\d+\.\ds\)/); + } finally { f.dispose(); } +}); + +test('a step keeps only the tail of its output and says what was dropped', async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('noisy', 'flood,code=1')}]}]`, '{maxOutputBytes:1000}')); + assert.equal(await bounded(run.exited, 15_000), 1, run.output()); + assert.match(run.output(), /\[\d+ bytes of earlier output dropped\]/); + assert.match(run.output(), /xEND/); + assert.ok((run.output().match(/x/g) ?? []).length < 1500, 'the dropped bytes are not printed'); + } finally { f.dispose(); } +}); + +test('a pass whose leftover process holds the pipes is abandoned after the grace period, not waited for', async () => { + const f = fixture(); + let session: number | undefined; + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'print,session=hang')}]}]`, '{graceMs:500}')); + assert.ok(await until(() => f.pid('one-session') !== undefined)); + session = f.pid('one-session'); + assert.equal(await bounded(run.exited, 10_000), 0, run.output()); + assert.match(run.output(), /^passed a: one/m); + assert.match(run.output(), /held its output open/); + } finally { if (session && alive(session)) process.kill(session, 'SIGKILL'); f.dispose(); } +}); + +test('steps are stopped with a process group on POSIX and with taskkill /T /F on Windows, and only POSIX detaches them', () => { + assert.deepEqual(lanes.stopPlan('linux', 4242, false), { kind: 'group', group: -4242, signal: 'SIGTERM' }); + assert.deepEqual(lanes.stopPlan('darwin', 4242, true), { kind: 'group', group: -4242, signal: 'SIGKILL' }); + for (const force of [false, true]) assert.deepEqual(lanes.stopPlan('win32', 4242, force), { kind: 'command', command: 'taskkill', args: ['/T', '/F', '/PID', '4242'] }); + assert.equal(lanes.detachesSteps('linux'), true); + assert.equal(lanes.detachesSteps('darwin'), true); + assert.equal(lanes.detachesSteps('win32'), false); +}); + +test('lane runner starts a yielding step at lower priority and an ordinary one at the usual priority', async () => { + const f = fixture(); + try { + const file = join(f.root, 'priority.ts'); + writeFileSync(file, `import {runLanes} from ${moduleUrl}; +const show=(label:string,lowPriority?:boolean)=>({label,command:process.execPath,args:['-e',"console.log('PRIORITY '+require('node:os').getPriority())"],lowPriority}); +await runLanes([{name:'a',steps:[show('usual')]},{name:'b',steps:[show('yielding',true)]}]);`); + const run = launch(file); + assert.equal(await bounded(run.exited, 20_000), 0, run.output()); + const seen = (label: string) => Number(new RegExp(`=== ${label} ===\\nPRIORITY (-?\\d+)`).exec(run.output())?.[1]); + assert.ok(seen('b: yielding') > seen('a: usual'), 'the yielding step has the larger niceness'); + } finally { f.dispose(); } +}); + +test('lane runner stops every lane at the first failure and reports it without interleaving', async () => { + const f = fixture(); + try { + const file = join(f.root, 'first-failure.ts'); + writeFileSync(file, `import {runLanes} from ${moduleUrl}; +const node=(label:string,code:string)=>({label,command:process.execPath,args:['-e',code]}); +try{await runLanes([ + {name:'slow',steps:[node('hangs',"setTimeout(()=>{},60000)"),node('never starts',"console.log('LATE-STEP')")]}, + {name:'fails',steps:[node('breaks',"console.log('FAIL-OUTPUT');process.exit(3)")]}, + {name:'ok',steps:[node('passes',"console.log('PASS-OUTPUT')"),node('skipped after failure',"setTimeout(()=>console.log('LATE-STEP'),5000)")]}, +]);}catch(error){console.log('REJECTED '+(error as Error).message);process.exit(1);}`); + const started = performance.now(); + const run = launch(file); + assert.equal(await bounded(run.exited, 20_000), 1); + assert.ok(performance.now() - started < 15_000, 'the hanging lane is stopped, not waited for'); + assert.match(run.output(), /FAIL-OUTPUT/); assert.match(run.output(), /Exited with code 3/); assert.match(run.output(), /REJECTED Portable check failed/); + assert.doesNotMatch(run.output(), /LATE-STEP/); + assert.match(run.output(), /stopped\s+[\d.]+s\s+slow: hangs/); + assert.match(run.output(), /failed\s+[\d.]+s\s+fails: breaks/); + } finally { f.dispose(); } +}); diff --git a/tests/automation/workflow-triggers.test.ts b/tests/automation/workflow-triggers.test.ts index 3b8d1b39..eb08ca7f 100644 --- a/tests/automation/workflow-triggers.test.ts +++ b/tests/automation/workflow-triggers.test.ts @@ -199,9 +199,13 @@ test('Windows selects fast PR/push checks and full release, weekly and manual jo assert.match(windows, /name: Fetch released Windows migration fixture\n if: env.AP_WINDOWS_FULL == '1'\n run: git fetch --no-tags origin tag v1\.1\.1/); assert.match(windows, /\$runs = @\('windows-platform', 'windows-service'\)/); assert.match(windows, /--test-concurrency=1 tests\/backend\/\$\(\$run\.File\)\.test\.ts/); - assert.ok(!windows.includes(' -Wait '), 'both native files are waited for before either result is read'); - assert.match(windows, /foreach \(\$run in \$runs\) \{ \$run\.Process\.WaitForExit\(\) \}/); - assert.match(windows, /Where-Object \{ \$_\.Process\.ExitCode -ne 0 \}/); + // Each file runs under Start-Process -Wait, which waits for the whole process tree, inside its own thread job. + assert.match(windows, /Start-ThreadJob[^\n]*-ScriptBlock \{/); + assert.match(windows, /-WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput \$run\.Output -RedirectStandardError \$run\.Errors/); + assert.ok(!windows.includes('WaitForExit'), 'a bare process wait would not wait for what the process started'); + assert.match(windows, /\$null = \$runs\.Job \| Wait-Job/); + assert.match(windows, /Receive-Job -Job \$run\.Job -ErrorAction Stop/); + assert.match(windows, /Where-Object \{ \$_\.ExitCode -ne 0 \}/); const root = mkdtempSync(join(tmpdir(), 'workflow-windows-')); const git = (...args: string[]) => { From 3c73f892d8a3cd18b7eb74a76582739a9149307f Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:20:39 +1100 Subject: [PATCH 11/15] ci: retry a refused simultaneous start of the two Windows test processes The secondary logon service creates one process at a time and refused the second Start-Process -Credential when both thread jobs started at once. Retry only that refusal, up to thirty times a second apart. --- .github/workflows/ci.yml | 15 +++++++++++++-- tests/automation/workflow-triggers.test.ts | 5 ++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 004cb4fc..68b61b87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,7 +224,8 @@ jobs: # which are what take Setup coordination ports; the platform file # drives the setup controller directly. Each runs under Start-Process # -Wait inside a thread job, so it is waited for with everything it - # started, exactly as when they ran one after the other. + # started, exactly as when they ran one after the other. Starting the + # two at the same instant can be refused once, so the start is retried. $runs = @('windows-platform', 'windows-service') | ForEach-Object { [pscustomobject]@{ File = $_ @@ -245,7 +246,17 @@ jobs: foreach ($run in $runs) { $run.Job = Start-ThreadJob -ArgumentList $run, $credential, $env:GITHUB_WORKSPACE, (Get-Command node).Source -ScriptBlock { param($run, $credential, $workspace, $node) - $process = Start-Process -FilePath $node -ArgumentList "--experimental-strip-types --test --test-concurrency=1 tests/backend/$($run.File).test.ts" -WorkingDirectory $workspace -Credential $credential -LoadUserProfile -WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput $run.Output -RedirectStandardError $run.Errors + # The secondary logon service creates one process at a time and + # refuses a second request while it is busy; retry only that refusal. + $process = $null + for ($attempt = 1; -not $process; $attempt++) { + try { + $process = Start-Process -FilePath $node -ArgumentList "--experimental-strip-types --test --test-concurrency=1 tests/backend/$($run.File).test.ts" -WorkingDirectory $workspace -Credential $credential -LoadUserProfile -WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput $run.Output -RedirectStandardError $run.Errors -ErrorAction Stop + } catch { + if ($attempt -ge 30 -or $_.Exception.Message -notmatch 'already running') { throw } + Start-Sleep -Seconds 1 + } + } $process.ExitCode } } diff --git a/tests/automation/workflow-triggers.test.ts b/tests/automation/workflow-triggers.test.ts index eb08ca7f..67567873 100644 --- a/tests/automation/workflow-triggers.test.ts +++ b/tests/automation/workflow-triggers.test.ts @@ -201,7 +201,10 @@ test('Windows selects fast PR/push checks and full release, weekly and manual jo assert.match(windows, /--test-concurrency=1 tests\/backend\/\$\(\$run\.File\)\.test\.ts/); // Each file runs under Start-Process -Wait, which waits for the whole process tree, inside its own thread job. assert.match(windows, /Start-ThreadJob[^\n]*-ScriptBlock \{/); - assert.match(windows, /-WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput \$run\.Output -RedirectStandardError \$run\.Errors/); + assert.match(windows, /-WindowStyle Hidden -Wait -PassThru -RedirectStandardOutput \$run\.Output -RedirectStandardError \$run\.Errors -ErrorAction Stop/); + // The secondary logon service refuses a second simultaneous start once; only that refusal is retried. + assert.match(windows, /\$_\.Exception\.Message -notmatch 'already running'\) \{ throw \}/); + assert.match(windows, /\$attempt -ge 30/); assert.ok(!windows.includes('WaitForExit'), 'a bare process wait would not wait for what the process started'); assert.match(windows, /\$null = \$runs\.Job \| Wait-Job/); assert.match(windows, /Receive-Job -Job \$run\.Job -ErrorAction Stop/); From 4ada2045fe947cb7427d2d2c28cf1c3d53e014fc Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:47:30 +1100 Subject: [PATCH 12/15] test: stop every recorded group when the runner returns, passing or not A leader that exits 0 while a same-group descendant keeps a pipe open let the grace period abandon the pipes and forget the group, so the descendant (and any Setup port it held) outlived the run, and an interrupt during the grace period did the same. Keep every group recorded whatever the leader's exit status, stop it (SIGTERM, then SIGKILL after a short bound) before runLanes returns, and signal the group of a leader that has already exited when the run is stopped. Skip the process-group and signal cases of the runner tests on Windows, and make every case stop the fixture processes it recorded, not only delete its files. --- CONTRIBUTING.md | 6 ++- openspec/changes/release-1-5-2/tasks.md | 2 +- scripts/test-lanes.ts | 30 +++++++++--- tests/automation/test-lanes.test.ts | 65 ++++++++++++++++++++----- 4 files changed, 82 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 793692ce..9c2b2ec0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,13 +127,15 @@ SIGTERM, SIGHUP or a closed output pipe (`test:all | head`) stops both lanes and running steps had written. Each step runs in its own process group, so the runner stops the groups itself: a repeated signal, or a step that has not stopped ten seconds after the first, is killed outright and abandoned even if a leftover process still holds its output pipes, and every group -the runner started is killed once the steps settle and again if the runner itself crashes. An +the runner started is stopped (SIGTERM, then SIGKILL after a short bound) once the steps settle, whether they passed or not, because a leader that exits 0 can leave descendants holding a port, and killed again if the runner itself crashes. An interrupted run exits 128 plus the signal number (130, 143, 129, 141), a failed one 1. A step the runner stopped is reported as stopped; one that fails on its own is reported as failed with its output, whenever it does so. On Windows steps are not detached and are stopped with `taskkill /T /F`; that path is covered by a unit test of the stop command but has not been run on Windows, and the typecheck step's `./node_modules/.bin/tsc` is a POSIX path, so on Windows run the -individual commands listed under Local development by hand. +individual commands listed under Local development by hand. The process-group and signal cases of +`tests/automation/test-lanes.test.ts` are skipped on Windows (its stop-command case runs), and every case +stops the processes it started even when it fails. Both runners also turn on Node's compile cache for every step they start (`NODE_COMPILE_CACHE`, a folder under the ignored `node_modules/.cache/`). It keeps diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index dc4256cc..b5c6c9c2 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -67,7 +67,7 @@ Each numbered group is one pull request into the development branch. It carries ## 9. Faster test suites - [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. -- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT, SIGTERM, SIGHUP or a closed output pipe, keeping the output of steps still running and killing every group the runner started (including after a crash or a repeated signal), exit with 128 plus the signal number when interrupted, tail-limit and heartbeat the step output, mark a step the runner stopped as stopped and one that fails by itself as failed, and keep the second build comparison and the package checks after all tests. Show with a preload probe on the listen call, covering worker threads, child processes and the bundled service, that the parallel backend group and the browser and automation tests take no Setup port, and fence the one file that does when `CP_BUILT_SERVICE=1` is set. Give the plan a core count so one to three cores are tested. +- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT, SIGTERM, SIGHUP or a closed output pipe, keeping the output of steps still running and stopping every group the runner started once the steps settle, passed or not (and killing them after a crash or a repeated signal), exit with 128 plus the signal number when interrupted, tail-limit and heartbeat the step output, mark a step the runner stopped as stopped and one that fails by itself as failed, and keep the second build comparison and the package checks after all tests. Show with a preload probe on the listen call, covering worker threads, child processes and the bundled service, that the parallel backend group and the browser and automation tests take no Setup port, and fence the one file that does when `CP_BUILT_SERVICE=1` is set. Give the plan a core count so one to three cores are tested. - [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. - [x] 9.4 Turn on Node's compile cache, in a folder under the ignored dependency folder, for every step of the CI and local runners. It checks each cached file against the source's hash, so no test input changes. Record its effect from CI step timings and from the heaviest managed-step test run alone, and say that the typecheck times did not move. - [x] 9.5 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts index ace9d735..117c447f 100644 --- a/scripts/test-lanes.ts +++ b/scripts/test-lanes.ts @@ -9,8 +9,9 @@ * left running would keep holding the Setup ports the next run needs. A repeated * signal, or a step that ignores the first one for too long, is killed outright * and abandoned even if its output pipes are still held open; every group the - * runner started is killed again once the steps settle and when the process - * exits, however it exits. On Windows steps are not detached and are stopped with + * runner started is stopped again (SIGTERM, then SIGKILL) once the steps settle, + * passing or not, since a leader that exits 0 can leave descendants holding + * ports, and again when the process exits, however it exits. On Windows steps are not detached and are stopped with * `taskkill /T /F`; that path is untested on Windows. */ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; @@ -28,6 +29,8 @@ export type RunOptions = { graceMs?: number; /** The most output kept per step; the earliest is dropped first. */ maxOutputBytes?: number; + /** How long the closing sweep waits for a group to leave after SIGTERM, and again after SIGKILL. */ + sweepMs?: number; }; const SIGNAL_NUMBER: Record = { SIGHUP: 1, SIGINT: 2, SIGPIPE: 13, SIGTERM: 15 }; @@ -51,6 +54,13 @@ export function stopPlan(platform: NodeJS.Platform, pid: number, force: boolean) /** Steps get their own process group everywhere but Windows, which has none to use. */ export const detachesSteps = (platform: NodeJS.Platform) => platform !== 'win32'; +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +/** Whether anything is left in the step's group. Only POSIX has groups to look in. */ +function groupAlive(pid: number) { + if (process.platform === 'win32') return false; + try { process.kill(-pid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code === 'EPERM'; } +} + function stop(pid: number, force: boolean) { const plan = stopPlan(process.platform, pid, force); try { @@ -94,7 +104,7 @@ function print(label: string, text: string, ok: boolean) { type Live = { label: string; started: number; child: ChildProcess; signalled: boolean; printed: boolean; dump: () => void; force: (note: string) => void }; export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise { - const { heartbeatMs = 60_000, killAfterMs = 10_000, graceMs = 3_000, maxOutputBytes = 2 * 1024 * 1024 } = options; + const { heartbeatMs = 60_000, killAfterMs = 10_000, graceMs = 3_000, maxOutputBytes = 2 * 1024 * 1024, sweepMs = 2_000 } = options; const results: StepResult[] = []; const groups = new Set(); const live = new Map(); @@ -109,8 +119,9 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise }; const beginStop = () => { for (const [pid, step] of live) { - // A step that has already exited on its own is not ours to call stopped. - if (step.child.exitCode === null && step.child.signalCode === null) { step.signalled = true; stop(pid, false); } + // A leader that has exited on its own is not ours to call stopped, but its group may still be running. + if (step.child.exitCode === null && step.child.signalCode === null) step.signalled = true; + stop(pid, false); } // Not unref'd: steps are still running, and the run must outlast them. killTimer ??= setTimeout(escalate, killAfterMs); @@ -163,7 +174,6 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise settled = true; clearTimeout(grace); if (pid) { live.delete(pid); - if (ok) groups.delete(pid); // a clean pass leaves nothing to sweep if (!live.size) { clearTimeout(killTimer); killTimer = undefined; } } const seconds = (performance.now() - started) / 1000; @@ -193,7 +203,13 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise })); } finally { clearInterval(heartbeat); clearTimeout(killTimer); - if (failed || interrupted) for (const pid of groups) stop(pid, true); + // However each step ended, and whatever its leader's exit status, nothing it started may outlive the + // run: a descendant still holding a Setup port would refuse the next one. ESRCH just means it is gone. + const left = [...groups]; + for (const pid of left) stop(pid, false); + for (const end = Date.now() + sweepMs; Date.now() < end && left.some(groupAlive);) await sleep(25); + for (const pid of left) if (groupAlive(pid)) stop(pid, true); + for (const end = Date.now() + sweepMs; Date.now() < end && left.some(groupAlive);) await sleep(25); groups.clear(); for (const [signal, handler] of handlers) process.off(signal, handler); process.stdout.off('error', onStdoutError); diff --git a/tests/automation/test-lanes.test.ts b/tests/automation/test-lanes.test.ts index 6f5daea4..6b23bcf2 100644 --- a/tests/automation/test-lanes.test.ts +++ b/tests/automation/test-lanes.test.ts @@ -1,14 +1,23 @@ -import { test } from 'node:test'; +import { test, after } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { mkdtempSync, writeFileSync, readFileSync, readdirSync, existsSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { spawn } from 'node:child_process'; +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import * as lanes from '../../scripts/test-lanes.ts'; +// The runner is POSIX-first: its process groups and the catchable SIGINT, SIGHUP and SIGTERM these tests send +// do not exist on Windows, where ChildProcess.kill() cannot deliver them. The stop-command test below runs everywhere. +const posixOnly = { skip: process.platform === 'win32' && 'process groups and catchable signals are POSIX-only' }; const moduleUrl = JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href); const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); +const killTree = (pid: number) => { + if (process.platform === 'win32') spawnSync('taskkill', ['/T', '/F', '/PID', String(pid)], { stdio: 'ignore' }); + else { try { process.kill(-pid, 'SIGKILL'); } catch { /* not a group leader */ } try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } } +}; +const launched = new Set(); +after(() => { for (const child of launched) if (child.pid) killTree(child.pid); }); const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; async function until(condition: () => boolean, ms = 10_000) { for (const end = Date.now() + ms; Date.now() < end;) { if (condition()) return true; await sleep(25); } @@ -41,11 +50,17 @@ else if (!modes.includes('hang')) process.exit(Number(value('code') ?? 0)); const step = (label: string, mode: string) => `{label:${JSON.stringify(label)},command:process.execPath,args:[${JSON.stringify(join(root, 'child.cjs'))}],env:{ROOT:${JSON.stringify(root)},NAME:${JSON.stringify(label)},MODE:${JSON.stringify(mode)}}}`; const pid = (name: string) => (existsSync(join(root, name + '.pid')) ? Number(readFileSync(join(root, name + '.pid'), 'utf8')) : undefined); - return { root, step, pid, dispose: () => rmSync(root, { recursive: true, force: true }) }; + // Every fixture process records its pid here: stop whatever a failed test left running, then delete the files. + const dispose = () => { + for (const name of readdirSync(root)) if (name.endsWith('.pid')) killTree(Number(readFileSync(join(root, name), 'utf8'))); + rmSync(root, { recursive: true, force: true }); + }; + return { root, step, pid, dispose }; } function launch(script: string, env: Record = {}) { const child = spawn(process.execPath, ['--experimental-strip-types', script], { env: { ...process.env, GITHUB_ACTIONS: '', ...env }, stdio: ['ignore', 'pipe', 'pipe'] }); + launched.add(child); let out = ''; child.stdout.on('data', c => { out += c; }); child.stderr.on('data', c => { out += c; }); @@ -64,7 +79,7 @@ catch (error) { console.log('REJECTED ' + (error as Error).message); process.exi const bounded = async (promise: Promise, ms: number) => Promise.race([promise, sleep(ms).then(() => 'hung' as const)]); for (const [signal, code] of [['SIGINT', 130], ['SIGTERM', 143], ['SIGHUP', 129]] as const) { - test(`${signal} stops every step, keeps what they wrote and exits ${code}`, async () => { + test(`${signal} stops every step, keeps what they wrote and exits ${code}`, posixOnly, async () => { const f = fixture(); try { const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'print,hang')}]},{name:'b',steps:[${s('two', 'print,hang')}]}]`)); @@ -81,7 +96,7 @@ for (const [signal, code] of [['SIGINT', 130], ['SIGTERM', 143], ['SIGHUP', 129] }); } -test('a grandchild that ignores SIGTERM and a descendant that holds the pipes from its own session do not hold the runner', async () => { +test('a grandchild that ignores SIGTERM and a descendant that holds the pipes from its own session do not hold the runner', posixOnly, async () => { const f = fixture(); let session: number | undefined; try { @@ -98,7 +113,35 @@ test('a grandchild that ignores SIGTERM and a descendant that holds the pipes fr } finally { if (session && alive(session)) process.kill(session, 'SIGKILL'); f.dispose(); } }); -test('a repeated SIGINT is not swallowed: it kills steps that ignore the first one', async () => { +test('a leader that exits 0 leaving a same-group descendant on the pipes does not leave the descendant running when the runner returns', posixOnly, async () => { + const f = fixture(); + try { + // The leader exits at once; its grandchild (same group, ignores SIGTERM) holds the pipes. The grace period + // abandons the pipes, and the step passes, but the group is still alive. + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'grandchild')}]}]`, '{graceMs:400,sweepMs:600}')); + assert.ok(await until(() => f.pid('one-grandchild') !== undefined)); + const descendant = f.pid('one-grandchild')!; + assert.equal(await bounded(run.exited, 15_000), 0, run.output()); + assert.match(run.output(), /^passed a: one/m); + assert.ok(await until(() => !alive(descendant), 3000), 'the descendant outlives a passing run'); + } finally { f.dispose(); } +}); + +test('an interrupt during the grace period, which ends before the kill timer, still stops the descendant of a leader that already exited', posixOnly, async () => { + const f = fixture(); + try { + const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'grandchild')}]}]`, '{graceMs:1500,killAfterMs:30000,sweepMs:600}')); + assert.ok(await until(() => f.pid('one-grandchild') !== undefined)); + const descendant = f.pid('one-grandchild')!; + await sleep(300); + assert.ok(alive(descendant), 'the grace period is still running'); + run.child.kill('SIGTERM'); + assert.equal(await bounded(run.exited, 15_000), 143, run.output()); + assert.ok(await until(() => !alive(descendant), 3000), 'the descendant outlives the interrupted run'); + } finally { f.dispose(); } +}); + +test('a repeated SIGINT is not swallowed: it kills steps that ignore the first one', posixOnly, async () => { const f = fixture(); try { const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'ignore,hang,grandchild')}]}]`, '{killAfterMs:60000}')); @@ -114,7 +157,7 @@ test('a repeated SIGINT is not swallowed: it kills steps that ignore the first o } finally { f.dispose(); } }); -test('a closed output pipe stops the steps instead of orphaning them', async () => { +test('a closed output pipe stops the steps instead of orphaning them', posixOnly, async () => { const f = fixture(); try { const run = launch(program(f, s => `[{name:'a',steps:[${s('one', 'ignore,hang')}]}]`, '{heartbeatMs:100,killAfterMs:1000}')); @@ -126,7 +169,7 @@ test('a closed output pipe stops the steps instead of orphaning them', async () } finally { f.dispose(); } }); -test('a crash of the runner still kills every group it started', async () => { +test('a crash of the runner still kills every group it started', posixOnly, async () => { const f = fixture(); try { const file = join(f.root, 'crash.ts'); @@ -200,7 +243,7 @@ test('a step keeps only the tail of its output and says what was dropped', async } finally { f.dispose(); } }); -test('a pass whose leftover process holds the pipes is abandoned after the grace period, not waited for', async () => { +test('a pass whose leftover process holds the pipes is abandoned after the grace period, not waited for', posixOnly, async () => { const f = fixture(); let session: number | undefined; try { @@ -222,7 +265,7 @@ test('steps are stopped with a process group on POSIX and with taskkill /T /F on assert.equal(lanes.detachesSteps('win32'), false); }); -test('lane runner starts a yielding step at lower priority and an ordinary one at the usual priority', async () => { +test('lane runner starts a yielding step at lower priority and an ordinary one at the usual priority', posixOnly, async () => { const f = fixture(); try { const file = join(f.root, 'priority.ts'); From 985805e810369876b425b69f94a0be3b858ae7af Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:04:57 +1100 Subject: [PATCH 13/15] test: retire process groups when empty and guard every pid the runner or its tests signal The runner kept the id of a passing step's group until both lanes finished, so a reused id could be signalled by the closing sweep (and on Windows taskkill could hit an exited PID). Sweep each step's group right after its leader exits and its pipes close, retire a group the moment a probe finds it empty, re-probe before every signal, never signal a retired group, and leave only unconfirmed groups to the closing sweep and the exit hook. On Windows run taskkill only against a child whose exit has not fired. Route every probe and signal through an injectable kill function and refuse a pid that is not a whole number above 1, itself or its parent. In the tests, write pid files atomically and have the cleanup accept only a safe pid, so an empty file (Number('') is 0, the whole process group) is ignored. --- CONTRIBUTING.md | 9 ++- openspec/changes/release-1-5-2/tasks.md | 2 +- scripts/test-lanes.ts | 93 ++++++++++++++++--------- tests/automation/test-lanes.test.ts | 77 ++++++++++++++++++-- 4 files changed, 144 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c2b2ec0..47d9a953 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,7 +127,14 @@ SIGTERM, SIGHUP or a closed output pipe (`test:all | head`) stops both lanes and running steps had written. Each step runs in its own process group, so the runner stops the groups itself: a repeated signal, or a step that has not stopped ten seconds after the first, is killed outright and abandoned even if a leftover process still holds its output pipes, and every group -the runner started is stopped (SIGTERM, then SIGKILL after a short bound) once the steps settle, whether they passed or not, because a leader that exits 0 can leave descendants holding a port, and killed again if the runner itself crashes. An +the runner started is swept as soon as its leader has exited and its pipes have closed (SIGTERM, a short +bound, SIGKILL, a short bound, then a probe that finds the group empty), whether the step passed or not, +because a leader that exits 0 can leave descendants holding a port. A group leaves the record the moment a +probe finds it empty and is never signalled afterwards, since its id may then belong to an unrelated +process; every signal is preceded by a fresh probe, and the closing sweep and the exit hook touch only +groups not yet confirmed empty. Group ids are checked before use (a whole number above 1 that is neither +the runner nor its parent), and a crash of the runner kills what is still recorded. On Windows `taskkill` +goes only to a child whose `exit` has not fired. An interrupted run exits 128 plus the signal number (130, 143, 129, 141), a failed one 1. A step the runner stopped is reported as stopped; one that fails on its own is reported as failed with its output, whenever it does so. On Windows steps are not detached and are stopped with diff --git a/openspec/changes/release-1-5-2/tasks.md b/openspec/changes/release-1-5-2/tasks.md index b5c6c9c2..a090f5b2 100644 --- a/openspec/changes/release-1-5-2/tasks.md +++ b/openspec/changes/release-1-5-2/tasks.md @@ -67,7 +67,7 @@ Each numbered group is one pull request into the development branch. It carries ## 9. Faster test suites - [x] 9.1 Measure where the portable run spends its time before changing anything: the serial backend group that acquires Setup coordination ports is about two thirds of the Linux step, and one file in it is most of that group. Record the per-step durations of recent CI runs and the per-file durations of that group. -- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT, SIGTERM, SIGHUP or a closed output pipe, keeping the output of steps still running and stopping every group the runner started once the steps settle, passed or not (and killing them after a crash or a repeated signal), exit with 128 plus the signal number when interrupted, tail-limit and heartbeat the step output, mark a step the runner stopped as stopped and one that fails by itself as failed, and keep the second build comparison and the package checks after all tests. Show with a preload probe on the listen call, covering worker threads, child processes and the bundled service, that the parallel backend group and the browser and automation tests take no Setup port, and fence the one file that does when `CP_BUILT_SERVICE=1` is set. Give the plan a core count so one to three cores are tested. +- [x] 9.2 Run the serial Setup-port backend group beside the typechecks, the parallel backend group and the browser-model and automation tests instead of before them, in portable CI and in a new local runner that shares one plan with it. Keep the serial group as one `--test-concurrency=1` invocation that nothing else joins, cap the other lane at the machine's cores minus two, print each step's output whole under its own heading, stop every lane at the first failure or on SIGINT, SIGTERM, SIGHUP or a closed output pipe, keeping the output of steps still running and sweeping each step's group as soon as its leader has exited and its pipes closed, passed or not, retiring a group the moment it is empty and never signalling it afterwards, and sweeping only unconfirmed groups at the end (and killing them after a crash or a repeated signal), exit with 128 plus the signal number when interrupted, tail-limit and heartbeat the step output, mark a step the runner stopped as stopped and one that fails by itself as failed, and keep the second build comparison and the package checks after all tests. Show with a preload probe on the listen call, covering worker threads, child processes and the bundled service, that the parallel backend group and the browser and automation tests take no Setup port, and fence the one file that does when `CP_BUILT_SERVICE=1` is set. Give the plan a core count so one to three cores are tested. - [x] 9.3 Add `bun run test:fast` (both typechecks, the parallel backend group, browser-model and automation tests) and `bun run test:all` (everything portable CI tests except packaging and the rebuild comparison), document them in the contributor guide, and cover the plan and the runner with automation tests: every portable file runs once, the serial group is alone and serial, and a failure stops the other lane. - [x] 9.4 Turn on Node's compile cache, in a folder under the ignored dependency folder, for every step of the CI and local runners. It checks each cached file against the source's hash, so no test input changes. Record its effect from CI step timings and from the heaviest managed-step test run alone, and say that the typecheck times did not move. - [x] 9.5 Skip installing the ZIP tools in the two Linux CI jobs when the runner image already has them. Keep every job and check name, the secrets scan and the second build unchanged. diff --git a/scripts/test-lanes.ts b/scripts/test-lanes.ts index 117c447f..261a2337 100644 --- a/scripts/test-lanes.ts +++ b/scripts/test-lanes.ts @@ -9,9 +9,12 @@ * left running would keep holding the Setup ports the next run needs. A repeated * signal, or a step that ignores the first one for too long, is killed outright * and abandoned even if its output pipes are still held open; every group the - * runner started is stopped again (SIGTERM, then SIGKILL) once the steps settle, - * passing or not, since a leader that exits 0 can leave descendants holding - * ports, and again when the process exits, however it exits. On Windows steps are not detached and are stopped with + * runner started is swept as soon as its leader has exited and its pipes have + * closed, passing or not, since a leader that exits 0 can leave descendants + * holding ports. A group is retired the moment a probe finds it empty and never + * signalled afterwards (its id may then belong to someone else); every signal is + * preceded by a probe, and the exit hook and closing sweep touch only groups not + * yet confirmed empty. On Windows steps are not detached and are stopped with * `taskkill /T /F`; that path is untested on Windows. */ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; @@ -29,10 +32,14 @@ export type RunOptions = { graceMs?: number; /** The most output kept per step; the earliest is dropped first. */ maxOutputBytes?: number; - /** How long the closing sweep waits for a group to leave after SIGTERM, and again after SIGKILL. */ + /** How long a group sweep waits for the group to leave after SIGTERM, and again after SIGKILL. */ sweepMs?: number; + /** Replaces process.kill for every probe and signal sent to a process group; for tests. */ + kill?: Kill; }; +export type Kill = (pid: number, signal: NodeJS.Signals | 0) => void; + const SIGNAL_NUMBER: Record = { SIGHUP: 1, SIGINT: 2, SIGPIPE: 13, SIGTERM: 15 }; /** `exitCode` is 1 for a failed run and 128 plus the signal number for an interrupted one. */ @@ -55,18 +62,19 @@ export function stopPlan(platform: NodeJS.Platform, pid: number, force: boolean) export const detachesSteps = (platform: NodeJS.Platform) => platform !== 'win32'; const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); -/** Whether anything is left in the step's group. Only POSIX has groups to look in. */ -function groupAlive(pid: number) { - if (process.platform === 'win32') return false; - try { process.kill(-pid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code === 'EPERM'; } + +/** A pid or group id the runner may signal: a whole number above 1 that is neither this process nor its + * parent. Zero would be this process's own group, a negative number another group, and 1 is init. */ +export function validPid(pid: unknown): pid is number { + return typeof pid === 'number' && Number.isInteger(pid) && pid > 1 && pid !== process.pid && pid !== process.ppid; } -function stop(pid: number, force: boolean) { - const plan = stopPlan(process.platform, pid, force); - try { - if (plan.kind === 'group') process.kill(plan.group, plan.signal); - else spawnSync(plan.command, plan.args, { stdio: 'ignore', windowsHide: true }); - } catch { /* already gone */ } +/** What to run to stop a step, if anything. On Windows `taskkill /T /F` goes only to a child whose `exit` has + * not fired: its PID may have been reused since. On POSIX a group is signalled only after a fresh probe. */ +export function stopAction(platform: NodeJS.Platform, exited: boolean, pid: number, force: boolean): StopPlan | undefined { + if (!validPid(pid)) return undefined; + if (platform === 'win32' && exited) return undefined; + return stopPlan(platform, pid, force); } /** The last `max` bytes written, with a note of what was dropped. */ @@ -101,27 +109,51 @@ function print(label: string, text: string, ok: boolean) { process.stdout.write(github ? `::group::${label}\n${body}::endgroup::\n` : `\n=== ${label} ${ok ? '' : '(failed) '}===\n${body}`); } -type Live = { label: string; started: number; child: ChildProcess; signalled: boolean; printed: boolean; dump: () => void; force: (note: string) => void }; +type Live = { label: string; started: number; child: ChildProcess; exited: boolean; signalled: boolean; printed: boolean; dump: () => void; force: (note: string) => void }; export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise { const { heartbeatMs = 60_000, killAfterMs = 10_000, graceMs = 3_000, maxOutputBytes = 2 * 1024 * 1024, sweepMs = 2_000 } = options; + const kill: Kill = options.kill ?? ((pid, signal) => { process.kill(pid, signal); }); const results: StepResult[] = []; + // Groups not yet known to be empty. A group leaves as soon as a probe finds it empty, because its id may + // then be reused by an unrelated process; nothing is ever signalled once it has left. const groups = new Set(); const live = new Map(); let failed = false; let interrupted: { signal: string; code: number } | undefined; let killTimer: NodeJS.Timeout | undefined; + const groupAlive = (group: number) => { + try { kill(-group, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code === 'EPERM'; } + }; + /** Signal a recorded group, probing it first; an empty group is retired instead. */ + const signalGroup = (group: number, signal: 'SIGTERM' | 'SIGKILL') => { + if (!groups.has(group) || !validPid(group)) return; + if (!groupAlive(group)) { groups.delete(group); return; } + try { kill(-group, signal); } catch { /* gone between probe and signal */ } + }; + const stopStep = (pid: number, step: Live, force: boolean) => { + const plan = stopAction(process.platform, step.exited, pid, force); + if (plan?.kind === 'group') signalGroup(pid, plan.signal); + else if (plan?.kind === 'command') spawnSync(plan.command, plan.args, { stdio: 'ignore', windowsHide: true }); + }; + /** TERM, a short bound, KILL, a short bound; retire the group once it is confirmed empty. */ + const sweepGroup = async (group: number) => { + signalGroup(group, 'SIGTERM'); + for (const end = Date.now() + sweepMs; groups.has(group) && Date.now() < end;) { if (groupAlive(group)) await sleep(25); else groups.delete(group); } + signalGroup(group, 'SIGKILL'); + for (const end = Date.now() + sweepMs; groups.has(group) && Date.now() < end;) { if (groupAlive(group)) await sleep(25); else groups.delete(group); } + }; const escalate = () => { clearTimeout(killTimer); killTimer = undefined; - for (const [pid, step] of live) { step.signalled = true; stop(pid, true); } + for (const [pid, step] of live) { step.signalled = true; stopStep(pid, step, true); } for (const step of [...live.values()]) step.force('Did not stop in time; killed and abandoned.\n'); }; const beginStop = () => { for (const [pid, step] of live) { // A leader that has exited on its own is not ours to call stopped, but its group may still be running. - if (step.child.exitCode === null && step.child.signalCode === null) step.signalled = true; - stop(pid, false); + if (!step.exited && step.child.exitCode === null && step.child.signalCode === null) step.signalled = true; + stopStep(pid, step, false); } // Not unref'd: steps are still running, and the run must outlast them. killTimer ??= setTimeout(escalate, killAfterMs); @@ -138,7 +170,10 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise const onStdoutError = (error: NodeJS.ErrnoException) => { if (error.code === 'EPIPE') onSignal('SIGPIPE'); }; process.stdout.on('error', onStdoutError); // A crash or process.exit must not orphan a group either. - const onExit = () => { for (const pid of groups) stop(pid, true); }; + const onExit = () => { + for (const pid of [...groups]) signalGroup(pid, 'SIGKILL'); + for (const [pid, step] of live) stopStep(pid, step, true); + }; process.on('exit', onExit); const heartbeat = setInterval(() => { const names = [...live.values()].map(step => `${step.label} (${stamp((performance.now() - step.started) / 1000)})`); @@ -159,11 +194,11 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise let exit: { code: number | null; signal: NodeJS.Signals | null } | undefined; let grace: NodeJS.Timeout | undefined; const entry: Live = { - label, started, child, signalled: false, printed: false, + label, started, child, exited: false, signalled: false, printed: false, dump: () => { entry.printed = true; print(label, `${tail.text()}Interrupted before it ended.\n`, false); }, force: note => { child.stdout?.destroy(); child.stderr?.destroy(); finish(exit?.code === 0, note); }, }; - if (pid) { groups.add(pid); live.set(pid, entry); } + if (pid) { live.set(pid, entry); if (detachesSteps(process.platform)) groups.add(pid); } child.stdout?.on('data', chunk => tail.push(chunk)); child.stderr?.on('data', chunk => tail.push(chunk)); child.stdout?.on('error', () => {}); @@ -183,11 +218,13 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise if ((status === 'passed' || status === 'failed') && !entry.printed) print(label, tail.text() + note, ok); console.log(`${status} ${label} (${stamp(seconds)})`); results.push({ lane: lane.name, label: step.label, seconds, status }); - resolve(ok); + // Sweep the group now that its leader is gone and the pipes are closed, so a descendant (and a Setup + // port it holds) cannot outlive the step, and the group id is retired while it is still ours. + void (pid ? sweepGroup(pid) : Promise.resolve()).then(() => resolve(ok)); }; child.on('error', error => finish(false, `Could not start: ${error.message}\n`)); child.on('exit', (code, signal) => { - exit = { code, signal }; + exit = { code, signal }; entry.exited = true; // A leftover process holding our pipes must not keep the step open for ever. grace = setTimeout(() => entry.force('A leftover process held its output open; abandoned.\n'), graceMs); }); @@ -203,14 +240,8 @@ export async function runLanes(lanes: Lane[], options: RunOptions = {}): Promise })); } finally { clearInterval(heartbeat); clearTimeout(killTimer); - // However each step ended, and whatever its leader's exit status, nothing it started may outlive the - // run: a descendant still holding a Setup port would refuse the next one. ESRCH just means it is gone. - const left = [...groups]; - for (const pid of left) stop(pid, false); - for (const end = Date.now() + sweepMs; Date.now() < end && left.some(groupAlive);) await sleep(25); - for (const pid of left) if (groupAlive(pid)) stop(pid, true); - for (const end = Date.now() + sweepMs; Date.now() < end && left.some(groupAlive);) await sleep(25); - groups.clear(); + // Only groups not yet confirmed empty are left: each step's own group was swept when it ended. + await Promise.all([...groups].map(sweepGroup)); for (const [signal, handler] of handlers) process.off(signal, handler); process.stdout.off('error', onStdoutError); process.stdout.on('error', () => {}); // a late EPIPE while exiting is not a crash diff --git a/tests/automation/test-lanes.test.ts b/tests/automation/test-lanes.test.ts index 6b23bcf2..f0563cc7 100644 --- a/tests/automation/test-lanes.test.ts +++ b/tests/automation/test-lanes.test.ts @@ -12,7 +12,17 @@ import * as lanes from '../../scripts/test-lanes.ts'; const posixOnly = { skip: process.platform === 'win32' && 'process groups and catchable signals are POSIX-only' }; const moduleUrl = JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href); const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); -const killTree = (pid: number) => { +// The harness's own process group, found once: a recorded pid must never lead to a signal for it. +const ownGroup = process.platform === 'win32' ? undefined : Number(spawnSync('ps', ['-o', 'pgid=', '-p', String(process.pid)], { encoding: 'utf8' }).stdout.trim()) || undefined; +/** A pid read back from a fixture file, or undefined for anything that is not a whole number above 1 that is + * neither the harness, its parent nor its group: an empty file is Number('') === 0, which is the whole group. */ +function safePid(text: string): number | undefined { + if (!/^[0-9]+$/.test(text.trim())) return undefined; + const pid = Number(text.trim()); + return Number.isSafeInteger(pid) && pid > 1 && pid !== process.pid && pid !== process.ppid && pid !== ownGroup ? pid : undefined; +} +const killTree = (pid: number | undefined) => { + if (pid === undefined || safePid(String(pid)) === undefined) return; if (process.platform === 'win32') spawnSync('taskkill', ['/T', '/F', '/PID', String(pid)], { stdio: 'ignore' }); else { try { process.kill(-pid, 'SIGKILL'); } catch { /* not a group leader */ } try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } } }; @@ -37,7 +47,8 @@ const { ROOT, NAME, MODE = '' } = process.env; const modes = MODE.split(','); const value = key => (modes.find(m => m.startsWith(key + '=')) ?? '').split('=').slice(1).join('=') || undefined; if (modes.includes('ignore')) process.on('SIGTERM', () => {}); -fs.writeFileSync(ROOT + '/' + NAME + '.pid', String(process.pid)); +fs.writeFileSync(ROOT + '/' + NAME + '.pid.tmp', String(process.pid)); +fs.renameSync(ROOT + '/' + NAME + '.pid.tmp', ROOT + '/' + NAME + '.pid'); const again = (suffix, mode, options = {}) => cp.spawn(process.execPath, [__filename], { env: { ...process.env, NAME: NAME + '-' + suffix, MODE: mode }, stdio: ['ignore', 'inherit', 'inherit'], ...options }); if (modes.includes('grandchild')) again('grandchild', 'ignore,hang'); if (value('session')) again('session', value('session').replaceAll('+', ','), { detached: true }); @@ -49,10 +60,11 @@ else if (!modes.includes('hang')) process.exit(Number(value('code') ?? 0)); `); const step = (label: string, mode: string) => `{label:${JSON.stringify(label)},command:process.execPath,args:[${JSON.stringify(join(root, 'child.cjs'))}],env:{ROOT:${JSON.stringify(root)},NAME:${JSON.stringify(label)},MODE:${JSON.stringify(mode)}}}`; - const pid = (name: string) => (existsSync(join(root, name + '.pid')) ? Number(readFileSync(join(root, name + '.pid'), 'utf8')) : undefined); + const pid = (name: string) => (existsSync(join(root, name + '.pid')) ? safePid(readFileSync(join(root, name + '.pid'), 'utf8')) : undefined); // Every fixture process records its pid here: stop whatever a failed test left running, then delete the files. const dispose = () => { - for (const name of readdirSync(root)) if (name.endsWith('.pid')) killTree(Number(readFileSync(join(root, name), 'utf8'))); + if (!existsSync(root)) return; + for (const name of readdirSync(root)) if (name.endsWith('.pid')) killTree(safePid(readFileSync(join(root, name), 'utf8'))); rmSync(root, { recursive: true, force: true }); }; return { root, step, pid, dispose }; @@ -256,6 +268,63 @@ test('a pass whose leftover process holds the pipes is abandoned after the grace } finally { if (session && alive(session)) process.kill(session, 'SIGKILL'); f.dispose(); } }); +test('an empty or garbage pid file is ignored by the fixture cleanup and the harness survives', async () => { + const f = fixture(); + const sleeper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: process.platform !== 'win32' }); + try { + for (const [name, text] of Object.entries({ empty: '', blank: ' \n', zero: '0', init: '1', word: 'abc', negative: '-5', fraction: '2.5', harness: String(process.pid), parent: String(process.ppid) })) + writeFileSync(join(f.root, `${name}.pid`), text); + writeFileSync(join(f.root, 'real.pid'), String(sleeper.pid)); + for (const text of ['', ' ', '0', '1', 'abc', '-5', '2.5', String(process.pid), String(process.ppid)]) assert.equal(safePid(text), undefined, JSON.stringify(text)); + assert.equal(f.pid('empty'), undefined, 'a half-written file is not pid 0'); + f.dispose(); + assert.ok(await until(() => !alive(sleeper.pid!), 5000), 'the recorded fixture process is stopped'); + assert.ok(alive(process.pid), 'the harness is still here'); + assert.ok(alive(process.ppid), 'so is its parent'); + } finally { killTree(sleeper.pid); f.dispose(); } +}); + +test('the runner signals only a whole number above 1 that is not itself or its parent, and Windows stops only a child that has not exited', () => { + for (const bad of [0, 1, -1, -3, 1.5, NaN, Infinity, '12', null, undefined, process.pid, process.ppid]) assert.equal(lanes.validPid(bad), false, String(bad)); + assert.equal(lanes.validPid(4242), true); + assert.equal(lanes.stopAction('win32', true, 4242, false), undefined, 'an exited PID may belong to someone else now'); + assert.deepEqual(lanes.stopAction('win32', false, 4242, true), { kind: 'command', command: 'taskkill', args: ['/T', '/F', '/PID', '4242'] }); + assert.deepEqual(lanes.stopAction('linux', true, 4242, false), { kind: 'group', group: -4242, signal: 'SIGTERM' }, 'POSIX groups are probed before any signal, not skipped here'); + assert.equal(lanes.stopAction('linux', false, 0, false), undefined); + assert.equal(lanes.stopAction('win32', false, process.pid, true), undefined); +}); + +test('a group is retired as soon as its step is swept and nothing is sent to it afterwards, even if its id is reused', posixOnly, async () => { + const f = fixture(); + try { + const file = join(f.root, 'retire.ts'); + writeFileSync(file, `import * as lanes from ${moduleUrl}; +const calls: string[] = []; +const retired = new Set(); +// Once a group answered ESRCH it is retired; any later call to that id stands for an unrelated process that reused it. +const kill = (pid: number, signal: NodeJS.Signals | 0) => { + const group = Math.abs(pid); + if (retired.has(group)) { calls.push(group + ' AFTER-RETIRED ' + signal); return; } + try { process.kill(pid, signal); calls.push(group + ' ' + signal + ' ok'); } + catch (error) { const code = (error as NodeJS.ErrnoException).code; calls.push(group + ' ' + signal + ' ' + code); if (code === 'ESRCH') retired.add(group); throw error; } +}; +await lanes.runLanes([{name:'a',steps:[${f.step('one', 'code=0')},${f.step('two', 'grandchild')}]}], {graceMs:300,sweepMs:800,kill}); +console.log('CALLS ' + JSON.stringify(calls));`); + const run = launch(file); + assert.equal(await bounded(run.exited, 20_000), 0, run.output()); + const calls: string[] = JSON.parse(/CALLS (.*)/.exec(run.output())?.[1] ?? '[]'); + const one = f.pid('one')!, two = f.pid('two')!; + const of = (group: number) => calls.filter(c => c.startsWith(group + ' ')); + assert.ok(calls.length > 0, 'the runner reports its probes and signals through the injected function'); + assert.equal(calls.filter(c => c.includes('AFTER-RETIRED')).length, 0, calls.join('; ')); + assert.deepEqual(of(one), [`${one} 0 ESRCH`], 'an empty group is probed once and retired without a signal'); + assert.ok(of(two).some(c => c.endsWith('SIGTERM ok')) && of(two).some(c => c.endsWith('SIGKILL ok')), 'a stubborn descendant gets TERM then KILL'); + assert.equal(of(two).at(-1), `${two} 0 ESRCH`, 'and the group is confirmed empty'); + assert.ok(calls.findIndex(c => c.startsWith(two + ' ')) > calls.findIndex(c => c.startsWith(one + ' ')) , 'the first step is swept before the next one starts'); + for (const c of calls) assert.ok(Number(c.split(' ')[0]) > 1, c); + } finally { f.dispose(); } +}); + test('steps are stopped with a process group on POSIX and with taskkill /T /F on Windows, and only POSIX detaches them', () => { assert.deepEqual(lanes.stopPlan('linux', 4242, false), { kind: 'group', group: -4242, signal: 'SIGTERM' }); assert.deepEqual(lanes.stopPlan('darwin', 4242, true), { kind: 'group', group: -4242, signal: 'SIGKILL' }); From 05f89c68275e7326fefb50dc949d90a4067b7939 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:13:44 +1100 Subject: [PATCH 14/15] test: survive a failed lookup of the harness process group When ps cannot run, spawnSync has no stdout and trimming it threw while the module loaded, so no test registered. Read the group with optional chaining and treat a failed lookup as unknown; cleanup then sends no group signal and signals only the validated pid itself. --- tests/automation/test-lanes.test.ts | 41 ++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/tests/automation/test-lanes.test.ts b/tests/automation/test-lanes.test.ts index f0563cc7..4fb7ff04 100644 --- a/tests/automation/test-lanes.test.ts +++ b/tests/automation/test-lanes.test.ts @@ -12,8 +12,14 @@ import * as lanes from '../../scripts/test-lanes.ts'; const posixOnly = { skip: process.platform === 'win32' && 'process groups and catchable signals are POSIX-only' }; const moduleUrl = JSON.stringify(pathToFileURL(resolve('scripts/test-lanes.ts')).href); const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); -// The harness's own process group, found once: a recorded pid must never lead to a signal for it. -const ownGroup = process.platform === 'win32' ? undefined : Number(spawnSync('ps', ['-o', 'pgid=', '-p', String(process.pid)], { encoding: 'utf8' }).stdout.trim()) || undefined; +// The harness's own process group, found once: a recorded pid must never lead to a signal for it. If `ps` cannot +// run (no binary, no permission) the result has no stdout, and the group is simply unknown. +function lookupGroup(run: typeof spawnSync = spawnSync, platform: NodeJS.Platform = process.platform): number | undefined { + if (platform === 'win32') return undefined; + const group = Number(run('ps', ['-o', 'pgid=', '-p', String(process.pid)], { encoding: 'utf8' })?.stdout?.trim()); + return Number.isSafeInteger(group) && group > 1 ? group : undefined; +} +const ownGroup = lookupGroup(); /** A pid read back from a fixture file, or undefined for anything that is not a whole number above 1 that is * neither the harness, its parent nor its group: an empty file is Number('') === 0, which is the whole group. */ function safePid(text: string): number | undefined { @@ -21,11 +27,15 @@ function safePid(text: string): number | undefined { const pid = Number(text.trim()); return Number.isSafeInteger(pid) && pid > 1 && pid !== process.pid && pid !== process.ppid && pid !== ownGroup ? pid : undefined; } -const killTree = (pid: number | undefined) => { +type Send = (pid: number, signal: NodeJS.Signals) => void; +/** Stop a fixture process and, on POSIX, the group it leads. With the harness's own group unknown no group signal is + * sent at all, only one to the validated pid itself: it could otherwise be the harness's group. */ +function killTree(pid: number | undefined, harness: { group: number | undefined } = { group: ownGroup }, send: Send = (target, signal) => { process.kill(target, signal); }, platform: NodeJS.Platform = process.platform) { if (pid === undefined || safePid(String(pid)) === undefined) return; - if (process.platform === 'win32') spawnSync('taskkill', ['/T', '/F', '/PID', String(pid)], { stdio: 'ignore' }); - else { try { process.kill(-pid, 'SIGKILL'); } catch { /* not a group leader */ } try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } } -}; + if (platform === 'win32') { spawnSync('taskkill', ['/T', '/F', '/PID', String(pid)], { stdio: 'ignore' }); return; } + if (harness.group !== undefined && pid !== harness.group) { try { send(-pid, 'SIGKILL'); } catch { /* not a group leader */ } } + try { send(pid, 'SIGKILL'); } catch { /* gone */ } +} const launched = new Set(); after(() => { for (const child of launched) if (child.pid) killTree(child.pid); }); const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; @@ -268,6 +278,25 @@ test('a pass whose leftover process holds the pipes is abandoned after the grace } finally { if (session && alive(session)) process.kill(session, 'SIGKILL'); f.dispose(); } }); +test('a failed lookup of the harness group leaves it unknown, the module still loads, and cleanup then sends no group signal', () => { + const failed = (() => ({ pid: 0, output: null, stdout: null, stderr: null, status: null, signal: null, error: Object.assign(new Error('spawn ps ENOENT'), { code: 'ENOENT' }) })) as unknown as typeof spawnSync; + assert.equal(lookupGroup(failed), undefined); + assert.equal(lookupGroup((() => undefined) as unknown as typeof spawnSync), undefined); + assert.equal(lookupGroup((() => ({ stdout: '' })) as unknown as typeof spawnSync), undefined, 'an empty answer is not group 0'); + assert.equal(lookupGroup((() => ({ stdout: ' 4321\n' })) as unknown as typeof spawnSync, 'linux'), 4321); + assert.equal(lookupGroup(failed, 'win32'), undefined); + const sent: number[] = []; + killTree(4242, { group: undefined }, pid => { sent.push(pid); }); + assert.deepEqual(sent, [4242], 'only the validated pid itself, never -4242'); + killTree(0, { group: undefined }, pid => { sent.push(pid); }); killTree(process.pid, { group: undefined }, pid => { sent.push(pid); }); + assert.deepEqual(sent, [4242], 'an unsafe pid is not signalled at all'); + const known: number[] = []; + killTree(4242, { group: 777 }, pid => { known.push(pid); }); + assert.deepEqual(known, [-4242, 4242], 'with the group known a leader is stopped as a group too'); + killTree(777, { group: 777 }, pid => { known.push(pid); }); + assert.deepEqual(known, [-4242, 4242, 777], 'but never the harness group itself'); +}); + test('an empty or garbage pid file is ignored by the fixture cleanup and the harness survives', async () => { const f = fixture(); const sleeper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: process.platform !== 'win32' }); From 8a5b6175d907c5b9cc702828e6327866250585f1 Mon Sep 17 00:00:00 2001 From: Phototonic <35551999+Phototonic@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:23:52 +1100 Subject: [PATCH 15/15] test: keep the mocked cleanup calls off the host's real kill and taskkill Name the platform on every mocked killTree call and give killTree an injectable command runner for the Windows path, so no call can shell out to a real taskkill. Add a mocked win32 call asserting the taskkill command and arguments. --- tests/automation/test-lanes.test.ts | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/automation/test-lanes.test.ts b/tests/automation/test-lanes.test.ts index 4fb7ff04..696e53c5 100644 --- a/tests/automation/test-lanes.test.ts +++ b/tests/automation/test-lanes.test.ts @@ -28,11 +28,13 @@ function safePid(text: string): number | undefined { return Number.isSafeInteger(pid) && pid > 1 && pid !== process.pid && pid !== process.ppid && pid !== ownGroup ? pid : undefined; } type Send = (pid: number, signal: NodeJS.Signals) => void; +type Command = (command: string, args: string[]) => void; +const runCommand: Command = (command, args) => { spawnSync(command, args, { stdio: 'ignore' }); }; /** Stop a fixture process and, on POSIX, the group it leads. With the harness's own group unknown no group signal is * sent at all, only one to the validated pid itself: it could otherwise be the harness's group. */ -function killTree(pid: number | undefined, harness: { group: number | undefined } = { group: ownGroup }, send: Send = (target, signal) => { process.kill(target, signal); }, platform: NodeJS.Platform = process.platform) { +function killTree(pid: number | undefined, harness: { group: number | undefined } = { group: ownGroup }, send: Send = (target, signal) => { process.kill(target, signal); }, platform: NodeJS.Platform = process.platform, command: Command = runCommand) { if (pid === undefined || safePid(String(pid)) === undefined) return; - if (platform === 'win32') { spawnSync('taskkill', ['/T', '/F', '/PID', String(pid)], { stdio: 'ignore' }); return; } + if (platform === 'win32') { command('taskkill', ['/T', '/F', '/PID', String(pid)]); return; } if (harness.group !== undefined && pid !== harness.group) { try { send(-pid, 'SIGKILL'); } catch { /* not a group leader */ } } try { send(pid, 'SIGKILL'); } catch { /* gone */ } } @@ -280,21 +282,29 @@ test('a pass whose leftover process holds the pipes is abandoned after the grace test('a failed lookup of the harness group leaves it unknown, the module still loads, and cleanup then sends no group signal', () => { const failed = (() => ({ pid: 0, output: null, stdout: null, stderr: null, status: null, signal: null, error: Object.assign(new Error('spawn ps ENOENT'), { code: 'ENOENT' }) })) as unknown as typeof spawnSync; - assert.equal(lookupGroup(failed), undefined); - assert.equal(lookupGroup((() => undefined) as unknown as typeof spawnSync), undefined); - assert.equal(lookupGroup((() => ({ stdout: '' })) as unknown as typeof spawnSync), undefined, 'an empty answer is not group 0'); + assert.equal(lookupGroup(failed, 'linux'), undefined); + assert.equal(lookupGroup((() => undefined) as unknown as typeof spawnSync, 'linux'), undefined); + assert.equal(lookupGroup((() => ({ stdout: '' })) as unknown as typeof spawnSync, 'linux'), undefined, 'an empty answer is not group 0'); assert.equal(lookupGroup((() => ({ stdout: ' 4321\n' })) as unknown as typeof spawnSync, 'linux'), 4321); assert.equal(lookupGroup(failed, 'win32'), undefined); + // Every call below names its platform and injects both the signal and the command: none can reach a real kill or taskkill. + const noCommand: Command = (name) => { assert.fail(`${name} must not run on this platform`); }; + const noSignal: Send = () => { assert.fail('no signal may be sent on this platform'); }; const sent: number[] = []; - killTree(4242, { group: undefined }, pid => { sent.push(pid); }); + killTree(4242, { group: undefined }, pid => { sent.push(pid); }, 'linux', noCommand); assert.deepEqual(sent, [4242], 'only the validated pid itself, never -4242'); - killTree(0, { group: undefined }, pid => { sent.push(pid); }); killTree(process.pid, { group: undefined }, pid => { sent.push(pid); }); + killTree(0, { group: undefined }, pid => { sent.push(pid); }, 'linux', noCommand); killTree(process.pid, { group: undefined }, pid => { sent.push(pid); }, 'linux', noCommand); assert.deepEqual(sent, [4242], 'an unsafe pid is not signalled at all'); const known: number[] = []; - killTree(4242, { group: 777 }, pid => { known.push(pid); }); + killTree(4242, { group: 777 }, pid => { known.push(pid); }, 'linux', noCommand); assert.deepEqual(known, [-4242, 4242], 'with the group known a leader is stopped as a group too'); - killTree(777, { group: 777 }, pid => { known.push(pid); }); + killTree(777, { group: 777 }, pid => { known.push(pid); }, 'linux', noCommand); assert.deepEqual(known, [-4242, 4242, 777], 'but never the harness group itself'); + const commands: [string, string[]][] = []; + killTree(4242, { group: 777 }, noSignal, 'win32', (name, args) => { commands.push([name, args]); }); + assert.deepEqual(commands, [['taskkill', ['/T', '/F', '/PID', '4242']]], 'on Windows a safe pid goes to taskkill, through the injected runner only'); + killTree(0, { group: undefined }, noSignal, 'win32', (name, args) => { commands.push([name, args]); }); + assert.equal(commands.length, 1, 'and an unsafe pid to nothing'); }); test('an empty or garbage pid file is ignored by the fixture cleanup and the harness survives', async () => {