From 9b12dcbd9bd7d5864404329929e269957e65de5f Mon Sep 17 00:00:00 2001 From: kylerankin Date: Fri, 20 Nov 2026 11:10:47 +0000 Subject: [PATCH 1/3] fix: fail the test gate when the runner cannot run tests The baseline gate only counted individual failed assertions and ignored the test runner's own exit code, so a suite that failed to load, a setup error, or a runner crash - any failure that stops tests from running - was reported as no new failures. Contributors could therefore see a green result while their code never actually executed. The gate now captures the runner's exit code and rejects a run that ends without any completed failed assertions, since a healthy red suite always has real failures to explain its exit. It also validates that the JSON report is complete before reading it or recording an updated baseline, so an incomplete report can no longer overwrite the baseline or pass the gate. Recorded, expected failures still pass, so the normal red suite is unaffected. Tests: regression cases for failed suites without assertions, runner errors, incomplete reports, and refusal to overwrite the baseline on such failures, with passing controls for healthy runs and recorded failures. Assisted-by: pi (lemonade/ornith-1.5-35b-a3b-gguf-q6_k) Hive-Run: projectbluefin/website#1053 Hive-Plan: repair-test-baseline-runner-error Hive-Spec: interface-violation Signed-off-by: kylerankin --- scripts/test-baseline.mjs | 38 ++++++- scripts/tests/test-baseline.test.ts | 150 ++++++++++++++++++++++++++-- 2 files changed, 173 insertions(+), 15 deletions(-) diff --git a/scripts/test-baseline.mjs b/scripts/test-baseline.mjs index 6e014830..b8d6cc2c 100644 --- a/scripts/test-baseline.mjs +++ b/scripts/test-baseline.mjs @@ -30,6 +30,11 @@ const shouldUpdate = process.argv.includes('--update') const reportPath = join(mkdtempSync(join(tmpdir(), 'wolves-baseline-')), 'vitest.json') +// Captured instead of swallowed: a non-zero exit is only acceptable when +// completed failed assertions account for it. A runner that failed to finish +// (suite-load error, setup error, import error) also exits non-zero, and that +// is exactly the case the gate used to report as passing. +let childExit = 0 try { execFileSync( 'npx', @@ -44,8 +49,10 @@ try { }, ) } -catch { - // vitest exits non-zero whenever tests fail, which is the normal case here. +catch (error) { + childExit = Number.isInteger(error.status) + ? error.status + : 1 } if (!existsSync(reportPath)) { @@ -54,12 +61,23 @@ if (!existsSync(reportPath)) { } const report = JSON.parse(readFileSync(reportPath, 'utf8')) + +// Validate the report before interpreting it or writing an updated baseline. +// An incomplete report (no `testResults`) means nothing ran to completion, so +// it must never be read as "no new failures" nor recorded as a baseline. +if (!report || typeof report !== 'object' || !Array.isArray(report.testResults)) { + console.error('test-baseline: vitest report is incomplete or malformed') + process.exit(2) +} + const cwd = `${process.cwd()}/` const current = new Set() -for (const file of report.testResults ?? []) { +for (const file of report.testResults) { for (const assertion of file.assertionResults ?? []) { - if (assertion.status === 'failed') { + // A suite that failed to load reports a failed assertion with no name; + // it is not a completed assertion and must not enter the failure set. + if (assertion.status === 'failed' && assertion.fullName) { current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`) } } @@ -67,6 +85,18 @@ for (const file of report.testResults ?? []) { const sorted = [...current].sort() +// A non-zero child exit with no completed failed assertions is a run that did +// not finish: a suite failed to load, setup threw, or the runner errored. The +// normal red suite always has completed failed assertions to explain its exit +// code, so the absence of any is itself a failure. Reject it before the +// comparison and before any `--update` could record an empty baseline. +if (childExit !== 0 && current.size === 0) { + console.error( + 'test-baseline: vitest failed to run to completion and reported no completed failures', + ) + process.exit(2) +} + if (shouldUpdate) { writeFileSync(BASELINE, `${sorted.join('\n')}\n`) console.info(`test-baseline: recorded ${sorted.length} known failures`) diff --git a/scripts/tests/test-baseline.test.ts b/scripts/tests/test-baseline.test.ts index a49136dc..0bba9b4f 100644 --- a/scripts/tests/test-baseline.test.ts +++ b/scripts/tests/test-baseline.test.ts @@ -329,21 +329,149 @@ describe('scripts/test-baseline.mjs (npm run test:gate)', () => { expect(stdout).toContain('test-baseline: no new failures (0 known, baseline 0).') }) - it('tolerates a report with no testResults and a file with no assertionResults', () => { + it('rejects an incomplete report with no testResults, even on a clean exit', () => { + // An empty report means nothing ran to completion. It must fail the gate + // (exit 2) regardless of the child exit code, and never read as "no new + // failures". const root = makeRepo() writeBaseline(root, '') - expect(runGate(root, { vitestRc: 0, report: '{}' }).status).toBe(0) - expect(runGate(root, { + const { status, stderr } = runGate(root, { vitestRc: 0, report: '{}' }) + expect(status).toBe(2) + expect(stderr).toContain('test-baseline: vitest report is incomplete or malformed') + + // The same report with a non-zero child exit is rejected the same way. + expect(runGate(root, { vitestRc: 1, report: '{}' }).status).toBe(2) + }) + + it('tolerates a file with no assertionResults on a clean exit', () => { + // A completed run that simply reports a file with no assertions still has + // no failures to report; only an unexplained non-zero exit is a problem. + const root = makeRepo() + writeBaseline(root, '') + + expect( + runGate(root, { + vitestRc: 0, + report: JSON.stringify({ + testResults: [ + { + name: join(root, 'src/tests/a.test.ts') + } + ], + }), + }).status, + ).toBe(0) + }) + + it('rejects a failed suite with no completed assertions', () => { + // Import / suite-load failure: vitest exits non-zero, the suite contributes + // no completed failed assertions, and success is false. + const root = makeRepo() + writeBaseline(root, '') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/broken.test.ts'), + assertionResults: [] + } + ], + }) + + const { status, stderr } = runGate(root, { vitestRc: 1, report }) + expect(status).toBe(2) + expect(stderr).toContain('test-baseline: vitest failed to run to completion') + }) + + it('rejects a runner error even when one assertion passed', () => { + // A passing assertion plus a runner error still means the run did not + // complete; the single pass does not explain the non-zero exit. + const root = makeRepo() + writeBaseline(root, '') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/a.test.ts'), + assertionResults: [ + { status: 'passed', fullName: 'a passes' } + ], + }, + ], + }) + + expect(runGate(root, { vitestRc: 1, report }).status).toBe(2) + }) + + it('rejects a suite-load failure reported with a nameless failed assertion', () => { + // Real vitest emits a failed entry with an empty fullName for a suite that + // failed to import. That is not a completed assertion and must not be + // counted, so the non-zero exit stays unexplained and the gate rejects it. + const root = makeRepo() + writeBaseline(root, '') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/broken.test.ts'), + assertionResults: [ + { status: 'failed', fullName: '' } + ], + }, + ], + }) + + expect(runGate(root, { vitestRc: 1, report }).status).toBe(2) + }) + + it('refuses to overwrite the baseline under --update on a runner error', () => { + // The update mode must not record an empty baseline from an incomplete run. + const root = makeRepo() + writeBaseline(root, 'src/tests/a.test.ts :: a fails\n') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/broken.test.ts'), + assertionResults: [] + } + ], + }) + + const { status } = runGate(root, { + args: ['--update'], + vitestRc: 1, + report + }) + expect(status).toBe(2) + // The recorded baseline is left untouched. + expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n') + }) + + it('still passes a healthy run with a passing assertion', () => { + // Passing control: a clean, complete run is unaffected by the new checks. + const root = makeRepo() + writeBaseline(root, '') + + const { status, stdout } = runGate(root, { vitestRc: 0, - report: JSON.stringify({ - testResults: [ - { - name: join(root, 'src/tests/a.test.ts') - } - ] - }), - }).status).toBe(0) + report: report(root, [ + { + name: 'src/tests/a.test.ts', + assertions: [ + ['passed', 'a passes'] + ] + } + ]), + }) + + expect(status).toBe(0) + expect(stdout).toContain('no new failures') }) it('ignores blank and whitespace-padded baseline lines', () => { From a4a60099d2d94ae86e88b0c020fbd467be8c034e Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 20 Nov 2026 08:36:01 -0500 Subject: [PATCH 2/3] fix: reject per-file suite-load failures in test gate The run-wide "non-zero exit and no failures" check let a file that failed to import hide behind any unrelated known failure, and --update would drop that file's tests from the baseline. Track named failures per file and reject any file that failed (or reported a nameless failed assertion) without a completed failed assertion. Add a regression test with a non-empty baseline. Hive-Fix-Attempt: 1/6 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- scripts/test-baseline.mjs | 29 +++++++++++++++++++++++++++-- scripts/tests/test-baseline.test.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/scripts/test-baseline.mjs b/scripts/test-baseline.mjs index b8d6cc2c..79fd026f 100644 --- a/scripts/test-baseline.mjs +++ b/scripts/test-baseline.mjs @@ -73,14 +73,28 @@ if (!report || typeof report !== 'object' || !Array.isArray(report.testResults)) const cwd = `${process.cwd()}/` const current = new Set() +// Files that failed (or reported a nameless failed assertion) without any +// completed failed assertion to explain it: suite-load / import / setup errors. +const unexplained = [] for (const file of report.testResults) { + let named = 0 + let nameless = 0 for (const assertion of file.assertionResults ?? []) { // A suite that failed to load reports a failed assertion with no name; // it is not a completed assertion and must not enter the failure set. - if (assertion.status === 'failed' && assertion.fullName) { - current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`) + if (assertion.status === 'failed') { + if (assertion.fullName) { + named++ + current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`) + } + else { + nameless++ + } } } + if ((file.status === 'failed' || nameless > 0) && named === 0) { + unexplained.push(String(file.name).replace(cwd, '')) + } } const sorted = [...current].sort() @@ -90,6 +104,17 @@ const sorted = [...current].sort() // normal red suite always has completed failed assertions to explain its exit // code, so the absence of any is itself a failure. Reject it before the // comparison and before any `--update` could record an empty baseline. +// The check is also per file: an unrelated known failure elsewhere must not +// mask a file that failed to load, so any such file rejects the run too. +if (unexplained.length > 0) { + console.error('test-baseline: file(s) failed without any completed failed assertion:') + for (const name of unexplained) { + console.error(` - ${name}`) + } + console.error('test-baseline: vitest failed to run to completion') + process.exit(2) +} + if (childExit !== 0 && current.size === 0) { console.error( 'test-baseline: vitest failed to run to completion and reported no completed failures', diff --git a/scripts/tests/test-baseline.test.ts b/scripts/tests/test-baseline.test.ts index 0bba9b4f..e4254c35 100644 --- a/scripts/tests/test-baseline.test.ts +++ b/scripts/tests/test-baseline.test.ts @@ -428,6 +428,35 @@ describe('scripts/test-baseline.mjs (npm run test:gate)', () => { expect(runGate(root, { vitestRc: 1, report }).status).toBe(2) }) + it('rejects a suite-load failure even when a known failure exists elsewhere', () => { + // A non-empty baseline plus an unrelated known failure must not mask a + // file that failed to import. + const root = makeRepo() + writeBaseline(root, 'src/tests/a.test.ts :: a fails\n') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/a.test.ts'), + status: 'failed', + assertionResults: [{ status: 'failed', fullName: 'a fails' }], + }, + { + name: join(root, 'src/tests/broken.test.ts'), + status: 'failed', + assertionResults: [], + }, + ], + }) + + const { status, stderr } = runGate(root, { vitestRc: 1, report }) + expect(status).toBe(2) + expect(stderr).toContain('src/tests/broken.test.ts') + expect(runGate(root, { vitestRc: 1, report, args: ['--update'] }).status).toBe(2) + expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n') + }) + it('refuses to overwrite the baseline under --update on a runner error', () => { // The update mode must not record an empty baseline from an incomplete run. const root = makeRepo() From fa974616424d0a9b0635dd05777f300b6f96b33b Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Fri, 20 Nov 2026 08:49:50 -0500 Subject: [PATCH 3/3] fix: flag files with a file-level error even when named failures exist A hook error reported on the file (message) was masked when the file also had a baseline-known named failure. Flag any file with a file-level error, add a regression test, and fix antfu lint errors in the test. Hive-Fix-Attempt: 2/6 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- scripts/test-baseline.mjs | 6 +++++- scripts/tests/test-baseline.test.ts | 33 +++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/scripts/test-baseline.mjs b/scripts/test-baseline.mjs index 79fd026f..2791b6b9 100644 --- a/scripts/test-baseline.mjs +++ b/scripts/test-baseline.mjs @@ -92,7 +92,11 @@ for (const file of report.testResults) { } } } - if ((file.status === 'failed' || nameless > 0) && named === 0) { + // A file-level error message (hook throw, import failure) is reported on the + // file itself, never on an assertion, so it flags the file even when other + // named failures in it are baseline-known. + const fileError = typeof file.message === 'string' && file.message.trim() !== '' + if (fileError || ((file.status === 'failed' || nameless > 0) && named === 0)) { unexplained.push(String(file.name).replace(cwd, '')) } } diff --git a/scripts/tests/test-baseline.test.ts b/scripts/tests/test-baseline.test.ts index e4254c35..8e870d47 100644 --- a/scripts/tests/test-baseline.test.ts +++ b/scripts/tests/test-baseline.test.ts @@ -440,7 +440,9 @@ describe('scripts/test-baseline.mjs (npm run test:gate)', () => { { name: join(root, 'src/tests/a.test.ts'), status: 'failed', - assertionResults: [{ status: 'failed', fullName: 'a fails' }], + assertionResults: [ + { status: 'failed', fullName: 'a fails' } + ], }, { name: join(root, 'src/tests/broken.test.ts'), @@ -453,10 +455,37 @@ describe('scripts/test-baseline.mjs (npm run test:gate)', () => { const { status, stderr } = runGate(root, { vitestRc: 1, report }) expect(status).toBe(2) expect(stderr).toContain('src/tests/broken.test.ts') - expect(runGate(root, { vitestRc: 1, report, args: ['--update'] }).status).toBe(2) + expect(runGate(root, { + vitestRc: 1, + report, + args: ['--update'] + }).status).toBe(2) expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n') }) + it('rejects a file with a named failure plus a file-level hook error', () => { + const root = makeRepo() + writeBaseline(root, 'src/tests/a.test.ts :: a fails\n') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/a.test.ts'), + status: 'failed', + message: 'afterAll hook threw', + assertionResults: [ + { status: 'failed', fullName: 'a fails' } + ], + }, + ], + }) + + const { status, stderr } = runGate(root, { vitestRc: 1, report }) + expect(status).toBe(2) + expect(stderr).toContain('src/tests/a.test.ts') + }) + it('refuses to overwrite the baseline under --update on a runner error', () => { // The update mode must not record an empty baseline from an incomplete run. const root = makeRepo()