diff --git a/scripts/test-baseline.mjs b/scripts/test-baseline.mjs index 6e014830..2791b6b9 100644 --- a/scripts/test-baseline.mjs +++ b/scripts/test-baseline.mjs @@ -30,6 +30,11 @@ const shouldUpdate = process.argv.includes('--update') const reportPath = join(mkdtempSync(join(tmpdir(), 'wolves-baseline-')), 'vitest.json') +// Captured instead of swallowed: a non-zero exit is only acceptable when +// completed failed assertions account for it. A runner that failed to finish +// (suite-load error, setup error, import error) also exits non-zero, and that +// is exactly the case the gate used to report as passing. +let childExit = 0 try { execFileSync( 'npx', @@ -44,8 +49,10 @@ try { }, ) } -catch { - // vitest exits non-zero whenever tests fail, which is the normal case here. +catch (error) { + childExit = Number.isInteger(error.status) + ? error.status + : 1 } if (!existsSync(reportPath)) { @@ -54,19 +61,71 @@ if (!existsSync(reportPath)) { } const report = JSON.parse(readFileSync(reportPath, 'utf8')) + +// Validate the report before interpreting it or writing an updated baseline. +// An incomplete report (no `testResults`) means nothing ran to completion, so +// it must never be read as "no new failures" nor recorded as a baseline. +if (!report || typeof report !== 'object' || !Array.isArray(report.testResults)) { + console.error('test-baseline: vitest report is incomplete or malformed') + process.exit(2) +} + const cwd = `${process.cwd()}/` const current = new Set() -for (const file of report.testResults ?? []) { +// Files that failed (or reported a nameless failed assertion) without any +// completed failed assertion to explain it: suite-load / import / setup errors. +const unexplained = [] +for (const file of report.testResults) { + let named = 0 + let nameless = 0 for (const assertion of file.assertionResults ?? []) { + // A suite that failed to load reports a failed assertion with no name; + // it is not a completed assertion and must not enter the failure set. if (assertion.status === 'failed') { - current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`) + if (assertion.fullName) { + named++ + current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`) + } + else { + nameless++ + } } } + // A file-level error message (hook throw, import failure) is reported on the + // file itself, never on an assertion, so it flags the file even when other + // named failures in it are baseline-known. + const fileError = typeof file.message === 'string' && file.message.trim() !== '' + if (fileError || ((file.status === 'failed' || nameless > 0) && named === 0)) { + unexplained.push(String(file.name).replace(cwd, '')) + } } const sorted = [...current].sort() +// A non-zero child exit with no completed failed assertions is a run that did +// not finish: a suite failed to load, setup threw, or the runner errored. The +// normal red suite always has completed failed assertions to explain its exit +// code, so the absence of any is itself a failure. Reject it before the +// comparison and before any `--update` could record an empty baseline. +// The check is also per file: an unrelated known failure elsewhere must not +// mask a file that failed to load, so any such file rejects the run too. +if (unexplained.length > 0) { + console.error('test-baseline: file(s) failed without any completed failed assertion:') + for (const name of unexplained) { + console.error(` - ${name}`) + } + console.error('test-baseline: vitest failed to run to completion') + process.exit(2) +} + +if (childExit !== 0 && current.size === 0) { + console.error( + 'test-baseline: vitest failed to run to completion and reported no completed failures', + ) + process.exit(2) +} + if (shouldUpdate) { writeFileSync(BASELINE, `${sorted.join('\n')}\n`) console.info(`test-baseline: recorded ${sorted.length} known failures`) diff --git a/scripts/tests/test-baseline.test.ts b/scripts/tests/test-baseline.test.ts index a49136dc..8e870d47 100644 --- a/scripts/tests/test-baseline.test.ts +++ b/scripts/tests/test-baseline.test.ts @@ -329,21 +329,207 @@ describe('scripts/test-baseline.mjs (npm run test:gate)', () => { expect(stdout).toContain('test-baseline: no new failures (0 known, baseline 0).') }) - it('tolerates a report with no testResults and a file with no assertionResults', () => { + it('rejects an incomplete report with no testResults, even on a clean exit', () => { + // An empty report means nothing ran to completion. It must fail the gate + // (exit 2) regardless of the child exit code, and never read as "no new + // failures". const root = makeRepo() writeBaseline(root, '') - expect(runGate(root, { vitestRc: 0, report: '{}' }).status).toBe(0) + const { status, stderr } = runGate(root, { vitestRc: 0, report: '{}' }) + expect(status).toBe(2) + expect(stderr).toContain('test-baseline: vitest report is incomplete or malformed') + + // The same report with a non-zero child exit is rejected the same way. + expect(runGate(root, { vitestRc: 1, report: '{}' }).status).toBe(2) + }) + + it('tolerates a file with no assertionResults on a clean exit', () => { + // A completed run that simply reports a file with no assertions still has + // no failures to report; only an unexplained non-zero exit is a problem. + const root = makeRepo() + writeBaseline(root, '') + + expect( + runGate(root, { + vitestRc: 0, + report: JSON.stringify({ + testResults: [ + { + name: join(root, 'src/tests/a.test.ts') + } + ], + }), + }).status, + ).toBe(0) + }) + + it('rejects a failed suite with no completed assertions', () => { + // Import / suite-load failure: vitest exits non-zero, the suite contributes + // no completed failed assertions, and success is false. + const root = makeRepo() + writeBaseline(root, '') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/broken.test.ts'), + assertionResults: [] + } + ], + }) + + const { status, stderr } = runGate(root, { vitestRc: 1, report }) + expect(status).toBe(2) + expect(stderr).toContain('test-baseline: vitest failed to run to completion') + }) + + it('rejects a runner error even when one assertion passed', () => { + // A passing assertion plus a runner error still means the run did not + // complete; the single pass does not explain the non-zero exit. + const root = makeRepo() + writeBaseline(root, '') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/a.test.ts'), + assertionResults: [ + { status: 'passed', fullName: 'a passes' } + ], + }, + ], + }) + + expect(runGate(root, { vitestRc: 1, report }).status).toBe(2) + }) + + it('rejects a suite-load failure reported with a nameless failed assertion', () => { + // Real vitest emits a failed entry with an empty fullName for a suite that + // failed to import. That is not a completed assertion and must not be + // counted, so the non-zero exit stays unexplained and the gate rejects it. + const root = makeRepo() + writeBaseline(root, '') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/broken.test.ts'), + assertionResults: [ + { status: 'failed', fullName: '' } + ], + }, + ], + }) + + expect(runGate(root, { vitestRc: 1, report }).status).toBe(2) + }) + + it('rejects a suite-load failure even when a known failure exists elsewhere', () => { + // A non-empty baseline plus an unrelated known failure must not mask a + // file that failed to import. + const root = makeRepo() + writeBaseline(root, 'src/tests/a.test.ts :: a fails\n') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/a.test.ts'), + status: 'failed', + assertionResults: [ + { status: 'failed', fullName: 'a fails' } + ], + }, + { + name: join(root, 'src/tests/broken.test.ts'), + status: 'failed', + assertionResults: [], + }, + ], + }) + + const { status, stderr } = runGate(root, { vitestRc: 1, report }) + expect(status).toBe(2) + expect(stderr).toContain('src/tests/broken.test.ts') expect(runGate(root, { + vitestRc: 1, + report, + args: ['--update'] + }).status).toBe(2) + expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n') + }) + + it('rejects a file with a named failure plus a file-level hook error', () => { + const root = makeRepo() + writeBaseline(root, 'src/tests/a.test.ts :: a fails\n') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/a.test.ts'), + status: 'failed', + message: 'afterAll hook threw', + assertionResults: [ + { status: 'failed', fullName: 'a fails' } + ], + }, + ], + }) + + const { status, stderr } = runGate(root, { vitestRc: 1, report }) + expect(status).toBe(2) + expect(stderr).toContain('src/tests/a.test.ts') + }) + + it('refuses to overwrite the baseline under --update on a runner error', () => { + // The update mode must not record an empty baseline from an incomplete run. + const root = makeRepo() + writeBaseline(root, 'src/tests/a.test.ts :: a fails\n') + + const report = JSON.stringify({ + success: false, + testResults: [ + { + name: join(root, 'src/tests/broken.test.ts'), + assertionResults: [] + } + ], + }) + + const { status } = runGate(root, { + args: ['--update'], + vitestRc: 1, + report + }) + expect(status).toBe(2) + // The recorded baseline is left untouched. + expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n') + }) + + it('still passes a healthy run with a passing assertion', () => { + // Passing control: a clean, complete run is unaffected by the new checks. + const root = makeRepo() + writeBaseline(root, '') + + const { status, stdout } = runGate(root, { vitestRc: 0, - report: JSON.stringify({ - testResults: [ - { - name: join(root, 'src/tests/a.test.ts') - } - ] - }), - }).status).toBe(0) + report: report(root, [ + { + name: 'src/tests/a.test.ts', + assertions: [ + ['passed', 'a passes'] + ] + } + ]), + }) + + expect(status).toBe(0) + expect(stdout).toContain('no new failures') }) it('ignores blank and whitespace-padded baseline lines', () => {