Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 63 additions & 4 deletions scripts/test-baseline.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ const shouldUpdate = process.argv.includes('--update')

const reportPath = join(mkdtempSync(join(tmpdir(), 'wolves-baseline-')), 'vitest.json')

// Captured instead of swallowed: a non-zero exit is only acceptable when
// completed failed assertions account for it. A runner that failed to finish
// (suite-load error, setup error, import error) also exits non-zero, and that
// is exactly the case the gate used to report as passing.
let childExit = 0
try {
execFileSync(
'npx',
Expand All @@ -44,8 +49,10 @@ try {
},
)
}
catch {
// vitest exits non-zero whenever tests fail, which is the normal case here.
catch (error) {
childExit = Number.isInteger(error.status)
? error.status
: 1
}

if (!existsSync(reportPath)) {
Expand All @@ -54,19 +61,71 @@ if (!existsSync(reportPath)) {
}

const report = JSON.parse(readFileSync(reportPath, 'utf8'))

// Validate the report before interpreting it or writing an updated baseline.
// An incomplete report (no `testResults`) means nothing ran to completion, so
// it must never be read as "no new failures" nor recorded as a baseline.
if (!report || typeof report !== 'object' || !Array.isArray(report.testResults)) {
console.error('test-baseline: vitest report is incomplete or malformed')
process.exit(2)
}

const cwd = `${process.cwd()}/`

const current = new Set()
for (const file of report.testResults ?? []) {
// Files that failed (or reported a nameless failed assertion) without any
// completed failed assertion to explain it: suite-load / import / setup errors.
const unexplained = []
for (const file of report.testResults) {
let named = 0
let nameless = 0
for (const assertion of file.assertionResults ?? []) {
// A suite that failed to load reports a failed assertion with no name;
// it is not a completed assertion and must not enter the failure set.
if (assertion.status === 'failed') {
current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`)
if (assertion.fullName) {
named++
current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`)
}
else {
nameless++
}
}
}
// A file-level error message (hook throw, import failure) is reported on the
// file itself, never on an assertion, so it flags the file even when other
// named failures in it are baseline-known.
const fileError = typeof file.message === 'string' && file.message.trim() !== ''
if (fileError || ((file.status === 'failed' || nameless > 0) && named === 0)) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style (low): A file now enters unexplained because of a file-level message, even when it has completed named failures (the new hook-error case). The diagnostic at the unexplained branch still prints file(s) failed without any completed failed assertion, and the comment above the unexplained declaration says the same. For the hook-error case that text is wrong and can send a contributor looking for a load failure. Suggest wording like file(s) failed to run to completion (load/setup/hook error): and updating the comment to match. Non-blocking.

unexplained.push(String(file.name).replace(cwd, ''))
}
}

const sorted = [...current].sort()

// A non-zero child exit with no completed failed assertions is a run that did
// not finish: a suite failed to load, setup threw, or the runner errored. The
// normal red suite always has completed failed assertions to explain its exit
// code, so the absence of any is itself a failure. Reject it before the
// comparison and before any `--update` could record an empty baseline.
// The check is also per file: an unrelated known failure elsewhere must not
// mask a file that failed to load, so any such file rejects the run too.
if (unexplained.length > 0) {
console.error('test-baseline: file(s) failed without any completed failed assertion:')
for (const name of unexplained) {
console.error(` - ${name}`)
}
console.error('test-baseline: vitest failed to run to completion')
process.exit(2)
}

if (childExit !== 0 && current.size === 0) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

correctness - medium: this guard only fires when the whole run has zero named failures, so one known failure anywhere hides a suite that failed to load.

Reproduced at this head with a stubbed npx: baseline src/tests/a.test.ts :: a fails; report has that failure plus src/tests/broken.test.ts with "status": "failed" and "assertionResults": []; child exit 1. Result: test-baseline: no new failures (1 known, baseline 1)., exit 0. --update would also drop broken.test.ts's tests from the baseline.

Proposed fix: flag each file that vitest marks failed but that has no named failed assertion, and reject the run if any exist (replaces lines 75–98):

 const current = new Set()
+const unexplained = []
 for (const file of report.testResults) {
+  let named = 0
   for (const assertion of file.assertionResults ?? []) {
     // A suite that failed to load reports a failed assertion with no name;
     // it is not a completed assertion and must not enter the failure set.
     if (assertion.status === 'failed' && assertion.fullName) {
       current.add(`${file.name.replace(cwd, '')} :: ${assertion.fullName}`)
+      named++
     }
   }
+  // A file vitest marks failed with no completed failed assertion did not run
+  // (import/setup error). Failures in other files do not explain it.
+  if (file.status === 'failed' && named === 0) {
+    unexplained.push(file.name.replace(cwd, ''))
+  }
 }
 
 const sorted = [...current].sort()
 
-if (childExit !== 0 && current.size === 0) {
+if (unexplained.length > 0 || (childExit !== 0 && current.size === 0)) {
   console.error(
     'test-baseline: vitest failed to run to completion and reported no completed failures',
   )
+  for (const name of unexplained) {
+    console.error(`  ! ${name} failed without completed assertions`)
+  }
   process.exit(2)
 }

Plus a regression test: non-empty baseline with one known failure in a.test.ts, a second file with status: 'failed' and empty assertionResults, vitestRc: 1 → expect status 2 and the baseline unchanged under --update. The existing test fixtures in this PR omit the file-level status, so add status: 'failed' to the failed-suite reports there too.

console.error(
'test-baseline: vitest failed to run to completion and reported no completed failures',
)
process.exit(2)
}

if (shouldUpdate) {
writeFileSync(BASELINE, `${sorted.join('\n')}\n`)
console.info(`test-baseline: recorded ${sorted.length} known failures`)
Expand Down
206 changes: 196 additions & 10 deletions scripts/tests/test-baseline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -329,21 +329,207 @@ describe('scripts/test-baseline.mjs (npm run test:gate)', () => {
expect(stdout).toContain('test-baseline: no new failures (0 known, baseline 0).')
})

it('tolerates a report with no testResults and a file with no assertionResults', () => {
it('rejects an incomplete report with no testResults, even on a clean exit', () => {
// An empty report means nothing ran to completion. It must fail the gate
// (exit 2) regardless of the child exit code, and never read as "no new
// failures".
const root = makeRepo()
writeBaseline(root, '')

expect(runGate(root, { vitestRc: 0, report: '{}' }).status).toBe(0)
const { status, stderr } = runGate(root, { vitestRc: 0, report: '{}' })
expect(status).toBe(2)
expect(stderr).toContain('test-baseline: vitest report is incomplete or malformed')

// The same report with a non-zero child exit is rejected the same way.
expect(runGate(root, { vitestRc: 1, report: '{}' }).status).toBe(2)
})

it('tolerates a file with no assertionResults on a clean exit', () => {
// A completed run that simply reports a file with no assertions still has
// no failures to report; only an unexplained non-zero exit is a problem.
const root = makeRepo()
writeBaseline(root, '')

expect(
runGate(root, {
vitestRc: 0,
report: JSON.stringify({
testResults: [
{
name: join(root, 'src/tests/a.test.ts')
}
],
}),
}).status,
).toBe(0)
})

it('rejects a failed suite with no completed assertions', () => {
// Import / suite-load failure: vitest exits non-zero, the suite contributes
// no completed failed assertions, and success is false.
const root = makeRepo()
writeBaseline(root, '')

const report = JSON.stringify({
success: false,
testResults: [
{
name: join(root, 'src/tests/broken.test.ts'),
assertionResults: []
}
],
})

const { status, stderr } = runGate(root, { vitestRc: 1, report })
expect(status).toBe(2)
expect(stderr).toContain('test-baseline: vitest failed to run to completion')
})

it('rejects a runner error even when one assertion passed', () => {
// A passing assertion plus a runner error still means the run did not
// complete; the single pass does not explain the non-zero exit.
const root = makeRepo()
writeBaseline(root, '')

const report = JSON.stringify({
success: false,
testResults: [
{
name: join(root, 'src/tests/a.test.ts'),
assertionResults: [
{ status: 'passed', fullName: 'a passes' }
],
},
],
})

expect(runGate(root, { vitestRc: 1, report }).status).toBe(2)
})

it('rejects a suite-load failure reported with a nameless failed assertion', () => {
// Real vitest emits a failed entry with an empty fullName for a suite that
// failed to import. That is not a completed assertion and must not be
// counted, so the non-zero exit stays unexplained and the gate rejects it.
const root = makeRepo()
writeBaseline(root, '')

const report = JSON.stringify({
success: false,
testResults: [
{
name: join(root, 'src/tests/broken.test.ts'),
assertionResults: [
{ status: 'failed', fullName: '' }
],
},
],
})

expect(runGate(root, { vitestRc: 1, report }).status).toBe(2)
})

it('rejects a suite-load failure even when a known failure exists elsewhere', () => {
// A non-empty baseline plus an unrelated known failure must not mask a
// file that failed to import.
const root = makeRepo()
writeBaseline(root, 'src/tests/a.test.ts :: a fails\n')

const report = JSON.stringify({
success: false,
testResults: [
{
name: join(root, 'src/tests/a.test.ts'),
status: 'failed',
assertionResults: [
{ status: 'failed', fullName: 'a fails' }
],
},
{
name: join(root, 'src/tests/broken.test.ts'),
status: 'failed',
assertionResults: [],
},
],
})

const { status, stderr } = runGate(root, { vitestRc: 1, report })
expect(status).toBe(2)
expect(stderr).toContain('src/tests/broken.test.ts')
expect(runGate(root, {
vitestRc: 1,
report,
args: ['--update']
}).status).toBe(2)
expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n')
})

it('rejects a file with a named failure plus a file-level hook error', () => {
const root = makeRepo()
writeBaseline(root, 'src/tests/a.test.ts :: a fails\n')

const report = JSON.stringify({
success: false,
testResults: [
{
name: join(root, 'src/tests/a.test.ts'),
status: 'failed',
message: 'afterAll hook threw',
assertionResults: [
{ status: 'failed', fullName: 'a fails' }
],
},
],
})

const { status, stderr } = runGate(root, { vitestRc: 1, report })
expect(status).toBe(2)
expect(stderr).toContain('src/tests/a.test.ts')
})

it('refuses to overwrite the baseline under --update on a runner error', () => {
// The update mode must not record an empty baseline from an incomplete run.
const root = makeRepo()
writeBaseline(root, 'src/tests/a.test.ts :: a fails\n')

const report = JSON.stringify({
success: false,
testResults: [
{
name: join(root, 'src/tests/broken.test.ts'),
assertionResults: []
}
],
})

const { status } = runGate(root, {
args: ['--update'],
vitestRc: 1,
report
})
expect(status).toBe(2)
// The recorded baseline is left untouched.
expect(readFileSync(baselinePath(root), 'utf8')).toBe('src/tests/a.test.ts :: a fails\n')
})

it('still passes a healthy run with a passing assertion', () => {
// Passing control: a clean, complete run is unaffected by the new checks.
const root = makeRepo()
writeBaseline(root, '')

const { status, stdout } = runGate(root, {
vitestRc: 0,
report: JSON.stringify({
testResults: [
{
name: join(root, 'src/tests/a.test.ts')
}
]
}),
}).status).toBe(0)
report: report(root, [
{
name: 'src/tests/a.test.ts',
assertions: [
['passed', 'a passes']
]
}
]),
})

expect(status).toBe(0)
expect(stdout).toContain('no new failures')
})

it('ignores blank and whitespace-padded baseline lines', () => {
Expand Down
Loading