Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions .github/scripts/check-pass-rate.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { readFileSync } from 'node:fs'

const [, , reportPath, thresholdArg] = process.argv

if (!reportPath) {
console.error('Usage: check-pass-rate.mjs <vitest-json-report> [thresholdPercent]')
process.exit(2)
}

const threshold = Number(thresholdArg ?? '95')

if (Number.isNaN(threshold) || threshold < 0 || threshold > 100) {
console.error(`Invalid threshold "${thresholdArg}". Pass a number between 0 and 100.`)
process.exit(2)
}

let report
try {
report = JSON.parse(readFileSync(reportPath, 'utf8'))
} catch (error) {
console.error(`Could not read the vitest report at "${reportPath}": ${error.message}`)
console.error('The test run probably crashed before writing results. Treating as failure.')
process.exit(1)
}
Comment on lines +17 to +24

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If the JSON report file is empty or contains null, JSON.parse will succeed but return null (or a non-object like a string/number). This will cause a TypeError when attempting to access properties like report.numPassedTests on line 26, crashing the script with an unhandled exception.

We should explicitly validate that report is a non-null object after parsing.

Suggested change
let report
try {
report = JSON.parse(readFileSync(reportPath, 'utf8'))
} catch (error) {
console.error(`Could not read the vitest report at "${reportPath}": ${error.message}`)
console.error('The test run probably crashed before writing results. Treating as failure.')
process.exit(1)
}
let report
try {
report = JSON.parse(readFileSync(reportPath, 'utf8'))
if (!report || typeof report !== 'object') {
throw new Error('Parsed JSON is not an object')
}
} catch (error) {
console.error('Could not read or parse the vitest report at "' + reportPath + '": ' + error.message)
console.error('The test run probably crashed before writing results. Treating as failure.')
process.exit(1)
}

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e364354. Added a non-null object check right after the parse, so an empty or null report exits 1 cleanly instead of throwing on report.numPassedTests.


if (!report || typeof report !== 'object') {
console.error(`The vitest report at "${reportPath}" is empty or not a JSON object. Treating as failure.`)
process.exit(1)
}

const passed = report.numPassedTests ?? 0
const failed = report.numFailedTests ?? 0
const skipped = report.numPendingTests ?? 0
const todo = report.numTodoTests ?? 0
const total = report.numTotalTests ?? 0

const executed = passed + failed

console.log(
`Tests: ${passed} passed, ${failed} failed, ${skipped} skipped, ${todo} todo (${total} total across ${report.numTotalTestSuites ?? 0} suites)`,
)

// A suite that fails to compile/import/run never produces assertion results, so its
// tests silently vanish from the denominator instead of counting as failures. Catch
// those per-suite — checking only the run-wide count misses the case where one suite
// errors while others pass.
const erroredSuites = (report.testResults ?? []).filter(
(suite) => suite.status === 'failed' && (suite.assertionResults?.length ?? 0) === 0,
)

if (erroredSuites.length > 0) {
console.error(`${erroredSuites.length} test suite(s) failed to compile, import, or run before any test executed:`)
for (const suite of erroredSuites) {
console.error(` - ${suite.name}`)
}
process.exit(1)
}

if (executed === 0) {
console.error('No tests were executed. Treating as failure.')
process.exit(1)
}

const passRate = (passed / executed) * 100
console.log(`Pass rate (of executed tests): ${passRate.toFixed(2)}% — required: ${threshold}%`)

const epsilon = 1e-9
if (passRate + epsilon < threshold) {
console.error(`Pass rate ${passRate.toFixed(2)}% is below the ${threshold}% threshold.`)
process.exit(1)
}

console.log('Pass-rate gate satisfied.')
58 changes: 58 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
name: CI

on:
push:
branches: [master, dev]
pull_request:
workflow_dispatch:

# Cancel superseded runs on the same ref to save CI minutes.
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
verify:
name: Lint and unit tests
runs-on: ubuntu-latest
# Everything here runs in-process: no Postgres, no Docker/testcontainers, no
# network, no PDF fixtures. tests/integration (testcontainers-backed) and
# tests/perf need external resources, so they are not run in CI.
env:
NODE_ENV: test
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest

- name: Cache Bun install
uses: actions/cache@v4
with:
path: ~/.bun/install/cache
key: ${{ runner.os }}-bun-${{ hashFiles('bun.lockb') }}
restore-keys: |
${{ runner.os }}-bun-

- name: Install dependencies
run: bun install --frozen-lockfile

- name: Lint (oxlint)
run: bun run lint

- name: Unit tests
# Don't let a failing test abort the step — the pass-rate gate decides.
run: |
bunx vitest run tests/unit \
--reporter=default \
--reporter=json --outputFile=vitest-report.json \
|| true

- name: Enforce 95% pass rate
run: node .github/scripts/check-pass-rate.mjs vitest-report.json 95
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ docs/train/

# Brainstorming visual companion (local-only)
.superpowers/

# CI test report artifact
vitest-report.json
2 changes: 1 addition & 1 deletion .oxlintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
},
"overrides": [
{
"files": ["**/*.test.{ts,tsx}", "tests/**", "cli/**", ".claude/scripts/**"],
"files": ["**/*.test.{ts,tsx}", "tests/**", "cli/**", ".claude/scripts/**", ".github/scripts/**"],
"rules": {
"no-console": "off"
}
Expand Down
Loading