From 9781b0ac9bb6e2a1f51fda36db8effe746dca3b7 Mon Sep 17 00:00:00 2001 From: devanfer Date: Fri, 29 May 2026 21:36:25 +0700 Subject: [PATCH 1/2] ci: lint + unit tests with 95% pass-rate gate on every PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a CI workflow that runs on every pull request (any base branch), on pushes to master/dev, and on manual dispatch. Two gates, both fully in-process — no Postgres, Docker/testcontainers, network, or PDF fixtures: - oxlint (hard fail on lint errors) - tests/unit via vitest, then a pass-rate check: the build fails if fewer than 95% of executed tests pass The pass-rate gate (.github/scripts/check-pass-rate.mjs) parses vitest's JSON report so the test step itself doesn't abort the job; it also fails on a missing report or suites that error before any test runs (import or compile errors). tests/integration (testcontainers-backed) and tests/perf are intentionally excluded since they need external resources. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/check-pass-rate.mjs | 58 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 58 +++++++++++++++++++++++++++++ .gitignore | 3 ++ .oxlintrc.json | 2 +- 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/check-pass-rate.mjs create mode 100644 .github/workflows/ci.yml diff --git a/.github/scripts/check-pass-rate.mjs b/.github/scripts/check-pass-rate.mjs new file mode 100644 index 0000000..93ba272 --- /dev/null +++ b/.github/scripts/check-pass-rate.mjs @@ -0,0 +1,58 @@ +import { readFileSync } from 'node:fs' + +const [, , reportPath, thresholdArg] = process.argv + +if (!reportPath) { + console.error('Usage: check-pass-rate.mjs [thresholdPercent]') + process.exit(2) +} + +const threshold = Number(thresholdArg ?? '95') + +if (Number.isNaN(threshold) || threshold < 0 || threshold > 100) { + console.error(`Invalid threshold "${thresholdArg}". Pass a number between 0 and 100.`) + process.exit(2) +} + +let report +try { + report = JSON.parse(readFileSync(reportPath, 'utf8')) +} catch (error) { + console.error(`Could not read the vitest report at "${reportPath}": ${error.message}`) + console.error('The test run probably crashed before writing results. Treating as failure.') + process.exit(1) +} + +const passed = report.numPassedTests ?? 0 +const failed = report.numFailedTests ?? 0 +const skipped = report.numPendingTests ?? 0 +const todo = report.numTodoTests ?? 0 +const total = report.numTotalTests ?? 0 +const failedSuites = report.numFailedTestSuites ?? 0 + +const executed = passed + failed + +console.log( + `Tests: ${passed} passed, ${failed} failed, ${skipped} skipped, ${todo} todo (${total} total across ${report.numTotalTestSuites ?? 0} suites)`, +) + +if (failedSuites > 0 && executed === 0) { + console.error(`${failedSuites} test suite(s) errored before any test ran (likely an import or compile error).`) + process.exit(1) +} + +if (executed === 0) { + console.error('No tests were executed. Treating as failure.') + process.exit(1) +} + +const passRate = (passed / executed) * 100 +console.log(`Pass rate (of executed tests): ${passRate.toFixed(2)}% — required: ${threshold}%`) + +const epsilon = 1e-9 +if (passRate + epsilon < threshold) { + console.error(`Pass rate ${passRate.toFixed(2)}% is below the ${threshold}% threshold.`) + process.exit(1) +} + +console.log('Pass-rate gate satisfied.') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..306e97b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: CI + +on: + push: + branches: [master, dev] + pull_request: + workflow_dispatch: + +# Cancel superseded runs on the same ref to save CI minutes. +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + verify: + name: Lint and unit tests + runs-on: ubuntu-latest + # Everything here runs in-process: no Postgres, no Docker/testcontainers, no + # network, no PDF fixtures. tests/integration (testcontainers-backed) and + # tests/perf need external resources, so they are not run in CI. + env: + NODE_ENV: test + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Cache Bun install + uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-${{ hashFiles('bun.lockb') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Lint (oxlint) + run: bun run lint + + - name: Unit tests + # Don't let a failing test abort the step — the pass-rate gate decides. + run: | + bunx vitest run tests/unit \ + --reporter=default \ + --reporter=json --outputFile=vitest-report.json \ + || true + + - name: Enforce 95% pass rate + run: node .github/scripts/check-pass-rate.mjs vitest-report.json 95 diff --git a/.gitignore b/.gitignore index b290c3f..2be014c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ docs/train/ # Brainstorming visual companion (local-only) .superpowers/ + +# CI test report artifact +vitest-report.json diff --git a/.oxlintrc.json b/.oxlintrc.json index 321b7e2..26b4c76 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -25,7 +25,7 @@ }, "overrides": [ { - "files": ["**/*.test.{ts,tsx}", "tests/**", "cli/**", ".claude/scripts/**"], + "files": ["**/*.test.{ts,tsx}", "tests/**", "cli/**", ".claude/scripts/**", ".github/scripts/**"], "rules": { "no-console": "off" } From e364354883d2b3b8976798ec22c12a741564859f Mon Sep 17 00:00:00 2001 From: devanfer Date: Fri, 29 May 2026 22:00:32 +0700 Subject: [PATCH 2/2] fix(ci): catch per-suite import errors and guard non-object reports Address Gemini review on the pass-rate gate: - A suite that fails to compile/import produces no assertion results, so its tests drop out of the denominator instead of counting as failures. The old guard only fired when zero tests ran across the whole job, so a broken suite could pass CI as long as the surviving suites cleared 95%. Now scan testResults per suite for status 'failed' with empty assertionResults and fail with the offending file names. - An empty or null report parses to null; reading numPassedTests off it threw a TypeError. Validate it's a non-null object before use. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/check-pass-rate.mjs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/scripts/check-pass-rate.mjs b/.github/scripts/check-pass-rate.mjs index 93ba272..53a76ff 100644 --- a/.github/scripts/check-pass-rate.mjs +++ b/.github/scripts/check-pass-rate.mjs @@ -23,12 +23,16 @@ try { process.exit(1) } +if (!report || typeof report !== 'object') { + console.error(`The vitest report at "${reportPath}" is empty or not a JSON object. Treating as failure.`) + process.exit(1) +} + const passed = report.numPassedTests ?? 0 const failed = report.numFailedTests ?? 0 const skipped = report.numPendingTests ?? 0 const todo = report.numTodoTests ?? 0 const total = report.numTotalTests ?? 0 -const failedSuites = report.numFailedTestSuites ?? 0 const executed = passed + failed @@ -36,8 +40,19 @@ console.log( `Tests: ${passed} passed, ${failed} failed, ${skipped} skipped, ${todo} todo (${total} total across ${report.numTotalTestSuites ?? 0} suites)`, ) -if (failedSuites > 0 && executed === 0) { - console.error(`${failedSuites} test suite(s) errored before any test ran (likely an import or compile error).`) +// A suite that fails to compile/import/run never produces assertion results, so its +// tests silently vanish from the denominator instead of counting as failures. Catch +// those per-suite — checking only the run-wide count misses the case where one suite +// errors while others pass. +const erroredSuites = (report.testResults ?? []).filter( + (suite) => suite.status === 'failed' && (suite.assertionResults?.length ?? 0) === 0, +) + +if (erroredSuites.length > 0) { + console.error(`${erroredSuites.length} test suite(s) failed to compile, import, or run before any test executed:`) + for (const suite of erroredSuites) { + console.error(` - ${suite.name}`) + } process.exit(1) }