From 9314b7cd33690880eb3a21755b4fdc49e88e6440 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 01:34:29 +0000 Subject: [PATCH 1/3] ci(repo): render mutation job failures from the failure record The mutation job decodes the stream's terminal record, renders the summary and annotations through the contract renderers, builds RecordMissing, JobTimedOut and BinaryMissing records itself, reports a run that evaluated no mutants as exactly that, and no longer calls a failed run an infrastructure failure Verdict-Semantics: unchanged --- .changeset/proud-bananas-work.md | 5 + .../src/render-annotations.workflow.ts | 20 +- packages/stryker-js/src/run-request.cell.ts | 20 + scripts/deno.json | 6 +- scripts/deno.lock | 366 +++++++++++++++++- scripts/lib/mutation-plan.test.ts | 82 ++-- scripts/lib/mutation-plan.ts | 153 +++++--- scripts/mutation-job.ts | 2 +- 8 files changed, 579 insertions(+), 75 deletions(-) create mode 100644 .changeset/proud-bananas-work.md diff --git a/.changeset/proud-bananas-work.md b/.changeset/proud-bananas-work.md new file mode 100644 index 000000000..826744cf9 --- /dev/null +++ b/.changeset/proud-bananas-work.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +stryker annotate also renders the GitHub annotations of reports/mutation/failure.json when a run failed. diff --git a/packages/stryker-js/src/render-annotations.workflow.ts b/packages/stryker-js/src/render-annotations.workflow.ts index cf2a199c6..f51e3099f 100644 --- a/packages/stryker-js/src/render-annotations.workflow.ts +++ b/packages/stryker-js/src/render-annotations.workflow.ts @@ -59,6 +59,7 @@ export class RenderAnnotationsCommand extends S.TaggedClass ].join(',') }::${escapeMessage(messageOf(entry))}` -const linesOf = (command: RenderAnnotationsCommand): ReadonlyArray => { - const byId = entriesByIdOf(command.report) - const baseline = HashSet.fromIterable(command.baseline) +const survivorLinesOf = ( + report: Report.MutationTestResult, + survivors: ReadonlyArray, + baseline: ReadonlyArray, +): ReadonlyArray => { + const byId = entriesByIdOf(report) + const baselineSet = HashSet.fromIterable(baseline) return Arr.flatMap( - command.survivors, + survivors, (ref) => - Boolean.match(HashSet.has(baseline, ref.id), { + Boolean.match(HashSet.has(baselineSet, ref.id), { onTrue: (): ReadonlyArray => [], onFalse: () => Arr.flatMap(Option.toArray(HashMap.get(byId, ref.id)), (entry) => [lineOf(entry)]), }), ) } +const linesOf = (command: RenderAnnotationsCommand): ReadonlyArray => [ + ...survivorLinesOf(command.report, command.survivors, command.baseline), + ...Option.fromUndefinedOr(command.failureAnnotations).pipe(Option.getOrElse((): ReadonlyArray => [])), +] + const decide = (command: RenderAnnotationsCommand): RenderAnnotationsDecision => { const lines = linesOf(command) return Boolean.match(Arr.length(lines) === 0, { diff --git a/packages/stryker-js/src/run-request.cell.ts b/packages/stryker-js/src/run-request.cell.ts index 037e6910b..7056a5454 100644 --- a/packages/stryker-js/src/run-request.cell.ts +++ b/packages/stryker-js/src/run-request.cell.ts @@ -1,6 +1,7 @@ import { Cell, Sandwich } from '@systemfsoftware/effect-cell-types' import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, Report } from '@systemfsoftware/stryker-js-plugin-interface' import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import * as CliError from 'effect/cli/CliError' @@ -61,6 +62,7 @@ import { mutationTestCell } from './run/run-stages.cell.js' import { RunEnvironment } from './run/RunEnvironment.service.js' import { serveMutationServer, type ServeRequest } from './Serve/Serve.cell.js' import { StrykerError } from './stryker-error.schema.js' +import { FAILURE_RECORD_FILE } from './stryker-outputs.js' import { annotationLinesOf, surfacedSurvivorsOf } from './surfacing.js' import { type SurfacingCaps, SurfacingFields } from './surfacing.schema.js' import type { SurvivorsAdmissionInput, SurvivorsSettlement } from './Survivors/mod.js' @@ -385,6 +387,7 @@ const surfacingCapsOf = (report: Report.MutationTestResult): SurfacingCaps => Option.getOrElse(Option.map(surfacingFieldsOf(report), capsOf), () => SURFACING_DEFAULTS) const decodeAnnotateReport = S.decodeUnknownResult(S.fromJsonString(Report.MutationTestResult)) +const decodeAnnotateFailure = S.decodeUnknownOption(FailureRecord.FailureRecordFile) const readAnnotateReport = ( file: string, @@ -409,6 +412,21 @@ const readAnnotateReport = ( ), )) +const readAnnotateFailureAnnotations = ( + file: string, +): Effect.Effect, never, FileSystem.FileSystem> => + Effect.flatMap(FileSystem.FileSystem, (fs) => + fs.readFileString(file).pipe( + Effect.asSome, + Effect.catchTag('PlatformError', () => Effect.succeed(Option.none())), + Effect.map((text) => + Option.flatMap(text, decodeAnnotateFailure).pipe( + Option.map(FailureRecord.annotationsOf), + Option.getOrElse((): ReadonlyArray => []), + ) + ), + )) + const readAnnotateBaseline = ( file: string, ): Effect.Effect, AnnotationsUnusable, FileSystem.FileSystem> => @@ -435,6 +453,7 @@ const annotateReport = ( const path = yield* Path.Path const basePath = channel.environment.basePath const report = yield* readAnnotateReport(path.resolve(basePath, GATE_REPORT_FILE)) + const failureAnnotations = yield* readAnnotateFailureAnnotations(path.resolve(basePath, FAILURE_RECORD_FILE)) const baseline = yield* Effect.forEach( Option.toArray(Option.fromUndefinedOr(annotate.baseline)), (file) => readAnnotateBaseline(path.resolve(basePath, file)), @@ -445,6 +464,7 @@ const annotateReport = ( report, survivors: surfacedSurvivorsOf(report, surfacingCapsOf(report)), baseline: baseline.flat(), + failureAnnotations, }), ), ) diff --git a/scripts/deno.json b/scripts/deno.json index a56f64f5a..1543a704a 100644 --- a/scripts/deno.json +++ b/scripts/deno.json @@ -1,8 +1,10 @@ { "compilerOptions": { "strict": true, - "noImplicitOverride": true + "noImplicitOverride": true, + "types": ["vitest/importMeta"] }, + "unstable": ["sloppy-imports"], "minimumDependencyAge": { "age": "P1D", "exclude": ["npm:effect"] @@ -19,6 +21,8 @@ "@std/fs/expand-glob": "jsr:@std/fs@1/expand-glob", "@std/path": "jsr:@std/path@1", "@std/yaml": "jsr:@std/yaml@1", + "@systemfsoftware/vitest": "npm:@systemfsoftware/vitest@^1.0.0", + "vitest": "npm:vitest@^5", "effect": "npm:effect@^4.0.0", "octokit": "npm:octokit@^4" } diff --git a/scripts/deno.lock b/scripts/deno.lock index 2ff9f8d39..dfdcd5624 100644 --- a/scripts/deno.lock +++ b/scripts/deno.lock @@ -17,8 +17,10 @@ "jsr:@std/toml@^1.0.3": "1.0.11", "jsr:@std/yaml@1": "1.2.0", "jsr:@std/yaml@^1.0.5": "1.2.0", + "npm:@systemfsoftware/vitest@1": "1.0.0_effect@4.0.0_vitest@5.0.1", "npm:effect@4": "4.0.0", - "npm:octokit@4": "4.1.4" + "npm:octokit@4": "4.1.4", + "npm:vitest@5": "5.0.1_vite@8.3.1" }, "jsr": { "@std/assert@1.0.19": { @@ -73,6 +75,19 @@ } }, "npm": { + "@jridgewell/resolve-uri@3.1.2": { + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==" + }, + "@jridgewell/sourcemap-codec@1.6.0": { + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==" + }, + "@jridgewell/trace-mapping@0.3.31": { + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dependencies": [ + "@jridgewell/resolve-uri", + "@jridgewell/sourcemap-codec" + ] + }, "@octokit/app@15.1.6": { "integrity": "sha512-WELCamoCJo9SN0lf3SWZccf68CF0sBNPQuLYmZ/n87p5qvBJDe9aBtr5dHkh7T9nxWZ608pizwsUbypSzZAiUw==", "dependencies": [ @@ -265,21 +280,260 @@ "@octokit/webhooks-methods" ] }, + "@oxc-project/types@0.152.0": { + "integrity": "sha512-oM/5rLBm2tPkg0iBgkH/FOeR3PCDpY19GTgAZjMFM8h9WI9VW7cLgzp6nwtarYKmovavIQZ+Fe/RKX/8C8O/Rw==" + }, + "@rolldown/binding-android-arm-eabi@1.2.12": { + "integrity": "sha512-dB/a1214qKfHMXCpgqR4OZT+jS4kTyEXbQGJPqzobt5EwH5rX080pxE37alt3RzvR1bf1Yz/yGqRfrYAxuPw0A==", + "os": ["android"], + "cpu": ["arm"] + }, + "@rolldown/binding-android-arm64@1.2.12": { + "integrity": "sha512-7KHFgQ5VJxIHcLlrwrc3Xbds7oTNQT7Pgi9gQCJKrd2VGab/UksIOYp6VD8MzCstGxOKMgNamPwUCfxPdP1OHg==", + "os": ["android"], + "cpu": ["arm64"] + }, + "@rolldown/binding-darwin-arm64@1.2.12": { + "integrity": "sha512-3YIhqHD96nA5SaYNRBR16HnGv4oavZvXfD/ayHM+oYZ0WD/8lBAtf6zQua4kEyAvpqrluKXl0lnOBoiNby7x9w==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "@rolldown/binding-darwin-x64@1.2.12": { + "integrity": "sha512-UuuJ35MFw4gmFOrE9pEqIV+K3syIKveph+Qc1/ljHZVdoDW4pz/JHR/eMVom+TZGl/5OOvGJOWaOCVt3ZfqhxA==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "@rolldown/binding-freebsd-x64@1.2.12": { + "integrity": "sha512-uMvssit0a4W+/7D8CbHUvG719mH3R2jwXAlh/XcPvuHTE0g++LymF88DCGNX0HM2rBOn0xrzgXktIB6fLSJBTQ==", + "os": ["freebsd"], + "cpu": ["x64"] + }, + "@rolldown/binding-linux-arm-gnueabihf@1.2.12": { + "integrity": "sha512-XcFu0R0xWnwzSf4IQgFH1rJIckPN1pLy2R+4r9IDB7Yfu/ys9cVqfa4pBrMHj7a3gl8mIR4nRNPg0e5IvEVs6g==", + "os": ["linux"], + "cpu": ["arm"] + }, + "@rolldown/binding-linux-arm64-gnu@1.2.12": { + "integrity": "sha512-260UrKgn8tz39ak+SMDOirKzr7V04M9dWPw5llW00SwBivCZoWcRBKV1d8cXnRkUmSZA3BdiUmBHWk7734Ulpw==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@rolldown/binding-linux-arm64-musl@1.2.12": { + "integrity": "sha512-5YK1I9SqDkbPgc1IA8BgDl34suqUS2q0KWnBrirm0E51YjOs6eo6dV6jbQfNE/argHRSvd0QUGgtpIoYx+WWpw==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@rolldown/binding-linux-ppc64-gnu@1.2.12": { + "integrity": "sha512-Rkcrmp7eFRg74yL5fXEU91JEWbdEPLevWwGtXpmhbjlD1StScbWTmO94Bhly+Mo+ketKYkdmM1vNUKeWSlx8cQ==", + "os": ["linux"], + "cpu": ["ppc64"] + }, + "@rolldown/binding-linux-s390x-gnu@1.2.12": { + "integrity": "sha512-qvK4DuAsQc2BSjlx+Xr+IzOIvvxbGZqxFwdWfG6F518Erj0GGISyQbJ6pIappnOxlNPzNHvo/L0BwB30GZ+zVw==", + "os": ["linux"], + "cpu": ["s390x"] + }, + "@rolldown/binding-linux-x64-gnu@1.2.12": { + "integrity": "sha512-Q9uLBO53Xd4QIq1WOycVQyPP1O4HhraEV2qqb3uTrnVw6QZih9duY4vNXOivL1xoUS1/z+W8eF4NMfl2a8Sdjw==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@rolldown/binding-linux-x64-musl@1.2.12": { + "integrity": "sha512-3IBxWFMjbOZskDPKv8Lf9BCnahlKuHthWkYnyIxOH/QcJrFcS4EmcenthApkwr/5+nEqZlLzeYbxeMaX7A5u4g==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@rolldown/binding-openharmony-arm64@1.2.12": { + "integrity": "sha512-xtX61xg4LKPkPWilZU1ynKClz5Gj4bf74LML4r3eVLWumKnGjoEr1OSHQhMdbBDoYTi+yjrujvpZe2pUnqCrrA==", + "os": ["openharmony"], + "cpu": ["arm64"] + }, + "@rolldown/binding-win32-arm64-msvc@1.2.12": { + "integrity": "sha512-At7fPB6PCaIjzgIhEZFxuT+BBFqiQibJDT4d3PhiR3f4E7bbMZF4aKblbFfEM3sETRDd1YiQx/+U/g/B/ou5Ew==", + "os": ["win32"], + "cpu": ["arm64"] + }, + "@rolldown/binding-win32-x64-msvc@1.2.12": { + "integrity": "sha512-WIw2haVKwjuYdXkHaoC0mF8Le71TuCBxjrdKqLbJGctbBABj+ClfmNvtbOnzpq3RokNo5+V1qhtSzJyXorsklQ==", + "os": ["win32"], + "cpu": ["x64"] + }, + "@rolldown/pluginutils@1.0.1": { + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==" + }, + "@systemfsoftware/vitest@1.0.0_effect@4.0.0_vitest@5.0.1": { + "integrity": "sha512-B3v+wlfcAZwG6HdHuDC4pVMi9O9c5Zvrr8pcCBxtS9zVypvWDAvW8f0iXzlRMSQHaH04GOiX9uv0S4HkMPHnqA==", + "dependencies": [ + "effect", + "vitest" + ] + }, "@types/aws-lambda@8.10.163": { "integrity": "sha512-+4zuoEB3S8RIhimtOFT7zAEk2SbpwrKjjGl9CyYnQR6k08uynxfauIIB0pDU1ssr05F8oyGiETwpn+8eXZMqPw==" }, + "@types/chai@5.2.3": { + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dependencies": [ + "@types/deep-eql", + "assertion-error" + ] + }, + "@types/deep-eql@4.0.2": { + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==" + }, + "@types/estree@1.0.9": { + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==" + }, + "@vitest/mocker@5.0.1_vite@8.3.1": { + "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", + "dependencies": [ + "@jridgewell/trace-mapping", + "@vitest/spy", + "estree-walker", + "magic-string", + "vite" + ], + "optionalPeers": [ + "vite" + ] + }, + "@vitest/spy@5.0.1": { + "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==" + }, + "assertion-error@2.0.1": { + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==" + }, "before-after-hook@3.0.2": { "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==" }, "bottleneck@2.19.5": { "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, + "chai@6.3.0": { + "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==" + }, + "detect-libc@2.1.2": { + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==" + }, "effect@4.0.0": { "integrity": "sha512-ooc1TG5t+FfzgYnFz2ff6BBKyZ7EwBRVXC7c4RhQUAD6/TZ2gTXXMeb4WX7a19ozQo4J73/QW+S00YAIresoMQ==" }, + "es-module-lexer@2.3.2": { + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==" + }, + "estree-walker@3.0.3": { + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dependencies": [ + "@types/estree" + ] + }, + "expect-type@1.4.0": { + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==" + }, "fast-content-type-parse@2.0.1": { "integrity": "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q==" }, + "fdir@6.5.0_picomatch@4.0.7": { + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dependencies": [ + "picomatch" + ], + "optionalPeers": [ + "picomatch" + ] + }, + "fsevents@2.3.3": { + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "os": ["darwin"], + "scripts": true + }, + "lightningcss-android-arm64@1.33.0": { + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "os": ["android"], + "cpu": ["arm64"] + }, + "lightningcss-darwin-arm64@1.33.0": { + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "lightningcss-darwin-x64@1.33.0": { + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "lightningcss-freebsd-x64@1.33.0": { + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "os": ["freebsd"], + "cpu": ["x64"] + }, + "lightningcss-linux-arm-gnueabihf@1.33.0": { + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "os": ["linux"], + "cpu": ["arm"] + }, + "lightningcss-linux-arm64-gnu@1.33.0": { + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "lightningcss-linux-arm64-musl@1.33.0": { + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "lightningcss-linux-x64-gnu@1.33.0": { + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "os": ["linux"], + "cpu": ["x64"] + }, + "lightningcss-linux-x64-musl@1.33.0": { + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "os": ["linux"], + "cpu": ["x64"] + }, + "lightningcss-win32-arm64-msvc@1.33.0": { + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "os": ["win32"], + "cpu": ["arm64"] + }, + "lightningcss-win32-x64-msvc@1.33.0": { + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "os": ["win32"], + "cpu": ["x64"] + }, + "lightningcss@1.33.0": { + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dependencies": [ + "detect-libc" + ], + "optionalDependencies": [ + "lightningcss-android-arm64", + "lightningcss-darwin-arm64", + "lightningcss-darwin-x64", + "lightningcss-freebsd-x64", + "lightningcss-linux-arm-gnueabihf", + "lightningcss-linux-arm64-gnu", + "lightningcss-linux-arm64-musl", + "lightningcss-linux-x64-gnu", + "lightningcss-linux-x64-musl", + "lightningcss-win32-arm64-msvc", + "lightningcss-win32-x64-msvc" + ] + }, + "magic-string@1.4.2": { + "integrity": "sha512-vG+rjFRj1PqdIBozIxAGMjPlOhaVe+GXpbttY/iSK7rGcJRMlwNJO7dcUwmUqkymsFLJiNGI06t4D7Fr7yRC9g==", + "dependencies": [ + "@jridgewell/sourcemap-codec" + ] + }, + "nanoid@3.3.19": { + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "bin": true + }, + "obug@2.2.1": { + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==" + }, "octokit@4.1.4": { "integrity": "sha512-cRvxRte6FU3vAHRC9+PMSY3D+mRAs2Rd9emMoqp70UGRvJRM3sbAoim2IXRZNNsf8wVfn4sGxVBHRAP+JBVX/g==", "dependencies": [ @@ -296,6 +550,70 @@ "@octokit/webhooks" ] }, + "picocolors@1.1.1": { + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "picomatch@4.0.7": { + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==" + }, + "postcss@8.5.28": { + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dependencies": [ + "nanoid", + "picocolors", + "source-map-js" + ] + }, + "rolldown@1.2.12": { + "integrity": "sha512-8wafseiaG80xmXSfqidUNqZcylTlhmPZZt+za2m+js2sFZ8dTNlhIOV2WcbIPx2hgwPBJpEUGFAMZ9bgBBLTSQ==", + "dependencies": [ + "@oxc-project/types", + "@rolldown/pluginutils" + ], + "optionalDependencies": [ + "@rolldown/binding-android-arm-eabi", + "@rolldown/binding-android-arm64", + "@rolldown/binding-darwin-arm64", + "@rolldown/binding-darwin-x64", + "@rolldown/binding-freebsd-x64", + "@rolldown/binding-linux-arm-gnueabihf", + "@rolldown/binding-linux-arm64-gnu", + "@rolldown/binding-linux-arm64-musl", + "@rolldown/binding-linux-ppc64-gnu", + "@rolldown/binding-linux-s390x-gnu", + "@rolldown/binding-linux-x64-gnu", + "@rolldown/binding-linux-x64-musl", + "@rolldown/binding-openharmony-arm64", + "@rolldown/binding-win32-arm64-msvc", + "@rolldown/binding-win32-x64-msvc" + ], + "bin": true + }, + "siginfo@2.0.0": { + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==" + }, + "source-map-js@1.2.2": { + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==" + }, + "stackback@0.0.2": { + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==" + }, + "std-env@4.3.0": { + "integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==" + }, + "tinybench@6.1.4": { + "integrity": "sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==" + }, + "tinyexec@1.3.0": { + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==" + }, + "tinyglobby@0.2.17": { + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dependencies": [ + "fdir", + "picomatch" + ] + }, "toad-cache@3.7.4": { "integrity": "sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==" }, @@ -304,6 +622,48 @@ }, "universal-user-agent@7.0.3": { "integrity": "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==" + }, + "vite@8.3.1": { + "integrity": "sha512-/bvH9E9tmCXRGp2uXY3WbOldqpTwFkbha/8ANaEQ6VkxhH60KyqLwgZq6lG2y+4uT55x9+9eUHMpQ7uGnOCKjA==", + "dependencies": [ + "lightningcss", + "picomatch", + "postcss", + "rolldown", + "tinyglobby" + ], + "optionalDependencies": [ + "fsevents" + ], + "bin": true + }, + "vitest@5.0.1_vite@8.3.1": { + "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", + "dependencies": [ + "@types/chai", + "@vitest/mocker", + "chai", + "es-module-lexer", + "expect-type", + "magic-string", + "obug", + "picomatch", + "std-env", + "tinybench", + "tinyexec", + "tinyglobby", + "vite", + "why-is-node-running" + ], + "bin": true + }, + "why-is-node-running@2.3.0": { + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dependencies": [ + "siginfo", + "stackback" + ], + "bin": true } }, "workspace": { @@ -317,8 +677,10 @@ "jsr:@std/fs@1", "jsr:@std/path@1", "jsr:@std/yaml@1", + "npm:@systemfsoftware/vitest@1", "npm:effect@4", - "npm:octokit@4" + "npm:octokit@4", + "npm:vitest@5" ] } } diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index 9589bcb89..33facc40d 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -17,6 +17,7 @@ import { PnpmWorkspaceSchema, ReportSchema, type StagedPart, + type SummaryInput, type TimingRecord, } from './mutation-plan.ts' @@ -25,21 +26,26 @@ const readFileFor = (files: Record) => (path: string): Promise { - const base = { package: 'packages/x', outcome: 'failure', reportsDir: dir } as const - - const empty = readFileFor({}) - const zero = buildRequireError(base, await loadState(dir, empty)) - assertStringIncludes(zero ?? '', 'zero mutant results') +const inputOf = (overrides: Partial = {}): SummaryInput => ({ + package: 'packages/x', + outcome: 'failure', + reportsDir: dir, + exitCode: 3, + cwd: '/repo', + ...overrides, +}) - const streaming = readFileFor({ - [`${dir}/mutation-stream.jsonl`]: '{"_tag":"phase"}\n{"_tag":"mutant","id":"m1"}\n{"_tag":"mutant","id":"m2"}\n', - }) - const partial = buildRequireError(base, await loadState(dir, streaming)) - assertStringIncludes(partial ?? '', 'after 2 completed mutant(s)') +Deno.test('a package whose reports dir has no report requires one, naming the exit code', async () => { + const state = await loadState(dir, readFileFor({})) + assertStringIncludes(buildRequireError(inputOf({ exitCode: 7 }), state) ?? '', 'exit code: 7') +}) - const complete = readFileFor({ [`${dir}/mutation-report.json`]: '{"schemaVersion":"1.0","files":{}}' }) - assertEquals(buildRequireError(base, await loadState(dir, complete)), null) +Deno.test('a complete report needs no record and no gate failure', async () => { + const state = await loadState( + dir, + readFileFor({ [`${dir}/mutation-report.json`]: '{"schemaVersion":"1.0","files":{}}' }), + ) + assertEquals(buildRequireError(inputOf(), state), null) }) const shardPart = (index: number, files: string[]): StagedPart => ({ @@ -102,7 +108,28 @@ Deno.test('mergeRecord keeps the previous duration when a shard is missing, sums assertEquals(mergeRecord(previous, complete, 'new').packages['p'], { seconds: 200, sha: 'new' }) }) -const input = { package: 'packages/x', outcome: 'failure', reportsDir: dir } as const +const input = inputOf() + +const failureRecord = { + _tag: 'BaselineTestsFailed', + stage: 'dryRun', + testCount: 1, + tests: [{ + id: 'math.test.ts > adds numbers', + name: 'adds numbers', + file: 'src/math.test.ts', + location: { file: 'src/math.test.ts', line: 12, column: 3 }, + message: 'expected 3 to be 4', + stack: null, + reproduce: ['vitest', 'run', 'src/math.test.ts', '-t', 'adds numbers'], + }], + cause: [{ kind: 'AssertionError', message: 'expected 3 to be 4', stack: null }], + capsule: { _tag: 'DoesNotReplay', why: 'interrupted', standIn: 'npx vitest run src/math.test.ts' }, + nextAction: { primary: 'fixCode', otherwise: 'fixTest' }, + traceId: null, +} as const + +const failureStream = `${JSON.stringify({ _tag: 'error', schemaVersion: '3.0', code: 5, record: failureRecord })}\n` Deno.test('buildSummary marks a report with schemaVersion and files as complete', async () => { const state = await loadState( @@ -124,19 +151,30 @@ Deno.test('buildSummary flags a report that is not a valid Stryker report', asyn assertStringIncludes(buildSummary(input, state), 'not a valid Stryker report') }) -Deno.test('buildSummary reports zero mutants when there is no report and no stream', async () => { +Deno.test('a terminal BaselineTestsFailed record yields a summary with the code, test location and capsule', async () => { + const state = await loadState(dir, readFileFor({ [`${dir}/mutation-stream.jsonl`]: failureStream })) + const summary = buildSummary(input, state) + assertStringIncludes(summary, 'BaselineTestsFailed') + assertStringIncludes(summary, 'src/math.test.ts:12:3') + assertStringIncludes(summary, 'adds numbers') + assertStringIncludes(summary, '**Replay:**') + assertEquals(summary.includes('infrastructure'), false) +}) + +Deno.test('a run with neither report nor terminal record yields a RecordMissing summary naming the exit code', async () => { const state = await loadState(dir, readFileFor({})) - assertStringIncludes(buildSummary(input, state), 'zero completed mutants') + const summary = buildSummary(inputOf({ exitCode: 6 }), state) + assertStringIncludes(summary, 'RecordMissing') + assertStringIncludes(summary, 'exit code: 6') }) -Deno.test('buildSummary counts completed mutants from a partial stream', async () => { +Deno.test('a reuse line with ran: 0 reports the evaluated-none outcome and a non-zero gate', async () => { const state = await loadState( dir, - readFileFor({ - [`${dir}/mutation-stream.jsonl`]: '{"_tag":"mutant","id":"m1"}\n{"_tag":"phase"}\n{"_tag":"mutant","id":"m2"}\n', - }), + readFileFor({ [`${dir}/mutation-stream.jsonl`]: '{"_tag":"reuse","reused":4,"ran":0,"refused":{}}\n' }), ) - assertStringIncludes(buildSummary(input, state), '2 completed mutant(s)') + assertStringIncludes(buildSummary(input, state), 'evaluated no mutants') + assertEquals(buildRequireError(input, state) !== null, true) }) Deno.test('a cleared reports dir cannot satisfy the no-report gate', async () => { @@ -149,7 +187,7 @@ Deno.test('a cleared reports dir cannot satisfy the no-report gate', async () => assertEquals(buildRequireError(input, stale), null) const cleared = await loadState(dir, readFileFor({})) - assertStringIncludes(buildRequireError(input, cleared) ?? '', 'zero mutant results') + assertEquals(buildRequireError(input, cleared) !== null, true) }) Deno.test('decodeJson rejects a malformed part meta, naming the source', () => { diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index 215c46a74..16055ca5e 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -1,7 +1,17 @@ +import * as Arr from 'effect/Array' +import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { type RecordContext, recordOf } from '../../packages/stryker-js-cli-contract/src/failure-catalog.ts' +import { + type Capsule, + type FailureEvidence, + FailureRecord, +} from '../../packages/stryker-js-cli-contract/src/failure-record.schema.ts' +import { annotationsOf, markdownOf } from '../../packages/stryker-js-cli-contract/src/render-failure.ts' + export const ShardSchema = S.Struct({ index: S.Int, count: S.Int }) export type Shard = S.Schema.Type @@ -205,27 +215,6 @@ export const summaryTable = (parts: readonly Part[], target: number, taskName = ].join('\n') } -const tagOf = (value: unknown): unknown => { - if (typeof value !== 'object' || value === null || !('_tag' in value)) return undefined - return value._tag -} - -const countMutantLines = (text: string): number => { - let n = 0 - for (const raw of text.split('\n')) { - const line = raw.trim() - if (line.length === 0) continue - let parsed: unknown - try { - parsed = JSON.parse(line) - } catch { - continue - } - if (tagOf(parsed) === 'mutant') n += 1 - } - return n -} - const isCompleteReport = (text: string): boolean => { let parsed: unknown try { @@ -240,7 +229,39 @@ const isCompleteReport = (text: string): boolean => { typeof files === 'object' && files !== null } -export type ReportState = { reportText: string | null; streamText: string | null } +type FailureRecordType = S.Schema.Type + +const RunFailedLineSchema = S.Struct({ + _tag: S.Literal('error'), + schemaVersion: S.Literal('3.0'), + code: S.Int, + record: FailureRecord, +}) + +const ReuseLineSchema = S.Struct({ + _tag: S.Literal('reuse'), + ran: S.Int.check(S.isGreaterThanOrEqualTo(0)), +}) + +const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder): ReadonlyArray => + (streamText ?? '').split('\n').flatMap((line) => + Option.toArray(S.decodeUnknownOption(S.fromJsonString(schema))(line)) + ) + +const terminalRecordOf = (streamText: string | null): Option.Option => + Option.map(Arr.last(decodedLines(streamText, RunFailedLineSchema)), (line) => line.record) + +export type ReuseState = { readonly ran: number } + +const reuseOf = (streamText: string | null): Option.Option => + Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran })) + +export type ReportState = { + reportText: string | null + streamText: string | null + failure: Option.Option + reuse: Option.Option +} export const loadState = async ( reportsDir: string, @@ -248,19 +269,80 @@ export const loadState = async ( ): Promise => { const reportText = await readFile(`${reportsDir}/mutation-report.json`).catch(() => null) const streamText = await readFile(`${reportsDir}/mutation-stream.jsonl`).catch(() => null) - return { reportText, streamText } + return { reportText, streamText, failure: terminalRecordOf(streamText), reuse: reuseOf(streamText) } } export interface SummaryInput { readonly package: string readonly outcome: Outcome readonly reportsDir: string + readonly exitCode: number | null + readonly cwd: string + readonly limitSeconds?: number } +const TIMEOUT_EXIT_CODE = 124 +const BINARY_MISSING_EXIT_CODE = 127 +const CI_BINARY = 'pnpm' + +type CiEvidence = Extract + +const ciEvidenceOf = (input: SummaryInput): CiEvidence => + Match.value(input.exitCode).pipe( + Match.when(TIMEOUT_EXIT_CODE, (): CiEvidence => ({ + _tag: 'JobTimedOut', + stage: 'ci', + limitSeconds: input.limitSeconds ?? 0, + })), + Match.when(BINARY_MISSING_EXIT_CODE, (): CiEvidence => ({ + _tag: 'BinaryMissing', + stage: 'ci', + binary: CI_BINARY, + })), + Match.orElse((): CiEvidence => ({ _tag: 'RecordMissing', stage: 'ci', exitCode: input.exitCode })), + ) + +const standInOf = (input: SummaryInput): string => + `stryker mutation --filter ${input.package} (exit ${input.exitCode === null ? 'unknown' : input.exitCode})` + +const capsuleOf = (evidence: CiEvidence, input: SummaryInput): Capsule => + Match.value(evidence).pipe( + Match.tagsExhaustive({ + RecordMissing: (): Capsule => ({ _tag: 'DoesNotReplay', why: 'recordMissing', standIn: standInOf(input) }), + JobTimedOut: (): Capsule => ({ _tag: 'DoesNotReplay', why: 'jobTimedOut', standIn: standInOf(input) }), + BinaryMissing: (): Capsule => ({ _tag: 'DoesNotReplay', why: 'binaryMissing', standIn: standInOf(input) }), + }), + ) + +const ciRecordOf = (input: SummaryInput): FailureRecordType => { + const evidence = ciEvidenceOf(input) + const context: RecordContext = { + cause: [], + cwd: input.cwd, + argv: ['stryker', 'mutation', '--filter', input.package], + env: [], + capsule: capsuleOf(evidence, input), + traceId: null, + } + return recordOf(evidence, context) +} + +export const recordFor = (input: SummaryInput, state: ReportState): FailureRecordType => + Option.getOrElse(state.failure, () => ciRecordOf(input)) + +const outcomeLabelOf = (input: SummaryInput, state: ReportState): string => + Option.match(state.reuse, { + onNone: () => input.outcome, + onSome: (reuse) => (reuse.ran === 0 ? 'evaluated no mutants' : input.outcome), + }) + export const buildSummary = (input: SummaryInput, state: ReportState): string => { const reportPath = `${input.reportsDir}/mutation-report.json` - const streamPath = `${input.reportsDir}/mutation-stream.jsonl` - const lines = [`#### Mutation · **${input.package}**`, '', `- **Stryker outcome**: **${input.outcome}**`] + const lines = [ + `#### Mutation · **${input.package}**`, + '', + `- **Stryker outcome**: **${outcomeLabelOf(input, state)}**`, + ] if (state.reportText !== null) { if (isCompleteReport(state.reportText)) { @@ -273,30 +355,13 @@ export const buildSummary = (input: SummaryInput, state: ReportState): string => return `${lines.join('\n')}\n` } - const mutants = state.streamText === null ? 0 : countMutantLines(state.streamText) - if (mutants === 0) { - lines.push( - `- **Result**: no final report and zero completed mutants — infrastructure failure (missing binary, crashed run or timeout). Stream: **${streamPath}**`, - ) - } else { - lines.push( - `- **Result**: no final report (run interrupted) — ${mutants} completed mutant(s) recorded, marked incomplete in the merged report. Stream: **${streamPath}**`, - ) - } + lines.push(markdownOf(recordFor(input, state))) return `${lines.join('\n')}\n` } export const buildRequireError = (input: SummaryInput, state: ReportState): string | null => { if (state.reportText !== null) return null - const mutants = state.streamText === null ? 0 : countMutantLines(state.streamText) - if (mutants === 0) { - return [ - `::error title=Mutation produced no report::${input.package}: stryker exited '${input.outcome}' with zero mutant results — infrastructure failure (missing binary, crashed run or timeout), not a score outcome. Stream artifact: ${input.reportsDir}/mutation-stream.jsonl`, - ].join('') - } - return [ - `::error title=Mutation produced no report::${input.package}: stryker exited '${input.outcome}' after ${mutants} completed mutant(s) without a final report — infrastructure failure, not a score outcome. Partial stream: ${input.reportsDir}/mutation-stream.jsonl`, - ].join('') + return annotationsOf(recordFor(input, state)).join('\n') } export const combineParts = (parts: readonly StagedPart[]): Map => { diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index 2ff63a63d..4114dfcc2 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -114,7 +114,7 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom } const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' const reportsDir = join(dir, 'reports') - const input = { package: labelOf(dir, shard), outcome, reportsDir } + const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } const state = await loadState(reportsDir, readText) console.log(buildSummary(input, state)) const missing = buildRequireError(input, state) From af3cc030d5000f10442c428cc57cd6e19eb863a5 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 02:17:55 +0000 Subject: [PATCH 2/3] ci(repo): keep a failed outcome visible when a run reused every verdict The evaluated-none label replaced the stryker outcome, so a verdict-failed run served entirely from cache read as evaluated no mutants instead of failure Verdict-Semantics: unchanged --- scripts/lib/mutation-plan.test.ts | 14 ++++++++++++++ scripts/lib/mutation-plan.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index 33facc40d..40732c9a4 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -177,6 +177,20 @@ Deno.test('a reuse line with ran: 0 reports the evaluated-none outcome and a non assertEquals(buildRequireError(input, state) !== null, true) }) +Deno.test('a failed run that reused every verdict keeps its failure outcome next to evaluated-none', async () => { + const state = await loadState( + dir, + readFileFor({ + [`${dir}/mutation-report.json`]: '{"schemaVersion":"1.0","files":{}}', + [`${dir}/mutation-stream.jsonl`]: '{"_tag":"reuse","reused":4,"ran":0,"refused":{}}\n', + }), + ) + assertStringIncludes( + buildSummary(inputOf({ outcome: 'failure', exitCode: 1 }), state), + '**failure, evaluated no mutants**', + ) +}) + Deno.test('a cleared reports dir cannot satisfy the no-report gate', async () => { const stale = await loadState( dir, diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index 16055ca5e..a9ab490d0 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -333,7 +333,7 @@ export const recordFor = (input: SummaryInput, state: ReportState): FailureRecor const outcomeLabelOf = (input: SummaryInput, state: ReportState): string => Option.match(state.reuse, { onNone: () => input.outcome, - onSome: (reuse) => (reuse.ran === 0 ? 'evaluated no mutants' : input.outcome), + onSome: (reuse) => (reuse.ran === 0 ? `${input.outcome}, evaluated no mutants` : input.outcome), }) export const buildSummary = (input: SummaryInput, state: ReportState): string => { From e4b2179db0f8e1dbab75c6f0aa28717015283f02 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 02:23:21 +0000 Subject: [PATCH 3/3] docs(solutions): a failure record is the only rendering of a failure --- ...cord-is-the-only-rendering-of-a-failure.md | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md diff --git a/docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md b/docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md new file mode 100644 index 000000000..86ace7c35 --- /dev/null +++ b/docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md @@ -0,0 +1,79 @@ +--- +title: A failure record is the only rendering of a failure; nothing next to it may restate it +date: 2026-10-02 +category: best-practices +module: stryker-js failure reporting +problem_type: best_practice +component: tooling +severity: high +applies_when: + - adding a new failure path that raises a `RunFailure` or a tagged error with an `evidence` getter + - writing the `message` of an error class that also declares evidence + - adding stderr logging, CI summary text or annotations about a failed run + - asserting failure output in tests or e2e journeys +symptoms: + - "A gate refusal printed the new survivor id four times on stderr: a logError line, the record's evidence, the carrier's message and the carrier's stack header" + - "A failed dry run printed `Initial test run failed ...` from a logError before the record said the same thing" + - "A dry-run cause link dumped the whole `DryRunFailed` decision object, stack included, through the Formatter fallback" + - "The CI summary called a vacuous-property dry-run failure an infrastructure failure" +tags: + - failure-record + - diagnostics + - stderr + - cause-chain +--- + +# A failure record is the only rendering of a failure; nothing next to it may restate it + +## Context + +A failed run ends in one `FailureRecord.FailureRecord` from `@systemfsoftware/stryker-js-cli-contract`. That record is printed on stderr through `terminalTextOf`, emitted as the schema-3.0 `error` stream event, written to the run's `failure.json` under its mutation reports directory (`FAILURE_RECORD_FILE`), and turned into the CI summary and annotations by `markdownOf`/`annotationsOf` (`render-failure.ts`). Stack PRs #146-#151 introduced it after mutation run 36935602456 reported a vacuous in-source property in the dry run as an "infrastructure failure (missing binary, crashed run or timeout)". + +The cutover worked on the first pass but still produced noisy, contradictory output, because old renderings survived next to the record. + +## Guidance + +1. **No log line about a failure the record carries.** A cell that fails with a `RunFailure` must not also `Effect.logError` the same fact. The removed offenders were `explainGateRefusal` in `run-request.cell.ts` and the `Initial test run failed. N of M test(s) failed` log in `dry-run.cell.ts`. Integration tests that captured those logs were pinning a duplicate; assert on the evidence instead. +2. **An error that declares evidence keeps a one-line summary `message`.** The record renders the evidence. If the message lists the same data, every cause link repeats it. `GateRejected.message` is now `stryker gate: N new survivor(s) absent from the committed baseline`, and the survivor list lives only in `NewSurvivors.survivors`. +3. **`RunFailure.cause` holds a real error, never the decision data.** `writeDryRunFailed` used to pass the `DryRunFailed` decision as the cause. `messageOf` in `conclude-run.ts` falls back to `Formatter.format` for non-Error objects, so the whole decision, stacks included, landed in the cause chain. +4. **A cause link's stack keeps only its `at` frames.** A V8 stack starts with `Name: message`, so storing it whole prints the message twice. `linkOf` in `conclude-run.ts` drops the lines before the first `at` frame. +5. **Stacks show on the terminal only when nothing else locates the failure.** `terminalTextOf` and `markdownOf` print cause stacks only for `CatalogGap` records, and a failed test's stack only when it has no `location`. `failure.json` keeps every stack. +6. **Paths in evidence are project-relative.** The runner reports paths inside `.stryker-tmp/sandbox-*`, which is deleted after the run. `projectTestOf` in `dry-run.cell.ts` relativizes the test `file` and strips the sandbox prefix from the stack. The Vitest runner names file-level failures by their project-relative path. +7. **CI text comes only from the record.** The mutation job's `buildSummary` and `buildRequireError` (`mutation-plan.ts`) render the stream's terminal record, or build `RecordMissing`, `JobTimedOut` or `BinaryMissing` through `recordFor` when no record exists. A summary label must never replace the outcome: "evaluated no mutants" is appended to "failure", never shown instead of it. + +## Why This Matters + +Each duplicate is a second, drifting description of the same failure. An agent reading stderr cannot tell which line is authoritative. Duplicated or missing data also breaks the exact-count assertions that consumers write: the gate test counts survivor ids, and annotations must cover exactly the located evidence. And output that claims a cause the run never had sends the reader to fix the wrong thing. + +## When to Apply + +- Any new code path that ends a run in failure, or any new error class with an `evidence` getter. +- Any change to `render-failure.ts`, `conclude-run.ts` (`linkOf`, `messageOf`), or the CI scripts' summary. + +## Examples + +Gate refusal on stderr, before (one id, four times): + +```text +ERROR (#1): stryker gate: 1 new survivor(s) absent from the committed baseline: + src/sum.js:4 4d4d4d4d4d4d4d4d +... +GateRejected: stryker gate: 1 new survivor(s) absent from the committed baseline: + src/sum.js:4 4d4d4d4d4d4d4d4d + GateRejected: stryker gate: ... <- stack header repeating the message +``` + +After: + +```text +NewSurvivors: Mutants the change can affect survived every test and are absent from the accepted survivor baseline. +survivors: 1 listed, 2 unchecked +src/sum.js:4 4d4d4d4d4d4d4d4d +GateRejected: stryker gate: 1 new survivor(s) absent from the committed baseline +next: killSurvivor +``` + +## Related + +- `docs/plans/2026-10-01-2306-feat-agent-ready-failure-diagnostics-plan.md` +- `docs/solutions/test-failures/stream-schema-must-not-carry-json-rest-records.md`