From aab9915c59ac633b12fe8356ce77d82e8717e2b7 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 01:31:13 +0000 Subject: [PATCH 1/2] feat(repo): give every failure record a one-step reproduction The Vitest runner reports a reproduce argv per failed test, and a pure workflow decides each record's capsule: the runner command under the mutation-worker budget, the dry-run fallback, the --mutant rerun for new survivors, or DoesNotReplay with its stand-in Verdict-Semantics: unchanged --- .changeset/busy-mammals-clean.md | 5 + .changeset/fine-pandas-sit.md | 5 + .changeset/quiet-clowns-remain.md | 5 + .changeset/sixty-wombats-write.md | 5 + .../contract/stream.schema.json | 44 +++++- .../src/failure-catalog.ts | 10 +- .../src/failure-record.schema.ts | 1 + .../src/TestRunner.schema.ts | 2 + .../src/vitest-test-run.ts | 54 ++++++- ...sify-run-outcome.workflow.property.test.ts | 13 -- ...-run-observation.workflow.property.test.ts | 2 + ...oduction-capsule.workflow.property.test.ts | 104 ++++++++++++++ packages/stryker-js/src/conclude-run.ts | 26 +++- .../interpret-dry-run-observation.workflow.ts | 1 + .../src/reproduction-capsule.workflow.ts | 135 ++++++++++++++++++ .../framework-stream.integration.test.ts | 9 +- .../stryker-js/tests/gate.integration.test.ts | 12 +- .../run-event-stream.integration.test.ts | 9 +- 18 files changed, 413 insertions(+), 29 deletions(-) create mode 100644 .changeset/busy-mammals-clean.md create mode 100644 .changeset/fine-pandas-sit.md create mode 100644 .changeset/quiet-clowns-remain.md create mode 100644 .changeset/sixty-wombats-write.md create mode 100644 packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts create mode 100644 packages/stryker-js/src/reproduction-capsule.workflow.ts diff --git a/.changeset/busy-mammals-clean.md b/.changeset/busy-mammals-clean.md new file mode 100644 index 000000000..be813fbc7 --- /dev/null +++ b/.changeset/busy-mammals-clean.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-vitest-runner": minor +--- + +Every failed test reports a reproduce argv: vitest run -t , or vitest run for a file-level failure. diff --git a/.changeset/fine-pandas-sit.md b/.changeset/fine-pandas-sit.md new file mode 100644 index 000000000..5a2455456 --- /dev/null +++ b/.changeset/fine-pandas-sit.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +Each failure record carries a one-step reproduction: the failing test's own runner command under the mutation-worker budget, stryker run --dryRunOnly when the runner gives none, stryker run --mutant for new survivors, or DoesNotReplay naming why and what stands in (worker kind, pid and exit code for crashed workers). diff --git a/.changeset/quiet-clowns-remain.md b/.changeset/quiet-clowns-remain.md new file mode 100644 index 000000000..88657706d --- /dev/null +++ b/.changeset/quiet-clowns-remain.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-plugin-interface": minor +--- + +FailedTestResult gains an optional reproduce argv: the command that re-runs exactly that failing test. diff --git a/.changeset/sixty-wombats-write.md b/.changeset/sixty-wombats-write.md new file mode 100644 index 000000000..95bd76ea5 --- /dev/null +++ b/.changeset/sixty-wombats-write.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-cli-contract": minor +--- + +FailedTestEvidence carries the runner's reproduce argv, and recordOf takes the decided capsule from its RecordContext instead of deriving it. diff --git a/packages/stryker-js-cli-contract/contract/stream.schema.json b/packages/stryker-js-cli-contract/contract/stream.schema.json index 204b07a33..8f346e092 100644 --- a/packages/stryker-js-cli-contract/contract/stream.schema.json +++ b/packages/stryker-js-cli-contract/contract/stream.schema.json @@ -3701,6 +3701,25 @@ "type": "null" } ] + }, + "reproduce": { + "anyOf": [ + { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + { + "type": "null" + } + ] } }, "required": [ @@ -3709,7 +3728,8 @@ "file", "location", "message", - "stack" + "stack", + "reproduce" ], "additionalProperties": true } @@ -3775,6 +3795,25 @@ "type": "null" } ] + }, + "reproduce": { + "anyOf": [ + { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + { + "type": "null" + } + ] } }, "required": [ @@ -3783,7 +3822,8 @@ "file", "location", "message", - "stack" + "stack", + "reproduce" ], "additionalProperties": true } diff --git a/packages/stryker-js-cli-contract/src/failure-catalog.ts b/packages/stryker-js-cli-contract/src/failure-catalog.ts index a92ebe773..8e5acba72 100644 --- a/packages/stryker-js-cli-contract/src/failure-catalog.ts +++ b/packages/stryker-js-cli-contract/src/failure-catalog.ts @@ -1,6 +1,6 @@ import { dual } from 'effect/Function' -import type { CapsuleRule, CatalogEntry } from './failure-catalog.schema.js' +import type { CatalogEntry } from './failure-catalog.schema.js' import { type Capsule, type CauseLink, @@ -183,21 +183,17 @@ export interface RecordContext { readonly cwd: string readonly argv: readonly [string, ...Array] readonly env: ReadonlyArray + readonly capsule: Capsule readonly traceId: TraceId | null } -const capsuleOf = (rule: CapsuleRule, context: RecordContext): Capsule => - rule === 'replays' - ? { _tag: 'Replays', cwd: context.cwd, argv: context.argv, env: context.env } - : { _tag: 'DoesNotReplay', why: rule, standIn: context.argv.join(' ') } - export const recordOf: { (context: RecordContext): (evidence: FailureEvidence) => FailureRecord (evidence: FailureEvidence, context: RecordContext): FailureRecord } = dual(2, (evidence: FailureEvidence, context: RecordContext): FailureRecord => ({ ...evidence, cause: context.cause, - capsule: capsuleOf(FailureCatalog[evidence._tag].capsule, context), + capsule: context.capsule, nextAction: FailureCatalog[evidence._tag].nextAction, traceId: context.traceId, })) diff --git a/packages/stryker-js-cli-contract/src/failure-record.schema.ts b/packages/stryker-js-cli-contract/src/failure-record.schema.ts index 12997e51f..bfa40bdf4 100644 --- a/packages/stryker-js-cli-contract/src/failure-record.schema.ts +++ b/packages/stryker-js-cli-contract/src/failure-record.schema.ts @@ -73,6 +73,7 @@ export const FailedTestEvidence = S.Struct({ location: S.NullOr(SourceLocation), message: S.String, stack: S.NullOr(S.String), + reproduce: S.String.pipe(S.NonEmptyArray, S.NullOr), }) export type FailedTestEvidence = typeof FailedTestEvidence.Type diff --git a/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts b/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts index 1e313c937..211b226ff 100644 --- a/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts +++ b/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts @@ -40,6 +40,7 @@ export const TestResultSchema = S.Union([ failureMessage: S.String, location: S.optionalKey(TestFailureLocation), stack: S.optionalKey(S.String), + reproduce: S.String.pipe(S.NonEmptyArray, S.optionalKey), }), S.Struct({ ...TestResultBase, status: S.Literal('skipped') }), S.Struct({ ...TestResultBase, status: S.Literal('success') }), @@ -114,6 +115,7 @@ export interface FailedTestResult extends BaseTestResult { readonly failureMessage: string readonly location?: TestFailureLocation readonly stack?: string + readonly reproduce?: readonly [string, ...string[]] } export interface SkippedTestResult extends BaseTestResult { diff --git a/packages/stryker-js-vitest-runner/src/vitest-test-run.ts b/packages/stryker-js-vitest-runner/src/vitest-test-run.ts index 92a7c68f4..6070ddc44 100644 --- a/packages/stryker-js-vitest-runner/src/vitest-test-run.ts +++ b/packages/stryker-js-vitest-runner/src/vitest-test-run.ts @@ -39,6 +39,15 @@ const projectRelativeId = (id: string, projectRoot: string): string => { }) } +const projectRelativeFileOf = (record: VitestTestRecord, projectRoot: string): string => + projectRelativePath(stripProjectRoot(fileNameOf(record), projectRoot)) + +const reproduceArgvOf = (testName: string, file: string): readonly [string, ...string[]] => + Boolean.match(testName === file, { + onTrue: (): readonly [string, ...string[]] => ['vitest', 'run', file], + onFalse: (): readonly [string, ...string[]] => ['vitest', 'run', file, '-t', testName], + }) + const testNameOf = (record: VitestTestRecord): string => Option.getOrElse( Option.fromNullishOr(record.fullTestName), @@ -121,9 +130,11 @@ const failureEvidenceOfRecord = ( failureEvidenceOf(locationOf(record.errorFrames, projectRoot), record.errorStack) const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEncoded => { + const name = testNameOf(record) + const reproduce = reproduceArgvOf(name, projectRelativeFileOf(record, projectRoot)) const base = { - id: projectRelativeId(`${fileNameOf(record)}#${testNameOf(record)}`, projectRoot), - name: testNameOf(record), + id: projectRelativeId(`${fileNameOf(record)}#${name}`, projectRoot), + name, timeSpentMs: timeSpentOf(record), ...fileNameFieldOf(record), } @@ -134,6 +145,7 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco ...base, status: 'failed', failureMessage: failureMessageOf(record), + reproduce, ...failureEvidenceOfRecord(record, projectRoot), }), ), @@ -146,6 +158,7 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco ...base, status: 'failed', failureMessage, + reproduce, ...failureEvidenceOfRecord(record, projectRoot), }), }), @@ -163,6 +176,7 @@ const fileFailureResultOf = (failure: VitestFileFailure, projectRoot: string): T status: 'failed', failureMessage: failure.message, fileName: failure.fileName, + reproduce: reproduceArgvOf(file, file), ...failureEvidenceOf(locationOf(failure.frames, projectRoot), failure.stack), } } @@ -194,6 +208,12 @@ if (import.meta.vitest !== void 0) { ): Option.Option> => Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.location)) + const reproduceOf = (only: TestResultEncoded): Option.Option => + Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.reproduce)) + + const sameArgv = (actual: readonly string[], expected: readonly string[]): boolean => + actual.length === expected.length && expected.every((value, index) => actual[index] === value) + const positionsEqual = ( location: NonNullable, line: number, @@ -286,6 +306,19 @@ if (import.meta.vitest !== void 0) { }, ) + it.prop( + '∀f_FileFailure_≡ReproduceRunsTheFile', + { of: [FileStem], subject: interpretVitestTestRun }, + (subject, [stem]) => { + const [only] = subject({ + projectRoot: '/project', + records: [], + fileFailures: [{ fileName: `/project/tests/${stem}.spec.ts`, message: 'load failed' }], + }) + return Option.exists(reproduceOf(only), (argv) => sameArgv(argv, ['vitest', 'run', `tests/${stem}.spec.ts`])) + }, + ) + const SourceOrdinal = S.Int.check(S.isGreaterThanOrEqualTo(1)) const failingRecord = (overrides: Partial): VitestTestRecord => ({ @@ -294,6 +327,23 @@ if (import.meta.vitest !== void 0) { ...overrides, }) + const TestName = S.String.check(S.isPattern(/^[a-z][a-z ]{0,11}$/)) + + it.prop( + '∀r_FailedTest_≡ReproduceNamesFileAndTest', + { of: [TestName, FileStem], subject: interpretVitestTestRun }, + (subject, [name, stem]) => { + const file = `tests/${stem}.spec.ts` + const record = failingRecord({ name, fullTestName: name, fileName: `/project/${file}` }) + const [only] = subject(singleRecordRun(record, '/project')) + const expected = Boolean.match(name === file, { + onTrue: (): readonly string[] => ['vitest', 'run', file], + onFalse: (): readonly string[] => ['vitest', 'run', file, '-t', name], + }) + return Option.exists(reproduceOf(only), (argv) => sameArgv(argv, expected)) + }, + ) + it.prop( '∀r_ProjectFrame_≡LocatedThere', { of: [SourceOrdinal, SourceOrdinal], subject: interpretVitestTestRun }, diff --git a/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts b/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts index 70fafdc8f..23e67a882 100644 --- a/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts @@ -79,19 +79,6 @@ describe('classifyRunOutcome', () => { }, ) - it.prop( - '∀evidence_RunOutcome_≡ReplayableRecordsReplayTheSameCommandAndOthersSayWhyNot', - { of: [FailureRecord.FailureEvidence], subject: outcomeOf }, - (subject, [evidence]) => { - const capsule = failedOf(subject(Cause.fail({ evidence })))?.record.capsule - const rule = FailureRecord.FailureCatalog[evidence._tag].capsule - return rule === 'replays' - ? Predicate.isTagged(capsule, 'Replays') && capsule.cwd === CWD && - capsule.argv.join(' ') === ['stryker', ...ARGS].join(' ') - : Predicate.isTagged(capsule, 'DoesNotReplay') && capsule.why === rule - }, - ) - it.prop( '∀fiber_RunOutcome_≡AnInterruptOnlyRunExits130WithARunInterruptedRecord', { of: [S.Int.check(S.isGreaterThanOrEqualTo(0))], subject: outcomeOf }, diff --git a/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts b/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts index 504273fd0..c1a13a648 100644 --- a/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts @@ -1,6 +1,7 @@ import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' +import * as Equal from 'effect/Equal' import * as Result from 'effect/Result' import * as S from 'effect/Schema' @@ -48,6 +49,7 @@ describe('interpretDryRunObservation', () => { evidence.location.line === location.line && evidence.location.column === location.column) && evidence.message === expected.failureMessage && + Equal.equals(evidence.reproduce ?? null, expected.reproduce ?? null) && evidence.stack === (expected.stack ?? null) }) }, diff --git a/packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts b/packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts new file mode 100644 index 000000000..899129ce2 --- /dev/null +++ b/packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts @@ -0,0 +1,104 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { describe, it } from '@systemfsoftware/vitest' +import * as Arr from 'effect/Array' +import * as Predicate from 'effect/Predicate' +import * as Result from 'effect/Result' + +import { + type CapsuleDoesNotReplay, + type CapsuleReplays, + reproductionCapsule, + ReproductionCapsuleCommand, + type ReproductionCapsuleDecision, +} from '../reproduction-capsule.workflow.js' + +const CWD = '/project' +const RUN_ARGV = ['stryker', 'run'] as const +const DRY_RUN_FLAG = '--dryRunOnly' +const MARKER: FailureRecord.EnvEntry = { name: 'STRYKER_WORKER_DIR', value: '.stryker-tmp' } + +const capsuleOf = (evidence: FailureRecord.FailureEvidence): ReproductionCapsuleDecision => + Result.getOrElse( + reproductionCapsule( + ReproductionCapsuleCommand.make({ evidence, argv: RUN_ARGV, cwd: CWD, envMarker: MARKER }), + ), + (impossible) => impossible, + ) + +const replaysOf = (decision: ReproductionCapsuleDecision): CapsuleReplays | undefined => + Predicate.isTagged(decision, 'Replays') ? decision : undefined + +const refusalOf = (decision: ReproductionCapsuleDecision): CapsuleDoesNotReplay | undefined => + Predicate.isTagged(decision, 'DoesNotReplay') ? decision : undefined + +const baselineEvidenceOf = (test: FailureRecord.FailedTestEvidence): FailureRecord.FailureEvidence => ({ + _tag: 'BaselineTestsFailed', + stage: 'dryRun', + testCount: 1, + tests: [test], +}) + +const mutantArgvOf = (evidence: FailureRecord.FailureEvidence): string => + Predicate.isTagged(evidence, 'NewSurvivors') + ? ['stryker', 'run', '--mutant', Arr.join(Arr.map(evidence.survivors, (each) => each.mutantId), ',')].join(' ') + : '' + +describe('reproductionCapsule', () => { + it.prop( + '∀evidence_ReproductionCapsule_≡ReplaysExactlyWhenItsCatalogCodeReplays', + { of: [FailureRecord.FailureEvidence], subject: capsuleOf }, + (subject, [evidence]) => { + const decision = subject(evidence) + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + return rule === 'replays' + ? replaysOf(decision)?.cwd === CWD + : refusalOf(decision)?.why === rule + }, + ) + + it.prop( + '∀evidence_ReproductionCapsule_≡NewSurvivorFailuresReplayTheNamedMutantIds', + { of: [FailureRecord.FailureEvidence], subject: capsuleOf }, + (subject, [evidence]) => { + const decision = subject(evidence) + if (!Predicate.isTagged(evidence, 'NewSurvivors')) { + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + return rule === 'replays' ? replaysOf(decision) !== undefined : refusalOf(decision) !== undefined + } + return replaysOf(decision)?.argv.join(' ') === mutantArgvOf(evidence) + }, + ) + + it.prop( + '∀failedTest_ReproductionCapsule_≡BaselineFailuresReplayTheRunnerArgvOrFallBackToTheDryRun', + { of: [FailureRecord.FailedTestEvidence], subject: capsuleOf }, + (subject, [failedTest]) => { + const replays = replaysOf(subject(baselineEvidenceOf(failedTest))) + const expectedArgv = failedTest.reproduce ?? [...RUN_ARGV, DRY_RUN_FLAG] + return replays !== undefined && replays.cwd === CWD && + replays.argv.join(' ') === expectedArgv.join(' ') && + Arr.every(replays.env, (entry) => entry.name === MARKER.name && entry.value === MARKER.value) + }, + ) + + it.prop( + '∀evidence_ReproductionCapsule_≡NonReplayableRecordsStandInTheirOwnEvidence', + { of: [FailureRecord.FailureEvidence], subject: capsuleOf }, + (subject, [evidence]) => { + const decision = subject(evidence) + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + if (rule === 'replays') return replaysOf(decision) !== undefined + const refusal = refusalOf(decision) + if (refusal === undefined) return false + if (Predicate.isTagged(evidence, 'WorkerOutOfMemory')) { + return refusal.standIn === `${evidence.workerKind} pid ${evidence.pid} exited ${evidence.exitCode}` + } + if (Predicate.isTagged(evidence, 'WorkerCrashed')) { + return refusal.standIn.includes(`${evidence.workerKind} pid ${evidence.pid}`) + } + if (Predicate.isTagged(evidence, 'JobTimedOut')) return refusal.standIn.includes(`${evidence.limitSeconds}`) + if (Predicate.isTagged(evidence, 'BinaryMissing')) return refusal.standIn.includes(evidence.binary) + return true + }, + ) +}) diff --git a/packages/stryker-js/src/conclude-run.ts b/packages/stryker-js/src/conclude-run.ts index b4fcddbb7..a76b63946 100644 --- a/packages/stryker-js/src/conclude-run.ts +++ b/packages/stryker-js/src/conclude-run.ts @@ -1,5 +1,5 @@ import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' -import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import { Options, Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Cause from 'effect/Cause' import * as CliError from 'effect/cli/CliError' @@ -9,8 +9,10 @@ import { pipe } from 'effect/Function' import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Predicate from 'effect/Predicate' +import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { reproductionCapsule, ReproductionCapsuleCommand } from './reproduction-capsule.workflow.js' import { type ObservedFailure, RunFailedObservation, @@ -24,6 +26,11 @@ import { const CLI_BIN = 'stryker' const MAX_CAUSE_DEPTH = 10 +const WORKER_ENV_MARKER: FailureRecord.EnvEntry = { + name: 'STRYKER_WORKER_DIR', + value: Options.StrykerTempDirName.literal, +} + const asEvidence = Option.liftPredicate(S.is(FailureRecord.FailureEvidence)) const asExitClass = Option.liftPredicate(S.is(Plugin.ExitClass)) @@ -132,6 +139,22 @@ interface RunContext { readonly traceId: FailureRecord.TraceId | null } +const decidedCapsuleOf = ( + evidence: FailureRecord.FailureEvidence, + context: RunContext, +): FailureRecord.Capsule => + Result.getOrElse( + reproductionCapsule( + ReproductionCapsuleCommand.make({ + evidence, + argv: [CLI_BIN, ...context.argv], + cwd: context.cwd, + envMarker: WORKER_ENV_MARKER, + }), + ), + (impossible: never) => impossible, + ) + const observedOf = ( evidence: FailureRecord.FailureEvidence, cause: ReadonlyArray, @@ -142,6 +165,7 @@ const observedOf = ( cwd: context.cwd, argv: [CLI_BIN, ...context.argv], env: [], + capsule: decidedCapsuleOf(evidence, context), traceId: context.traceId, }), exitCode: FailureRecord.FailureCatalog[evidence._tag].exitCode ?? CI_ONLY_EXIT_CODE, diff --git a/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts b/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts index fe7b742e0..a862103ea 100644 --- a/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts +++ b/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts @@ -49,6 +49,7 @@ const failedTestEvidenceOf = (test: TestRunner.FailedTestResult): FailureRecord. location: orNullOf(test.location), message: test.failureMessage, stack: orNullOf(test.stack), + reproduce: orNullOf(test.reproduce), }) const failedTestEvidencesOf = ( diff --git a/packages/stryker-js/src/reproduction-capsule.workflow.ts b/packages/stryker-js/src/reproduction-capsule.workflow.ts new file mode 100644 index 000000000..163cbedfe --- /dev/null +++ b/packages/stryker-js/src/reproduction-capsule.workflow.ts @@ -0,0 +1,135 @@ +import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import * as Arr from 'effect/Array' +import * as Boolean from 'effect/Boolean' +import * as Match from 'effect/Match' +import * as Option from 'effect/Option' +import * as Result from 'effect/Result' +import * as S from 'effect/Schema' + +const CLI_BIN = 'stryker' +const DRY_RUN_FLAG = '--dryRunOnly' +const MUTANT_FLAG = '--mutant' + +const ReproductionCapsuleTypeId: unique symbol = Symbol.for( + '@systemfsoftware/stryker-js/ReproductionCapsule', +) +type ReproductionCapsuleTypeId = typeof ReproductionCapsuleTypeId + +export class CapsuleReplays extends S.TaggedClass()('Replays', { + cwd: S.NonEmptyString, + argv: S.NonEmptyArray(S.String), + env: S.Array(FailureRecord.EnvEntry), +}) { + readonly [ReproductionCapsuleTypeId] = ReproductionCapsuleTypeId +} + +export class CapsuleDoesNotReplay extends S.TaggedClass()('DoesNotReplay', { + why: FailureRecord.NonReplayReason, + standIn: S.NonEmptyString, +}) { + readonly [ReproductionCapsuleTypeId] = ReproductionCapsuleTypeId +} + +export type ReproductionCapsuleDecision = CapsuleReplays | CapsuleDoesNotReplay + +export class ReproductionCapsuleCommand extends S.TaggedClass()( + 'ReproductionCapsuleCommand', + { + evidence: FailureRecord.FailureEvidence, + argv: S.NonEmptyArray(S.String), + cwd: S.NonEmptyString, + envMarker: FailureRecord.EnvEntry, + }, +) { + static readonly [Workflow.InstrumentationBrand] = {} as const +} + +const replays = ( + argv: readonly [string, ...Array], + command: ReproductionCapsuleCommand, +): CapsuleReplays => CapsuleReplays.make({ cwd: command.cwd, argv, env: [command.envMarker] }) + +const doesNotReplay = ( + why: FailureRecord.NonReplayReason, + standIn: string, +): CapsuleDoesNotReplay => CapsuleDoesNotReplay.make({ why, standIn }) + +const dryRunArgv = (argv: readonly [string, ...Array]): readonly [string, ...Array] => + Boolean.match(Arr.contains(argv, DRY_RUN_FLAG), { + onTrue: () => argv, + onFalse: () => Arr.append(argv, DRY_RUN_FLAG), + }) + +const crashedStandIn = ( + workerKind: FailureRecord.WorkerKind, + pid: number, + exitCode: number | null, + signal: string | null, +): string => + Option.match(Option.fromNullishOr(exitCode), { + onSome: (code) => `${workerKind} pid ${pid} exited ${code}`, + onNone: () => + Option.match(Option.fromNullishOr(signal), { + onSome: (name) => `${workerKind} pid ${pid} was killed by ${name}`, + onNone: () => `${workerKind} pid ${pid} exited without a code`, + }), + }) + +const missingStandIn = (exitCode: number | null): string => + Option.match(Option.fromNullishOr(exitCode), { + onSome: (code) => `the job wrote neither a failure record nor a report (exit code ${code})`, + onNone: () => 'the job wrote neither a failure record nor a report', + }) + +const ownReplays = (command: ReproductionCapsuleCommand): CapsuleReplays => replays(command.argv, command) + +const capsuleFor = (command: ReproductionCapsuleCommand): ReproductionCapsuleDecision => + Match.value(command.evidence).pipe( + Match.tagsExhaustive({ + WorkerOutOfMemory: ({ workerKind, pid, exitCode }) => + doesNotReplay('outOfMemory', `${workerKind} pid ${pid} exited ${exitCode}`), + WorkerCrashed: ({ workerKind, pid, exitCode, signal }) => + doesNotReplay('workerCrashed', crashedStandIn(workerKind, pid, exitCode, signal)), + RunInterrupted: () => doesNotReplay('interrupted', 'the run was interrupted; re-run it with the same command'), + RecordMissing: ({ exitCode }) => doesNotReplay('recordMissing', missingStandIn(exitCode)), + JobTimedOut: ({ limitSeconds }) => + doesNotReplay('jobTimedOut', `the job exceeded its ${limitSeconds}s time limit`), + BinaryMissing: ({ binary }) => doesNotReplay('binaryMissing', `the required binary "${binary}" was not found`), + BaselineTestsFailed: ({ tests }) => + Option.match(Option.fromNullishOr(Arr.headNonEmpty(tests).reproduce), { + onSome: (reproduce) => replays(reproduce, command), + onNone: () => replays(dryRunArgv(command.argv), command), + }), + NewSurvivors: ({ survivors }) => + replays([CLI_BIN, 'run', MUTANT_FLAG, Arr.join(Arr.map(survivors, (each) => each.mutantId), ',')], command), + BaselineTimedOut: () => replays(dryRunArgv(command.argv), command), + BaselineErrored: () => replays(dryRunArgv(command.argv), command), + BaselineFoundNoTests: () => replays(dryRunArgv(command.argv), command), + ArgumentsInvalid: () => ownReplays(command), + ConfigInvalid: () => ownReplays(command), + PluginNotFound: () => ownReplays(command), + PluginLoadFailed: () => ownReplays(command), + PluginImportFailed: () => ownReplays(command), + SurvivorsUnavailable: () => ownReplays(command), + NoInputFiles: () => ownReplays(command), + SandboxPreparationFailed: () => ownReplays(command), + InstrumentationFailed: () => ownReplays(command), + CheckerFailed: () => ownReplays(command), + TestRunnerFailed: () => ownReplays(command), + WorkerBootTimedOut: () => ownReplays(command), + ReporterFailed: () => ownReplays(command), + InvariantBroken: () => ownReplays(command), + CatalogGap: () => ownReplays(command), + }), + ) + +const decide = (command: ReproductionCapsuleCommand): Result.Result => + Result.succeed(capsuleFor(command)) + +export const reproductionCapsule = Workflow.make({ + command: ReproductionCapsuleCommand, + decision: S.Union([CapsuleReplays, CapsuleDoesNotReplay]), + error: S.Never, + decide, +}) diff --git a/packages/stryker-js/tests/framework-stream.integration.test.ts b/packages/stryker-js/tests/framework-stream.integration.test.ts index b8ce43b8f..ac3f5b3cc 100644 --- a/packages/stryker-js/tests/framework-stream.integration.test.ts +++ b/packages/stryker-js/tests/framework-stream.integration.test.ts @@ -53,7 +53,14 @@ const refusal = RunEvent.RunFailed.make({ descriptor: frameworkModule, reason: { _tag: 'PeerMissing', peer: frameworkModule }, }, - { cause: [], cwd: '/project', argv: ['stryker', 'run'], env: [], traceId: null }, + { + cause: [], + cwd: '/project', + argv: ['stryker', 'run'], + env: [], + capsule: { _tag: 'Replays', cwd: '/project', argv: ['stryker', 'run'], env: [] }, + traceId: null, + }, ), }) diff --git a/packages/stryker-js/tests/gate.integration.test.ts b/packages/stryker-js/tests/gate.integration.test.ts index 20f3f1bab..4b51f2618 100644 --- a/packages/stryker-js/tests/gate.integration.test.ts +++ b/packages/stryker-js/tests/gate.integration.test.ts @@ -165,12 +165,20 @@ Feature('Gating a pull request on survivors the committed baseline has never see 'ran', (s) => runGate(s.project.root, 'human', ['--baseline', COMMITTED_BASELINE]), ), - Then('the process exits 1 and stderr names only the new survivor once')((s, expect) => + Then('the process exits 1, replays the new survivor, and never suggests --update-baseline')((s, expect) => expect({ exitCode: s.ran.exitCode, namesOnlyTheNewSurvivor: namesOnlyTheNewSurvivor(s.ran.stderr), newIdAppearances: s.ran.stderr.split(NEW_SURVIVOR).length - 1, - }).toStrictEqual({ exitCode: 1, namesOnlyTheNewSurvivor: true, newIdAppearances: 1 }) + replaysTheSurvivor: s.ran.stderr.includes(`stryker run --mutant ${NEW_SURVIVOR}`), + neverSuggestsUpdateBaseline: !s.ran.stderr.includes('--update-baseline'), + }).toStrictEqual({ + exitCode: 1, + namesOnlyTheNewSurvivor: true, + newIdAppearances: 2, + replaysTheSurvivor: true, + neverSuggestsUpdateBaseline: true, + }) ), ), ) diff --git a/packages/stryker-js/tests/run-event-stream.integration.test.ts b/packages/stryker-js/tests/run-event-stream.integration.test.ts index 0ae5b3e6d..744a58f59 100644 --- a/packages/stryker-js/tests/run-event-stream.integration.test.ts +++ b/packages/stryker-js/tests/run-event-stream.integration.test.ts @@ -22,7 +22,14 @@ const HEARTBEAT = CliContract.Heartbeat.make({ elapsedMs: 2, completed: 1, total const HELP_RENDERED = CliContract.HelpRendered.make({ schemaVersion: '3.0', code: 0, help: 'usage' }) const RUN_FAILED_RECORD = FailureRecord.recordOf( { _tag: 'TestRunnerFailed', stage: 'mutationTest' }, - { cause: [], cwd: '/base', argv: ['stryker', 'run'], env: [], traceId: null }, + { + cause: [], + cwd: '/base', + argv: ['stryker', 'run'], + env: [], + capsule: { _tag: 'Replays', cwd: '/base', argv: ['stryker', 'run'], env: [] }, + traceId: null, + }, ) const RUN_FAILED = CliContract.RunFailed.make({ schemaVersion: '3.0', From 80a604c9bd5c1c9981e4d7dca8fd3068af8cd2cd Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 02:39:51 +0000 Subject: [PATCH 2/2] test(repo): expect the runner's vitest replay in the failing-run capsule The capsule layer replays a failed baseline test through the runner's vitest run -t argv; the e2e journey still expected the stryker run fallback Verdict-Semantics: unchanged --- test/e2e/tests/failing-run.e2e.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/tests/failing-run.e2e.test.ts b/test/e2e/tests/failing-run.e2e.test.ts index fe477fdae..147958426 100644 --- a/test/e2e/tests/failing-run.e2e.test.ts +++ b/test/e2e/tests/failing-run.e2e.test.ts @@ -13,7 +13,7 @@ const FAILING_TEST_NAME = 'isEven reports three as even' const FAILING_TEST_FILE = 'src/thing.test.ts' const FAILING_TEST_LINE = 7 const FAILURE_RECORD_FILE = 'reports/mutation/failure.json' -const FAILING_DRY_RUN_ARGV: ReadonlyArray = ['stryker', 'run'] +const FAILING_TEST_REPLAY_ARGV: ReadonlyArray = ['vitest', 'run', FAILING_TEST_FILE, '-t', FAILING_TEST_NAME] const BASELINE_TESTS_FAILED_NEXT_ACTION: FailureRecord.NextAction = { primary: 'fixCode', otherwise: 'fixTest' } const failureRecordOf = (terminal: RunEvent.RunEvent): FailureRecord.FailureRecord | undefined => @@ -76,7 +76,7 @@ const verifyTypedErrorDocument = ( recordTag: 'BaselineTestsFailed', stage: 'dryRun', capsuleReplays: true, - argv: FAILING_DRY_RUN_ARGV, + argv: FAILING_TEST_REPLAY_ARGV, nextAction: BASELINE_TESTS_FAILED_NEXT_ACTION, carriesVerdict: false, testCount: 1,