feat(repo): give every failure record a one-step reproduction

.changeset/busy-mammals-clean.md

@@ -0,0 +1,5 @@
+---
+"@systemfsoftware/stryker-js-vitest-runner": minor
+---
+
+Every failed test reports a reproduce argv: vitest run <file> -t <full test name>, or vitest run <file> for a file-level failure.

.changeset/fine-pandas-sit.md

@@ -0,0 +1,5 @@
+---
+"@systemfsoftware/stryker-js": minor
+---
+
+Each failure record carries a one-step reproduction: the failing test's own runner command under the mutation-worker budget, stryker run --dryRunOnly when the runner gives none, stryker run --mutant <ids> for new survivors, or DoesNotReplay naming why and what stands in (worker kind, pid and exit code for crashed workers).

.changeset/quiet-clowns-remain.md

@@ -0,0 +1,5 @@
+---
+"@systemfsoftware/stryker-js-plugin-interface": minor
+---
+
+FailedTestResult gains an optional reproduce argv: the command that re-runs exactly that failing test.

.changeset/sixty-wombats-write.md

@@ -0,0 +1,5 @@
+---
+"@systemfsoftware/stryker-js-cli-contract": minor
+---
+
+FailedTestEvidence carries the runner's reproduce argv, and recordOf takes the decided capsule from its RecordContext instead of deriving it.

packages/stryker-js-cli-contract/contract/stream.schema.json

@@ -3701,6 +3701,25 @@
                               "type": "null"
                             }
                           ]
+                        },
+                        "reproduce": {
+                          "anyOf": [
+                            {
+                              "type": "array",
+                              "minItems": 1,
+                              "items": [
+                                {
+                                  "type": "string"
+                                }
+                              ],
+                              "additionalItems": {
+                                "type": "string"
+                              }
+                            },
+                            {
+                              "type": "null"
+                            }
+                          ]
                         }
                       },
                       "required": [
@@ -3709,7 +3728,8 @@
                         "file",
                         "location",
                         "message",
-                        "stack"
+                        "stack",
+                        "reproduce"
                       ],
                       "additionalProperties": true
                     }
@@ -3775,6 +3795,25 @@
                             "type": "null"
                           }
                         ]
+                      },
+                      "reproduce": {
+                        "anyOf": [
+                          {
+                            "type": "array",
+                            "minItems": 1,
+                            "items": [
+                              {
+                                "type": "string"
+                              }
+                            ],
+                            "additionalItems": {
+                              "type": "string"
+                            }
+                          },
+                          {
+                            "type": "null"
+                          }
+                        ]
                       }
                     },
                     "required": [
@@ -3783,7 +3822,8 @@
                       "file",
                       "location",
                       "message",
-                      "stack"
+                      "stack",
+                      "reproduce"
                     ],
                     "additionalProperties": true
                   }

packages/stryker-js-cli-contract/src/failure-catalog.ts

@@ -1,6 +1,6 @@
 import { dual } from 'effect/Function'
 
-import type { CapsuleRule, CatalogEntry } from './failure-catalog.schema.js'
+import type { CatalogEntry } from './failure-catalog.schema.js'
 import {
   type Capsule,
   type CauseLink,
@@ -183,21 +183,17 @@ export interface RecordContext {
   readonly cwd: string
   readonly argv: readonly [string, ...Array<string>]
   readonly env: ReadonlyArray<EnvEntry>
+  readonly capsule: Capsule
   readonly traceId: TraceId | null
 }
 
-const capsuleOf = (rule: CapsuleRule, context: RecordContext): Capsule =>
-  rule === 'replays'
-    ? { _tag: 'Replays', cwd: context.cwd, argv: context.argv, env: context.env }
-    : { _tag: 'DoesNotReplay', why: rule, standIn: context.argv.join(' ') }
-
 export const recordOf: {
   (context: RecordContext): (evidence: FailureEvidence) => FailureRecord
   (evidence: FailureEvidence, context: RecordContext): FailureRecord
 } = dual(2, (evidence: FailureEvidence, context: RecordContext): FailureRecord => ({
   ...evidence,
   cause: context.cause,
-  capsule: capsuleOf(FailureCatalog[evidence._tag].capsule, context),
+  capsule: context.capsule,
   nextAction: FailureCatalog[evidence._tag].nextAction,
   traceId: context.traceId,
 }))

packages/stryker-js-cli-contract/src/failure-record.schema.ts

@@ -73,6 +73,7 @@ export const FailedTestEvidence = S.Struct({
   location: S.NullOr(SourceLocation),
   message: S.String,
   stack: S.NullOr(S.String),
+  reproduce: S.String.pipe(S.NonEmptyArray, S.NullOr),
 })
 export type FailedTestEvidence = typeof FailedTestEvidence.Type
 

packages/stryker-js-plugin-interface/src/TestRunner.schema.ts

@@ -40,6 +40,7 @@ export const TestResultSchema = S.Union([
     failureMessage: S.String,
     location: S.optionalKey(TestFailureLocation),
     stack: S.optionalKey(S.String),
+    reproduce: S.String.pipe(S.NonEmptyArray, S.optionalKey),
   }),
   S.Struct({ ...TestResultBase, status: S.Literal('skipped') }),
   S.Struct({ ...TestResultBase, status: S.Literal('success') }),
@@ -114,6 +115,7 @@ export interface FailedTestResult extends BaseTestResult {
   readonly failureMessage: string
   readonly location?: TestFailureLocation
   readonly stack?: string
+  readonly reproduce?: readonly [string, ...string[]]
 }
 
 export interface SkippedTestResult extends BaseTestResult {

packages/stryker-js-vitest-runner/src/vitest-test-run.ts

@@ -39,6 +39,15 @@ const projectRelativeId = (id: string, projectRoot: string): string => {
   })
 }
 
+const projectRelativeFileOf = (record: VitestTestRecord, projectRoot: string): string =>
+  projectRelativePath(stripProjectRoot(fileNameOf(record), projectRoot))
+
+const reproduceArgvOf = (testName: string, file: string): readonly [string, ...string[]] =>
+  Boolean.match(testName === file, {
+    onTrue: (): readonly [string, ...string[]] => ['vitest', 'run', file],
+    onFalse: (): readonly [string, ...string[]] => ['vitest', 'run', file, '-t', testName],
+  })
+
 const testNameOf = (record: VitestTestRecord): string =>
   Option.getOrElse(
     Option.fromNullishOr(record.fullTestName),
@@ -121,9 +130,11 @@ const failureEvidenceOfRecord = (
   failureEvidenceOf(locationOf(record.errorFrames, projectRoot), record.errorStack)
 
 const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEncoded => {
+  const name = testNameOf(record)
+  const reproduce = reproduceArgvOf(name, projectRelativeFileOf(record, projectRoot))
   const base = {
-    id: projectRelativeId(`${fileNameOf(record)}#${testNameOf(record)}`, projectRoot),
-    name: testNameOf(record),
+    id: projectRelativeId(`${fileNameOf(record)}#${name}`, projectRoot),
+    name,
     timeSpentMs: timeSpentOf(record),
     ...fileNameFieldOf(record),
   }
@@ -134,6 +145,7 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco
         ...base,
         status: 'failed',
         failureMessage: failureMessageOf(record),
+        reproduce,
         ...failureEvidenceOfRecord(record, projectRoot),
       }),
     ),
@@ -146,6 +158,7 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco
             ...base,
             status: 'failed',
             failureMessage,
+            reproduce,
             ...failureEvidenceOfRecord(record, projectRoot),
           }),
         }),
@@ -163,6 +176,7 @@ const fileFailureResultOf = (failure: VitestFileFailure, projectRoot: string): T
     status: 'failed',
     failureMessage: failure.message,
     fileName: failure.fileName,
+    reproduce: reproduceArgvOf(file, file),
     ...failureEvidenceOf(locationOf(failure.frames, projectRoot), failure.stack),
   }
 }
@@ -194,6 +208,12 @@ if (import.meta.vitest !== void 0) {
   ): Option.Option<NonNullable<FailedTestResultEncoded['location']>> =>
     Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.location))
 
+  const reproduceOf = (only: TestResultEncoded): Option.Option<readonly string[]> =>
+    Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.reproduce))
+
+  const sameArgv = (actual: readonly string[], expected: readonly string[]): boolean =>
+    actual.length === expected.length && expected.every((value, index) => actual[index] === value)
+
   const positionsEqual = (
     location: NonNullable<FailedTestResultEncoded['location']>,
     line: number,
@@ -286,6 +306,19 @@ if (import.meta.vitest !== void 0) {
     },
   )
 
+  it.prop(
+    '∀f_FileFailure_≡ReproduceRunsTheFile',
+    { of: [FileStem], subject: interpretVitestTestRun },
+    (subject, [stem]) => {
+      const [only] = subject({
+        projectRoot: '/project',
+        records: [],
+        fileFailures: [{ fileName: `/project/tests/${stem}.spec.ts`, message: 'load failed' }],
+      })
+      return Option.exists(reproduceOf(only), (argv) => sameArgv(argv, ['vitest', 'run', `tests/${stem}.spec.ts`]))
+    },
+  )
+
   const SourceOrdinal = S.Int.check(S.isGreaterThanOrEqualTo(1))
 
   const failingRecord = (overrides: Partial<VitestTestRecord>): VitestTestRecord => ({
@@ -294,6 +327,23 @@ if (import.meta.vitest !== void 0) {
     ...overrides,
   })
 
+  const TestName = S.String.check(S.isPattern(/^[a-z][a-z ]{0,11}$/))
+
+  it.prop(
+    '∀r_FailedTest_≡ReproduceNamesFileAndTest',
+    { of: [TestName, FileStem], subject: interpretVitestTestRun },
+    (subject, [name, stem]) => {
+      const file = `tests/${stem}.spec.ts`
+      const record = failingRecord({ name, fullTestName: name, fileName: `/project/${file}` })
+      const [only] = subject(singleRecordRun(record, '/project'))
+      const expected = Boolean.match(name === file, {
+        onTrue: (): readonly string[] => ['vitest', 'run', file],
+        onFalse: (): readonly string[] => ['vitest', 'run', file, '-t', name],
+      })
+      return Option.exists(reproduceOf(only), (argv) => sameArgv(argv, expected))
+    },
+  )
+
   it.prop(
     '∀r_ProjectFrame_≡LocatedThere',
     { of: [SourceOrdinal, SourceOrdinal], subject: interpretVitestTestRun },

packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts

@@ -79,19 +79,6 @@ describe('classifyRunOutcome', () => {
     },
   )
 
-  it.prop(
-    '∀evidence_RunOutcome_≡ReplayableRecordsReplayTheSameCommandAndOthersSayWhyNot',
-    { of: [FailureRecord.FailureEvidence], subject: outcomeOf },
-    (subject, [evidence]) => {
-      const capsule = failedOf(subject(Cause.fail({ evidence })))?.record.capsule
-      const rule = FailureRecord.FailureCatalog[evidence._tag].capsule
-      return rule === 'replays'
-        ? Predicate.isTagged(capsule, 'Replays') && capsule.cwd === CWD &&
-          capsule.argv.join(' ') === ['stryker', ...ARGS].join(' ')
-        : Predicate.isTagged(capsule, 'DoesNotReplay') && capsule.why === rule
-    },
-  )
-
   it.prop(
     '∀fiber_RunOutcome_≡AnInterruptOnlyRunExits130WithARunInterruptedRecord',
     { of: [S.Int.check(S.isGreaterThanOrEqualTo(0))], subject: outcomeOf },

packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts

@@ -1,6 +1,7 @@
 import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface'
 import { describe, it } from '@systemfsoftware/vitest'
 import * as Arbitrary from 'effect/Arbitrary'
+import * as Equal from 'effect/Equal'
 import * as Result from 'effect/Result'
 import * as S from 'effect/Schema'
 
@@ -48,6 +49,7 @@ describe('interpretDryRunObservation', () => {
                     evidence.location.line === location.line &&
                     evidence.location.column === location.column) &&
                 evidence.message === expected.failureMessage &&
+                Equal.equals(evidence.reproduce ?? null, expected.reproduce ?? null) &&
                 evidence.stack === (expected.stack ?? null)
             })
         },