kiro-systemf · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026
diff --git a/.changeset/record-check-time-for-untested-verdicts.md b/.changeset/record-check-time-for-untested-verdicts.md
@@ -0,0 +1,5 @@
+---
+"@systemfsoftware/stryker-js": minor
+---
+
+A verdict the engine decided without running a test now records the cost it measured. A mutant whose checker rejected it, or which no test covered while a checker looked at it, records its share of that check call's wall time; a mutant every configured checker ignored, or one left uncovered with no checker configured, records zero. A static mutant that a checker rejected carries that check time in the `StaticVerdict` event's `costMs` as well. A shard plan therefore balances on recorded times instead of pricing those verdicts at a whole-suite prediction, which previously dominated the plan. Killed, Survived and Timeout verdicts keep the measured test time they already recorded, and the run budget still counts only the verdicts that ran a test.
diff --git a/...mance-issues/shard-plan-priced-untested-verdicts-at-a-whole-suite-prediction.md b/...mance-issues/shard-plan-priced-untested-verdicts-at-a-whole-suite-prediction.md
@@ -0,0 +1,48 @@
+---
+title: Shard plans balanced on CompileError counts because untested verdicts recorded no cost
+date: 2026-10-07
+category: performance-issues
+module: shard-planning
+problem_type: performance_issue
+component: tooling
+symptoms:
+  - "Mutation shards finish minutes apart although the plan predicts equal shard seconds"
+  - "Every shard receives an almost identical count of CompileError mutants"
+  - "The incremental report's costs entries for CompileError, NoCoverage and Ignored mutants have actualMs null"
+root_cause: logic_error
+resolution_type: code_fix
+severity: medium
+tags: [mutation-testing, sharding, lpt, cost-record, compile-error, incremental-report]
+---
+
+# Shard plans balanced on CompileError counts because untested verdicts recorded no cost
+
+## Problem
+
+`stryker plan` bin-packs mutants longest-first at `costs[id].actualMs ?? costs[id].predictedMs`. A verdict reached without running a test (CompileError, NoCoverage, Ignored) recorded no cost, so `actualMs` was null. The plan then priced it at `predictedMs`, the summed time of every test covering that mutant, even though no test would run for it.
+
+## Symptoms
+
+Main Mutation run 37602076567: stryker-js had 3547 CompileError mutants priced at about 68 s each, 241,914 s in total. All its Killed and Survived mutants together measured 3,216 s. The planner therefore balanced CompileError counts (177 or 178 per shard), while the test time each shard actually ran ranged from 88 s to 237 s. Shard wall time follows the test time (99 s + 1.51 × test seconds, r = 0.91), so the Mutation step ranged from 271 s to 498 s across shards.
+
+## What Didn't Work
+
+- **Pricing untested verdicts at 0.** LPT places each zero-cost item into the bin that is currently least loaded, and that bin does not change while only zero-cost items are being placed. Every check would land in a single shard.
+- **Capping CompileErrors per shard.** This is a second heuristic layered on the first, not a measurement.
+
+## Solution
+
+Record what each verdict measurably cost. The checker pool times each `check` call and splits the elapsed time evenly across that call's mutants. A mutant that several checkers see sums its shares. A CompileError, or a NoCoverage verdict that a checker looked at, carries `{ fixedOverheadMs: checkShare, testBodyMs: 0, testsExecuted: 0 }` as its verdict cost. An Ignored verdict, or NoCoverage with no checker, records 0 because nothing ran. The run budget still prices only verdicts that ran a test (`decidedWithoutATest` in `mutant-cost.ts`), so budget gates are unchanged.
+
+## Why This Works
+
+The planner's fallback assumes a mutant with no recorded cost will run its covering tests. That holds for a mutant that was never measured. It does not hold for a mutant whose verdict a checker or the coverage map has already decided. Once these verdicts carry a measured cost, the plan prices the work that will actually happen.
+
+## Prevention
+
+- A new verdict path must record a finite `costs[id].actualMs`. If it records null, the planner falls back to the whole-suite prediction. Gate: `pnpm --filter @systemfsoftware/stryker-js exec vitest run tests/check-cost-record.integration.test.ts tests/plan-recorded-costs.integration.test.ts`.
+  - wrong: a new early verdict is returned without a `cost`.
+  - right: it carries `checkOnlyCostOf(measuredMs)`, or 0 when nothing ran.
+- Judge plan quality by comparing per-shard Mutation-step seconds (from the run's jobs API) with per-shard recorded test seconds (from the merged report). The plan's `predictedSeconds` is not evidence. Gate: review. The reviewer cites both numbers by run id.
+  - wrong: "the plan predicts equal shards."
+  - right: "run 37602076567: Mutation step 271–498 s, recorded test time 88–237 s per shard."
diff --git a/packages/stryker-js/src/Checker/checker-pool.handle.ts b/packages/stryker-js/src/Checker/checker-pool.handle.ts
@@ -3,6 +3,7 @@ import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract'
 import type { Mutant } from '@systemfsoftware/stryker-js-plugin-interface'
 import { Checker } from '@systemfsoftware/stryker-js-plugin-interface'
 import * as Array from 'effect/Array'
+import * as Duration from 'effect/Duration'
 import * as Effect from 'effect/Effect'
 import * as Exit from 'effect/Exit'
 import { dual } from 'effect/Function'
@@ -36,6 +37,7 @@ export interface CheckedPlans {
   readonly passedPlans: readonly Mutant.MutantRunPlan[]
   readonly failedChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.FailedCheckResult])[]
   readonly ignoredChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.IgnoredCheckResult])[]
+  readonly checkMsByMutantId: Readonly<Record<string, number>>
 }
 
 const CheckerPoolHandle = Handle.make<Record<never, never>, CheckerPool>()(TypeId)
@@ -112,6 +114,28 @@ const noFailedChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.FailedCh
 
 const noIgnoredChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.IgnoredCheckResult])[] = []
 
+const noCheckMs: Readonly<Record<string, number>> = {}
+
+const checkMsOf = (checkMsByMutantId: Readonly<Record<string, number>>, mutantId: string): number =>
+  Option.getOrElse(Option.fromUndefinedOr(checkMsByMutantId[mutantId]), () => 0)
+
+const chargedWith = (
+  carried: Readonly<Record<string, number>>,
+  group: readonly Mutant.MutantRunPlan[],
+  elapsedMs: number,
+): Readonly<Record<string, number>> => {
+  if (group.length === 0) {
+    return carried
+  }
+  const shareMs = elapsedMs / group.length
+  return {
+    ...carried,
+    ...Object.fromEntries(
+      group.map((plan) => [plan.mutant.id, checkMsOf(carried, plan.mutant.id) + shareMs] as const),
+    ),
+  }
+}
+
 export const splitCheckedPlans = Effect.fn(SpanTaxonomy.Spans.checkerPoolSplitChecked.name)(function*(
   checked: readonly (readonly [Mutant.MutantRunPlan, Checker.CheckResult])[],
 ) {
@@ -146,42 +170,56 @@ export const splitCheckedPlans = Effect.fn(SpanTaxonomy.Spans.checkerPoolSplitCh
           ),
       }),
   )
-  return { passedPlans, failedChecks, ignoredChecks }
+  return { passedPlans, failedChecks, ignoredChecks } satisfies Omit<CheckedPlans, 'checkMsByMutantId'>
 })
 
 const checkerNamesOf = (pool: CheckerPool): Effect.Effect<readonly string[], StageError | CheckerCrash> =>
   Pool.use(pool, (slot) => Effect.succeed(slot.map(({ checkerName }) => checkerName)))
 
 const failedElement = (
   failedChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.FailedCheckResult])[],
+  checkMsByMutantId: Readonly<Record<string, number>>,
 ): Stream.Stream<CheckedPlans, StageError | CheckerCrash> =>
   Option.match(Array.head(failedChecks), {
     onNone: () => Stream.empty,
     onSome: () =>
-      Stream.succeed<CheckedPlans>({ passedPlans: noPassedPlans, failedChecks, ignoredChecks: noIgnoredChecks }),
+      Stream.succeed<CheckedPlans>({
+        passedPlans: noPassedPlans,
+        failedChecks,
+        ignoredChecks: noIgnoredChecks,
+        checkMsByMutantId,
+      }),
   })
 
 const ignoredElement = (
   ignoredChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.IgnoredCheckResult])[],
+  checkMsByMutantId: Readonly<Record<string, number>>,
 ): Stream.Stream<CheckedPlans, StageError | CheckerCrash> =>
   Option.match(Array.head(ignoredChecks), {
     onNone: () => Stream.empty,
     onSome: () =>
-      Stream.succeed<CheckedPlans>({ passedPlans: noPassedPlans, failedChecks: noFailedChecks, ignoredChecks }),
+      Stream.succeed<CheckedPlans>({
+        passedPlans: noPassedPlans,
+        failedChecks: noFailedChecks,
+        ignoredChecks,
+        checkMsByMutantId,
+      }),
   })
 
 const checkedGroupsFor = (
   pool: CheckerPool,
   checkerNames: readonly string[],
   checkerIndex: number,
   plans: readonly Mutant.MutantRunPlan[],
+  carried: Readonly<Record<string, number>>,
 ): Stream.Stream<CheckedPlans, StageError | CheckerCrash> =>
   Option.match(Array.get(checkerNames, checkerIndex), {
     onNone: () =>
       Stream.succeed<CheckedPlans>({
         passedPlans: plans,
         failedChecks: noFailedChecks,
         ignoredChecks: noIgnoredChecks,
+        checkMsByMutantId: carried,
       }),
     onSome: (checkerName) =>
       Stream.unwrap(
@@ -190,18 +228,26 @@ const checkedGroupsFor = (
             Stream.fromIterable(groups).pipe(
               Stream.mapEffect(
                 (group) =>
-                  onCheckerSlot(pool, checkerIndex, (checker) => checkPlansWithChecker(checker, checkerName, group))
-                    .pipe(
-                      Effect.flatMap((checked) => splitCheckedPlans(checked)),
-                    ),
+                  onCheckerSlot(pool, checkerIndex, (checker) =>
+                    Effect.timed(checkPlansWithChecker(checker, checkerName, group)).pipe(
+                      Effect.flatMap(([elapsed, checked]) =>
+                        Effect.map(
+                          splitCheckedPlans(checked),
+                          (split) => ({
+                            split,
+                            charged: chargedWith(carried, group, Duration.toMillis(elapsed)),
+                          }),
+                        )
+                      ),
+                    )),
                 { concurrency: 'unbounded', unordered: true },
               ),
-              Stream.flatMap((split) =>
+              Stream.flatMap(({ split, charged }) =>
                 Stream.concat(
-                  ignoredElement(split.ignoredChecks),
+                  ignoredElement(split.ignoredChecks, charged),
                   Stream.concat(
-                    failedElement(split.failedChecks),
-                    checkedGroupsFor(pool, checkerNames, checkerIndex + 1, split.passedPlans),
+                    failedElement(split.failedChecks, charged),
+                    checkedGroupsFor(pool, checkerNames, checkerIndex + 1, split.passedPlans, charged),
                   ),
                 )
               ),
@@ -232,10 +278,13 @@ export const checkPlansStream: {
             passedPlans: plans,
             failedChecks: noFailedChecks,
             ignoredChecks: noIgnoredChecks,
+            checkMsByMutantId: noCheckMs,
           }),
         onSome: (handle) =>
           checkerNamesOf(CheckerPoolHandle.slot(handle)).pipe(
-            Effect.map((checkerNames) => checkedGroupsFor(CheckerPoolHandle.slot(handle), checkerNames, 0, plans)),
+            Effect.map((checkerNames) =>
+              checkedGroupsFor(CheckerPoolHandle.slot(handle), checkerNames, 0, plans, noCheckMs)
+            ),
             Stream.unwrap,
           ),
       }),
@@ -251,18 +300,24 @@ export const checkPlans = Effect.fnUntraced(function*(
   const passedPlans: Mutant.MutantRunPlan[] = []
   const failedChecks: (readonly [Mutant.MutantRunPlan, Checker.FailedCheckResult])[] = []
   const ignoredChecks: (readonly [Mutant.MutantRunPlan, Checker.IgnoredCheckResult])[] = []
+  const checkMsByMutantId: Record<string, number> = {}
   for (const group of checkedGroups) {
     passedPlans.push(...group.passedPlans)
     failedChecks.push(...group.failedChecks)
     ignoredChecks.push(...group.ignoredChecks)
+    Object.assign(checkMsByMutantId, group.checkMsByMutantId)
   }
-  return { passedPlans, failedChecks, ignoredChecks } satisfies CheckedPlans
+  return { passedPlans, failedChecks, ignoredChecks, checkMsByMutantId } satisfies CheckedPlans
 })
 
 export interface CheckedPlansExecution<A, E> {
-  readonly settleFailure: (plan: Mutant.MutantRunPlan, result: Checker.FailedCheckResult) => Effect.Effect<A, E>
+  readonly settleFailure: (
+    plan: Mutant.MutantRunPlan,
+    result: Checker.FailedCheckResult,
+    checkMs: number,
+  ) => Effect.Effect<A, E>
   readonly settleIgnored: (plan: Mutant.MutantRunPlan, result: Checker.IgnoredCheckResult) => Effect.Effect<A, E>
-  readonly runPlan: (plan: Mutant.MutantRunPlan) => Effect.Effect<A, E>
+  readonly runPlan: (plan: Mutant.MutantRunPlan, checkMs: number) => Effect.Effect<A, E>
   readonly concurrency: number
 }
 
@@ -283,11 +338,13 @@ export const runCheckedPlans: {
     execution: CheckedPlansExecution<A, E>,
   ): Stream.Stream<A, E | StageError | CheckerCrash> =>
     self.pipe(
-      Stream.flatMap(({ passedPlans, failedChecks, ignoredChecks }) =>
+      Stream.flatMap(({ passedPlans, failedChecks, ignoredChecks, checkMsByMutantId }) =>
         Stream.fromIterable<() => Effect.Effect<A, E>>([
-          ...failedChecks.map(([plan, result]) => () => execution.settleFailure(plan, result)),
+          ...failedChecks.map(([plan, result]) => () =>
+            execution.settleFailure(plan, result, checkMsOf(checkMsByMutantId, plan.mutant.id))
+          ),
           ...ignoredChecks.map(([plan, result]) => () => execution.settleIgnored(plan, result)),
-          ...passedPlans.map((plan) => () => execution.runPlan(plan)),
+          ...passedPlans.map((plan) => () => execution.runPlan(plan, checkMsOf(checkMsByMutantId, plan.mutant.id))),
         ])
       ),
       Stream.mapEffect((work) => work(), { concurrency: Math.max(1, execution.concurrency), unordered: true }),

diff --git a/packages/stryker-js/src/budget.ts b/packages/stryker-js/src/budget.ts
@@ -1,8 +1,9 @@
 import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract'
 import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface'
 import * as Arr from 'effect/Array'
+import * as Boolean from 'effect/Boolean'
 
-import { costTotalMsOf } from './mutant-cost.js'
+import { costOrZero, decidedWithoutATest } from './mutant-cost.js'
 
 const MILLIS_PER_SECOND = 1000
 
@@ -13,11 +14,15 @@ export interface BudgetInput {
   readonly planPredictedSeconds?: number | undefined
 }
 
-const costMsOf = (result: Mutant.RunMutantResult): number => costTotalMsOf(result.cost) ?? 0
+const testWorkMsOf = (result: Mutant.RunMutantResult): number =>
+  Boolean.match(decidedWithoutATest(result.status), {
+    onTrue: () => 0,
+    onFalse: () => costOrZero(result.cost),
+  })
 
 const predictedSecondsOf = (results: readonly Mutant.RunMutantResult[], concurrency: number): number => {
   const lanes = concurrency > 0 ? concurrency : 1
-  return Arr.reduce(results, 0, (total, result) => total + costMsOf(result)) / lanes / MILLIS_PER_SECOND
+  return Arr.reduce(results, 0, (total, result) => total + testWorkMsOf(result)) / lanes / MILLIS_PER_SECOND
 }
 
 export const budgetOf = (input: BudgetInput): RunEvent.Budget => ({

diff --git a/packages/stryker-js/src/mutant-cost.ts b/packages/stryker-js/src/mutant-cost.ts
@@ -1,5 +1,6 @@
 import type { Mutant, TestRunner } from '@systemfsoftware/stryker-js-plugin-interface'
 import * as Match from 'effect/Match'
+import * as Option from 'effect/Option'
 
 export interface MutantCostInput {
   readonly elapsedMs: number
@@ -23,6 +24,24 @@ const bodyTimeWithinWall = (elapsedMs: number, testBodyMs: number): number => Ma
 export const costTotalMsOf = (cost: Mutant.MutantCost | undefined): number | undefined =>
   cost === undefined ? undefined : cost.fixedOverheadMs + cost.testBodyMs
 
+export const costOrZero = (cost: Mutant.MutantCost | undefined): number =>
+  Option.getOrElse(Option.fromUndefinedOr(costTotalMsOf(cost)), () => 0)
+
+const DECIDED_WITHOUT_A_TEST: Partial<Record<Mutant.MutantStatus, true>> = {
+  CompileError: true,
+  NoCoverage: true,
+  Ignored: true,
+}
+
+export const decidedWithoutATest = (status: Mutant.MutantStatus): boolean => DECIDED_WITHOUT_A_TEST[status] === true
+
+export const checkOnlyCostOf = (checkMs: number): Mutant.MutantCost => ({
+  fixedOverheadMs: checkMs,
+  testBodyMs: 0,
+  testsExecuted: 0,
+  shared: false,
+})
+
 export const mutantCostOf = (input: MutantCostInput): Mutant.MutantCost => {
   const testBodyMs = bodyTimeWithinWall(input.elapsedMs, input.testBodyMs)
   return {

diff --git a/packages/stryker-js/src/reporting/static-verdict.ts b/packages/stryker-js/src/reporting/static-verdict.ts
@@ -3,10 +3,7 @@ import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface'
 import * as Arr from 'effect/Array'
 import * as Option from 'effect/Option'
 
-import { costTotalMsOf } from '../mutant-cost.js'
-
-const costOrZero = (cost: Mutant.MutantCost | undefined): number =>
-  Option.getOrElse(Option.fromUndefinedOr(costTotalMsOf(cost)), () => 0)
+import { costOrZero } from '../mutant-cost.js'
 
 export const staticVerdictOf = (results: readonly Mutant.RunMutantResult[]): RunEvent.StaticVerdict | null => {
   const statics = Arr.filter(results, (result) => result.static === true)

diff --git a/packages/stryker-js/src/run/mutation-test.cell.ts b/packages/stryker-js/src/run/mutation-test.cell.ts
@@ -2,6 +2,7 @@ import { Sandwich } from '@systemfsoftware/effect-cell-types'
 import { RunEvent, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract'
 import type { Mutant } from '@systemfsoftware/stryker-js-plugin-interface'
 import type { Plugin } from '@systemfsoftware/stryker-js-plugin-interface'
+import * as Boolean from 'effect/Boolean'
 import * as Clock from 'effect/Clock'
 import * as Duration from 'effect/Duration'
 import * as Effect from 'effect/Effect'
@@ -24,6 +25,7 @@ import {
   makeCheckerPoolHandle,
   runCheckedPlans,
 } from '../Checker/checker-pool.handle.js'
+import { checkOnlyCostOf, decidedWithoutATest } from '../mutant-cost.js'
 import { MutationReporting } from '../mutation-reporting.service.js'
 import { MutationTestCommand } from '../MutationTest.schema.js'
 import { withPhaseSpan } from '../reporter-stream.service.js'
@@ -60,6 +62,12 @@ const countIgnoredByReason = (
   reason: string,
 ): number => results.filter((result) => result.status === 'Ignored' && result.statusReason === reason).length
 
+const withMeasuredCheckCost = (result: Mutant.RunMutantResult, checkMs: number): Mutant.RunMutantResult =>
+  Boolean.match(Boolean.and(result.cost === undefined, decidedWithoutATest(result.status)), {
+    onFalse: () => result,
+    onTrue: () => ({ ...result, cost: checkOnlyCostOf(checkMs) }),
+  })
+
 export interface MutationTestDone {
   readonly results: readonly Mutant.RunMutantResult[]
   readonly verdict: Plugin.ExitClass | null
@@ -169,7 +177,10 @@ const proceedPipeline = Effect.fnUntraced(function*(raw: MutationTestRaw) {
     plannedMutants: [...rememberedResults, ...reuse.mutants],
     pathService,
   }
-  const settledResults = [...rememberedResults, ...plan.earlyResults]
+  const settledResults = [
+    ...rememberedResults,
+    ...plan.earlyResults.map((result) => withMeasuredCheckCost(result, 0)),
+  ]
   yield* Effect.forEach(settledResults, (result) => announceSettledMutant(context, result), {
     concurrency: 1,
     discard: true,
@@ -187,19 +198,22 @@ const proceedPipeline = Effect.fnUntraced(function*(raw: MutationTestRaw) {
       { total: plan.plannedTotal, testRunners: prev.concurrency.testRunners },
       () =>
         runCheckedPlans(checkedPlans, {
-          settleFailure: (mutantPlan, result) =>
+          settleFailure: (mutantPlan, result, checkMs) =>
             reporting.reportCheckFailure(toReportedMutant(mutantPlan.mutant), result).pipe(
+              Effect.map((reported) => withMeasuredCheckCost(reported, checkMs)),
               Effect.flatMap(settleReported),
             ),
           settleIgnored: (mutantPlan, result) =>
             reporting.reportIgnored(toReportedMutant(mutantPlan.mutant), result).pipe(
+              Effect.map((reported) => withMeasuredCheckCost(reported, 0)),
               Effect.flatMap(settleReported),
             ),
-          runPlan: (runPlan) =>
+          runPlan: (runPlan, checkMs) =>
             Option.match(Option.liftPredicate(runPlan, isNoCoveragePlan), {
               onNone: () => Effect.scoped(mutantRunCell.run({ context, testRunnerPool, checkpoint, plan: runPlan })),
               onSome: (noCoverageRunPlan) =>
                 reporting.reportNoCoverage(toReportedMutant(noCoverageRunPlan.mutant)).pipe(
+                  Effect.map((reported) => withMeasuredCheckCost(reported, checkMs)),
                   Effect.flatMap(settleReported),
                 ),
             }),