From 25fdbe72b9ce87a53e899196e4e5f8d19430b8b6 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:42 -0400 Subject: [PATCH 1/8] fix(repo): draw every deciding timeout class in the timeout-fields property (#146) * docs(repo): plan agent-ready failure diagnostics Verdict-Semantics: unchanged * fix(repo): draw every deciding timeout class in the timeout-fields property Verdict-Semantics: unchanged --- .changeset/witty-spies-try.md | 5 + .changeset/young-memes-mix.md | 5 + ...at-agent-ready-failure-diagnostics-plan.md | 886 ++++++++++++++++++ .../src/mutant-timeout-reason.schema.ts | 13 +- .../src/mutation-reporting.service.ts | 101 +- 5 files changed, 962 insertions(+), 48 deletions(-) create mode 100644 .changeset/witty-spies-try.md create mode 100644 .changeset/young-memes-mix.md create mode 100644 docs/plans/2026-10-01-2306-feat-agent-ready-failure-diagnostics-plan.md diff --git a/.changeset/witty-spies-try.md b/.changeset/witty-spies-try.md new file mode 100644 index 00000000..8d0076bb --- /dev/null +++ b/.changeset/witty-spies-try.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-plugin-interface": major +--- + +TestRunner.HitLimitReasonText now accepts only the canonical `Hit limit reached (count/limit)` shape; text that merely starts with the prefix no longer validates. diff --git a/.changeset/young-memes-mix.md b/.changeset/young-memes-mix.md new file mode 100644 index 00000000..7bb292b3 --- /dev/null +++ b/.changeset/young-memes-mix.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": patch +--- + +Timeout reasons that only start with `Hit limit reached` no longer classify as hit-limit timeouts; the timeout-fields property now draws every deciding input class under the mutation-worker budget. diff --git a/docs/plans/2026-10-01-2306-feat-agent-ready-failure-diagnostics-plan.md b/docs/plans/2026-10-01-2306-feat-agent-ready-failure-diagnostics-plan.md new file mode 100644 index 00000000..691305be --- /dev/null +++ b/docs/plans/2026-10-01-2306-feat-agent-ready-failure-diagnostics-plan.md @@ -0,0 +1,886 @@ +--- +title: Agent-Ready Failure Diagnostics - Plan +type: feat +date: 2026-10-01 +topic: agent-ready-failure-diagnostics +artifact_contract: ce-unified-plan/v1 +product_contract_source: ce-brainstorm +execution: code +--- + +# Agent-Ready Failure Diagnostics - Plan + +## Goal Capsule + +- **Objective:** When any run in this repository fails, a coding agent with no human help can tell what failed, reproduce it in one step, and land the right fix, using only what the run published. Property tests and mutation lanes can no longer pass on one lane and fail on another. +- **Means:** one typed failure record per failure, rendered by one renderer onto every output surface (KTD1, KTD3); property lanes made sound by identity seeds, coverage classes and a scheduled sweep (KTD12-KTD14, KTD21); a PR lane that runs the mutation preflight and the mutants the change invalidates (KTD16, KTD17). +- **Product authority:** the user, in the 2026-10-01 debug and brainstorm session. Scope covers this repository plus the upstream `@systemfsoftware/vitest` property engine in `systemfsoftware/systemfsoftware`, tracked in systemfsoftware/systemfsoftware#576. Waking or dispatching the agent is not in scope. +- **Execution profile:** Deep. 19 units in three phases, shipped as one `gh stack` with one concern per layer (Sequencing). Phase A needs nothing outside this repository. Phase B needs the #576 release. Phase C edits `.github/workflows/` under the user's authorization for this plan (KTD19). +- **Stop conditions:** if #576 has no release, or its release lacks a property field R3 names, finish Phase A and leave U14 and U15 unstarted; never parse property facts out of message text to fill the gap. If a contract guard or api report demands a bump a unit did not plan, follow the guard; never edit a guard to pass. Workflow edits stay within U17-U19; no other unit touches `.github/workflows/`. +- **Who finishes:** `ce-work` executes every unit, including the Phase C workflow edits. The maintainer (A3) reviews the workflow diff, merges and cuts the releases. +- **Open blockers:** none for Phase A. Phase B waits on the #576 release (R29). + +--- + +## Product Contract + +### Summary + +Every failure in this repository, from a single property test up to a CI job, becomes one typed record carrying a stable code, its evidence, a one-step reproduction and the next action. The stream, the exit code, the CI summary and annotations, SARIF, the terminal, MCP and the Mutation Server Protocol all render that record. Property tests become sound under any seed: each property runs its own fixed seed in every lane, a random sweep on `main` feeds broken seeds back as regression seeds, and a property that never draws the inputs that decide its output fails for that reason directly. PRs run a mutation preflight and the mutants their change can affect, so lane-only failures and new surviving mutants are caught before merge, while the full mutation run stays on `main`. + +### Problem Frame + +On 2026-10-01 the Mutation workflow on `main` (run 36935602456, after PR #136 moved to effect 4.0.0 and `@systemfsoftware/vitest` 1.0.0) failed in five of six `stryker-js` shards, each with exit 3 after a dry run of more than a minute. The cause was a test-only defect. Mutation workers run every property at `runs: 30, seed: 1` (`packages/toolchain/vitest-config/lib/property-runs.js`). Seed 1 draws no `Timeout` status for the property at `packages/stryker-js/src/mutation-reporting.service.ts:1043`, so its subject always returned `{}`. The upstream constant-impostor gate then flagged the file as vacuous. PR CI had passed, because it uses random seeds at 100 runs, where this defect surfaces about 3% of the time. + +The run published almost nothing an agent could act on. The upstream message printed the frozen output as `[object Object]` and named no seed, no draws and no undrawn input class. Stryker's terminal stream envelope did hold the dry-run failure. It was prefixed with an internal tag, cut at 1024 characters, carried `reason: null`, and suggested reading a report file that a failed dry run never writes. The CI summary script ignored that envelope and reported "infrastructure failure (missing binary, crashed run or timeout)". This contradicts the repository's own recorded learning in `docs/solutions/workflow-issues/mutation-lane-green-while-every-job-failed.md`. The failing shards' artifacts were deleted by the report job minutes later. Every shard repeated the same failing dry run, a warning with no headline fired on every run, and no trace was exported for the lane. Reconstructing the cause took a local reproduction, a seed probe and a reading of upstream library source. + +`STRATEGY.md` names CI pipelines and AI coding agents as primary consumers and the NDJSON stream plus structured exit codes as the contract. Grounding found the same gap across the whole failure surface, not only on this path: + +- the stream schema can type only plugin-load failures, a CONST-D2 violation; +- dry-run failures reduce to `{name, failureMessage}`; +- worker out-of-memory and crash classifications are flattened into a generic stage error; +- the Mutation Server Protocol drops the exit class, and MCP rerun failures die; +- other existing properties whose deciding inputs are rare are candidates for the same vacuity. + +### Actors + +- A1. Coding agent: reads published failures, reproduces them, and lands fixes on its own. The primary reader for every requirement here. +- A2. CI pipeline: runs the per-commit, PR, `main` and scheduled lanes and publishes their records. +- A3. Maintainer: merges and cuts upstream releases. +- A4. Upstream property engine: `@systemfsoftware/vitest`, which produces the property failures this repository consumes. + +### Key Decisions + +- **One typed failure record is the source of truth, every record carries a reproduction capsule, and MCP and the Mutation Server Protocol are projections of it.** A record without a replay leaves the agent reading prose; a replay without a record leaves CI guessing. Governs R1, R2, R3, R4, R5, R8, R9, R10, R11, R12, R13. (session-settled: user-directed — chosen over record-only, reproduction-first, and query-first designs: the agent needs both a typed classification and a one-step replay, and the existing MCP and Mutation Server Protocol surfaces already lose the failure class.) +- **The primary reader is an agent that closes the loop alone.** Diagnostics are judged by whether an agent can classify, reproduce and fix a failure without a human or log digging. Governs R3, R4, R5, R9. (session-settled: user-directed — chosen over agent triage with human approval, human-first summaries, and equal weighting: the user wants failures fixed without involvement.) +- **Scope is the whole failure surface, including the upstream property engine.** Governs R21, R29. (session-settled: user-directed — chosen over limiting this work to one of run-failure diagnostics, property soundness, or the upstream message: leave nothing behind.) +- **Seeds follow the Hypothesis split: a fixed seed per property in every per-commit lane, random exploration on a schedule.** A seed-dependent property is a broken property; deterministic per-commit lanes make a red build mean "this change broke it", and the mutation lane needs reproducible verdicts. Governs R16, R17, R18. (session-settled: user-directed — chosen over random PR seeds with coverage checks only, and over adding multi-seed sweeps to every PR: matches Hypothesis's documented `derandomize` and `ci` profile design.) +- **PRs run a mutation preflight plus the mutants their change can affect; the full run stays on `main`.** Measured full runs cost 83-95 runner-minutes, and reusing `main`'s verdicts limits a PR to the mutants its change affects. `CONSTITUTION.md` CONST-T12 defines the PR set as change-relevant mutants, which the verdict cache's reuse refusal already computes; changed lines alone would miss mutants whose covering tests or imports changed. Governs R23, R24, R25, R27. (session-settled: user-directed — chosen over preflight-only and seed-split-only PR lanes, after the user rejected full mutation on PRs for cost: scoped runs reusing `main`'s verdicts are cheap.) +- **Automated actors never suppress surviving mutants; an equivalent mutant is resolved by refactoring the code so the mutant cannot exist**, which is CONST-T12's "deleting the dead branch it exploits". Governs R5, R26. (session-settled: user-directed — chosen over agent suppression backed by machine proof and agent suppression with human review: equivalent mutants signal redundant code.) +- **Hit-limit timeout reasons have one canonical shape, `Hit limit reached (count/limit)`.** Governs R22. (session-settled: user-directed — chosen over the prefix-only rule production uses today: the full shape is what the decoder and the Vitest runner already produce.) +- **Waking or dispatching the agent is outside this work.** This work guarantees that the published record is sufficient; something else decides when an agent reads it. (session-settled: user-directed — chosen over CI-started agents and published-for-pickup triggers: not the responsibility of this work.) +- **This is a breaking cutover.** The stream schema, exit codes, envelope and every consumer move together with no compatibility path, per `AGENTS.md` BREAK-1 and `CONSTITUTION.md` CONST-D2. Governs R2, R8. + +### Requirements + +```mermaid +flowchart TB + F[Failure at its source: test, property, stage, worker, config, CI] --> R[Typed failure record: code, evidence, reproduction capsule, next action] + R --> S1[NDJSON stream terminal event and exit code] + R --> S2[CI summary and file:line annotations] + R --> S3[SARIF] + R --> S4[MCP and Mutation Server Protocol responses] + R --> S5[Terminal output] + R --> S6[OTel span attributes and trace id] +``` + +**Typed failure record** + +- R1. Every failure that ends or degrades a run is described by one typed record: argument parsing, configuration, plugin load, sandbox preparation, instrumentation, checker, dry run, worker crash or out-of-memory, mutant-phase infrastructure, reporter, interruption, and every CI-level failure. +- R2. Each record carries a stable code from one closed, documented catalog; prose is a rendering of the record and never the only carrier of a fact. +- R3. Each record carries its evidence: the failing stage, the failing test's identity with file and line where one exists, and the full cause chain without truncation. For property failures it also carries the seed, run count, counterexample or structurally rendered frozen output, and the declared input classes that were never drawn. +- R4. Each record carries a reproduction capsule: one local command, including seed, lane budget and environment, that reproduces the failure. A failure that cannot replay (out-of-memory, missing binary) says so and names the evidence that stands in for the replay. +- R5. Each record names its next action from a closed set that distinguishes at least fixing a test, fixing code, adding a test to kill a survivor, refactoring away an equivalent mutant, fixing configuration, and retrying infrastructure. +- R6. No record or rendering contains internal type tags, framework prefixes, or placeholder text such as `[object Object]`, "Unknown failure", or remediation that does not apply to the failure at hand. +- R7. A failure the catalog does not classify carries a dedicated code that marks it as a catalog gap, with its full cause attached. + +**Surfaces projected from the record** + +- R8. The NDJSON stream emits the full record as its terminal failure event, and the exit code derives from the record's class, with a code of its own for a failing baseline or dry run. +- R9. The CI summary and annotations render from the record, including a file:line annotation for a failing test. CI reports infrastructure failure only when a record says so, and reports a run that produced no record as exactly that. +- R10. SARIF output includes run-failure records alongside surviving mutants, and CI uploads it. +- R11. MCP and Mutation Server Protocol responses carry the same record, with no loss of class and no crash on engine failure. +- R12. Human terminal output renders from the record. +- R13. Mutation lanes export traces the way the e2e lane does, OTel spans carry the record's code, and the record carries its trace id, so `AGENTS.md` OBS-1 applies to mutation failures. +- R14. Evidence a failed run produced, including its stream and record, stays downloadable after the run ends. +- R15. A warning fires only when its condition holds, and a failing run's log carries no lines that explain nothing about the run. + +**Property-test soundness** + +- R16. Each property runs under its own fixed seed derived from its identity, the same in every per-commit lane: local, PR CI and mutation workers. No seed is shared across all properties. +- R17. A scheduled job on `main` runs properties under random seeds; a seed that breaks a property appears in the sweep's record, and the fix for it commits that seed as a regression seed that every lane replays from then on. +- R18. Each property declares the input classes that decide its output, and a class never drawn fails the property directly under any seed, naming the class. +- R19. The vacuity check keeps judging the draws the property actually ran, and its verdict reports the evidence of every property that shares the subject. +- R20. Property failures (refuted, vacuous, under-covered, non-boolean) arrive as structured data attributed to the property that owns them, never as a file-level prose blob. +- R21. Every existing property in this repository meets R16-R20, and those the new checks flag are fixed, including test models that reimplement production rules. +- R22. Production, the Vitest runner, and every test model classify hit-limit timeouts by the canonical `Hit limit reached (count/limit)` shape. + +**PR and main lanes** + +- R23. Every PR runs one dry-run preflight under mutation-worker conditions before any mutant runs, and a preflight failure stops the lane with its record. +- R24. A PR evaluates exactly the mutants whose reused `main` verdict its change invalidates and reuses every other verdict. A dependency-only PR runs the preflight only, and a PR lane that evaluates no mutants reports that outcome explicitly rather than a pass. +- R25. A new surviving mutant among those the PR evaluated blocks the PR and arrives as a record naming the mutant, its location, its covering tests and its next action. +- R26. No automated actor suppresses a surviving mutant. +- R27. `main` keeps the full mutation run, and its failures and the scheduled sweep's failures publish records of the same shape as PR failures. +- R28. A mutation run performs its dry run once, not once per shard. + +**Upstream property engine** + +- R29. The upstream `@systemfsoftware/vitest` changes that R3, R4 and R16-R20 depend on ship as an upstream release, tracked in systemfsoftware/systemfsoftware#576, and this repository consumes that release. + +### Key Flows + +- F1. Property defect caught on a PR + - **Trigger:** A PR changes code or dependencies so that a property becomes vacuous or under-covered. + - **Actors:** A1, A2 + - **Steps:** PR CI runs the property under its fixed seed; it fails on an undrawn class or the vacuity check; CI publishes the record; the agent runs the capsule, sees the same failure, and fixes the property or the code. + - **Outcome:** The defect never reaches `main`. + - **Covered by:** R3, R4, R16, R18, R19, R20 +- F2. Preflight failure on a PR + - **Trigger:** A test fails only under mutation-worker conditions. + - **Actors:** A1, A2 + - **Steps:** The preflight runs once and fails; the lane stops; the summary shows the record's code, the failing test at file:line, and the capsule. + - **Outcome:** No shard time is spent, and nothing is labelled infrastructure. + - **Covered by:** R8, R9, R23, R28 +- F3. New survivor on a PR + - **Trigger:** Changed code introduces a mutant no test kills. + - **Actors:** A1, A2 + - **Steps:** PR mutation reports the survivor; the PR is blocked; the record names the action: add a killing test, or refactor away if the mutant is equivalent. + - **Outcome:** The agent unblocks the PR without suppressing anything. + - **Covered by:** R5, R24, R25, R26 +- F4. Sweep finds a breaking seed on `main` + - **Trigger:** The scheduled random run breaks a property. + - **Actors:** A2, A1 + - **Steps:** The sweep publishes the record with the breaking seed; the agent's fix commits that seed, and every lane replays it from then on. + - **Outcome:** A found counterexample can never pass silently again. + - **Covered by:** R17, R27 + +### Acceptance Examples + +- AE1. **Covers R16, R18, R20.** Given the tree at `4ec69700d` with the timeout-fields property unchanged, when PR CI runs, then the property fails on the PR under its own seed, naming the never-drawn `Timeout` class, and the record's capsule reproduces it locally. +- AE2. **Covers R8, R9, R23.** Given a test that fails only under mutation-worker conditions, when the preflight runs, then the exit code is the baseline-failure code and the CI summary shows the test at file:line with the capsule, and the word "infrastructure" appears nowhere. +- AE3. **Covers R1, R4, R5.** Given a worker killed for out-of-memory during the mutant phase, when the run ends, then the record's code is the out-of-memory class, the capsule states that the failure does not replay and names the memory evidence, and the next action is infrastructure. +- AE4. **Covers R5, R25, R26.** Given a survivor among a PR's evaluated mutants that no test can kill, when the PR lane reports it, then the next action is refactoring the code away, and no suppression is added by any automated actor. +- AE5. **Covers R15.** Given a configuration with `warnings.unknownOptions` enabled and no unknown options, when a run starts, then no unknown-options warning is printed. +- AE6. **Covers R24.** Given a PR that only bumps a dependency, when PR CI runs, then the preflight runs, no mutants run, and the lane reports that it evaluated none. +- AE7. **Covers R7.** Given a failure no catalog entry classifies, when it occurs, then the record carries the catalog-gap code with the full cause, not free text with a null reason. + +### Success Criteria + +- Replaying the 2026-10-01 incident, an agent given only the published record identifies the cause and produces the fix without reading raw logs or reading library source. +- No rendering of any record contains the strings `[object Object]`, "Unknown failure", or an internal type tag. +- Every catalog code has a documented meaning, next action and capsule rule. +- A PR whose change invalidates no reused verdict adds only the preflight to its CI time, and the preflight completes in a few minutes, in line with today's roughly one-minute dry run plus job setup. + +### Scope Boundaries + +- Which files are mutated, score thresholds, and verdict semantics are unchanged. +- Waking or dispatching an agent when a record is published is out of scope. +- No automated route to removing a mutant is added; the existing mutant-set policy rules are unchanged. +- Full mutation on PRs is out of scope. + +### Dependencies / Assumptions + +- The upstream release (R29) lands before the repository parts that consume structured property failures, per-property seeds and recorded-seed replay. +- GitHub Actions lets PR runs restore caches created on the base branch and forbids PR runs from writing the default branch's scope (GitHub docs, "Restrictions for accessing a cache"), so a PR can reuse `main`'s verdicts but never change them. +- The verdict cache refuses all reuse when a mutant's closure or run-inputs digest changes (`packages/stryker-js/src/incremental-diff.workflow.ts`), which is why dependency-only PRs run the preflight only. +- Stryker already has a dry-run-only mode (`packages/stryker-js/src/bin/cli-command.ts`). + +### Sources / Research + +- Incident: CI run 36935602456; local reproduction with `STRYKER_WORKER_DIR` set; a seed probe showing seed 1 draws zero `Timeout` statuses in 30 runs while seeds 2-10 draw 1-7. +- Failure path in this repository: `packages/stryker-js-cli-contract/src/run-event.schema.ts` (RunFailed reason), `packages/stryker-js/src/run-event-stream.service.ts`, `packages/stryker-js/src/reporting/run-failure.ts`, `packages/stryker-js/src/bin/main.ts` (1024-character limit), `packages/stryker-js/src/interpret-dry-run-observation.workflow.ts`, `packages/stryker-js/src/Checker/checker-pool.handle.ts`, `packages/stryker-js/src/Serve/msp.schema.ts`, `packages/stryker-js/src/Mcp/mcp-server.cell.ts`, `packages/stryker-js/src/run/validate-options-admission.workflow.ts`, `packages/stryker-js/src/sarif-report.workflow.ts` (existing SARIF reporter, not enabled in this repository). +- CI: `.github/workflows/mutation.yml`, `.github/workflows/ci.yml`, `scripts/mutation-job.ts`, `scripts/lib/mutation-plan.ts`. +- Property lanes: `packages/toolchain/vitest-config/lib/property-runs.js`, `packages/toolchain/vitest-config/lib/base.js`, `packages/toolchain/vitest-config/lib/schema-laws.js`, `packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts`, `packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts`, `packages/stryker-js-instrumenter/src/__tests__/mutant-set-policy.workflow.property.test.ts`. +- Upstream: `systemfsoftware/systemfsoftware` `packages/runner/vitest/src/internal/property/` (`engine.ts`, `impostor.ts`, `failure-record.ts`). +- Prior work: `docs/plans/2026-09-29-0427-feat-state-of-the-art-mutation-testing-plan.md` (R29 gate on new survivors, R30 SARIF, R35 Mutation Server Protocol, R40 and KTD19 OTel); `docs/plans/2026-09-25-0337-chore-systemfsoftware-latest-packages-plan.md` (KTD7 impostor gate); `docs/solutions/workflow-issues/mutation-lane-green-while-every-job-failed.md`; `docs/solutions/workflow-issues/exit-codes-through-runtime-teardown.md`; `docs/solutions/test-failures/stream-schema-must-not-carry-json-rest-records.md`; `docs/solutions/tooling-decisions/verdict-cache-content-keyed-reuse.md`. +- External: rustc JSON diagnostics (https://doc.rust-lang.org/stable/rustc/json.html); SARIF 2.1.0; OpenTelemetry CI/CD semantic conventions (https://opentelemetry.io/docs/specs/semconv/cicd/cicd-spans/); Hypothesis settings, `derandomize` and the `ci` profile (https://hypothesis.readthedocs.io/en/latest/_modules/hypothesis/_settings.html); Nelson Elhage, two kinds of testing (https://blog.nelhage.com/post/two-kinds-of-testing/); fast-check seed and path replay (https://fast-check.dev/docs/tutorials/quick-start/read-test-reports/); proptest failure persistence (https://proptest-rs.github.io/proptest/proptest/failure-persistence.html); cargo-mutants exit codes (https://mutants.rs/exit-codes.html). + +--- + +## Planning Contract + +### Key Technical Decisions + +**Record and catalog** + +- KTD1. **The record is a tagged union whose `_tag` is its catalog code.** Every variant carries `stage`, `cause`, `capsule`, `nextAction` and an optional `traceId`, plus evidence fields of its own. A `code` string beside a free-form evidence blob is the CONST-D2 violation this work removes. Lives in `packages/stryker-js-cli-contract/src/failure-record.schema.ts`. Governs R1, R2, R3. +- KTD2. **The catalog is a total map `Record`, generated to `packages/stryker-js-cli-contract/contract/failure-catalog.json`.** An entry holds meaning, exit class, next action and capsule rule, so the compiler refuses an undocumented code. `scripts/guards/check-contract-versions.ts` gains a `failure-catalog` kind classified like `stock-catalog.json`: a removed or renamed code demands a matching bump. Codes are PascalCase names such as `BaselineTestsFailed`; agents read names, and the guard gives them the stability rustc gets from numbers. Governs R2, R7. +- KTD3. **One renderer in the contract package writes every prose form** (terminal text, GitHub summary markdown, workflow annotations, SARIF notification text) from the record. No surface writes its own failure prose. Governs R6, R9, R12. +- KTD4. **`RunFailed` becomes `{schemaVersion: '3.0', code, record}`.** The `error`, `remediation` and `reason` fields are removed; plugin-load reasons become the evidence of `PluginLoadFailed`. `boundedErrorText` and its 1024-character limit in `packages/stryker-js/src/bin/main.ts` are deleted; the record is never truncated. Governs R3, R8. +- KTD5. **Exit codes derive from the record's catalog exit class, with one new class.** `BaselineFailed = 5` joins `ExitClass` and the `ExitCodeFromClass` code maps in `packages/stryker-js-plugin-interface/src/ExitClass.schema.ts`; `CatalogGap` maps to `InternalError = 4`, ending the generic fallback that shared code 1 with `VerdictFail`. When a cause holds several classes, precedence is `InternalError`, `RuntimeError`, `BaselineFailed`, `ConfigError`, `VerdictFail`, held as a total map over `ExitClass` so a future class cannot be skipped (today's `PRECEDENCE` array in `packages/stryker-js/src/conclude-run.ts` is not exhaustive). Publishing stays the single `RunExit` path (`docs/solutions/workflow-issues/exit-codes-through-runtime-teardown.md`). Governs R7, R8. +- KTD9. **`nextAction` is `{primary, otherwise}` over a closed set:** `fixTest`, `fixCode`, `killSurvivor`, `refactorAwayEquivalent`, `fixConfiguration`, `retryInfrastructure`, `reportToolDefect`. A survivor is `killSurvivor` otherwise `refactorAwayEquivalent`, because the tool cannot prove equivalence and the agent decides. A failing baseline test is `fixCode` otherwise `fixTest`; a vacuous or under-covered property is `fixTest`. Governs R5, R26. + +| Code | Stage | Exit | Next action (primary / otherwise) | Capsule | +| -------------------------- | ------------ | -------- | -------------------------------------- | --------------- | +| `ArgumentsInvalid` | cli | 2 | fixConfiguration | replays | +| `ConfigInvalid` | config | 2 | fixConfiguration | replays | +| `PluginLoadFailed` | config | 2 | fixConfiguration | replays | +| `PluginImportFailed` | config | 4 | reportToolDefect | replays | +| `SandboxPreparationFailed` | prepare | 3 | fixConfiguration / retryInfrastructure | replays | +| `InstrumentationFailed` | instrument | 3 | reportToolDefect | replays | +| `CheckerFailed` | check | 3 | fixConfiguration / reportToolDefect | replays | +| `BaselineTestsFailed` | dryRun | 5 | fixCode / fixTest | replays | +| `BaselineTimedOut` | dryRun | 5 | fixTest / retryInfrastructure | replays | +| `BaselineErrored` | dryRun | 5 | fixTest | replays | +| `BaselineFoundNoTests` | dryRun | 2 | fixConfiguration | replays | +| `WorkerOutOfMemory` | mutationTest | 3 | retryInfrastructure | does not replay | +| `WorkerCrashed` | mutationTest | 3 | retryInfrastructure / reportToolDefect | does not replay | +| `ReporterFailed` | report | 3 | fixConfiguration | replays | +| `RunInterrupted` | any | 130 | retryInfrastructure | does not replay | +| `SurvivingMutant` | gate | 1 | killSurvivor / refactorAwayEquivalent | replays | +| `CatalogGap` | any | 4 | reportToolDefect | replays | +| `RecordMissing` | ci | observed | reportToolDefect | does not replay | +| `JobTimedOut` | ci | observed | retryInfrastructure | does not replay | +| `BinaryMissing` | ci | observed | fixConfiguration | does not replay | + +The table is the starting catalog; U2 owns the final list. Property failures are evidence inside `BaselineTestsFailed`, not codes of their own. + +**Sources, evidence and capsule** + +- KTD6. **Failure sources raise typed errors carrying evidence, and `StageError{reason: string}` is removed.** A pure `classify-failure.workflow.ts` maps the run's `Cause` to a record with `Match.tag`; anything unmatched becomes `CatalogGap` with the full cause chain. It replaces the prototype traversal and `Cause.pretty` in `packages/stryker-js/src/conclude-run.ts`, and `failureTextOf`, `'Unknown failure'` and `DEFAULT_REMEDIATION` in `packages/stryker-js/src/reporting/run-failure.ts`. Out-of-memory and crash errors keep `{pid, exitCode, workerKind}` instead of the flattening in `mapMutationTestCause`. Governs R1, R3, R7. +- KTD7. **The runner reads Vitest's serialized error, not only its `message`.** `packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts` keeps `stack`, `stacks` (file, line, column), `name` and `cause`; the first frame in project source becomes `location`. The failed `TestResult` gains `location` and `stack`, replacing `startPosition`, which no producer sets. Property evidence is decoded by Schema from the serialized error's own fields once #576 adds them, never parsed from message text. Governs R3, R20. +- KTD8. **The test-runner plugin supplies the capsule's command; Stryker adds the lane environment.** The plugin interface gains an optional `reproduce(failedTest)` capability returning argv, because only the runner knows its CLI. Stryker adds `cwd`, the `STRYKER_WORKER_DIR` marker that selects the mutation-worker property budget, and, for property failures, the replay variable #576 defines. Without a runner capsule the command is `stryker run --dryRunOnly` with the run's configuration. Non-replayable codes carry `DoesNotReplay{why, standIn}`. Environment is a fixed-field array of `{name, value}` (`docs/solutions/test-failures/stream-schema-must-not-carry-json-rest-records.md`). Governs R4. +- KTD10. **The record is written to `reports/mutation/failure.json` and registered in `strykerOutputFilesOf`** (`packages/stryker-js/src/stryker-outputs.ts`), like `reproducers.json`. It is the durable copy MCP, CI and SARIF read, and stays out of verdict-cache digests. Governs R11, R14. +- KTD11. **Deno scripts import the record module from source.** `failure-record.schema.ts`, `failure-catalog.ts` and `render-failure.ts` import only `effect` and each other, so `scripts/` reaches them by relative path through the `effect` mapping in `scripts/deno.json`. CI-level failures become records of the same catalog, built by the script. If Deno rejects the `.js` relative specifiers, the fix is Deno configuration, never a second copy of the schema. Governs R1, R9. + +**Surfaces** + +- KTD22. **OTel spans carry `stryker.failure.code`, and the record carries `traceId` read from the current span at conclusion.** The truncated `stryker.run.error` attribute leaves `packages/stryker-js-cli-contract/src/SpanTaxonomy.ts`, which the span-taxonomy guard classifies as breaking. Governs R13. +- KTD23. **SARIF carries run failures as `runs[0].invocations[0]`** with `executionSuccessful: false` and one `toolExecutionNotifications` entry per record (`descriptor.id` is the code; the location comes from test evidence). The conclusion path writes it when the `sarif` reporter is configured, because a failed run never emits the `mutationTestReportReady` event the reporter listens to. Governs R10. +- KTD24. **The Mutation Server Protocol puts the record in `error.data`, and MCP returns it.** `packages/stryker-js/src/Serve/Serve.cell.ts` fills the existing optional `JsonRpcErrorObject.data`. `packages/stryker-js/src/Mcp/mcp-server.cell.ts` replaces `Effect.orDie` in `runRestricted` with a typed refusal carrying the record, and MCP gains a `get_failure` tool that reads `reports/mutation/failure.json`. Governs R11. + +**Property lanes** + +- KTD12. **Every property gets a seed derived from its identity in every per-commit lane, provided by #576.** `packages/toolchain/vitest-config/lib/property-runs.js` stops passing `seed: 1` and requests derandomized runs. Budgets still differ (100 runs normally, 30 in mutation workers); under one seed the worker draws are a prefix of the CI draws, and the PR preflight (KTD16) runs the 30-run budget before merge. Governs R16. +- KTD13. **Regression seeds live in `packages/toolchain/vitest-config/property-seeds.json`**, keyed by the property identity #576 defines and carried through the existing `@systemfsoftware/vitest:property-check` provide key (`packages/toolchain/vitest-config/lib/base.js:201`). The store's format is #576's. Governs R17. +- KTD14. **Coverage classes use the fork's existing `cover` option.** `cover` already exists in `@systemfsoftware/vitest` 1.0.0, so properties can declare classes before #576; #576 only makes the under-coverage failure structured. Governs R18, R21. +- KTD15. **Hit-limit reasons are classified by decoding `HitLimitReason`.** The check on `HitLimitReasonText` becomes the full `Hit limit reached (count/limit)` pattern, `timeoutKindIn` in `packages/stryker-js/src/mutation-reporting.service.ts` decodes instead of prefix-matching, and the runner test pins the full shape. Governs R22. +- KTD21. **The sweep runs daily on `main` at 1000 runs per property under random seeds**, in one job capped at 60 minutes, publishing one record per broken property with its seed. 1000 runs is the budget per-commit CI dropped for cost in PR #135; a nightly job is where Hypothesis places randomized exploration. Governs R17, R27. + +**Lanes** + +- KTD16. **The PR lane runs `stryker run --dryRunOnly` as its preflight, then an incremental run that restores `main`'s `.stryker-incremental` cache read-only.** The dry run already executes in a spawned worker with `STRYKER_WORKER_DIR` set (`packages/stryker-js/src/drivers/node.ts:63`), so the preflight runs under mutation-worker conditions by construction. "Evaluated none" is read from the stream's `reuse` line (`ran === 0`). Governs R23, R24. +- KTD17. **A new survivor is a survivor among the mutants the PR ran that is absent from `main`'s survivor baseline.** After merging its report, `main`'s lane writes that baseline with `stryker gate --baseline --update-baseline` and uploads it as the `mutation-survivor-baseline` artifact; the PR restores the latest one from `main` and runs `stryker gate --baseline` against it, never passing `--update-baseline`. The gate's refusal emits `SurvivingMutant` records, and its remediation line in `packages/stryker-js/src/run-request.cell.ts:347-348`, which tells the reader to accept survivors with `--update-baseline`, is replaced by the record's next action. The flag itself stays for the maintainer. Governs R25, R26. +- KTD18. **`main` runs the dry run once.** A preflight job runs `--dryRunOnly` and publishes its dry-run coverage; shards restore it through `incrementalSources`, so `dryRunReuse` returns `DryRunCoverageReused`. This relies on the run-inputs and closure digests being the same for every shard, which is inferred from `scopeOptionKeys` in `packages/stryker-js/src/verdict-semantics.ts` and proved in U17 before the workflow change lands. Governs R28. +- KTD19. **U17-U19 edit `.github/workflows/` directly; no other unit does.** `AGENTS.md` lists the directory as read-only for agents; the user lifted that for this plan's lane units, so the PR carries the full YAML diff for review instead of a patch for someone else to apply. The edits stay limited to the lane changes R10, R13, R14, R17 and R23-R28 name. Governs R10, R13, R14, R17, R23-R28. (session-settled: user-directed — chosen over maintainer-applied patches in the PR body: the agent finishes the lanes end to end and the workflow diff is reviewed in the PR.) +- KTD20. **A failed run's shard artifacts and record are kept 30 days, and the report job deletes per-shard artifacts only when every shard succeeded.** Today shard parts keep 1 day and are deleted after the merge. 30 days matches the timings record. Governs R14. + +### High-Level Technical Design + +Directional, not an implementation specification. + +```mermaid +flowchart TB + subgraph Sources + T[Vitest runner: serialized error, stacks, property fields] --> E[Typed failure errors with evidence] + W[Worker exit: pid, exit code] --> E + C[Config, plugin, sandbox, instrument, checker, reporter] --> E + end + E --> K[classify-failure workflow] + K --> R[FailureRecord: code, stage, evidence, cause, capsule, nextAction, traceId] + P[Runner reproduce capability plus lane env] --> R + R --> X[Exit code from catalog exit class] + R --> S[Stream terminal RunFailed v3.0] + R --> F[reports/mutation/failure.json] + R --> O[OTel span attribute and traceId] + F --> M[MCP get_failure and MSP error.data] + F --> Q[SARIF invocation notifications] + F --> G[stryker annotate and CI scripts] + R --> H[Terminal text via render-failure] +``` + +```ts +// Directional shape of failure-record.schema.ts +const Capsule = S.Union([ + S.TaggedStruct('Replays', { cwd: S.String, argv: S.NonEmptyArray(S.String), env: S.Array(EnvEntry) }), + S.TaggedStruct('DoesNotReplay', { why: NonReplayReason, standIn: S.String }), +]) +const Shared = { + stage: Stage, + cause: S.Array(CauseNode), + capsule: Capsule, + nextAction: NextAction, + traceId: S.optional(TraceId), +} +export const FailureRecord = S.Union([ + S.TaggedStruct('BaselineTestsFailed', { ...Shared, tests: S.NonEmptyArray(FailedTestEvidence) }), + S.TaggedStruct('WorkerOutOfMemory', { ...Shared, pid: S.Int, exitCode: S.Int, workerKind: WorkerKind }), + S.TaggedStruct('CatalogGap', { ...Shared }), + // one variant per catalog code +]) +``` + +### Sequencing + +Stack layers, bottom first. Each layer passes the Definition of Done on its own. + +```mermaid +flowchart TB + L1[L1 U1 incident property and hit-limit shape] --> L2[L2 U16 coverage classes and model audit] + L1 --> L3[L3 U2 U3 U4 U5 U6 record cutover] + L3 --> L4[L4 U7 capsule] + L4 --> L5[L5 U8 U9 U10 surfaces] + L4 --> L6[L6 U11 U13 noise and gate] + L5 --> L7[L7 U12 CI scripts] + L6 --> L7 + R576[Upstream issue 576 release] --> L8[L8 U14 U15 property evidence and seeds] + L7 --> L8 + L7 --> L9[L9 U17 U18 U19 lanes and workflows] + L8 --> L9 +``` + +- L1 lands first because `main`'s mutation lane stays red until the incident property is sound. +- U2-U6 share one layer: changing `RunFailed` breaks every producer and consumer, and BREAK-1 moves them together. +- The dogfood lane runs the published CLI (`AGENTS.md` START-6), so CLI-side surface changes reach `main`'s mutation lane only after a release. In-PR proof of CLI behavior comes from integration tests and the e2e lane. + +### Assumptions Under Challenge + +- **Vitest keeps a thrown error's own fields across the worker boundary.** KTD7 and U14 depend on it. U5 checks it against a real Vitest run before U14 builds on it; if the fields are dropped, #576 must put them in a channel Vitest keeps, and U14 waits. +- **Shards accept one preflight's dry-run coverage.** KTD18 depends on it. U17 proves it with two shards reusing one coverage file; if a digest differs per shard, U17 fixes the digest inputs or R28 is reported unmet. +- **The PR mutation set stays small.** KTD16 runs the PR's mutants unsharded. A change to a widely imported module invalidates many verdicts; U18 measures the worst recent PR and shards the PR run if it exceeds the 15-minute target set by `TARGET_SECONDS` in `.github/workflows/mutation.yml`. + +### System-Wide Impact + +- Contract documents `stream.schema.json`, `span-taxonomy.json` and the new `failure-catalog.json` change, and the contract-version guard demands bumps. Changesets: `@systemfsoftware/stryker-js-cli-contract` minor while 0.x; `@systemfsoftware/stryker-js`, `@systemfsoftware/stryker-js-plugin-interface` and `@systemfsoftware/stryker-js-vitest-runner` major (BREAK-1). +- api-extractor reports under `packages/*/etc/*.api.md` change; regenerate with `api:update` and export every type a public signature reaches (`docs/solutions/build-errors/api-check-alias-rotation-not-stable-across-invocations.md`). +- Consumers of `RunFailed` that move in U6: `packages/stryker-js/src/run-event-stream.service.ts`, `packages/stryker-js/src/frame-run-event.workflow.ts`, `packages/stryker-js/src/plugin-load-report.service.ts`, `test/e2e/tests/failing-run.e2e.test.ts`, `test/e2e/tests/__fixtures__/machine-stream.fixture.ts`, `packages/stryker-js-cli-contract/src/run-event-wire.schema.ts`, and `scripts/lib/mutation-plan.ts`. +- Plugin worker bundles must keep their `deps.onlyImport` closure (`AGENTS.md` PLUG-1) when the runner adds the `reproduce` capability. + +## Implementation Units + +| U-ID | Title | Key files | Depends on | +| ---- | ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| U1 | Incident property and canonical hit-limit shape | `packages/stryker-js/src/mutation-reporting.service.ts`, `packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts` | none | +| U2 | Failure record schema and catalog | `packages/stryker-js-cli-contract/src/failure-record.schema.ts`, `failure-catalog.ts` | U1 | +| U3 | One failure renderer | `packages/stryker-js-cli-contract/src/render-failure.ts` | U2 | +| U4 | Typed failure sources | `packages/stryker-js/src/Run.schema.ts`, `run/dry-run.cell.ts`, `run/mutation-test.cell.ts` | U2 | +| U5 | Runner failure evidence | `packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts`, `packages/stryker-js-plugin-interface/src/TestRunner.schema.ts` | U2 | +| U6 | Classification, exit codes, stream cutover | `packages/stryker-js/src/classify-failure.workflow.ts`, `conclude-run.ts`, `run-event-stream.service.ts` | U3, U4, U5 | +| U7 | Reproduction capsule | `packages/stryker-js/src/reproduction-capsule.workflow.ts`, plugin interface | U6 | +| U8 | OTel failure code and trace id | `packages/stryker-js-cli-contract/src/SpanTaxonomy.ts`, `packages/stryker-js/src/bin/main.ts` | U6 | +| U9 | MCP and Mutation Server Protocol | `packages/stryker-js/src/Mcp/`, `packages/stryker-js/src/Serve/` | U7 | +| U10 | SARIF run-failure notifications | `packages/stryker-js/src/sarif-report.workflow.ts` | U7 | +| U11 | Warnings and log noise | `packages/stryker-js/src/run/validate-options-admission.workflow.ts` | U6 | +| U12 | CI scripts render records | `scripts/lib/mutation-plan.ts`, `scripts/mutation-job.ts`, `packages/stryker-js/src/render-annotations.workflow.ts` | U7, U13 | +| U13 | Survivor records and gate remediation | `packages/stryker-js/src/gate-new-survivors.workflow.ts`, `run-request.cell.ts` | U7 | +| U14 | Structured property evidence from #576 | `packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts`, `pnpm-workspace.yaml` | U7, #576 release | +| U15 | Identity seeds and regression seed store | `packages/toolchain/vitest-config/lib/property-runs.js`, `property-seeds.json` | U14 | +| U16 | Coverage classes and model audit | property tests across `packages/` | U1 | +| U17 | Main lane: one dry run, traces, retention, SARIF | `scripts/mutation-job.ts`, `.github/workflows/mutation.yml` | U12 | +| U18 | PR mutation lane | `scripts/mutation-job.ts`, `.github/workflows/mutation.yml` | U12, U13, U17 | +| U19 | Scheduled property sweep | `packages/toolchain/vitest-config/lib/property-runs.js`, `.github/workflows/property-sweep.yml` | U15 | + +Test placement follows `docs/adr/0001-cell-architecture-module-taxonomy.md` and `skill://test-layer-selection`: workflow decisions get colocated `*.workflow.property.test.ts`, schemas state refusals in their own module, cells and handlers get Gherkin integration tests, and e2e stays at the existing capped journeys. Tests that would assert wiring, forwarding or non-emptiness are refused. + +### Phase A: the record (this repository only) + +### U1. Incident property and canonical hit-limit shape + +**Goal:** the timeout-fields property draws the inputs that decide its output, so `main`'s mutation lane passes under `{runs: 30, seed: 1}`. + +**Requirements:** R21, R22; AE1 for the incident property. + +**Dependencies:** none. + +**Files:** + +- `packages/stryker-js/src/mutation-reporting.service.ts` (property at `:1043`, `timeoutKindIn` at `:349-358`, test model at `:1004-1040`) +- `packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts` +- `packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts` +- `.changeset/*.md` + +**Approach:** + +1. Draw `status` with `Timeout` as its own weighted branch, `statusReason` from the wall-clock literal, encoded `HitLimitReason` text, arbitrary text and `undefined`, and `evidence` as `UndefinedOr(TimeoutEvidenceSchema)`. +2. Declare `cover` classes for the deciding combinations (KTD14): `Timeout` with a wall-clock reason, with a hit-limit reason, and with evidence only. +3. Apply KTD15 to the schema, `timeoutKindIn` and the runner test; the test model compares against the decoded `HitLimitReason`, not a regex. + +**Patterns to follow:** `packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts` `textOf` for the canonical encoding. + +**Test scenarios:** + +- Covers AE1. Under `STRYKER_WORKER_DIR` set, `vitest run src/mutation-reporting.service.ts` passes; with the generator reverted it fails naming the uncovered `Timeout` class. +- A reason `Hit limit reached (3/2)` classifies as `hitLimit`; `Hit limit reached` with no counts, or with trailing text, falls back to the evidence kind. +- The runner property pins the full `(count/limit)` shape for every hit-limit outcome it builds. + +**Verification:** the incident reproduction passes for seeds 1-10 at 30 runs; the full `stryker run --mutate src/mutation-reporting.service.ts` dry run in `packages/stryker-js` completes. + +### U16. Coverage classes and model audit + +**Goal:** every property in this repository declares the input classes that decide it, and no test model reimplements a production rule. + +**Requirements:** R18, R21. + +**Dependencies:** U1. Final R21 sign-off also needs U14 and U15. + +**Files:** property tests across `packages/` flagged by the audit, starting with `packages/stryker-js-instrumenter/src/__tests__/mutant-set-policy.workflow.property.test.ts` and `packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts`. + +**Approach:** + +1. Run every property file under seeds 1-20 at 30 runs with `STRYKER_WORKER_DIR` set and list the ones that fail or draw a deciding class fewer than once. +2. Declare `cover` classes on each flagged property (KTD14) and fix its generator. +3. Replace models that restate production rules with independent oracles (CONST-T10). + +**Execution note:** the seed sweep is a throwaway script in a gitignored scratch directory, deleted in the landing commit. + +**Test scenarios:** each flagged property gains its `cover` declaration; no new test files. + +**Verification:** the seed sweep reports zero failures and zero undrawn classes. + +### U2. Failure record schema and catalog + +**Goal:** a closed, documented record type exists in the contract package and is published as a contract document. + +**Requirements:** R1, R2, R3, R4, R5, R7. + +**Dependencies:** U1. + +**Files:** + +- `packages/stryker-js-cli-contract/src/failure-record.schema.ts` (new) +- `packages/stryker-js-cli-contract/src/failure-catalog.ts` (new) +- `packages/stryker-js-cli-contract/src/mod.ts` +- `packages/stryker-js-cli-contract/scripts/contract-documents.ts` +- `packages/stryker-js-cli-contract/contract/failure-catalog.json` (generated) +- `packages/stryker-js-cli-contract/tests/contract-documents.integration.test.ts` +- `scripts/guards/check-contract-versions.ts`, `scripts/lib/contract-compat.ts`, `scripts/lib/contract-compat.test.ts` +- `packages/stryker-js-plugin-interface/src/ExitClass.schema.ts` + +**Approach:** + +1. Write the union per KTD1 and the catalog per KTD2 and KTD9, importing only `effect` (KTD11). Exit classes are a local literal set mirrored by `ExitClass`, which gains `BaselineFailed` (KTD5). +2. Generate `failure-catalog.json` through the existing contract-documents generator. +3. Add the `failure-catalog` kind to the guard, reusing `compareCatalog`. + +**Patterns to follow:** `packages/stryker-js-cli-contract/src/StockCatalog.ts` and `contract/stock-catalog.json`; `plugin-load-failure-reason.schema.ts` for tagged evidence. + +**Test scenarios:** + +- Removing a catalog code from `failure-catalog.json` is classified incompatible by `contract-compat`; adding one is compatible. +- The record schema refuses a variant with an empty `argv`, an `env` entry with extra keys, and an unknown `_tag`. +- The regenerated catalog document equals the committed bytes (existing contract-documents integration test, extended to the new document). + +**Verification:** `pnpm guard:projects` passes with the new document and fails on a hand-deleted code. + +### U3. One failure renderer + +**Goal:** every prose form of a record comes from one pure module. + +**Requirements:** R6, R9, R12. + +**Dependencies:** U2. + +**Files:** + +- `packages/stryker-js-cli-contract/src/render-failure.ts` (new, in-source property block) + +**Approach:** export `terminalTextOf`, `markdownOf`, `annotationsOf` and `sarifTextOf` (KTD3). Annotations emit `::error file=,line=,col=` for evidence with a location. Values render structurally; nothing calls `String()` on an object. + +**Patterns to follow:** `packages/stryker-js/src/render-annotations.workflow.ts` for the workflow-command grammar and escaping. + +**Test scenarios:** + +- For any record, no rendering contains `[object Object]`, `Unknown failure`, a `~`-prefixed tag or a `_tag` key. +- For any record whose evidence has a test location, `annotationsOf` emits exactly one `::error` per located test, with file and line equal to the evidence. +- Rendering escapes `%`, CR and LF in messages so one record is never split across workflow commands. + +**Verification:** properties pass under the worker budget and the default budget. + +### U4. Typed failure sources + +**Goal:** every failure source raises a tagged error that carries its evidence. + +**Requirements:** R1, R3. + +**Dependencies:** U2. + +**Files:** + +- `packages/stryker-js/src/Run.schema.ts` +- `packages/stryker-js/src/dry-run.workflow.ts`, `packages/stryker-js/src/interpret-dry-run-observation.workflow.ts` +- `packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts` +- `packages/stryker-js/src/run/dry-run.cell.ts`, `packages/stryker-js/src/run/mutation-test.cell.ts` +- `packages/stryker-js/src/Checker/checker-pool.handle.ts`, `packages/stryker-js/src/drivers/node.ts` + +**Approach:** + +1. Replace `StageError` with one tagged error per source (KTD6); each declares its catalog code and evidence fields. +2. `failedTestSummariesOf` keeps test id, file, location, message and stack instead of `{name, failureMessage}`. +3. `mapMutationTestCause` passes out-of-memory and crash errors through with `{pid, exitCode, workerKind}`. + +**Patterns to follow:** `packages/stryker-js/src/plugin-load-report.service.ts` for typed reasons; channel-E refusals (pack: cell-architecture, four-channel-contracts.md). + +**Test scenarios:** + +- A dry-run observation with failed tests yields a decision carrying each test's id, file and location unchanged (replaces the lossy `{name, failureMessage}` assertion). +- A worker exit of 134 or 137 surfaces as the out-of-memory error with the same pid and exit code; any other non-zero exit surfaces as a crash. + +**Verification:** `StageError` has no remaining references (`git grep -n StageError` exits 1). + +### U5. Runner failure evidence + +**Goal:** the Vitest runner reports where a test failed and why, not only the first message. + +**Requirements:** R3. + +**Dependencies:** U2. + +**Files:** + +- `packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts` +- `packages/stryker-js-vitest-runner/src/vitest-test-run.ts` (in-source properties) +- `packages/stryker-js-vitest-runner/src/VitestRunner.service.ts` +- `packages/stryker-js-plugin-interface/src/TestRunner.schema.ts` + +**Approach:** apply KTD7. File-level errors (`fileFailureMessages`) gain the same fields. + +**Execution note:** before anything builds on it, confirm with a real Vitest run that an error's own enumerable fields reach the runner (Assumptions Under Challenge). + +**Test scenarios:** + +- A test record whose first project-source stack frame is `src/a.ts:12:5` yields `location` `{file: 'src/a.ts', line: 12, column: 5}`; frames under `node_modules` are skipped. +- A failure with no project-source frame yields no `location` and keeps the stack. +- Replace `∀r_TestRecord_≡FailureMessageIffFailed` with the widened failure shape. + +**Verification:** a failing fixture test run through the runner reports its file and line. + +### U6. Classification, exit codes and stream cutover + +**Goal:** a failed run ends with one record on the stream, on disk, in the exit code and on the terminal. + +**Requirements:** R7, R8, R12, R14; AE7. + +**Dependencies:** U3, U4, U5. + +**Files:** + +- `packages/stryker-js/src/classify-failure.workflow.ts` (new) and `packages/stryker-js/src/__tests__/classify-failure.workflow.property.test.ts` (new) +- `packages/stryker-js/src/conclude-run.ts`, `packages/stryker-js/src/conclude-run.cell.ts`, `packages/stryker-js/src/classify-run-outcome.workflow.ts` +- `packages/stryker-js/src/reporting/run-failure.ts`, `packages/stryker-js/src/reporting/run-failure.schema.ts` +- `packages/stryker-js/src/run-event-stream.service.ts`, `packages/stryker-js/src/frame-run-event.workflow.ts`, `packages/stryker-js/src/bin/main.ts`, `packages/stryker-js/src/stryker-outputs.ts` +- `packages/stryker-js-cli-contract/src/run-event.schema.ts`, `stream-version.schema.ts`, `contract/stream.schema.json` +- `test/e2e/tests/failing-run.e2e.test.ts`, `test/e2e/tests/__fixtures__/machine-stream.fixture.ts` + +**Approach:** + +1. Add the classification workflow (KTD6) and derive the exit code from the catalog (KTD5). +2. Cut `RunFailed` over (KTD4); write `reports/mutation/failure.json` (KTD10). +3. Render the terminal line through U3 and read `MachineConsole` once, removing the second read at conclusion. +4. Delete `failureTextOf`, `capturedOrUnknown`, `DEFAULT_REMEDIATION` and `boundedErrorText`. +5. A run that ends without failure deletes any `reports/mutation/failure.json` left by an earlier run, so `stryker annotate` and MCP never read a stale record. + +**Patterns to follow:** `packages/stryker-js/src/classify-run-outcome.workflow.ts` for workflow shape; exit-code invariants in `docs/solutions/workflow-issues/exit-codes-through-runtime-teardown.md`. + +**Test scenarios:** + +- Covers AE7. A cause containing an error no variant matches classifies as `CatalogGap` whose cause chain holds every message in the chain, exit 4. +- For every catalog code, the decided exit code equals the catalog's exit class code; `BaselineTestsFailed` is 5 and `VerdictFail` stays 1. +- When a cause holds several typed errors, the record takes the one the existing severity precedence picks (replaces `HighestExitClass` in `conclude-run.ts`). +- e2e `failing-run`: exit 5, terminal event `error` with `schemaVersion` `3.0`, record `BaselineTestsFailed` naming the failing test with its file and line. + +**Verification:** a local `stryker run` on a fixture with a failing test prints the rendered record, exits 5 and leaves `reports/mutation/failure.json` decodable by `FailureRecord`. + +### U7. Reproduction capsule + +**Goal:** every record carries a one-step reproduction, or says it has none and why. + +**Requirements:** R4; AE3. + +**Dependencies:** U6. + +**Files:** + +- `packages/stryker-js/src/reproduction-capsule.workflow.ts` (new) and its property test +- `packages/stryker-js-plugin-interface/src/TestRunner.schema.ts` (optional `reproduce` capability) +- `packages/stryker-js-vitest-runner/src/VitestRunner.service.ts` + +**Approach:** apply KTD8. The Vitest runner answers `vitest run -t `; the workflow adds `cwd`, the worker marker and the run's configuration file. + +**Test scenarios:** + +- Covers AE3. A `WorkerOutOfMemory` record's capsule is `DoesNotReplay` with `why` out-of-memory and a stand-in naming the pid and exit code. +- A `BaselineTestsFailed` record whose runner supplies argv carries that argv with the `STRYKER_WORKER_DIR` marker in `env`. +- A record whose runner supplies no argv falls back to `stryker run --dryRunOnly` with the same configuration file. + +**Verification:** running the capsule from the U6 smoke fixture reproduces the same failing test under the same budget. + +### U8. OTel failure code and trace id + +**Goal:** a mutation failure can be found in traces from its record and the other way round. + +**Requirements:** R13 (CLI side). + +**Dependencies:** U6. + +**Files:** + +- `packages/stryker-js-cli-contract/src/SpanTaxonomy.ts`, `contract/span-taxonomy.json` +- `packages/stryker-js/src/bin/main.ts`, `packages/stryker-js/src/plan-run-conclusion.workflow.ts`, `packages/stryker-js/src/reporter-stream.service.ts` +- `packages/stryker-js/tests/trace-propagation.integration.test.ts` + +**Approach:** apply KTD22. + +**Test scenarios:** + +- With OTel enabled and a failing dry run, the run span carries `stryker.failure.code = BaselineTestsFailed` and the record's `traceId` equals the exported span's trace id. +- With OTel disabled the record has no `traceId`. + +**Verification:** the e2e telemetry export for the failing journey shows the code on the run span. + +### U9. MCP and Mutation Server Protocol carry the record + +**Goal:** agents on MCP or MSP receive the same record as the stream, and engine failures no longer kill the server. + +**Requirements:** R11. + +**Dependencies:** U7. + +**Files:** + +- `packages/stryker-js/src/Mcp/mcp-tools.ts`, `mcp-tools.schema.ts`, `mcp-server.cell.ts` +- `packages/stryker-js/src/Serve/Serve.cell.ts` +- `packages/stryker-js/tests/mcp-server.integration.test.ts`, `packages/stryker-js/tests/mutation-server.integration.test.ts` + +**Approach:** apply KTD24. + +**Test scenarios:** + +- An MSP `mutationTest` whose engine fails answers a JSON-RPC error whose `data` decodes as the record with the same code the stream would carry; the server keeps serving. +- An MCP `rerun_mutant` whose engine fails returns a refusal carrying the record instead of terminating the server. +- `get_failure` after a failed run returns the record from `reports/mutation/failure.json`; with no file it returns the no-failure answer. + +**Verification:** both integration suites pass with the new scenarios. + +### U10. SARIF run-failure notifications + +**Goal:** SARIF consumers see a failed run, not an empty results list. + +**Requirements:** R10 (CLI side). + +**Dependencies:** U7. + +**Files:** + +- `packages/stryker-js/src/sarif-report.workflow.ts`, `packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts` +- `packages/stryker-js/src/reporter-factories.ts` +- `packages/stryker-js/tests/sarif-reporter.integration.test.ts` + +**Approach:** apply KTD23. + +**Test scenarios:** + +- For any record, the SARIF log has one invocation with `executionSuccessful: false`, the run's exit code, and one notification whose `descriptor.id` is the code. +- A record with located tests gives the notification a physical location matching the evidence. + +**Verification:** a failed fixture run with the `sarif` reporter writes a log that validates against SARIF 2.1.0. + +### U11. Warnings and log noise + +**Goal:** a run's log holds only lines about that run. + +**Requirements:** R15; AE5. + +**Dependencies:** U6. + +**Files:** + +- `packages/stryker-js/src/run/validate-options-admission.workflow.ts`, `packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts` +- `packages/stryker-js/src/bin/main.ts` (process warning handling) + +**Approach:** append `POSSIBLE_CAUSES_OF` only when there is at least one unknown option (`:192-203`). The `ExperimentalWarning: WASI` line comes from the parser the bin bundles on purpose (`packages/stryker-js/tsdown.bin.config.ts` aliases `oxc-parser` to `@oxc-parser/binding-wasm32-wasi`), so the bin entry drops exactly that one warning and leaves every other process warning visible. + +**Test scenarios:** + +- Covers AE5. Options with no excess names yield no warning; options with one excess name yield a warning naming it plus the possible causes. + +**Verification:** a clean `stryker run` log has no "Possible causes" and no `ExperimentalWarning` line. + +### U13. Survivor records and gate remediation + +**Goal:** a gate refusal is a record per new survivor whose next action never suggests accepting it. + +**Requirements:** R25, R26; AE4. + +**Dependencies:** U7. + +**Files:** + +- `packages/stryker-js/src/gate-new-survivors.workflow.ts` and its property test +- `packages/stryker-js/src/run-request.cell.ts` +- `packages/stryker-js/tests/gate.integration.test.ts` + +**Approach:** apply KTD17 and KTD9. The capsule is `stryker run --mutant ` with the run's configuration. + +**Test scenarios:** + +- Covers AE4. Any new survivor yields a `SurvivingMutant` record with location, covering tests and next action `killSurvivor` otherwise `refactorAwayEquivalent`. +- No rendering of a gate refusal mentions `--update-baseline`. + +**Verification:** the gate integration suite passes and its refusal output is the rendered records. + +### U12. CI scripts render records + +**Goal:** the CI summary and annotations say what the record says, and a run without a record is reported as exactly that. + +**Requirements:** R1, R9, R24 (reporting); AE2 and AE6 reporting halves. + +**Dependencies:** U7, U13. + +**Files:** + +- `scripts/lib/mutation-plan.ts`, `scripts/lib/mutation-plan.test.ts` +- `scripts/mutation-job.ts` +- `packages/stryker-js/src/render-annotations.workflow.ts`, `packages/stryker-js/src/run-request.cell.ts` (`annotate` reads `failure.json`) + +**Approach:** + +1. `loadState` decodes the terminal `error` line and the `reuse` line with the contract schema (KTD11); `buildSummary` and `buildRequireError` render through U3. +2. The script builds `RecordMissing`, `JobTimedOut` and `BinaryMissing` records itself; the "infrastructure failure (missing binary, crashed run or timeout)" text is deleted. +3. A run whose `reuse` line says `ran === 0` reports "evaluated no mutants" as its outcome, not a pass. +4. `stryker annotate` also renders the failure record's annotations. + +**Patterns to follow:** existing `decodeJson` with Effect Schema in `scripts/lib/mutation-plan.ts`; `docs/solutions/workflow-issues/mutation-lane-green-while-every-job-failed.md`. + +**Test scenarios:** + +- Covers AE2 reporting. A stream ending in a `BaselineTestsFailed` record yields a summary with the code, the test at file:line and the capsule, and the word "infrastructure" appears nowhere. +- A run with neither report nor terminal record yields a `RecordMissing` summary naming the exit code. +- Covers AE6 reporting. A stream whose `reuse` line has `ran: 0` yields the evaluated-none outcome and a non-zero gate. + +**Verification:** `pnpm test:scripts` passes; `deno check` resolves the contract module from `scripts/`. + +### Phase B: property soundness (needs the #576 release) + +### U14. Structured property evidence from #576 + +**Goal:** property failures reach the record as data: property, verdict, seed, runs, witness and undrawn classes. + +**Requirements:** R3, R4, R20, R29. + +**Dependencies:** U7; the #576 release. + +**Files:** + +- `pnpm-workspace.yaml` (`@systemfsoftware/vitest` catalog pin) +- `packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts` +- `packages/stryker-js-cli-contract/src/failure-record.schema.ts` (`PropertyEvidence`) +- `packages/stryker-js/src/reproduction-capsule.workflow.ts` + +**Approach:** decode the released tagged errors by Schema (KTD7); add the replay variable to the capsule (KTD8). Every property-failure kind #576 exports maps to a `PropertyEvidence` variant. + +**Test scenarios:** + +- A serialized vacuous-property error decodes to evidence with the property names, seed, runs and frozen output rendered structurally. +- An error without property fields decodes to a plain test failure, never a partial property record. +- Covers AE1. The incident property with U1's generator reverted fails with a record naming the undrawn `Timeout` class, and its capsule reproduces it. + +**Verification:** the U1 revert experiment's record carries seed, runs and the undrawn class, and running its capsule fails the same way. + +### U15. Identity seeds and the regression seed store + +**Goal:** each property runs one seed of its own in every per-commit lane, after any committed regression seeds. + +**Requirements:** R16, R17. + +**Dependencies:** U14. + +**Files:** + +- `packages/toolchain/vitest-config/lib/property-runs.js`, `packages/toolchain/vitest-config/lib/base.js` +- `packages/toolchain/vitest-config/property-seeds.json` (new) +- `packages/toolchain/vitest-config/tests/in-source-schema-laws.integration.test.ts` + +**Approach:** apply KTD12 and KTD13; the generated schema laws use the same budget. + +**Test scenarios:** + +- A property run twice in a row, in a mutation worker and outside one, draws the same first 30 inputs. +- A seed added to the store is replayed before novel draws in both lanes. + +**Verification:** `git grep -n "seed: 1\|mutationWorkerSeed"` exits 1. + +### Phase C: lanes + +### U17. Main lane: one dry run, traces, retention, SARIF + +**Goal:** `main`'s mutation run dry-runs once, exports traces, keeps a failed run's evidence, and uploads SARIF. + +**Requirements:** R9, R10, R13, R14, R27, R28. + +**Dependencies:** U12. + +**Files:** + +- `scripts/mutation-job.ts` (preflight mode publishing dry-run coverage; shard mode consuming it) +- `scripts/lib/mutation-plan.ts`, `scripts/lib/mutation-plan.test.ts` +- `.github/workflows/mutation.yml` + +**Approach:** + +1. Preflight and shard reuse per KTD18. +2. Workflow: preflight job; OTel environment, `pnpm lgtm:up` and `test/e2e/scripts/export-traces.ts` per shard as in `.github/workflows/ci.yml`'s e2e job; retention and deletion per KTD20; SARIF upload; `stryker annotate` step; publish the survivor baseline artifact (KTD17). + +**Execution note:** prove shard reuse first: two local shard runs against one preflight's coverage must log `DryRunCoverageReused` and finish with the same verdicts as a cold run. + +**Test scenarios:** + +- A shard given a preflight coverage file whose digests match reuses it; a mismatched file is refused with its reason and the shard runs its own dry run. + +**Verification:** after merge, one `main` run shows one dry run, trace artifacts per shard, and failed-shard artifacts that outlive the report job. + +### U18. PR mutation lane + +**Goal:** every PR runs the preflight and the mutants its change invalidates, and new survivors block it. + +**Requirements:** R23, R24, R25, R26; AE2, AE4, AE6; F2, F3. + +**Dependencies:** U12, U13, U17. + +**Files:** + +- `scripts/mutation-job.ts` (PR mode: preflight, incremental run, gate against `main`'s baseline) +- `.github/workflows/mutation.yml` (new `pull_request` job) + +**Approach:** apply KTD16 and KTD17. The job's check name stays stable (`docs/solutions/workflow-issues/matrix-legs-rename-the-required-status-check.md`). Before the job lands, replay the scoped run on the recent PR with the largest mutate-scope change; if it exceeds the 15-minute `TARGET_SECONDS` target in `.github/workflows/mutation.yml`, the PR job shards its run through the existing `shardMutate` split. + +**Test scenarios:** + +- PR mode given a stream whose `reuse` line has `ran: 0` reports evaluated-none and does not pass. +- PR mode given a survivor absent from the baseline fails with its `SurvivingMutant` record; a survivor present in the baseline does not fail it. + +**Verification:** on this stack's own PR, the dependency-free layers show the preflight and the evaluated-none outcome; a throwaway PR that adds an unkilled branch is blocked with the survivor's record. + +### U19. Scheduled property sweep + +**Goal:** random exploration runs on `main` every day and turns any breaking seed into a committed regression seed. + +**Requirements:** R17, R27; F4. + +**Dependencies:** U15. + +**Files:** + +- `packages/toolchain/vitest-config/lib/property-runs.js` (sweep budget selected by an environment marker) +- `.github/workflows/property-sweep.yml` (new, scheduled and manually dispatchable) + +**Approach:** apply KTD21. A broken property publishes its record with the seed; the fix adds that seed to `property-seeds.json` (KTD13). + +**Test expectation:** none beyond U15's replay scenario; the sweep is configuration over the same budget mechanism. + +**Verification:** a manual dispatch of the sweep completes inside 60 minutes and publishes records for any failure. + +## Verification Contract + +| Gate | Command | Applies to | +| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | +| Format | `pnpm format:check` | every layer | +| Typecheck | `pnpm typecheck` | every layer | +| Tests | `pnpm test` | every layer | +| Build, api reports and guards | `pnpm check:ci` (includes `guard:projects`, `api:check`) | every layer | +| Scripts | `pnpm test:scripts` | U2, U12, U17, U18 | +| e2e | `pnpm test:e2e` | U6, U8 | +| Change intent | `./scripts/check-changeset.ts $(git merge-base HEAD origin/main)` | every layer touching `packages/` | +| Dogfood pin | `git grep -F 'catalog:stryker' -- packages/stryker-js/package.json packages/stryker-js-vitest-runner/package.json packages/stryker-js-typescript-checker/package.json` | every layer | +| Plugin bundles | `pnpm --filter @systemfsoftware/stryker-js-vitest-runner --filter @systemfsoftware/stryker-js-typescript-checker build` | U5, U7, U14 | + +Incident replay, run before declaring Phase A done: with U1's generator reverted on a scratch branch, a local `stryker run` in `packages/stryker-js` exits 5, prints the `BaselineTestsFailed` record naming `mutation-reporting.service.ts` with its line, writes `reports/mutation/failure.json`, and the record's capsule reproduces the failure. After U14, the same replay names seed, runs and the undrawn `Timeout` class. + +## Definition of Done + +- Every gate in the Verification Contract passes on each stack layer. +- No rendering of any record contains `[object Object]`, `Unknown failure` or an internal type tag, and `git grep -n -e StageError -e DEFAULT_REMEDIATION -e boundedErrorText -e 'infrastructure failure (missing binary'` exits 1. +- Every catalog code has meaning, exit class, next action and capsule rule, enforced by the total map in `failure-catalog.ts`. +- Each phase's Acceptance Examples hold: AE3, AE5 and AE7 in Phase A; AE1 in Phase B; AE2, AE4 and AE6 in Phase C. +- `.github/workflows/` changes appear only in U17-U19's layer. +- Abandoned approaches, scratch scripts and probes are deleted from the diff. diff --git a/packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts b/packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts index bfc14cad..67835983 100644 --- a/packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts +++ b/packages/stryker-js-plugin-interface/src/mutant-timeout-reason.schema.ts @@ -7,19 +7,10 @@ export const HitLimitReasonPrefix = S.Literal('Hit limit reached') export const WallClockTimeoutReason = S.Literal('wall-clock-timeout') -const HIT_LIMIT_REASON_PREFIX = HitLimitReasonPrefix.literal - -export const HitLimitReasonText = S.String.check( - S.isStartingWith(HIT_LIMIT_REASON_PREFIX, { - arbitraryConstraint: { - patterns: [{ source: `^${HIT_LIMIT_REASON_PREFIX}[\\s\\S]*$`, flags: '' }], - minLength: HIT_LIMIT_REASON_PREFIX.length + 2, - }, - }), -) - const HIT_LIMIT_REASON_SHAPE = /^Hit limit reached \((\d+)\/(\d+)\)$/ +export const HitLimitReasonText = S.String.check(S.isPattern(HIT_LIMIT_REASON_SHAPE)) + const limitsOf = SchemaGetter.transformEffect((text: string) => Option.match(Option.fromNullishOr(HIT_LIMIT_REASON_SHAPE.exec(text)), { onNone: () => Effect.fail(malformedHitLimit(text)), diff --git a/packages/stryker-js/src/mutation-reporting.service.ts b/packages/stryker-js/src/mutation-reporting.service.ts index c0f562d0..c36ab4e1 100644 --- a/packages/stryker-js/src/mutation-reporting.service.ts +++ b/packages/stryker-js/src/mutation-reporting.service.ts @@ -1001,51 +1001,78 @@ if (import.meta.vitest !== void 0) { (subject, [mutant, result]) => Effect.map(subject(mutant, result), (mapped) => carriesClassOutcome(result, mapped)), ) - const expectedTimeoutKind = ( - result: Parameters[0], - evidence: { readonly timeoutKind: TimeoutKind; readonly reproductions: number }, - ): TimeoutKind => - Match.value(result.statusReason).pipe( - Match.when('wall-clock-timeout', (): TimeoutKind => 'wallClock'), - Match.when( - (reason: string | undefined): boolean => - reason !== undefined && /^Hit limit reached \(\d+\/\d+\)$/.test(reason), - (): TimeoutKind => 'hitLimit', - ), - Match.orElse((): TimeoutKind => evidence.timeoutKind), - ) + const TimeoutStatus = S.Union([S.Literal('Timeout'), MutantStatusSchema]) + const TimeoutReason = S.UndefinedOr( + S.Union([TestRunner.WallClockTimeoutReason, TestRunner.HitLimitReasonText, S.String]), + ) + const isReason = (drawn: unknown): drawn is string | undefined => drawn === undefined || typeof drawn === 'string' + const isWallClockReason = (reason: string | undefined): boolean => + reason === TestRunner.WallClockTimeoutReason.literal + const isHitLimitReason = (reason: string | undefined): boolean => + Option.isSome(S.decodeUnknownOption(TestRunner.HitLimitReason)(reason)) + const isKindlessReason = (reason: string | undefined): boolean => + isWallClockReason(reason) === false && isHitLimitReason(reason) === false - const expectedReproductions = ( - timeoutKind: TimeoutKind, - evidence: { readonly timeoutKind: TimeoutKind }, - ): number => - Match.value(timeoutKind).pipe( - Match.when('wallClock', () => - Match.value(evidence.timeoutKind).pipe( - Match.when('wallClock', () => 1), - Match.orElse(() => 0), - )), - Match.orElse(() => 0), + const expectedTimeoutKind = ( + reason: string | undefined, + evidence: TimeoutEvidence | undefined, + ): TimeoutKind | undefined => + Match.value(reason).pipe( + Match.when(isWallClockReason, (): TimeoutKind | undefined => 'wallClock'), + Match.when(isHitLimitReason, (): TimeoutKind | undefined => 'hitLimit'), + Match.orElse((): TimeoutKind | undefined => evidence?.timeoutKind), ) const expectedTimeoutFields = ( - result: Parameters[0], - evidence: { readonly timeoutKind: TimeoutKind; readonly reproductions: number }, - ) => - Match.value(result.status === 'Timeout').pipe( - Match.when(true, () => ({ - timeoutKind: expectedTimeoutKind(result, evidence), - reproductions: expectedReproductions(expectedTimeoutKind(result, evidence), evidence), - })), - Match.orElse(() => ({ timeoutKind: undefined, reproductions: undefined })), + status: typeof TimeoutStatus.Type, + reason: string | undefined, + evidence: TimeoutEvidence | undefined, + ): { readonly timeoutKind?: TimeoutKind; readonly reproductions?: number } => + Option.match( + Option.filter(Option.fromUndefinedOr(expectedTimeoutKind(reason, evidence)), () => status === 'Timeout'), + { + onNone: () => ({}), + onSome: (timeoutKind) => ({ + timeoutKind, + reproductions: Number(holds([timeoutKind === 'wallClock', evidence?.timeoutKind === 'wallClock'])), + }), + }, ) it.prop( '∀mse_MutantStatusAndEvidence_≡PersistedTimeoutFieldsFollowTheReproductionRule', - { of: [Mutant, MutantStatusSchema, TimeoutEvidenceSchema], subject: timeoutFieldsOf }, - (subject, [mutant, status, evidence]) => { - const result: Parameters[0] = { ...mutant, status } - return JSON.stringify(subject(result, evidence)) === JSON.stringify(expectedTimeoutFields(result, evidence)) + { + of: [Mutant, TimeoutStatus, TimeoutReason, S.UndefinedOr(TimeoutEvidenceSchema)], + subject: timeoutFieldsOf, + cover: { + wallClockTimeout: [ + (_mutant, status, reason) => isReason(reason) && holds([status === 'Timeout', isWallClockReason(reason)]), + 0.05, + ], + hitLimitTimeout: [ + (_mutant, status, reason) => isReason(reason) && holds([status === 'Timeout', isHitLimitReason(reason)]), + 0.05, + ], + evidenceOnlyTimeout: [ + (_mutant, status, reason, evidence) => + isReason(reason) && holds([status === 'Timeout', isKindlessReason(reason), evidence !== undefined]), + 0.05, + ], + kindlessTimeout: [ + (_mutant, status, reason, evidence) => + isReason(reason) && holds([status === 'Timeout', isKindlessReason(reason), evidence === undefined]), + 0.02, + ], + }, + }, + (subject, [mutant, status, reason, evidence]) => { + const result: Parameters[0] = { + ...mutant, + status, + ...presentField('statusReason', reason), + } + return JSON.stringify(subject(result, evidence)) === + JSON.stringify(expectedTimeoutFields(status, reason, evidence)) }, ) } From 782fa8ec3653ab05dd7ec6359aa4241058311055 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:42 -0400 Subject: [PATCH 2/8] feat(repo): end every failed run in one typed failure record (#147) * feat(repo): end every failed run in one typed failure record A failed run now emits a FailureRecord (catalog code, stage, structured evidence, cause chain, next action, replay capsule, trace id) on stderr, as the schema-3.0 terminal stream event, and in reports/mutation/failure.json. StageError and PrepareError are replaced by RunFailure, exit codes come from the failure catalog, and failed baseline tests carry project-relative file, line and stack from the Vitest runner Verdict-Semantics: unchanged * chore(repo): guard the failure catalog as a versioned contract document Verdict-Semantics: unchanged * test(repo): assert the frozen-runner dry run through its failure record Trunk's frozen-runner scenario read the removed StageError stage and log capture; it now reads the RunFailure evidence stage Verdict-Semantics: unchanged * fix(repo): declare stryker.failure.code optional on the run span A successful run carries no failure code, so the e2e trace contract refused every passing run span with a missing key Verdict-Semantics: unchanged * refactor(repo): declare the optional span attribute schema in its schema module Verdict-Semantics: unchanged * test(repo): pin summary annotations to the record code and assert the failing-run record once The summary-annotation law accepted any single annotation, so a draw with no located evidence let a constant renderer pass and the fork refused the file as vacuous on macOS. The failing-run journey checked the terminal record in two consecutive steps, which the fork refuses as a second check on the same state Verdict-Semantics: unchanged * test(repo): show the run's failure record when a framework-run scenario expects success A CI rerun of the claimed-type scenario stopped after file discovery and the Then reported only that the incremental state was not JSON, so the failure that stopped the run was lost Verdict-Semantics: unchanged --- .changeset/green-kings-think.md | 5 + .changeset/moody-snails-watch.md | 5 + .changeset/small-results-greet.md | 5 + .changeset/upset-chicken-hammer.md | 5 + .../contract/failure-catalog.json | 288 + .../contract/span-taxonomy.json | 2 +- .../contract/stream.schema.json | 5847 ++++++++++++++++- .../etc/stryker-js-cli-contract.api.md | 5 + .../scripts/contract-documents.ts | 3 + .../scripts/generate-contract.ts | 8 +- .../src/FailureRecord/mod.ts | 23 + .../src/SpanTaxonomy.schema.ts | 2 + .../src/SpanTaxonomy.ts | 10 +- .../src/failure-catalog.schema.ts | 19 + .../src/failure-catalog.ts | 209 + .../src/failure-record.schema.ts | 151 + packages/stryker-js-cli-contract/src/mod.ts | 1 + .../src/render-failure.ts | 408 ++ .../src/run-event.schema.ts | 6 +- .../src/stream-version.schema.ts | 2 +- .../contract-documents.integration.test.ts | 18 + .../etc/stryker-js-plugin-interface.api.md | 2 +- .../src/TestRunner.schema.ts | 18 +- .../src/VitestRunner.service.ts | 35 +- .../src/drivers/vitest-node.ts | 72 +- .../src/vitest-run-command.schema.ts | 14 + .../src/vitest-test-run.ts | 215 +- packages/stryker-js/etc/stryker-js.api.md | 2 +- .../src/Checker/Checker.blueprint.ts | 1 + .../stryker-js/src/Checker/Checker.handle.ts | 2 +- .../src/Checker/checker-pool.blueprint.ts | 10 +- .../src/Checker/checker-pool.handle.ts | 59 +- packages/stryker-js/src/ConfigError.schema.ts | 29 +- packages/stryker-js/src/Engine/mod.ts | 2 +- .../stryker-js/src/PluginsError.schema.ts | 22 +- packages/stryker-js/src/Rerun/Rerun.cell.ts | 3 +- .../src/Rerun/admit-mutant-rerun.workflow.ts | 7 +- packages/stryker-js/src/Run.schema.ts | 44 +- .../src/RunOutcomeCommand.schema.ts | 50 +- .../stryker-js/src/TestRunner.blueprint.ts | 1 + packages/stryker-js/src/Worker.schema.ts | 42 +- .../stryker-js/src/WorkerLauncher.service.ts | 2 + ...it-mutation-test.workflow.property.test.ts | 8 +- .../checker-pool.workflow.property.test.ts | 46 +- ...sify-run-outcome.workflow.property.test.ts | 180 +- .../dry-run.workflow.property.test.ts | 19 +- ...-run-observation.workflow.property.test.ts | 27 +- ...n-run-conclusion.workflow.property.test.ts | 55 +- .../run-failure.workflow.property.test.ts | 41 - .../src/admit-mutation-test.workflow.ts | 24 +- .../src/admit-survivors-run.workflow.ts | 5 + packages/stryker-js/src/bin/main.ts | 67 +- .../src/classify-run-outcome.workflow.ts | 160 +- packages/stryker-js/src/conclude-run.cell.ts | 108 +- packages/stryker-js/src/conclude-run.ts | 489 +- packages/stryker-js/src/drivers/node.ts | 155 +- packages/stryker-js/src/dry-run.workflow.ts | 44 +- packages/stryker-js/src/events/mod.ts | 2 +- .../src/frame-run-event.workflow.ts | 34 +- .../src/gate-new-survivors.workflow.ts | 41 +- .../interpret-dry-run-observation.workflow.ts | 26 +- .../src/mutation-reporting.service.ts | 9 +- .../src/plan-run-conclusion.workflow.ts | 87 +- .../src/plugin-load-report.service.ts | 44 +- .../src/render-annotations.workflow.ts | 7 +- .../stryker-js/src/reporter-stream.service.ts | 1 + .../stryker-js/src/reporter-wiring.service.ts | 22 +- .../src/reporting/run-failure.schema.ts | 12 - .../stryker-js/src/reporting/run-failure.ts | 77 +- .../src/run-event-stream.service.ts | 76 +- packages/stryker-js/src/run-request.cell.ts | 8 +- packages/stryker-js/src/run/dry-run.cell.ts | 140 +- packages/stryker-js/src/run/host.service.ts | 4 +- .../src/run/incremental-reuse.cell.ts | 9 +- .../stryker-js/src/run/instrument.cell.ts | 32 +- .../stryker-js/src/run/mutant-run.cell.ts | 13 +- packages/stryker-js/src/run/mutant-run.ts | 4 +- .../src/run/mutation-test-plan.cell.ts | 24 +- .../stryker-js/src/run/mutation-test-plan.ts | 10 +- .../stryker-js/src/run/mutation-test.cell.ts | 39 +- packages/stryker-js/src/run/prepare.cell.ts | 81 +- .../stryker-js/src/run/run-stages.cell.ts | 20 +- packages/stryker-js/src/run/run-stages.ts | 6 +- .../src/run/test-runner-pool.blueprint.ts | 14 +- packages/stryker-js/src/stryker-outputs.ts | 2 + .../stryker-js/src/worker-client.blueprint.ts | 6 +- .../__fixtures__/dry-run-failure.schema.ts | 15 +- .../tests/__fixtures__/rerun-by-id.schema.ts | 2 - .../substituted-worker.fixture.ts | 5 +- .../tests/checker-rpc.integration.test.ts | 1 + .../tests/config-file.integration.test.ts | 6 +- .../tests/diff-scope.integration.test.ts | 20 +- .../tests/dry-run-failure.integration.test.ts | 95 +- .../empty-file-selection.integration.test.ts | 27 +- .../tests/framework-run.integration.test.ts | 177 +- .../framework-stream.integration.test.ts | 35 +- .../incremental-reuse.integration.test.ts | 2 +- .../instrument-stage.integration.test.ts | 26 +- ...mutant-location-parity.integration.test.ts | 2 +- ...lugin-mutator-provider.integration.test.ts | 23 +- .../tests/rerun-by-id.integration.test.ts | 26 +- .../run-event-stream.integration.test.ts | 26 +- ...porter-terminal-report.integration.test.ts | 2 +- .../tests/vm-parity.differential.test.ts | 6 +- .../wall-clock-timeout.integration.test.ts | 2 +- ...worker-connection-loss.integration.test.ts | 1 + .../tests/worker-launcher.integration.test.ts | 1 + scripts/guards/check-contract-versions.ts | 1 + .../typescript-checker.fixture.ts | 22 +- test/e2e/tests/failing-run.e2e.test.ts | 102 +- 110 files changed, 8765 insertions(+), 1792 deletions(-) create mode 100644 .changeset/green-kings-think.md create mode 100644 .changeset/moody-snails-watch.md create mode 100644 .changeset/small-results-greet.md create mode 100644 .changeset/upset-chicken-hammer.md create mode 100644 packages/stryker-js-cli-contract/contract/failure-catalog.json create mode 100644 packages/stryker-js-cli-contract/src/FailureRecord/mod.ts create mode 100644 packages/stryker-js-cli-contract/src/failure-catalog.schema.ts create mode 100644 packages/stryker-js-cli-contract/src/failure-catalog.ts create mode 100644 packages/stryker-js-cli-contract/src/failure-record.schema.ts create mode 100644 packages/stryker-js-cli-contract/src/render-failure.ts delete mode 100644 packages/stryker-js/src/__tests__/run-failure.workflow.property.test.ts diff --git a/.changeset/green-kings-think.md b/.changeset/green-kings-think.md new file mode 100644 index 00000000..bd5a6c66 --- /dev/null +++ b/.changeset/green-kings-think.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": major +--- + +Every failed run ends in one FailureRecord: printed on stderr, emitted as the terminal stream event, and written to reports/mutation/failure.json. StageError and PrepareError are replaced by RunFailure, exit codes come from the failure catalog (a failing baseline test exits 5), failed baseline tests are named by project-relative file and line, and the cli.run span carries stryker.failure.code. diff --git a/.changeset/moody-snails-watch.md b/.changeset/moody-snails-watch.md new file mode 100644 index 00000000..d53309ab --- /dev/null +++ b/.changeset/moody-snails-watch.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-cli-contract": minor +--- + +RunFailed now carries a typed FailureRecord (catalog code, stage, structured evidence, cause chain, next action, replay capsule, trace id) instead of error/remediation/reason prose; the stream schema is 3.0. The package publishes contract/failure-catalog.json and FailureRecord renderers for terminal, markdown, GitHub annotations and SARIF. diff --git a/.changeset/small-results-greet.md b/.changeset/small-results-greet.md new file mode 100644 index 00000000..3c8b9a1a --- /dev/null +++ b/.changeset/small-results-greet.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-plugin-interface": major +--- + +FailedTestResult carries an optional project-relative source location and the raw stack; the unused startPosition field is removed. diff --git a/.changeset/upset-chicken-hammer.md b/.changeset/upset-chicken-hammer.md new file mode 100644 index 00000000..b6c95a52 --- /dev/null +++ b/.changeset/upset-chicken-hammer.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-vitest-runner": minor +--- + +Failed tests report the first project source frame as their location and keep the raw stack; file-level failures are named and identified by their project-relative path. diff --git a/packages/stryker-js-cli-contract/contract/failure-catalog.json b/packages/stryker-js-cli-contract/contract/failure-catalog.json new file mode 100644 index 00000000..215cd0ed --- /dev/null +++ b/packages/stryker-js-cli-contract/contract/failure-catalog.json @@ -0,0 +1,288 @@ +[ + { + "id": "ArgumentsInvalid", + "name": "ArgumentsInvalid", + "meaning": "The command line named an option, argument or subcommand the CLI does not accept.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "ConfigInvalid", + "name": "ConfigInvalid", + "meaning": "The configuration could not be read or failed validation.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "PluginNotFound", + "name": "PluginNotFound", + "meaning": "The configuration names a plugin that is not installed or not registered.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "PluginLoadFailed", + "name": "PluginLoadFailed", + "meaning": "A configured plugin refused to load: a peer is missing or unsupported, or its contribution is invalid.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "PluginImportFailed", + "name": "PluginImportFailed", + "meaning": "A configured plugin module threw while it was being imported.", + "exitCode": 4, + "nextAction": { + "primary": "reportToolDefect", + "otherwise": "fixConfiguration" + }, + "capsule": "replays" + }, + { + "id": "SurvivorsUnavailable", + "name": "SurvivorsUnavailable", + "meaning": "Survivors were requested but no matching mutation report exists.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "NoInputFiles", + "name": "NoInputFiles", + "meaning": "The configured file patterns matched no input files.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "SandboxPreparationFailed", + "name": "SandboxPreparationFailed", + "meaning": "The project could not be read or copied into the sandbox.", + "exitCode": 3, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": "retryInfrastructure" + }, + "capsule": "replays" + }, + { + "id": "InstrumentationFailed", + "name": "InstrumentationFailed", + "meaning": "The instrumenter could not read or mutate a source file.", + "exitCode": 3, + "nextAction": { + "primary": "reportToolDefect", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "CheckerFailed", + "name": "CheckerFailed", + "meaning": "A checker failed or broke its protocol while checking mutants.", + "exitCode": 3, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": "reportToolDefect" + }, + "capsule": "replays" + }, + { + "id": "TestRunnerFailed", + "name": "TestRunnerFailed", + "meaning": "The test runner could not be started or failed outside of any test.", + "exitCode": 3, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": "reportToolDefect" + }, + "capsule": "replays" + }, + { + "id": "BaselineTestsFailed", + "name": "BaselineTestsFailed", + "meaning": "One or more tests failed in the initial test run, before any mutant was applied.", + "exitCode": 5, + "nextAction": { + "primary": "fixCode", + "otherwise": "fixTest" + }, + "capsule": "replays" + }, + { + "id": "BaselineTimedOut", + "name": "BaselineTimedOut", + "meaning": "The initial test run did not finish within its time limit.", + "exitCode": 5, + "nextAction": { + "primary": "fixTest", + "otherwise": "retryInfrastructure" + }, + "capsule": "replays" + }, + { + "id": "BaselineErrored", + "name": "BaselineErrored", + "meaning": "The initial test run errored outside of any test.", + "exitCode": 5, + "nextAction": { + "primary": "fixTest", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "BaselineFoundNoTests", + "name": "BaselineFoundNoTests", + "meaning": "The initial test run found no tests to run.", + "exitCode": 2, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "WorkerBootTimedOut", + "name": "WorkerBootTimedOut", + "meaning": "A worker process did not finish booting before its boot window closed.", + "exitCode": 3, + "nextAction": { + "primary": "retryInfrastructure", + "otherwise": "reportToolDefect" + }, + "capsule": "replays" + }, + { + "id": "WorkerOutOfMemory", + "name": "WorkerOutOfMemory", + "meaning": "A worker process was killed for running out of memory.", + "exitCode": 3, + "nextAction": { + "primary": "retryInfrastructure", + "otherwise": null + }, + "capsule": "outOfMemory" + }, + { + "id": "WorkerCrashed", + "name": "WorkerCrashed", + "meaning": "A worker process exited unexpectedly.", + "exitCode": 3, + "nextAction": { + "primary": "retryInfrastructure", + "otherwise": "reportToolDefect" + }, + "capsule": "workerCrashed" + }, + { + "id": "ReporterFailed", + "name": "ReporterFailed", + "meaning": "A reporter could not be started or failed while reporting.", + "exitCode": 3, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "RunInterrupted", + "name": "RunInterrupted", + "meaning": "The run was interrupted by a signal before it finished.", + "exitCode": 130, + "nextAction": { + "primary": "retryInfrastructure", + "otherwise": null + }, + "capsule": "interrupted" + }, + { + "id": "NewSurvivors", + "name": "NewSurvivors", + "meaning": "Mutants the change can affect survived every test and are absent from the accepted survivor baseline.", + "exitCode": 1, + "nextAction": { + "primary": "killSurvivor", + "otherwise": "refactorAwayEquivalent" + }, + "capsule": "replays" + }, + { + "id": "InvariantBroken", + "name": "InvariantBroken", + "meaning": "Stryker reached a state its own design rules out; this is a defect in Stryker.", + "exitCode": 4, + "nextAction": { + "primary": "reportToolDefect", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "CatalogGap", + "name": "CatalogGap", + "meaning": "A failure no catalog entry classifies; the record carries its full cause.", + "exitCode": 4, + "nextAction": { + "primary": "reportToolDefect", + "otherwise": null + }, + "capsule": "replays" + }, + { + "id": "RecordMissing", + "name": "RecordMissing", + "meaning": "A CI job ended without a failure record or a report.", + "exitCode": null, + "nextAction": { + "primary": "reportToolDefect", + "otherwise": null + }, + "capsule": "recordMissing" + }, + { + "id": "JobTimedOut", + "name": "JobTimedOut", + "meaning": "A CI job was killed for exceeding its time limit.", + "exitCode": null, + "nextAction": { + "primary": "retryInfrastructure", + "otherwise": null + }, + "capsule": "jobTimedOut" + }, + { + "id": "BinaryMissing", + "name": "BinaryMissing", + "meaning": "A CI job could not find a binary it needs.", + "exitCode": null, + "nextAction": { + "primary": "fixConfiguration", + "otherwise": null + }, + "capsule": "binaryMissing" + } +] diff --git a/packages/stryker-js-cli-contract/contract/span-taxonomy.json b/packages/stryker-js-cli-contract/contract/span-taxonomy.json index 98beda60..0044d809 100644 --- a/packages/stryker-js-cli-contract/contract/span-taxonomy.json +++ b/packages/stryker-js-cli-contract/contract/span-taxonomy.json @@ -63,7 +63,7 @@ "stryker.run.exit_code": { "type": "number" }, - "stryker.run.error": { + "stryker.failure.code": { "type": "string" } } diff --git a/packages/stryker-js-cli-contract/contract/stream.schema.json b/packages/stryker-js-cli-contract/contract/stream.schema.json index c204b747..204b07a3 100644 --- a/packages/stryker-js-cli-contract/contract/stream.schema.json +++ b/packages/stryker-js-cli-contract/contract/stream.schema.json @@ -57,7 +57,7 @@ "schemaVersion": { "type": "string", "enum": [ - "2.0" + "3.0" ] }, "runId": { @@ -724,7 +724,7 @@ "schemaVersion": { "type": "string", "enum": [ - "2.0" + "3.0" ] }, "runId": { @@ -1006,7 +1006,7 @@ "schemaVersion": { "type": "string", "enum": [ - "2.0" + "3.0" ] }, "code": { @@ -1014,117 +1014,5856 @@ "minimum": 0, "maximum": 255 }, - "error": { - "type": "string" - }, - "remediation": { - "type": "string" - }, - "reason": { + "record": { "anyOf": [ { - "anyOf": [ - { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "ArgumentsInvalid" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "argument": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "argument" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "ConfigInvalid" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "detail": { + "type": "string" + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "detail" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "PluginNotFound" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "descriptor": { + "type": "string" + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "descriptor" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "PluginLoadFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "descriptor": { + "type": "string" + }, + "reason": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "PeerMissing" + ] + }, + "peer": { + "type": "string" + } + }, + "required": [ + "_tag", + "peer" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "PeerVersionUnsupported" + ] + }, + "peer": { + "type": "string" + }, + "detail": { + "type": "string" + } + }, + "required": [ + "_tag", + "peer", + "detail" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "PeerUnrecognized" + ] + }, + "peer": { + "type": "string" + } + }, + "required": [ + "_tag", + "peer" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "InvalidContribution" + ] + }, + "detail": { + "type": "string" + } + }, + "required": [ + "_tag", + "detail" + ], + "additionalProperties": true + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "descriptor", + "reason" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "PluginImportFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "descriptor": { + "type": "string" + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "descriptor" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "SurvivorsUnavailable" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "reason": { + "type": "string", + "enum": [ + "no-report", + "mismatch" + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "reason" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "NoInputFiles" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "SandboxPreparationFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "InstrumentationFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "CheckerFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "checker": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "checker" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "TestRunnerFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "BaselineTestsFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "testCount": { + "type": "integer", + "minimum": 0 + }, + "tests": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "location": { + "anyOf": [ + { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "minimum": 1 + }, + "column": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "file", + "line", + "column" + ], + "additionalProperties": true + }, + { + "type": "null" + } + ] + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "name", + "file", + "location", + "message", + "stack" + ], + "additionalProperties": true + } + ], + "additionalItems": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "location": { + "anyOf": [ + { + "type": "object", + "properties": { + "file": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "minimum": 1 + }, + "column": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "file", + "line", + "column" + ], + "additionalProperties": true + }, + { + "type": "null" + } + ] + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "name", + "file", + "location", + "message", + "stack" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "testCount", + "tests" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "BaselineTimedOut" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "BaselineErrored" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "BaselineFoundNoTests" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "WorkerBootTimedOut" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "workerKind": { + "type": "string", + "enum": [ + "testRunner", + "checker", + "reporter" + ] + }, + "pid": { + "type": "integer" + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "workerKind", + "pid" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "WorkerOutOfMemory" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "workerKind": { + "type": "string", + "enum": [ + "testRunner", + "checker", + "reporter" + ] + }, + "pid": { + "type": "integer" + }, + "exitCode": { + "type": "integer" + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "workerKind", + "pid", + "exitCode" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "WorkerCrashed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "workerKind": { + "type": "string", + "enum": [ + "testRunner", + "checker", + "reporter" + ] + }, + "pid": { + "type": "integer" + }, + "exitCode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "signal": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "workerKind", + "pid", + "exitCode", + "signal" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "ReporterFailed" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "reporter": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "reporter" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "RunInterrupted" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "NewSurvivors" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { "type": "object", "properties": { - "_tag": { + "primary": { "type": "string", "enum": [ - "PeerMissing" + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" ] }, - "peer": { - "type": "string" + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] } }, "required": [ - "_tag", - "peer" + "primary", + "otherwise" ], "additionalProperties": true }, - { + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "survivors": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "object", + "properties": { + "mutantId": { + "type": "string", + "minLength": 1 + }, + "file": { + "type": "string" + }, + "line": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "mutantId", + "file", + "line" + ], + "additionalProperties": true + } + ], + "additionalItems": { + "type": "object", + "properties": { + "mutantId": { + "type": "string", + "minLength": 1 + }, + "file": { + "type": "string" + }, + "line": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "mutantId", + "file", + "line" + ], + "additionalProperties": true + } + }, + "unchecked": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "survivors", + "unchecked" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "InvariantBroken" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { "type": "object", "properties": { - "_tag": { + "primary": { "type": "string", "enum": [ - "PeerVersionUnsupported" + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" ] }, - "peer": { - "type": "string" - }, - "detail": { - "type": "string" + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] } }, "required": [ - "_tag", - "peer", - "detail" + "primary", + "otherwise" ], "additionalProperties": true }, - { + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "CatalogGap" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { "type": "object", "properties": { - "_tag": { + "primary": { "type": "string", "enum": [ - "PeerUnrecognized" + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" ] }, - "peer": { - "type": "string" + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] } }, "required": [ - "_tag", - "peer" + "primary", + "otherwise" ], "additionalProperties": true }, - { + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "RecordMissing" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { "type": "object", "properties": { - "_tag": { + "primary": { "type": "string", "enum": [ - "InvalidContribution" + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" ] }, - "detail": { - "type": "string" + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] } }, "required": [ - "_tag", - "detail" + "primary", + "otherwise" ], "additionalProperties": true }, - { + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "exitCode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "exitCode" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "JobTimedOut" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { "type": "object", "properties": { - "_tag": { + "primary": { "type": "string", "enum": [ - "ImportFailed" + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" ] }, - "cause": {} + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } }, "required": [ - "_tag", - "cause" + "primary", + "otherwise" ], "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "limitSeconds": { + "type": "integer", + "minimum": 0 } - ] + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "limitSeconds" + ], + "additionalProperties": true }, { - "type": "null" + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "BinaryMissing" + ] + }, + "stage": { + "type": "string", + "enum": [ + "cli", + "config", + "prepare", + "instrument", + "check", + "dryRun", + "mutationTest", + "report", + "gate", + "ci", + "run" + ] + }, + "cause": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "message", + "stack" + ], + "additionalProperties": true + } + }, + "capsule": { + "anyOf": [ + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "Replays" + ] + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "argv": { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": true + } + } + }, + "required": [ + "_tag", + "cwd", + "argv", + "env" + ], + "additionalProperties": true + }, + { + "type": "object", + "properties": { + "_tag": { + "type": "string", + "enum": [ + "DoesNotReplay" + ] + }, + "why": { + "type": "string", + "enum": [ + "outOfMemory", + "workerCrashed", + "interrupted", + "recordMissing", + "jobTimedOut", + "binaryMissing" + ] + }, + "standIn": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "_tag", + "why", + "standIn" + ], + "additionalProperties": true + } + ] + }, + "nextAction": { + "type": "object", + "properties": { + "primary": { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + "otherwise": { + "anyOf": [ + { + "type": "string", + "enum": [ + "fixTest", + "fixCode", + "killSurvivor", + "refactorAwayEquivalent", + "fixConfiguration", + "retryInfrastructure", + "reportToolDefect" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "primary", + "otherwise" + ], + "additionalProperties": true + }, + "traceId": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + { + "type": "null" + } + ] + }, + "binary": { + "type": "string" + } + }, + "required": [ + "_tag", + "stage", + "cause", + "capsule", + "nextAction", + "traceId", + "binary" + ], + "additionalProperties": true } ] } @@ -1133,9 +6872,7 @@ "_tag", "schemaVersion", "code", - "error", - "remediation", - "reason" + "record" ], "additionalProperties": true }, @@ -1151,7 +6888,7 @@ "schemaVersion": { "type": "string", "enum": [ - "2.0" + "3.0" ] }, "code": { diff --git a/packages/stryker-js-cli-contract/etc/stryker-js-cli-contract.api.md b/packages/stryker-js-cli-contract/etc/stryker-js-cli-contract.api.md index 1d11c669..8608e358 100644 --- a/packages/stryker-js-cli-contract/etc/stryker-js-cli-contract.api.md +++ b/packages/stryker-js-cli-contract/etc/stryker-js-cli-contract.api.md @@ -4,6 +4,11 @@ ```ts +// @public (undocumented) +export namespace FailureRecord { + export { Capsule, CapsuleRule, CatalogEntry, CatalogExitCode, CauseLink, DoesNotReplay, EnvEntry, FailedTestEvidence, FailureCatalog, FailureCode, FailureEvidence, FailureRecord, FailureRecordFile, FailureStage, NextAction, NextActionKind, NonReplayReason, PluginLoadRefusal, RecordContext, Replays, SourceLocation, TraceId, WorkerKind, annotationsOf, failureCatalogEntries, markdownOf, recordOf, sarifTextOf, terminalTextOf }; +} + // @public (undocumented) export namespace OutputMode { export { ModeSignal, OutputMode }; diff --git a/packages/stryker-js-cli-contract/scripts/contract-documents.ts b/packages/stryker-js-cli-contract/scripts/contract-documents.ts index b3c2cb57..4bf0821b 100644 --- a/packages/stryker-js-cli-contract/scripts/contract-documents.ts +++ b/packages/stryker-js-cli-contract/scripts/contract-documents.ts @@ -2,6 +2,7 @@ import * as JsonSchema from 'effect/JsonSchema' import * as Result from 'effect/Result' import * as Schema from 'effect/Schema' +import { failureCatalogEntries } from '../src/failure-catalog.js' import { RunEvent } from '../src/RunEvent/mod.js' import { spanMembers } from '../src/SpanTaxonomy.js' import { SpanDocuments, type SpanMember } from '../src/SpanTaxonomy.schema.js' @@ -19,6 +20,8 @@ export const streamDocumentSource = (): string => { export const stockCatalogDocumentSource = (): string => `${JSON.stringify(StockCatalog.entries, null, 2)}\n` +export const failureCatalogDocumentSource = (): string => `${JSON.stringify(failureCatalogEntries, null, 2)}\n` + export const spanDocumentOf = (member: SpanMember) => ({ id: member.name, name: member.name, diff --git a/packages/stryker-js-cli-contract/scripts/generate-contract.ts b/packages/stryker-js-cli-contract/scripts/generate-contract.ts index c768cc99..3dfceea4 100644 --- a/packages/stryker-js-cli-contract/scripts/generate-contract.ts +++ b/packages/stryker-js-cli-contract/scripts/generate-contract.ts @@ -2,7 +2,12 @@ import { mkdirSync, writeFileSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' -import { spanTaxonomyDocumentSource, stockCatalogDocumentSource, streamDocumentSource } from './contract-documents.js' +import { + failureCatalogDocumentSource, + spanTaxonomyDocumentSource, + stockCatalogDocumentSource, + streamDocumentSource, +} from './contract-documents.js' const packageRoot = fileURLToPath(new URL('..', import.meta.url)) @@ -15,3 +20,4 @@ const write = (relativePath: string, contents: string): void => { write('contract/stream.schema.json', streamDocumentSource()) write('contract/stock-catalog.json', stockCatalogDocumentSource()) write('contract/span-taxonomy.json', spanTaxonomyDocumentSource()) +write('contract/failure-catalog.json', failureCatalogDocumentSource()) diff --git a/packages/stryker-js-cli-contract/src/FailureRecord/mod.ts b/packages/stryker-js-cli-contract/src/FailureRecord/mod.ts new file mode 100644 index 00000000..029fbb98 --- /dev/null +++ b/packages/stryker-js-cli-contract/src/FailureRecord/mod.ts @@ -0,0 +1,23 @@ +export { FailureCatalog, failureCatalogEntries, type RecordContext, recordOf } from '../failure-catalog.js' +export { CapsuleRule, CatalogEntry, CatalogExitCode } from '../failure-catalog.schema.js' +export { + Capsule, + CauseLink, + DoesNotReplay, + EnvEntry, + FailedTestEvidence, + FailureCode, + FailureEvidence, + FailureRecord, + FailureRecordFile, + FailureStage, + NextAction, + NextActionKind, + NonReplayReason, + PluginLoadRefusal, + Replays, + SourceLocation, + TraceId, + WorkerKind, +} from '../failure-record.schema.js' +export { annotationsOf, markdownOf, sarifTextOf, terminalTextOf } from '../render-failure.js' diff --git a/packages/stryker-js-cli-contract/src/SpanTaxonomy.schema.ts b/packages/stryker-js-cli-contract/src/SpanTaxonomy.schema.ts index e0da51b7..150e4afa 100644 --- a/packages/stryker-js-cli-contract/src/SpanTaxonomy.schema.ts +++ b/packages/stryker-js-cli-contract/src/SpanTaxonomy.schema.ts @@ -27,6 +27,8 @@ export interface SpanMember { export const SpanAttributeText = S.String +export const SpanAttributeOptionalText = S.optionalKey(SpanAttributeText) + export const SpanAttributeCount = S.Finite export const SpanRunStatus = S.Literals(['success', 'failure', 'timed_out', 'aborted', 'skipped']) diff --git a/packages/stryker-js-cli-contract/src/SpanTaxonomy.ts b/packages/stryker-js-cli-contract/src/SpanTaxonomy.ts index 6f71ed67..6e1287fe 100644 --- a/packages/stryker-js-cli-contract/src/SpanTaxonomy.ts +++ b/packages/stryker-js-cli-contract/src/SpanTaxonomy.ts @@ -1,7 +1,13 @@ import { Mutant, Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as S from 'effect/Schema' -import { SpanAttributeCount, SpanAttributeText, type SpanMember, SpanRunStatus } from './SpanTaxonomy.schema.js' +import { + SpanAttributeCount, + SpanAttributeOptionalText, + SpanAttributeText, + type SpanMember, + SpanRunStatus, +} from './SpanTaxonomy.schema.js' const protocolMethods: ReadonlyArray = [ ...new Set([ @@ -31,7 +37,7 @@ export const Spans = { attributes: { 'stryker.run.outcome': SpanAttributeText, 'stryker.run.exit_code': SpanAttributeCount, - 'stryker.run.error': SpanAttributeText, + 'stryker.failure.code': SpanAttributeOptionalText, }, }, commandRunnerRun: { name: 'stryker.command_runner.run', attributes: {} }, diff --git a/packages/stryker-js-cli-contract/src/failure-catalog.schema.ts b/packages/stryker-js-cli-contract/src/failure-catalog.schema.ts new file mode 100644 index 00000000..48d229c0 --- /dev/null +++ b/packages/stryker-js-cli-contract/src/failure-catalog.schema.ts @@ -0,0 +1,19 @@ +import * as S from 'effect/Schema' + +import { FailureCode, NextAction, NonReplayReason } from './failure-record.schema.js' + +export const CapsuleRule = S.Union([S.Literal('replays'), NonReplayReason]) +export type CapsuleRule = typeof CapsuleRule.Type + +export const CatalogExitCode = S.Literals([1, 2, 3, 4, 5, 130]) +export type CatalogExitCode = typeof CatalogExitCode.Type + +export const CatalogEntry = S.Struct({ + id: FailureCode, + name: FailureCode, + meaning: S.NonEmptyString, + exitCode: S.NullOr(CatalogExitCode), + nextAction: NextAction, + capsule: CapsuleRule, +}) +export type CatalogEntry = typeof CatalogEntry.Type diff --git a/packages/stryker-js-cli-contract/src/failure-catalog.ts b/packages/stryker-js-cli-contract/src/failure-catalog.ts new file mode 100644 index 00000000..a92ebe77 --- /dev/null +++ b/packages/stryker-js-cli-contract/src/failure-catalog.ts @@ -0,0 +1,209 @@ +import { dual } from 'effect/Function' + +import type { CapsuleRule, CatalogEntry } from './failure-catalog.schema.js' +import { + type Capsule, + type CauseLink, + type EnvEntry, + FailureCode, + type FailureEvidence, + type FailureRecord, + type NextAction, + type TraceId, +} from './failure-record.schema.js' + +type CatalogFacts = Omit + +const action = (primary: NextAction['primary'], otherwise: NextAction['otherwise'] = null): NextAction => ({ + primary, + otherwise, +}) + +export const FailureCatalog = { + ArgumentsInvalid: { + meaning: 'The command line named an option, argument or subcommand the CLI does not accept.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + ConfigInvalid: { + meaning: 'The configuration could not be read or failed validation.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + PluginNotFound: { + meaning: 'The configuration names a plugin that is not installed or not registered.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + PluginLoadFailed: { + meaning: 'A configured plugin refused to load: a peer is missing or unsupported, or its contribution is invalid.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + PluginImportFailed: { + meaning: 'A configured plugin module threw while it was being imported.', + exitCode: 4, + nextAction: action('reportToolDefect', 'fixConfiguration'), + capsule: 'replays', + }, + SurvivorsUnavailable: { + meaning: 'Survivors were requested but no matching mutation report exists.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + NoInputFiles: { + meaning: 'The configured file patterns matched no input files.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + SandboxPreparationFailed: { + meaning: 'The project could not be read or copied into the sandbox.', + exitCode: 3, + nextAction: action('fixConfiguration', 'retryInfrastructure'), + capsule: 'replays', + }, + InstrumentationFailed: { + meaning: 'The instrumenter could not read or mutate a source file.', + exitCode: 3, + nextAction: action('reportToolDefect'), + capsule: 'replays', + }, + CheckerFailed: { + meaning: 'A checker failed or broke its protocol while checking mutants.', + exitCode: 3, + nextAction: action('fixConfiguration', 'reportToolDefect'), + capsule: 'replays', + }, + TestRunnerFailed: { + meaning: 'The test runner could not be started or failed outside of any test.', + exitCode: 3, + nextAction: action('fixConfiguration', 'reportToolDefect'), + capsule: 'replays', + }, + BaselineTestsFailed: { + meaning: 'One or more tests failed in the initial test run, before any mutant was applied.', + exitCode: 5, + nextAction: action('fixCode', 'fixTest'), + capsule: 'replays', + }, + BaselineTimedOut: { + meaning: 'The initial test run did not finish within its time limit.', + exitCode: 5, + nextAction: action('fixTest', 'retryInfrastructure'), + capsule: 'replays', + }, + BaselineErrored: { + meaning: 'The initial test run errored outside of any test.', + exitCode: 5, + nextAction: action('fixTest'), + capsule: 'replays', + }, + WorkerBootTimedOut: { + meaning: 'A worker process did not finish booting before its boot window closed.', + exitCode: 3, + nextAction: action('retryInfrastructure', 'reportToolDefect'), + capsule: 'replays', + }, + BaselineFoundNoTests: { + meaning: 'The initial test run found no tests to run.', + exitCode: 2, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + WorkerOutOfMemory: { + meaning: 'A worker process was killed for running out of memory.', + exitCode: 3, + nextAction: action('retryInfrastructure'), + capsule: 'outOfMemory', + }, + WorkerCrashed: { + meaning: 'A worker process exited unexpectedly.', + exitCode: 3, + nextAction: action('retryInfrastructure', 'reportToolDefect'), + capsule: 'workerCrashed', + }, + ReporterFailed: { + meaning: 'A reporter could not be started or failed while reporting.', + exitCode: 3, + nextAction: action('fixConfiguration'), + capsule: 'replays', + }, + RunInterrupted: { + meaning: 'The run was interrupted by a signal before it finished.', + exitCode: 130, + nextAction: action('retryInfrastructure'), + capsule: 'interrupted', + }, + NewSurvivors: { + meaning: 'Mutants the change can affect survived every test and are absent from the accepted survivor baseline.', + exitCode: 1, + nextAction: action('killSurvivor', 'refactorAwayEquivalent'), + capsule: 'replays', + }, + InvariantBroken: { + meaning: 'Stryker reached a state its own design rules out; this is a defect in Stryker.', + exitCode: 4, + nextAction: action('reportToolDefect'), + capsule: 'replays', + }, + CatalogGap: { + meaning: 'A failure no catalog entry classifies; the record carries its full cause.', + exitCode: 4, + nextAction: action('reportToolDefect'), + capsule: 'replays', + }, + RecordMissing: { + meaning: 'A CI job ended without a failure record or a report.', + exitCode: null, + nextAction: action('reportToolDefect'), + capsule: 'recordMissing', + }, + JobTimedOut: { + meaning: 'A CI job was killed for exceeding its time limit.', + exitCode: null, + nextAction: action('retryInfrastructure'), + capsule: 'jobTimedOut', + }, + BinaryMissing: { + meaning: 'A CI job could not find a binary it needs.', + exitCode: null, + nextAction: action('fixConfiguration'), + capsule: 'binaryMissing', + }, +} as const satisfies { readonly [Code in FailureCode]: CatalogFacts } + +export interface RecordContext { + readonly cause: ReadonlyArray + readonly cwd: string + readonly argv: readonly [string, ...Array] + readonly env: ReadonlyArray + readonly traceId: TraceId | null +} + +const capsuleOf = (rule: CapsuleRule, context: RecordContext): Capsule => + rule === 'replays' + ? { _tag: 'Replays', cwd: context.cwd, argv: context.argv, env: context.env } + : { _tag: 'DoesNotReplay', why: rule, standIn: context.argv.join(' ') } + +export const recordOf: { + (context: RecordContext): (evidence: FailureEvidence) => FailureRecord + (evidence: FailureEvidence, context: RecordContext): FailureRecord +} = dual(2, (evidence: FailureEvidence, context: RecordContext): FailureRecord => ({ + ...evidence, + cause: context.cause, + capsule: capsuleOf(FailureCatalog[evidence._tag].capsule, context), + nextAction: FailureCatalog[evidence._tag].nextAction, + traceId: context.traceId, +})) + +export const failureCatalogEntries: ReadonlyArray = FailureCode.literals.map((code) => ({ + id: code, + name: code, + ...FailureCatalog[code], +})) diff --git a/packages/stryker-js-cli-contract/src/failure-record.schema.ts b/packages/stryker-js-cli-contract/src/failure-record.schema.ts new file mode 100644 index 00000000..12997e51 --- /dev/null +++ b/packages/stryker-js-cli-contract/src/failure-record.schema.ts @@ -0,0 +1,151 @@ +import * as S from 'effect/Schema' + +export const FailureStage = S.Literals([ + 'cli', + 'config', + 'prepare', + 'instrument', + 'check', + 'dryRun', + 'mutationTest', + 'report', + 'gate', + 'ci', + 'run', +]) +export type FailureStage = typeof FailureStage.Type + +export const NextActionKind = S.Literals([ + 'fixTest', + 'fixCode', + 'killSurvivor', + 'refactorAwayEquivalent', + 'fixConfiguration', + 'retryInfrastructure', + 'reportToolDefect', +]) +export type NextActionKind = typeof NextActionKind.Type + +export const NextAction = S.Struct({ primary: NextActionKind, otherwise: S.NullOr(NextActionKind) }) +export type NextAction = typeof NextAction.Type + +export const CauseLink = S.Struct({ kind: S.String, message: S.String, stack: S.NullOr(S.String) }) +export type CauseLink = typeof CauseLink.Type + +export const EnvEntry = S.Struct({ name: S.NonEmptyString, value: S.String }) +export type EnvEntry = typeof EnvEntry.Type + +export const NonReplayReason = S.Literals([ + 'outOfMemory', + 'workerCrashed', + 'interrupted', + 'recordMissing', + 'jobTimedOut', + 'binaryMissing', +]) +export type NonReplayReason = typeof NonReplayReason.Type + +export const Replays = S.TaggedStruct('Replays', { + cwd: S.NonEmptyString, + argv: S.NonEmptyArray(S.String), + env: S.Array(EnvEntry), +}) +export type Replays = typeof Replays.Type + +export const DoesNotReplay = S.TaggedStruct('DoesNotReplay', { why: NonReplayReason, standIn: S.NonEmptyString }) +export type DoesNotReplay = typeof DoesNotReplay.Type + +export const Capsule = S.Union([Replays, DoesNotReplay]) +export type Capsule = typeof Capsule.Type + +export const TraceId = S.String.check(S.isPattern(/^[0-9a-f]{32}$/u)) +export type TraceId = typeof TraceId.Type + +const SourceLine = S.Int.check(S.isGreaterThanOrEqualTo(1)) + +export const SourceLocation = S.Struct({ file: S.NonEmptyString, line: SourceLine, column: SourceLine }) +export type SourceLocation = typeof SourceLocation.Type + +export const FailedTestEvidence = S.Struct({ + id: S.String, + name: S.String, + file: S.NullOr(S.String), + location: S.NullOr(SourceLocation), + message: S.String, + stack: S.NullOr(S.String), +}) +export type FailedTestEvidence = typeof FailedTestEvidence.Type + +export const WorkerKind = S.Literals(['testRunner', 'checker', 'reporter']) +export type WorkerKind = typeof WorkerKind.Type + +export const PluginLoadRefusal = S.Union([ + S.TaggedStruct('PeerMissing', { peer: S.String }), + S.TaggedStruct('PeerVersionUnsupported', { peer: S.String, detail: S.String }), + S.TaggedStruct('PeerUnrecognized', { peer: S.String }), + S.TaggedStruct('InvalidContribution', { detail: S.String }), +]) +export type PluginLoadRefusal = typeof PluginLoadRefusal.Type + +const NonNegativeInt = S.Int.check(S.isGreaterThanOrEqualTo(0)) + +const RecordShared = { + stage: FailureStage, + cause: S.Array(CauseLink), + capsule: Capsule, + nextAction: NextAction, + traceId: S.NullOr(TraceId), +} + +const variant = (tag: Tag, fields: Fields) => ({ + tag, + evidence: S.TaggedStruct(tag, { stage: FailureStage, ...fields }), + record: S.TaggedStruct(tag, { ...RecordShared, ...fields }), +}) + +const SurvivorEvidence = S.Struct({ mutantId: S.NonEmptyString, file: S.String, line: SourceLine }) + +const VARIANTS = [ + variant('ArgumentsInvalid', { argument: S.NullOr(S.String) }), + variant('ConfigInvalid', { detail: S.String }), + variant('PluginNotFound', { descriptor: S.String }), + variant('PluginLoadFailed', { descriptor: S.String, reason: PluginLoadRefusal }), + variant('PluginImportFailed', { descriptor: S.String }), + variant('SurvivorsUnavailable', { reason: S.Literals(['no-report', 'mismatch']) }), + variant('NoInputFiles', {}), + variant('SandboxPreparationFailed', {}), + variant('InstrumentationFailed', {}), + variant('CheckerFailed', { checker: S.NullOr(S.String) }), + variant('TestRunnerFailed', {}), + variant('BaselineTestsFailed', { testCount: NonNegativeInt, tests: S.NonEmptyArray(FailedTestEvidence) }), + variant('BaselineTimedOut', {}), + variant('BaselineErrored', {}), + variant('BaselineFoundNoTests', {}), + variant('WorkerBootTimedOut', { workerKind: WorkerKind, pid: S.Int }), + variant('WorkerOutOfMemory', { workerKind: WorkerKind, pid: S.Int, exitCode: S.Int }), + variant('WorkerCrashed', { + workerKind: WorkerKind, + pid: S.Int, + exitCode: S.NullOr(S.Int), + signal: S.NullOr(S.String), + }), + variant('ReporterFailed', { reporter: S.NullOr(S.String) }), + variant('RunInterrupted', {}), + variant('NewSurvivors', { survivors: S.NonEmptyArray(SurvivorEvidence), unchecked: NonNegativeInt }), + variant('InvariantBroken', {}), + variant('CatalogGap', {}), + variant('RecordMissing', { exitCode: S.NullOr(S.Int) }), + variant('JobTimedOut', { limitSeconds: NonNegativeInt }), + variant('BinaryMissing', { binary: S.String }), +] as const + +export const FailureEvidence = S.Union(VARIANTS.map((each) => each.evidence)) +export type FailureEvidence = typeof FailureEvidence.Type + +export const FailureRecord = S.Union(VARIANTS.map((each) => each.record)) +export type FailureRecord = typeof FailureRecord.Type + +export const FailureCode = S.Literals(VARIANTS.map((each) => each.tag)) +export type FailureCode = typeof FailureCode.Type + +export const FailureRecordFile = S.fromJsonString(FailureRecord) diff --git a/packages/stryker-js-cli-contract/src/mod.ts b/packages/stryker-js-cli-contract/src/mod.ts index 7953746a..fdf30aad 100644 --- a/packages/stryker-js-cli-contract/src/mod.ts +++ b/packages/stryker-js-cli-contract/src/mod.ts @@ -1,3 +1,4 @@ +export * as FailureRecord from './FailureRecord/mod.js' export * as OutputMode from './OutputMode/mod.js' export * as RunEvent from './RunEvent/mod.js' export * as SpanTaxonomy from './SpanTaxonomy/mod.js' diff --git a/packages/stryker-js-cli-contract/src/render-failure.ts b/packages/stryker-js-cli-contract/src/render-failure.ts new file mode 100644 index 00000000..7cc133f6 --- /dev/null +++ b/packages/stryker-js-cli-contract/src/render-failure.ts @@ -0,0 +1,408 @@ +import * as Arr from 'effect/Array' +import * as Match from 'effect/Match' +import * as Option from 'effect/Option' +import * as Predicate from 'effect/Predicate' + +import { FailureCatalog } from './failure-catalog.js' +import { + type Capsule, + type CauseLink, + type FailedTestEvidence, + type FailureCode, + FailureRecord, + type NextAction, + type PluginLoadRefusal, + type Replays, +} from './failure-record.schema.js' + +const meaningOf = (code: FailureCode): string => FailureCatalog[code].meaning + +const SHELL_SAFE = /^[A-Za-z0-9_@%+=:,./-]+$/u + +const singleQuoted = (value: string): string => `'${value.replaceAll("'", "'\\''")}'` + +const isShellSafe = (value: string): boolean => value.length > 0 && SHELL_SAFE.test(value) + +const shellQuote = (value: string): string => (isShellSafe(value) ? value : singleQuoted(value)) + +const escapeCommandData = (value: string): string => + value.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A') + +const escapeCommandProperty = (value: string): string => + escapeCommandData(value).replaceAll(':', '%3A').replaceAll(',', '%2C') + +const indentLines = (text: string, prefix: string): ReadonlyArray => + text.split('\n').map((line) => `${prefix}${line}`) + +const locationTextOf = (location: { readonly file: string; readonly line: number; readonly column: number }): string => + `${location.file}:${location.line}:${location.column}` + +const textOrFallback = (value: string | null, fallback: string): string => (value === null ? fallback : value) + +const numberOrFallback = (value: number | null, fallback: string): string => value === null ? fallback : `${value}` + +const refusalTextOf = (refusal: PluginLoadRefusal): string => + Match.value(refusal).pipe( + Match.tagsExhaustive({ + PeerMissing: (missing) => `the peer ${missing.peer} is not installed`, + PeerVersionUnsupported: (unsupported) => + `the peer ${unsupported.peer} is at an unsupported version: ${unsupported.detail}`, + PeerUnrecognized: (unrecognized) => `the peer ${unrecognized.peer} is not recognized`, + InvalidContribution: (invalid) => `its contribution is invalid: ${invalid.detail}`, + }), + ) + +const nonNullPrefixOf = (value: string | null): string => (value === null ? '' : `${value} `) + +const failedTestWhereOf = (test: FailedTestEvidence): string => + test.location === null ? nonNullPrefixOf(test.file) : `${locationTextOf(test.location)} ` + +const failedTestHeadingOf = (test: FailedTestEvidence): string => `${failedTestWhereOf(test)}${test.name}` + +const unlocatedStackOf = (test: FailedTestEvidence): string | null => (test.location === null ? test.stack : null) + +const baselineTestLinesOf = ( + testCount: number, + tests: ReadonlyArray, +): ReadonlyArray => [ + `${tests.length} of ${testCount} test(s) failed`, + ...Arr.flatMap(tests, (test) => [ + `- ${failedTestHeadingOf(test)}`, + ` message: ${test.message}`, + ...Option.match(Option.fromNullishOr(unlocatedStackOf(test)), { + onNone: (): ReadonlyArray => [], + onSome: (stack) => [' stack:', ...indentLines(stack, ' ')], + }), + ]), +] + +const evidenceLinesOf = (record: FailureRecord): ReadonlyArray => + Match.value(record).pipe( + Match.tagsExhaustive({ + ArgumentsInvalid: (evidence) => + evidence.argument === null ? ['no argument was named'] : [`argument: ${evidence.argument}`], + ConfigInvalid: (evidence) => [`detail: ${evidence.detail}`], + PluginNotFound: (evidence) => [`descriptor: ${evidence.descriptor}`], + PluginLoadFailed: (evidence) => [ + `descriptor: ${evidence.descriptor}`, + `refusal: ${refusalTextOf(evidence.reason)}`, + ], + PluginImportFailed: (evidence) => [`descriptor: ${evidence.descriptor}`], + SurvivorsUnavailable: (evidence) => [`reason: ${evidence.reason}`], + NoInputFiles: () => [], + SandboxPreparationFailed: () => [], + InstrumentationFailed: () => [], + CheckerFailed: (evidence) => [`checker: ${textOrFallback(evidence.checker, '(not named)')}`], + TestRunnerFailed: () => [], + BaselineTestsFailed: (evidence) => baselineTestLinesOf(evidence.testCount, evidence.tests), + BaselineTimedOut: () => [], + BaselineErrored: () => [], + BaselineFoundNoTests: () => [], + WorkerBootTimedOut: (evidence) => [ + `worker: ${evidence.workerKind} pid ${evidence.pid}`, + 'its boot window closed before it reported ready', + ], + WorkerOutOfMemory: (evidence) => [ + `worker: ${evidence.workerKind} pid ${evidence.pid}`, + `exit code: ${evidence.exitCode}`, + ], + WorkerCrashed: (evidence) => [ + `worker: ${evidence.workerKind} pid ${evidence.pid}`, + `exit code: ${numberOrFallback(evidence.exitCode, '(no exit code)')}`, + `signal: ${textOrFallback(evidence.signal, '(no signal)')}`, + ], + ReporterFailed: (evidence) => [`reporter: ${textOrFallback(evidence.reporter, '(not named)')}`], + RunInterrupted: () => [], + NewSurvivors: (evidence) => [ + `survivors: ${evidence.survivors.length} listed, ${evidence.unchecked} unchecked`, + ...evidence.survivors.map((survivor) => `${survivor.file}:${survivor.line} ${survivor.mutantId}`), + ], + InvariantBroken: () => [], + CatalogGap: () => [], + RecordMissing: (evidence) => [`exit code: ${numberOrFallback(evidence.exitCode, '(not recorded)')}`], + JobTimedOut: (evidence) => [`time limit: ${evidence.limitSeconds}s`], + BinaryMissing: (evidence) => [`binary: ${evidence.binary}`], + }), + ) + +const stackShownFor = (record: FailureRecord) => (link: CauseLink): string | null => + Predicate.isTagged(record, 'CatalogGap') ? link.stack : null + +const causeLinesOf = (record: FailureRecord): ReadonlyArray => + Arr.flatMap(record.cause, (link) => [ + `${link.kind}: ${link.message}`, + ...Option.match(Option.fromNullishOr(stackShownFor(record)(link)), { + onNone: (): ReadonlyArray => [], + onSome: (stack) => indentLines(stack, ' '), + }), + ]) + +const reproduceCommandOf = (replays: Replays): string => + `(cd ${shellQuote(replays.cwd)} && ${ + [ + ...replays.env.map((entry) => `${entry.name}=${shellQuote(entry.value)}`), + ...replays.argv.map(shellQuote), + ].join(' ') + })` + +const capsuleLinesOf = (capsule: Capsule): ReadonlyArray => + Match.value(capsule).pipe( + Match.tagsExhaustive({ + Replays: (replays) => [`reproduce: ${reproduceCommandOf(replays)}`], + DoesNotReplay: (refusal) => [`does not replay (${refusal.why}); instead: ${refusal.standIn}`], + }), + ) + +const otherwiseLinesOf = (record: FailureRecord): ReadonlyArray => + record.nextAction.otherwise === null ? [] : [`otherwise: ${record.nextAction.otherwise}`] + +const traceLinesOf = (record: FailureRecord): ReadonlyArray => + record.traceId === null ? [] : [`trace: ${record.traceId}`] + +export const terminalTextOf = (record: FailureRecord): string => + [ + `${record._tag}: ${meaningOf(record._tag)}`, + ...evidenceLinesOf(record), + ...causeLinesOf(record), + `next: ${record.nextAction.primary}`, + ...otherwiseLinesOf(record), + ...capsuleLinesOf(record.capsule), + ...traceLinesOf(record), + ].join('\n') + +const isBulletLine = (line: string): boolean => line.startsWith('- ') + +const indentedBulletOf = (line: string): string => (line.startsWith(' ') ? ` - ${line.trimStart()}` : `- ${line}`) + +const markdownBulletOf = (line: string): string => (isBulletLine(line) ? line : indentedBulletOf(line)) + +const otherwiseSuffixOf = (nextAction: NextAction): string => + nextAction.otherwise === null ? '' : ` (otherwise \`${nextAction.otherwise}\`)` + +const bulletsOf = (facts: ReadonlyArray): ReadonlyArray => + facts.length === 0 ? ['- no further evidence'] : facts.map(markdownBulletOf) + +const markdownCapsuleLinesOf = (capsule: Capsule, traceId: string | null): ReadonlyArray => [ + ...Match.value(capsule).pipe( + Match.tagsExhaustive({ + Replays: (replays) => ['**Replay:**', '', '```bash', reproduceCommandOf(replays), '```'], + DoesNotReplay: (refusal) => [`**Replay:** does not replay (${refusal.why}); instead: ${refusal.standIn}`], + }), + ), + ...(traceId === null ? [] : ['', `**Trace:** \`${traceId}\``]), +] + +export const markdownOf = (record: FailureRecord): string => { + const facts = [...evidenceLinesOf(record), ...causeLinesOf(record)] + return [ + `## \`${record._tag}\`: ${meaningOf(record._tag)}`, + '', + ...bulletsOf(facts), + '', + `**Next:** \`${record.nextAction.primary}\`${otherwiseSuffixOf(record.nextAction)}`, + '', + ...markdownCapsuleLinesOf(record.capsule, record.traceId), + '', + ].join('\n') +} + +interface LocatedEvidence { + readonly file: string + readonly line: number + readonly column: number | null + readonly message: string +} + +const locatedEvidenceOf = (record: FailureRecord): ReadonlyArray => + Match.value(record).pipe( + Match.tagsExhaustive({ + BaselineTestsFailed: (evidence) => + Arr.flatMap(evidence.tests, (test) => + test.location === null + ? [] + : [ + { + file: test.location.file, + line: test.location.line, + column: test.location.column, + message: `${test.name}: ${test.message}`, + }, + ]), + NewSurvivors: (evidence) => + evidence.survivors.map((survivor) => ({ + file: survivor.file, + line: survivor.line, + column: null, + message: `surviving mutant ${survivor.mutantId}`, + })), + ArgumentsInvalid: () => [], + ConfigInvalid: () => [], + PluginNotFound: () => [], + PluginLoadFailed: () => [], + PluginImportFailed: () => [], + SurvivorsUnavailable: () => [], + NoInputFiles: () => [], + SandboxPreparationFailed: () => [], + InstrumentationFailed: () => [], + CheckerFailed: () => [], + TestRunnerFailed: () => [], + BaselineTimedOut: () => [], + BaselineErrored: () => [], + BaselineFoundNoTests: () => [], + WorkerBootTimedOut: () => [], + WorkerOutOfMemory: () => [], + WorkerCrashed: () => [], + ReporterFailed: () => [], + RunInterrupted: () => [], + InvariantBroken: () => [], + CatalogGap: () => [], + RecordMissing: () => [], + JobTimedOut: () => [], + BinaryMissing: () => [], + }), + ) + +const annotationOf = (code: FailureCode, located: LocatedEvidence): string => { + const properties = [ + `title=${escapeCommandProperty(code)}`, + `file=${escapeCommandProperty(located.file)}`, + `line=${escapeCommandProperty(`${located.line}`)}`, + ...(located.column === null ? [] : [`col=${escapeCommandProperty(`${located.column}`)}`]), + ].join(',') + return `::error ${properties}::${escapeCommandData(located.message)}` +} + +const firstEvidenceLineOf = (record: FailureRecord): Option.Option => Arr.head(evidenceLinesOf(record)) + +const summaryAnnotationOf = (record: FailureRecord): string => { + const summary = Option.match(firstEvidenceLineOf(record), { + onNone: () => meaningOf(record._tag), + onSome: (first) => `${meaningOf(record._tag)} — ${first}`, + }) + return `::error title=${escapeCommandProperty(record._tag)}::${escapeCommandData(summary)}` +} + +export const annotationsOf = (record: FailureRecord): ReadonlyArray => { + const located = locatedEvidenceOf(record) + return located.length === 0 + ? [summaryAnnotationOf(record)] + : located.map((entry) => annotationOf(record._tag, entry)) +} + +export const sarifTextOf = (record: FailureRecord): string => + [ + `${record._tag}: ${meaningOf(record._tag)}`, + ...Option.toArray(firstEvidenceLineOf(record)), + `next: ${record.nextAction.primary}`, + ].join(' — ') + +if (import.meta.vitest !== void 0) { + const { it } = await import('@systemfsoftware/vitest') + + const FORBIDDEN = /\[object Object\]|Unknown failure|_tag|~(?:stryker|systemfsoftware|effect)\b/u + const LINE_BREAK = /[\r\n]/u + const JOIN = '\u0000' + + const locatedAnnotationTextOf = ( + code: FailureCode, + file: string, + line: number, + column: number | null, + message: string, + ): string => + `::error title=${escapeCommandProperty(code)},file=${escapeCommandProperty(file)},` + + `line=${escapeCommandProperty(`${line}`)}` + + `${column === null ? '' : `,col=${escapeCommandProperty(`${column}`)}`}::${escapeCommandData(message)}` + + const expectedAnnotationsOf = (record: FailureRecord): ReadonlyArray => + Match.value(record).pipe( + Match.tag('BaselineTestsFailed', (evidence) => + Arr.flatMap(evidence.tests, (test) => + test.location === null + ? [] + : [ + locatedAnnotationTextOf( + 'BaselineTestsFailed', + test.location.file, + test.location.line, + test.location.column, + `${test.name}: ${test.message}`, + ), + ])), + Match.tag('NewSurvivors', (evidence) => + evidence.survivors.map((survivor) => + locatedAnnotationTextOf( + 'NewSurvivors', + survivor.file, + survivor.line, + null, + `surviving mutant ${survivor.mutantId}`, + ) + )), + Match.orElse((): ReadonlyArray => []), + ) + + const summaryOnly = (annotations: ReadonlyArray, code: FailureCode): boolean => + annotations.length === 1 && Arr.every(annotations, (line) => line.startsWith(`::error title=${code}::`)) + + const sameAnnotations = (actual: ReadonlyArray, expected: ReadonlyArray): boolean => + actual.length === expected.length && actual.every((line, index) => line === expected[index]) + + it.prop( + '∀r_FailureRecord_≡AnnotationsCoverExactlyTheLocatedEvidence', + { of: [FailureRecord], subject: annotationsOf }, + (subject, [record]) => { + const expected = expectedAnnotationsOf(record) + const annotations = subject(record) + return expected.length === 0 ? summaryOnly(annotations, record._tag) : sameAnnotations(annotations, expected) + }, + ) + + it.prop( + '∀r_FailureRecord_≡AnnotationsLeakNoInternalTag', + { of: [FailureRecord], subject: annotationsOf }, + (subject, [record]) => !FORBIDDEN.test(subject(record).join(JOIN)), + ) + + it.prop( + '∀r_FailureRecord_≡AnnotationsNeverSplitAWorkflowCommand', + { of: [FailureRecord], subject: annotationsOf }, + (subject, [record]) => !LINE_BREAK.test(subject(record).join(JOIN)), + ) + + it.prop( + '∀r_FailureRecord_≡TerminalTextLeaksNoInternalTag', + { of: [FailureRecord], subject: terminalTextOf }, + (subject, [record]) => !FORBIDDEN.test(subject(record)), + ) + + it.prop( + '∀r_FailureRecord_≡TerminalTextStatesTheNextAction', + { of: [FailureRecord], subject: terminalTextOf }, + (subject, [record]) => subject(record).includes(`next: ${record.nextAction.primary}`), + ) + + it.prop( + '∀r_FailureRecord_≡MarkdownLeaksNoInternalTag', + { of: [FailureRecord], subject: markdownOf }, + (subject, [record]) => !FORBIDDEN.test(subject(record)), + ) + + it.prop( + '∀r_FailureRecord_≡MarkdownStatesTheNextAction', + { of: [FailureRecord], subject: markdownOf }, + (subject, [record]) => subject(record).includes(`\`${record.nextAction.primary}\``), + ) + + it.prop( + '∀r_FailureRecord_≡SarifLeaksNoInternalTag', + { of: [FailureRecord], subject: sarifTextOf }, + (subject, [record]) => !FORBIDDEN.test(subject(record)), + ) + + it.prop( + '∀r_FailureRecord_≡SarifStatesTheNextAction', + { of: [FailureRecord], subject: sarifTextOf }, + (subject, [record]) => subject(record).endsWith(`next: ${record.nextAction.primary}`), + ) +} diff --git a/packages/stryker-js-cli-contract/src/run-event.schema.ts b/packages/stryker-js-cli-contract/src/run-event.schema.ts index 432c826c..96909d8c 100644 --- a/packages/stryker-js-cli-contract/src/run-event.schema.ts +++ b/packages/stryker-js-cli-contract/src/run-event.schema.ts @@ -3,8 +3,8 @@ import { Plugin, Report } from '@systemfsoftware/stryker-js-plugin-interface' import { SchemaGetter } from 'effect' import * as S from 'effect/Schema' +import { FailureRecord } from './failure-record.schema.js' import { ModeSignal, OutputMode } from './output-mode.schema.js' -import { PluginLoadFailureReason } from './plugin-load-failure-reason.schema.js' import { StreamSchemaVersion } from './stream-version.schema.js' export const RunPhase = S.Literals(['prepare', 'instrument', 'dry-run', 'mutation-test']) @@ -256,9 +256,7 @@ export class FeedbackReported extends S.TaggedClass()('feedbac export class RunFailed extends S.TaggedClass()('error', { schemaVersion: StreamSchemaVersion, code: Plugin.ExitCode, - error: S.String, - remediation: S.String, - reason: S.NullOr(PluginLoadFailureReason), + record: FailureRecord, }) {} export class HelpRendered extends S.TaggedClass()('help', { diff --git a/packages/stryker-js-cli-contract/src/stream-version.schema.ts b/packages/stryker-js-cli-contract/src/stream-version.schema.ts index 9a764165..5378825a 100644 --- a/packages/stryker-js-cli-contract/src/stream-version.schema.ts +++ b/packages/stryker-js-cli-contract/src/stream-version.schema.ts @@ -1,4 +1,4 @@ import * as S from 'effect/Schema' -export const StreamSchemaVersion = S.Literal('2.0') +export const StreamSchemaVersion = S.Literal('3.0') export type StreamSchemaVersion = typeof StreamSchemaVersion.Type diff --git a/packages/stryker-js-cli-contract/tests/contract-documents.integration.test.ts b/packages/stryker-js-cli-contract/tests/contract-documents.integration.test.ts index 781ebcc2..c9dacab7 100644 --- a/packages/stryker-js-cli-contract/tests/contract-documents.integration.test.ts +++ b/packages/stryker-js-cli-contract/tests/contract-documents.integration.test.ts @@ -7,6 +7,7 @@ import * as Layer from 'effect/Layer' import * as Path from 'effect/Path' import { + failureCatalogDocumentSource, spanTaxonomyDocumentSource, stockCatalogDocumentSource, streamDocumentSource, @@ -56,6 +57,23 @@ Feature('Regenerating the published CLI contract documents') ), ) + scenario( + 'The committed failure catalog document is byte for byte what the generator writes from the catalog', + Gherkin.Do.pipe( + Given('the committed failure catalog document')( + 'committed', + () => readCommitted('contract/failure-catalog.json'), + ), + When('the generator reconstructs the document from the failure catalog')( + 'regenerated', + () => Effect.sync(() => failureCatalogDocumentSource()), + ), + Then('the regenerated document equals the committed bytes')((s, expect) => + expect(s.regenerated).toEqual(s.committed) + ), + ), + ) + scenario( 'The committed span taxonomy document is byte for byte what the generator writes from the taxonomy', Gherkin.Do.pipe( diff --git a/packages/stryker-js-plugin-interface/etc/stryker-js-plugin-interface.api.md b/packages/stryker-js-plugin-interface/etc/stryker-js-plugin-interface.api.md index 3951f1ba..0768e4d9 100644 --- a/packages/stryker-js-plugin-interface/etc/stryker-js-plugin-interface.api.md +++ b/packages/stryker-js-plugin-interface/etc/stryker-js-plugin-interface.api.md @@ -51,7 +51,7 @@ export namespace Reporter { // @public (undocumented) export namespace TestRunner { - export { BaseTestResult, CompleteDryRunResult, CoverageAnalysis, CoverageAnalysisSchema, DryRunOptions, DryRunOptionsSchema, DryRunResult, DryRunResultSchema, DryRunStatus, ErrorDryRunResult, ErrorMutantRunResult, ExecutedTest, ExecutedTestSchema, FailedTestResult, HitLimitReason, HitLimitReasonPrefix, HitLimitReasonText, KilledMutantRunResult, MutantRunResult, MutantRunResultSchema, MutantRunStatus, SkippedTestResult, SuccessTestResult, SurvivedMutantRunResult, TestId, TestResult, TestResultSchema, TestRunner, TestRunnerCapabilities, TestRunnerCapabilitiesSchema, TestRunnerFailed, TestRunnerService, TestStatus, TimeoutDryRunResult, TimeoutMutantRunResult, WallClockTimeoutReason }; + export { BaseTestResult, CompleteDryRunResult, CoverageAnalysis, CoverageAnalysisSchema, DryRunOptions, DryRunOptionsSchema, DryRunResult, DryRunResultSchema, DryRunStatus, ErrorDryRunResult, ErrorMutantRunResult, ExecutedTest, ExecutedTestSchema, FailedTestResult, HitLimitReason, HitLimitReasonPrefix, HitLimitReasonText, KilledMutantRunResult, MutantRunResult, MutantRunResultSchema, MutantRunStatus, SkippedTestResult, SuccessTestResult, SurvivedMutantRunResult, TestFailureLocation, TestId, TestResult, TestResultSchema, TestRunner, TestRunnerCapabilities, TestRunnerCapabilitiesSchema, TestRunnerFailed, TestRunnerService, TestStatus, TimeoutDryRunResult, TimeoutMutantRunResult, WallClockTimeoutReason }; } // @public (undocumented) diff --git a/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts b/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts index 1476819d..1e313c93 100644 --- a/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts +++ b/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts @@ -1,7 +1,6 @@ /// import * as S from 'effect/Schema' -import { Position } from './Location.schema.js' import { NonNegativeFinite, NonNegativeInt } from './Metrics.schema.js' import { MutantCoverageSchema, RunOptionsFields } from './Mutant.schema.js' import type { MutantCoverage, RunOptions } from './Mutant.schema.js' @@ -27,11 +26,21 @@ const TestResultBase = { name: S.String, timeSpentMs: NonNegativeFinite, fileName: S.optionalKey(S.String), - startPosition: S.optionalKey(Position), } +const SourceOrdinal = S.Int.check(S.isGreaterThanOrEqualTo(1)) + +export const TestFailureLocation = S.Struct({ file: S.NonEmptyString, line: SourceOrdinal, column: SourceOrdinal }) +export type TestFailureLocation = typeof TestFailureLocation.Type + export const TestResultSchema = S.Union([ - S.Struct({ ...TestResultBase, status: S.Literal('failed'), failureMessage: S.String }), + S.Struct({ + ...TestResultBase, + status: S.Literal('failed'), + failureMessage: S.String, + location: S.optionalKey(TestFailureLocation), + stack: S.optionalKey(S.String), + }), S.Struct({ ...TestResultBase, status: S.Literal('skipped') }), S.Struct({ ...TestResultBase, status: S.Literal('success') }), ]) @@ -98,12 +107,13 @@ export interface BaseTestResult { readonly name: string readonly timeSpentMs: number readonly fileName?: string - readonly startPosition?: Position } export interface FailedTestResult extends BaseTestResult { readonly status: 'failed' readonly failureMessage: string + readonly location?: TestFailureLocation + readonly stack?: string } export interface SkippedTestResult extends BaseTestResult { diff --git a/packages/stryker-js-vitest-runner/src/VitestRunner.service.ts b/packages/stryker-js-vitest-runner/src/VitestRunner.service.ts index d20ca61f..3cb4073b 100644 --- a/packages/stryker-js-vitest-runner/src/VitestRunner.service.ts +++ b/packages/stryker-js-vitest-runner/src/VitestRunner.service.ts @@ -15,10 +15,10 @@ import * as Result from 'effect/Result' import * as S from 'effect/Schema' import type { RunnerTask, RunnerTestCase, RunnerTestFile, RunnerTestSuite } from 'vitest' -import { testRecordOf } from './drivers/vitest-node.js' +import { errorTextOf, failureFramesOf, testRecordOf } from './drivers/vitest-node.js' import { interpretVitestDryRun } from './interpret-vitest-dry-run.workflow.js' import { makeMutantRunCell } from './MutantRun.cell.js' -import { VitestDryRunCommand } from './vitest-run-command.schema.js' +import { VitestDryRunCommand, type VitestFileFailure } from './vitest-run-command.schema.js' import { interpretVitestTestRun } from './vitest-test-run.js' import { CoverageDecodeFailed, type TestRunnerPhase } from './VitestRunner.schema.js' import { testFileOrder } from './VitestRuntime.blueprint.js' @@ -92,13 +92,11 @@ const collectTestsFromSuite = (suite: RunnerTestSuite): readonly RunnerTestCase[ }) ) -const messageOfError = (error: { readonly message?: string }): Option.Option => - Option.filter(Option.fromNullishOr(error.message), (message) => message.length > 0) - const fileFailureMessages = (file: RunnerTestFile): readonly string[] => Option.match(Option.fromNullishOr(file.result?.errors), { onNone: (): readonly string[] => [], - onSome: (errors) => errors.flatMap((error) => Option.toArray(messageOfError(error))), + onSome: (errors) => + errors.flatMap((error) => Option.toArray(Option.filter(errorTextOf(error), (message) => message.length > 0))), }) const fileFailureMessage = (file: RunnerTestFile): string => { @@ -106,6 +104,27 @@ const fileFailureMessage = (file: RunnerTestFile): string => { return messages.length === 0 ? 'StrykerJS: the test file failed to load' : messages.join('\n') } +const fileFailureOf = (file: RunnerTestFile): VitestFileFailure => { + const first = Option.flatMap( + Option.fromNullishOr(file.result?.errors), + (errors) => Option.fromNullishOr(errors[0]), + ) + return { + fileName: file.filepath, + message: fileFailureMessage(file), + ...Option.match(first, { + onNone: () => ({}), + onSome: (error) => ({ + ...Option.match(Option.fromNullishOr(error.stack), { + onNone: () => ({}), + onSome: (stack) => ({ stack }), + }), + frames: failureFramesOf(error), + }), + }), + } +} + const SKIPPED_TEST_MODES: ReadonlySet = new Set(['skip', 'todo']) const isCollectableTest = (test: RunnerTestCase): boolean => @@ -331,9 +350,7 @@ const makeRunner = Effect.fn(SpanTaxonomy.Spans.vitestRunnerMake.name)(function* })) const records = collected.flatMap((entry) => entry.tests).map(testRecordOf) const fileFailures = collected.flatMap(({ file, tests }) => - fileFailedWithoutFailingTest(file, tests) - ? [{ fileName: file.filepath, message: fileFailureMessage(file) }] - : [] + fileFailedWithoutFailingTest(file, tests) ? [fileFailureOf(file)] : [] ) const externalError = hasExternalErrors(self) yield* Effect.annotateCurrentSpan({ diff --git a/packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts b/packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts index 5516c55e..a808c3ac 100644 --- a/packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts +++ b/packages/stryker-js-vitest-runner/src/drivers/vitest-node.ts @@ -6,7 +6,7 @@ import * as Predicate from 'effect/Predicate' import * as S from 'effect/Schema' import type { Vitest } from 'vitest/node' -import type { VitestTestRecord } from '../vitest-run-command.schema.js' +import type { VitestFailureFrame, VitestTestRecord } from '../vitest-run-command.schema.js' export type VitestValue = S.Schema.Type @@ -160,14 +160,49 @@ const stringFieldOf = (value: A, key: string): Option.Option(value: A, key: string): Option.Option => Option.filter(propertyOf(value, key), Predicate.isNumber) -const firstErrorMessageOf = (result: A): Option.Option => +const firstErrorOf = (result: A): Option.Option => Option.flatMap( Option.flatMap(propertyOf(result, 'errors'), (errors) => Option.flatMap(Option.liftPredicate(errors, Array.isArray), (list) => Option.fromNullishOr(list[0]))), (first) => - stringFieldOf(first, 'message'), + Option.liftPredicate(first, Predicate.isObject), ) +const frameOf = (value: VitestValue): Option.Option => + Option.flatMap( + stringFieldOf(value, 'file'), + (file) => + Option.flatMap(numberFieldOf(value, 'line'), (line) => + Option.map(numberFieldOf(value, 'column'), (column) => ({ file, line, column }))), + ) + +export const failureFramesOf = (error: A): readonly VitestFailureFrame[] => + Option.getOrElse( + Option.map( + Option.filter(propertyOf(error, 'stacks'), Array.isArray), + (list) => list.flatMap((entry) => Option.toArray(frameOf(entry))), + ), + (): readonly VitestFailureFrame[] => [], + ) + +const causeTextsOf = (error: A): readonly string[] => + Option.match(propertyOf(error, 'cause'), { + onNone: (): readonly string[] => [], + onSome: (cause) => Option.toArray(errorTextOf(cause)), + }) + +export const errorTextOf = (error: A): Option.Option => + Option.map(stringFieldOf(error, 'message'), (message) => [message, ...causeTextsOf(error)].join('\n')) + +const errorTextOfResult = (result: A): Option.Option => + Option.flatMap(firstErrorOf(result), errorTextOf) + +const failureNameOf = (result: A): Option.Option => + Option.flatMap(firstErrorOf(result), (error) => stringFieldOf(error, 'name')) + +const failureStackOf = (result: A): Option.Option => + Option.flatMap(firstErrorOf(result), (error) => stringFieldOf(error, 'stack')) + const suiteNamesOf = (suite: A): readonly string[] => Option.match(recordOption(suite), { onNone: (): readonly string[] => [], @@ -182,20 +217,27 @@ const suiteErrorOf = (suite: A): string | undefined => Option.match(recordOption(suite), { onNone: (): string | undefined => undefined, onSome: (record) => - Option.match(firstErrorMessageOf(fieldOf(record, 'result')), { + Option.match(errorTextOfResult(fieldOf(record, 'result')), { onNone: () => suiteErrorOf(fieldOf(record, 'suite')), onSome: (message) => message, }), }) -export const testRecordOf = (test: A): VitestTestRecord => ({ - name: textFieldOf(test, 'name'), - fullTestName: Option.getOrUndefined(stringFieldOf(test, 'fullTestName')), - suiteNames: suiteNamesOf(fieldOf(test, 'suite')), - fileName: Option.getOrUndefined(stringFieldOf(fieldOf(test, 'file'), 'filepath')), - mode: Option.getOrUndefined(stringFieldOf(test, 'mode')), - state: Option.getOrUndefined(stringFieldOf(fieldOf(test, 'result'), 'state')), - durationMs: Option.getOrUndefined(numberFieldOf(fieldOf(test, 'result'), 'duration')), - errorMessage: Option.getOrUndefined(firstErrorMessageOf(fieldOf(test, 'result'))), - suiteErrorMessage: suiteErrorOf(fieldOf(test, 'suite')), -}) +export const testRecordOf = (test: A): VitestTestRecord => { + const result = fieldOf(test, 'result') + const error = firstErrorOf(result) + return { + name: textFieldOf(test, 'name'), + fullTestName: Option.getOrUndefined(stringFieldOf(test, 'fullTestName')), + suiteNames: suiteNamesOf(fieldOf(test, 'suite')), + fileName: Option.getOrUndefined(stringFieldOf(fieldOf(test, 'file'), 'filepath')), + mode: Option.getOrUndefined(stringFieldOf(test, 'mode')), + state: Option.getOrUndefined(stringFieldOf(result, 'state')), + durationMs: Option.getOrUndefined(numberFieldOf(result, 'duration')), + errorMessage: Option.getOrUndefined(errorTextOfResult(result)), + errorName: Option.getOrUndefined(failureNameOf(result)), + errorStack: Option.getOrUndefined(failureStackOf(result)), + errorFrames: Option.getOrUndefined(Option.map(error, failureFramesOf)), + suiteErrorMessage: suiteErrorOf(fieldOf(test, 'suite')), + } +} diff --git a/packages/stryker-js-vitest-runner/src/vitest-run-command.schema.ts b/packages/stryker-js-vitest-runner/src/vitest-run-command.schema.ts index bcb4618d..7bb35601 100644 --- a/packages/stryker-js-vitest-runner/src/vitest-run-command.schema.ts +++ b/packages/stryker-js-vitest-runner/src/vitest-run-command.schema.ts @@ -3,6 +3,15 @@ import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import { Report, TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import * as S from 'effect/Schema' +const VitestSourceOrdinal = S.Int.check(S.isGreaterThanOrEqualTo(1)) + +export const VitestFailureFrame = S.Struct({ + file: S.String, + line: VitestSourceOrdinal, + column: VitestSourceOrdinal, +}) +export type VitestFailureFrame = S.Schema.Type + export const VitestTestRecord = S.Struct({ name: S.String, fullTestName: S.optional(S.String), @@ -12,6 +21,9 @@ export const VitestTestRecord = S.Struct({ state: S.optional(S.String), durationMs: S.optional(Report.NonNegativeFinite), errorMessage: S.optional(S.String), + errorName: S.optional(S.String), + errorStack: S.optional(S.String), + errorFrames: S.Array(VitestFailureFrame).pipe(S.optional), suiteErrorMessage: S.optional(S.String), }) export type VitestTestRecord = S.Schema.Type @@ -19,6 +31,8 @@ export type VitestTestRecord = S.Schema.Type export const VitestFileFailure = S.Struct({ fileName: S.String, message: S.String, + stack: S.optional(S.String), + frames: S.Array(VitestFailureFrame).pipe(S.optional), }) export type VitestFileFailure = S.Schema.Type diff --git a/packages/stryker-js-vitest-runner/src/vitest-test-run.ts b/packages/stryker-js-vitest-runner/src/vitest-test-run.ts index d6ad23ac..92a7c68f 100644 --- a/packages/stryker-js-vitest-runner/src/vitest-test-run.ts +++ b/packages/stryker-js-vitest-runner/src/vitest-test-run.ts @@ -3,7 +3,12 @@ import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import * as Boolean from 'effect/Boolean' import * as Match from 'effect/Match' import * as Option from 'effect/Option' -import type { VitestFileFailure, VitestTestRecord, VitestTestRun } from './vitest-run-command.schema.js' +import type { + VitestFailureFrame, + VitestFileFailure, + VitestTestRecord, + VitestTestRun, +} from './vitest-run-command.schema.js' type TestResultEncoded = (typeof TestRunner.TestResultSchema)['Encoded'] @@ -40,6 +45,50 @@ const testNameOf = (record: VitestTestRecord): string => () => [...record.suiteNames, record.name].join(SUITE_SEPARATOR).trim(), ) +const isInternalFilePath = (file: string): boolean => file.startsWith('node:') || file.startsWith('file://') + +const isNodeModulesPath = (file: string): boolean => file.split(/[/\\]/).includes('node_modules') + +const hasSourcePosition = (frame: VitestFailureFrame): boolean => frame.line >= 1 && frame.column >= 1 + +const isExternalFrame = (frame: VitestFailureFrame): boolean => frame.file.length > 0 && !isInternalFilePath(frame.file) + +const isProjectFileFrame = (frame: VitestFailureFrame): boolean => + isExternalFrame(frame) && !isNodeModulesPath(frame.file) + +const isProjectSourceFrame = (frame: VitestFailureFrame): boolean => + isProjectFileFrame(frame) && hasSourcePosition(frame) + +const locationOf = ( + frames: readonly VitestFailureFrame[] | undefined, + projectRoot: string, +): Option.Option => + Option.flatMap( + Option.fromNullishOr(frames), + (present) => + Option.flatMap( + Option.fromNullishOr(present.find(isProjectSourceFrame)), + (frame) => { + const file = projectRelativePath(stripProjectRoot(frame.file, projectRoot)) + return file.length > 0 ? Option.some({ file, line: frame.line, column: frame.column }) : Option.none() + }, + ), + ) + +const failureEvidenceOf = ( + location: Option.Option, + stack: string | undefined, +): { readonly location?: TestRunner.TestFailureLocation; readonly stack?: string } => ({ + ...Option.match(location, { + onNone: () => ({}), + onSome: (present) => ({ location: present }), + }), + ...Option.match(Option.fromNullishOr(stack), { + onNone: () => ({}), + onSome: (present) => ({ stack: present }), + }), +}) + const testStatusOf = (record: VitestTestRecord): TestRunner.TestStatus => Boolean.match(isSkippedMode(record.mode) || isSkippedMode(record.state), { onTrue: (): TestRunner.TestStatus => 'skipped', @@ -65,6 +114,12 @@ const fileNameFieldOf = (record: VitestTestRecord): { readonly fileName?: string onSome: (fileName) => ({ fileName }), }) +const failureEvidenceOfRecord = ( + record: VitestTestRecord, + projectRoot: string, +): { readonly location?: TestRunner.TestFailureLocation; readonly stack?: string } => + failureEvidenceOf(locationOf(record.errorFrames, projectRoot), record.errorStack) + const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEncoded => { const base = { id: projectRelativeId(`${fileNameOf(record)}#${testNameOf(record)}`, projectRoot), @@ -75,28 +130,42 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco return Match.value(testStatusOf(record)).pipe( Match.when( 'failed', - (): TestResultEncoded => ({ ...base, status: 'failed', failureMessage: failureMessageOf(record) }), + (): TestResultEncoded => ({ + ...base, + status: 'failed', + failureMessage: failureMessageOf(record), + ...failureEvidenceOfRecord(record, projectRoot), + }), ), Match.when( 'skipped', (): TestResultEncoded => Option.match(Option.fromNullishOr(record.suiteErrorMessage), { onNone: (): TestResultEncoded => ({ ...base, status: 'skipped' }), - onSome: (failureMessage): TestResultEncoded => ({ ...base, status: 'failed', failureMessage }), + onSome: (failureMessage): TestResultEncoded => ({ + ...base, + status: 'failed', + failureMessage, + ...failureEvidenceOfRecord(record, projectRoot), + }), }), ), Match.orElse((): TestResultEncoded => ({ ...base, status: 'success' })), ) } -const fileFailureResultOf = (failure: VitestFileFailure, projectRoot: string): TestResultEncoded => ({ - id: projectRelativeId(`${failure.fileName}#${failure.fileName}`, projectRoot), - name: failure.fileName, - timeSpentMs: 0, - status: 'failed', - failureMessage: failure.message, - fileName: failure.fileName, -}) +const fileFailureResultOf = (failure: VitestFileFailure, projectRoot: string): TestResultEncoded => { + const file = projectRelativePath(stripProjectRoot(failure.fileName, projectRoot)) + return { + id: `${file}#${file}`, + name: file, + timeSpentMs: 0, + status: 'failed', + failureMessage: failure.message, + fileName: failure.fileName, + ...failureEvidenceOf(locationOf(failure.frames, projectRoot), failure.stack), + } +} export const interpretVitestTestRun = (run: VitestTestRun): readonly TestResultEncoded[] => [ ...run.records.map((record) => resultOf(record, run.projectRoot)), @@ -116,6 +185,46 @@ if (import.meta.vitest !== void 0) { fileFailures: [], }) + type FailedTestResultEncoded = Extract + + const isFailedResult = (only: TestResultEncoded): only is FailedTestResultEncoded => only.status === 'failed' + + const locationOfFailure = ( + only: TestResultEncoded, + ): Option.Option> => + Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.location)) + + const positionsEqual = ( + location: NonNullable, + line: number, + column: number, + ): boolean => location.line === line && location.column === column + + const locatedAt = (only: TestResultEncoded, file: string, line: number, column: number): boolean => + Option.exists( + locationOfFailure(only), + (location) => location.file === file && positionsEqual(location, line, column), + ) + + const keptStackWithoutLocation = (only: TestResultEncoded, stack: string): boolean => + Option.exists( + Option.liftPredicate(only, isFailedResult), + (failed) => failed.stack === stack && failed.location === undefined, + ) + + const hasFailureMessage = (only: TestResultEncoded): boolean => 'failureMessage' in only + + const hasOptionalEvidence = (only: TestResultEncoded): boolean => 'location' in only || 'stack' in only + + const unfailedShapeIsBare = (only: TestResultEncoded): boolean => + !hasFailureMessage(only) && !hasOptionalEvidence(only) + + const hasWidenedFailureShape = (only: TestResultEncoded): boolean => + Boolean.match(isFailedResult(only), { + onTrue: () => hasFailureMessage(only), + onFalse: () => unfailedShapeIsBare(only), + }) + it.prop( '∀r_TestRecord_≡FullTestNameOrSuitePath', { of: [Records.VitestTestRecord, S.String], subject: interpretVitestTestRun }, @@ -136,11 +245,11 @@ if (import.meta.vitest !== void 0) { ) it.prop( - '∀r_TestRecord_≡FailureMessageIffFailed', + '∀r_TestRecord_≡WidenedFailureShape', { of: [Records.VitestTestRecord, S.String], subject: interpretVitestTestRun }, (subject, [record, projectRoot]) => { const [only] = subject(singleRecordRun(record, projectRoot)) - return (only.status === 'failed') === ('failureMessage' in only) + return hasWidenedFailureShape(only) }, ) @@ -162,12 +271,82 @@ if (import.meta.vitest !== void 0) { }, ) + const FileStem = S.String.check(S.isPattern(/^[a-z]{1,8}$/)) + it.prop( - '∀f_FileFailure_≡NamedAfterTheFile', - { of: [Records.VitestFileFailure, S.String], subject: interpretVitestTestRun }, - (subject, [failure, projectRoot]) => { - const [only] = subject({ projectRoot, records: [], fileFailures: [failure] }) - return only.name === failure.fileName && only.fileName === failure.fileName + '∀f_FileFailure_≡NamedAfterTheProjectRelativeFile', + { of: [FileStem], subject: interpretVitestTestRun }, + (subject, [stem]) => { + const [only] = subject({ + projectRoot: '/project', + records: [], + fileFailures: [{ fileName: `/project/tests/${stem}.spec.ts`, message: 'load failed' }], + }) + return only.name === `tests/${stem}.spec.ts` && only.id === `tests/${stem}.spec.ts#tests/${stem}.spec.ts` + }, + ) + + const SourceOrdinal = S.Int.check(S.isGreaterThanOrEqualTo(1)) + + const failingRecord = (overrides: Partial): VitestTestRecord => ({ + name: 'fails', + suiteNames: [], + ...overrides, + }) + + it.prop( + '∀r_ProjectFrame_≡LocatedThere', + { of: [SourceOrdinal, SourceOrdinal], subject: interpretVitestTestRun }, + (subject, [line, column]) => { + const record = failingRecord({ + fileName: 'src/a.ts', + errorMessage: 'boom', + errorStack: 'Error: boom', + errorFrames: [ + { file: '/project/node_modules/vitest/dist/runner.js', line: 1, column: 1 }, + { file: '/project/src/a.ts', line, column }, + ], + }) + const [only] = subject(singleRecordRun(record, '/project')) + return locatedAt(only, 'src/a.ts', line, column) + }, + ) + + it.prop( + '∀r_NonProjectFrames_≡StackKeptWithoutLocation', + { of: [S.String, SourceOrdinal], subject: interpretVitestTestRun }, + (subject, [stack, line]) => { + const record = failingRecord({ + errorMessage: 'boom', + errorStack: stack, + errorFrames: [ + { file: 'node:internal/timers', line, column: line }, + { file: '/project/node_modules/x/y.js', line, column: line }, + ], + }) + const [only] = subject(singleRecordRun(record, '/project')) + return keptStackWithoutLocation(only, stack) + }, + ) + + it.prop( + '∀f_ProjectFrame_≡LocatedThere', + { of: [SourceOrdinal, SourceOrdinal], subject: interpretVitestTestRun }, + (subject, [line, column]) => { + const [only] = subject({ + projectRoot: '/project', + records: [], + fileFailures: [{ + fileName: 'tests/a.spec.ts', + message: 'load failed', + stack: 'SyntaxError: bad', + frames: [ + { file: '/project/node_modules/vite/dist/x.js', line: 4, column: 2 }, + { file: '/project/tests/a.spec.ts', line, column }, + ], + }], + }) + return locatedAt(only, 'tests/a.spec.ts', line, column) }, ) } diff --git a/packages/stryker-js/etc/stryker-js.api.md b/packages/stryker-js/etc/stryker-js.api.md index e9d0a4e4..1e8994a6 100644 --- a/packages/stryker-js/etc/stryker-js.api.md +++ b/packages/stryker-js/etc/stryker-js.api.md @@ -16,7 +16,7 @@ export namespace Configuration { // @public (undocumented) export namespace Engine { - export { DryRunDone, EnginePorts, IdGenerator, IdGeneratorShape, IncrementalReportSchema, InstrumentDone, MutationTestDone, PrepareDone, PrepareExecutorArgs, Project, ProjectFile, RunEnvironment, RunEnvironmentShape, RunStageServices, SandboxHandle, StageError, StageServices, StrykerError, StrykerRun, TestCoverage, WiredRunLayer, mutationTestCell, nodePlatformLayer, strykerCell }; + export { DryRunDone, EnginePorts, IdGenerator, IdGeneratorShape, IncrementalReportSchema, InstrumentDone, MutationTestDone, PrepareDone, PrepareExecutorArgs, Project, ProjectFile, RunEnvironment, RunEnvironmentShape, RunFailure, RunStageServices, SandboxHandle, StageServices, StrykerError, StrykerRun, TestCoverage, WiredRunLayer, mutationTestCell, nodePlatformLayer, strykerCell }; } // @public (undocumented) diff --git a/packages/stryker-js/src/Checker/Checker.blueprint.ts b/packages/stryker-js/src/Checker/Checker.blueprint.ts index a0609b4e..f580a376 100644 --- a/packages/stryker-js/src/Checker/Checker.blueprint.ts +++ b/packages/stryker-js/src/Checker/Checker.blueprint.ts @@ -52,6 +52,7 @@ const acquire = Effect.fnUntraced(function*(spec: CheckerSpec) { workingDirectory: spec.workingDirectory, execArgv: [...nodeArgsOf(spec.options)], tempDirPrefix: TEMP_DIR_PREFIX, + workerKind: 'checker', }).pipe( Effect.mapError((error) => Match.value(error).pipe( diff --git a/packages/stryker-js/src/Checker/Checker.handle.ts b/packages/stryker-js/src/Checker/Checker.handle.ts index 794edf10..8a63d485 100644 --- a/packages/stryker-js/src/Checker/Checker.handle.ts +++ b/packages/stryker-js/src/Checker/Checker.handle.ts @@ -63,7 +63,7 @@ export interface CheckerResourceService { } export const connectionCrashed = (cause: string): ChildProcessCrashedError => - ChildProcessCrashedError.make({ pid: 0, exit: { _tag: 'Code', code: 1 }, cause }) + ChildProcessCrashedError.make({ pid: 0, exit: { _tag: 'Code', code: 1 }, workerKind: 'checker', cause }) const recordCheckerCall = ( span: SpanTaxonomy.SpanMember, diff --git a/packages/stryker-js/src/Checker/checker-pool.blueprint.ts b/packages/stryker-js/src/Checker/checker-pool.blueprint.ts index 95d2727a..ae40f53a 100644 --- a/packages/stryker-js/src/Checker/checker-pool.blueprint.ts +++ b/packages/stryker-js/src/Checker/checker-pool.blueprint.ts @@ -10,7 +10,7 @@ import type * as Scope from 'effect/Scope' import type { LoadedPlugins } from '../Plugins.schema.js' import { PluginNotFoundError } from '../PluginsError.schema.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { ConfiguredPluginModulePath, resolveConfiguredPlugin, @@ -36,15 +36,15 @@ const CHECKER_ACQUIRE_RETRIES = 2 const checkerWorkerSpawnOf = ( loaded: Pick, configured: ConfiguredPluginModulePath, -): Effect.Effect => +): Effect.Effect => Effect.mapError( Effect.fromResult( resolveConfiguredPlugin(WorkerSpawnCommand.make({ sources: loaded.pluginSources, kind: 'Checker', configured })), ), (missing) => - StageError.make({ - stage: 'mutationTest', - reason: missing.reason, + RunFailure.make({ + evidence: { _tag: 'PluginNotFound', stage: 'config', descriptor: missing.descriptor }, + detail: missing.reason, cause: PluginNotFoundError.make({ descriptor: missing.descriptor }), }), ) diff --git a/packages/stryker-js/src/Checker/checker-pool.handle.ts b/packages/stryker-js/src/Checker/checker-pool.handle.ts index 7e20b105..27d53ee9 100644 --- a/packages/stryker-js/src/Checker/checker-pool.handle.ts +++ b/packages/stryker-js/src/Checker/checker-pool.handle.ts @@ -14,7 +14,7 @@ import * as Scope from 'effect/Scope' import * as Stream from 'effect/Stream' import type { CheckerContractBroken } from '../admit-checker-answer.workflow.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import type { CheckerCrash, CheckerResourceService } from './Checker.handle.js' import { checkPlans as checkPlansWithChecker, groupPlans as groupPlansWithChecker } from './Checker.plans.js' import { @@ -29,7 +29,7 @@ export type TypeId = typeof TypeId export type CheckerSlot = { readonly checkerName: string; readonly checker: CheckerResourceService }[] -export type CheckerPool = Pool.Pool +export type CheckerPool = Pool.Pool export interface CheckedPlans { readonly passedPlans: readonly Mutant.MutantRunPlan[] @@ -44,31 +44,36 @@ export const isCheckerPoolHandle = CheckerPoolHandle.is export const makeCheckerPoolHandle = (pool: CheckerPool): CheckerPoolHandle => CheckerPoolHandle.make({}, pool) -export const isCheckerCrash = (error: StageError | CheckerCrash): boolean => +export const isCheckerCrash = (error: RunFailure | CheckerCrash): boolean => Match.value(error).pipe( Match.tag('ChildProcessCrashedError', 'OutOfMemoryError', () => true), Match.orElse(() => false), ) -const checkerBreachToStageError = (error: CheckerContractBroken | Checker.CheckerFailed): StageError => +const checkerBreachToRunFailure = (error: CheckerContractBroken | Checker.CheckerFailed): RunFailure => Match.value(error).pipe( Match.tag( 'CheckerFailed', - (failed) => StageError.make({ stage: 'mutationTest', reason: failed.cause, cause: failed }), + (failed) => + RunFailure.make({ + evidence: { _tag: 'CheckerFailed', stage: 'check', checker: failed.checkerName }, + detail: failed.cause, + cause: failed, + }), ), Match.tag('CheckerAnsweredUnrequested', (breach) => - StageError.make({ - stage: 'mutationTest', - reason: + RunFailure.make({ + evidence: { _tag: 'CheckerFailed', stage: 'check', checker: breach.checkerName }, + detail: `Checker "${breach.checkerName}" answered about mutants it was not asked about (${breach.phase} phase): ${ breach.unrequestedIds.join(', ') }`, cause: breach, })), Match.tag('CheckerSkippedRequested', (breach) => - StageError.make({ - stage: 'mutationTest', - reason: `Checker "${breach.checkerName}" skipped requested mutants (${breach.phase} phase): ${ + RunFailure.make({ + evidence: { _tag: 'CheckerFailed', stage: 'check', checker: breach.checkerName }, + detail: `Checker "${breach.checkerName}" skipped requested mutants (${breach.phase} phase): ${ breach.missingIds.join(', ') }`, cause: breach, @@ -88,7 +93,7 @@ const onCheckerSlot = ( run: ( checker: CheckerResourceService, ) => Effect.Effect, -): Effect.Effect => +): Effect.Effect => Pool.use(pool, (slot) => Option.match(Option.fromUndefinedOr(slot[checkerIndex]), { onNone: () => Effect.die(new Error(`checker slot has no entry at index ${checkerIndex}`)), @@ -97,9 +102,9 @@ const onCheckerSlot = ( Effect.catchTags({ OutOfMemoryError: (error) => invalidateSlot(pool, slot, error), ChildProcessCrashedError: (error) => invalidateSlot(pool, slot, error), - CheckerFailed: (error) => error.pipe(checkerBreachToStageError, Effect.fail), - CheckerAnsweredUnrequested: (error) => error.pipe(checkerBreachToStageError, Effect.fail), - CheckerSkippedRequested: (error) => error.pipe(checkerBreachToStageError, Effect.fail), + CheckerFailed: (error) => error.pipe(checkerBreachToRunFailure, Effect.fail), + CheckerAnsweredUnrequested: (error) => error.pipe(checkerBreachToRunFailure, Effect.fail), + CheckerSkippedRequested: (error) => error.pipe(checkerBreachToRunFailure, Effect.fail), }), ), })) @@ -135,12 +140,12 @@ export const splitCheckedPlans = Effect.fn(SpanTaxonomy.Spans.checkerPoolSplitCh return { passedPlans, failedChecks } }) -const checkerNamesOf = (pool: CheckerPool): Effect.Effect => +const checkerNamesOf = (pool: CheckerPool): Effect.Effect => Pool.use(pool, (slot) => Effect.succeed(slot.map(({ checkerName }) => checkerName))) const failedElement = ( failedChecks: readonly (readonly [Mutant.MutantRunPlan, Checker.FailedCheckResult])[], -): Stream.Stream => +): Stream.Stream => Option.match(Array.head(failedChecks), { onNone: () => Stream.empty, onSome: () => Stream.succeed({ passedPlans: noPassedPlans, failedChecks }), @@ -151,7 +156,7 @@ const checkedGroupsFor = ( checkerNames: readonly string[], checkerIndex: number, plans: readonly Mutant.MutantRunPlan[], -): Stream.Stream => +): Stream.Stream => Option.match(Array.get(checkerNames, checkerIndex), { onNone: () => Stream.succeed({ passedPlans: plans, failedChecks: noFailedChecks }), onSome: (checkerName) => @@ -182,17 +187,17 @@ const checkedGroupsFor = ( export const checkPlansStream: { ( plans: readonly Mutant.MutantRunPlan[], - ): (checkerPool: CheckerPoolHandle | undefined) => Stream.Stream + ): (checkerPool: CheckerPoolHandle | undefined) => Stream.Stream ( checkerPool: CheckerPoolHandle | undefined, plans: readonly Mutant.MutantRunPlan[], - ): Stream.Stream + ): Stream.Stream } = dual( 2, ( checkerPool: CheckerPoolHandle | undefined, plans: readonly Mutant.MutantRunPlan[], - ): Stream.Stream => + ): Stream.Stream => Option.fromNullishOr(checkerPool).pipe( Option.match({ onNone: () => Stream.succeed({ passedPlans: plans, failedChecks: noFailedChecks }), @@ -230,18 +235,18 @@ export const runCheckedPlans: { ( execution: CheckedPlansExecution, ): ( - self: Stream.Stream, - ) => Stream.Stream + self: Stream.Stream, + ) => Stream.Stream ( - self: Stream.Stream, + self: Stream.Stream, execution: CheckedPlansExecution, - ): Stream.Stream + ): Stream.Stream } = dual( 2, ( - self: Stream.Stream, + self: Stream.Stream, execution: CheckedPlansExecution, - ): Stream.Stream => + ): Stream.Stream => self.pipe( Stream.flatMap(({ passedPlans, failedChecks }) => Stream.fromIterable<() => Effect.Effect>([ diff --git a/packages/stryker-js/src/ConfigError.schema.ts b/packages/stryker-js/src/ConfigError.schema.ts index aec54a94..9b219195 100644 --- a/packages/stryker-js/src/ConfigError.schema.ts +++ b/packages/stryker-js/src/ConfigError.schema.ts @@ -1,3 +1,4 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import * as S from 'effect/Schema' export class ConfigFileNotFoundError extends S.TaggedError()( @@ -6,7 +7,9 @@ export class ConfigFileNotFoundError extends S.TaggedError()('C message: S.String, cause: S.Defect(), }) { - readonly exitClass = 'ConfigError' as const + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'ConfigInvalid', stage: 'config', detail: this.message } + } } export class ConfigModuleUnloadable extends S.TaggedError()('ConfigModuleUnloadable', { @@ -68,13 +79,17 @@ export class ConfigModuleUnloadable extends S.TaggedError()('ConfigError', { message: S.String, }) { - readonly exitClass = 'ConfigError' as const + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'ConfigInvalid', stage: 'config', detail: this.message } + } } export type ConfigReadError = diff --git a/packages/stryker-js/src/Engine/mod.ts b/packages/stryker-js/src/Engine/mod.ts index f9a9aca9..ce25b245 100644 --- a/packages/stryker-js/src/Engine/mod.ts +++ b/packages/stryker-js/src/Engine/mod.ts @@ -1,7 +1,7 @@ export { nodePlatformLayer } from '../drivers/node.js' export { IncrementalReportSchema } from '../IncrementalReport.schema.js' export type { Project, ProjectFile } from '../Project.schema.js' -export { StageError } from '../Run.schema.js' +export { RunFailure } from '../Run.schema.js' export type { DryRunDone } from '../run/dry-run.cell.js' export type { StrykerRun } from '../run/host.service.js' export type { InstrumentDone } from '../run/instrument.cell.js' diff --git a/packages/stryker-js/src/PluginsError.schema.ts b/packages/stryker-js/src/PluginsError.schema.ts index aed28d19..80b68c32 100644 --- a/packages/stryker-js/src/PluginsError.schema.ts +++ b/packages/stryker-js/src/PluginsError.schema.ts @@ -1,12 +1,14 @@ import { Schema as S } from 'effect' +import * as Predicate from 'effect/Predicate' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' -import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' export class PluginNotFoundError extends S.TaggedError()('PluginNotFoundError', { descriptor: S.String, }) { - readonly exitClass = 'ConfigError' as const + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'PluginNotFound', stage: 'config', descriptor: this.descriptor } + } override get message(): string { return `Plugin "${this.descriptor}" was not found` @@ -16,14 +18,6 @@ export class PluginNotFoundError extends S.TaggedError()('P export const PeerFailureTag = S.Literals(['PeerMissing', 'PeerVersionUnsupported', 'PeerUnrecognized']) export type PeerFailureTag = typeof PeerFailureTag.Type -const FAILURE_EXIT_CLASS: Record = { - PeerMissing: 'ConfigError', - PeerVersionUnsupported: 'ConfigError', - PeerUnrecognized: 'ConfigError', - InvalidContribution: 'ConfigError', - ImportFailed: 'InternalError', -} - export class PluginLoadRefusedError extends S.TaggedError()( 'PluginLoadRefusedError', { @@ -31,8 +25,10 @@ export class PluginLoadRefusedError extends S.TaggedError Effect.flatMap(S.decodeEffect(RerunRefused)(refused), (decoded) => Effect.fail(decoded)), - CommandRejected: ({ issue }) => - Effect.fail(RerunRefused.make({ exitClass: 'ConfigError', unknownIds: [], reason: issue })), + CommandRejected: ({ issue }) => Effect.fail(RerunRefused.make({ unknownIds: [], reason: issue })), }) diff --git a/packages/stryker-js/src/Rerun/admit-mutant-rerun.workflow.ts b/packages/stryker-js/src/Rerun/admit-mutant-rerun.workflow.ts index 1fcefd92..2833c35f 100644 --- a/packages/stryker-js/src/Rerun/admit-mutant-rerun.workflow.ts +++ b/packages/stryker-js/src/Rerun/admit-mutant-rerun.workflow.ts @@ -1,4 +1,5 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Option from 'effect/Option' @@ -29,11 +30,14 @@ export class RerunAdmitted extends S.TaggedClass()('RerunAdmitted } export class RerunRefused extends S.TaggedClass()('RerunRefused', { - exitClass: S.Literal('ConfigError'), unknownIds: S.Array(Mutant.MutantId), reason: S.String, }) { readonly [MutantRerunTypeId] = MutantRerunTypeId + + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'ConfigInvalid', stage: 'config', detail: this.reason } + } } export const MutantRerunAdmission = S.Union([RerunAdmitted, RerunRefused]) @@ -57,7 +61,6 @@ const mutateSpansOf = (command: AdmitMutantRerunCommand): ReadonlyArray const refusalOf = (command: AdmitMutantRerunCommand, unknownIds: ReadonlyArray): RerunRefused => RerunRefused.make({ - exitClass: 'ConfigError', unknownIds, reason: `Unknown mutant id(s): ${ Arr.join(unknownIds, ', ') diff --git a/packages/stryker-js/src/Run.schema.ts b/packages/stryker-js/src/Run.schema.ts index 11bf395e..b1aadbb5 100644 --- a/packages/stryker-js/src/Run.schema.ts +++ b/packages/stryker-js/src/Run.schema.ts @@ -1,46 +1,12 @@ -import type { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' -import * as Match from 'effect/Match' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import * as S from 'effect/Schema' -const TypeId = '~stryker/mutation-run/StageError' as const - -export class StageError extends S.TaggedError(TypeId)('StageError', { - stage: S.Literals(['prepare', 'instrument', 'dryRun', 'dryRunNoTests', 'mutationTest']), - reason: S.String, +export class RunFailure extends S.TaggedError()('RunFailure', { + evidence: FailureRecord.FailureEvidence, + detail: S.String, cause: S.optional(S.Defect()), - command: S.optional(S.String), -}) { - readonly [TypeId] = TypeId - - get exitClass(): Plugin.ExitClass { - return STAGE_PRESENTATION[this.stage].exitClass - } - - override get message(): string { - const failure = `${STAGE_PRESENTATION[this.stage].label} failed: ${this.reason}` - return Match.value(this.command).pipe( - Match.when(Match.nonEmptyString, (present) => `${failure} (command: ${present})`), - Match.orElse(() => failure), - ) - } -} - -const STAGE_PRESENTATION: Record< - StageError['stage'], - { readonly label: string; readonly exitClass: Plugin.ExitClass } -> = { - prepare: { label: 'Prepare', exitClass: 'ConfigError' }, - instrument: { label: 'Instrument', exitClass: 'RuntimeError' }, - dryRun: { label: 'Dry run', exitClass: 'RuntimeError' }, - dryRunNoTests: { label: 'Dry run', exitClass: 'ConfigError' }, - mutationTest: { label: 'Mutation testing', exitClass: 'RuntimeError' }, -} - -export class PrepareError extends S.TaggedError()('PrepareError', { - stage: S.Literal('prepare'), - reason: S.String, }) { override get message(): string { - return this.reason + return this.detail } } diff --git a/packages/stryker-js/src/RunOutcomeCommand.schema.ts b/packages/stryker-js/src/RunOutcomeCommand.schema.ts index bae1c739..c3f6bf61 100644 --- a/packages/stryker-js/src/RunOutcomeCommand.schema.ts +++ b/packages/stryker-js/src/RunOutcomeCommand.schema.ts @@ -1,64 +1,36 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' -import { Plugin, Report } from '@systemfsoftware/stryker-js-plugin-interface' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as S from 'effect/Schema' export const RunSucceededClean = S.TaggedStruct('RunSucceededClean', {}) export const RunSucceededVerdict = S.TaggedStruct('RunSucceededVerdict', { exitClass: Plugin.ExitClass, - diagnostic: S.NullOr(S.String), }) -export const RunInterruptedObservation = S.TaggedStruct('RunInterruptedObservation', {}) +export const RunHelpObservation = S.TaggedStruct('RunHelpObservation', {}) -export const RunHelpObservation = S.TaggedStruct('RunHelpObservation', { - errorCount: Report.NonNegativeInt, - unrecognized: S.NullOr(S.String), +export const ObservedFailure = S.Struct({ + record: FailureRecord.FailureRecord, + exitCode: Plugin.ExitCode, }) +export type ObservedFailure = typeof ObservedFailure.Type -export const RunCliErrorObservation = S.TaggedStruct('RunCliErrorObservation', { - unrecognized: S.NullOr(S.String), -}) - -export const RunSurvivorsRejectedObservation = S.TaggedStruct('RunSurvivorsRejectedObservation', { - reason: S.Literals(['no-report', 'mismatch']), - diagnostic: S.NullOr(S.String), -}) - -export const RunSchemaErrorObservation = S.TaggedStruct('RunSchemaErrorObservation', { - configDetail: S.NullOr(S.String), -}) - -export const RunClassedObservation = S.TaggedStruct('RunClassedObservation', { - exitClass: Plugin.ExitClass, - configDetail: S.NullOr(S.String), - diagnostic: S.NullOr(S.String), -}) - -export const RunGenericFailureObservation = S.TaggedStruct('RunGenericFailureObservation', { - diagnostic: S.NullOr(S.String), +export const RunFailedObservation = S.TaggedStruct('RunFailedObservation', { + failures: S.NonEmptyArray(ObservedFailure), }) export const RunOutcomeObservation = S.Union([ RunSucceededClean, RunSucceededVerdict, - RunInterruptedObservation, RunHelpObservation, - RunCliErrorObservation, - RunSurvivorsRejectedObservation, - RunSchemaErrorObservation, - RunClassedObservation, - RunGenericFailureObservation, + RunFailedObservation, ]) export type RunOutcomeObservation = typeof RunOutcomeObservation.Type export type RunSucceededVerdict = typeof RunSucceededVerdict.Type -export type RunHelpObservation = typeof RunHelpObservation.Type -export type RunCliErrorObservation = typeof RunCliErrorObservation.Type -export type RunSurvivorsRejectedObservation = typeof RunSurvivorsRejectedObservation.Type -export type RunSchemaErrorObservation = typeof RunSchemaErrorObservation.Type -export type RunClassedObservation = typeof RunClassedObservation.Type -export type RunGenericFailureObservation = typeof RunGenericFailureObservation.Type +export type RunFailedObservation = typeof RunFailedObservation.Type export class RunOutcomeCommand extends S.TaggedClass()('RunOutcomeCommand', { observation: RunOutcomeObservation, diff --git a/packages/stryker-js/src/TestRunner.blueprint.ts b/packages/stryker-js/src/TestRunner.blueprint.ts index bdd4b4d7..29339d75 100644 --- a/packages/stryker-js/src/TestRunner.blueprint.ts +++ b/packages/stryker-js/src/TestRunner.blueprint.ts @@ -107,6 +107,7 @@ const scopedOf: ( workingDirectory: params.sandboxWorkingDirectory, execArgv: [...execArgv], tempDirPrefix: 'stryker-test-runner-', + workerKind: 'testRunner', env: { NODE_ENV: 'test', VITEST: '1', diff --git a/packages/stryker-js/src/Worker.schema.ts b/packages/stryker-js/src/Worker.schema.ts index a7bbdf2a..6e217ff8 100644 --- a/packages/stryker-js/src/Worker.schema.ts +++ b/packages/stryker-js/src/Worker.schema.ts @@ -1,4 +1,5 @@ /// +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Match, Schema as S } from 'effect' // --------------------------------------------------------------------------- @@ -46,11 +47,11 @@ export class ChildProcessCrashedError extends S.TaggedError ({ exitCode: code, signal: null }), + Signal: ({ signal }) => ({ exitCode: null, signal }), + }) + return { + _tag: 'WorkerCrashed', + stage: 'mutationTest', + workerKind: this.workerKind, + pid: this.pid, + exitCode, + signal, + } + } } export class OutOfMemoryError extends S.TaggedError()('OutOfMemoryError', { pid: ProcessId, exitCode: ChildExitCode, + workerKind: FailureRecord.WorkerKind, }) { readonly [WorkerExitTypeId] = WorkerExitTypeId - readonly exitClass = 'RuntimeError' as const override get message(): string { return `Worker process ${this.pid} ran out of memory (exit code ${this.exitCode})` } + + get evidence(): FailureRecord.FailureEvidence { + return { + _tag: 'WorkerOutOfMemory', + stage: 'mutationTest', + workerKind: this.workerKind, + pid: this.pid, + exitCode: this.exitCode, + } + } } export type WorkerExit = ChildProcessCrashedError | OutOfMemoryError @@ -81,13 +107,21 @@ export class WorkerBootTimeoutError extends S.TaggedError> | undefined } diff --git a/packages/stryker-js/src/__tests__/admit-mutation-test.workflow.property.test.ts b/packages/stryker-js/src/__tests__/admit-mutation-test.workflow.property.test.ts index cce6d6e8..1870f18e 100644 --- a/packages/stryker-js/src/__tests__/admit-mutation-test.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/admit-mutation-test.workflow.property.test.ts @@ -1,15 +1,16 @@ import { describe, it } from '@systemfsoftware/vitest' +import * as Predicate from 'effect/Predicate' import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { admitMutationTest, MutationTestDryRunOnly, - MutationTestError, MutationTestNoTests, MutationTestProceed, } from '../admit-mutation-test.workflow.js' import { MutationTestCommand } from '../MutationTest.schema.js' +import { RunFailure } from '../Run.schema.js' describe('admitMutationTest', () => { it.prop( @@ -18,7 +19,10 @@ describe('admitMutationTest', () => { (subject, [command]) => { const result = subject(command) if (command.testCount < 0) { - return Result.isFailure(result) && S.is(MutationTestError)(result.failure) + return Result.isFailure(result) && + S.is(RunFailure)(result.failure) && + Predicate.isTagged(result.failure.evidence, 'InvariantBroken') && + result.failure.evidence.stage === 'mutationTest' } if (command.dryRunOnly) { return Result.isSuccess(result) && S.is(MutationTestDryRunOnly)(result.success) diff --git a/packages/stryker-js/src/__tests__/checker-pool.workflow.property.test.ts b/packages/stryker-js/src/__tests__/checker-pool.workflow.property.test.ts index c72e5913..9bc8899d 100644 --- a/packages/stryker-js/src/__tests__/checker-pool.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/checker-pool.workflow.property.test.ts @@ -38,7 +38,7 @@ import { partitionCheckedPlans, PartitionCheckedPlansCommand, } from '../Checker/partition-checked-plans.workflow.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { ChildProcessCrashedError, OutOfMemoryError } from '../Worker.schema.js' type CheckedPlans = readonly (readonly [Mutant.MutantRunPlan, Checker.CheckResult])[] @@ -113,34 +113,34 @@ const checkerSlotOf = (checkerName: string, checker: CheckerResourceService): Ch const checkerSlotPoolOf = ( size: number, - acquire: Effect.Effect, + acquire: Effect.Effect, ): Effect.Effect => - Pool.make({ + Pool.make({ acquire, size, }) -const causeTagOf = (stageError: StageError): string => +const causeTagOf = (runFailure: RunFailure): string => Option.match( - Option.flatMap(Option.fromNullishOr(stageError.cause), S.decodeUnknownOption(S.Struct({ _tag: S.String }))), + Option.flatMap(Option.fromNullishOr(runFailure.cause), S.decodeUnknownOption(S.Struct({ _tag: S.String }))), { onNone: () => 'none', onSome: (decoded) => decoded._tag }, ) -const stageErrorShapeOf = (error: StageError | CheckerCrash) => +const runFailureShapeOf = (error: RunFailure | CheckerCrash) => Match.value(error).pipe( - Match.tag('StageError', (stageError) => ({ - tag: stageError._tag, - stage: stageError.stage, - cause: causeTagOf(stageError), + Match.tag('RunFailure', (runFailure) => ({ + tag: runFailure._tag, + stage: runFailure.evidence.stage, + cause: causeTagOf(runFailure), })), Match.orElse((crash) => ({ tag: crash._tag, stage: 'none', cause: 'none' })), ) -const crashTagOf = (error: StageError | CheckerCrash): string => +const crashTagOf = (error: RunFailure | CheckerCrash): string => Match.value(error).pipe( Match.tag('OutOfMemoryError', () => 'OutOfMemoryError'), Match.tag('ChildProcessCrashedError', () => 'ChildProcessCrashedError'), - Match.orElse(() => 'StageError'), + Match.orElse(() => 'RunFailure'), ) const poolBoundVerdictOf = (observed: { @@ -167,7 +167,7 @@ const crashVerdictOf = (observed: { const breachVerdictOf = ( observed: { readonly tag: string; readonly stage: string; readonly cause: string }, breachTag: string, -) => holds([observed.tag === 'StageError', observed.stage === 'mutationTest', observed.cause === breachTag]) +) => holds([observed.tag === 'RunFailure', observed.stage === 'check', observed.cause === breachTag]) const backgroundReleaseVerdictOf = (observed: { readonly beforeRelease: number @@ -185,8 +185,13 @@ const groupPlansOf = (prefix: string, seed: number) => { const crashOf = (tag: string) => tag === 'OutOfMemoryError' - ? OutOfMemoryError.make({ pid: 1, exitCode: 137 }) - : ChildProcessCrashedError.make({ pid: 2, exit: { _tag: 'Code', code: 1 }, cause: 'the checker died' }) + ? OutOfMemoryError.make({ pid: 1, exitCode: 137, workerKind: 'checker' }) + : ChildProcessCrashedError.make({ + pid: 2, + exit: { _tag: 'Code', code: 1 }, + workerKind: 'checker', + cause: 'the checker died', + }) const resultOf = (outcome: CheckedHistoryEntry['outcome']): Checker.CheckResult => outcome === 'passed' ? { status: 'passed' } : { status: 'compileError', reason: outcome } @@ -294,7 +299,7 @@ describe('checker pool', () => { ) it.effect.prop( - '∀breach_CheckerBreach_≡StageError', + '∀breach_CheckerBreach_≡RunFailure', { of: [S.Literals(['CheckerFailed', 'CheckerAnsweredUnrequested', 'CheckerSkippedRequested'])], subject: checkPlans, @@ -324,7 +329,7 @@ describe('checker pool', () => { }) const pool = yield* checkerSlotPoolOf(1, Effect.succeed(checkerSlotOf('c', checker))) const error = yield* subject(makeCheckerPoolHandle(pool), plans).pipe(Effect.flip) - return breachVerdictOf(stageErrorShapeOf(error), breachTag) + return breachVerdictOf(runFailureShapeOf(error), breachTag) }), ) @@ -353,7 +358,12 @@ describe('checker pool', () => { Effect.gen(function*() { const failed = Effect.andThen( subject(acquire), - Effect.fail(StageError.make({ stage: 'mutationTest', reason: 'the checks failed' })), + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'TestRunnerFailed', stage: 'mutationTest' }, + detail: 'the checks failed', + }), + ), ) const outcome = yield* Effect.scoped(failed).pipe(Effect.exit) const releasedAfter = yield* Ref.get(released) diff --git a/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts b/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts index 08007a47..70fafdc8 100644 --- a/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts @@ -1,103 +1,103 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { describe, it } from '@systemfsoftware/vitest' -import * as Match from 'effect/Match' +import * as Arr from 'effect/Array' +import * as Cause from 'effect/Cause' +import * as Exit from 'effect/Exit' +import * as Predicate from 'effect/Predicate' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -import { - classifyRunOutcome, - RunConfigFailed, - RunFailed, - RunInterrupted, - RunOk, - type RunOutcomeDecision, - type RunOutcomeError, - RunParseFailed, - RunSurvivorsRejected, -} from '../classify-run-outcome.workflow.js' -import { RunOutcomeCommand } from '../RunOutcomeCommand.schema.js' +import { classifyRunOutcome, RunFailed, type RunOutcomeDecision } from '../classify-run-outcome.workflow.js' +import { runOutcomeCommandOf } from '../conclude-run.js' -const classCode = (exitClass: 'VerdictFail' | 'ConfigError' | 'RuntimeError' | 'InternalError'): number => { - if (exitClass === 'VerdictFail') { - return 1 - } - if (exitClass === 'ConfigError') { - return 2 - } - if (exitClass === 'RuntimeError') { - return 3 - } - return 4 +const CWD = '/project' +const ARGS = ['run', '--dryRunOnly'] + +const outcomeOf = (cause: Cause.Cause): RunOutcomeDecision => + Result.getOrElse( + classifyRunOutcome(runOutcomeCommandOf({ exit: Exit.failCause(cause), argv: ARGS, cwd: CWD, traceId: null })), + (impossible) => impossible, + ) + +const failedOf = (outcome: RunOutcomeDecision): RunFailed | undefined => S.is(RunFailed)(outcome) ? outcome : undefined + +const errorChainOf = (messages: Arr.NonEmptyReadonlyArray, innermost: object): object => + Arr.reduce( + [...messages].reverse(), + innermost, + (inner, message) => Object.assign(new Error(message), { cause: inner }), + ) + +const plainErrorChainOf = (messages: Arr.NonEmptyReadonlyArray): Error => { + const [innermost, ...outer] = [...messages].reverse() + return Arr.reduce( + outer, + new Error(innermost), + (inner, message) => Object.assign(new Error(message), { cause: inner }), + ) } -const isRunFailed = ( - result: Result.Result, - code: number, - diagnostic: string | undefined, -): boolean => - Result.isSuccess(result) && - S.is(RunFailed)(result.success) && - result.success.code === code && - result.success.diagnostic === diagnostic +const PRECEDENCE_LOW_TO_HIGH: ReadonlyArray = [130, 1, 2, 5, 3, 4] + +const rankOf = (code: FailureRecord.FailureCode): number => + PRECEDENCE_LOW_TO_HIGH.indexOf(FailureRecord.FailureCatalog[code].exitCode ?? 4) + +const shortChain = () => S.NonEmptyArray(S.String).check(S.isMaxLength(8)) describe('classifyRunOutcome', () => { it.prop( - '∀c_Command_≡TaggedOutcome', - { of: [RunOutcomeCommand], subject: classifyRunOutcome }, - (subject, [command]) => { - const result = subject(command) - return Match.value(command.observation).pipe( - Match.tag( - 'RunSucceededClean', - () => Result.isSuccess(result) && S.is(RunOk)(result.success) && result.success.help === false, - ), - Match.tag( - 'RunSucceededVerdict', - (observation) => isRunFailed(result, classCode(observation.exitClass), observation.diagnostic ?? undefined), - ), - Match.tag( - 'RunInterruptedObservation', - () => Result.isFailure(result) && S.is(RunInterrupted)(result.failure) && result.failure.code === 130, - ), - Match.tag('RunHelpObservation', (observation) => - observation.errorCount > 0 - ? Result.isSuccess(result) && - S.is(RunParseFailed)(result.success) && - result.success.unrecognized === (observation.unrecognized ?? undefined) - : Result.isSuccess(result) && S.is(RunOk)(result.success) && result.success.help === true), - Match.tag( - 'RunCliErrorObservation', - (observation) => - Result.isSuccess(result) && - S.is(RunParseFailed)(result.success) && - result.success.unrecognized === (observation.unrecognized ?? undefined), - ), - Match.tag( - 'RunSurvivorsRejectedObservation', - (observation) => - Result.isSuccess(result) && - S.is(RunSurvivorsRejected)(result.success) && - result.success.reason === observation.reason && - result.success.diagnostic === (observation.diagnostic ?? undefined), - ), - Match.tag( - 'RunSchemaErrorObservation', - (observation) => - Result.isSuccess(result) && - S.is(RunConfigFailed)(result.success) && - result.success.detail === (observation.configDetail ?? undefined), - ), - Match.tag('RunClassedObservation', (observation) => - observation.exitClass === 'ConfigError' - ? Result.isSuccess(result) && - S.is(RunConfigFailed)(result.success) && - result.success.detail === (observation.configDetail ?? undefined) - : isRunFailed(result, classCode(observation.exitClass), observation.diagnostic ?? undefined)), - Match.tag( - 'RunGenericFailureObservation', - (observation) => isRunFailed(result, 1, observation.diagnostic ?? undefined), - ), - Match.exhaustive, - ) + '∀evidence_RunOutcome_≡RecordOfTheDeclaredEvidenceWithItsCatalogExitCode', + { of: [FailureRecord.FailureEvidence, shortChain()], subject: outcomeOf }, + (subject, [evidence, wrappers]) => { + const failed = failedOf(subject(Cause.fail(errorChainOf(wrappers, { evidence })))) + const facts = FailureRecord.FailureCatalog[evidence._tag] + return failed !== undefined && Predicate.isTagged(failed.record, evidence._tag) && + failed.record.stage === evidence.stage && failed.code === (facts.exitCode ?? 4) && + failed.record.nextAction.primary === facts.nextAction.primary && + failed.record.nextAction.otherwise === facts.nextAction.otherwise && + failed.record.cause.length === wrappers.length + 1 + }, + ) + + it.prop( + '∀messages_RunOutcome_≡UnclassifiedFailureIsACatalogGapCarryingEveryMessage', + { of: [shortChain()], subject: outcomeOf }, + (subject, [messages]) => { + const failed = failedOf(subject(Cause.fail(plainErrorChainOf(messages)))) + return failed !== undefined && failed.code === 4 && Predicate.isTagged(failed.record, 'CatalogGap') && + failed.record.cause.map((link) => link.message).join('\u0000') === messages.join('\u0000') + }, + ) + + it.prop( + '∀first,second_RunOutcome_≡TheMoreSevereFailureDecidesTheRun', + { of: [FailureRecord.FailureEvidence, FailureRecord.FailureEvidence], subject: outcomeOf }, + (subject, [first, second]) => { + const failed = failedOf(subject(Cause.combine(Cause.fail({ evidence: first }), Cause.fail({ evidence: second })))) + const expected = rankOf(second._tag) > rankOf(first._tag) ? second : first + return failed !== undefined && Predicate.isTagged(failed.record, expected._tag) + }, + ) + + it.prop( + '∀evidence_RunOutcome_≡ReplayableRecordsReplayTheSameCommandAndOthersSayWhyNot', + { of: [FailureRecord.FailureEvidence], subject: outcomeOf }, + (subject, [evidence]) => { + const capsule = failedOf(subject(Cause.fail({ evidence })))?.record.capsule + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + return rule === 'replays' + ? Predicate.isTagged(capsule, 'Replays') && capsule.cwd === CWD && + capsule.argv.join(' ') === ['stryker', ...ARGS].join(' ') + : Predicate.isTagged(capsule, 'DoesNotReplay') && capsule.why === rule + }, + ) + + it.prop( + '∀fiber_RunOutcome_≡AnInterruptOnlyRunExits130WithARunInterruptedRecord', + { of: [S.Int.check(S.isGreaterThanOrEqualTo(0))], subject: outcomeOf }, + (subject, [fiberId]) => { + const failed = failedOf(subject(Cause.interrupt(fiberId))) + return failed?.code === 130 && Predicate.isTagged(failed.record, 'RunInterrupted') }, ) }) diff --git a/packages/stryker-js/src/__tests__/dry-run.workflow.property.test.ts b/packages/stryker-js/src/__tests__/dry-run.workflow.property.test.ts index 13e870a5..68dddb7b 100644 --- a/packages/stryker-js/src/__tests__/dry-run.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/dry-run.workflow.property.test.ts @@ -1,9 +1,11 @@ import { describe, it } from '@systemfsoftware/vitest' import * as Equal from 'effect/Equal' +import * as Predicate from 'effect/Predicate' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -import { dryRun, DryRunCommand, DryRunError, DryRunFailed, DryRunPassed } from '../dry-run.workflow.js' +import { dryRun, DryRunCommand, DryRunFailed, DryRunPassed } from '../dry-run.workflow.js' +import { RunFailure } from '../Run.schema.js' describe('dryRun', () => { it.prop( @@ -14,22 +16,25 @@ describe('dryRun', () => { if (command.status === 'Error') { return ( Result.isFailure(result) && - S.is(DryRunError)(result.failure) && - result.failure.stage === 'dryRun' + S.is(RunFailure)(result.failure) && + Predicate.isTagged(result.failure.evidence, 'BaselineErrored') && + result.failure.evidence.stage === 'dryRun' ) } if (command.status === 'Timeout') { return ( Result.isFailure(result) && - S.is(DryRunError)(result.failure) && - result.failure.stage === 'dryRun' + S.is(RunFailure)(result.failure) && + Predicate.isTagged(result.failure.evidence, 'BaselineTimedOut') && + result.failure.evidence.stage === 'dryRun' ) } if (command.testCount === 0 && command.allowEmpty === false) { return ( Result.isFailure(result) && - S.is(DryRunError)(result.failure) && - result.failure.stage === 'dryRunNoTests' + S.is(RunFailure)(result.failure) && + Predicate.isTagged(result.failure.evidence, 'BaselineFoundNoTests') && + result.failure.evidence.stage === 'dryRun' ) } if (command.failedTestCount > 0) { diff --git a/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts b/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts index bfe8d9c7..504273fd 100644 --- a/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts @@ -1,3 +1,4 @@ +import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' import * as Result from 'effect/Result' @@ -15,7 +16,7 @@ const observationArb = Arbitrary.schema(DryRunObservation) describe('interpretDryRunObservation', () => { it.prop( - '∀o_Complete_≡CountsTestsAndFailures', + '∀o_Complete_≡CarriesFailedTestEvidenceInOrder', { of: [observationArb], subject: interpretDryRunObservation }, (subject, [observation]) => Result.match(subject(observation), { @@ -24,16 +25,30 @@ describe('interpretDryRunObservation', () => { if (observation.dryRunResult.status !== 'complete') { return true } - const failed = observation.dryRunResult.tests.filter((test) => test.status === 'failed') + const failed = observation.dryRunResult.tests.filter( + (test): test is TestRunner.FailedTestResult => test.status === 'failed', + ) return S.is(DryRunObservedComplete)(decision) && decision.testCount === observation.dryRunResult.tests.length && decision.failedTestCount === failed.length && decision.failedTests.length === failed.length && - decision.failedTests.every((summary, index) => { + decision.failedTests.every((evidence, index) => { const expected = failed[index] - return expected !== undefined && - summary.name === expected.name && - summary.failureMessage === expected.failureMessage + if (expected === undefined) { + return false + } + const location = expected.location + return evidence.id === expected.id && + evidence.name === expected.name && + evidence.file === (expected.fileName ?? null) && + (location === undefined + ? evidence.location === null + : evidence.location !== null && + evidence.location.file === location.file && + evidence.location.line === location.line && + evidence.location.column === location.column) && + evidence.message === expected.failureMessage && + evidence.stack === (expected.stack ?? null) }) }, }), diff --git a/packages/stryker-js/src/__tests__/plan-run-conclusion.workflow.property.test.ts b/packages/stryker-js/src/__tests__/plan-run-conclusion.workflow.property.test.ts index 0905586e..7277cdaf 100644 --- a/packages/stryker-js/src/__tests__/plan-run-conclusion.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/plan-run-conclusion.workflow.property.test.ts @@ -1,5 +1,5 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { describe, it } from '@systemfsoftware/vitest' -import * as Equal from 'effect/Equal' import * as Match from 'effect/Match' import * as Result from 'effect/Result' import * as S from 'effect/Schema' @@ -8,28 +8,12 @@ import { planRunConclusion, PlanRunConclusionCommand, type PlanRunConclusionDecision, - RunConclusionEmittedFailed, RunConclusionEmittedOk, - RunConclusionQuietFailed, + RunConclusionFailed, RunConclusionQuietOk, + RunConclusionVerdictFailed, } from '../plan-run-conclusion.workflow.js' -type ConclusionVariant = 'emitted-ok' | 'emitted-failed' | 'quiet-ok' | 'quiet-failed' - -const variantOf = (machine: boolean, exitCode: number): ConclusionVariant => - machine - ? (exitCode === 0 ? 'emitted-ok' : 'emitted-failed') - : (exitCode === 0 ? 'quiet-ok' : 'quiet-failed') - -const isVariant = (variant: ConclusionVariant, decision: PlanRunConclusionDecision): boolean => - Match.value(variant).pipe( - Match.when('emitted-ok', () => S.is(RunConclusionEmittedOk)(decision)), - Match.when('emitted-failed', () => S.is(RunConclusionEmittedFailed)(decision)), - Match.when('quiet-ok', () => S.is(RunConclusionQuietOk)(decision)), - Match.when('quiet-failed', () => S.is(RunConclusionQuietFailed)(decision)), - Match.exhaustive, - ) - const decisionOf = ( subject: typeof planRunConclusion, command: PlanRunConclusionCommand, @@ -39,33 +23,32 @@ const decisionOf = ( onSuccess: (decision) => decision, }) -const commandCarried = (command: PlanRunConclusionCommand, decision: PlanRunConclusionDecision): boolean => - Match.value(variantOf(command.machine, command.exitCode)).pipe( - Match.when( - 'emitted-ok', - () => S.is(RunConclusionEmittedOk)(decision) && Equal.equals(decision.command, command.command), +const sameRecord = (left: FailureRecord.FailureRecord, right: FailureRecord.FailureRecord): boolean => + S.toEquivalence(FailureRecord.FailureRecord)(left, right) + +const concludesAsPlanned = (command: PlanRunConclusionCommand, decision: PlanRunConclusionDecision): boolean => + Match.value(command.decision).pipe( + Match.tag( + 'RunOk', + () => command.machine ? S.is(RunConclusionEmittedOk)(decision) : S.is(RunConclusionQuietOk)(decision), ), - Match.when('emitted-failed', () => - S.is(RunConclusionEmittedFailed)(decision) && - Equal.equals(decision.command, command.command) && - decision.exitCode === command.exitCode), - Match.when('quiet-ok', () => S.is(RunConclusionQuietOk)(decision)), - Match.when( - 'quiet-failed', - () => S.is(RunConclusionQuietFailed)(decision) && decision.exitCode === command.exitCode, + Match.tag( + 'RunVerdictFailed', + (failed) => S.is(RunConclusionVerdictFailed)(decision) && decision.exitCode === failed.code, ), + Match.tag('RunFailed', (failed) => + S.is(RunConclusionFailed)(decision) && decision.exitCode === failed.code && + sameRecord(decision.record, failed.record)), Match.exhaustive, ) describe('planRunConclusion', () => { it.prop( - '∀command_Plan_≡VariantAndEffectFollowMachineAndExitCode', + '∀command_Plan_≡AFailedRunConcludesWithItsRecordInEveryModeAndOnlyMachineModeEmitsSuccess', { of: [PlanRunConclusionCommand], subject: planRunConclusion }, (subject, [command]) => { const decision = decisionOf(subject, command) - return decision !== undefined && - isVariant(variantOf(command.machine, command.exitCode), decision) && - commandCarried(command, decision) + return decision !== undefined && concludesAsPlanned(command, decision) }, ) }) diff --git a/packages/stryker-js/src/__tests__/run-failure.workflow.property.test.ts b/packages/stryker-js/src/__tests__/run-failure.workflow.property.test.ts deleted file mode 100644 index 7240afe2..00000000 --- a/packages/stryker-js/src/__tests__/run-failure.workflow.property.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, it } from '@systemfsoftware/vitest' -import * as Match from 'effect/Match' -import * as S from 'effect/Schema' - -import { - RunConfigFailed, - RunFailed, - RunInterrupted, - RunOk, - RunParseFailed, - RunSurvivorsRejected, -} from '../classify-run-outcome.workflow.js' -import { exitCodeOf } from '../reporting/run-failure.js' - -const FROZEN_CONFIG_CODE = 2 - -describe('exitCodeOf', () => { - it.prop( - '∀outcome_ExitCode_≡FrozenCodes', - { - of: [ - S.Union([ - RunOk, - RunInterrupted, - RunParseFailed, - RunSurvivorsRejected, - RunConfigFailed, - RunFailed, - ]), - ], - subject: exitCodeOf, - }, - (subject, [outcome]) => - Match.value(outcome).pipe( - Match.tag('RunOk', () => subject(outcome) === 0), - Match.tag('RunInterrupted', (interrupted) => subject(outcome) === interrupted.code), - Match.tag('RunFailed', (failed) => subject(outcome) === failed.code), - Match.orElse(() => subject(outcome) === FROZEN_CONFIG_CODE), - ), - ) -}) diff --git a/packages/stryker-js/src/admit-mutation-test.workflow.ts b/packages/stryker-js/src/admit-mutation-test.workflow.ts index e887506d..4f101f98 100644 --- a/packages/stryker-js/src/admit-mutation-test.workflow.ts +++ b/packages/stryker-js/src/admit-mutation-test.workflow.ts @@ -4,15 +4,7 @@ import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { MutationTestCommand } from './MutationTest.schema.js' - -export class MutationTestError extends S.TaggedError()('MutationTestError', { - stage: S.Literal('mutationTest'), - reason: S.String, -}) { - override get message(): string { - return this.reason - } -} +import { RunFailure } from './Run.schema.js' const MutationTestDecisionTypeId: unique symbol = Symbol.for('@systemfsoftware/stryker-js/MutationTestDecision') type MutationTestDecisionTypeId = typeof MutationTestDecisionTypeId @@ -31,7 +23,7 @@ export class MutationTestNoTests extends S.TaggedClass()('M export type MutationTestDecision = MutationTestProceed | MutationTestDryRunOnly | MutationTestNoTests -const decide = (command: MutationTestCommand): Result.Result => +const decide = (command: MutationTestCommand): Result.Result => Match.value({ invalid: command.testCount < 0, dryRunOnly: command.dryRunOnly, @@ -39,9 +31,13 @@ const decide = (command: MutationTestCommand): Result.Result - Result.fail(MutationTestError.make({ stage: 'mutationTest', reason: 'Invalid test count' }))), - Match.when({ dryRunOnly: true }, () => - Result.succeed(MutationTestDryRunOnly.make({}))), + Result.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, + detail: 'Invalid test count', + }), + )), + Match.when({ dryRunOnly: true }, () => Result.succeed(MutationTestDryRunOnly.make({}))), Match.when({ isZero: true, allowEmpty: true }, () => Result.succeed(MutationTestNoTests.make({}))), Match.orElse(() => Result.succeed(MutationTestProceed.make({}))), ) @@ -49,6 +45,6 @@ const decide = (command: MutationTestCommand): Result.Result()('Sur }) { readonly [SurvivorsAdmissionTypeId] = SurvivorsAdmissionTypeId + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'SurvivorsUnavailable', stage: 'config', reason: this.reason } + } + override get message(): string { return this.remediation } diff --git a/packages/stryker-js/src/bin/main.ts b/packages/stryker-js/src/bin/main.ts index e34a71f2..ee5bb4c0 100755 --- a/packages/stryker-js/src/bin/main.ts +++ b/packages/stryker-js/src/bin/main.ts @@ -7,7 +7,7 @@ import { AggregationTemporalityPreference, OTLPMetricExporter } from '@opentelem import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics' import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' -import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { HtmlReporter } from '@systemfsoftware/stryker-js-html-reporter' import cliPkgJson from '@systemfsoftware/stryker-js/package.json' with { type: 'json' } import * as Boolean from 'effect/Boolean' @@ -21,6 +21,7 @@ import * as Console from 'effect/Console' import * as EffectDuration from 'effect/Duration' import * as Effect from 'effect/Effect' import * as Exit from 'effect/Exit' +import * as FileSystem from 'effect/FileSystem' import * as Layer from 'effect/Layer' import * as Logger from 'effect/Logger' import * as Match from 'effect/Match' @@ -33,16 +34,15 @@ import * as Scope from 'effect/Scope' import * as Stdio from 'effect/Stdio' import { inheritableCompileCacheDirectory } from './enable-compile-cache.js' -import { classifyRunOutcome, RunExit, RunParseFailed } from '../classify-run-outcome.workflow.js' +import { classifyRunOutcome, RunExit, type RunOutcomeDecision } from '../classify-run-outcome.workflow.js' import { concludeRunCell } from '../conclude-run.cell.js' import { runOutcomeCommandOf } from '../conclude-run.js' import { makeNodePlatformLayer } from '../drivers/node.js' import { OutputModeProbe, OutputModeProbeLive } from '../output-mode-probe.service.js' import type { ResolvedMode } from '../output-mode.schema.js' -import { FailedRunOutcomeSchema } from '../plan-run-conclusion.workflow.js' import { environmentParentContext } from '../reporter-stream.service.js' import { MachineConsole } from '../reporting/machine-console.service.js' -import { errorEnvelopeFromOutcome, runExitCodeFromOutcome } from '../reporting/run-failure.js' +import { runExitCodeFromOutcome } from '../reporting/run-failure.js' import { RunEventDrain, RunEventStreamPort, RunEventStreamPortTag } from '../run-event-stream.service.js' import { type CliAnswer, type CliEnvironment } from '../run-request.cell.js' import { RunEnvironment } from '../run/RunEnvironment.service.js' @@ -118,9 +118,11 @@ const otlpTelemetryLayer = (options: { ) } +const tracingEnabled = Config.Boolean('OTEL_ENABLED').pipe(Config.withDefault(false)) + const telemetryLayer: Layer.Layer = Layer.unwrap( Effect.all([ - Config.Boolean('OTEL_ENABLED').pipe(Config.withDefault(false)), + tracingEnabled, Config.String('OTEL_SERVICE_NAME').pipe(Config.withDefault('stryker-js')), Config.String('OTEL_EXPORTER_OTLP_ENDPOINT').pipe(Config.withDefault('http://127.0.0.1:4318')), Config.Number('OTEL_METRIC_EXPORT_INTERVAL').pipe(Config.withDefault(DEFAULT_METRIC_EXPORT_INTERVAL_MILLIS)), @@ -168,7 +170,7 @@ const cliLayer = Layer.mergeAll( NodeTerminal.layer, ).pipe(Layer.provideMerge(nodePlatform)) -const USAGE_EXIT_CODE = runExitCodeFromOutcome(RunParseFailed.make({})).code +const USAGE_EXIT_CODE = FailureRecord.FailureCatalog.ArgumentsInvalid.exitCode const PROTOCOL_SERVER_SUBCOMMANDS: ReadonlyArray = ['serve', 'mcp'] @@ -180,11 +182,23 @@ const servedInvocation = (argv: ReadonlyArray): boolean => (subcommand) => PROTOCOL_SERVER_SUBCOMMANDS.some((served) => served === subcommand), ) -const SPAN_ERROR_LIMIT = 1024 -const TRUNCATION_SUFFIX = '…[truncated]' +const exportedTraceId = Effect.gen(function*() { + const enabled = yield* Effect.orElseSucceed(tracingEnabled, () => false) + const span = yield* Effect.option(Effect.currentSpan) + return Option.getOrNull( + Option.flatMap( + Option.filter(span, () => enabled), + (active) => S.decodeOption(FailureRecord.TraceId)(active.traceId), + ), + ) +}) -const boundedErrorText = (text: string): string => - text.length > SPAN_ERROR_LIMIT ? text.slice(0, SPAN_ERROR_LIMIT) + TRUNCATION_SUFFIX : text +const failureCodeAttributeOf = (decision: RunOutcomeDecision): Record => + Match.valueTags(decision, { + RunOk: () => ({}), + RunVerdictFailed: () => ({}), + RunFailed: ({ record }) => ({ 'stryker.failure.code': record._tag }), + }) const strykerProgram = Effect.gen(function*() { const stdio = yield* Stdio.Stdio @@ -240,35 +254,26 @@ const strykerProgram = Effect.gen(function*() { ), ), ) - const conclusionCommand = runOutcomeCommandOf({ exit, argv: args }) - const outcome = classifyRunOutcome(conclusionCommand) - const classified = Result.getOrElse( - outcome, - (interrupted) => interrupted, - ) - const machineConsoleService = yield* MachineConsole - const errorText = Option.getOrElse( - Option.map( - Option.liftPredicate(S.is(FailedRunOutcomeSchema))(classified), - (failure) => - boundedErrorText( - errorEnvelopeFromOutcome({ error: failure, captured: machineConsoleService.read() }).error, - ), - ), - () => '', - ) + const conclusionCommand = runOutcomeCommandOf({ + exit, + argv: args, + cwd: host.basePath, + traceId: yield* exportedTraceId, + }) + const decision = Result.getOrElse(classifyRunOutcome(conclusionCommand), (impossible) => impossible) yield* Effect.annotateCurrentSpan({ - 'stryker.run.outcome': classified._tag, - 'stryker.run.exit_code': runExitCodeFromOutcome(classified).code, - 'stryker.run.error': errorText, + 'stryker.run.outcome': decision._tag, + 'stryker.run.exit_code': runExitCodeFromOutcome(decision).code, + ...failureCodeAttributeOf(decision), }) return yield* concludeRunCell.run({ mode, stream, basePath: host.basePath, pathService, + fileSystem: yield* FileSystem.FileSystem, runEvents, - concluded: { command: conclusionCommand, outcome, error: errorText }, + decision, }) }), ) diff --git a/packages/stryker-js/src/classify-run-outcome.workflow.ts b/packages/stryker-js/src/classify-run-outcome.workflow.ts index 319e809e..f3a046f6 100644 --- a/packages/stryker-js/src/classify-run-outcome.workflow.ts +++ b/packages/stryker-js/src/classify-run-outcome.workflow.ts @@ -1,21 +1,18 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' -import * as Boolean from 'effect/Boolean' +import * as Arr from 'effect/Array' import * as Match from 'effect/Match' -import * as Option from 'effect/Option' +import * as Order from 'effect/Order' import * as Result from 'effect/Result' import * as Runtime from 'effect/Runtime' import * as S from 'effect/Schema' import { - RunClassedObservation, - RunCliErrorObservation, - RunGenericFailureObservation, - RunHelpObservation, + type ObservedFailure, + type RunFailedObservation, RunOutcomeCommand, - RunSchemaErrorObservation, - RunSucceededVerdict, - RunSurvivorsRejectedObservation, + type RunSucceededVerdict, } from './RunOutcomeCommand.schema.js' export class RunExit extends S.TaggedError()('RunExit', { code: Plugin.ExitCode }) { @@ -28,17 +25,6 @@ export class RunExit extends S.TaggedError()('RunExit', { code: Plugin. } } -const CONFIG_CODE = 2 - -const classCode = (exitClass: Plugin.ExitClass): number => - Match.value(exitClass).pipe( - Match.when('VerdictFail', () => 1), - Match.when('ConfigError', () => CONFIG_CODE), - Match.when('RuntimeError', () => 3), - Match.when('InternalError', () => 4), - Match.exhaustive, - ) - const RunOutcomeTypeId: unique symbol = Symbol.for('@systemfsoftware/stryker-js/RunOutcome') type RunOutcomeTypeId = typeof RunOutcomeTypeId @@ -48,117 +34,63 @@ export class RunOk extends S.TaggedClass()('RunOk', { readonly [RunOutcomeTypeId] = RunOutcomeTypeId } -export class RunInterrupted extends S.TaggedError()('RunInterrupted', { +export class RunVerdictFailed extends S.TaggedClass()('RunVerdictFailed', { code: Plugin.ExitCode, -}) { - readonly [RunOutcomeTypeId] = RunOutcomeTypeId - - override get message(): string { - return `Run interrupted by a signal (exit code ${this.code})` - } -} - -export class RunParseFailed extends S.TaggedClass()('RunParseFailed', { - unrecognized: S.optional(S.String), -}) { - readonly [RunOutcomeTypeId] = RunOutcomeTypeId -} - -export class RunSurvivorsRejected extends S.TaggedClass()('RunSurvivorsRejected', { - reason: S.Literals(['no-report', 'mismatch']), - diagnostic: S.optional(S.String), -}) { - readonly [RunOutcomeTypeId] = RunOutcomeTypeId -} - -export class RunConfigFailed extends S.TaggedClass()('RunConfigFailed', { - detail: S.optional(S.String), }) { readonly [RunOutcomeTypeId] = RunOutcomeTypeId } export class RunFailed extends S.TaggedClass()('RunFailed', { code: Plugin.ExitCode, - diagnostic: S.optional(S.String), + record: FailureRecord.FailureRecord, }) { readonly [RunOutcomeTypeId] = RunOutcomeTypeId } -export type RunOutcomeDecision = - | RunOk - | RunParseFailed - | RunSurvivorsRejected - | RunConfigFailed - | RunFailed - -export type RunOutcomeError = RunInterrupted - -export type FailedRunOutcome = Exclude | RunOutcomeError - -const succeededCleanOutcome = (): RunOutcomeDecision => RunOk.make({ help: false }) - -const succeededVerdictOutcome = (observation: RunSucceededVerdict): RunOutcomeDecision => - RunFailed.make({ - code: classCode(observation.exitClass), - diagnostic: Option.getOrUndefined(Option.fromNullishOr(observation.diagnostic)), - }) - -const helpOutcome = (observation: RunHelpObservation): RunOutcomeDecision => - Boolean.match(observation.errorCount > 0, { - onTrue: () => - RunParseFailed.make({ unrecognized: Option.getOrUndefined(Option.fromNullishOr(observation.unrecognized)) }), - onFalse: () => RunOk.make({ help: true }), - }) - -const cliErrorOutcome = (observation: RunCliErrorObservation): RunOutcomeDecision => - RunParseFailed.make({ unrecognized: Option.getOrUndefined(Option.fromNullishOr(observation.unrecognized)) }) - -const survivorsRejectedOutcome = (observation: RunSurvivorsRejectedObservation): RunOutcomeDecision => - RunSurvivorsRejected.make({ - reason: observation.reason, - diagnostic: Option.getOrUndefined(Option.fromNullishOr(observation.diagnostic)), - }) - -const schemaErrorOutcome = (observation: RunSchemaErrorObservation): RunOutcomeDecision => - RunConfigFailed.make({ detail: Option.getOrUndefined(Option.fromNullishOr(observation.configDetail)) }) - -const classedOutcome = (observation: RunClassedObservation): RunOutcomeDecision => - Match.value(observation.exitClass).pipe( - Match.when( - 'ConfigError', - () => RunConfigFailed.make({ detail: Option.getOrUndefined(Option.fromNullishOr(observation.configDetail)) }), - ), - Match.orElse((exitClass) => - RunFailed.make({ - code: classCode(exitClass), - diagnostic: Option.getOrUndefined(Option.fromNullishOr(observation.diagnostic)), - }) - ), +export const RunOutcomeDecision = S.Union([RunOk, RunVerdictFailed, RunFailed]) +export type RunOutcomeDecision = typeof RunOutcomeDecision.Type + +const verdictCode = (exitClass: Plugin.ExitClass): Plugin.ExitCode => + Match.value(exitClass).pipe( + Match.when('VerdictFail', () => 1), + Match.when('ConfigError', () => 2), + Match.when('RuntimeError', () => 3), + Match.when('InternalError', () => 4), + Match.exhaustive, ) -const genericFailureOutcome = (observation: RunGenericFailureObservation): RunOutcomeDecision => - RunFailed.make({ code: 1, diagnostic: Option.getOrUndefined(Option.fromNullishOr(observation.diagnostic)) }) - -const decide = (command: RunOutcomeCommand): Result.Result => - Match.value(command.observation).pipe( - Match.tag('RunSucceededClean', () => Result.succeed(succeededCleanOutcome())), - Match.tag('RunSucceededVerdict', (observation) => Result.succeed(succeededVerdictOutcome(observation))), - Match.tag('RunInterruptedObservation', () => Result.fail(RunInterrupted.make({ code: 130 }))), - Match.tag('RunHelpObservation', (observation) => Result.succeed(helpOutcome(observation))), - Match.tag('RunCliErrorObservation', (observation) => Result.succeed(cliErrorOutcome(observation))), - Match.tag( - 'RunSurvivorsRejectedObservation', - (observation) => Result.succeed(survivorsRejectedOutcome(observation)), +const PRECEDENCE_LOW_TO_HIGH: ReadonlyArray = [130, 1, 2, 5, 3, 4] + +const bySeverity: Order.Order = Order.mapInput( + Order.Number, + (failure: ObservedFailure) => PRECEDENCE_LOW_TO_HIGH.indexOf(failure.exitCode), +) + +const decidingFailureOf = (observation: RunFailedObservation): ObservedFailure => + Arr.headNonEmpty(Arr.sort(observation.failures, Order.flip(bySeverity))) + +const failedOutcome = (observation: RunFailedObservation): RunOutcomeDecision => { + const deciding = decidingFailureOf(observation) + return RunFailed.make({ code: deciding.exitCode, record: deciding.record }) +} + +const verdictOutcome = (observation: RunSucceededVerdict): RunOutcomeDecision => + RunVerdictFailed.make({ code: verdictCode(observation.exitClass) }) + +const decide = (command: RunOutcomeCommand): Result.Result => + Result.succeed( + Match.value(command.observation).pipe( + Match.tag('RunSucceededClean', () => RunOk.make({ help: false })), + Match.tag('RunHelpObservation', () => RunOk.make({ help: true })), + Match.tag('RunSucceededVerdict', verdictOutcome), + Match.tag('RunFailedObservation', failedOutcome), + Match.exhaustive, ), - Match.tag('RunSchemaErrorObservation', (observation) => Result.succeed(schemaErrorOutcome(observation))), - Match.tag('RunClassedObservation', (observation) => Result.succeed(classedOutcome(observation))), - Match.tag('RunGenericFailureObservation', (observation) => Result.succeed(genericFailureOutcome(observation))), - Match.exhaustive, ) export const classifyRunOutcome = Workflow.make({ command: RunOutcomeCommand, - decision: S.Union([RunOk, RunParseFailed, RunSurvivorsRejected, RunConfigFailed, RunFailed]), - error: RunInterrupted, + decision: RunOutcomeDecision, + error: S.Never, decide, }) diff --git a/packages/stryker-js/src/conclude-run.cell.ts b/packages/stryker-js/src/conclude-run.cell.ts index 43fb8e38..f5c45cf8 100644 --- a/packages/stryker-js/src/conclude-run.cell.ts +++ b/packages/stryker-js/src/conclude-run.cell.ts @@ -1,92 +1,98 @@ import { Sandwich } from '@systemfsoftware/effect-cell-types' -import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import * as Effect from 'effect/Effect' +import type * as FileSystem from 'effect/FileSystem' import type * as Path from 'effect/Path' -import * as Result from 'effect/Result' +import * as S from 'effect/Schema' -import { RunExit, type RunOutcomeDecision, type RunOutcomeError } from './classify-run-outcome.workflow.js' +import { RunExit, RunOutcomeDecision } from './classify-run-outcome.workflow.js' import type { ResolvedMode } from './output-mode.schema.js' import { planRunConclusion, type PlanRunConclusionCommand } from './plan-run-conclusion.workflow.js' -import { runExitCodeFromOutcome } from './reporting/run-failure.js' import type { RunEventDrain, RunEventStream, RunEventStreamPort } from './run-event-stream.service.js' -import { RunOutcomeCommand } from './RunOutcomeCommand.schema.js' import { StrykerError } from './stryker-error.schema.js' - -export interface RunConclusion { - readonly command: RunOutcomeCommand - readonly outcome: Result.Result - readonly error: string -} +import { FAILURE_RECORD_FILE } from './stryker-outputs.js' export interface RunConclusionInput { readonly mode: ResolvedMode readonly stream: RunEventStream readonly basePath: string readonly pathService: Path.Path + readonly fileSystem: FileSystem.FileSystem readonly runEvents: RunEventStreamPort - readonly concluded: RunConclusion + readonly decision: RunOutcomeDecision } export type RunConclusionRaw = (typeof PlanRunConclusionCommand)['Encoded'] & { readonly conclusion: RunConclusionInput } -const encodedCommandOf = ( - command: RunOutcomeCommand, -): (typeof RunOutcomeCommand)['Encoded'] => ({ - _tag: 'RunOutcomeCommand', - observation: command.observation, -}) - const readConclusion = Effect.fn(SpanTaxonomy.Spans.runConclusionRead.name)(function*( input: RunConclusionInput, ): Effect.fn.Return { - const classified = Result.getOrElse(input.concluded.outcome, (interrupted) => interrupted) yield* input.stream.open return { _tag: 'PlanRunConclusionCommand' as const, - command: encodedCommandOf(input.concluded.command), + decision: yield* Effect.orDie(S.encodeEffect(RunOutcomeDecision)(input.decision)), machine: input.mode.mode === 'machine', - exitCode: runExitCodeFromOutcome(classified).code, - outcome: classified._tag, - error: input.concluded.error, conclusion: input, } }) +const failureRecordPathOf = (conclusion: RunConclusionInput): string => + conclusion.pathService.resolve(conclusion.basePath, FAILURE_RECORD_FILE) + +const writeFailureRecord = (conclusion: RunConclusionInput, record: FailureRecord.FailureRecord) => + Effect.gen(function*() { + const file = failureRecordPathOf(conclusion) + const text = yield* S.encodeEffect(FailureRecord.FailureRecordFile)(record) + yield* conclusion.fileSystem.makeDirectory(conclusion.pathService.dirname(file), { recursive: true }) + yield* conclusion.fileSystem.writeFileString(file, `${text}\n`) + }).pipe( + Effect.tapError((error) => Effect.logWarning(`could not write ${FAILURE_RECORD_FILE}: ${error.message}`)), + Effect.ignore, + ) + +const removeStaleFailureRecord = (conclusion: RunConclusionInput) => + conclusion.fileSystem.remove(failureRecordPathOf(conclusion), { force: true }).pipe( + Effect.tapError((error) => Effect.logWarning(`could not remove a stale ${FAILURE_RECORD_FILE}: ${error.message}`)), + Effect.ignore, + ) + +const closeAndExit = (conclusion: RunConclusionInput, exitCode: number) => + Effect.andThen(conclusion.stream.closeAndDrain, Effect.fail(RunExit.make({ code: exitCode }))) + export const concludeRunCell = Sandwich.named(SpanTaxonomy.Spans.runConclude.name)(readConclusion) .decide(planRunConclusion) .write({ - RunConclusionEmittedOk: (_decision, raw) => - Effect.andThen( - raw.conclusion.runEvents.emitMachineModeOutput({ - stream: raw.conclusion.stream, - mode: raw.conclusion.mode, - outcome: raw.conclusion.concluded.outcome, - basePath: raw.conclusion.basePath, - pathService: raw.conclusion.pathService, - }), - Effect.andThen(raw.conclusion.stream.closeAndDrain, Effect.void), - ), - RunConclusionEmittedFailed: (decision, raw) => - Effect.andThen( - raw.conclusion.runEvents.emitMachineModeOutput({ - stream: raw.conclusion.stream, - mode: raw.conclusion.mode, - outcome: raw.conclusion.concluded.outcome, - basePath: raw.conclusion.basePath, - pathService: raw.conclusion.pathService, - }), + RunConclusionEmittedOk: (decision, raw) => + removeStaleFailureRecord(raw.conclusion).pipe( Effect.andThen( - raw.conclusion.stream.closeAndDrain, - Effect.fail(RunExit.make({ code: decision.exitCode })), + raw.conclusion.runEvents.emitMachineModeOutput({ + stream: raw.conclusion.stream, + mode: raw.conclusion.mode, + ok: decision.ok, + basePath: raw.conclusion.basePath, + pathService: raw.conclusion.pathService, + }), ), + Effect.andThen(raw.conclusion.stream.closeAndDrain), + Effect.asVoid, + ), + RunConclusionQuietOk: (_decision, raw) => + removeStaleFailureRecord(raw.conclusion).pipe( + Effect.andThen(raw.conclusion.stream.closeAndDrain), + Effect.asVoid, ), - RunConclusionQuietOk: (_decision, raw) => Effect.andThen(raw.conclusion.stream.closeAndDrain, Effect.void), - RunConclusionQuietFailed: (decision, raw) => - Effect.andThen( - raw.conclusion.stream.closeAndDrain, - Effect.fail(RunExit.make({ code: decision.exitCode })), + RunConclusionVerdictFailed: (decision, raw) => + removeStaleFailureRecord(raw.conclusion).pipe( + Effect.andThen(closeAndExit(raw.conclusion, decision.exitCode)), + ), + RunConclusionFailed: (decision, raw) => + writeFailureRecord(raw.conclusion, decision.record).pipe( + Effect.andThen( + raw.conclusion.runEvents.emitFailureRecord(raw.conclusion.stream, decision.exitCode, decision.record), + ), + Effect.andThen(closeAndExit(raw.conclusion, decision.exitCode)), ), CommandRejected: ({ issue }) => Effect.fail(StrykerError.make({ message: `the run conclusion command was rejected: ${issue}` })), diff --git a/packages/stryker-js/src/conclude-run.ts b/packages/stryker-js/src/conclude-run.ts index ae2b3114..b4fcddbb 100644 --- a/packages/stryker-js/src/conclude-run.ts +++ b/packages/stryker-js/src/conclude-run.ts @@ -1,240 +1,88 @@ -/// -import { ErrorText } from '@systemfsoftware/stryker-js-instrumenter' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' -import * as Boolean from 'effect/Boolean' import * as Cause from 'effect/Cause' import * as CliError from 'effect/cli/CliError' import * as Exit from 'effect/Exit' +import * as Formatter from 'effect/Formatter' +import { pipe } from 'effect/Function' import * as Match from 'effect/Match' import * as Option from 'effect/Option' -import * as Order from 'effect/Order' import * as Predicate from 'effect/Predicate' import * as S from 'effect/Schema' -import { RunOutcomeCommand } from './RunOutcomeCommand.schema.js' import { - RunClassedObservation, - RunCliErrorObservation, - RunGenericFailureObservation, + type ObservedFailure, + RunFailedObservation, RunHelpObservation, - RunInterruptedObservation, + RunOutcomeCommand, type RunOutcomeObservation, - RunSchemaErrorObservation, RunSucceededClean, RunSucceededVerdict, - RunSurvivorsRejectedObservation, } from './RunOutcomeCommand.schema.js' -import { SurvivorsRejection } from './Survivors/mod.js' -const UNKNOWN_FAILURE = 'Unknown failure' -const MAX_TRAVERSAL_DEPTH = 10 +const CLI_BIN = 'stryker' +const MAX_CAUSE_DEPTH = 10 +const asEvidence = Option.liftPredicate(S.is(FailureRecord.FailureEvidence)) const asExitClass = Option.liftPredicate(S.is(Plugin.ExitClass)) -const nonEmptyText = Option.liftPredicate(S.is(S.NonEmptyString)) - -const failedExit = Option.liftPredicate(Exit.isFailure) - -const hasExitClass = Predicate.hasProperty('exitClass') - -const hasVerdict = Predicate.hasProperty('verdict') - -const hasCause = Predicate.hasProperty('cause') - -const hasReason = Predicate.hasProperty('reason') - -const hasMessageField = Predicate.hasProperty('message') - -const isUnseenObject = (value: A, seen: WeakSet): value is A & object => - Predicate.isObjectOrArray(value) && !seen.has(value) -const isReachableValue = (value: A, depth: number, seen: WeakSet): value is A & object => - depth <= MAX_TRAVERSAL_DEPTH && isUnseenObject(value, seen) - -const causeChildrenOf = (value: object): ReadonlyArray => - Option.match(Option.liftPredicate(value, hasCause), { - onNone: () => [], - onSome: (carrier) => Arr.filter(Arr.ensure(carrier.cause), Predicate.isObjectOrArray), - }) - -const visitReachableValue = ( - value: A, - depth: number, - seen: WeakSet, - visit: (node: object) => void, -): void => - Option.match( - Option.liftPredicate( - value, - (candidate): candidate is A & object => isReachableValue(candidate, depth, seen), - ), - { - onSome: (reachable) => { - seen.add(reachable) - visit(reachable) - causeChildrenOf(reachable).forEach((child) => visitReachableValue(child, depth + 1, seen, visit)) - }, - onNone: () => undefined, - }, +const textFieldOf = (node: A, key: K): Option.Option => + Option.flatMap( + Option.liftPredicate(node, Predicate.hasProperty(key)), + (carrier) => Option.liftPredicate(carrier[key], Predicate.isString), ) -const findReachableValue = ( - value: A, - depth: number, - seen: WeakSet, - read: (node: object) => Option.Option, -): Option.Option => - Option.match( - Option.liftPredicate( - value, - (candidate): candidate is A & object => isReachableValue(candidate, depth, seen), - ), - { - onSome: (reachable) => { - seen.add(reachable) - return Option.orElse( - read(reachable), - () => Arr.findFirst(causeChildrenOf(reachable), (child) => findReachableValue(child, depth + 1, seen, read)), - ) - }, - onNone: () => Option.none(), - }, +const declaredEvidenceOf = (node: A): Option.Option => + Option.flatMap( + Option.liftPredicate(node, Predicate.hasProperty('evidence')), + (carrier) => asEvidence(carrier.evidence), ) -const causePayloadOf = (reason: Cause.Reason): E | object | undefined => - Cause.isFailReason(reason) ? reason.error : objectPayloadOf(reason) - -const objectPayloadOf = (reason: Cause.Reason): object | undefined => - Option.getOrUndefined(Option.filter(dieDefectOf(reason), Predicate.isObject)) +const causeOf = (node: A, depth: number) => + Option.flatMap( + Option.filter(Option.liftPredicate(node, Predicate.hasProperty('cause')), () => depth < MAX_CAUSE_DEPTH), + (carrier) => Option.fromUndefinedOr(carrier.cause), + ) -const dieDefectOf = (reason: Cause.Reason) => - Cause.isDieReason(reason) ? Option.some(reason.defect) : Option.none() +const kindOf = (node: A): string => + Option.getOrElse(Option.orElse(textFieldOf(node, '_tag'), () => textFieldOf(node, 'name')), () => typeof node) -const failurePayloads = (exit: Exit.Exit): ReadonlyArray => +const messageOf = (node: A): string => Option.getOrElse( - Option.map(failedExit(exit), (failure) => failure.cause.reasons.map(causePayloadOf)), - (): ReadonlyArray => [], - ) - -const exitClassOf = (value: A): Plugin.ExitClass | undefined => - Option.getOrUndefined( - Option.flatMap( - Option.liftPredicate(value, hasExitClass), - (carrier) => asExitClass(carrier.exitClass), - ), + Option.orElse(textFieldOf(node, 'message'), () => Option.liftPredicate(node, Predicate.isString)), + () => Formatter.format(node, { ignoreToString: true }), ) -const appendExitClass = (value: object, out: Array): void => - Option.match(Option.fromUndefinedOr(exitClassOf(value)), { - onSome: (declared) => { - out.push(declared) - }, - onNone: () => undefined, - }) - -const collectExitClasses = (exit: Exit.Exit): Array => { - const out: Array = [] - const seen = new WeakSet() - failurePayloads(exit).forEach((payload) => - visitReachableValue(payload, 0, seen, (node) => appendExitClass(node, out)) - ) - return out -} +const STACK_FRAME = /^\s+at / -const causeTextOf = (value: A): Option.Option => - Option.map(ErrorText.causeTextOf(hasCause(value) ? value.cause : undefined), (decoded) => decoded.text) - -const reasonOf = (value: A): string | undefined => { - const declared = hasReason(value) ? value.reason : undefined - return Option.getOrUndefined( - Option.map(nonEmptyText(declared), (reason) => - Option.match(causeTextOf(value), { - onNone: () => reason, - onSome: (detail) => `${reason}: ${detail}`, - })), +const framesOf = (stack: string): Option.Option => + Option.liftPredicate( + Arr.dropWhile(stack.split('\n'), (line) => !STACK_FRAME.test(line)).join('\n'), + (frames) => frames.length > 0, ) -} - -const reasonTextFieldOf = (value: A): Option.Option => - Option.flatMap(Option.liftPredicate(value, hasReason), (carrier) => nonEmptyText(carrier.reason)) -const messageTextFieldOf = (value: A): Option.Option => - Option.flatMap(Option.liftPredicate(value, hasMessageField), (carrier) => nonEmptyText(carrier.message)) +const linkOf = (node: A): FailureRecord.CauseLink => ({ + kind: kindOf(node), + message: messageOf(node), + stack: Option.getOrNull(Option.flatMap(textFieldOf(node, 'stack'), framesOf)), +}) -const firstConfiguredText = (value: A): Option.Option => - Option.orElse(reasonTextFieldOf(value), () => messageTextFieldOf(value)) - -const configDetailAt = (value: object): Option.Option => - exitClassOf(value) === 'ConfigError' ? firstConfiguredText(value) : Option.none() - -const firstConfigErrorDetail = (exit: Exit.Exit): string | undefined => { - const seen = new WeakSet() - const roots = [...failurePayloads(exit)].reverse() - return Option.getOrUndefined( - Arr.findFirst(roots, (root) => findReachableValue(root, 0, seen, configDetailAt)), +const causeLinksOf = (node: A, depth: number): ReadonlyArray => + Arr.prepend( + Option.match(causeOf(node, depth), { + onNone: (): ReadonlyArray => [], + onSome: (child) => causeLinksOf(child, depth + 1), + }), + linkOf(node), ) -} - -const PRIMITIVE_REFINEMENTS = [ - Predicate.isString, - Predicate.isNumber, - Predicate.isBoolean, - Predicate.isBigInt, - Predicate.isSymbol, -] - -const isPrimitiveText = Predicate.some(PRIMITIVE_REFINEMENTS) - -const declaresReasonText = (value: A): boolean => hasReason(value) && Option.isSome(nonEmptyText(value.reason)) - -const remediationTextOf = (value: A): Option.Option => - S.is(SurvivorsRejection)(value) ? Option.some(value.remediation) : Option.none() -const reasonTextOf = (value: A): Option.Option => - declaresReasonText(value) ? Option.fromNullishOr(reasonOf(value)) : Option.none() - -const errorMessageTextOf = (value: A): Option.Option => - Option.flatMap(Option.filter(Option.some(value), Predicate.isError), (error) => nonEmptyText(error.message)) - -const primitiveTextOf = (value: A): Option.Option => - isPrimitiveText(value) ? Option.some(String(value)) : Option.none() - -const failureValueDescription = (value: A): Option.Option => - remediationTextOf(value).pipe( - Option.orElse(() => reasonTextOf(value)), - Option.orElse(() => errorMessageTextOf(value)), - Option.orElse(() => primitiveTextOf(value)), - ) - -const failureValueOf = (exit: Exit.Exit): E | undefined => - exit.pipe(failedExit, Option.flatMap((failure) => Cause.findErrorOption(failure.cause)), Option.getOrUndefined) - -const failureDescriptionOf = (exit: Exit.Exit): Option.Option => - exit.pipe( - failureValueOf, - failureValueDescription, - Option.orElse(() => exit.pipe(firstConfigErrorDetail, Option.fromNullishOr)), - Option.orElse(() => - exit.pipe(failedExit, Option.flatMap((failure) => failure.cause.pipe(Cause.pretty, nonEmptyText))) - ), +const evidenceWithin = (node: A, depth: number): Option.Option => + Option.orElse( + declaredEvidenceOf(node), + () => Option.flatMap(causeOf(node, depth), (child) => evidenceWithin(child, depth + 1)), ) -const describeFailureOf = (exit: Exit.Exit): string => - Option.getOrElse(failureDescriptionOf(exit), () => UNKNOWN_FAILURE) - -const showHelpErrors = (help: CliError.ShowHelp): ReadonlyArray => help.errors - -const showHelpErrorsOf = (value: A): Option.Option> => - Option.map(Option.liftPredicate(value, S.is(CliError.ShowHelp)), showHelpErrors) - -const cliErrorListOf = (exit: Exit.Exit): Option.Option> => { - const value = failureValueOf(exit) - return Option.match(showHelpErrorsOf(value), { - onSome: (errors) => Option.some(errors), - onNone: () => (CliError.isCliError(value) ? Option.some([value]) : Option.none()), - }) -} - const followingArgumentOf = (argv: readonly string[], option: string): Option.Option => Match.value(argv.indexOf(option)).pipe( Match.when((at) => at >= 0, (at) => Option.fromNullishOr(argv[at + 1])), @@ -255,160 +103,109 @@ const argumentHintOf = (error: CliError.CliError, argv: readonly string[]): Opti Match.orElse(() => Option.none()), ) -const unrecognizedHintOf = (exit: Exit.Exit, argv: readonly string[]): string | undefined => - exit.pipe( - cliErrorListOf, - Option.flatMap((errors) => Arr.findFirst(errors, (error) => argumentHintOf(error, argv))), - Option.getOrUndefined, +const cliErrorsOf = (error: CliError.CliError): ReadonlyArray => + S.is(CliError.ShowHelp)(error) ? error.errors : [error] + +const argumentsInvalidOf = (error: CliError.CliError, argv: readonly string[]): FailureRecord.FailureEvidence => ({ + _tag: 'ArgumentsInvalid', + stage: 'cli', + argument: pipe( + cliErrorsOf(error), + Arr.findFirst((each: CliError.CliError) => argumentHintOf(each, argv)), + Option.getOrNull, + ), +}) + +const configInvalidOf = (error: A): FailureRecord.FailureEvidence => ({ + _tag: 'ConfigInvalid', + stage: 'config', + detail: messageOf(error), +}) + +const CATALOG_GAP: FailureRecord.FailureEvidence = { _tag: 'CatalogGap', stage: 'run' } +const INTERRUPTED: FailureRecord.FailureEvidence = { _tag: 'RunInterrupted', stage: 'run' } +const CI_ONLY_EXIT_CODE = 4 + +interface RunContext { + readonly argv: readonly string[] + readonly cwd: string + readonly traceId: FailureRecord.TraceId | null +} + +const observedOf = ( + evidence: FailureRecord.FailureEvidence, + cause: ReadonlyArray, + context: RunContext, +): ObservedFailure => ({ + record: FailureRecord.recordOf(evidence, { + cause, + cwd: context.cwd, + argv: [CLI_BIN, ...context.argv], + env: [], + traceId: context.traceId, + }), + exitCode: FailureRecord.FailureCatalog[evidence._tag].exitCode ?? CI_ONLY_EXIT_CODE, +}) + +const undeclaredEvidenceOf = (payload: A, argv: readonly string[]): FailureRecord.FailureEvidence => + Option.getOrElse( + Option.orElse( + Option.map(Option.liftPredicate(payload, CliError.isCliError), (error) => argumentsInvalidOf(error, argv)), + () => Option.map(Option.liftPredicate(payload, S.isSchemaError), configInvalidOf), + ), + () => CATALOG_GAP, ) -const survivorsRejectionOf = (value: A): SurvivorsRejection | undefined => - S.is(SurvivorsRejection)(value) ? value : undefined - -const survivorsReasonOf = ( - survivors: SurvivorsRejection | undefined, -): 'no-report' | 'mismatch' | undefined => survivors?.reason - -const survivorsDiagnosticOf = (survivors: SurvivorsRejection | undefined): string | undefined => survivors?.remediation - -const omitUnknownFailure = (diagnostic: string): string | undefined => - diagnostic === UNKNOWN_FAILURE ? undefined : diagnostic - -const hasOnlyInterruptsOf = (exit: Exit.Exit): boolean => - Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) - -const carriesCliError = (value: A): boolean => value !== undefined && CliError.isCliError(value) - -const carriesSchemaError = (value: A): boolean => value !== undefined && S.isSchemaError(value) - -const helpErrorCountOf = (value: A): number | undefined => - S.is(CliError.ShowHelp)(value) ? value.errors.length : undefined - -const verdictExitClassOf = (value: A): Plugin.ExitClass | undefined => - hasVerdict(value) ? Option.getOrUndefined(asExitClass(value.verdict)) : undefined - -const successExitClassOf = (exit: Exit.Exit): Plugin.ExitClass | undefined => - Exit.isSuccess(exit) ? verdictExitClassOf(exit.value) : undefined - -const bySeverity: Order.Order = Order.mapInput( - Order.Number, - (exitClass: Plugin.ExitClass) => Plugin.ExitClass.literals.indexOf(exitClass), -) - -const highestExitClassOf = (pending: ReadonlyArray): Plugin.ExitClass | undefined => - Option.getOrUndefined(Arr.last(Arr.sort(pending, bySeverity))) - -const observationOf = ( - { exit, value, survivors, argv }: { - readonly exit: Exit.Exit - readonly value: E | object | undefined - readonly survivors: SurvivorsRejection | undefined - readonly argv: readonly string[] - }, -): RunOutcomeObservation => { - const unrecognized = Option.getOrNull(Option.fromUndefinedOr(unrecognizedHintOf(exit, argv))) - const configDetail = exit.pipe(firstConfigErrorDetail, Option.fromUndefinedOr, Option.getOrNull) - const diagnostic = exit.pipe(describeFailureOf, omitUnknownFailure, Option.fromUndefinedOr, Option.getOrNull) - - return Boolean.match(Exit.isSuccess(exit), { - onTrue: () => - exit.pipe( - successExitClassOf, - Option.fromUndefinedOr, - Option.match({ - onNone: () => RunSucceededClean.make({}), - onSome: (exitClass) => RunSucceededVerdict.make({ exitClass, diagnostic }), - }), - ), - onFalse: () => - Boolean.match(hasOnlyInterruptsOf(exit), { - onTrue: () => RunInterruptedObservation.make({}), - onFalse: () => - Option.match(Option.fromUndefinedOr(helpErrorCountOf(value)), { - onSome: (errorCount) => RunHelpObservation.make({ errorCount, unrecognized }), - onNone: () => - Boolean.match(carriesCliError(value), { - onTrue: () => RunCliErrorObservation.make({ unrecognized }), - onFalse: () => - Option.match(Option.fromUndefinedOr(survivorsReasonOf(survivors)), { - onSome: (reason) => - RunSurvivorsRejectedObservation.make({ - reason, - diagnostic: Option.getOrNull(Option.fromUndefinedOr(survivorsDiagnosticOf(survivors))), - }), - onNone: () => - Boolean.match(carriesSchemaError(value), { - onTrue: () => RunSchemaErrorObservation.make({ configDetail }), - onFalse: () => - exit.pipe( - collectExitClasses, - highestExitClassOf, - Option.fromUndefinedOr, - Option.match({ - onSome: (exitClass) => - RunClassedObservation.make({ exitClass, configDetail, diagnostic }), - onNone: () => RunGenericFailureObservation.make({ diagnostic }), - }), - ), - }), - }), - }), - }), - }), - }) -} +const observedFailureOf = (payload: A, context: RunContext): ObservedFailure => + observedOf( + Option.getOrElse(evidenceWithin(payload, 0), () => undeclaredEvidenceOf(payload, context.argv)), + causeLinksOf(payload, 0), + context, + ) -export const runOutcomeCommandOf = ( - { argv, exit }: { readonly exit: Exit.Exit; readonly argv: readonly string[] }, -): RunOutcomeCommand => { - const value = failureValueOf(exit) - return RunOutcomeCommand.make({ - observation: observationOf({ exit, value, survivors: survivorsRejectionOf(value), argv }), - }) -} +const payloadOf = (reason: Cause.Reason) => + Cause.isFailReason(reason) + ? Option.some(reason.error) + : Option.map(Option.liftPredicate(reason, Cause.isDieReason), (die) => die.defect) -if (import.meta.vitest !== void 0) { - const { it } = await import('@systemfsoftware/vitest') +const unpayloadedFailureOf = (cause: Cause.Cause, context: RunContext): ObservedFailure => + Cause.hasInterruptsOnly(cause) + ? observedOf(INTERRUPTED, [], context) + : observedOf(CATALOG_GAP, [cause.pipe(Cause.pretty, linkOf)], context) - const severityOf = (exitClass: Plugin.ExitClass): number => Plugin.ExitClass.literals.indexOf(exitClass) +const failuresOf = (cause: Cause.Cause, context: RunContext): Arr.NonEmptyReadonlyArray => + Option.getOrElse( + Arr.match(Arr.flatMap(cause.reasons, (reason) => Option.toArray(payloadOf(reason))), { + onEmpty: () => Option.none(), + onNonEmpty: (payloads) => Option.some(Arr.map(payloads, (payload) => observedFailureOf(payload, context))), + }), + () => Arr.of(unpayloadedFailureOf(cause, context)), + ) - const classedKeyOf = (command: RunOutcomeCommand): string => - S.is(RunClassedObservation)(command.observation) - ? `${command.observation.exitClass}|${command.observation.configDetail}` - : 'not-classed' +const isHelpRequest = (cause: Cause.Cause): boolean => + Option.exists(Cause.findErrorOption(cause), (error) => S.is(CliError.ShowHelp)(error) && error.errors.length === 0) - const genericDiagnosticOf = (command: RunOutcomeCommand): string | null => - S.is(RunGenericFailureObservation)(command.observation) ? command.observation.diagnostic : null +const failureObservationOf = (cause: Cause.Cause, context: RunContext): RunOutcomeObservation => + isHelpRequest(cause) + ? RunHelpObservation.make({}) + : RunFailedObservation.make({ failures: failuresOf(cause, context) }) - it.prop( - '∀text_RunOutcomeCommand_≡PrimitiveFailureDiagnostic', - { of: [S.String], subject: runOutcomeCommandOf }, - (subject, [text]) => { - const command = subject({ exit: Exit.fail(text), argv: [] }) - return genericDiagnosticOf(command) === (text === UNKNOWN_FAILURE ? null : text) - }, +const verdictExitClassOf = (value: A): Option.Option => + Option.flatMap( + Option.liftPredicate(value, Predicate.hasProperty('verdict')), + (carrier) => asExitClass(carrier.verdict), ) - it.prop( - '∀detail_RunOutcomeCommand_≡ConfigDetail', - { of: [S.NonEmptyString], subject: runOutcomeCommandOf }, - (subject, [detail]) => { - const command = subject({ exit: Exit.fail({ exitClass: 'ConfigError', reason: detail }), argv: [] }) - return classedKeyOf(command) === `ConfigError|${detail}` - }, - ) +const successObservationOf = (value: A): RunOutcomeObservation => + Option.match(verdictExitClassOf(value), { + onNone: () => RunSucceededClean.make({}), + onSome: (exitClass) => RunSucceededVerdict.make({ exitClass }), + }) - it.prop( - '∀ec_RunOutcomeCommand_≡HighestExitClass', - { of: [Plugin.ExitClass, Plugin.ExitClass, Plugin.ExitClass], subject: runOutcomeCommandOf }, - (subject, [left, middle, right]) => { - const exit = Exit.fail({ exitClass: left, cause: { exitClass: middle, cause: { exitClass: right } } }) - const command = subject({ exit, argv: [] }) - return ( - S.is(RunClassedObservation)(command.observation) && - severityOf(command.observation.exitClass) === - Math.max(severityOf(left), severityOf(middle), severityOf(right)) - ) - }, - ) -} +export const runOutcomeCommandOf = (input: RunContext & { readonly exit: Exit.Exit }): RunOutcomeCommand => + RunOutcomeCommand.make({ + observation: Exit.match(input.exit, { + onSuccess: successObservationOf, + onFailure: (cause) => failureObservationOf(cause, input), + }), + }) diff --git a/packages/stryker-js/src/drivers/node.ts b/packages/stryker-js/src/drivers/node.ts index 6b3c8ca4..efd9fd3f 100644 --- a/packages/stryker-js/src/drivers/node.ts +++ b/packages/stryker-js/src/drivers/node.ts @@ -1,7 +1,7 @@ import { NodeFileSystem, NodePath, NodeSocket, NodeStdio } from '@effect/platform-node' import * as NodeChildProcessSpawner from '@effect/platform-node-shared/NodeChildProcessSpawner' import * as NodeCrypto from '@effect/platform-node-shared/NodeCrypto' -import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import * as Crypto from 'effect/Crypto' import * as Duration from 'effect/Duration' import * as Effect from 'effect/Effect' @@ -14,16 +14,42 @@ import * as ChildProcessSpawner from 'effect/process/ChildProcessSpawner' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -import { classifyWorkerExit, ClassifyWorkerExitCommand } from '../classify-worker-exit.workflow.js' +import { + classifyWorkerExit, + ClassifyWorkerExitCommand, + type ClassifyWorkerExitDecision, +} from '../classify-worker-exit.workflow.js' import { GitDiff } from '../git-diff.service.js' import type { EnginePorts } from '../run/StageServices.service.js' import { make as makeSpawnedSocketWorker } from '../spawned-socket-worker.handle.js' import { layerWorkerProtocol } from '../worker-protocol.blueprint.js' -import { ChildProcessCrashedError, OutOfMemoryError } from '../Worker.schema.js' +import { ChildProcessCrashedError, OutOfMemoryError, type WorkerExit } from '../Worker.schema.js' import { WorkerLauncher, type WorkerSpawnParams } from '../WorkerLauncher.service.js' const WORKER_TERMINATION_GRACE = Duration.seconds(5) +const workerExitError = ( + decision: ClassifyWorkerExitDecision, + workerKind: FailureRecord.WorkerKind, +): WorkerExit => + Match.value(decision).pipe( + Match.tag( + 'WorkerOutOfMemory', + ({ pid, exitCode }): WorkerExit => OutOfMemoryError.make({ pid, exitCode, workerKind }), + ), + Match.tag( + 'WorkerCrashed', + ({ pid, exitCode }): WorkerExit => + ChildProcessCrashedError.make({ + pid, + exit: { _tag: 'Code', code: exitCode }, + workerKind, + cause: 'worker exited before it accepted the RPC connection', + }), + ), + Match.exhaustive, + ) + const restrictToOwnerOrWarn = (fs: FileSystem.FileSystem, file: string) => fs.chmod(file, 0o600).pipe( Effect.tapError((cause) => @@ -43,82 +69,61 @@ const nodeWorkerLauncherLayer = (childEnv: Readonly>) => const path = yield* Path.Path const spawner = yield* ChildProcessSpawner.ChildProcessSpawner - return { - spawn: Effect.fn(SpanTaxonomy.Spans.workerSpawn.name)( - function*(params: WorkerSpawnParams) { - const workerDir = yield* fs.makeTempDirectoryScoped({ prefix: params.tempDirPrefix }) - const workerId = yield* crypto.randomUUIDv4 - const socketPath = Match.value(globalThis.process.platform).pipe( - Match.when('win32', () => `\\\\.\\pipe\\stryker-worker-${workerId}`), - Match.orElse(() => path.join(workerDir, 'worker.sock')), - ) - const optionsFile = path.join(workerDir, 'options.json') - yield* fs.writeFileString(optionsFile, params.optionsJson) - yield* restrictToOwnerOrWarn(fs, optionsFile) - - const entrypointPath = yield* path.fromFileUrl(new URL(params.entrypoint)) - const handle = yield* ChildProcess.make( - globalThis.process.execPath, - [...params.execArgv, entrypointPath], + const spawnWorker = Effect.fn(SpanTaxonomy.Spans.workerSpawn.name)(function*(params: WorkerSpawnParams) { + const workerDir = yield* fs.makeTempDirectoryScoped({ prefix: params.tempDirPrefix }) + const workerId = yield* crypto.randomUUIDv4 + const socketPath = Match.value(globalThis.process.platform).pipe( + Match.when('win32', () => `\\\\.\\pipe\\stryker-worker-${workerId}`), + Match.orElse(() => path.join(workerDir, 'worker.sock')), + ) + const optionsFile = path.join(workerDir, 'options.json') + yield* fs.writeFileString(optionsFile, params.optionsJson) + yield* restrictToOwnerOrWarn(fs, optionsFile) + + const entrypointPath = yield* path.fromFileUrl(new URL(params.entrypoint)) + const handle = yield* ChildProcess.make( + globalThis.process.execPath, + [...params.execArgv, entrypointPath], + { + cwd: params.workingDirectory, + extendEnv: true, + env: { STRYKER_WORKER_DIR: workerDir, STRYKER_SOCKET: socketPath, ...childEnv, ...params.env }, + stderr: 'inherit', + forceKillAfter: WORKER_TERMINATION_GRACE, + }, + ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)) + + const clientLayer = layerWorkerProtocol(NodeSocket.layerNet({ path: socketPath })) + + const exited = handle.exitCode.pipe( + Effect.orDie, + Effect.flatMap((exitCode) => + Result.match( + classifyWorkerExit(ClassifyWorkerExitCommand.make({ pid: Number(handle.pid), exitCode })), { - cwd: params.workingDirectory, - extendEnv: true, - env: { STRYKER_WORKER_DIR: workerDir, STRYKER_SOCKET: socketPath, ...childEnv, ...params.env }, - stderr: 'inherit', - forceKillAfter: WORKER_TERMINATION_GRACE, + onFailure: (refused) => Effect.fail(refused), + onSuccess: (decision) => Effect.fail(workerExitError(decision, params.workerKind)), }, - ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)) - - const clientLayer = layerWorkerProtocol(NodeSocket.layerNet({ path: socketPath })) - - const exited = handle.exitCode.pipe( - Effect.orDie, - Effect.flatMap((exitCode) => - Result.match( - classifyWorkerExit(ClassifyWorkerExitCommand.make({ pid: Number(handle.pid), exitCode })), - { - onFailure: (refused) => Effect.fail(refused), - onSuccess: (decision) => - Match.value(decision).pipe( - Match.tag( - 'WorkerOutOfMemory', - (outOfMemory) => - Effect.fail( - OutOfMemoryError.make({ pid: outOfMemory.pid, exitCode: outOfMemory.exitCode }), - ), - ), - Match.tag( - 'WorkerCrashed', - (crashed) => - Effect.fail( - ChildProcessCrashedError.make({ - pid: crashed.pid, - exit: { _tag: 'Code', code: crashed.exitCode }, - cause: 'worker exited before it accepted the RPC connection', - }), - ), - ), - Match.exhaustive, - ), - }, - ) - ), ) + ), + ) - return makeSpawnedSocketWorker({ pid: Number(handle.pid), clientLayer, exited }) - }, - (spawned) => - spawned.pipe( - Effect.catchIf(S.is(ChildProcessCrashedError), (error) => Effect.fail(error), () => - Effect.fail( - ChildProcessCrashedError.make({ - pid: 0, - exit: { _tag: 'Code', code: 1 }, - cause: 'worker spawn failed', - }), - )), - ), - ), + return makeSpawnedSocketWorker({ pid: Number(handle.pid), clientLayer, exited }) + }) + + return { + spawn: (params: WorkerSpawnParams) => + spawnWorker(params).pipe( + Effect.catchIf(S.is(ChildProcessCrashedError), (error) => Effect.fail(error), () => + Effect.fail( + ChildProcessCrashedError.make({ + pid: 0, + workerKind: params.workerKind, + exit: { _tag: 'Code', code: 1 }, + cause: 'worker spawn failed', + }), + )), + ), } }), ) diff --git a/packages/stryker-js/src/dry-run.workflow.ts b/packages/stryker-js/src/dry-run.workflow.ts index 7e810cab..67e7e1c0 100644 --- a/packages/stryker-js/src/dry-run.workflow.ts +++ b/packages/stryker-js/src/dry-run.workflow.ts @@ -1,27 +1,16 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import * as Match from 'effect/Match' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -export class DryRunError extends S.TaggedError()('DryRunError', { - stage: S.Literals(['dryRun', 'dryRunNoTests']), - reason: S.String, -}) { - override get message(): string { - return this.reason - } -} - -export class FailedTestSummary extends S.Class('FailedTestSummary')({ - name: S.String, - failureMessage: S.String, -}) {} +import { RunFailure } from './Run.schema.js' export class DryRunCommand extends S.TaggedClass()('DryRunCommand', { status: S.Literals(['Complete', 'Error', 'Timeout']), testCount: S.Finite, failedTestCount: S.Finite, - failedTests: S.Array(FailedTestSummary), + failedTests: S.Array(FailureRecord.FailedTestEvidence), allowEmpty: S.Boolean, errorMessage: S.optional(S.String), reason: S.optional(S.String), @@ -46,7 +35,7 @@ export class DryRunPassed extends S.TaggedClass()('DryRunPassed', export class DryRunFailed extends S.TaggedClass()('DryRunFailed', { testCount: S.Finite, failedTestCount: S.Finite, - failedTests: S.Array(FailedTestSummary), + failedTests: S.Array(FailureRecord.FailedTestEvidence), }) { readonly [DryRunDecisionTypeId] = DryRunDecisionTypeId } @@ -62,7 +51,7 @@ const completeOutcomeOf = (command: DryRunCommand): CompleteOutcome => Match.orElse((): CompleteOutcome => 'passed'), ) -const decideComplete = (command: DryRunCommand): Result.Result => +const decideComplete = (command: DryRunCommand): Result.Result => Match.value({ outcome: completeOutcomeOf(command), testCount: command.testCount, @@ -71,9 +60,9 @@ const decideComplete = (command: DryRunCommand): Result.Result Result.fail( - DryRunError.make({ - stage: 'dryRunNoTests', - reason: 'No tests were executed. Stryker will exit prematurely. Please check your configuration.', + RunFailure.make({ + evidence: { _tag: 'BaselineFoundNoTests', stage: 'dryRun' }, + detail: 'No tests were executed. Stryker will exit prematurely. Please check your configuration.', }), )), Match.when({ outcome: 'failed' }, ({ testCount, failedTestCount, failedTests }) => @@ -99,28 +88,29 @@ const reasonOrDefault = (reason: string | undefined, fallback: string): string = Match.orElse((present) => present), ) -const decide = (command: DryRunCommand): Result.Result => +const decide = (command: DryRunCommand): Result.Result => Match.value(command.status).pipe( Match.when('Error', () => Result.fail( - DryRunError.make({ - stage: 'dryRun', - reason: reasonOrDefault(command.errorMessage, 'Dry run error'), + RunFailure.make({ + evidence: { _tag: 'BaselineErrored', stage: 'dryRun' }, + detail: reasonOrDefault(command.errorMessage, 'Dry run error'), }), )), Match.when('Timeout', () => Result.fail( - DryRunError.make({ - stage: 'dryRun', - reason: reasonOrDefault(command.reason, 'Initial test run timed out'), + RunFailure.make({ + evidence: { _tag: 'BaselineTimedOut', stage: 'dryRun' }, + detail: reasonOrDefault(command.reason, 'Initial test run timed out'), }), )), Match.when('Complete', () => decideComplete(command)), Match.exhaustive, ) + export const dryRun = Workflow.make({ command: DryRunCommand, decision: S.Union([DryRunPassed, DryRunFailed]), - error: DryRunError, + error: RunFailure, decide, }) diff --git a/packages/stryker-js/src/events/mod.ts b/packages/stryker-js/src/events/mod.ts index 6010a038..4765707e 100644 --- a/packages/stryker-js/src/events/mod.ts +++ b/packages/stryker-js/src/events/mod.ts @@ -1 +1 @@ -export { PrepareError, StageError } from '../Run.schema.js' +export { RunFailure } from '../Run.schema.js' diff --git a/packages/stryker-js/src/frame-run-event.workflow.ts b/packages/stryker-js/src/frame-run-event.workflow.ts index c35933f4..a69675a5 100644 --- a/packages/stryker-js/src/frame-run-event.workflow.ts +++ b/packages/stryker-js/src/frame-run-event.workflow.ts @@ -1,5 +1,5 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' -import { OutputMode, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, OutputMode, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { Report } from '@systemfsoftware/stryker-js-plugin-interface' import * as Boolean from 'effect/Boolean' import * as Match from 'effect/Match' @@ -32,10 +32,16 @@ export class FrameRunEventCommand extends S.TaggedClass()( static readonly [Workflow.InstrumentationBrand] = {} as const } +const StderrOutput = S.Union([ + S.TaggedStruct('StderrText', { text: S.String }), + S.TaggedStruct('StderrRecord', { record: FailureRecord.FailureRecord }), +]) +type StderrOutput = typeof StderrOutput.Type + export class EventFramed extends S.TaggedClass()('EventFramed', { state: FramingState, event: RunEvent.RunEvent, - stderrLine: S.NullOr(S.String), + stderr: S.NullOr(StderrOutput), }) { readonly [FrameRunEventTypeId] = FrameRunEventTypeId } @@ -44,7 +50,7 @@ export class EventSuppressed extends S.TaggedClass()( 'EventSuppressed', { state: FramingState, - stderrLine: S.NullOr(S.String), + stderr: S.NullOr(StderrOutput), }, ) { readonly [FrameRunEventTypeId] = FrameRunEventTypeId @@ -111,19 +117,21 @@ const formatTotal = (total: number | null): string => onSome: (val) => String(val), }) -const formatStderrEvent = (event: RunEvent.RunEvent): string | null => +const text = (line: string): StderrOutput => ({ _tag: 'StderrText', text: line }) + +const formatStderrEvent = (event: RunEvent.RunEvent): StderrOutput | null => Match.value(event).pipe( - Match.tag('plan', (e) => `plan ${e.total} mutants`), - Match.tag('phase', (e) => `phase ${e.phase}`), + Match.tag('plan', (e) => text(`plan ${e.total} mutants`)), + Match.tag('phase', (e) => text(`phase ${e.phase}`)), Match.tag( 'tick', - (e) => `${e.completed}/${formatTotal(e.total)} elapsed ${e.elapsedMs}ms`, + (e) => text(`${e.completed}/${formatTotal(e.total)} elapsed ${e.elapsedMs}ms`), ), Match.tag( 'verdict', - (e) => `score ${formatScore(e.score)} killed ${e.counts.killed} survived ${e.counts.survived}`, + (e) => text(`score ${formatScore(e.score)} killed ${e.counts.killed} survived ${e.counts.survived}`), ), - Match.tag('error', (e) => `error ${e.error}`), + Match.tag('error', (e): StderrOutput => ({ _tag: 'StderrRecord', record: e.record })), Match.tag('stream', () => null), Match.tag('mutantTested', () => null), Match.tag('help', () => null), @@ -134,7 +142,7 @@ const formatStderrEvent = (event: RunEvent.RunEvent): string | null => Match.exhaustive, ) -const stderrLineFor = (state: FramingState, event: RunEvent.RunEvent): string | null => +const stderrFor = (state: FramingState, event: RunEvent.RunEvent): StderrOutput | null => Boolean.match(Boolean.and(state.mode === 'human', !state.terminalSeen), { onTrue: () => formatStderrEvent(event), onFalse: () => null, @@ -146,7 +154,7 @@ const decideFrame = ( command: FrameRunEventCommand, ): Result.Result => { const nextState = noteState(command.state, command.event) - const stderrLine = stderrLineFor(command.state, command.event) + const stderr = stderrFor(command.state, command.event) const framed = shouldFrame(command.state) return Match.value(framed).pipe( @@ -155,14 +163,14 @@ const decideFrame = ( EventFramed.make({ state: nextState, event: command.event, - stderrLine, + stderr, }), )), Match.when(false, () => Result.succeed( EventSuppressed.make({ state: nextState, - stderrLine, + stderr, }), )), Match.exhaustive, diff --git a/packages/stryker-js/src/gate-new-survivors.workflow.ts b/packages/stryker-js/src/gate-new-survivors.workflow.ts index b402569d..741fc463 100644 --- a/packages/stryker-js/src/gate-new-survivors.workflow.ts +++ b/packages/stryker-js/src/gate-new-survivors.workflow.ts @@ -1,5 +1,6 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' -import { Mutant, Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Boolean from 'effect/Boolean' import * as HashSet from 'effect/HashSet' @@ -19,9 +20,6 @@ export const GateEntry = S.Struct({ export type GateEntry = typeof GateEntry.Type -const GATE_EXIT_CLASS = 'VerdictFail' satisfies Plugin.ExitClass -const GATE_REMEDIATION = 'kill the new survivors, or accept them with `stryker gate --update-baseline`' - /** * R28: `Survived` and `NoCoverage` are survivors; a `Timeout` or a runtime * error is not. Written out here rather than read off `SurvivedStatusSchema` @@ -55,28 +53,33 @@ export class GateNewSurvivorsCommand extends S.TaggedClass()('GateRejected', { - newSurvivors: S.Array(GateEntry), + newSurvivors: S.NonEmptyArray(GateEntry), unchecked: Mutant.MutantId.pipe(S.Array), }) { - readonly exitClass = GATE_EXIT_CLASS + get evidence(): FailureRecord.FailureEvidence { + return { + _tag: 'NewSurvivors', + stage: 'gate', + survivors: Arr.map(this.newSurvivors, (entry) => ({ + mutantId: entry.id, + file: entry.fileName, + line: entry.line, + })), + unchecked: this.unchecked.length, + } + } override get message(): string { - return [ - `stryker gate: ${this.newSurvivors.length} new survivor(s) absent from the committed baseline:`, - ...this.newSurvivors.map((entry) => ` ${entry.fileName}:${entry.line} ${entry.id}`), - ...Option.match(Option.liftPredicate(this.unchecked, (unchecked) => unchecked.length > 0), { - onNone: (): ReadonlyArray => [], - onSome: (unchecked) => [`${unchecked.length} mutant(s) unchecked`], - }), - GATE_REMEDIATION, - ].join('\n') + return `stryker gate: ${this.newSurvivors.length} new survivor(s) absent from the committed baseline` } } export class GateInputUnusable extends S.TaggedError()('GateInputUnusable', { reason: S.String, }) { - readonly exitClass = 'ConfigError' satisfies Plugin.ExitClass + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'ConfigInvalid', stage: 'gate', detail: this.reason } + } override get message(): string { return `stryker gate: ${this.reason}` @@ -130,9 +133,9 @@ const clearedOrRejected = ( ): Result.Result => { const unchecked = uncheckedIdsOf(command.entries) const newSurvivors = newSurvivorsOf(command.entries, committed) - return Option.match(Arr.head(newSurvivors), { - onNone: () => Result.succeed(GateCleared.make({ baseline: null, unchecked })), - onSome: () => Result.fail(GateRejected.make({ newSurvivors, unchecked })), + return Arr.match(newSurvivors, { + onEmpty: () => Result.succeed(GateCleared.make({ baseline: null, unchecked })), + onNonEmpty: (survivors) => Result.fail(GateRejected.make({ newSurvivors: survivors, unchecked })), }) } diff --git a/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts b/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts index 283fdcd6..fe7b742e 100644 --- a/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts +++ b/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts @@ -1,4 +1,5 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import * as Match from 'effect/Match' import * as Option from 'effect/Option' @@ -12,8 +13,6 @@ export class DryRunObservation extends S.TaggedClass()('DryRu static readonly [Workflow.InstrumentationBrand] = {} as const } -const FailureSummarySchema = S.Struct({ name: S.String, failureMessage: S.String }) - const DryRunObservationDecisionTypeId: unique symbol = Symbol.for( '@systemfsoftware/stryker-js/DryRunObservationDecision', ) @@ -22,7 +21,7 @@ type DryRunObservationDecisionTypeId = typeof DryRunObservationDecisionTypeId export class DryRunObservedComplete extends S.TaggedClass()('DryRunObservedComplete', { testCount: S.Finite, failedTestCount: S.Finite, - failedTests: S.Array(FailureSummarySchema), + failedTests: S.Array(FailureRecord.FailedTestEvidence), }) { readonly [DryRunObservationDecisionTypeId] = DryRunObservationDecisionTypeId } @@ -41,12 +40,21 @@ export class DryRunObservedTimedOut extends S.TaggedClass(value: A | undefined): A | null => Option.fromUndefinedOr(value).pipe(Option.getOrNull) + +const failedTestEvidenceOf = (test: TestRunner.FailedTestResult): FailureRecord.FailedTestEvidence => ({ + id: test.id, + name: test.name, + file: orNullOf(test.fileName), + location: orNullOf(test.location), + message: test.failureMessage, + stack: orNullOf(test.stack), +}) + +const failedTestEvidencesOf = ( tests: readonly TestRunner.TestResult[], -): readonly (typeof FailureSummarySchema.Type)[] => - tests - .filter((test): test is TestRunner.FailedTestResult => test.status === 'failed') - .map((test) => ({ name: test.name, failureMessage: test.failureMessage })) +): readonly FailureRecord.FailedTestEvidence[] => + tests.filter((test): test is TestRunner.FailedTestResult => test.status === 'failed').map(failedTestEvidenceOf) const decide = (command: DryRunObservation): Result.Result => Match.value(command.dryRunResult).pipe( @@ -55,7 +63,7 @@ const decide = (command: DryRunObservation): Result.Result test.status === 'failed').length, - failedTests: failedTestSummariesOf(complete.tests), + failedTests: failedTestEvidencesOf(complete.tests), }), )), Match.discriminator('status')( diff --git a/packages/stryker-js/src/mutation-reporting.service.ts b/packages/stryker-js/src/mutation-reporting.service.ts index c36ab4e1..a0d1180f 100644 --- a/packages/stryker-js/src/mutation-reporting.service.ts +++ b/packages/stryker-js/src/mutation-reporting.service.ts @@ -415,11 +415,10 @@ const reportMutantOf = ( ...timeoutFieldsOf(mutant, evidence), }) -const reportTestOf = (test: TestRunner.TestResult, remap: TestIdRemap) => - Option.match(Option.fromUndefinedOr(test.startPosition), { - onNone: () => ({ id: remap.testId(test.id), name: test.name }), - onSome: (start) => ({ id: remap.testId(test.id), name: test.name, location: { start } }), - }) +const reportTestOf = (test: TestRunner.TestResult, remap: TestIdRemap) => ({ + id: remap.testId(test.id), + name: test.name, +}) const groupMutants = (input: FileResultsInput): Effect.Effect> => Stream.fromIterable(input.mutants).pipe( diff --git a/packages/stryker-js/src/plan-run-conclusion.workflow.ts b/packages/stryker-js/src/plan-run-conclusion.workflow.ts index 5f9e2652..137aa805 100644 --- a/packages/stryker-js/src/plan-run-conclusion.workflow.ts +++ b/packages/stryker-js/src/plan-run-conclusion.workflow.ts @@ -1,106 +1,77 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as Boolean from 'effect/Boolean' +import * as Match from 'effect/Match' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -import { - RunConfigFailed, - RunFailed, - RunInterrupted, - RunParseFailed, - RunSurvivorsRejected, -} from './classify-run-outcome.workflow.js' -import { RunOutcomeCommand } from './RunOutcomeCommand.schema.js' +import { type RunFailed, RunOk, RunOutcomeDecision, type RunVerdictFailed } from './classify-run-outcome.workflow.js' const PlanRunConclusionTypeId: unique symbol = Symbol.for('@systemfsoftware/stryker-js/PlanRunConclusion') type PlanRunConclusionTypeId = typeof PlanRunConclusionTypeId -export const FailedRunOutcomeSchema = S.Union([ - RunParseFailed, - RunSurvivorsRejected, - RunConfigFailed, - RunFailed, - RunInterrupted, -]) - -export const RunOutcomeTag = S.Literals([ - 'RunOk', - 'RunParseFailed', - 'RunSurvivorsRejected', - 'RunConfigFailed', - 'RunFailed', - 'RunInterrupted', -]) -export type RunOutcomeTag = typeof RunOutcomeTag.Type - export class PlanRunConclusionCommand extends S.TaggedClass()('PlanRunConclusionCommand', { - command: RunOutcomeCommand, + decision: RunOutcomeDecision, machine: S.Boolean, - exitCode: Plugin.ExitCode, - outcome: RunOutcomeTag, - error: S.String, }) { static readonly [Workflow.InstrumentationBrand] = { machine: 'stryker.run_conclusion.machine', - exitCode: 'stryker.run.exit_code', - outcome: 'stryker.run.outcome', - error: 'stryker.run.error', } as const } export class RunConclusionEmittedOk extends S.TaggedClass()('RunConclusionEmittedOk', { - command: RunOutcomeCommand, + ok: RunOk, }) { readonly [PlanRunConclusionTypeId] = PlanRunConclusionTypeId } -export class RunConclusionEmittedFailed extends S.TaggedClass()( - 'RunConclusionEmittedFailed', - { - command: RunOutcomeCommand, - exitCode: Plugin.ExitCode, - }, -) { +export class RunConclusionQuietOk extends S.TaggedClass()('RunConclusionQuietOk', {}) { readonly [PlanRunConclusionTypeId] = PlanRunConclusionTypeId } -export class RunConclusionQuietOk extends S.TaggedClass()('RunConclusionQuietOk', {}) { +export class RunConclusionVerdictFailed extends S.TaggedClass()( + 'RunConclusionVerdictFailed', + { exitCode: Plugin.ExitCode }, +) { readonly [PlanRunConclusionTypeId] = PlanRunConclusionTypeId } -export class RunConclusionQuietFailed extends S.TaggedClass()('RunConclusionQuietFailed', { +export class RunConclusionFailed extends S.TaggedClass()('RunConclusionFailed', { exitCode: Plugin.ExitCode, + record: FailureRecord.FailureRecord, }) { readonly [PlanRunConclusionTypeId] = PlanRunConclusionTypeId } export const PlanRunConclusionDecision = S.Union([ RunConclusionEmittedOk, - RunConclusionEmittedFailed, RunConclusionQuietOk, - RunConclusionQuietFailed, + RunConclusionVerdictFailed, + RunConclusionFailed, ]) export type PlanRunConclusionDecision = typeof PlanRunConclusionDecision.Type -const emittedOf = (command: PlanRunConclusionCommand): PlanRunConclusionDecision => - Boolean.match(command.exitCode === 0, { - onTrue: () => RunConclusionEmittedOk.make({ command: command.command }), - onFalse: () => RunConclusionEmittedFailed.make({ command: command.command, exitCode: command.exitCode }), +const okConclusionOf = (ok: RunOk, machine: boolean): PlanRunConclusionDecision => + Boolean.match(machine, { + onTrue: () => RunConclusionEmittedOk.make({ ok }), + onFalse: () => RunConclusionQuietOk.make({}), }) -const quietOf = (command: PlanRunConclusionCommand): PlanRunConclusionDecision => - Boolean.match(command.exitCode === 0, { - onTrue: () => RunConclusionQuietOk.make({}), - onFalse: () => RunConclusionQuietFailed.make({ exitCode: command.exitCode }), - }) +const verdictConclusionOf = (failed: RunVerdictFailed): PlanRunConclusionDecision => + RunConclusionVerdictFailed.make({ exitCode: failed.code }) + +const failedConclusionOf = (failed: RunFailed): PlanRunConclusionDecision => + RunConclusionFailed.make({ exitCode: failed.code, record: failed.record }) const decide = (command: PlanRunConclusionCommand): Result.Result => Result.succeed( - Boolean.match(command.machine, { - onTrue: () => emittedOf(command), - onFalse: () => quietOf(command), - }), + Match.value(command.decision).pipe( + Match.tag('RunOk', (ok) => okConclusionOf(ok, command.machine)), + Match.tag('RunVerdictFailed', verdictConclusionOf), + Match.tag('RunFailed', failedConclusionOf), + Match.exhaustive, + ), ) export const planRunConclusion = Workflow.make({ diff --git a/packages/stryker-js/src/plugin-load-report.service.ts b/packages/stryker-js/src/plugin-load-report.service.ts index 51efa5d5..f0e71707 100644 --- a/packages/stryker-js/src/plugin-load-report.service.ts +++ b/packages/stryker-js/src/plugin-load-report.service.ts @@ -1,6 +1,5 @@ import { RunEvent, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { Format } from '@systemfsoftware/stryker-js-instrumenter' -import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as Array from 'effect/Array' import type * as Cause from 'effect/Cause' import * as Effect from 'effect/Effect' @@ -8,50 +7,11 @@ import { dual } from 'effect/Function' import * as MutableHashMap from 'effect/MutableHashMap' import * as Option from 'effect/Option' import * as Queue from 'effect/Queue' -import * as S from 'effect/Schema' import type { LoadedPlugins } from './Plugins.schema.js' -import { type PluginLoadRefusedError } from './PluginsError.schema.js' -const PLUGIN_FAILURE_REMEDIATION: Record = { - PeerMissing: 'install the peer dependency the plugin needs', - PeerVersionUnsupported: 'install a supported version of the peer dependency', - PeerUnrecognized: 'install a peer version the plugin recognizes, or a matching plugin version', - InvalidContribution: 'fix the contribution the plugin declares', - ImportFailed: 'fix the plugin so that it imports cleanly', -} - -const exitCodeOfClass = (exitClass: Plugin.ExitClass): Effect.Effect => - Effect.orDie(S.decodeEffect(Plugin.ExitCodeFromClass)(exitClass)) - -export const pluginLoadFailureEvents: { - ( - elapsedMs: number, - ): (error: PluginLoadRefusedError) => Effect.Effect - ( - error: PluginLoadRefusedError, - elapsedMs: number, - ): Effect.Effect -} = dual( - 2, - ( - error: PluginLoadRefusedError, - elapsedMs: number, - ): Effect.Effect => - Effect.map( - exitCodeOfClass(error.exitClass), - (code): readonly [RunEvent.PhaseEntered, RunEvent.RunFailed] => [ - RunEvent.PhaseEntered.make({ phase: 'prepare', elapsedMs }), - RunEvent.RunFailed.make({ - schemaVersion: RunEvent.StreamSchemaVersion.literal, - code, - error: error.message, - remediation: PLUGIN_FAILURE_REMEDIATION[error.reason._tag], - reason: error.reason, - }), - ], - ), -) +export const pluginLoadPhaseEvent = (elapsedMs: number): RunEvent.PhaseEntered => + RunEvent.PhaseEntered.make({ phase: 'prepare', elapsedMs }) type FrameworkContributionModule = LoadedPlugins['frameworks'][number] diff --git a/packages/stryker-js/src/render-annotations.workflow.ts b/packages/stryker-js/src/render-annotations.workflow.ts index c757e73e..cf2a199c 100644 --- a/packages/stryker-js/src/render-annotations.workflow.ts +++ b/packages/stryker-js/src/render-annotations.workflow.ts @@ -1,5 +1,6 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' -import { Mutant, Plugin, Report } from '@systemfsoftware/stryker-js-plugin-interface' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Mutant, Report } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Boolean from 'effect/Boolean' import * as HashMap from 'effect/HashMap' @@ -27,7 +28,9 @@ const decodeMutantStatus = S.decodeOption(MUTANT_STATUSES) export class AnnotationsUnusable extends S.TaggedError()('AnnotationsUnusable', { reason: S.String, }) { - readonly exitClass = 'ConfigError' satisfies Plugin.ExitClass + get evidence(): FailureRecord.FailureEvidence { + return { _tag: 'ConfigInvalid', stage: 'config', detail: this.reason } + } override get message(): string { return `stryker annotate: ${this.reason}` diff --git a/packages/stryker-js/src/reporter-stream.service.ts b/packages/stryker-js/src/reporter-stream.service.ts index 787b6fc9..67c085ea 100644 --- a/packages/stryker-js/src/reporter-stream.service.ts +++ b/packages/stryker-js/src/reporter-stream.service.ts @@ -257,6 +257,7 @@ export const spawnReporterWorker = ( workingDirectory: params.projectBasePath, execArgv: [...params.execArgv], tempDirPrefix: params.tempDirPrefix, + workerKind: 'reporter', }) const warnEventDropped = (attachment: ReporterAttachment): Effect.Effect => diff --git a/packages/stryker-js/src/reporter-wiring.service.ts b/packages/stryker-js/src/reporter-wiring.service.ts index 57f085a8..29278071 100644 --- a/packages/stryker-js/src/reporter-wiring.service.ts +++ b/packages/stryker-js/src/reporter-wiring.service.ts @@ -12,7 +12,7 @@ import * as Scope from 'effect/Scope' import type { LoadedPlugins } from './Plugins.schema.js' import { PluginNotFoundError } from './PluginsError.schema.js' import { type AttachReporterInput, reporterWorkerFactory, spawnReporterWorker } from './reporter-stream.service.js' -import { StageError } from './Run.schema.js' +import { RunFailure } from './Run.schema.js' import { ConfiguredPluginName, resolveConfiguredPlugin, @@ -33,7 +33,7 @@ const spawnPluginReporterFactory = Effect.fn(SpanTaxonomy.Spans.reporterWiringSp options: Options.StrykerOptions, ): Effect.fn.Return< InterfaceReporter.ReporterFactory, - StageError, + RunFailure, Scope.Scope | WorkerLauncher | FileSystem.FileSystem | Path.Path > { const entry = yield* Effect.mapError( @@ -47,9 +47,9 @@ const spawnPluginReporterFactory = Effect.fn(SpanTaxonomy.Spans.reporterWiringSp ), ), (missing) => - StageError.make({ - stage: 'prepare', - reason: missing.reason, + RunFailure.make({ + evidence: { _tag: 'PluginNotFound', stage: 'config', descriptor: missing.descriptor }, + detail: missing.reason, cause: PluginNotFoundError.make({ descriptor: missing.descriptor }), }), ) @@ -61,7 +61,11 @@ const spawnPluginReporterFactory = Effect.fn(SpanTaxonomy.Spans.reporterWiringSp tempDirPrefix: 'stryker-reporter-', }).pipe( Effect.mapError((cause) => - StageError.make({ stage: 'prepare', reason: `Failed to start the reporter worker "${name}"`, cause }) + RunFailure.make({ + evidence: { _tag: 'ReporterFailed', stage: 'report', reporter: name }, + detail: `Failed to start the reporter worker "${name}"`, + cause, + }) ), ) return reporterWorkerFactory(client) @@ -96,7 +100,7 @@ export const reporterInputsOf: { names: readonly string[], ) => Effect.Effect< readonly AttachReporterInput[], - StageError, + RunFailure, Scope.Scope | WorkerLauncher | FileSystem.FileSystem | Path.Path > ( @@ -107,7 +111,7 @@ export const reporterInputsOf: { options: Options.StrykerOptions, ): Effect.Effect< readonly AttachReporterInput[], - StageError, + RunFailure, Scope.Scope | WorkerLauncher | FileSystem.FileSystem | Path.Path > } = dual( @@ -120,7 +124,7 @@ export const reporterInputsOf: { options: Options.StrykerOptions, ): Effect.Effect< readonly AttachReporterInput[], - StageError, + RunFailure, Scope.Scope | WorkerLauncher | FileSystem.FileSystem | Path.Path > => Effect.forEach( diff --git a/packages/stryker-js/src/reporting/run-failure.schema.ts b/packages/stryker-js/src/reporting/run-failure.schema.ts index 445f1f07..cfcd77ad 100644 --- a/packages/stryker-js/src/reporting/run-failure.schema.ts +++ b/packages/stryker-js/src/reporting/run-failure.schema.ts @@ -1,18 +1,6 @@ -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as S from 'effect/Schema' -export class ErrorEnvelope extends S.Class('ErrorEnvelope')({ - schemaVersion: RunEvent.StreamSchemaVersion, - code: Plugin.ExitCode, - error: S.NonEmptyString, - remediation: S.NonEmptyString, -}) { - get envelopeText(): string { - return this.error - } -} - export class RunExitCode extends S.Class('RunExitCode')({ code: Plugin.ExitCode, }) {} diff --git a/packages/stryker-js/src/reporting/run-failure.ts b/packages/stryker-js/src/reporting/run-failure.ts index f9fa997d..2b6d3374 100644 --- a/packages/stryker-js/src/reporting/run-failure.ts +++ b/packages/stryker-js/src/reporting/run-failure.ts @@ -1,81 +1,16 @@ -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' -import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import type { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as Match from 'effect/Match' -import * as Option from 'effect/Option' -import * as S from 'effect/Schema' -import type { FailedRunOutcome, RunOutcomeDecision, RunOutcomeError } from '../classify-run-outcome.workflow.js' -import { ErrorEnvelope, RunExitCode } from './run-failure.schema.js' +import type { RunOutcomeDecision } from '../classify-run-outcome.workflow.js' +import { RunExitCode } from './run-failure.schema.js' -const CONFIG_CODE = 2 -const UNKNOWN_FAILURE = 'Unknown failure' -const SIGNAL_REMEDIATION = 'the run was interrupted by a signal; re-run it to continue' -const PARSE_REMEDIATION = 're-run with --help to see the full usage' -const DEFAULT_REMEDIATION = 'see --reportFile or the verdict envelope on stdout' - -export const exitCodeOf = (outcome: RunOutcomeDecision | RunOutcomeError): Plugin.ExitCode => +export const exitCodeOf = (outcome: RunOutcomeDecision): Plugin.ExitCode => Match.value(outcome).pipe( Match.tag('RunOk', () => 0), - Match.tag('RunInterrupted', (error) => error.code), - Match.tag('RunParseFailed', () => CONFIG_CODE), - Match.tag('RunSurvivorsRejected', () => CONFIG_CODE), - Match.tag('RunConfigFailed', () => CONFIG_CODE), + Match.tag('RunVerdictFailed', (failed) => failed.code), Match.tag('RunFailed', (failed) => failed.code), Match.exhaustive, ) -const nonEmptyText = Option.liftPredicate(S.is(S.NonEmptyString)) - -const capturedOrUnknown = (captured: string) => Option.getOrElse(nonEmptyText(captured), () => UNKNOWN_FAILURE) - -const capturedThenRecorded = (captured: string, recorded: string | undefined) => - Option.getOrElse( - nonEmptyText(captured), - () => Option.getOrElse(Option.fromNullishOr(recorded), () => UNKNOWN_FAILURE), - ) - -const failureTextOf = (error: FailedRunOutcome, captured: string) => - Match.value(error).pipe( - Match.tag('RunParseFailed', (failed) => - Option.getOrElse( - Option.map(Option.fromNullishOr(failed.unrecognized), (value) => `Received unknown argument: '${value}'`), - () => capturedOrUnknown(captured), - )), - Match.tag('RunSurvivorsRejected', (failed) => - Option.getOrElse(Option.fromNullishOr(failed.diagnostic), () => UNKNOWN_FAILURE)), - Match.tag('RunInterrupted', () => - capturedOrUnknown(captured)), - Match.tag('RunConfigFailed', (failed) => capturedThenRecorded(captured, failed.detail)), - Match.tag('RunFailed', (failed) => capturedThenRecorded(captured, failed.diagnostic)), - Match.exhaustive, - ) - -const remediationTextOf = (error: FailedRunOutcome) => - Match.value(error).pipe( - Match.tag('RunInterrupted', () => SIGNAL_REMEDIATION), - Match.tag('RunParseFailed', () => PARSE_REMEDIATION), - Match.tag('RunSurvivorsRejected', (failed) => - Option.getOrElse(Option.fromNullishOr(failed.diagnostic), () => DEFAULT_REMEDIATION)), - Match.tag('RunConfigFailed', (failed) => - Option.match(Option.fromNullishOr(failed.detail), { - onSome: (detail) => - `check the config file: ${detail}`, - onNone: () => 'check the config file', - })), - Match.tag('RunFailed', () => DEFAULT_REMEDIATION), - Match.exhaustive, - ) - -export const errorEnvelopeFromOutcome = (input: { - readonly error: FailedRunOutcome - readonly captured: string -}): ErrorEnvelope => - ErrorEnvelope.make({ - schemaVersion: RunEvent.StreamSchemaVersion.literal, - code: exitCodeOf(input.error), - error: failureTextOf(input.error, input.captured), - remediation: remediationTextOf(input.error), - }) - -export const runExitCodeFromOutcome = (outcome: RunOutcomeDecision | RunOutcomeError): RunExitCode => +export const runExitCodeFromOutcome = (outcome: RunOutcomeDecision): RunExitCode => RunExitCode.make({ code: exitCodeOf(outcome) }) diff --git a/packages/stryker-js/src/run-event-stream.service.ts b/packages/stryker-js/src/run-event-stream.service.ts index 32976f62..3c433e30 100644 --- a/packages/stryker-js/src/run-event-stream.service.ts +++ b/packages/stryker-js/src/run-event-stream.service.ts @@ -18,9 +18,10 @@ import * as Stdio from 'effect/Stdio' import * as Stream from 'effect/Stream' import * as SynchronizedRef from 'effect/SynchronizedRef' -import { RunEvent, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import type { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import { Report } from '@systemfsoftware/stryker-js-plugin-interface' -import type { FailedRunOutcome, RunOk, RunOutcomeDecision, RunOutcomeError } from './classify-run-outcome.workflow.js' +import type { RunOk } from './classify-run-outcome.workflow.js' import { defaultOptions } from './config/default-options.js' import { frameRunEvent, @@ -31,12 +32,19 @@ import { } from './frame-run-event.workflow.js' import type { ResolvedMode } from './output-mode.schema.js' import { MachineConsole } from './reporting/machine-console.service.js' -import { errorEnvelopeFromOutcome } from './reporting/run-failure.js' import { buildVerdictEnvelope, generateRunId } from './reporting/verdict-envelope.js' import { StrykerPackage } from './stryker-package.schema.js' export type { ResolvedModeInput } from './frame-run-event.workflow.js' +type RunOkFacts = (typeof RunOk)['Encoded'] + +const stderrTextOf = (stderr: NonNullable): string => + Match.valueTags(stderr, { + StderrText: ({ text }) => text, + StderrRecord: ({ record }) => FailureRecord.terminalTextOf(record), + }) + export const TICK_INTERVAL_MS = 10_000 export const initialFramingState = (resolved: ResolvedModeInput) => @@ -222,7 +230,7 @@ export interface EmitNullScoreVerdictOptions { export interface EmitMachineModeOutputOptions { readonly stream: RunEventStream readonly mode: ResolvedMode - readonly outcome: Result.Result + readonly ok: RunOkFacts readonly basePath: string readonly pathService: Path.Path } @@ -266,23 +274,15 @@ const emitNullScoreVerdict = (params: EmitNullScoreVerdictOpti ) } -const offerFailureEnvelope = ( +const emitFailureRecord = ( stream: RunEventStream, - failed: FailedRunOutcome, - captured: string, -): Effect.Effect => { - const envelope = errorEnvelopeFromOutcome({ error: failed, captured }) - return Queue.offer( + code: Plugin.ExitCode, + record: FailureRecord.FailureRecord, +): Effect.Effect => + Queue.offer( stream.queue, - RunEvent.RunFailed.make({ - schemaVersion: envelope.schemaVersion, - code: envelope.code, - error: envelope.error, - remediation: envelope.remediation, - reason: null, - }), - ) -} + RunEvent.RunFailed.make({ schemaVersion: RunEvent.StreamSchemaVersion.literal, code, record }), + ).pipe(Effect.asVoid) const emitHelpEnvelope = (stream: RunEventStream, help: string): Effect.Effect => Queue.offer( @@ -294,7 +294,7 @@ const emitHelpEnvelope = (stream: RunEventStream, help: string): Effect.Effect => +const helpPayload = (ok: RunOkFacts, captured: string): Option.Option => Boolean.match(ok.help || captured.length > 0, { onTrue: () => Option.some(captured), onFalse: () => Option.none(), @@ -332,29 +332,11 @@ const emitNullScoreVerdictWhenOpen = ( const emitMachineModeOutput = Effect.fn(SpanTaxonomy.Spans.runEventStreamEmitMachineModeOutput.name)(function*( params: EmitMachineModeOutputOptions, ) { - const { stream, mode, outcome, basePath, pathService } = params + const { stream, mode, ok, basePath, pathService } = params const captured = (yield* MachineConsole).read() - return yield* Result.match(outcome, { - onSuccess: (decision) => - Match.value(decision).pipe( - Match.tag('RunOk', (ok): Effect.Effect => - Option.match(helpPayload(ok, captured), { - onSome: (help) => emitHelpEnvelope(stream, help), - onNone: () => - emitNullScoreVerdictWhenOpen( - stream, - mode, - basePath, - pathService, - ), - })), - Match.tag('RunParseFailed', (failed) => offerFailureEnvelope(stream, failed, captured)), - Match.tag('RunSurvivorsRejected', (failed) => offerFailureEnvelope(stream, failed, captured)), - Match.tag('RunConfigFailed', (failed) => offerFailureEnvelope(stream, failed, captured)), - Match.tag('RunFailed', (failed) => offerFailureEnvelope(stream, failed, captured)), - Match.exhaustive, - ), - onFailure: (failure) => offerFailureEnvelope(stream, failure, captured), + return yield* Option.match(helpPayload(ok, captured), { + onSome: (help) => emitHelpEnvelope(stream, help), + onNone: () => emitNullScoreVerdictWhenOpen(stream, mode, basePath, pathService), }) }) @@ -366,6 +348,11 @@ export interface RunEventStreamPort { params: EmitNullScoreVerdictOptions, ) => Effect.Effect readonly emitMachineModeOutput: (params: EmitMachineModeOutputOptions) => Effect.Effect + readonly emitFailureRecord: ( + stream: RunEventStream, + code: Plugin.ExitCode, + record: FailureRecord.FailureRecord, + ) => Effect.Effect } export class RunEventStreamPortTag extends Context.Service()( @@ -377,6 +364,7 @@ export class RunEventStreamPortTag extends Context.Service makeRunEventStream(resolved), emitNullScoreVerdict, emitMachineModeOutput, + emitFailureRecord, }), ) } @@ -473,9 +461,9 @@ export const makeRunEventStream = Effect.fn(SpanTaxonomy.Spans.runEventStreamMak Option.match(decision, { onNone: () => Effect.void, onSome: (d) => - Option.match(Option.fromNullishOr(d.stderrLine), { + Option.match(Option.fromNullishOr(d.stderr), { onNone: () => Effect.void, - onSome: (line) => writeStderr(stdio, line), + onSome: (stderr) => writeStderr(stdio, stderrTextOf(stderr)), }), }) ), diff --git a/packages/stryker-js/src/run-request.cell.ts b/packages/stryker-js/src/run-request.cell.ts index cef62d81..037e6910 100644 --- a/packages/stryker-js/src/run-request.cell.ts +++ b/packages/stryker-js/src/run-request.cell.ts @@ -344,12 +344,6 @@ const writeDecidedBaseline = ( const reportUnchecked = (unchecked: ReadonlyArray): Effect.Effect => Effect.logInfo(`stryker gate: ${unchecked.length} mutant(s) unchecked (in scope with no verdict)`) -const GATE_REMEDIATION_LINE = - 'accept the new survivors with `stryker gate --update-baseline`, or kill them before the next run' - -const explainGateRefusal = (failure: GateRejected | GateInputUnusable): Effect.Effect => - Effect.andThen(Effect.logError(failure.message), Effect.logInfo(GATE_REMEDIATION_LINE)) - const gateReport = ( gate: { readonly baseline: string; readonly updateBaseline: boolean }, channel: CliRead, @@ -499,7 +493,7 @@ export const runRequestCell = Sandwich.named(SpanTaxonomy.Spans.runRequest.name) mode: channel.environment.mode.mode, }), CliCompareRequested: (compare) => compareReports(compare), - CliGateRequested: (gate, channel) => gateReport(gate, channel).pipe(Effect.tapError(explainGateRefusal)), + CliGateRequested: (gate, channel) => gateReport(gate, channel), CliAnnotateRequested: (annotate, channel) => annotateReport(annotate, channel), CliFeedbackRequested: (feedback, channel) => feedbackRoute(feedback, channel), CliMcpRequested: (_, channel) => diff --git a/packages/stryker-js/src/run/dry-run.cell.ts b/packages/stryker-js/src/run/dry-run.cell.ts index 9031fdfd..a7cbe925 100644 --- a/packages/stryker-js/src/run/dry-run.cell.ts +++ b/packages/stryker-js/src/run/dry-run.cell.ts @@ -1,7 +1,7 @@ import { sha256 } from '@noble/hashes/sha2.js' import { bytesToHex, utf8ToBytes } from '@noble/hashes/utils.js' import { type Cell, Sandwich } from '@systemfsoftware/effect-cell-types' -import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import type { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import { Options, type Plugin, Reporter, type TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' @@ -28,7 +28,7 @@ import { DryRunReuseCommand, type DryRunReuseDecision, } from '../dry-run-reuse.workflow.js' -import { dryRun, DryRunCommand, DryRunError, DryRunFailed, FailedTestSummary } from '../dry-run.workflow.js' +import { dryRun, DryRunCommand } from '../dry-run.workflow.js' import { analyzeImportClosure, type ImportClosureAnalysis } from '../import-closure.cell.js' import { DryRunObservation, @@ -39,7 +39,7 @@ import type { LoadedPlugins } from '../Plugins.schema.js' import { PluginNotFoundError } from '../PluginsError.schema.js' import { offerReporterEvent, withPhaseSpan } from '../reporter-stream.service.js' import type { RunEvents } from '../run-events.service.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { originalFileFor, sandboxFileFor, type SandboxHandle } from '../Sandbox.handle.js' import type { TestCoverage } from '../test-coverage.schema.js' import { buildTestRunner, makeChildProcessTestRunner } from '../TestRunner.blueprint.js' @@ -92,19 +92,18 @@ const configuredPluginOf = (configured: string | { readonly plugin: string }) => ) const workerSpawnOf = ( - stage: StageError['stage'], loaded: Pick, kind: Plugin.WorkerPluginKind, configured: ConfiguredPluginName | ConfiguredPluginModulePath, -): Effect.Effect => +): Effect.Effect => Effect.mapError( Effect.fromResult( resolveConfiguredPlugin(WorkerSpawnCommand.make({ sources: loaded.pluginSources, kind, configured })), ), (missing) => - StageError.make({ - stage, - reason: missing.reason, + RunFailure.make({ + evidence: { _tag: 'PluginNotFound', stage: 'config', descriptor: missing.descriptor }, + detail: missing.reason, cause: PluginNotFoundError.make({ descriptor: missing.descriptor }), }), ) @@ -129,7 +128,13 @@ const buildDryRunFiles = (command: InstrumentDone) => const resolveDryRunFiles = Effect.fn(SpanTaxonomy.Spans.dryRunResolveFiles.name)(function*(command: InstrumentDone) { return yield* Effect.fromResult(buildDryRunFiles(command)).pipe( - Effect.mapError((cause) => StageError.make({ stage: 'dryRun', reason: 'Failed to resolve sandbox files', cause })), + Effect.mapError((cause) => + RunFailure.make({ + evidence: { _tag: 'SandboxPreparationFailed', stage: 'dryRun' }, + detail: 'Failed to resolve sandbox files', + cause, + }) + ), ) }) @@ -548,9 +553,19 @@ const completeDryRunPassed = (raw: DryRunRaw) => Match.value(raw.rawResult).pipe( Match.discriminator('status')('complete', (rawResult) => completeDryRunResultOf(raw, rawResult)), Match.discriminator('status')('error', () => - Effect.fail(StageError.make({ stage: 'dryRun', reason: 'Unexpected dry-run status after decision' }))), + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'dryRun' }, + detail: 'Unexpected dry-run status after decision', + }), + )), Match.discriminator('status')('timeout', () => - Effect.fail(StageError.make({ stage: 'dryRun', reason: 'Unexpected dry-run status after decision' }))), + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'dryRun' }, + detail: 'Unexpected dry-run status after decision', + }), + )), Match.exhaustive, ) @@ -571,7 +586,7 @@ const runFreshDryRun = Effect.fnUntraced(function*( Effect.gen(function*() { const childRunnerEffect = Effect.suspend(() => { const runnerConfigured = command.options.testRunner - return workerSpawnOf('dryRun', command.loadedPlugins, 'TestRunner', configuredPluginOf(runnerConfigured)) + return workerSpawnOf(command.loadedPlugins, 'TestRunner', configuredPluginOf(runnerConfigured)) .pipe( Effect.flatMap((resolved) => makeChildProcessTestRunner({ @@ -598,14 +613,24 @@ const runFreshDryRun = Effect.fnUntraced(function*( ) const capabilities = yield* runner.capabilities.pipe( Effect.mapError((cause) => - StageError.make({ stage: 'dryRun', reason: 'Failed to get test runner capabilities', cause }) + RunFailure.make({ + evidence: { _tag: 'TestRunnerFailed', stage: 'dryRun' }, + detail: 'Failed to get test runner capabilities', + cause, + }) ), ) const timedPass = Effect.timed( runner .dryRun(options) .pipe( - Effect.mapError((cause) => StageError.make({ stage: 'dryRun', reason: 'Dry run failed', cause })), + Effect.mapError((cause) => + RunFailure.make({ + evidence: { _tag: 'TestRunnerFailed', stage: 'dryRun' }, + detail: 'Dry run failed', + cause, + }) + ), ), ) const firstPass = yield* timedPass @@ -624,11 +649,15 @@ const runFreshDryRun = Effect.fnUntraced(function*( Effect.mapError((cause) => Match.value({ cause }).pipe( Match.when( - { cause: (candidate: unknown): candidate is StageError => S.is(StageError)(candidate) }, + { cause: (candidate: unknown): candidate is RunFailure => S.is(RunFailure)(candidate) }, ({ cause }) => cause, ), Match.orElse(({ cause }) => - StageError.make({ stage: 'dryRun', reason: 'Dry run failed to start test runner', cause }) + RunFailure.make({ + evidence: { _tag: 'TestRunnerFailed', stage: 'dryRun' }, + detail: 'Dry run failed to start test runner', + cause, + }) ), ) ), @@ -651,7 +680,7 @@ const runFreshDryRun = Effect.fnUntraced(function*( const readDryRun: (command: InstrumentDone) => Effect.Effect< DryRunRaw, - StageError, + RunFailure, | Scope.Scope | IdGenerator | ChildProcessSpawner.ChildProcessSpawner @@ -718,7 +747,12 @@ const readDryRun: (command: InstrumentDone) => Effect.Effect< Match.tag('DryRunCoverageReused', () => Option.match(prior, { onNone: () => - Effect.die(StageError.make({ stage: 'dryRun', reason: 'Coverage reuse decided without prior coverage' })), + Effect.die( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'dryRun' }, + detail: 'Coverage reuse decided without prior coverage', + }), + ), onSome: (coverage) => Effect.succeed(reusedRawOf(command, coverage)), })), Match.tag( @@ -739,48 +773,68 @@ const writeDryRunPassed = Effect.fn(SpanTaxonomy.Spans.dryRunWritePassed.name)(f const FAILED_TESTS_REASON = 'There were failed tests in the initial test run.' -const failedTestsDetail = (failedTests: readonly FailedTestSummary[]): string => - failedTests - .map((test) => ` ${test.name}${test.failureMessage.length > 0 ? `: ${test.failureMessage}` : ''}`) - .join('\n') +const projectFileOf = (path: Path.Path, sandboxDirectory: string) => (file: string): string => + Option.getOrElse( + Option.filter(Option.some(path.relative(sandboxDirectory, file)), (relative) => !relative.startsWith('..')), + () => file, + ) -const reasonWithFailedTests = (detail: string): string => - detail.length > 0 ? `${FAILED_TESTS_REASON}\n${detail}` : FAILED_TESTS_REASON +const projectTestOf = + (path: Path.Path, sandboxDirectory: string) => + (test: FailureRecord.FailedTestEvidence): FailureRecord.FailedTestEvidence => ({ + ...test, + file: Option.getOrNull(Option.map(Option.fromNullishOr(test.file), projectFileOf(path, sandboxDirectory))), + stack: Option.getOrNull( + Option.map(Option.fromNullishOr(test.stack), (stack) => stack.replaceAll(`${sandboxDirectory}${path.sep}`, '')), + ), + }) const writeDryRunFailed = Effect.fn(SpanTaxonomy.Spans.dryRunWriteFailed.name)(function*({ testCount, - failedTestCount, failedTests, + sandboxDirectory, }: { readonly testCount: number - readonly failedTestCount: number - readonly failedTests: readonly FailedTestSummary[] + readonly failedTests: readonly FailureRecord.FailedTestEvidence[] + readonly sandboxDirectory: string }) { + const path = yield* Path.Path + const projectTests = failedTests.map(projectTestOf(path, sandboxDirectory)) return yield* withPhaseSpan( SpanTaxonomy.Spans.dryRunPhase, {}, () => - Effect.gen(function*() { - const detail = failedTestsDetail(failedTests) - yield* Effect.logError( - `Initial test run failed. ${failedTestCount} of ${testCount} test(s) failed:\n${detail}`, - ) - return yield* StageError.make({ - stage: 'dryRun', - reason: reasonWithFailedTests(detail), - cause: DryRunFailed.make({ testCount, failedTestCount, failedTests }), - }) + Arr.match(projectTests, { + onEmpty: () => + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'dryRun' }, + detail: 'The dry run reported failed tests but carried none', + }), + ), + onNonEmpty: (tests) => + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'BaselineTestsFailed', stage: 'dryRun', testCount, tests }, + detail: FAILED_TESTS_REASON, + }), + ), }), ) }) -export const dryRunCell: Cell.Cell = Sandwich.named( +export const dryRunCell: Cell.Cell = Sandwich.named( SpanTaxonomy.Spans.dryRun.name, )(readDryRun).decide(dryRun).write({ DryRunPassed: (_decision, raw) => writeDryRunPassed(raw), - DryRunFailed: ({ testCount, failedTestCount, failedTests }, _raw) => - writeDryRunFailed({ testCount, failedTestCount, failedTests }), - DryRunError: ({ stage, reason }) => - Effect.fail(StageError.make({ stage, reason, cause: DryRunError.make({ stage, reason }) })), - CommandRejected: ({ issue }) => Effect.fail(StageError.make({ stage: 'dryRun', reason: issue })), + DryRunFailed: ({ testCount, failedTests }, raw) => + writeDryRunFailed({ testCount, failedTests, sandboxDirectory: raw.prev.sandbox.workingDirectory }), + RunFailure: (failure) => Effect.fail(RunFailure.make(failure)), + CommandRejected: ({ issue }) => + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'dryRun' }, + detail: issue, + }), + ), }) diff --git a/packages/stryker-js/src/run/host.service.ts b/packages/stryker-js/src/run/host.service.ts index 9b56e0d4..afc8d461 100644 --- a/packages/stryker-js/src/run/host.service.ts +++ b/packages/stryker-js/src/run/host.service.ts @@ -4,7 +4,7 @@ import type * as Cause from 'effect/Cause' import type * as Effect from 'effect/Effect' import type * as Queue from 'effect/Queue' -import type { StageError } from '../Run.schema.js' +import type { RunFailure } from '../Run.schema.js' import type { MutationTestDone } from './mutation-test.cell.js' import type { RunEnvironmentShape } from './RunEnvironment.service.js' @@ -16,4 +16,4 @@ export interface HostServices { export type StrykerRun = ( options: Options.PartialStrykerOptions, targetMutatePatterns?: string[], -) => Effect.Effect +) => Effect.Effect diff --git a/packages/stryker-js/src/run/incremental-reuse.cell.ts b/packages/stryker-js/src/run/incremental-reuse.cell.ts index 3777db67..c2c49295 100644 --- a/packages/stryker-js/src/run/incremental-reuse.cell.ts +++ b/packages/stryker-js/src/run/incremental-reuse.cell.ts @@ -34,7 +34,7 @@ import { } from '../IncrementalDiff.schema.js' import type { Project } from '../Project.schema.js' import { reportTestIds, ResolveReportTestIds } from '../report-test-ids.workflow.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { originalFileFor, type SandboxHandle } from '../Sandbox.handle.js' import type { TestCoverage } from '../test-coverage.schema.js' import { runInputsDigestOf, VERDICT_SEMANTICS_VERSION } from '../verdict-semantics.js' @@ -477,7 +477,12 @@ const incrementalReuseCell = Sandwich.named(SpanTaxonomy.Spans.incrementalReuse. MutantToRun: (decision, command) => mutantToRunPart(decision, command), MutantRemembered: (decision, command) => rememberedMutantPart(decision, command), CommandRejected: ({ issue }) => - Effect.die(StageError.make({ stage: 'mutationTest', reason: `incremental reuse command rejected: ${issue}` })), + Effect.die( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, + detail: `incremental reuse command rejected: ${issue}`, + }), + ), }) export interface IncrementalReuseInput { diff --git a/packages/stryker-js/src/run/instrument.cell.ts b/packages/stryker-js/src/run/instrument.cell.ts index fdc5a22b..d1f6fe70 100644 --- a/packages/stryker-js/src/run/instrument.cell.ts +++ b/packages/stryker-js/src/run/instrument.cell.ts @@ -20,7 +20,7 @@ import { ProjectFiles } from '../project-files.service.js' import type { Project, ProjectFile } from '../Project.schema.js' import { withPhaseSpan } from '../reporter-stream.service.js' import { RunEvents } from '../run-events.service.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { makeSandbox } from '../Sandbox.blueprint.js' import type { SandboxHandle } from '../Sandbox.handle.js' import { explainFileSkip, ExplainFileSkipCommand, type FrameworkClaimant } from './explain-file-skip.workflow.js' @@ -162,7 +162,11 @@ const readInstrument = Effect.fn(SpanTaxonomy.Spans.instrumentGather.name)(funct (readFiles) => readFiles.map(([file, content]) => ({ content, mutate: file.mutate, name: file.name })), ).pipe( Effect.mapError((cause) => - StageError.make({ stage: 'instrument', reason: 'Failed to read files to mutate', cause }) + RunFailure.make({ + evidence: { _tag: 'InstrumentationFailed', stage: 'instrument' }, + detail: 'Failed to read files to mutate', + cause, + }) ), ) @@ -179,7 +183,13 @@ const readInstrument = Effect.fn(SpanTaxonomy.Spans.instrumentGather.name)(funct optInMutations, ), }, command.formatRegistry).pipe( - Effect.mapError((cause) => StageError.make({ stage: 'instrument', reason: 'Instrumenter failed', cause })), + Effect.mapError((cause) => + RunFailure.make({ + evidence: { _tag: 'InstrumentationFailed', stage: 'instrument' }, + detail: 'Instrumenter failed', + cause, + }) + ), ) const instrumentedProject = withInstrumentedFiles(command.project, instrumentResult.files) @@ -194,7 +204,11 @@ const readInstrument = Effect.fn(SpanTaxonomy.Spans.instrumentGather.name)(funct formatRegistry: command.formatRegistry, }).pipe( Effect.mapError((cause) => - StageError.make({ stage: 'instrument', reason: 'Sandbox initialization failed', cause }) + RunFailure.make({ + evidence: { _tag: 'SandboxPreparationFailed', stage: 'instrument' }, + detail: 'Sandbox initialization failed', + cause, + }) ), ) @@ -216,7 +230,7 @@ const readInstrument = Effect.fn(SpanTaxonomy.Spans.instrumentGather.name)(funct export const instrumentCell: Cell.Cell< PrepareDone & { readonly concurrency: { readonly testRunners: number; readonly checkers: number } }, InstrumentDone, - StageError, + RunFailure, | Scope.Scope | PhaseClock | RunEnvironment @@ -228,7 +242,13 @@ export const instrumentCell: Cell.Cell< > = Sandwich.named(SpanTaxonomy.Spans.instrument.name)(readInstrument).decide(planInstrumentation).write({ InPlaceInstrument: (_decision, raw) => writeInstrument(raw), EphemeralInstrument: (_decision, raw) => writeInstrument(raw), - CommandRejected: ({ issue }) => Effect.fail(StageError.make({ stage: 'instrument', reason: issue })), + CommandRejected: ({ issue }) => + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'instrument' }, + detail: issue, + }), + ), }) const projectOf = (seeds: readonly ProjectFile[]): Project => { diff --git a/packages/stryker-js/src/run/mutant-run.cell.ts b/packages/stryker-js/src/run/mutant-run.cell.ts index d9ffbc73..ad12e2fd 100644 --- a/packages/stryker-js/src/run/mutant-run.cell.ts +++ b/packages/stryker-js/src/run/mutant-run.cell.ts @@ -12,7 +12,7 @@ import type * as Scope from 'effect/Scope' import { interpretMutantRun, NoCoveringTestExecutedReason } from '../interpret-mutant-run.workflow.js' import { cicdAttributesOf, spanRunStatusOf, testSuiteNameOf } from '../mutant-run-span.js' import { invalidatesRunnerPool } from '../pooled-test-runner.handle.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import type { PooledTestRunnerError } from '../TestRunner.schema.js' import { recycleAndSettleMutantRun, type RunOnePlanArgs, settleMutantRun } from './mutant-run.js' @@ -110,7 +110,7 @@ const readMutantRunAttempt = Effect.fnUntraced(function*(input: MutantRunAttempt const attemptCell: Cell.Cell< MutantRunAttemptArgs, Mutant.RunMutantResult, - StageError | PooledTestRunnerError, + RunFailure | PooledTestRunnerError, Scope.Scope > = Cell.suspend(() => Sandwich.named(SpanTaxonomy.Spans.mutantRun.name)(readMutantRunAttempt) @@ -133,13 +133,18 @@ const attemptCell: Cell.Cell< }), ), CommandRejected: ({ issue }) => - Effect.fail(StageError.make({ stage: 'mutationTest', reason: `mutant run command rejected: ${issue}` })), + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, + detail: `mutant run command rejected: ${issue}`, + }), + ), }) ) export const mutantRunCell: Cell.Cell< RunOnePlanArgs, Mutant.RunMutantResult, - StageError | PooledTestRunnerError, + RunFailure | PooledTestRunnerError, Scope.Scope > = Cell.mapInput(attemptCell, (args: RunOnePlanArgs): MutantRunAttemptArgs => ({ ...args, attempt: 0 })) diff --git a/packages/stryker-js/src/run/mutant-run.ts b/packages/stryker-js/src/run/mutant-run.ts index 768763d2..1f4953c9 100644 --- a/packages/stryker-js/src/run/mutant-run.ts +++ b/packages/stryker-js/src/run/mutant-run.ts @@ -16,7 +16,7 @@ import { mutantCostOf, testBodyMsOf } from '../mutant-cost.js' import { type MutationReportingInput, type MutationReportingService } from '../mutation-reporting.service.js' import { type PooledTestRunner } from '../pooled-test-runner.handle.js' import { offerReporterEvent } from '../reporter-stream.service.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import type { PooledTestRunnerError } from '../TestRunner.schema.js' import type { DryRunDone } from './dry-run.cell.js' import { isMutantStatus, toReportedMutant, type ValidMutantStatus } from './mutation-test-plan.js' @@ -196,7 +196,7 @@ export const makeCheckpointWriter = Effect.fnUntraced(function*( export interface RunOnePlanArgs { readonly context: RunContext - readonly testRunnerPool: Pool.Pool + readonly testRunnerPool: Pool.Pool readonly checkpoint: CheckpointWriter readonly plan: Mutant.MutantRunPlan } diff --git a/packages/stryker-js/src/run/mutation-test-plan.cell.ts b/packages/stryker-js/src/run/mutation-test-plan.cell.ts index cca2a672..1dad8a77 100644 --- a/packages/stryker-js/src/run/mutation-test-plan.cell.ts +++ b/packages/stryker-js/src/run/mutation-test-plan.cell.ts @@ -23,7 +23,7 @@ import { import type { Project } from '../Project.schema.js' import { offerReporterEvent, type ReporterStage } from '../reporter-stream.service.js' import { RunEvents } from '../run-events.service.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import type { SandboxHandle } from '../Sandbox.handle.js' import { OrderedRunPlan, SortRunPlans, sortRunPlans } from '../sort-run-plans.workflow.js' import type { TestCoverage } from '../test-coverage.schema.js' @@ -125,7 +125,7 @@ const plannedPlanOf = ( const materializeDecision = Effect.fnUntraced(function*( decision: EncodedPlannedDecision, command: MutantTestPlanRaw, -): Effect.fn.Return { +): Effect.fn.Return { const mutant = yield* Option.match(Record.get(command.mutantsById, decision.mutantId), { onNone: () => Effect.die( @@ -180,21 +180,27 @@ const planMutantTestsCell = Sandwich.named(SpanTaxonomy.Spans.mutationTestPlanMu PlannedEarlyResultMutant: (decision, command) => materializeDecision(decision, command), CoveredMutantHitCountMissing: ({ missingIds }) => Effect.fail( - StageError.make({ - stage: 'mutationTest', - reason: `covered mutant missing dry-run hit count: ${missingIds.join(', ')}`, + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, + detail: `covered mutant missing dry-run hit count: ${missingIds.join(', ')}`, cause: CoveredMutantHitCountMissing.make({ missingIds }), }), ), MutantTimeoutNotFinite: ({ mutantId }) => Effect.fail( - StageError.make({ - stage: 'mutationTest', - reason: `mutant ${mutantId} has a non-finite timeout`, + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, + detail: `mutant ${mutantId} has a non-finite timeout`, cause: MutantTimeoutNotFinite.make({ mutantId: Mutant.MutantId.make(mutantId) }), }), ), - CommandRejected: ({ issue }) => Effect.fail(StageError.make({ stage: 'mutationTest', reason: issue })), + CommandRejected: ({ issue }) => + Effect.fail( + RunFailure.make({ + evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, + detail: issue, + }), + ), }) export interface MutationTestPlanInput { diff --git a/packages/stryker-js/src/run/mutation-test-plan.ts b/packages/stryker-js/src/run/mutation-test-plan.ts index a603a975..82e52b1f 100644 --- a/packages/stryker-js/src/run/mutation-test-plan.ts +++ b/packages/stryker-js/src/run/mutation-test-plan.ts @@ -10,7 +10,7 @@ import * as S from 'effect/Schema' import { checkerMutantsSkipped } from '../Checker/Checker.handle.js' import { CheckerMutantFromMutant } from '../Checker/Checker.schema.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { sandboxFileFor, type SandboxHandle } from '../Sandbox.handle.js' export const isMutantStatus = S.is(Mutant.MutantStatusSchema) @@ -48,12 +48,16 @@ export interface SandboxFilesInput { export const sandboxFilesOf: ( input: SandboxFilesInput, -) => Effect.Effect = Effect.fn( +) => Effect.Effect = Effect.fn( SpanTaxonomy.Spans.mutationTestSandboxFiles.name, )(function*(input: SandboxFilesInput) { return yield* Effect.fromResult(sandboxFilePairsOf(input.sandbox, input.fileNames)).pipe( Effect.mapError((cause) => - StageError.make({ stage: 'mutationTest', reason: 'Failed to resolve sandbox file', cause }) + RunFailure.make({ + evidence: { _tag: 'SandboxPreparationFailed', stage: 'mutationTest' }, + detail: 'Failed to resolve sandbox file', + cause, + }) ), ) }) diff --git a/packages/stryker-js/src/run/mutation-test.cell.ts b/packages/stryker-js/src/run/mutation-test.cell.ts index 2821f488..ceb53165 100644 --- a/packages/stryker-js/src/run/mutation-test.cell.ts +++ b/packages/stryker-js/src/run/mutation-test.cell.ts @@ -16,7 +16,7 @@ import * as S from 'effect/Schema' import * as Scope from 'effect/Scope' import * as Stream from 'effect/Stream' -import { admitMutationTest, MutationTestError } from '../admit-mutation-test.workflow.js' +import { admitMutationTest } from '../admit-mutation-test.workflow.js' import { scoped as checkerPoolsScoped } from '../Checker/checker-pool.blueprint.js' import { checkPlansStream as checkPlansWithConfiguredCheckers, @@ -29,8 +29,9 @@ import { MutationTestCommand } from '../MutationTest.schema.js' import { withPhaseSpan } from '../reporter-stream.service.js' import { mutantDetailEventsOf, requestedIdsOf, restrictedToRequestedIds } from '../Rerun/rerun-selection.js' import { RunEvents } from '../run-events.service.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import type { PooledTestRunnerError } from '../TestRunner.schema.js' +import { ChildProcessCrashedError, OutOfMemoryError } from '../Worker.schema.js' import { IdGenerator } from '../Worker.service.js' import type { DryRunDone } from './dry-run.cell.js' import { readIncrementalReuse } from './incremental-reuse.cell.js' @@ -212,18 +213,24 @@ const proceedPipeline = Effect.fnUntraced(function*(raw: MutationTestRaw) { return outcomeResult }) -const mapMutationTestCause = ( - cause: PooledTestRunnerError | PlatformError | StageError, -): StageError => - Match.value({ cause }).pipe( - Match.when({ cause: (candidate: unknown): candidate is StageError => S.is(StageError)(candidate) }, ({ cause }) => - cause), - Match.orElse(({ cause }) => - StageError.make({ stage: 'mutationTest', reason: 'Mutation testing failed', cause }) +const workerFailureOf = (error: ChildProcessCrashedError | OutOfMemoryError): RunFailure => + RunFailure.make({ evidence: error.evidence, detail: error.message, cause: error }) + +const mapMutationTestCause = (cause: PooledTestRunnerError | PlatformError | RunFailure): RunFailure => + Match.value(cause).pipe( + Match.tag('RunFailure', (failure) => failure), + Match.tag('ChildProcessCrashedError', (crashed) => workerFailureOf(crashed)), + Match.tag('OutOfMemoryError', (outOfMemory) => workerFailureOf(outOfMemory)), + Match.orElse((cause) => + RunFailure.make({ + evidence: { _tag: 'TestRunnerFailed', stage: 'mutationTest' }, + detail: 'Mutation testing failed', + cause, + }) ), ) -const writeMutationTestProceed = (raw: MutationTestRaw): Effect.Effect => +const writeMutationTestProceed = (raw: MutationTestRaw): Effect.Effect => proceedPipeline(raw).pipe(Effect.mapError(mapMutationTestCause)) const writeMutationTestOutcome = ({ @@ -231,8 +238,8 @@ const writeMutationTestOutcome = ({ outcome, }: { readonly raw: MutationTestRaw - readonly outcome: Effect.Effect -}): Effect.Effect => + readonly outcome: Effect.Effect +}): Effect.Effect => withPhaseSpan( SpanTaxonomy.Spans.mutationTestPhase, { @@ -268,7 +275,7 @@ export const mutationTestCell = Sandwich.named( MutationTestDryRunOnly: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestDryRunOnly() }), MutationTestNoTests: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestNoTests() }), MutationTestProceed: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestProceed(raw) }), - MutationTestError: ({ stage, reason }) => - Effect.fail(StageError.make({ stage, reason, cause: MutationTestError.make({ stage, reason }) })), - CommandRejected: ({ issue }) => Effect.fail(StageError.make({ stage: 'mutationTest', reason: issue })), + RunFailure: (failure) => Effect.fail(RunFailure.make(failure)), + CommandRejected: ({ issue }) => + Effect.fail(RunFailure.make({ evidence: { _tag: 'InvariantBroken', stage: 'mutationTest' }, detail: issue })), }) diff --git a/packages/stryker-js/src/run/prepare.cell.ts b/packages/stryker-js/src/run/prepare.cell.ts index 3f7fbe7c..019d760f 100644 --- a/packages/stryker-js/src/run/prepare.cell.ts +++ b/packages/stryker-js/src/run/prepare.cell.ts @@ -36,7 +36,7 @@ import { planMutatorCatalogs, PlanMutatorCatalogsCommand, } from '../plan-mutator-catalogs.workflow.js' -import { pluginLoadFailureEvents, reportPluginLoad } from '../plugin-load-report.service.js' +import { pluginLoadPhaseEvent, reportPluginLoad } from '../plugin-load-report.service.js' import { loadPlugins, pluginUrlsFromOptions } from '../plugin-loader.service.js' import { type LoadedPlugins, type PluginDescriptor } from '../Plugins.schema.js' import type { Project } from '../Project.schema.js' @@ -53,7 +53,7 @@ import { type ReporterChoice, reporterInputsOf } from '../reporter-wiring.servic import { Reporter } from '../reporter.service.js' import { AnsiCode } from '../reporting/ansi.schema.js' import { RunEvents } from '../run-events.service.js' -import { PrepareError, StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { TemporaryDirectory } from '../Sandbox.service.js' import { WorkerLauncher } from '../WorkerLauncher.service.js' import { admitNonEmptyProject, NonEmptyProjectCommand, ProjectEmpty } from './admit-non-empty-project.workflow.js' @@ -134,7 +134,7 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( command: ReadProjectDone, ): Effect.fn.Return< PrepareRaw, - StageError, + RunFailure, Scope.Scope | RunEnvironment | RunEvents | WorkerLauncher | FileSystem.FileSystem | Path.Path | Reporter > { yield* Scope.Scope @@ -160,13 +160,13 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( } const descriptors: readonly string[] = pluginUrlsFromOptions(options) const loaded = yield* loadPlugins(descriptors, env.basePath).pipe( - Effect.tapError((error) => + Effect.tapError(() => Clock.currentTimeMillis.pipe( - Effect.flatMap((failedAt) => pluginLoadFailureEvents(error, failedAt - env.runStartedAt)), - Effect.flatMap((events) => Effect.forEach(events, (event) => Queue.offer(queue, event), { discard: true })), + Effect.map((failedAt) => pluginLoadPhaseEvent(failedAt - env.runStartedAt)), + Effect.flatMap((event) => Queue.offer(queue, event)), ) ), - Effect.mapError((cause) => StageError.make({ stage: 'prepare', reason: 'Failed to load plugins', cause })), + Effect.mapError((cause) => RunFailure.make({ evidence: cause.evidence, detail: 'Failed to load plugins', cause })), ) const stockCatalog = yield* Effect.orDie(S.decodeEffect(MutatorCatalog.Catalog)(StockCatalog.StockCatalog)) const providers = loaded.mutators.map((provider) => ({ @@ -183,7 +183,14 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( const plannedCatalogs = yield* Effect.fromResult( Result.mapError( planMutatorCatalogs(PlanMutatorCatalogsCommand.make({ stock: stockCatalog, providers })), - (refused) => StageError.make({ stage: 'prepare', reason: refused.message, cause: refused }), + (refused) => { + const detail = refused.message + return RunFailure.make({ + evidence: { _tag: 'ConfigInvalid', stage: 'config', detail }, + detail, + cause: refused, + }) + }, ), ) const mutatorCatalogs = plannedCatalogs.catalogs @@ -196,7 +203,14 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( optInMutations: [...options.mutator.optInMutations], }), ), - (refused) => StageError.make({ stage: 'prepare', reason: refused.message, cause: refused }), + (refused) => { + const detail = refused.message + return RunFailure.make({ + evidence: { _tag: 'ConfigInvalid', stage: 'config', detail }, + detail, + cause: refused, + }) + }, ), ) const registry = Format.registerEntries( @@ -208,12 +222,14 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( const record = { ...options } yield* validateOptions(record, mergedSchema).pipe( Effect.mapError( - (cause) => - StageError.make({ - stage: 'prepare', - reason: 'Failed to revalidate options with plugin schema', + (cause) => { + const detail = 'Failed to revalidate options with plugin schema' + return RunFailure.make({ + evidence: { _tag: 'ConfigInvalid', stage: 'config', detail }, + detail, cause, - }), + }) + }, ), ) const ignorers: readonly Ignorer[] = loaded.ignorers @@ -264,7 +280,7 @@ const applyPrepare = Effect.fn(SpanTaxonomy.Spans.prepareApply.name)(function*( span: PhaseSpan, reporters: readonly string[], raw: PrepareRaw, -): Effect.fn.Return { +): Effect.fn.Return { const mutateCount = pipe(raw.project.filesToMutate, MutableHashMap.size) yield* announceSummary({ env: raw.env, @@ -274,25 +290,31 @@ const applyPrepare = Effect.fn(SpanTaxonomy.Spans.prepareApply.name)(function*( raw.options.reporters, [...HashMap.values(raw.reporterChoicesByName)].map((choice) => choice.name), ).pipe( - Effect.mapError((cause) => StageError.make({ stage: 'prepare', reason: cause.message, cause })), + Effect.mapError((cause) => { + const detail = cause.message + return RunFailure.make({ + evidence: { _tag: 'ConfigInvalid', stage: 'config', detail }, + detail, + cause, + }) + }), ) const admission = admitNonEmptyProject( NonEmptyProjectCommand.make({ fileCount: pipe(raw.project.files, MutableHashMap.size) }), ) const emptyAdmission = Option.filter(Result.getSuccess(admission), S.is(ProjectEmpty)) - const emptyProjectGuard: Result.Result = Option.getOrElse( + const emptyProjectGuard: Result.Result = Option.getOrElse( Option.map( emptyAdmission, - (): Result.Result => + (): Result.Result => Result.fail( - StageError.make({ - stage: 'prepare', - reason: 'No input files found.', - cause: PrepareError.make({ stage: 'prepare', reason: 'No input files found.' }), + RunFailure.make({ + evidence: { _tag: 'NoInputFiles', stage: 'prepare' }, + detail: 'No input files found.', }), ), ), - (): Result.Result => Result.succeed(undefined), + (): Result.Result => Result.succeed(undefined), ) yield* Effect.fromResult(emptyProjectGuard) const temporaryDirectoryPath = yield* Effect.map( @@ -300,7 +322,11 @@ const applyPrepare = Effect.fn(SpanTaxonomy.Spans.prepareApply.name)(function*( (temporaryDirectory) => Context.get(temporaryDirectory, TemporaryDirectory).path, ).pipe( Effect.mapError((cause) => - StageError.make({ stage: 'prepare', reason: 'Failed to create temporary directory', cause }) + RunFailure.make({ + evidence: { _tag: 'SandboxPreparationFailed', stage: 'prepare' }, + detail: 'Failed to create temporary directory', + cause, + }) ), ) const reporterInputs = yield* reporterInputsOf( @@ -331,18 +357,19 @@ const applyPrepare = Effect.fn(SpanTaxonomy.Spans.prepareApply.name)(function*( const writePrepare = ( reporters: readonly string[], raw: PrepareRaw, -): Effect.Effect => +): Effect.Effect => withPhaseSpan(SpanTaxonomy.Spans.preparePhase, {}, (span) => applyPrepare(span, reporters, raw)) export const prepareCell: Cell.Cell< ReadProjectDone, PrepareDone, - StageError, + RunFailure, Scope.Scope | RunEnvironment | RunEvents | WorkerLauncher | FileSystem.FileSystem | Path.Path | Reporter > = Sandwich.named(SpanTaxonomy.Spans.prepare.name)(readPrepare) .decide(planPrepare) .write({ HumanReporters: ({ reporters }, raw) => writePrepare(reporters, raw), MachineReporters: ({ reporters }, raw) => writePrepare(reporters, raw), - CommandRejected: ({ issue }) => Effect.fail(StageError.make({ stage: 'prepare', reason: issue })), + CommandRejected: ({ issue }) => + Effect.fail(RunFailure.make({ evidence: { _tag: 'InvariantBroken', stage: 'prepare' }, detail: issue })), }) diff --git a/packages/stryker-js/src/run/run-stages.cell.ts b/packages/stryker-js/src/run/run-stages.cell.ts index 0d3d7918..d509aed3 100644 --- a/packages/stryker-js/src/run/run-stages.cell.ts +++ b/packages/stryker-js/src/run/run-stages.cell.ts @@ -4,7 +4,7 @@ import * as Match from 'effect/Match' import { concurrencyCell } from '../concurrency.cell.js' import type { ConfigReadError } from '../ConfigError.schema.js' import { readProjectCell } from '../read-project.cell.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { dryRunCell } from './dry-run.cell.js' import { instrumentCell } from './instrument.cell.js' import { loadConfigCell } from './load-config.cell.js' @@ -25,11 +25,23 @@ const prepareStageCell = Cell.andThen( Cell.andThen( Cell.mapError( loadConfigCell, - (cause) => StageError.make({ stage: 'prepare', reason: configReadReasonOf(cause), cause }), + (cause) => { + const detail = configReadReasonOf(cause) + return RunFailure.make({ + evidence: { _tag: 'ConfigInvalid', stage: 'config', detail }, + detail, + cause, + }) + }, ), Cell.mapError( readProjectCell, - (cause) => StageError.make({ stage: 'prepare', reason: 'Failed to read project', cause }), + (cause) => + RunFailure.make({ + evidence: { _tag: 'SandboxPreparationFailed', stage: 'prepare' }, + detail: 'Failed to read project', + cause, + }), ), ), prepareCell, @@ -40,5 +52,5 @@ const prepareStageCell = Cell.andThen( ), ) -export const mutationTestCell: Cell.Cell = +export const mutationTestCell: Cell.Cell = prepareStageCell diff --git a/packages/stryker-js/src/run/run-stages.ts b/packages/stryker-js/src/run/run-stages.ts index 36c136ce..f7da375a 100644 --- a/packages/stryker-js/src/run/run-stages.ts +++ b/packages/stryker-js/src/run/run-stages.ts @@ -10,7 +10,7 @@ import * as Predicate from 'effect/Predicate' import type { ResolvedMode } from '../output-mode.schema.js' import { makeRunEventStream, RunEventDrainLive } from '../run-event-stream.service.js' -import { StageError } from '../Run.schema.js' +import type { RunFailure } from '../Run.schema.js' import type { MutationTestDone } from './mutation-test.cell.js' import { mutationTestCell } from './run-stages.cell.js' import { RunEnvironment } from './RunEnvironment.service.js' @@ -33,12 +33,12 @@ export const strykerCell: { ( options: Options.PartialStrykerOptions, targetMutatePatterns?: readonly string[], - ): Effect.Effect + ): Effect.Effect ( targetMutatePatterns?: readonly string[], ): ( options: Options.PartialStrykerOptions, - ) => Effect.Effect + ) => Effect.Effect } = dual( (args) => Predicate.isObject(args[0]), (options: Options.PartialStrykerOptions, targetMutatePatterns?: readonly string[]) => diff --git a/packages/stryker-js/src/run/test-runner-pool.blueprint.ts b/packages/stryker-js/src/run/test-runner-pool.blueprint.ts index 5528020d..6fcb0ae2 100644 --- a/packages/stryker-js/src/run/test-runner-pool.blueprint.ts +++ b/packages/stryker-js/src/run/test-runner-pool.blueprint.ts @@ -12,7 +12,7 @@ import type * as Scope from 'effect/Scope' import type { LoadedPlugins } from '../Plugins.schema.js' import { PluginNotFoundError } from '../PluginsError.schema.js' import type { PooledTestRunner } from '../pooled-test-runner.handle.js' -import { StageError } from '../Run.schema.js' +import { RunFailure } from '../Run.schema.js' import { ConfiguredPluginModulePath, ConfiguredPluginName, @@ -51,7 +51,7 @@ const configuredPluginOf = (configured: string | { readonly plugin: string }) => const testRunnerWorkerSpawnOf = ( loaded: Pick, configured: ConfiguredPluginName | ConfiguredPluginModulePath, -): Effect.Effect => +): Effect.Effect => Effect.mapError( Effect.fromResult( resolveConfiguredPlugin( @@ -59,9 +59,9 @@ const testRunnerWorkerSpawnOf = ( ), ), (missing) => - StageError.make({ - stage: 'mutationTest', - reason: missing.reason, + RunFailure.make({ + evidence: { _tag: 'PluginNotFound', stage: 'config', descriptor: missing.descriptor }, + detail: missing.reason, cause: PluginNotFoundError.make({ descriptor: missing.descriptor }), }), ) @@ -69,7 +69,7 @@ const testRunnerWorkerSpawnOf = ( const acquire: ( spec: TestRunnerPoolSpec, ) => Effect.Effect< - Pool.Pool, + Pool.Pool, never, Scope.Scope | ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | WorkerLauncher > = Effect.fnUntraced(function*(spec: TestRunnerPoolSpec) { @@ -113,7 +113,7 @@ export type TestRunnerPoolBlueprint = Blueprint.Of export const scoped = ( spec: TestRunnerPoolSpec, ): Effect.Effect< - Pool.Pool, + Pool.Pool, never, Scope.Scope | ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | WorkerLauncher > => TestRunnerPools.of(spec).scoped diff --git a/packages/stryker-js/src/stryker-outputs.ts b/packages/stryker-js/src/stryker-outputs.ts index 190b2537..289d5ec5 100644 --- a/packages/stryker-js/src/stryker-outputs.ts +++ b/packages/stryker-js/src/stryker-outputs.ts @@ -1,6 +1,7 @@ import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' export const REPRODUCERS_FILE = 'reports/mutation/reproducers.json' +export const FAILURE_RECORD_FILE = 'reports/mutation/failure.json' export const sarifFileNameOf = (jsonReportFileName: string): string => `${jsonReportFileName.replace(/\.[^./\\]*$/, '')}.sarif` @@ -18,4 +19,5 @@ export const strykerOutputFilesOf = ( options.jsonReporter.fileName, sarifFileNameOf(options.jsonReporter.fileName), REPRODUCERS_FILE, + FAILURE_RECORD_FILE, ] diff --git a/packages/stryker-js/src/worker-client.blueprint.ts b/packages/stryker-js/src/worker-client.blueprint.ts index b55c7a98..b7f77de2 100644 --- a/packages/stryker-js/src/worker-client.blueprint.ts +++ b/packages/stryker-js/src/worker-client.blueprint.ts @@ -1,4 +1,5 @@ import { Blueprint } from '@systemfsoftware/effect-cell-types' +import type { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import { Trace, Worker } from '@systemfsoftware/stryker-js-plugin-runtime' import * as Context from 'effect/Context' @@ -29,6 +30,7 @@ export interface WorkerClientParams { readonly workingDirectory: string readonly execArgv: readonly string[] readonly tempDirPrefix: string + readonly workerKind: FailureRecord.WorkerKind readonly env?: Readonly> | undefined } @@ -45,6 +47,7 @@ const WorkerClients = () => execArgv: params.execArgv, optionsJson, tempDirPrefix: params.tempDirPrefix, + workerKind: params.workerKind, env: params.env, }) const protocol = yield* worker.pipe( @@ -52,7 +55,8 @@ const WorkerClients = () => Layer.build, Effect.retry(connectRetry), Effect.raceFirst(worker.exited), - Effect.catchTag('SocketError', () => Effect.fail(WorkerBootTimeoutError.make({ pid: worker.pid }))), + Effect.catchTag('SocketError', () => + Effect.fail(WorkerBootTimeoutError.make({ pid: worker.pid, workerKind: params.workerKind }))), ) const traceContext = yield* Layer.build(Trace.layerTraceContextClient) diff --git a/packages/stryker-js/tests/__fixtures__/dry-run-failure.schema.ts b/packages/stryker-js/tests/__fixtures__/dry-run-failure.schema.ts index 9785dfcc..36b2d760 100644 --- a/packages/stryker-js/tests/__fixtures__/dry-run-failure.schema.ts +++ b/packages/stryker-js/tests/__fixtures__/dry-run-failure.schema.ts @@ -1,9 +1,14 @@ import * as S from 'effect/Schema' -export const DryRunFailedCause = S.TaggedStruct('DryRunFailed', { +export const BaselineTestsFailedEvidence = S.TaggedStruct('BaselineTestsFailed', { + stage: S.Literal('dryRun'), testCount: S.Finite, - failedTestCount: S.Finite, - failedTests: S.Array(S.Struct({ name: S.String, failureMessage: S.String })), + tests: S.Array(S.Struct({ + id: S.String, + name: S.String, + file: S.NullOr(S.String), + location: S.NullOr(S.Struct({ file: S.String, line: S.Finite, column: S.Finite })), + message: S.String, + stack: S.NullOr(S.String), + })), }) - -export type DryRunFailedView = S.Schema.Type diff --git a/packages/stryker-js/tests/__fixtures__/rerun-by-id.schema.ts b/packages/stryker-js/tests/__fixtures__/rerun-by-id.schema.ts index fbbc3d08..b2db371f 100644 --- a/packages/stryker-js/tests/__fixtures__/rerun-by-id.schema.ts +++ b/packages/stryker-js/tests/__fixtures__/rerun-by-id.schema.ts @@ -13,5 +13,3 @@ export const ReportSurvivors = S.Struct({ S.Struct({ mutants: S.Array(S.Struct({ id: S.String, status: S.String })) }), ), }) - -export const ErrorEnvelope = S.TaggedStruct('error', { code: S.Finite, error: S.String }) diff --git a/packages/stryker-js/tests/__fixtures__/substituted-worker.fixture.ts b/packages/stryker-js/tests/__fixtures__/substituted-worker.fixture.ts index 67b20079..27b62df1 100644 --- a/packages/stryker-js/tests/__fixtures__/substituted-worker.fixture.ts +++ b/packages/stryker-js/tests/__fixtures__/substituted-worker.fixture.ts @@ -96,12 +96,15 @@ const exitOf = (behaviour: ChildBehaviour): Effect.Effect rpcs: Plugin.CheckerRpcs, tempDirPrefix: 'checker-', workingDirectory: '/project', + workerKind: 'checker', }).pipe(Effect.provide(launcherLayer)) return { client, receivedRef } satisfies CheckerHarness diff --git a/packages/stryker-js/tests/config-file.integration.test.ts b/packages/stryker-js/tests/config-file.integration.test.ts index 9fa54a22..cd860db4 100644 --- a/packages/stryker-js/tests/config-file.integration.test.ts +++ b/packages/stryker-js/tests/config-file.integration.test.ts @@ -228,14 +228,16 @@ Feature('Configuring a Stryker run from a module config file') const hint = hintOf(failure) return expect({ tag: failure['_tag'], - exitClass: failure['exitClass'], + evidenceTag: failure.evidence._tag, + evidenceStage: failure.evidence.stage, namesTheAbandonedFile: fileOf(failure).endsWith(row.file), hintNamesLegacyFormats: hint.includes('JSON or CommonJS'), hintNamesEveryModuleFormat: ['.ts', '.mts', '.js', '.mjs'].every((extension) => hint.includes(extension)), hintNamesModuleDefaultExport: hint.includes('export default'), }).toEqual({ tag: 'ConfigFileUnsupportedError', - exitClass: 'ConfigError', + evidenceTag: 'ConfigInvalid', + evidenceStage: 'config', namesTheAbandonedFile: true, hintNamesLegacyFormats: true, hintNamesEveryModuleFormat: true, diff --git a/packages/stryker-js/tests/diff-scope.integration.test.ts b/packages/stryker-js/tests/diff-scope.integration.test.ts index 3bdfba9d..a52a34f5 100644 --- a/packages/stryker-js/tests/diff-scope.integration.test.ts +++ b/packages/stryker-js/tests/diff-scope.integration.test.ts @@ -46,7 +46,7 @@ interface Observation { readonly scope: string | undefined readonly mutatedFiles: readonly string[] readonly mutantLines: readonly number[] - readonly failure: { readonly stage: string; readonly exitClass: string; readonly namesRef: boolean } | undefined + readonly failure: { readonly evidenceTag: string; readonly stage: string; readonly namesRef: boolean } | undefined readonly refsSeen: readonly string[] } @@ -137,10 +137,10 @@ const runProject = ( onFailure: (cause) => Option.match(Cause.findErrorOption(cause), { onNone: () => undefined, - onSome: (stageError) => ({ - stage: stageError.stage, - exitClass: stageError.exitClass, - namesRef: S.is(GitDiffSchema.GitRefUnresolved)(stageError.cause), + onSome: (failure) => ({ + evidenceTag: failure.evidence._tag, + stage: failure.evidence.stage, + namesRef: S.is(GitDiffSchema.GitRefUnresolved)(failure.cause), }), }), onSuccess: () => undefined, @@ -215,7 +215,7 @@ Feature('Scoping a mutation run to the changed lines since a git ref') ) scenario( - 'An unknown ref fails the run with a named preparation error and exit class 2', + 'An unknown ref fails the run while preparing, naming the ref', Gherkin.Do.pipe( Given('a project whose git ref cannot be resolved')( 'observation', @@ -226,8 +226,12 @@ Feature('Scoping a mutation run to the changed lines since a git ref') (ref) => Effect.fail(GitDiffSchema.GitRefUnresolved.make({ ref, detail: 'unknown revision' })), ), ), - Then('the run fails while preparing, naming the ref, with the configuration exit class')((s, expect) => - expect(s.observation.failure).toEqual({ stage: 'prepare', exitClass: 'ConfigError', namesRef: true }) + Then('the run fails while preparing, naming the unresolvable ref')((s, expect) => + expect(s.observation.failure).toEqual({ + evidenceTag: 'SandboxPreparationFailed', + stage: 'prepare', + namesRef: true, + }) ), ), ) diff --git a/packages/stryker-js/tests/dry-run-failure.integration.test.ts b/packages/stryker-js/tests/dry-run-failure.integration.test.ts index e9909937..83b38f41 100644 --- a/packages/stryker-js/tests/dry-run-failure.integration.test.ts +++ b/packages/stryker-js/tests/dry-run-failure.integration.test.ts @@ -4,7 +4,6 @@ import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import * as Effect from 'effect/Effect' import * as FileSystem from 'effect/FileSystem' import * as Layer from 'effect/Layer' -import * as Logger from 'effect/Logger' import * as Option from 'effect/Option' import * as Path from 'effect/Path' import type { PlatformError } from 'effect/PlatformError' @@ -12,7 +11,7 @@ import * as Result from 'effect/Result' import * as S from 'effect/Schema' import * as Stdio from 'effect/Stdio' -import { DryRunFailedCause, type DryRunFailedView } from './__fixtures__/dry-run-failure.schema.js' +import { BaselineTestsFailedEvidence } from './__fixtures__/dry-run-failure.schema.js' const Feature = makeFeature({ it }) @@ -42,14 +41,8 @@ const TEST_CONTENT = [ '})', ].join('\n') -interface LogEntry { - readonly level: string - readonly text: string -} - interface ProjectFixture { readonly root: string - readonly logs: Array } const writeProject = (): Effect.Effect => @@ -61,7 +54,7 @@ const writeProject = (): Effect.Effect => @@ -69,21 +62,11 @@ const removeProject = (root: string): Effect.Effect fs.remove(root, { recursive: true })), )) -const capturingLogger = (fixture: ProjectFixture): Layer.Layer => - Logger.layer([ - Logger.make((entry) => { - fixture.logs.push({ - level: entry.logLevel, - text: [entry.message].flat().map(String).join(' '), - }) - }), - ]) - const runFromProject = ( root: string, options: Options.PartialStrykerOptions, ): Effect.Effect< - Result.Result, + Result.Result, never, Engine.EnginePorts > => @@ -101,20 +84,17 @@ const runFromProject = ( ) const failureOf = ( - outcome: Result.Result, -): Engine.StageError => { + outcome: Result.Result, +): Engine.RunFailure => { if (Result.isSuccess(outcome)) { throw new Error('the run was expected to fail its dry run, but it completed') } - if (!S.is(Engine.StageError)(outcome.failure)) { - throw new Error(`the run was expected to fail as a stage, not a platform failure: ${String(outcome.failure)}`) + if (!S.is(Engine.RunFailure)(outcome.failure)) { + throw new Error(`the run was expected to fail as a run failure, not a platform failure: ${String(outcome.failure)}`) } return outcome.failure } -const dryRunFailedCauseOf = (cause: Engine.StageError['cause']): Option.Option => - S.decodeUnknownOption(DryRunFailedCause)(cause) - const OPTIONS: Options.PartialStrykerOptions = { testRunner: 'vm', testFiles: ['test/**/*.mjs'], @@ -126,7 +106,7 @@ const OPTIONS: Options.PartialStrykerOptions = { const FROZEN_RUNNER_PLUGIN = new URL('./__fixtures__/frozen-runner/index.mjs', import.meta.url).href interface FrozenRunnerOutcome { - readonly outcome: Result.Result + readonly outcome: Result.Result readonly workerPid: number } @@ -159,46 +139,35 @@ Feature('Reporting why a dry run failed') Given('a project whose test suite fails two of its three tests')('project', () => writeProject()), When('the mutation run performs its initial test run')( 'outcome', - (s) => - runFromProject(s.project.root, OPTIONS).pipe( - Effect.provide(capturingLogger(s.project)), - Effect.ensuring(removeProject(s.project.root)), - ), + (s) => runFromProject(s.project.root, OPTIONS).pipe(Effect.ensuring(removeProject(s.project.root))), ), Then( - 'the refusal names every failing test with its failure message, in the cause, the reason, and the log', + 'the refusal reports every failing test by id, name and file', )((s, expect) => { const failure = failureOf(s.outcome) - const cause = Option.getOrUndefined(dryRunFailedCauseOf(failure.cause)) - const messages = cause?.failedTests.map((test) => test.failureMessage) ?? [] - const errorLog = s.project.logs - .filter((entry) => entry.level === 'Error') - .map((entry) => entry.text) - .join('\n') + const evidence = Option.getOrUndefined( + S.decodeUnknownOption(BaselineTestsFailedEvidence)(failure.evidence), + ) + const tests = evidence?.tests ?? [] + const messages = tests.map((test) => test.message) return expect({ - stage: failure.stage, - causeTag: cause?._tag, - causeTestCount: cause?.testCount, - causeFailedTestCount: cause?.failedTestCount, - causeFailedNames: cause?.failedTests.map((test) => test.name), - reasonNamesEveryFailure: FAILING_TEST_NAMES.every((name) => failure.reason.includes(name)), - reasonCarriesEveryMessage: messages.every((message) => failure.reason.includes(message)), + evidenceTag: failure.evidence._tag, + stage: failure.evidence.stage, + testCount: evidence?.testCount, + failedTestCount: tests.length, + failedNames: tests.map((test) => test.name), + idsNameTheirFileAndTest: tests.every((test) => test.id === `${TEST_FILE}#${test.name}`), + filesNameTheirTestFile: tests.every((test) => test.file === TEST_FILE), messagesAreAssertions: messages.every((message) => message.includes('expected')), - logNamesEveryFailure: FAILING_TEST_NAMES.every((name) => errorLog.includes(name)), - logCarriesEveryMessage: messages.every((message) => errorLog.includes(message)), - logCountsTheFailures: errorLog.includes('2 of 3 test(s) failed'), }).toEqual({ + evidenceTag: 'BaselineTestsFailed', stage: 'dryRun', - causeTag: 'DryRunFailed', - causeTestCount: 3, - causeFailedTestCount: 2, - causeFailedNames: FAILING_TEST_NAMES, - reasonNamesEveryFailure: true, - reasonCarriesEveryMessage: true, + testCount: 3, + failedTestCount: 2, + failedNames: FAILING_TEST_NAMES, + idsNameTheirFileAndTest: true, + filesNameTheirTestFile: true, messagesAreAssertions: true, - logNamesEveryFailure: true, - logCarriesEveryMessage: true, - logCountsTheFailures: true, }) }), ), @@ -212,15 +181,11 @@ Feature('Reporting why a dry run failed') ), When('the mutation run performs its initial test run')( 'run', - (s) => - runWithFrozenRunner(s.project).pipe( - Effect.provide(capturingLogger(s.project)), - Effect.ensuring(removeProject(s.project.root)), - ), + (s) => runWithFrozenRunner(s.project).pipe(Effect.ensuring(removeProject(s.project.root))), ), Then('the dry run fails and the unresponsive test runner does not outlive the run')((s, expect) => expect({ - stage: failureOf(s.run.outcome).stage, + stage: failureOf(s.run.outcome).evidence.stage, workerAlive: isProcessAlive(s.run.workerPid), }).toEqual({ stage: 'dryRun', workerAlive: false }) ), diff --git a/packages/stryker-js/tests/empty-file-selection.integration.test.ts b/packages/stryker-js/tests/empty-file-selection.integration.test.ts index 4d98f32e..e2ebd6f1 100644 --- a/packages/stryker-js/tests/empty-file-selection.integration.test.ts +++ b/packages/stryker-js/tests/empty-file-selection.integration.test.ts @@ -1,12 +1,10 @@ import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Engine } from '@systemfsoftware/stryker-js' import { type Options, Report } from '@systemfsoftware/stryker-js-plugin-interface' -import { PrepareError } from '@systemfsoftware/stryker-js/events' import * as Effect from 'effect/Effect' import * as FileSystem from 'effect/FileSystem' import * as Layer from 'effect/Layer' import * as Match from 'effect/Match' -import * as Option from 'effect/Option' import * as Path from 'effect/Path' import type { PlatformError } from 'effect/PlatformError' import * as Result from 'effect/Result' @@ -50,7 +48,7 @@ const runFromProject = ( root: string, options: Options.PartialStrykerOptions, ): Effect.Effect< - Result.Result, + Result.Result, never, Engine.EnginePorts > => @@ -78,26 +76,25 @@ const reportedFilesOf = ( return report.files }).pipe(Effect.orDie) -const verdictOf = (outcome: Result.Result) => +const verdictOf = (outcome: Result.Result) => Result.match(outcome, { onFailure: (failure) => `refused: ${failure.message}`, onSuccess: (done) => done.verdict, }) -const refusalOf = (outcome: Result.Result) => +const refusalOf = (outcome: Result.Result) => Result.match(outcome, { onFailure: (failure) => Match.value(failure).pipe( - Match.tag('StageError', (refused) => ({ - stage: refused.stage, - causeMessage: Option.getOrNull( - Option.map(S.decodeUnknownOption(PrepareError)(refused.cause), (cause) => cause.message), - ), + Match.tag('RunFailure', (refused) => ({ + evidenceTag: refused.evidence._tag, + stage: refused.evidence.stage, + detail: refused.detail, })), - Match.tag('PlatformError', (platform) => ({ stage: null, causeMessage: platform.message })), + Match.tag('PlatformError', (platform) => ({ evidenceTag: null, stage: null, detail: platform.message })), Match.exhaustive, ), - onSuccess: () => ({ stage: null, causeMessage: null }), + onSuccess: () => ({ evidenceTag: null, stage: null, detail: null }), }) const runLayer = Layer.mergeAll(Engine.nodePlatformLayer, Stdio.layerTest({})) @@ -119,7 +116,11 @@ Feature('Running mutation testing when no file is selected') runFromProject(s.root, { testRunner: 'vm', checkers: [] }).pipe(Effect.ensuring(removeProject(s.root))), ), Then('the run stops while preparing, naming the missing input files')((s, expect) => - expect(refusalOf(s.outcome)).toEqual({ stage: 'prepare', causeMessage: 'No input files found.' }) + expect(refusalOf(s.outcome)).toEqual({ + evidenceTag: 'NoInputFiles', + stage: 'prepare', + detail: 'No input files found.', + }) ), ), ) diff --git a/packages/stryker-js/tests/framework-run.integration.test.ts b/packages/stryker-js/tests/framework-run.integration.test.ts index 4fe60aac..46c56dfc 100644 --- a/packages/stryker-js/tests/framework-run.integration.test.ts +++ b/packages/stryker-js/tests/framework-run.integration.test.ts @@ -1,13 +1,15 @@ import { NodeFileSystem, NodePath } from '@effect/platform-node' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Engine } from '@systemfsoftware/stryker-js' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' -import type * as Cause from 'effect/Cause' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import * as Cause from 'effect/Cause' import * as Effect from 'effect/Effect' import * as Exit from 'effect/Exit' import * as FileSystem from 'effect/FileSystem' import * as Layer from 'effect/Layer' +import * as Option from 'effect/Option' import * as Path from 'effect/Path' +import * as Predicate from 'effect/Predicate' import * as Queue from 'effect/Queue' import * as S from 'effect/Schema' @@ -175,11 +177,19 @@ const environmentFor = (directory: string): Engine.RunEnvironmentShape => ({ const incrementalFileOf = (directory: string): string => `${directory}/reports/stryker-incremental.json` interface RunObservation { - readonly exit: Exit.Exit + readonly exit: Exit.Exit readonly events: ReadonlyArray readonly incrementalState: string } +const failureEvidenceOf = (run: RunObservation): FailureRecord.FailureEvidence | undefined => + Option.getOrUndefined( + Exit.match(run.exit, { + onFailure: (cause) => Option.map(Cause.findErrorOption(cause), (failure) => failure.evidence), + onSuccess: () => Option.none(), + }), + ) + const incrementalStateOf = (directory: string): Effect.Effect => Effect.flatMap(FileSystem.FileSystem, (fs) => fs.readFileString(incrementalFileOf(directory))).pipe( Effect.orElseSucceed(() => ''), @@ -254,6 +264,7 @@ Feature('Framework plugins joining a mutation run') const row = skipped?.files.find((file) => file.file.endsWith('widget.svelte')) return expect({ runSucceeded: Exit.isSuccess(s.observation.exit), + failure: failureEvidenceOf(s.observation), verdictCount: verdicts.length, skippedExtension: row?.extension, skippedReasonNamesFramework: row?.reason.includes('@systemfsoftware/stryker-js-svelte') ?? false, @@ -262,6 +273,7 @@ Feature('Framework plugins joining a mutation run') verdictHasSvelteMutant: verdicts[0]?.mutants.some((mutant) => mutant.file.endsWith('.svelte')) ?? true, }).toEqual({ runSucceeded: true, + failure: undefined, verdictCount: 1, skippedExtension: '.svelte', skippedReasonNamesFramework: true, @@ -292,27 +304,30 @@ Feature('Framework plugins joining a mutation run') 'the run refuses before touching any file, naming the missing peer, with no verdict and no files examined', )( (s, expect) => { - const failure = s.observation.events.find((event): event is RunEvent.RunFailed => - S.is(RunEvent.RunFailed)(event) - ) + const evidence = failureEvidenceOf(s.observation) + const load = evidence !== undefined && Predicate.isTagged(evidence, 'PluginLoadFailed') + ? evidence + : undefined const phases = s.observation.events .filter((event): event is RunEvent.PhaseEntered => S.is(RunEvent.PhaseEntered)(event)) .map((phase) => phase.phase) return expect({ runFailed: Exit.isFailure(s.observation.exit), - reason: failure?.reason?._tag, - code: failure?.code, - errorNamesPeer: failure?.error.includes('peer-missing') ?? false, - remediationNamesPeerDependency: failure?.remediation.includes('peer dependency') ?? false, + tag: evidence?._tag, + stage: evidence?.stage, + code: evidence === undefined ? undefined : FailureRecord.FailureCatalog[evidence._tag].exitCode, + descriptorNamesPeer: load?.descriptor.includes('peer-missing') ?? false, + reasonTag: load?.reason._tag, phases, reachedVerdict: s.observation.events.some((event) => S.is(RunEvent.VerdictReached)(event)), reportedSkippedFiles: s.observation.events.some((event) => S.is(RunEvent.SkippedReported)(event)), }).toEqual({ runFailed: true, - reason: 'PeerMissing', + tag: 'PluginLoadFailed', + stage: 'config', code: 2, - errorNamesPeer: true, - remediationNamesPeerDependency: true, + descriptorNamesPeer: true, + reasonTag: 'PeerMissing', phases: ['prepare'], reachedVerdict: false, reportedSkippedFiles: false, @@ -351,43 +366,44 @@ Feature('Framework plugins joining a mutation run') Then( 'the resolved-formats report names the claimed type, its mutant is exercised, and the run state records the claim', )( - (s, expect) => - Effect.map( - S.decodeEffect(S.fromJsonString(Engine.IncrementalReportSchema))(s.observation.incrementalState), - (state) => { - const formats = s.observation.events.find( - (event): event is RunEvent.FormatRegistryResolved => S.is(RunEvent.FormatRegistryResolved)(event), - ) - const formatRow = formats?.rows.find((candidate) => candidate.extension === '.fixture') - const tested = s.observation.events.filter( - (event): event is RunEvent.RunMutantTested => S.is(RunEvent.RunMutantTested)(event), - ) - const fromClaimed = tested.find((mutant) => mutant.fileName.endsWith('widget.fixture')) - return expect({ - runSucceeded: Exit.isSuccess(s.observation.exit), - formatOwner: formatRow?.ownerModule, - formatId: formatRow?.formatId, - formatLanguage: formatRow?.language, - claimedMutantStatus: fromClaimed?.status, - stateLanguage: state.files['src/widget.fixture']?.language, - stateFormatIdentity: state.files['src/widget.fixture']?.formatIdentity, - scriptLanguage: state.files['src/math.js']?.language, - }).toEqual({ - runSucceeded: true, - formatOwner: pluginUrlOf('valid-framework.fixture.mjs'), - formatId: 'fixture', - formatLanguage: 'fixture', - claimedMutantStatus: 'NoCoverage', - stateLanguage: 'fixture', - stateFormatIdentity: { - formatId: 'fixture', - ownerModule: pluginUrlOf('valid-framework.fixture.mjs'), - ownerVersion: '1.0.0', - }, - scriptLanguage: 'javascript', - }) + (s, expect) => { + const state = Option.getOrUndefined( + S.decodeOption(S.fromJsonString(Engine.IncrementalReportSchema))(s.observation.incrementalState), + ) + const formats = s.observation.events.find( + (event): event is RunEvent.FormatRegistryResolved => S.is(RunEvent.FormatRegistryResolved)(event), + ) + const formatRow = formats?.rows.find((candidate) => candidate.extension === '.fixture') + const tested = s.observation.events.filter( + (event): event is RunEvent.RunMutantTested => S.is(RunEvent.RunMutantTested)(event), + ) + const fromClaimed = tested.find((mutant) => mutant.fileName.endsWith('widget.fixture')) + return expect({ + runSucceeded: Exit.isSuccess(s.observation.exit), + failure: failureEvidenceOf(s.observation), + formatOwner: formatRow?.ownerModule, + formatId: formatRow?.formatId, + formatLanguage: formatRow?.language, + claimedMutantStatus: fromClaimed?.status, + stateLanguage: state?.files['src/widget.fixture']?.language, + stateFormatIdentity: state?.files['src/widget.fixture']?.formatIdentity, + scriptLanguage: state?.files['src/math.js']?.language, + }).toEqual({ + runSucceeded: true, + failure: undefined, + formatOwner: pluginUrlOf('valid-framework.fixture.mjs'), + formatId: 'fixture', + formatLanguage: 'fixture', + claimedMutantStatus: 'NoCoverage', + stateLanguage: 'fixture', + stateFormatIdentity: { + formatId: 'fixture', + ownerModule: pluginUrlOf('valid-framework.fixture.mjs'), + ownerVersion: '1.0.0', }, - ), + scriptLanguage: 'javascript', + }) + }, ), ), ) @@ -464,19 +480,22 @@ Feature('Framework plugins joining a mutation run') (s) => runOver(s.workspace), ), Then('the run refuses with an internal-error exit code naming the failing module')((s, expect) => { - const failure = s.observation.events.find((event): event is RunEvent.RunFailed => - S.is(RunEvent.RunFailed)(event) - ) + const evidence = failureEvidenceOf(s.observation) + const imported = evidence !== undefined && Predicate.isTagged(evidence, 'PluginImportFailed') + ? evidence + : undefined return expect({ runFailed: Exit.isFailure(s.observation.exit), - reason: failure?.reason?._tag, - code: failure?.code, - errorNamesModule: failure?.error.includes('throws-on-import') ?? false, + tag: evidence?._tag, + stage: evidence?.stage, + code: evidence === undefined ? undefined : FailureRecord.FailureCatalog[evidence._tag].exitCode, + descriptorNamesModule: imported?.descriptor.includes('throws-on-import') ?? false, }).toEqual({ runFailed: true, - reason: 'ImportFailed', + tag: 'PluginImportFailed', + stage: 'config', code: 4, - errorNamesModule: true, + descriptorNamesModule: true, }) }), ), @@ -498,19 +517,24 @@ Feature('Framework plugins joining a mutation run') (s) => runOver(s.workspace), ), Then('the run refuses with a configuration exit code naming the plugin module')((s, expect) => { - const failure = s.observation.events.find((event): event is RunEvent.RunFailed => - S.is(RunEvent.RunFailed)(event) - ) + const evidence = failureEvidenceOf(s.observation) + const load = evidence !== undefined && Predicate.isTagged(evidence, 'PluginLoadFailed') + ? evidence + : undefined return expect({ runFailed: Exit.isFailure(s.observation.exit), - reason: failure?.reason?._tag, - code: failure?.code, - errorNamesModule: failure?.error.includes('malformed') ?? false, + tag: evidence?._tag, + stage: evidence?.stage, + code: evidence === undefined ? undefined : FailureRecord.FailureCatalog[evidence._tag].exitCode, + descriptorNamesModule: load?.descriptor.includes('malformed') ?? false, + reasonTag: load?.reason._tag, }).toEqual({ runFailed: true, - reason: 'InvalidContribution', + tag: 'PluginLoadFailed', + stage: 'config', code: 2, - errorNamesModule: true, + descriptorNamesModule: true, + reasonTag: 'InvalidContribution', }) }), ), @@ -535,27 +559,30 @@ Feature('Framework plugins joining a mutation run') 'the run refuses before touching any file, naming the unrecognized peer, with no verdict and no files examined', )( (s, expect) => { - const failure = s.observation.events.find((event): event is RunEvent.RunFailed => - S.is(RunEvent.RunFailed)(event) - ) + const evidence = failureEvidenceOf(s.observation) + const load = evidence !== undefined && Predicate.isTagged(evidence, 'PluginLoadFailed') + ? evidence + : undefined const phases = s.observation.events .filter((event): event is RunEvent.PhaseEntered => S.is(RunEvent.PhaseEntered)(event)) .map((phase) => phase.phase) return expect({ runFailed: Exit.isFailure(s.observation.exit), - reason: failure?.reason?._tag, - code: failure?.code, - errorNamesPeer: failure?.error.includes('peer-unrecognized') ?? false, - remediationNamesRecognition: failure?.remediation.includes('recognizes') ?? false, + tag: evidence?._tag, + stage: evidence?.stage, + code: evidence === undefined ? undefined : FailureRecord.FailureCatalog[evidence._tag].exitCode, + descriptorNamesPeer: load?.descriptor.includes('peer-unrecognized') ?? false, + reasonTag: load?.reason._tag, phases, reachedVerdict: s.observation.events.some((event) => S.is(RunEvent.VerdictReached)(event)), reportedSkippedFiles: s.observation.events.some((event) => S.is(RunEvent.SkippedReported)(event)), }).toEqual({ runFailed: true, - reason: 'PeerUnrecognized', + tag: 'PluginLoadFailed', + stage: 'config', code: 2, - errorNamesPeer: true, - remediationNamesRecognition: true, + descriptorNamesPeer: true, + reasonTag: 'PeerUnrecognized', phases: ['prepare'], reachedVerdict: false, reportedSkippedFiles: false, diff --git a/packages/stryker-js/tests/framework-stream.integration.test.ts b/packages/stryker-js/tests/framework-stream.integration.test.ts index 7e61d9ed..b8ce43b8 100644 --- a/packages/stryker-js/tests/framework-stream.integration.test.ts +++ b/packages/stryker-js/tests/framework-stream.integration.test.ts @@ -1,7 +1,8 @@ import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import * as Effect from 'effect/Effect' import * as Layer from 'effect/Layer' +import * as Predicate from 'effect/Predicate' import * as S from 'effect/Schema' const Feature = makeFeature({ it }) @@ -43,11 +44,17 @@ const skippedReport = RunEvent.SkippedReported.make({ }) const refusal = RunEvent.RunFailed.make({ - schemaVersion: '2.0', + schemaVersion: '3.0', code: 2, - error: `Failed to load plugin "${frameworkModule}" (PeerMissing)`, - remediation: 'install the peer dependency the plugin needs', - reason: { _tag: 'PeerMissing', peer: frameworkModule }, + record: FailureRecord.recordOf( + { + _tag: 'PluginLoadFailed', + stage: 'config', + descriptor: frameworkModule, + reason: { _tag: 'PeerMissing', peer: frameworkModule }, + }, + { cause: [], cwd: '/project', argv: ['stryker', 'run'], env: [], traceId: null }, + ), }) type ReportEvent = @@ -113,13 +120,23 @@ Feature('Reporting framework plugins on the machine wire') 'seen', (s) => decodedOf(s.line), ), - Then('the failure names the reason, the configuration code, and the remedy')((s, expect) => - expect({ reason: s.seen.reason, code: s.seen.code, remediation: s.seen.remediation }).toEqual({ + Then('the failure names the reason, the configuration code, and the remedy')((s, expect) => { + const record = s.seen.record + const load = Predicate.isTagged(record, 'PluginLoadFailed') ? record : undefined + return expect({ + tag: record._tag, + descriptor: load?.descriptor, + reason: load?.reason, + code: s.seen.code, + next: record.nextAction.primary, + }).toEqual({ + tag: 'PluginLoadFailed', + descriptor: frameworkModule, reason: { _tag: 'PeerMissing', peer: frameworkModule }, code: 2, - remediation: 'install the peer dependency the plugin needs', + next: 'fixConfiguration', }) - ), + }), ), ) diff --git a/packages/stryker-js/tests/incremental-reuse.integration.test.ts b/packages/stryker-js/tests/incremental-reuse.integration.test.ts index 46b40fd3..c8cbf584 100644 --- a/packages/stryker-js/tests/incremental-reuse.integration.test.ts +++ b/packages/stryker-js/tests/incremental-reuse.integration.test.ts @@ -44,7 +44,7 @@ const mutantsOf = (text: string): readonly RecordedMutant[] => }) interface RunObservation { - readonly exit: Exit.Exit + readonly exit: Exit.Exit readonly events: ReadonlyArray readonly reuse: RunEvent.ReuseReported | undefined readonly verdict: RunEvent.VerdictReached | undefined diff --git a/packages/stryker-js/tests/instrument-stage.integration.test.ts b/packages/stryker-js/tests/instrument-stage.integration.test.ts index f06851b7..4479d8d3 100644 --- a/packages/stryker-js/tests/instrument-stage.integration.test.ts +++ b/packages/stryker-js/tests/instrument-stage.integration.test.ts @@ -40,7 +40,7 @@ const runFromProject = ( root: string, options: Options.PartialStrykerOptions, ): Effect.Effect< - Result.Result, + Result.Result, never, Engine.EnginePorts > => @@ -58,13 +58,15 @@ const runFromProject = ( ) const failureOf = ( - outcome: Result.Result, -): Engine.StageError => { + outcome: Result.Result, +): Engine.RunFailure => { if (Result.isSuccess(outcome)) { throw new Error('the run was expected to be refused, but it completed') } - if (!S.is(Engine.StageError)(outcome.failure)) { - throw new Error(`the run was expected to be refused as a stage, not a platform failure: ${String(outcome.failure)}`) + if (!S.is(Engine.RunFailure)(outcome.failure)) { + throw new Error( + `the run was expected to be refused as a run failure, not a platform failure: ${String(outcome.failure)}`, + ) } return outcome.failure } @@ -72,7 +74,7 @@ const failureOf = ( const carriesMessage = (value: unknown): value is { readonly message: string } => typeof value === 'object' && value !== null && 'message' in value && typeof value.message === 'string' -const textOf = (cause: Engine.StageError['cause']): string => +const textOf = (cause: Engine.RunFailure['cause']): string => carriesMessage(cause) ? cause.message : 'the refused cause carried no message' const runLayer = Layer.mergeAll(Engine.nodePlatformLayer, Stdio.layerTest({})) @@ -99,14 +101,14 @@ Feature('Opting a mutation run into extra mutations') const failure = failureOf(s.outcome) const cause = textOf(failure.cause) return expect({ - stage: failure.stage, - exitClass: failure['exitClass'], - unknownEntryNamed: failure.message.includes(UNKNOWN_NAME), + evidenceTag: failure.evidence._tag, + stage: failure.evidence.stage, + detailNamesTheUnknownEntry: failure.detail.includes(UNKNOWN_NAME), causeNamesTheUnknownEntry: cause.includes(UNKNOWN_NAME), }).toEqual({ - stage: 'prepare', - exitClass: 'ConfigError', - unknownEntryNamed: true, + evidenceTag: 'ConfigInvalid', + stage: 'config', + detailNamesTheUnknownEntry: true, causeNamesTheUnknownEntry: true, }) }, diff --git a/packages/stryker-js/tests/mutant-location-parity.integration.test.ts b/packages/stryker-js/tests/mutant-location-parity.integration.test.ts index b663cb14..8416cdd8 100644 --- a/packages/stryker-js/tests/mutant-location-parity.integration.test.ts +++ b/packages/stryker-js/tests/mutant-location-parity.integration.test.ts @@ -95,7 +95,7 @@ const environmentFor = (directory: string): Engine.RunEnvironmentShape => ({ }) interface ObservedRun { - readonly exit: Exit.Exit + readonly exit: Exit.Exit readonly events: ReadonlyArray readonly report: Option.Option } diff --git a/packages/stryker-js/tests/plugin-mutator-provider.integration.test.ts b/packages/stryker-js/tests/plugin-mutator-provider.integration.test.ts index 11d92830..1f2003bc 100644 --- a/packages/stryker-js/tests/plugin-mutator-provider.integration.test.ts +++ b/packages/stryker-js/tests/plugin-mutator-provider.integration.test.ts @@ -67,7 +67,7 @@ const environmentFor = (directory: string): Engine.RunEnvironmentShape => ({ }) interface ObservedRun { - readonly outcome: Result.Result + readonly outcome: Result.Result readonly events: ReadonlyArray readonly report: Option.Option } @@ -153,12 +153,19 @@ const reportOf = (run: ObservedRun): Report.MutationTestResult => Option.getOrThrowWith(run.report, () => new Error('the run wrote no JSON report')) const failureTextOf = (run: ObservedRun): string => - Result.isFailure(run.outcome) && S.is(Engine.StageError)(run.outcome.failure) - ? run.outcome.failure.message + Result.isFailure(run.outcome) && S.is(Engine.RunFailure)(run.outcome.failure) + ? run.outcome.failure.detail : 'no prepare refusal' const stageOf = (run: ObservedRun): string | null => - Result.isFailure(run.outcome) && S.is(Engine.StageError)(run.outcome.failure) ? run.outcome.failure.stage : null + Result.isFailure(run.outcome) && S.is(Engine.RunFailure)(run.outcome.failure) + ? run.outcome.failure.evidence.stage + : null + +const tagOf = (run: ObservedRun): string | null => + Result.isFailure(run.outcome) && S.is(Engine.RunFailure)(run.outcome.failure) + ? run.outcome.failure.evidence._tag + : null Feature('A plugin contributing a namespaced mutator catalog') .withLayer(Layer.empty) @@ -193,19 +200,19 @@ Feature('A plugin contributing a namespaced mutator catalog') Gherkin.Do.pipe( Given('two provider modules declaring the same namespace')('plugins', () => Effect.succeed([PROVIDER, RIVAL])), When('a mutation run loads both')('observation', (s) => runAndClean(s.plugins)), - Then('the run is refused while preparing, and the refusal names both modules')((s, expect) => { + Then('the run is refused as a configuration error, and the refusal names both modules')((s, expect) => { const message = failureTextOf(s.observation) return expect({ refused: Result.isFailure(s.observation.outcome), + tag: tagOf(s.observation), stage: stageOf(s.observation), - prepareFailureNamed: message.includes('Prepare failed'), providerNamed: message.includes('mutator-provider/index.mjs'), rivalNamed: message.includes('mutator-provider/rival.mjs'), namespaceNamed: message.includes('acme'), }).toEqual({ refused: true, - stage: 'prepare', - prepareFailureNamed: true, + tag: 'ConfigInvalid', + stage: 'config', providerNamed: true, rivalNamed: true, namespaceNamed: true, diff --git a/packages/stryker-js/tests/rerun-by-id.integration.test.ts b/packages/stryker-js/tests/rerun-by-id.integration.test.ts index 794b9172..4cbac598 100644 --- a/packages/stryker-js/tests/rerun-by-id.integration.test.ts +++ b/packages/stryker-js/tests/rerun-by-id.integration.test.ts @@ -7,13 +7,14 @@ import * as Fiber from 'effect/Fiber' import * as FileSystem from 'effect/FileSystem' import * as Option from 'effect/Option' import * as Path from 'effect/Path' +import * as Predicate from 'effect/Predicate' import * as ChildProcess from 'effect/process/ChildProcess' import * as ChildProcessSpawner from 'effect/process/ChildProcessSpawner' import * as S from 'effect/Schema' import * as Scope from 'effect/Scope' import * as Stream from 'effect/Stream' -import { ErrorEnvelope, ReportSurvivors, ReproducerList } from './__fixtures__/rerun-by-id.schema.js' +import { ReportSurvivors, ReproducerList } from './__fixtures__/rerun-by-id.schema.js' const Feature = makeFeature({ it }) @@ -123,8 +124,8 @@ const readFirstSurvivorId = (root: string) => const eventLinesOf = (stdout: string): ReadonlyArray => stdout.split('\n').flatMap((line) => (line.trimStart().startsWith('{') ? [line.trim()] : [])) -const errorEnvelopesOf = (stdout: string): ReadonlyArray => - eventLinesOf(stdout).flatMap((line) => Option.toArray(S.decodeOption(S.fromJsonString(ErrorEnvelope))(line))) +const runFailuresOf = (stdout: string): ReadonlyArray => + eventLinesOf(stdout).flatMap((line) => Option.toArray(S.decodeOption(S.fromJsonString(RunEvent.RunFailed))(line))) const mutantDetailsOf = (stdout: string): ReadonlyArray => eventLinesOf(stdout).flatMap((line) => @@ -195,27 +196,36 @@ Feature('Re-running one mutant by its id', { timeout: 180_000 }) (s) => runStryker(s.project.root, 'machine', ['run', '--mutant', UNKNOWN_ID]), ), Then( - 'both runs are refused with the configuration exit code, the person is told the id and the remedy exactly once, and the machine reads the same refusal', + 'both runs are refused with the configuration exit code, the person is told the id and the remedy, and the machine reads the same refusal', )( (s, expect) => { - const failure = Arr.head(errorEnvelopesOf(s.machine.stdout)) + const failure = Arr.head(runFailuresOf(s.machine.stdout)) const says = (needle: string): boolean => - Option.getOrElse(Option.map(failure, (envelope) => envelope.error.includes(needle)), () => false) + Option.getOrElse( + Option.map(failure, (envelope) => { + const record = envelope.record + return Predicate.isTagged(record, 'ConfigInvalid') ? record.detail.includes(needle) : false + }), + () => false, + ) return expect({ humanExitCode: s.human.exitCode, humanNamesTheId: s.human.stderr.includes(UNKNOWN_ID), humanNamesTheRemediation: s.human.stderr.includes('stryker run'), - humanSaysItOnce: s.human.stderr.split(UNKNOWN_ID).length - 1, machineExitCode: s.machine.exitCode, machineCode: Option.map(failure, (envelope) => envelope.code), + machineIsConfigInvalid: Option.exists( + failure, + (envelope) => Predicate.isTagged(envelope.record, 'ConfigInvalid'), + ), machineNamesTheId: says(UNKNOWN_ID), }).toStrictEqual({ humanExitCode: 2, humanNamesTheId: true, humanNamesTheRemediation: true, - humanSaysItOnce: 1, machineExitCode: 2, machineCode: Option.some(2), + machineIsConfigInvalid: true, machineNamesTheId: true, }) }, diff --git a/packages/stryker-js/tests/run-event-stream.integration.test.ts b/packages/stryker-js/tests/run-event-stream.integration.test.ts index 98e13d41..0ae5b3e6 100644 --- a/packages/stryker-js/tests/run-event-stream.integration.test.ts +++ b/packages/stryker-js/tests/run-event-stream.integration.test.ts @@ -1,6 +1,6 @@ import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { RunEvent } from '@systemfsoftware/stryker-js' -import { RunEvent as CliContract } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent as CliContract } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, Report } from '@systemfsoftware/stryker-js-plugin-interface' import * as Effect from 'effect/Effect' import * as Layer from 'effect/Layer' @@ -19,14 +19,17 @@ const Feature = makeFeature({ it }) const PLAN_KNOWN = CliContract.PlanKnown.make({ total: 4 }) const PHASE_ENTERED = CliContract.PhaseEntered.make({ phase: 'dry-run', elapsedMs: 1 }) const HEARTBEAT = CliContract.Heartbeat.make({ elapsedMs: 2, completed: 1, total: 4 }) -const HELP_RENDERED = CliContract.HelpRendered.make({ schemaVersion: '2.0', code: 0, help: 'usage' }) +const HELP_RENDERED = CliContract.HelpRendered.make({ schemaVersion: '3.0', code: 0, help: 'usage' }) +const RUN_FAILED_RECORD = FailureRecord.recordOf( + { _tag: 'TestRunnerFailed', stage: 'mutationTest' }, + { cause: [], cwd: '/base', argv: ['stryker', 'run'], env: [], traceId: null }, +) const RUN_FAILED = CliContract.RunFailed.make({ - schemaVersion: '2.0', + schemaVersion: '3.0', code: 3, - error: 'x', - remediation: 'y', - reason: null, + record: RUN_FAILED_RECORD, }) +const RUN_FAILED_TEXT = FailureRecord.terminalTextOf(RUN_FAILED_RECORD) const pathService = Effect.runSync(Effect.provide(Path.Path, Path.layer)) @@ -255,7 +258,7 @@ Feature('Streaming a run to machine readers') tags: ['stream', 'plan', 'phase', 'tick', 'help'], newlineTerminated: true, stderr: [], - opening: { mode: 'machine', signal: 'tty', schemaVersion: '2.0', runIdIsNonEmpty: true }, + opening: { mode: 'machine', signal: 'tty', schemaVersion: '3.0', runIdIsNonEmpty: true }, }) }), ), @@ -287,7 +290,7 @@ Feature('Streaming a run to machine readers') open: s.result.open, }).toEqual({ stdout: [], - stderr: ['plan 4 mutants', 'phase dry-run', 'error x'], + stderr: ['plan 4 mutants', 'phase dry-run', RUN_FAILED_TEXT], open: false, }) ), @@ -364,7 +367,7 @@ Feature('Streaming a run to machine readers') }), ), Then( - 'the consumer receives only the opening header and the error document, which carries the failure code, error message, and remediation guidance, and the stream is permanently closed', + 'the consumer receives only the opening header and the failure document, which carries the failure code and its structured record, and the stream is permanently closed', )((s, expect) => { const failure = Option.filter(decodedEventAt(s.result.stdout, 1), S.is(CliContract.RunFailed)) return expect({ @@ -373,15 +376,14 @@ Feature('Streaming a run to machine readers') onNone: () => null, onSome: (failed) => ({ code: failed.code, - error: failed.error, - remediation: failed.remediation, + tag: failed.record._tag, schemaVersion: failed.schemaVersion, }), }), open: s.result.open, }).toEqual({ tags: ['stream', 'error'], - failure: { code: 3, error: 'x', remediation: 'y', schemaVersion: '2.0' }, + failure: { code: 3, tag: 'TestRunnerFailed', schemaVersion: '3.0' }, open: false, }) }), diff --git a/packages/stryker-js/tests/slow-reporter-terminal-report.integration.test.ts b/packages/stryker-js/tests/slow-reporter-terminal-report.integration.test.ts index 5342fb3b..2c126fe2 100644 --- a/packages/stryker-js/tests/slow-reporter-terminal-report.integration.test.ts +++ b/packages/stryker-js/tests/slow-reporter-terminal-report.integration.test.ts @@ -35,7 +35,7 @@ interface Workspace { } interface ObservedRun { - readonly exit: Exit.Exit + readonly exit: Exit.Exit readonly observation: Option.Option readonly report: Option.Option } diff --git a/packages/stryker-js/tests/vm-parity.differential.test.ts b/packages/stryker-js/tests/vm-parity.differential.test.ts index e0354af5..a681a1c3 100644 --- a/packages/stryker-js/tests/vm-parity.differential.test.ts +++ b/packages/stryker-js/tests/vm-parity.differential.test.ts @@ -279,7 +279,7 @@ const withVmRunner = ( const describeDryRunFailure = (dry: Exclude): string => dry.status === 'error' ? `the vm dry run failed: ${dry.errorMessage}` : `the vm dry run ended in ${dry.status}` -const vmTestOutcomes = (root: string, path: Path.Path): Effect.Effect => +const vmTestOutcomes = (root: string): Effect.Effect => serialized( withVmRunner(root, (runner) => Effect.gen(function*() { @@ -292,7 +292,7 @@ const vmTestOutcomes = (root: string, path: Path.Path): Effect.Effect return recordOf(dry.tests.map((test): readonly [string, string] => { const hash = test.id.lastIndexOf('#') const file = hash === -1 ? '' : test.id.slice(0, hash) - const name = test.name === path.join(root, file) ? FILE_FAILED_WITHOUT_FAILING_TEST : test.name + const name = test.name === file ? FILE_FAILED_WITHOUT_FAILING_TEST : test.name return [`${file}#${name}`, test.status] })) })), @@ -658,7 +658,7 @@ const generatedRealOutcomes = (suite: GeneratedSuite): Effect.Effect = ) const generatedVmOutcomes = (suite: GeneratedSuite): Effect.Effect => - withGeneratedSuite(suite, (directory, path) => vmTestOutcomes(directory, path)).pipe( + withGeneratedSuite(suite, (directory) => vmTestOutcomes(directory)).pipe( Effect.provide(Engine.nodePlatformLayer), dieOnFailure, labelled('the vm runner failed'), diff --git a/packages/stryker-js/tests/wall-clock-timeout.integration.test.ts b/packages/stryker-js/tests/wall-clock-timeout.integration.test.ts index e9053b3a..a193531a 100644 --- a/packages/stryker-js/tests/wall-clock-timeout.integration.test.ts +++ b/packages/stryker-js/tests/wall-clock-timeout.integration.test.ts @@ -112,7 +112,7 @@ interface MutantRow { } interface ObservedRun { - readonly exit: Exit.Exit + readonly exit: Exit.Exit readonly events: ReadonlyArray readonly daemonMutants: ReadonlyArray } diff --git a/packages/stryker-js/tests/worker-connection-loss.integration.test.ts b/packages/stryker-js/tests/worker-connection-loss.integration.test.ts index be48e4d3..dc2e4579 100644 --- a/packages/stryker-js/tests/worker-connection-loss.integration.test.ts +++ b/packages/stryker-js/tests/worker-connection-loss.integration.test.ts @@ -109,6 +109,7 @@ const makeHarness = (bootRefusals = 0) => rpcs: Plugin.CheckerRpcs, tempDirPrefix: 'checker-', workingDirectory: '/project', + workerKind: 'checker', }).pipe(Effect.provide(launcherLayer)) return { client, gate, received } satisfies SilentWorkerHarness diff --git a/packages/stryker-js/tests/worker-launcher.integration.test.ts b/packages/stryker-js/tests/worker-launcher.integration.test.ts index bcdb3ac0..a5324060 100644 --- a/packages/stryker-js/tests/worker-launcher.integration.test.ts +++ b/packages/stryker-js/tests/worker-launcher.integration.test.ts @@ -43,6 +43,7 @@ const bootPingWorker = ( workingDirectory: WORKING_DIRECTORY, execArgv: EXEC_ARGV, tempDirPrefix: TEMP_DIR_PREFIX, + workerKind: 'testRunner', env: undefined, }).pipe( Effect.flatMap((client) => client.ping({ message: 'boot' })), diff --git a/scripts/guards/check-contract-versions.ts b/scripts/guards/check-contract-versions.ts index c8b9ecbb..f7672565 100755 --- a/scripts/guards/check-contract-versions.ts +++ b/scripts/guards/check-contract-versions.ts @@ -15,6 +15,7 @@ const KIND_BY_BASENAME: Record = { 'stream.schema.json': 'json-schema', 'report.schema.json': 'json-schema', 'stock-catalog.json': 'catalog', + 'failure-catalog.json': 'catalog', 'span-taxonomy.json': 'taxonomy', } diff --git a/test/e2e/tests/__fixtures__/typescript-checker.fixture.ts b/test/e2e/tests/__fixtures__/typescript-checker.fixture.ts index 8d50d88c..d68f56ef 100644 --- a/test/e2e/tests/__fixtures__/typescript-checker.fixture.ts +++ b/test/e2e/tests/__fixtures__/typescript-checker.fixture.ts @@ -1,4 +1,4 @@ -import type { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, type RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { Report } from '@systemfsoftware/stryker-js-plugin-interface' import type { Check, Expect } from '@systemfsoftware/vitest' import type { ExecResult } from '../../src/Harness/guest-job.schema.js' @@ -86,6 +86,9 @@ export const reportEnvelopeOf = (run: ExecResult, report: Report.MutationTestRes export const verifyReportEnvelope = (expect: Expect, envelope: ReportEnvelope): Check => expect(envelope).toStrictEqual({ exitCode: 0, schemaVersion: '1.0' }) +const BROKEN_CHECKER_EXIT_CODE = 3 +const MISSING_TSCONFIG = 'non-existent-tsconfig.json' + export const verifyBrokenCheckerError = ( expect: Expect, run: ExecResult, @@ -93,20 +96,27 @@ export const verifyBrokenCheckerError = ( terminal: RunEvent.RunEvent, ): Check => { const kinds = events.map((event) => event._tag) - const errorDocument: RunEvent.RunFailed | undefined = terminal._tag === 'error' ? terminal : undefined + const failure: RunEvent.RunFailed | undefined = terminal._tag === 'error' ? terminal : undefined + const record: FailureRecord.FailureRecord | undefined = failure?.record return expect({ - exitCodeIsZero: run.exitCode === 0, + exitCode: run.exitCode, lastKind: kinds.at(-1), carriesVerdict: kinds.includes('verdict'), terminalTag: terminal._tag, - errorNamesTsconfig: (errorDocument?.error ?? '').includes('non-existent-tsconfig.json'), + schemaVersion: failure?.schemaVersion, + recordTag: record?._tag, + checker: record?._tag === 'CheckerFailed' ? record.checker : undefined, + causeNamesTsconfig: (record?.cause ?? []).some((link) => link.message.includes(MISSING_TSCONFIG)), }).toStrictEqual({ - exitCodeIsZero: false, + exitCode: BROKEN_CHECKER_EXIT_CODE, lastKind: 'error', carriesVerdict: false, terminalTag: 'error', - errorNamesTsconfig: true, + schemaVersion: '3.0', + recordTag: 'CheckerFailed', + checker: 'typescript', + causeNamesTsconfig: true, }) } diff --git a/test/e2e/tests/failing-run.e2e.test.ts b/test/e2e/tests/failing-run.e2e.test.ts index b9ca9538..fe477fda 100644 --- a/test/e2e/tests/failing-run.e2e.test.ts +++ b/test/e2e/tests/failing-run.e2e.test.ts @@ -1,17 +1,38 @@ import { Gherkin, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' -import type { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, type RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import type { Check, Expect } from '@systemfsoftware/vitest' -import { Effect } from 'effect' +import { Effect, Schema } from 'effect' import type { ExecResult } from '../src/Harness/guest-job.schema.js' import { E2eHarnessLive, runStryker } from './__fixtures__/e2e-harness.fixture.js' -import { decodeStream, terminalEvent } from './__fixtures__/machine-stream.fixture.js' +import { decodeStream, MachineStreamError, terminalEvent } from './__fixtures__/machine-stream.fixture.js' -const FAILING_DRY_RUN_RUNTIME_ERROR_CODE = 3 +const BASELINE_TESTS_FAILED_EXIT_CODE = 5 +const STREAM_SCHEMA_VERSION = '3.0' const FAILING_FIXTURE_URL = new URL('../testResources/failing-fixture', import.meta.url) const FAILING_TEST_NAME = 'isEven reports three as even' +const FAILING_TEST_FILE = 'src/thing.test.ts' +const FAILING_TEST_LINE = 7 +const FAILURE_RECORD_FILE = 'reports/mutation/failure.json' +const FAILING_DRY_RUN_ARGV: ReadonlyArray = ['stryker', 'run'] +const BASELINE_TESTS_FAILED_NEXT_ACTION: FailureRecord.NextAction = { primary: 'fixCode', otherwise: 'fixTest' } + +const failureRecordOf = (terminal: RunEvent.RunEvent): FailureRecord.FailureRecord | undefined => + terminal._tag === 'error' ? terminal.record : undefined + +const failingTestOf = ( + record: FailureRecord.FailureRecord | undefined, +): FailureRecord.FailedTestEvidence | undefined => record?._tag === 'BaselineTestsFailed' ? record.tests[0] : undefined + +const decodeFailureRecord = ( + text: string, +): Effect.Effect => + Effect.mapError( + Schema.decodeUnknownEffect(FailureRecord.FailureRecordFile)(text), + (issue) => new MachineStreamError({ line: text.slice(0, 400), detail: `failure record: ${issue.message}` }), + ) const verifyFailingDryRunExit = (expect: Expect, run: ExecResult): Check => - expect(run.exitCode).toBe(FAILING_DRY_RUN_RUNTIME_ERROR_CODE) + expect(run.exitCode).toBe(BASELINE_TESTS_FAILED_EXIT_CODE) const verifyStreamCleanliness = (expect: Expect, events: ReadonlyArray): Check => expect({ @@ -24,31 +45,56 @@ const verifyTypedErrorDocument = ( terminal: RunEvent.RunEvent, events: ReadonlyArray, ): Check => { - const errorDocument: RunEvent.RunFailed | undefined = terminal._tag === 'error' ? terminal : undefined + const failure: RunEvent.RunFailed | undefined = terminal._tag === 'error' ? terminal : undefined + const record = failureRecordOf(terminal) + const capsule = record?.capsule const tags = events.map((event) => event._tag) + const failed = failingTestOf(record) return expect({ terminalTag: terminal._tag, - schemaVersion: errorDocument?.schemaVersion, - code: errorDocument?.code, - errorIsString: typeof errorDocument?.error === 'string', - remediationHasContent: /\S/.test(errorDocument?.remediation ?? ''), + schemaVersion: failure?.schemaVersion, + code: failure?.code, + recordTag: record?._tag, + stage: record?.stage, + capsuleReplays: capsule?._tag === 'Replays', + argv: capsule?._tag === 'Replays' ? capsule.argv : undefined, + nextAction: record?.nextAction, carriesVerdict: tags.includes('verdict'), + testCount: record?._tag === 'BaselineTestsFailed' ? record.testCount : undefined, + failedTestCount: record?._tag === 'BaselineTestsFailed' ? record.tests.length : undefined, + name: failed?.name, + fileNameNamesTheFailingTest: (failed?.file ?? '').endsWith(FAILING_TEST_FILE), + locationFile: failed?.location?.file, + locationLine: failed?.location?.line, + locationColumnIsPositive: (failed?.location?.column ?? 0) >= 1, + messageNamesTheFailure: /\S/.test(failed?.message ?? '') && (failed?.message ?? '').includes('expected'), }).toStrictEqual({ terminalTag: 'error', - schemaVersion: '2.0', - code: FAILING_DRY_RUN_RUNTIME_ERROR_CODE, - errorIsString: true, - remediationHasContent: true, + schemaVersion: STREAM_SCHEMA_VERSION, + code: BASELINE_TESTS_FAILED_EXIT_CODE, + recordTag: 'BaselineTestsFailed', + stage: 'dryRun', + capsuleReplays: true, + argv: FAILING_DRY_RUN_ARGV, + nextAction: BASELINE_TESTS_FAILED_NEXT_ACTION, carriesVerdict: false, + testCount: 1, + failedTestCount: 1, + name: FAILING_TEST_NAME, + fileNameNamesTheFailingTest: true, + locationFile: FAILING_TEST_FILE, + locationLine: FAILING_TEST_LINE, + locationColumnIsPositive: true, + messageNamesTheFailure: true, }) } -const verifyErrorDocumentNamesTheFailingTest = (expect: Expect, errorText: string): Check => - expect({ - namesFailingTest: errorText.includes(FAILING_TEST_NAME), - carriesFailureMessage: /\S/.test(errorText) && errorText.includes('expected'), - }).toStrictEqual({ namesFailingTest: true, carriesFailureMessage: true }) +const verifyPersistedFailureRecord = ( + expect: Expect, + terminal: RunEvent.RunEvent, + persisted: FailureRecord.FailureRecord, +): Check => expect(persisted).toStrictEqual(failureRecordOf(terminal)) const Feature = makeFeature({ it }) @@ -65,7 +111,7 @@ Feature('Failing a mutation run at the process boundary') 'run', () => runStryker({ fixture: FAILING_FIXTURE_URL, label: 'failing-fixture', args: ['run'] }), ), - Then('the process exits with the failing dry run code')((s, expect) => + Then('the process exits with the baseline tests failed code')((s, expect) => verifyFailingDryRunExit(expect, s.run.output.result) ), When('the stdout event stream decodes to run events')( @@ -74,15 +120,15 @@ Feature('Failing a mutation run at the process boundary') ), Then('every machine event is a tagged record')((s, expect) => verifyStreamCleanliness(expect, s.events)), When('the terminal event of the decoded stream is read')('terminal', (s) => terminalEvent(s.events)), - Then('the run emits a structured error document and no verdict')((s, expect) => - verifyTypedErrorDocument(expect, s.terminal, s.events) - ), - When('the failure text carried by the error document is read')( - 'errorText', - (s) => Effect.succeed(s.terminal._tag === 'error' ? s.terminal.error : ''), + Then( + 'the run emits a structured error document whose baseline failure record names the failing test with its file, line and failure message, and no verdict', + )((s, expect) => verifyTypedErrorDocument(expect, s.terminal, s.events)), + When('the failure record the run wrote to disk is read')( + 'persistedRecord', + (s) => Effect.flatMap(s.run.output.readFile(FAILURE_RECORD_FILE), decodeFailureRecord), ), - Then('the error document names the failing test with its failure message')((s, expect) => - verifyErrorDocumentNamesTheFailingTest(expect, s.errorText) + Then('the persisted failure record is the one the terminal event carries')((s, expect) => + verifyPersistedFailureRecord(expect, s.terminal, s.persistedRecord) ), ), ) From 8ec5920314752e94166de130c948ea5ecd221758 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:43 -0400 Subject: [PATCH 3/8] test(repo): declare the deciding coverage classes of two under-drawn properties (#148) mutant-set-policy and interpret-vitest-mutant-run now declare cover classes for every branch that decides their subject, and their generators reach each class constructively under the mutation-worker budget Verdict-Semantics: unchanged --- ...utant-set-policy.workflow.property.test.ts | 125 +++++++++++++++++- ...itest-mutant-run.workflow.property.test.ts | 113 ++++++++++++++-- 2 files changed, 227 insertions(+), 11 deletions(-) diff --git a/packages/stryker-js-instrumenter/src/__tests__/mutant-set-policy.workflow.property.test.ts b/packages/stryker-js-instrumenter/src/__tests__/mutant-set-policy.workflow.property.test.ts index efa5bdc1..870c1735 100644 --- a/packages/stryker-js-instrumenter/src/__tests__/mutant-set-policy.workflow.property.test.ts +++ b/packages/stryker-js-instrumenter/src/__tests__/mutant-set-policy.workflow.property.test.ts @@ -1,9 +1,11 @@ import { it } from '@systemfsoftware/vitest' +import * as Arbitrary from 'effect/Arbitrary' import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { MutantKept, + type MutantSetFacts, MutantSetFactsSchema, type MutantSetOutcome, mutantSetPolicy, @@ -11,6 +13,8 @@ import { MutantSuppressed, } from '../mutant-set-policy.workflow.js' +const holds = (conditions: readonly boolean[]): boolean => conditions.every((condition) => condition) + const outcomeOf = ( decided: Result.Result, ): MutantSetOutcome | undefined => @@ -19,9 +23,106 @@ const outcomeOf = ( onSuccess: (outcomes) => (outcomes.length === 1 ? outcomes[0] : undefined), }) +const isFacts = (value: unknown): value is MutantSetFacts => typeof value === 'object' && value !== null + +const literalCode = (code: string): boolean => + holds([code.length > 0, code.includes('(') === false, code.includes(')') === false, code.trim() === code]) + +const sameLiteralCode = (facts: MutantSetFacts): boolean => + holds([facts.replacementCode === facts.originalCode, literalCode(facts.originalCode)]) + +const sameReplacementAsEarlier = ( + candidates: readonly MutantSetFacts[], + facts: MutantSetFacts, + index: number, +): boolean => + holds([ + literalCode(facts.replacementCode), + candidates.slice(0, index).some((earlier) => earlier.replacementCode === facts.replacementCode), + ]) + +const repeatsAReplacement = (candidates: readonly MutantSetFacts[]): boolean => + candidates.some((facts, index) => sameReplacementAsEarlier(candidates, facts, index)) + +const tokenCharacterArb = Arbitrary.schema(S.Literals(['a', 'b', 'c', '0', '1', 'Z'])) + +const tokenArb = Arbitrary.array(tokenCharacterArb, { minLength: 1, maxLength: 6 }).pipe( + Arbitrary.map((characters) => characters.join('')), +) + +const plainFactsArb: Arbitrary.Arbitrary = Arbitrary.schema(MutantSetFactsSchema) + +const literalFactsArb: Arbitrary.Arbitrary = Arbitrary.all({ + code: tokenArb, + relationalSufficient: Arbitrary.schema(S.Boolean), +}).pipe( + Arbitrary.map(({ code, relationalSufficient }) => + MutantSetFactsSchema.make({ originalCode: code, replacementCode: code, relationalSufficient }) + ), +) + +const candidateFactsArb: Arbitrary.Arbitrary = Arbitrary.schema(S.Boolean).pipe( + Arbitrary.flatMap((literal) => (literal ? literalFactsArb : plainFactsArb)), +) + +const relationalInsufficientFactsArb: Arbitrary.Arbitrary = plainFactsArb.pipe( + Arbitrary.map((facts) => MutantSetFactsSchema.make({ ...facts, relationalSufficient: false })), +) + +const candidateArbs = { + plain: plainFactsArb, + literal: literalFactsArb, + relational: relationalInsufficientFactsArb, +} as const + +const candidateArb: Arbitrary.Arbitrary = Arbitrary.schema( + S.Literals(['plain', 'literal', 'relational']), +).pipe(Arbitrary.flatMap((kind) => candidateArbs[kind])) + +const candidateListArb = Arbitrary.array(candidateArb, { maxLength: 3 }) + +const duplicatedListArb = Arbitrary.all({ + head: literalFactsArb, + tail: candidateListArb, +}).pipe(Arbitrary.map(({ head, tail }) => [head, head, ...tail])) + +const candidatesArb: Arbitrary.Arbitrary = Arbitrary.schema(S.Boolean).pipe( + Arbitrary.flatMap((duplicated) => (duplicated ? duplicatedListArb : candidateListArb)), +) + +const fullPolicyCommandArb: Arbitrary.Arbitrary = candidatesArb.pipe( + Arbitrary.map((candidates) => MutantSetPolicyCommand.make({ policy: 'full', candidates })), +) + it.prop( '∀pf_PolicyAndFacts_≡TheRelationalVerdictDecidesTheSingleCandidateOutcome', - { of: [S.Literals(['default', 'full']), MutantSetFactsSchema], subject: mutantSetPolicy }, + { + of: [S.Literals(['default', 'full']), candidateFactsArb], + subject: mutantSetPolicy, + cover: { + fullPolicy: [(policy) => policy === 'full', 0.2], + relationalSuppression: [ + (policy, facts) => holds([policy === 'default', isFacts(facts) && facts.relationalSufficient === false]), + 0.1, + ], + equivalentSuppression: [ + (policy, facts) => + holds([ + policy === 'default', + isFacts(facts) && facts.relationalSufficient === true && sameLiteralCode(facts), + ]), + 0.05, + ], + keptUnderDefault: [ + (policy, facts) => + holds([ + policy === 'default', + isFacts(facts) && facts.relationalSufficient === true && sameLiteralCode(facts) === false, + ]), + 0.05, + ], + }, + }, (subject, [policy, facts]) => { const outcome = outcomeOf(subject(MutantSetPolicyCommand.make({ policy, candidates: [facts] }))) if (policy === 'full') { @@ -30,13 +131,31 @@ it.prop( if (facts.relationalSufficient === false) { return outcome !== undefined && S.is(MutantSuppressed)(outcome) && outcome.ruleId === 'redundant-relational' } - return true + if (sameLiteralCode(facts)) { + return outcome !== undefined && S.is(MutantSuppressed)(outcome) && outcome.ruleId === 'equivalent-to-original' + } + return ( + outcome !== undefined && + (S.is(MutantKept)(outcome) || (S.is(MutantSuppressed)(outcome) && outcome.ruleId === 'equivalent-to-original')) + ) }, ) it.prop( '∀c_Command_≡TheFullPolicySuppressesNothing', - { of: [MutantSetPolicyCommand], subject: mutantSetPolicy }, + { + of: [fullPolicyCommandArb], + subject: mutantSetPolicy, + cover: { + emptyCandidates: [(command) => command.candidates.length === 0, 0.05], + relationalInsufficientCandidate: [ + (command) => command.candidates.some((facts) => facts.relationalSufficient === false), + 0.2, + ], + equivalentCandidate: [(command) => command.candidates.some((facts) => sameLiteralCode(facts)), 0.2], + duplicateReplacement: [(command) => repeatsAReplacement(command.candidates), 0.15], + }, + }, (subject, [command]) => Result.match( subject(MutantSetPolicyCommand.make({ policy: 'full', candidates: [...command.candidates] })), diff --git a/packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts b/packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts index f239020b..188ce49e 100644 --- a/packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts +++ b/packages/stryker-js-vitest-runner/src/__tests__/interpret-vitest-mutant-run.workflow.property.test.ts @@ -1,6 +1,7 @@ import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import { describe } from '@systemfsoftware/vitest' +import * as Arbitrary from 'effect/Arbitrary' import * as Result from 'effect/Result' import * as S from 'effect/Schema' @@ -17,6 +18,43 @@ const TRAP_MUTANT_ID = Mutant.MutantId.make('0000000000000000') const OTHER_MUTANT_ID = Mutant.MutantId.make('0000000000000001') const TRAP_FILE = 'b.ts' +const holds = (conditions: readonly boolean[]): boolean => conditions.every((condition) => condition) + +type MutantRunCoverValue = VitestMutantRunCommand | TestRunner.TestResult | string + +const isTestResult = (value: MutantRunCoverValue): value is TestRunner.TestResult => + typeof value === 'object' && 'status' in value + +const isCommand = (value: MutantRunCoverValue): value is VitestMutantRunCommand => + typeof value === 'object' && ('status' in value) === false + +const commandOf = (value: MutantRunCoverValue): VitestMutantRunCommand | undefined => + isCommand(value) ? value : undefined + +const limitOf = (value: MutantRunCoverValue): number => commandOf(value)?.hitLimit ?? 0 + +const externalError = (value: MutantRunCoverValue): boolean => commandOf(value)?.hasExternalError === true + +const testsOf = (value: MutantRunCoverValue): readonly TestRunner.TestResult[] => commandOf(value)?.tests ?? [] + +const anyFailedTest = (tests: readonly TestRunner.TestResult[]): boolean => + tests.some((test) => test.status === 'failed') + +const anySkippedTest = (tests: readonly TestRunner.TestResult[]): boolean => + tests.some((test) => test.status === 'skipped') + +const statusOf = (value: MutantRunCoverValue): TestRunner.TestStatus | undefined => + isTestResult(value) ? value.status : undefined + +const textCharacterArb = Arbitrary.schema(S.Literals(['a', 'b', 'x', '1', '-', ' '])) + +const errorTextArb: Arbitrary.Arbitrary = Arbitrary.schema( + S.Int.check(S.isBetween({ minimum: 1, maximum: 32 })), +).pipe( + Arbitrary.flatMap((length) => Arbitrary.array(textCharacterArb, { minLength: length, maxLength: length })), + Arbitrary.map((characters) => characters.join('')), +) + const commandWith = ( input: VitestMutantRunCommand, override: { @@ -47,7 +85,14 @@ const commandWith = ( describe('interpretVitestMutantRun', (it) => { it.prop( '→h_HitLimitOnNamedTrap_=Timeout', - { of: [VitestMutantRunCommand], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand], + subject: interpretVitestMutantRun, + cover: { + hitLimitAtZero: [(input) => limitOf(input) === 0, 0.2], + hitLimitPositive: [(input) => limitOf(input) > 0, 0.05], + }, + }, (subject, [input]) => { const hitLimit = input.hitLimit ?? 0 const hitCount = hitLimit + 1 @@ -67,7 +112,14 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '→h_HitLimitOnTrapFile_=Timeout', - { of: [VitestMutantRunCommand], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand], + subject: interpretVitestMutantRun, + cover: { + hitLimitAtZero: [(input) => limitOf(input) === 0, 0.2], + hitLimitPositive: [(input) => limitOf(input) > 0, 0.05], + }, + }, (subject, [input]) => { const hitLimit = input.hitLimit ?? 0 const command = commandWith(input, { @@ -85,7 +137,14 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '→h_HitLimitOnOtherMutant_=Killed', - { of: [VitestMutantRunCommand], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand], + subject: interpretVitestMutantRun, + cover: { + hitLimitAtZero: [(input) => limitOf(input) === 0, 0.2], + hitLimitPositive: [(input) => limitOf(input) > 0, 0.05], + }, + }, (subject, [input]) => { const hitLimit = input.hitLimit ?? 0 const command = commandWith(input, { @@ -106,7 +165,18 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '→h_HitCountAtBound_≠Timeout', - { of: [VitestMutantRunCommand], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand], + subject: interpretVitestMutantRun, + cover: { + noHitKilled: [(input) => anyFailedTest(testsOf(input)), 0.2], + noHitExternalError: [(input) => holds([anyFailedTest(testsOf(input)) === false, externalError(input)]), 0.05], + noHitSurvived: [ + (input) => holds([anyFailedTest(testsOf(input)) === false, externalError(input) === false]), + 0.05, + ], + }, + }, (subject, [input]) => { const hitLimit = input.hitLimit ?? 0 const command = commandWith(input, { @@ -123,7 +193,14 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '→f_FailedTestPlusHitBound_=Killed', - { of: [VitestMutantRunCommand, TestRunner.TestResultSchema], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand, TestRunner.TestResultSchema], + subject: interpretVitestMutantRun, + cover: { + hitLimitAtZero: [(input) => limitOf(input) === 0, 0.2], + hitLimitPositive: [(input) => limitOf(input) > 0, 0.05], + }, + }, (subject, [input, test]) => { const hitLimit = input.hitLimit ?? 0 const command = commandWith(input, { @@ -144,7 +221,15 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '∀t_TestResult_≡KilledIffFailed', - { of: [VitestMutantRunCommand, TestRunner.TestResultSchema], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand, TestRunner.TestResultSchema], + subject: interpretVitestMutantRun, + cover: { + failedTest: [(_input, test) => statusOf(test) === 'failed', 0.1], + skippedTest: [(_input, test) => statusOf(test) === 'skipped', 0.1], + successTest: [(_input, test) => statusOf(test) === 'success', 0.1], + }, + }, (subject, [input, test]) => { const command = commandWith(input, { tests: [test], @@ -167,8 +252,12 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '∀c_MutantRunCommand_≡DryErrorNamesTheExternalError', { - of: [VitestMutantRunCommand, S.String.check(S.isMinLength(1), S.isMaxLength(32))], + of: [VitestMutantRunCommand, errorTextArb], subject: interpretVitestMutantRun, + cover: { + shortExternalErrorText: [(_input, text) => typeof text === 'string' && text.length <= 16, 0.2], + longExternalErrorText: [(_input, text) => typeof text === 'string' && text.length > 16, 0.1], + }, }, (subject, [input, externalErrorText]) => { const command = commandWith(input, { @@ -189,7 +278,15 @@ describe('interpretVitestMutantRun', (it) => { it.prop( '∀c_MutantRunCommand_≡ScorableOutcomesReportEveryExecutedTest', - { of: [VitestMutantRunCommand], subject: interpretVitestMutantRun }, + { + of: [VitestMutantRunCommand], + subject: interpretVitestMutantRun, + cover: { + anyFailedTest: [(input) => anyFailedTest(testsOf(input)), 0.2], + noFailedTest: [(input) => anyFailedTest(testsOf(input)) === false, 0.05], + anySkippedTest: [(input) => anySkippedTest(testsOf(input)), 0.2], + }, + }, (subject, [input]) => { const command = commandWith(input, { tests: input.tests, From 3d5586185bde60ba6fa5f94b793a3c0f62fe14a5 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:44 -0400 Subject: [PATCH 4/8] feat(repo): give every failure record a one-step reproduction (#149) * feat(repo): give every failure record a one-step reproduction The Vitest runner reports a reproduce argv per failed test, and a pure workflow decides each record's capsule: the runner command under the mutation-worker budget, the dry-run fallback, the --mutant rerun for new survivors, or DoesNotReplay with its stand-in Verdict-Semantics: unchanged * test(repo): expect the runner's vitest replay in the failing-run capsule The capsule layer replays a failed baseline test through the runner's vitest run -t argv; the e2e journey still expected the stryker run fallback Verdict-Semantics: unchanged --- .changeset/busy-mammals-clean.md | 5 + .changeset/fine-pandas-sit.md | 5 + .changeset/quiet-clowns-remain.md | 5 + .changeset/sixty-wombats-write.md | 5 + .../contract/stream.schema.json | 44 +++++- .../src/failure-catalog.ts | 10 +- .../src/failure-record.schema.ts | 1 + .../src/TestRunner.schema.ts | 2 + .../src/vitest-test-run.ts | 54 ++++++- ...sify-run-outcome.workflow.property.test.ts | 13 -- ...-run-observation.workflow.property.test.ts | 2 + ...oduction-capsule.workflow.property.test.ts | 104 ++++++++++++++ packages/stryker-js/src/conclude-run.ts | 26 +++- .../interpret-dry-run-observation.workflow.ts | 1 + .../src/reproduction-capsule.workflow.ts | 135 ++++++++++++++++++ .../framework-stream.integration.test.ts | 9 +- .../stryker-js/tests/gate.integration.test.ts | 12 +- .../run-event-stream.integration.test.ts | 9 +- test/e2e/tests/failing-run.e2e.test.ts | 4 +- 19 files changed, 415 insertions(+), 31 deletions(-) create mode 100644 .changeset/busy-mammals-clean.md create mode 100644 .changeset/fine-pandas-sit.md create mode 100644 .changeset/quiet-clowns-remain.md create mode 100644 .changeset/sixty-wombats-write.md create mode 100644 packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts create mode 100644 packages/stryker-js/src/reproduction-capsule.workflow.ts diff --git a/.changeset/busy-mammals-clean.md b/.changeset/busy-mammals-clean.md new file mode 100644 index 00000000..be813fbc --- /dev/null +++ b/.changeset/busy-mammals-clean.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-vitest-runner": minor +--- + +Every failed test reports a reproduce argv: vitest run -t , or vitest run for a file-level failure. diff --git a/.changeset/fine-pandas-sit.md b/.changeset/fine-pandas-sit.md new file mode 100644 index 00000000..5a245545 --- /dev/null +++ b/.changeset/fine-pandas-sit.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +Each failure record carries a one-step reproduction: the failing test's own runner command under the mutation-worker budget, stryker run --dryRunOnly when the runner gives none, stryker run --mutant for new survivors, or DoesNotReplay naming why and what stands in (worker kind, pid and exit code for crashed workers). diff --git a/.changeset/quiet-clowns-remain.md b/.changeset/quiet-clowns-remain.md new file mode 100644 index 00000000..88657706 --- /dev/null +++ b/.changeset/quiet-clowns-remain.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-plugin-interface": minor +--- + +FailedTestResult gains an optional reproduce argv: the command that re-runs exactly that failing test. diff --git a/.changeset/sixty-wombats-write.md b/.changeset/sixty-wombats-write.md new file mode 100644 index 00000000..95bd76ea --- /dev/null +++ b/.changeset/sixty-wombats-write.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js-cli-contract": minor +--- + +FailedTestEvidence carries the runner's reproduce argv, and recordOf takes the decided capsule from its RecordContext instead of deriving it. diff --git a/packages/stryker-js-cli-contract/contract/stream.schema.json b/packages/stryker-js-cli-contract/contract/stream.schema.json index 204b07a3..8f346e09 100644 --- a/packages/stryker-js-cli-contract/contract/stream.schema.json +++ b/packages/stryker-js-cli-contract/contract/stream.schema.json @@ -3701,6 +3701,25 @@ "type": "null" } ] + }, + "reproduce": { + "anyOf": [ + { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + { + "type": "null" + } + ] } }, "required": [ @@ -3709,7 +3728,8 @@ "file", "location", "message", - "stack" + "stack", + "reproduce" ], "additionalProperties": true } @@ -3775,6 +3795,25 @@ "type": "null" } ] + }, + "reproduce": { + "anyOf": [ + { + "type": "array", + "minItems": 1, + "items": [ + { + "type": "string" + } + ], + "additionalItems": { + "type": "string" + } + }, + { + "type": "null" + } + ] } }, "required": [ @@ -3783,7 +3822,8 @@ "file", "location", "message", - "stack" + "stack", + "reproduce" ], "additionalProperties": true } diff --git a/packages/stryker-js-cli-contract/src/failure-catalog.ts b/packages/stryker-js-cli-contract/src/failure-catalog.ts index a92ebe77..8e5acba7 100644 --- a/packages/stryker-js-cli-contract/src/failure-catalog.ts +++ b/packages/stryker-js-cli-contract/src/failure-catalog.ts @@ -1,6 +1,6 @@ import { dual } from 'effect/Function' -import type { CapsuleRule, CatalogEntry } from './failure-catalog.schema.js' +import type { CatalogEntry } from './failure-catalog.schema.js' import { type Capsule, type CauseLink, @@ -183,21 +183,17 @@ export interface RecordContext { readonly cwd: string readonly argv: readonly [string, ...Array] readonly env: ReadonlyArray + readonly capsule: Capsule readonly traceId: TraceId | null } -const capsuleOf = (rule: CapsuleRule, context: RecordContext): Capsule => - rule === 'replays' - ? { _tag: 'Replays', cwd: context.cwd, argv: context.argv, env: context.env } - : { _tag: 'DoesNotReplay', why: rule, standIn: context.argv.join(' ') } - export const recordOf: { (context: RecordContext): (evidence: FailureEvidence) => FailureRecord (evidence: FailureEvidence, context: RecordContext): FailureRecord } = dual(2, (evidence: FailureEvidence, context: RecordContext): FailureRecord => ({ ...evidence, cause: context.cause, - capsule: capsuleOf(FailureCatalog[evidence._tag].capsule, context), + capsule: context.capsule, nextAction: FailureCatalog[evidence._tag].nextAction, traceId: context.traceId, })) diff --git a/packages/stryker-js-cli-contract/src/failure-record.schema.ts b/packages/stryker-js-cli-contract/src/failure-record.schema.ts index 12997e51..bfa40bdf 100644 --- a/packages/stryker-js-cli-contract/src/failure-record.schema.ts +++ b/packages/stryker-js-cli-contract/src/failure-record.schema.ts @@ -73,6 +73,7 @@ export const FailedTestEvidence = S.Struct({ location: S.NullOr(SourceLocation), message: S.String, stack: S.NullOr(S.String), + reproduce: S.String.pipe(S.NonEmptyArray, S.NullOr), }) export type FailedTestEvidence = typeof FailedTestEvidence.Type diff --git a/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts b/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts index 1e313c93..211b226f 100644 --- a/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts +++ b/packages/stryker-js-plugin-interface/src/TestRunner.schema.ts @@ -40,6 +40,7 @@ export const TestResultSchema = S.Union([ failureMessage: S.String, location: S.optionalKey(TestFailureLocation), stack: S.optionalKey(S.String), + reproduce: S.String.pipe(S.NonEmptyArray, S.optionalKey), }), S.Struct({ ...TestResultBase, status: S.Literal('skipped') }), S.Struct({ ...TestResultBase, status: S.Literal('success') }), @@ -114,6 +115,7 @@ export interface FailedTestResult extends BaseTestResult { readonly failureMessage: string readonly location?: TestFailureLocation readonly stack?: string + readonly reproduce?: readonly [string, ...string[]] } export interface SkippedTestResult extends BaseTestResult { diff --git a/packages/stryker-js-vitest-runner/src/vitest-test-run.ts b/packages/stryker-js-vitest-runner/src/vitest-test-run.ts index 92a7c68f..6070ddc4 100644 --- a/packages/stryker-js-vitest-runner/src/vitest-test-run.ts +++ b/packages/stryker-js-vitest-runner/src/vitest-test-run.ts @@ -39,6 +39,15 @@ const projectRelativeId = (id: string, projectRoot: string): string => { }) } +const projectRelativeFileOf = (record: VitestTestRecord, projectRoot: string): string => + projectRelativePath(stripProjectRoot(fileNameOf(record), projectRoot)) + +const reproduceArgvOf = (testName: string, file: string): readonly [string, ...string[]] => + Boolean.match(testName === file, { + onTrue: (): readonly [string, ...string[]] => ['vitest', 'run', file], + onFalse: (): readonly [string, ...string[]] => ['vitest', 'run', file, '-t', testName], + }) + const testNameOf = (record: VitestTestRecord): string => Option.getOrElse( Option.fromNullishOr(record.fullTestName), @@ -121,9 +130,11 @@ const failureEvidenceOfRecord = ( failureEvidenceOf(locationOf(record.errorFrames, projectRoot), record.errorStack) const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEncoded => { + const name = testNameOf(record) + const reproduce = reproduceArgvOf(name, projectRelativeFileOf(record, projectRoot)) const base = { - id: projectRelativeId(`${fileNameOf(record)}#${testNameOf(record)}`, projectRoot), - name: testNameOf(record), + id: projectRelativeId(`${fileNameOf(record)}#${name}`, projectRoot), + name, timeSpentMs: timeSpentOf(record), ...fileNameFieldOf(record), } @@ -134,6 +145,7 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco ...base, status: 'failed', failureMessage: failureMessageOf(record), + reproduce, ...failureEvidenceOfRecord(record, projectRoot), }), ), @@ -146,6 +158,7 @@ const resultOf = (record: VitestTestRecord, projectRoot: string): TestResultEnco ...base, status: 'failed', failureMessage, + reproduce, ...failureEvidenceOfRecord(record, projectRoot), }), }), @@ -163,6 +176,7 @@ const fileFailureResultOf = (failure: VitestFileFailure, projectRoot: string): T status: 'failed', failureMessage: failure.message, fileName: failure.fileName, + reproduce: reproduceArgvOf(file, file), ...failureEvidenceOf(locationOf(failure.frames, projectRoot), failure.stack), } } @@ -194,6 +208,12 @@ if (import.meta.vitest !== void 0) { ): Option.Option> => Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.location)) + const reproduceOf = (only: TestResultEncoded): Option.Option => + Option.flatMap(Option.liftPredicate(only, isFailedResult), (failed) => Option.fromNullishOr(failed.reproduce)) + + const sameArgv = (actual: readonly string[], expected: readonly string[]): boolean => + actual.length === expected.length && expected.every((value, index) => actual[index] === value) + const positionsEqual = ( location: NonNullable, line: number, @@ -286,6 +306,19 @@ if (import.meta.vitest !== void 0) { }, ) + it.prop( + '∀f_FileFailure_≡ReproduceRunsTheFile', + { of: [FileStem], subject: interpretVitestTestRun }, + (subject, [stem]) => { + const [only] = subject({ + projectRoot: '/project', + records: [], + fileFailures: [{ fileName: `/project/tests/${stem}.spec.ts`, message: 'load failed' }], + }) + return Option.exists(reproduceOf(only), (argv) => sameArgv(argv, ['vitest', 'run', `tests/${stem}.spec.ts`])) + }, + ) + const SourceOrdinal = S.Int.check(S.isGreaterThanOrEqualTo(1)) const failingRecord = (overrides: Partial): VitestTestRecord => ({ @@ -294,6 +327,23 @@ if (import.meta.vitest !== void 0) { ...overrides, }) + const TestName = S.String.check(S.isPattern(/^[a-z][a-z ]{0,11}$/)) + + it.prop( + '∀r_FailedTest_≡ReproduceNamesFileAndTest', + { of: [TestName, FileStem], subject: interpretVitestTestRun }, + (subject, [name, stem]) => { + const file = `tests/${stem}.spec.ts` + const record = failingRecord({ name, fullTestName: name, fileName: `/project/${file}` }) + const [only] = subject(singleRecordRun(record, '/project')) + const expected = Boolean.match(name === file, { + onTrue: (): readonly string[] => ['vitest', 'run', file], + onFalse: (): readonly string[] => ['vitest', 'run', file, '-t', name], + }) + return Option.exists(reproduceOf(only), (argv) => sameArgv(argv, expected)) + }, + ) + it.prop( '∀r_ProjectFrame_≡LocatedThere', { of: [SourceOrdinal, SourceOrdinal], subject: interpretVitestTestRun }, diff --git a/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts b/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts index 70fafdc8..23e67a88 100644 --- a/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/classify-run-outcome.workflow.property.test.ts @@ -79,19 +79,6 @@ describe('classifyRunOutcome', () => { }, ) - it.prop( - '∀evidence_RunOutcome_≡ReplayableRecordsReplayTheSameCommandAndOthersSayWhyNot', - { of: [FailureRecord.FailureEvidence], subject: outcomeOf }, - (subject, [evidence]) => { - const capsule = failedOf(subject(Cause.fail({ evidence })))?.record.capsule - const rule = FailureRecord.FailureCatalog[evidence._tag].capsule - return rule === 'replays' - ? Predicate.isTagged(capsule, 'Replays') && capsule.cwd === CWD && - capsule.argv.join(' ') === ['stryker', ...ARGS].join(' ') - : Predicate.isTagged(capsule, 'DoesNotReplay') && capsule.why === rule - }, - ) - it.prop( '∀fiber_RunOutcome_≡AnInterruptOnlyRunExits130WithARunInterruptedRecord', { of: [S.Int.check(S.isGreaterThanOrEqualTo(0))], subject: outcomeOf }, diff --git a/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts b/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts index 504273fd..c1a13a64 100644 --- a/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/interpret-dry-run-observation.workflow.property.test.ts @@ -1,6 +1,7 @@ import { TestRunner } from '@systemfsoftware/stryker-js-plugin-interface' import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' +import * as Equal from 'effect/Equal' import * as Result from 'effect/Result' import * as S from 'effect/Schema' @@ -48,6 +49,7 @@ describe('interpretDryRunObservation', () => { evidence.location.line === location.line && evidence.location.column === location.column) && evidence.message === expected.failureMessage && + Equal.equals(evidence.reproduce ?? null, expected.reproduce ?? null) && evidence.stack === (expected.stack ?? null) }) }, diff --git a/packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts b/packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts new file mode 100644 index 00000000..899129ce --- /dev/null +++ b/packages/stryker-js/src/__tests__/reproduction-capsule.workflow.property.test.ts @@ -0,0 +1,104 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { describe, it } from '@systemfsoftware/vitest' +import * as Arr from 'effect/Array' +import * as Predicate from 'effect/Predicate' +import * as Result from 'effect/Result' + +import { + type CapsuleDoesNotReplay, + type CapsuleReplays, + reproductionCapsule, + ReproductionCapsuleCommand, + type ReproductionCapsuleDecision, +} from '../reproduction-capsule.workflow.js' + +const CWD = '/project' +const RUN_ARGV = ['stryker', 'run'] as const +const DRY_RUN_FLAG = '--dryRunOnly' +const MARKER: FailureRecord.EnvEntry = { name: 'STRYKER_WORKER_DIR', value: '.stryker-tmp' } + +const capsuleOf = (evidence: FailureRecord.FailureEvidence): ReproductionCapsuleDecision => + Result.getOrElse( + reproductionCapsule( + ReproductionCapsuleCommand.make({ evidence, argv: RUN_ARGV, cwd: CWD, envMarker: MARKER }), + ), + (impossible) => impossible, + ) + +const replaysOf = (decision: ReproductionCapsuleDecision): CapsuleReplays | undefined => + Predicate.isTagged(decision, 'Replays') ? decision : undefined + +const refusalOf = (decision: ReproductionCapsuleDecision): CapsuleDoesNotReplay | undefined => + Predicate.isTagged(decision, 'DoesNotReplay') ? decision : undefined + +const baselineEvidenceOf = (test: FailureRecord.FailedTestEvidence): FailureRecord.FailureEvidence => ({ + _tag: 'BaselineTestsFailed', + stage: 'dryRun', + testCount: 1, + tests: [test], +}) + +const mutantArgvOf = (evidence: FailureRecord.FailureEvidence): string => + Predicate.isTagged(evidence, 'NewSurvivors') + ? ['stryker', 'run', '--mutant', Arr.join(Arr.map(evidence.survivors, (each) => each.mutantId), ',')].join(' ') + : '' + +describe('reproductionCapsule', () => { + it.prop( + '∀evidence_ReproductionCapsule_≡ReplaysExactlyWhenItsCatalogCodeReplays', + { of: [FailureRecord.FailureEvidence], subject: capsuleOf }, + (subject, [evidence]) => { + const decision = subject(evidence) + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + return rule === 'replays' + ? replaysOf(decision)?.cwd === CWD + : refusalOf(decision)?.why === rule + }, + ) + + it.prop( + '∀evidence_ReproductionCapsule_≡NewSurvivorFailuresReplayTheNamedMutantIds', + { of: [FailureRecord.FailureEvidence], subject: capsuleOf }, + (subject, [evidence]) => { + const decision = subject(evidence) + if (!Predicate.isTagged(evidence, 'NewSurvivors')) { + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + return rule === 'replays' ? replaysOf(decision) !== undefined : refusalOf(decision) !== undefined + } + return replaysOf(decision)?.argv.join(' ') === mutantArgvOf(evidence) + }, + ) + + it.prop( + '∀failedTest_ReproductionCapsule_≡BaselineFailuresReplayTheRunnerArgvOrFallBackToTheDryRun', + { of: [FailureRecord.FailedTestEvidence], subject: capsuleOf }, + (subject, [failedTest]) => { + const replays = replaysOf(subject(baselineEvidenceOf(failedTest))) + const expectedArgv = failedTest.reproduce ?? [...RUN_ARGV, DRY_RUN_FLAG] + return replays !== undefined && replays.cwd === CWD && + replays.argv.join(' ') === expectedArgv.join(' ') && + Arr.every(replays.env, (entry) => entry.name === MARKER.name && entry.value === MARKER.value) + }, + ) + + it.prop( + '∀evidence_ReproductionCapsule_≡NonReplayableRecordsStandInTheirOwnEvidence', + { of: [FailureRecord.FailureEvidence], subject: capsuleOf }, + (subject, [evidence]) => { + const decision = subject(evidence) + const rule = FailureRecord.FailureCatalog[evidence._tag].capsule + if (rule === 'replays') return replaysOf(decision) !== undefined + const refusal = refusalOf(decision) + if (refusal === undefined) return false + if (Predicate.isTagged(evidence, 'WorkerOutOfMemory')) { + return refusal.standIn === `${evidence.workerKind} pid ${evidence.pid} exited ${evidence.exitCode}` + } + if (Predicate.isTagged(evidence, 'WorkerCrashed')) { + return refusal.standIn.includes(`${evidence.workerKind} pid ${evidence.pid}`) + } + if (Predicate.isTagged(evidence, 'JobTimedOut')) return refusal.standIn.includes(`${evidence.limitSeconds}`) + if (Predicate.isTagged(evidence, 'BinaryMissing')) return refusal.standIn.includes(evidence.binary) + return true + }, + ) +}) diff --git a/packages/stryker-js/src/conclude-run.ts b/packages/stryker-js/src/conclude-run.ts index b4fcddbb..a76b6394 100644 --- a/packages/stryker-js/src/conclude-run.ts +++ b/packages/stryker-js/src/conclude-run.ts @@ -1,5 +1,5 @@ import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' -import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import { Options, Plugin } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Cause from 'effect/Cause' import * as CliError from 'effect/cli/CliError' @@ -9,8 +9,10 @@ import { pipe } from 'effect/Function' import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Predicate from 'effect/Predicate' +import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { reproductionCapsule, ReproductionCapsuleCommand } from './reproduction-capsule.workflow.js' import { type ObservedFailure, RunFailedObservation, @@ -24,6 +26,11 @@ import { const CLI_BIN = 'stryker' const MAX_CAUSE_DEPTH = 10 +const WORKER_ENV_MARKER: FailureRecord.EnvEntry = { + name: 'STRYKER_WORKER_DIR', + value: Options.StrykerTempDirName.literal, +} + const asEvidence = Option.liftPredicate(S.is(FailureRecord.FailureEvidence)) const asExitClass = Option.liftPredicate(S.is(Plugin.ExitClass)) @@ -132,6 +139,22 @@ interface RunContext { readonly traceId: FailureRecord.TraceId | null } +const decidedCapsuleOf = ( + evidence: FailureRecord.FailureEvidence, + context: RunContext, +): FailureRecord.Capsule => + Result.getOrElse( + reproductionCapsule( + ReproductionCapsuleCommand.make({ + evidence, + argv: [CLI_BIN, ...context.argv], + cwd: context.cwd, + envMarker: WORKER_ENV_MARKER, + }), + ), + (impossible: never) => impossible, + ) + const observedOf = ( evidence: FailureRecord.FailureEvidence, cause: ReadonlyArray, @@ -142,6 +165,7 @@ const observedOf = ( cwd: context.cwd, argv: [CLI_BIN, ...context.argv], env: [], + capsule: decidedCapsuleOf(evidence, context), traceId: context.traceId, }), exitCode: FailureRecord.FailureCatalog[evidence._tag].exitCode ?? CI_ONLY_EXIT_CODE, diff --git a/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts b/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts index fe7b742e..a862103e 100644 --- a/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts +++ b/packages/stryker-js/src/interpret-dry-run-observation.workflow.ts @@ -49,6 +49,7 @@ const failedTestEvidenceOf = (test: TestRunner.FailedTestResult): FailureRecord. location: orNullOf(test.location), message: test.failureMessage, stack: orNullOf(test.stack), + reproduce: orNullOf(test.reproduce), }) const failedTestEvidencesOf = ( diff --git a/packages/stryker-js/src/reproduction-capsule.workflow.ts b/packages/stryker-js/src/reproduction-capsule.workflow.ts new file mode 100644 index 00000000..163cbedf --- /dev/null +++ b/packages/stryker-js/src/reproduction-capsule.workflow.ts @@ -0,0 +1,135 @@ +import { Workflow } from '@systemfsoftware/effect-cell-types' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import * as Arr from 'effect/Array' +import * as Boolean from 'effect/Boolean' +import * as Match from 'effect/Match' +import * as Option from 'effect/Option' +import * as Result from 'effect/Result' +import * as S from 'effect/Schema' + +const CLI_BIN = 'stryker' +const DRY_RUN_FLAG = '--dryRunOnly' +const MUTANT_FLAG = '--mutant' + +const ReproductionCapsuleTypeId: unique symbol = Symbol.for( + '@systemfsoftware/stryker-js/ReproductionCapsule', +) +type ReproductionCapsuleTypeId = typeof ReproductionCapsuleTypeId + +export class CapsuleReplays extends S.TaggedClass()('Replays', { + cwd: S.NonEmptyString, + argv: S.NonEmptyArray(S.String), + env: S.Array(FailureRecord.EnvEntry), +}) { + readonly [ReproductionCapsuleTypeId] = ReproductionCapsuleTypeId +} + +export class CapsuleDoesNotReplay extends S.TaggedClass()('DoesNotReplay', { + why: FailureRecord.NonReplayReason, + standIn: S.NonEmptyString, +}) { + readonly [ReproductionCapsuleTypeId] = ReproductionCapsuleTypeId +} + +export type ReproductionCapsuleDecision = CapsuleReplays | CapsuleDoesNotReplay + +export class ReproductionCapsuleCommand extends S.TaggedClass()( + 'ReproductionCapsuleCommand', + { + evidence: FailureRecord.FailureEvidence, + argv: S.NonEmptyArray(S.String), + cwd: S.NonEmptyString, + envMarker: FailureRecord.EnvEntry, + }, +) { + static readonly [Workflow.InstrumentationBrand] = {} as const +} + +const replays = ( + argv: readonly [string, ...Array], + command: ReproductionCapsuleCommand, +): CapsuleReplays => CapsuleReplays.make({ cwd: command.cwd, argv, env: [command.envMarker] }) + +const doesNotReplay = ( + why: FailureRecord.NonReplayReason, + standIn: string, +): CapsuleDoesNotReplay => CapsuleDoesNotReplay.make({ why, standIn }) + +const dryRunArgv = (argv: readonly [string, ...Array]): readonly [string, ...Array] => + Boolean.match(Arr.contains(argv, DRY_RUN_FLAG), { + onTrue: () => argv, + onFalse: () => Arr.append(argv, DRY_RUN_FLAG), + }) + +const crashedStandIn = ( + workerKind: FailureRecord.WorkerKind, + pid: number, + exitCode: number | null, + signal: string | null, +): string => + Option.match(Option.fromNullishOr(exitCode), { + onSome: (code) => `${workerKind} pid ${pid} exited ${code}`, + onNone: () => + Option.match(Option.fromNullishOr(signal), { + onSome: (name) => `${workerKind} pid ${pid} was killed by ${name}`, + onNone: () => `${workerKind} pid ${pid} exited without a code`, + }), + }) + +const missingStandIn = (exitCode: number | null): string => + Option.match(Option.fromNullishOr(exitCode), { + onSome: (code) => `the job wrote neither a failure record nor a report (exit code ${code})`, + onNone: () => 'the job wrote neither a failure record nor a report', + }) + +const ownReplays = (command: ReproductionCapsuleCommand): CapsuleReplays => replays(command.argv, command) + +const capsuleFor = (command: ReproductionCapsuleCommand): ReproductionCapsuleDecision => + Match.value(command.evidence).pipe( + Match.tagsExhaustive({ + WorkerOutOfMemory: ({ workerKind, pid, exitCode }) => + doesNotReplay('outOfMemory', `${workerKind} pid ${pid} exited ${exitCode}`), + WorkerCrashed: ({ workerKind, pid, exitCode, signal }) => + doesNotReplay('workerCrashed', crashedStandIn(workerKind, pid, exitCode, signal)), + RunInterrupted: () => doesNotReplay('interrupted', 'the run was interrupted; re-run it with the same command'), + RecordMissing: ({ exitCode }) => doesNotReplay('recordMissing', missingStandIn(exitCode)), + JobTimedOut: ({ limitSeconds }) => + doesNotReplay('jobTimedOut', `the job exceeded its ${limitSeconds}s time limit`), + BinaryMissing: ({ binary }) => doesNotReplay('binaryMissing', `the required binary "${binary}" was not found`), + BaselineTestsFailed: ({ tests }) => + Option.match(Option.fromNullishOr(Arr.headNonEmpty(tests).reproduce), { + onSome: (reproduce) => replays(reproduce, command), + onNone: () => replays(dryRunArgv(command.argv), command), + }), + NewSurvivors: ({ survivors }) => + replays([CLI_BIN, 'run', MUTANT_FLAG, Arr.join(Arr.map(survivors, (each) => each.mutantId), ',')], command), + BaselineTimedOut: () => replays(dryRunArgv(command.argv), command), + BaselineErrored: () => replays(dryRunArgv(command.argv), command), + BaselineFoundNoTests: () => replays(dryRunArgv(command.argv), command), + ArgumentsInvalid: () => ownReplays(command), + ConfigInvalid: () => ownReplays(command), + PluginNotFound: () => ownReplays(command), + PluginLoadFailed: () => ownReplays(command), + PluginImportFailed: () => ownReplays(command), + SurvivorsUnavailable: () => ownReplays(command), + NoInputFiles: () => ownReplays(command), + SandboxPreparationFailed: () => ownReplays(command), + InstrumentationFailed: () => ownReplays(command), + CheckerFailed: () => ownReplays(command), + TestRunnerFailed: () => ownReplays(command), + WorkerBootTimedOut: () => ownReplays(command), + ReporterFailed: () => ownReplays(command), + InvariantBroken: () => ownReplays(command), + CatalogGap: () => ownReplays(command), + }), + ) + +const decide = (command: ReproductionCapsuleCommand): Result.Result => + Result.succeed(capsuleFor(command)) + +export const reproductionCapsule = Workflow.make({ + command: ReproductionCapsuleCommand, + decision: S.Union([CapsuleReplays, CapsuleDoesNotReplay]), + error: S.Never, + decide, +}) diff --git a/packages/stryker-js/tests/framework-stream.integration.test.ts b/packages/stryker-js/tests/framework-stream.integration.test.ts index b8ce43b8..ac3f5b3c 100644 --- a/packages/stryker-js/tests/framework-stream.integration.test.ts +++ b/packages/stryker-js/tests/framework-stream.integration.test.ts @@ -53,7 +53,14 @@ const refusal = RunEvent.RunFailed.make({ descriptor: frameworkModule, reason: { _tag: 'PeerMissing', peer: frameworkModule }, }, - { cause: [], cwd: '/project', argv: ['stryker', 'run'], env: [], traceId: null }, + { + cause: [], + cwd: '/project', + argv: ['stryker', 'run'], + env: [], + capsule: { _tag: 'Replays', cwd: '/project', argv: ['stryker', 'run'], env: [] }, + traceId: null, + }, ), }) diff --git a/packages/stryker-js/tests/gate.integration.test.ts b/packages/stryker-js/tests/gate.integration.test.ts index 20f3f1ba..4b51f261 100644 --- a/packages/stryker-js/tests/gate.integration.test.ts +++ b/packages/stryker-js/tests/gate.integration.test.ts @@ -165,12 +165,20 @@ Feature('Gating a pull request on survivors the committed baseline has never see 'ran', (s) => runGate(s.project.root, 'human', ['--baseline', COMMITTED_BASELINE]), ), - Then('the process exits 1 and stderr names only the new survivor once')((s, expect) => + Then('the process exits 1, replays the new survivor, and never suggests --update-baseline')((s, expect) => expect({ exitCode: s.ran.exitCode, namesOnlyTheNewSurvivor: namesOnlyTheNewSurvivor(s.ran.stderr), newIdAppearances: s.ran.stderr.split(NEW_SURVIVOR).length - 1, - }).toStrictEqual({ exitCode: 1, namesOnlyTheNewSurvivor: true, newIdAppearances: 1 }) + replaysTheSurvivor: s.ran.stderr.includes(`stryker run --mutant ${NEW_SURVIVOR}`), + neverSuggestsUpdateBaseline: !s.ran.stderr.includes('--update-baseline'), + }).toStrictEqual({ + exitCode: 1, + namesOnlyTheNewSurvivor: true, + newIdAppearances: 2, + replaysTheSurvivor: true, + neverSuggestsUpdateBaseline: true, + }) ), ), ) diff --git a/packages/stryker-js/tests/run-event-stream.integration.test.ts b/packages/stryker-js/tests/run-event-stream.integration.test.ts index 0ae5b3e6..744a58f5 100644 --- a/packages/stryker-js/tests/run-event-stream.integration.test.ts +++ b/packages/stryker-js/tests/run-event-stream.integration.test.ts @@ -22,7 +22,14 @@ const HEARTBEAT = CliContract.Heartbeat.make({ elapsedMs: 2, completed: 1, total const HELP_RENDERED = CliContract.HelpRendered.make({ schemaVersion: '3.0', code: 0, help: 'usage' }) const RUN_FAILED_RECORD = FailureRecord.recordOf( { _tag: 'TestRunnerFailed', stage: 'mutationTest' }, - { cause: [], cwd: '/base', argv: ['stryker', 'run'], env: [], traceId: null }, + { + cause: [], + cwd: '/base', + argv: ['stryker', 'run'], + env: [], + capsule: { _tag: 'Replays', cwd: '/base', argv: ['stryker', 'run'], env: [] }, + traceId: null, + }, ) const RUN_FAILED = CliContract.RunFailed.make({ schemaVersion: '3.0', diff --git a/test/e2e/tests/failing-run.e2e.test.ts b/test/e2e/tests/failing-run.e2e.test.ts index fe477fda..14795842 100644 --- a/test/e2e/tests/failing-run.e2e.test.ts +++ b/test/e2e/tests/failing-run.e2e.test.ts @@ -13,7 +13,7 @@ const FAILING_TEST_NAME = 'isEven reports three as even' const FAILING_TEST_FILE = 'src/thing.test.ts' const FAILING_TEST_LINE = 7 const FAILURE_RECORD_FILE = 'reports/mutation/failure.json' -const FAILING_DRY_RUN_ARGV: ReadonlyArray = ['stryker', 'run'] +const FAILING_TEST_REPLAY_ARGV: ReadonlyArray = ['vitest', 'run', FAILING_TEST_FILE, '-t', FAILING_TEST_NAME] const BASELINE_TESTS_FAILED_NEXT_ACTION: FailureRecord.NextAction = { primary: 'fixCode', otherwise: 'fixTest' } const failureRecordOf = (terminal: RunEvent.RunEvent): FailureRecord.FailureRecord | undefined => @@ -76,7 +76,7 @@ const verifyTypedErrorDocument = ( recordTag: 'BaselineTestsFailed', stage: 'dryRun', capsuleReplays: true, - argv: FAILING_DRY_RUN_ARGV, + argv: FAILING_TEST_REPLAY_ARGV, nextAction: BASELINE_TESTS_FAILED_NEXT_ACTION, carriesVerdict: false, testCount: 1, From 15dde0b8d3f142f762c7d5bdc351c063bf0bf13f Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:44 -0400 Subject: [PATCH 5/8] feat(repo): carry the failure record to MCP, MSP, SARIF and traces (#150) MSP answers engine failures with the record in error.data and keeps serving; MCP refuses reruns with the record and gains get_failure; a failed run with the sarif reporter writes a failed invocation per record; trace tests pin stryker.failure.code and the record's traceId; the run log drops the orphan possible-causes block and the bundled WASI warning Verdict-Semantics: unchanged --- .changeset/humble-coats-teach.md | 5 + packages/stryker-js/etc/stryker-js.api.md | 2 +- .../stryker-js/src/Mcp/mcp-server.cell.ts | 81 +++++- .../stryker-js/src/Mcp/mcp-tools.schema.ts | 10 +- packages/stryker-js/src/Mcp/mcp-tools.ts | 11 +- packages/stryker-js/src/Mcp/mod.ts | 11 +- packages/stryker-js/src/Rerun/Rerun.cell.ts | 3 +- packages/stryker-js/src/Rerun/mod.ts | 1 + packages/stryker-js/src/Rerun/rerun.schema.ts | 8 + packages/stryker-js/src/Serve/Serve.cell.ts | 93 +++++-- .../sarif-report.workflow.property.test.ts | 111 +++++++- ...ptions-admission.workflow.property.test.ts | 49 +++- packages/stryker-js/src/bin/main.ts | 25 ++ packages/stryker-js/src/conclude-run.cell.ts | 55 +++- packages/stryker-js/src/reporter-factories.ts | 26 +- .../src/run/RunEnvironment.service.ts | 17 +- packages/stryker-js/src/run/prepare.cell.ts | 11 + .../validate-options-admission.workflow.ts | 10 +- .../stryker-js/src/sarif-failure.schema.ts | 53 ++++ .../stryker-js/src/sarif-report.workflow.ts | 112 ++++++-- .../__fixtures__/mutation-server.schema.ts | 2 +- .../tests/__fixtures__/otlp-spans.schema.ts | 17 ++ .../__fixtures__/sarif-reporter.schema.ts | 24 ++ .../tests/mcp-server.integration.test.ts | 197 +++++++++++++- .../tests/mutation-server.integration.test.ts | 81 +++++- .../tests/sarif-reporter.integration.test.ts | 91 ++++++- .../trace-propagation.integration.test.ts | 242 +++++++++++++++++- 27 files changed, 1231 insertions(+), 117 deletions(-) create mode 100644 .changeset/humble-coats-teach.md create mode 100644 packages/stryker-js/src/Rerun/rerun.schema.ts create mode 100644 packages/stryker-js/src/sarif-failure.schema.ts create mode 100644 packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts diff --git a/.changeset/humble-coats-teach.md b/.changeset/humble-coats-teach.md new file mode 100644 index 00000000..7881d77c --- /dev/null +++ b/.changeset/humble-coats-teach.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +A failed run's record reaches every surface: the Mutation Server Protocol answers engine failures with the record in error.data and keeps serving, MCP rerun_mutant returns a RerunEngineUnusable refusal carrying it and the new get_failure tool reads reports/mutation/failure.json, a configured sarif reporter writes a SARIF log whose invocation is executionSuccessful false with one notification per record, and the run log no longer prints an orphan 'Possible causes' block or the bundled parser's WASI ExperimentalWarning. diff --git a/packages/stryker-js/etc/stryker-js.api.md b/packages/stryker-js/etc/stryker-js.api.md index 1e8994a6..08334102 100644 --- a/packages/stryker-js/etc/stryker-js.api.md +++ b/packages/stryker-js/etc/stryker-js.api.md @@ -36,7 +36,7 @@ export namespace ImportClosure { // @public (undocumented) export namespace Mcp { - export { McpServerOptions, MutantDetail, MutantUnusable, RerunMutantFailure, ShowMutantFailure, mcpServerLayer, mcpToolkit }; + export { FailureLookup, GetFailure, McpServerOptions, MutantDetail, MutantUnusable, RerunEngineUnusable, RerunMutantFailure, ShowMutantFailure, mcpServerLayer, mcpToolkit }; } // @public (undocumented) diff --git a/packages/stryker-js/src/Mcp/mcp-server.cell.ts b/packages/stryker-js/src/Mcp/mcp-server.cell.ts index 2c25db38..b3b20dea 100644 --- a/packages/stryker-js/src/Mcp/mcp-server.cell.ts +++ b/packages/stryker-js/src/Mcp/mcp-server.cell.ts @@ -1,39 +1,45 @@ -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, type Options, Report } from '@systemfsoftware/stryker-js-plugin-interface' import { McpProtocol, McpServer } from 'effect/ai' import * as Arr from 'effect/Array' import * as Cause from 'effect/Cause' import * as Effect from 'effect/Effect' +import * as Exit from 'effect/Exit' import * as FileSystem from 'effect/FileSystem' import * as Layer from 'effect/Layer' +import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Path from 'effect/Path' import * as Queue from 'effect/Queue' +import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { ReproducerSchema } from '../build-reproducers.workflow.js' +import { classifyRunOutcome, type RunOutcomeDecision } from '../classify-run-outcome.workflow.js' +import { runOutcomeCommandOf } from '../conclude-run.js' import type { ConfigReadError } from '../ConfigError.schema.js' import { recordFeedbackCell } from '../Feedback/Feedback.cell.js' import { FeedbackUnusable } from '../Feedback/Feedback.schema.js' import { readMutationReport, readSurfacedSurvivors } from '../Feedback/read-report.js' import { ResolvedMode } from '../output-mode.schema.js' import { MachineConsole } from '../reporting/machine-console.service.js' -import { mutantRerunAdmissionCell, RerunRefused } from '../Rerun/mod.js' +import { mutantRerunAdmissionCell, RerunEngineUnusable, RerunRefused } from '../Rerun/mod.js' import { mutantDetailEventsOf } from '../Rerun/rerun-selection.js' import type { MutationTestDone } from '../run/mutation-test.cell.js' import { mutationTestCell } from '../run/run-stages.cell.js' import { RunEnvironment, type RunEnvironmentShape } from '../run/RunEnvironment.service.js' import type { EnginePorts } from '../run/StageServices.service.js' +import { FAILURE_RECORD_FILE } from '../stryker-outputs.js' import { StrykerPackage } from '../stryker-package.schema.js' import { mcpToolkit } from './mcp-tools.js' -import { MutantDetail, MutantUnusable, RerunUnusable } from './mcp-tools.schema.js' +import { FailureLookup, MutantDetail, MutantUnusable, RerunMutantFailure, RerunUnusable } from './mcp-tools.schema.js' const MCP_RUN_ID = RunEvent.RunId.make('01ARZ3NDEKTSV4RRFFQ69G5FAM') const REPRODUCERS_FILE = 'reports/mutation/reproducers.json' const MCP_SERVER_INSTRUCTIONS = - 'Stryker mutation testing. Use list_survivors to read the surfaced survivors of the finished report, show_mutant to inspect one, rerun_mutant to re-run one through the run pipeline, and report_usefulness to record whether a survivor was worth acting on.' + 'Stryker mutation testing. Use list_survivors to read the surfaced survivors of the finished report, show_mutant to inspect one, rerun_mutant to re-run one through the run pipeline, report_usefulness to record whether a survivor was worth acting on, and get_failure to read the failure record the last failed run left behind.' const MCP_MODE = ResolvedMode.make({ mode: 'machine', signal: 'tool', stdoutIsTTY: false }) @@ -123,10 +129,30 @@ const restrictedOptionsOf = ({ incremental: true, }) +const engineDecisionOf = (exit: Exit.Exit, basePath: string): RunOutcomeDecision => + Result.getOrElse( + classifyRunOutcome(runOutcomeCommandOf({ exit, argv: [], cwd: basePath, traceId: null })), + (impossible) => impossible, + ) + +const engineRefusalOf = ( + exit: Exit.Exit, + basePath: string, +): Option.Option => + Match.value(engineDecisionOf(exit, basePath)).pipe( + Match.tag('RunFailed', (failed) => + Option.some(RerunEngineUnusable.make({ code: failed.code, record: failed.record }))), + Match.orElse(() => + Option.none() + ), + ) + +const NO_RESULTS: MutationTestDone = { results: [], verdict: null } + const runRestricted = ( basePath: string, options: Options.PartialStrykerOptions, -): Effect.Effect => +): Effect.Effect => Effect.gen(function*() { const queue = yield* Queue.bounded(RunEvent.RunEvent.QUEUE_BOUND) const environment: RunEnvironmentShape = { @@ -141,13 +167,23 @@ const runRestricted = ( RunEnvironment.stage(environment, queue), MachineConsole.captureLayer.pipe(Layer.provide(MachineConsole.layer)), ) - return yield* mutationTestCell - .run({ cliOptions: options, targetMutatePatterns: undefined }) - .pipe(Effect.provide(runLayer), Effect.scoped, Effect.orDie) + const exit = yield* Effect.exit( + mutationTestCell + .run({ cliOptions: options, targetMutatePatterns: undefined }) + .pipe(Effect.provide(runLayer), Effect.scoped), + ) + return yield* Exit.match(exit, { + onSuccess: (done) => Effect.succeed(done), + onFailure: (cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Option.match(engineRefusalOf(Exit.failCause(cause), basePath), { + onNone: () => Effect.die(new Error('an engine failure did not classify as a run failure')), + onSome: (refusal) => Effect.fail(refusal), + }), + }) }) -const NO_RESULTS: MutationTestDone = { results: [], verdict: null } - const resultsOf = (settled: void | MutationTestDone): ReadonlyArray => (settled ?? NO_RESULTS).results @@ -162,14 +198,20 @@ const reportedOf = ( const reasonOfRerun = (error: RerunRefused | ConfigReadError): string => S.is(RerunRefused)(error) ? error.reason : error.message -const rerunFailureOf = - (id: Mutant.MutantId) => (error: RerunRefused | ConfigReadError | MutantUnusable): MutantUnusable | RerunUnusable => - S.is(MutantUnusable)(error) ? error : RerunUnusable.make({ id, reason: reasonOfRerun(error) }) +const isRerunRefusal = (error: unknown): error is MutantUnusable | RerunEngineUnusable => + S.is(MutantUnusable)(error) || S.is(RerunEngineUnusable)(error) + +const rerunFailureOf = ( + id: Mutant.MutantId, +) => +( + error: RerunRefused | ConfigReadError | MutantUnusable | RerunEngineUnusable, +): RerunMutantFailure => isRerunRefusal(error) ? error : RerunUnusable.make({ id, reason: reasonOfRerun(error) }) const rerunMutant = ( basePath: string, id: Mutant.MutantId, -): Effect.Effect => +): Effect.Effect => Effect.gen(function*() { const settled = yield* mutantRerunAdmissionCell.run({ ids: [id], @@ -196,6 +238,16 @@ const rerunMutant = ( } }) +const readFailure = (basePath: string): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const text = yield* fs.readFileString(path.resolve(basePath, FAILURE_RECORD_FILE)).pipe( + Effect.orElseSucceed(() => ''), + ) + return { failure: Option.getOrNull(S.decodeOption(FailureRecord.FailureRecordFile)(text)) } + }) + const handlers = (basePath: string) => mcpToolkit.toLayer({ list_survivors: () => readSurfacedSurvivors(basePath), @@ -203,6 +255,7 @@ const handlers = (basePath: string) => rerun_mutant: ({ id }) => rerunMutant(basePath, id), report_usefulness: ({ id, judgment, reason }) => recordFeedbackCell({ basePath, id, judgment, reason: reason ?? null }), + get_failure: () => readFailure(basePath), }) export const mcpServerLayer = ({ basePath }: McpServerOptions) => diff --git a/packages/stryker-js/src/Mcp/mcp-tools.schema.ts b/packages/stryker-js/src/Mcp/mcp-tools.schema.ts index 33b372b4..4704fe94 100644 --- a/packages/stryker-js/src/Mcp/mcp-tools.schema.ts +++ b/packages/stryker-js/src/Mcp/mcp-tools.schema.ts @@ -1,7 +1,9 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import * as S from 'effect/Schema' import { FeedbackUnusable } from '../Feedback/Feedback.schema.js' +import { RerunEngineUnusable } from '../Rerun/rerun.schema.js' export const MutantDetail = S.Struct({ id: Mutant.MutantId, @@ -31,6 +33,12 @@ export class RerunUnusable extends S.TaggedError()('RerunUnusable } } +export const FailureLookup = S.Struct({ failure: S.NullOr(FailureRecord.FailureRecord) }) + +export type FailureLookup = typeof FailureLookup.Type + export const ShowMutantFailure = S.Union([MutantUnusable, FeedbackUnusable]) -export const RerunMutantFailure = S.Union([MutantUnusable, RerunUnusable]) +export const RerunMutantFailure = S.Union([MutantUnusable, RerunUnusable, RerunEngineUnusable]) + +export type RerunMutantFailure = typeof RerunMutantFailure.Type diff --git a/packages/stryker-js/src/Mcp/mcp-tools.ts b/packages/stryker-js/src/Mcp/mcp-tools.ts index ea2fb3fd..8aad737e 100644 --- a/packages/stryker-js/src/Mcp/mcp-tools.ts +++ b/packages/stryker-js/src/Mcp/mcp-tools.ts @@ -11,7 +11,7 @@ import { FeedbackUnusable } from '../Feedback/Feedback.schema.js' import { GitDiff } from '../git-diff.service.js' import { SurvivorRef } from '../surfacing.schema.js' import { WorkerLauncher } from '../WorkerLauncher.service.js' -import { MutantDetail, RerunMutantFailure, ShowMutantFailure } from './mcp-tools.schema.js' +import { FailureLookup, MutantDetail, RerunMutantFailure, ShowMutantFailure } from './mcp-tools.schema.js' const reportDependencies = [FileSystem.FileSystem, Path.Path] @@ -63,4 +63,11 @@ export const ReportUsefulness = Tool.make('report_usefulness', { dependencies: reportDependencies, }) -export const mcpToolkit = Toolkit.make(ListSurvivors, ShowMutant, RerunMutant, ReportUsefulness) +export const GetFailure = Tool.make('get_failure', { + description: + 'Read the failure record the last finished run wrote to reports/mutation/failure.json: its stable code, evidence, reproduction and next action. Answers null when no failed run left a record.', + success: FailureLookup, + dependencies: reportDependencies, +}) + +export const mcpToolkit = Toolkit.make(ListSurvivors, ShowMutant, RerunMutant, ReportUsefulness, GetFailure) diff --git a/packages/stryker-js/src/Mcp/mod.ts b/packages/stryker-js/src/Mcp/mod.ts index 6bea3e78..9a587490 100644 --- a/packages/stryker-js/src/Mcp/mod.ts +++ b/packages/stryker-js/src/Mcp/mod.ts @@ -1,3 +1,10 @@ +export { RerunEngineUnusable } from '../Rerun/rerun.schema.js' export { mcpServerLayer, type McpServerOptions } from './mcp-server.cell.js' -export { mcpToolkit } from './mcp-tools.js' -export { MutantDetail, MutantUnusable, RerunMutantFailure, ShowMutantFailure } from './mcp-tools.schema.js' +export { GetFailure, mcpToolkit } from './mcp-tools.js' +export { + FailureLookup, + MutantDetail, + MutantUnusable, + RerunMutantFailure, + ShowMutantFailure, +} from './mcp-tools.schema.js' diff --git a/packages/stryker-js/src/Rerun/Rerun.cell.ts b/packages/stryker-js/src/Rerun/Rerun.cell.ts index a35a2306..d363b84c 100644 --- a/packages/stryker-js/src/Rerun/Rerun.cell.ts +++ b/packages/stryker-js/src/Rerun/Rerun.cell.ts @@ -13,6 +13,7 @@ import type { MutationTestDone } from '../run/mutation-test.cell.js' import type { EnginePorts } from '../run/StageServices.service.js' import { PriorReportDocument } from '../Survivors/Survivors.schema.js' import { admitMutantRerun, type PriorMutantShape, RerunRefused } from './admit-mutant-rerun.workflow.js' +import type { RerunEngineUnusable } from './rerun.schema.js' export type MutantRerunSettled = void | MutationTestDone @@ -23,7 +24,7 @@ export interface MutantRerunRun { } export interface MutantRerunSettlement { - readonly runAdmitted: (run: MutantRerunRun) => Effect.Effect + readonly runAdmitted: (run: MutantRerunRun) => Effect.Effect } export interface MutantRerunInput { diff --git a/packages/stryker-js/src/Rerun/mod.ts b/packages/stryker-js/src/Rerun/mod.ts index a89677c5..438a4a2e 100644 --- a/packages/stryker-js/src/Rerun/mod.ts +++ b/packages/stryker-js/src/Rerun/mod.ts @@ -6,3 +6,4 @@ export { type MutantRerunSettled, type MutantRerunSettlement, } from './Rerun.cell.js' +export { RerunEngineUnusable } from './rerun.schema.js' diff --git a/packages/stryker-js/src/Rerun/rerun.schema.ts b/packages/stryker-js/src/Rerun/rerun.schema.ts new file mode 100644 index 00000000..6a2f9ad3 --- /dev/null +++ b/packages/stryker-js/src/Rerun/rerun.schema.ts @@ -0,0 +1,8 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import * as S from 'effect/Schema' + +export class RerunEngineUnusable extends S.TaggedClass()('RerunEngineUnusable', { + code: Plugin.ExitCode, + record: FailureRecord.FailureRecord, +}) {} diff --git a/packages/stryker-js/src/Serve/Serve.cell.ts b/packages/stryker-js/src/Serve/Serve.cell.ts index 4dcc51f6..36f0265e 100644 --- a/packages/stryker-js/src/Serve/Serve.cell.ts +++ b/packages/stryker-js/src/Serve/Serve.cell.ts @@ -14,14 +14,15 @@ */ import { NodeSocketServer } from '@effect/platform-node' import { Cell } from '@systemfsoftware/effect-cell-types' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { HtmlReporter } from '@systemfsoftware/stryker-js-html-reporter' -import type { Mutant, Options } from '@systemfsoftware/stryker-js-plugin-interface' -import type * as Cause from 'effect/Cause' +import { type Mutant, type Options } from '@systemfsoftware/stryker-js-plugin-interface' +import * as Cause from 'effect/Cause' import * as Clock from 'effect/Clock' import * as Context from 'effect/Context' import * as DateTime from 'effect/DateTime' import * as Effect from 'effect/Effect' +import * as Exit from 'effect/Exit' import * as Fiber from 'effect/Fiber' import * as FiberSet from 'effect/FiberSet' import * as FileSystem from 'effect/FileSystem' @@ -39,6 +40,8 @@ import * as SocketServer from 'effect/socket/SocketServer' import * as Stdio from 'effect/Stdio' import * as Stream from 'effect/Stream' +import { classifyRunOutcome, type RunOutcomeDecision } from '../classify-run-outcome.workflow.js' +import { runOutcomeCommandOf } from '../conclude-run.js' import { concurrencyCell } from '../concurrency.cell.js' import type { ResolvedMode } from '../output-mode.schema.js' import { readProjectCell } from '../read-project.cell.js' @@ -108,6 +111,28 @@ interface EngineRun { readonly context: Context.Context } +const engineDecisionOf = (exit: Exit.Exit, basePath: string): RunOutcomeDecision => + Result.getOrElse( + classifyRunOutcome(runOutcomeCommandOf({ exit, argv: [], cwd: basePath, traceId: null })), + (impossible) => impossible, + ) + +const engineFailureOf = ( + exit: Exit.Exit, + basePath: string, + reason: string, +): Effect.Effect => + Exit.match(exit, { + onSuccess: (value) => Effect.succeed(value), + onFailure: (cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Match.value(engineDecisionOf(Exit.failCause(cause), basePath)).pipe( + Match.tag('RunFailed', (failed) => Effect.fail(ServeError.make({ reason, cause: failed.record }))), + Match.orElse(() => Effect.die(new Error(reason))), + ), + }) + const serveEnvironment = (basePath: string, startedAt: number): RunEnvironmentShape => ({ runId: generateRunId(DateTime.makeUnsafe(startedAt)), resolvedMode: HEADLESS_MODE, @@ -299,14 +324,15 @@ const runDiscover = ( withEngine(() => Effect.void, ({ env, context }) => Effect.gen(function*() { const path = yield* Path.Path - const done = yield* Cell.provideContext(discoverStageCell, context).run({ - cliOptions: options, - targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], - }) + const exit = yield* Effect.exit( + Cell.provideContext(discoverStageCell, context).run({ + cliOptions: options, + targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], + }), + ) + const done = yield* engineFailureOf(exit, env.basePath, 'discovery failed') return { files: discoveredFilesOf(done.mutants, (fileName) => path.relative(env.basePath, fileName)) } - }).pipe( - Effect.mapError((failure) => ServeError.make({ reason: describe(failure, 'discovery failed'), cause: failure })), - )) + })) const runMutationTest = ( options: ServeOptions, @@ -322,16 +348,15 @@ const runMutationTest = ( ({ env, context }) => Effect.gen(function*() { const path = yield* Path.Path - const done = yield* Cell.provideContext(mutationTestCell, context).run({ - cliOptions: options, - targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], - }) + const exit = yield* Effect.exit( + Cell.provideContext(mutationTestCell, context).run({ + cliOptions: options, + targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], + }), + ) + const done = yield* engineFailureOf(exit, env.basePath, 'the mutation test failed') return { files: mutationTestFilesOf(done.results, (fileName) => path.relative(env.basePath, fileName)) } - }).pipe( - Effect.mapError((failure) => - ServeError.make({ reason: describe(failure, 'the mutation test failed'), cause: failure }) - ), - ), + }), ) const respond = (connection: Connection, id: JsonRpcId, result: S.Json): Effect.Effect => @@ -345,12 +370,36 @@ const respondError = ( id: JsonRpcId | null, code: number, message: string, + data?: S.Json, ): Effect.Effect => Effect.flatMap( - encodedOr(encodeJsonRpcResponse({ jsonrpc: '2.0' as const, id, error: { code, message } }), 'an error response'), + encodedOr( + encodeJsonRpcResponse({ + jsonrpc: '2.0' as const, + id, + error: { code, message, ...(data === undefined ? {} : { data }) }, + }), + 'an error response', + ), (payload) => connection.write(payload), ) +const respondFailure = ( + connection: Connection, + id: JsonRpcId, + failure: ServeError, +): Effect.Effect => + Option.match( + Option.flatMap( + Option.fromUndefinedOr(failure.cause), + (cause) => S.decodeUnknownOption(FailureRecord.FailureRecord)(cause), + ), + { + onNone: () => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason), + onSome: (record) => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason, record), + }, + ) + const notifyProgress = ( connection: Connection, notification: ReportMutationTestProgress, @@ -382,7 +431,7 @@ const runRequested = ( runDiscover(optionsOf(invocation, yield* state), restrictPatternsOf(params.files)), ) yield* Result.match(result, { - onFailure: (failure) => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason), + onFailure: (failure) => respondFailure(connection, id, failure), onSuccess: (files) => respond(connection, id, files), }) })), @@ -396,7 +445,7 @@ const runRequested = ( ), ) yield* Result.match(result, { - onFailure: (failure) => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason), + onFailure: (failure) => respondFailure(connection, id, failure), onSuccess: (files) => respond(connection, id, files), }) })), diff --git a/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts b/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts index d38b67a3..36940f15 100644 --- a/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts @@ -1,16 +1,27 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, type Report } from '@systemfsoftware/stryker-js-plugin-interface' import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' import * as Arr from 'effect/Array' import * as Match from 'effect/Match' import * as Option from 'effect/Option' +import * as Predicate from 'effect/Predicate' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -import { sarifReport, SarifReportCommand, SarifReportRendered } from '../sarif-report.workflow.js' +import { + FailureReportSource, + SarifFailureReportRendered, + sarifReport, + SarifReportCommand, + type SarifReportDecision, + SarifReportRendered, + SurvivorsReportSource, +} from '../sarif-report.workflow.js' import type { SurvivorRef } from '../surfacing.schema.js' const GENEROUS_BUDGET = Number.MAX_SAFE_INTEGER +const TEST_TOOL = { name: 'StrykerJS', version: '0.0.0', informationUri: 'https://stryker-mutator.io' } as const type Subject = typeof sarifReport const isSurvivorStatus = (status: string): boolean => status === 'Survived' || status === 'NoCoverage' @@ -76,37 +87,69 @@ const refsOf = (report: Report.MutationTestResult): ReadonlyArray = const commandArb: Arbitrary.Arbitrary = Arbitrary.all({ report: reportArb, - tool: Arbitrary.all({ - name: Arbitrary.schema(S.String), - version: Arbitrary.schema(S.String), - informationUri: Arbitrary.schema(S.String), - }), maxResults: Arbitrary.schema(S.Natural), }).pipe( - Arbitrary.map(({ report, ...rest }) => SarifReportCommand.make({ report, survivors: refsOf(report), ...rest })), + Arbitrary.map(({ report, maxResults }) => + SarifReportCommand.make({ + source: SurvivorsReportSource.make({ report, survivors: refsOf(report), maxResults }), + tool: TEST_TOOL, + }) + ), ) const withBudget = ( command: SarifReportCommand, survivors: ReadonlyArray, maxResults: number, -): SarifReportCommand => SarifReportCommand.make({ report: command.report, survivors, tool: command.tool, maxResults }) +): SarifReportCommand => { + const source = command.source + return SarifReportCommand.make({ + source: S.is(SurvivorsReportSource)(source) + ? SurvivorsReportSource.make({ report: source.report, survivors, maxResults }) + : source, + tool: command.tool, + }) +} + +const survivorsOf = (command: SarifReportCommand): ReadonlyArray => + S.is(SurvivorsReportSource)(command.source) ? [...command.source.survivors] : [] const decisionOf = (subject: Subject, command: SarifReportCommand) => subject(command).pipe(Result.getOrThrow) const runOf = (subject: Subject, command: SarifReportCommand) => Option.getOrThrow(Arr.head(decisionOf(subject, command).log.runs)) +const failureDecisionOf = (source: FailureReportSource): SarifReportDecision => + sarifReport(SarifReportCommand.make({ source, tool: TEST_TOOL })).pipe(Result.getOrThrow) + +const failureRunOf = (decision: SarifReportDecision) => + Option.getOrThrow( + Arr.head(Option.getOrThrow(Option.liftPredicate(decision, S.is(SarifFailureReportRendered))).log.runs), + ) + +const failureCodeOf = (record: FailureRecord.FailureRecord): FailureRecord.FailureCode => + Option.getOrThrow(Arr.findFirst(FailureRecord.FailureCode.literals, (code) => Predicate.isTagged(record, code))) + +const recordLocationsOf = (record: FailureRecord.FailureRecord): ReadonlyArray => + Match.value(record).pipe( + Match.tag( + 'BaselineTestsFailed', + (evidence) => Arr.flatMap(evidence.tests, (test) => Option.toArray(Option.fromNullishOr(test.location))), + ), + Match.orElse((): ReadonlyArray => []), + ) + describe('sarifReport', () => { it.prop( '∀c_SarifReportCommand_≡AGenerousBudgetEmitsOneResultPerCommandedSurvivor', { of: [commandArb], subject: sarifReport }, (subject, [command]) => { - const results = runOf(subject, withBudget(command, command.survivors, GENEROUS_BUDGET)).results + const survivors = survivorsOf(command) + const results = runOf(subject, withBudget(command, survivors, GENEROUS_BUDGET)).results const fingerprints = Arr.map(results, (result) => result.partialFingerprints.primaryLocationLineHash) - return results.length === command.survivors.length && + return results.length === survivors.length && new Set(fingerprints).size === results.length && - fingerprints.every((id) => command.survivors.some((ref) => ref.id === id)) + fingerprints.every((id) => survivors.some((ref) => ref.id === id)) }, ) @@ -123,20 +166,60 @@ describe('sarifReport', () => { '∀c_SarifReportCommand_≡AGenerousBudgetNeverTruncates', { of: [commandArb], subject: sarifReport }, (subject, [command]) => - S.is(SarifReportRendered)(decisionOf(subject, withBudget(command, command.survivors, GENEROUS_BUDGET))), + S.is(SarifReportRendered)(decisionOf(subject, withBudget(command, survivorsOf(command), GENEROUS_BUDGET))), ) it.prop( '∀c_SarifReportCommand_≡AZeroResultBudgetEmitsNoResultAndDeclaresTheOmission', { of: [commandArb], subject: sarifReport }, (subject, [command]) => { - const decision = decisionOf(subject, withBudget(command, command.survivors, 0)) + const decision = decisionOf(subject, withBudget(command, survivorsOf(command), 0)) const results = Option.getOrThrow(Arr.head(decision.log.runs)).results return Match.value(decision).pipe( - Match.tag('SarifReportRendered', () => results.length === 0 && command.survivors.length === 0), + Match.tag('SarifReportRendered', () => results.length === 0 && survivorsOf(command).length === 0), Match.tag('SarifReportTruncated', (truncated) => results.length === 0 && truncated.omitted > 0), + Match.tag('SarifFailureReportRendered', () => false), Match.exhaustive, ) }, ) }) + +describe('sarifReport failure notifications', () => { + it.prop( + '∀s_FailureReportSource_≡AFailedInvocationNotifiesOncePerRecord', + { of: [FailureReportSource], subject: failureDecisionOf }, + (subject, [source]) => { + const invocation = Option.getOrThrow(Arr.head(failureRunOf(subject(source)).invocations)) + return invocation.exitCode === source.exitCode && + invocation.toolExecutionNotifications.length === source.records.length && + Arr.every( + Arr.zip(source.records, invocation.toolExecutionNotifications), + ([record, notification]) => + notification.descriptor.id === failureCodeOf(record) && + notification.message.text === FailureRecord.sarifTextOf(record), + ) + }, + ) + + it.prop( + '∀s_FailureReportSource_≡OnlyLocatedBaselineTestsCarryAPhysicalLocation', + { of: [FailureReportSource], subject: failureDecisionOf }, + (subject, [source]) => { + const notifications = Option.getOrThrow(Arr.head(failureRunOf(subject(source)).invocations)) + .toolExecutionNotifications + return notifications.length === source.records.length && + Arr.every(Arr.zip(source.records, notifications), ([record, notification]) => { + const expected = recordLocationsOf(record) + return notification.locations.length === expected.length && + Arr.every( + Arr.zip(expected, notification.locations), + ([location, projected]) => + decodeURIComponent(projected.physicalLocation.artifactLocation.uri) === location.file.toWellFormed() && + projected.physicalLocation.region.startLine === location.line && + projected.physicalLocation.region.startColumn === location.column, + ) + }) + }, + ) +}) diff --git a/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts b/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts index c98122cc..0fe5d4a8 100644 --- a/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts @@ -1,25 +1,45 @@ import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' +import * as Match from 'effect/Match' import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { forkOptionsSchema } from '../Config.schema.js' import { OptionsRefused, OptionsValidated, type OptionsValidationDecision, validateOptionsAdmission, ValidateOptionsCommand, + type ValidationSchemaDocument, } from '../run/validate-options-admission.workflow.js' -const commandOf = (options: Record): ValidateOptionsCommand => - ValidateOptionsCommand.make({ options, schema: {} }) +const CORE_SCHEMA_DOCUMENT: ValidationSchemaDocument = S.toJsonSchemaDocument(forkOptionsSchema).schema -const decide = (subject: typeof validateOptionsAdmission, options: Record) => - subject(commandOf(options)).pipe(Result.getOrElse((neverError) => neverError)) +const commandOf = ( + options: Record, + schema: ValidationSchemaDocument = {}, +): ValidateOptionsCommand => ValidateOptionsCommand.make({ options, schema }) + +const decide = ( + subject: typeof validateOptionsAdmission, + options: Record, + schema?: ValidationSchemaDocument, +) => subject(commandOf(options, schema)).pipe(Result.getOrElse((neverError) => neverError)) const refusedWith = (decision: OptionsValidationDecision, fragment: string): boolean => S.is(OptionsRefused)(decision) && decision.errors.some((error) => error.includes(fragment)) +const warningsIn = (decision: OptionsValidationDecision): readonly string[] => + Match.valueTags(decision, { + OptionsRefused: (refused) => refused.warnings, + OptionsUndecodable: (undecodable) => undecodable.warnings, + OptionsValidated: (validated) => validated.warnings, + }) + +const hasUnknownOptionWarning = (warnings: readonly string[]): boolean => + warnings.some((warning) => warning.includes('Unknown stryker config option') || warning.includes('Possible causes')) + const rangeArb = Arbitrary.schema(S.Struct({ startLine: S.Int.check(S.isBetween({ minimum: 0, maximum: 8 })), endLine: S.Int.check(S.isBetween({ minimum: 0, maximum: 8 })), @@ -30,6 +50,16 @@ const globArb = Arbitrary.schema(S.Struct({ name: S.String.check(S.isPattern(/^[a-z][a-z0-9]{0,4}$/)), })) +const unknownOptionArb = Arbitrary.schema(S.Struct({ + include: S.Boolean, + name: S.String.check(S.isPattern(/^k[a-z0-9]{0,4}$/)), +})) + +const optionsWith = (include: boolean, name: string): Record => { + const known: Record = { warnings: true } + return include ? { ...known, [name]: 42 } : known +} + describe('validateOptionsAdmission', () => { it.prop( '∀m_Mutate_≡RangeBoundsAreEnforced', @@ -84,4 +114,15 @@ describe('validateOptionsAdmission', () => { : S.is(OptionsValidated)(decision) }, ) + + it.prop( + '∀k_Included_≡UnknownOptionIsWarnedExactlyWhenTheSchemaOmitsIt', + { of: [unknownOptionArb], subject: validateOptionsAdmission }, + (subject, [{ include, name }]) => { + const decision = decide(subject, optionsWith(include, name), CORE_SCHEMA_DOCUMENT) + const warnings = warningsIn(decision) + const namesTheOption = warnings.some((warning) => warning.includes(`Unknown stryker config option "${name}".`)) + return namesTheOption === include && hasUnknownOptionWarning(warnings) === include + }, + ) }) diff --git a/packages/stryker-js/src/bin/main.ts b/packages/stryker-js/src/bin/main.ts index ee5bb4c0..584360fb 100755 --- a/packages/stryker-js/src/bin/main.ts +++ b/packages/stryker-js/src/bin/main.ts @@ -9,6 +9,7 @@ import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics' import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { HtmlReporter } from '@systemfsoftware/stryker-js-html-reporter' +import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import cliPkgJson from '@systemfsoftware/stryker-js/package.json' with { type: 'json' } import * as Boolean from 'effect/Boolean' import * as Cause from 'effect/Cause' @@ -50,6 +51,25 @@ import { makeStrykerCommand } from './cli-command.js' globalThis.process.title = 'stryker' +const WASI_WARNING_MENTION = 'WASI' + +const isWasiExperimentalWarning = (warning: Error): boolean => + Boolean.and(warning.name === 'ExperimentalWarning', warning.message.includes(WASI_WARNING_MENTION)) + +const NODE_WARNING_PRINTERS = globalThis.process.listeners('warning') + +globalThis.process.removeAllListeners('warning') + +globalThis.process.on('warning', (warning) => + Boolean.match(isWasiExperimentalWarning(warning), { + onFalse: () => { + for (const print of NODE_WARNING_PRINTERS) { + print(warning) + } + }, + onTrue: () => undefined, + })) + const EXPORT_TIMEOUT_MILLIS = 5000 const SHUTDOWN_TIMEOUT = EffectDuration.millis(EXPORT_TIMEOUT_MILLIS + 1_000) @@ -266,11 +286,16 @@ const strykerProgram = Effect.gen(function*() { 'stryker.run.exit_code': runExitCodeFromOutcome(decision).code, ...failureCodeAttributeOf(decision), }) + const resolvedOptions = yield* Option.match(Option.fromUndefinedOr(host.resolvedOptions), { + onNone: () => Effect.succeed(null), + onSome: (resolved) => Ref.get(resolved), + }) return yield* concludeRunCell.run({ mode, stream, basePath: host.basePath, pathService, + resolvedOptions, fileSystem: yield* FileSystem.FileSystem, runEvents, decision, diff --git a/packages/stryker-js/src/conclude-run.cell.ts b/packages/stryker-js/src/conclude-run.cell.ts index f5c45cf8..a180dc6f 100644 --- a/packages/stryker-js/src/conclude-run.cell.ts +++ b/packages/stryker-js/src/conclude-run.cell.ts @@ -1,16 +1,21 @@ import { Sandwich } from '@systemfsoftware/effect-cell-types' import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import * as Effect from 'effect/Effect' import type * as FileSystem from 'effect/FileSystem' +import * as Option from 'effect/Option' import type * as Path from 'effect/Path' +import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { RunExit, RunOutcomeDecision } from './classify-run-outcome.workflow.js' import type { ResolvedMode } from './output-mode.schema.js' import { planRunConclusion, type PlanRunConclusionCommand } from './plan-run-conclusion.workflow.js' +import { SARIF_TOOL } from './reporter-factories.js' import type { RunEventDrain, RunEventStream, RunEventStreamPort } from './run-event-stream.service.js' +import { FailureReportSource, sarifReport, SarifReportCommand } from './sarif-report.workflow.js' import { StrykerError } from './stryker-error.schema.js' -import { FAILURE_RECORD_FILE } from './stryker-outputs.js' +import { FAILURE_RECORD_FILE, sarifFileNameOf } from './stryker-outputs.js' export interface RunConclusionInput { readonly mode: ResolvedMode @@ -20,6 +25,7 @@ export interface RunConclusionInput { readonly fileSystem: FileSystem.FileSystem readonly runEvents: RunEventStreamPort readonly decision: RunOutcomeDecision + readonly resolvedOptions: Options.StrykerOptions | null } export type RunConclusionRaw = (typeof PlanRunConclusionCommand)['Encoded'] & { @@ -58,6 +64,52 @@ const removeStaleFailureRecord = (conclusion: RunConclusionInput) => Effect.ignore, ) +const SARIF_REPORTER_NAME = 'sarif' + +const sarifFailureFileOf = (conclusion: RunConclusionInput): string | null => + Option.getOrNull( + Option.map( + Option.filter( + Option.fromNullishOr(conclusion.resolvedOptions), + (options) => options.reporters.includes(SARIF_REPORTER_NAME), + ), + (options) => conclusion.pathService.resolve(conclusion.basePath, sarifFileNameOf(options.jsonReporter.fileName)), + ), + ) + +const writeSarifFailureLog = ( + conclusion: RunConclusionInput, + file: string, + exitCode: number, + record: FailureRecord.FailureRecord, +) => + Effect.gen(function*() { + const rendered = Result.getOrThrow( + sarifReport( + SarifReportCommand.make({ + source: FailureReportSource.make({ records: [record], exitCode }), + tool: SARIF_TOOL, + }), + ), + ) + const json = yield* S.encodeEffect(S.fromJsonString(S.Unknown, { space: 2 }))(rendered.log) + yield* conclusion.fileSystem.makeDirectory(conclusion.pathService.dirname(file), { recursive: true }) + yield* conclusion.fileSystem.writeFileString(file, json) + }).pipe( + Effect.tapError((error) => Effect.logWarning(`could not write the SARIF failure log: ${error.message}`)), + Effect.ignore, + ) + +const writeFailureSarif = ( + conclusion: RunConclusionInput, + exitCode: number, + record: FailureRecord.FailureRecord, +) => + Effect.forEach( + Option.toArray(Option.fromNullishOr(sarifFailureFileOf(conclusion))), + (file) => writeSarifFailureLog(conclusion, file, exitCode, record), + ).pipe(Effect.asVoid) + const closeAndExit = (conclusion: RunConclusionInput, exitCode: number) => Effect.andThen(conclusion.stream.closeAndDrain, Effect.fail(RunExit.make({ code: exitCode }))) @@ -89,6 +141,7 @@ export const concludeRunCell = Sandwich.named(SpanTaxonomy.Spans.runConclude.nam ), RunConclusionFailed: (decision, raw) => writeFailureRecord(raw.conclusion, decision.record).pipe( + Effect.andThen(writeFailureSarif(raw.conclusion, decision.exitCode, decision.record)), Effect.andThen( raw.conclusion.runEvents.emitFailureRecord(raw.conclusion.stream, decision.exitCode, decision.record), ), diff --git a/packages/stryker-js/src/reporter-factories.ts b/packages/stryker-js/src/reporter-factories.ts index 85ca6051..2c67063c 100644 --- a/packages/stryker-js/src/reporter-factories.ts +++ b/packages/stryker-js/src/reporter-factories.ts @@ -21,7 +21,7 @@ import { progressReportCell } from './progress-report.cell.js' import type { ProgressState, ProgressTally } from './render-progress-report.workflow.js' import { failAsProgress } from './reporter-failures.js' import { ReporterOutput } from './reporter-output.service.js' -import { sarifReport, SarifReportCommand } from './sarif-report.workflow.js' +import { sarifReport, SarifReportCommand, SurvivorsReportSource } from './sarif-report.workflow.js' import { sarifFileNameOf } from './stryker-outputs.js' import { StrykerPackage } from './stryker-package.schema.js' import { surfacedSurvivorsOf } from './surfacing.js' @@ -85,6 +85,12 @@ export const progressReporterFactory = ( const SARIF_MAX_RESULTS = 5000 const STRYKER_INFORMATION_URI = 'https://stryker-mutator.io' +export const SARIF_TOOL = { + name: 'StrykerJS', + version: StrykerPackage.version, + informationUri: STRYKER_INFORMATION_URI, +} as const + const failAsSarif = (cause: E): Reporter.ReporterFailed => Reporter.ReporterFailed.make({ reporterName: 'sarif', @@ -117,17 +123,15 @@ const writeSarifReport = (options: Options.StrykerOptions, report: Report.Mutati const rendered = Result.getOrThrow( sarifReport( SarifReportCommand.make({ - report, - survivors: surfacedSurvivorsOf(report, { - perLine: options.surfacing.perLine, - perFile: options.surfacing.perFile, + source: SurvivorsReportSource.make({ + report, + survivors: surfacedSurvivorsOf(report, { + perLine: options.surfacing.perLine, + perFile: options.surfacing.perFile, + }), + maxResults: SARIF_MAX_RESULTS, }), - tool: { - name: 'StrykerJS', - version: StrykerPackage.version, - informationUri: STRYKER_INFORMATION_URI, - }, - maxResults: SARIF_MAX_RESULTS, + tool: SARIF_TOOL, }), ), ) diff --git a/packages/stryker-js/src/run/RunEnvironment.service.ts b/packages/stryker-js/src/run/RunEnvironment.service.ts index b11bfb76..5ac3e38c 100644 --- a/packages/stryker-js/src/run/RunEnvironment.service.ts +++ b/packages/stryker-js/src/run/RunEnvironment.service.ts @@ -1,5 +1,5 @@ import { RunEvent, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' -import type { Reporter as InterfaceReporter } from '@systemfsoftware/stryker-js-plugin-interface' +import type { Options, Reporter as InterfaceReporter } from '@systemfsoftware/stryker-js-plugin-interface' import type * as Cause from 'effect/Cause' import * as Clock from 'effect/Clock' import * as Context from 'effect/Context' @@ -13,6 +13,7 @@ import * as Option from 'effect/Option' import type { PlatformError } from 'effect/PlatformError' import * as Predicate from 'effect/Predicate' import * as Queue from 'effect/Queue' +import * as Ref from 'effect/Ref' import * as S from 'effect/Schema' import * as Scope from 'effect/Scope' @@ -34,6 +35,7 @@ export interface RunEnvironmentShape { readonly basePath: string readonly builtinReporters: Readonly> readonly allowConsoleColors: boolean + readonly resolvedOptions?: Ref.Ref | undefined } export class RunEnvironment extends Context.Service()( @@ -81,9 +83,11 @@ export class RunEnvironment extends Context.Service> }, ): Effect.Effect => - Effect.map( - Effect.flatMap(FileSystem.FileSystem, (fs) => fs.realPath('.')), - (basePath) => ({ + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const basePath = yield* fs.realPath('.') + const resolvedOptions = yield* Ref.make(null) + return { runId: stream.runId, resolvedMode: mode, runStartedAt: stream.startedAt, @@ -91,8 +95,9 @@ export class RunEnvironment extends Context.Service( core, ) +const publishResolvedOptions = ( + env: RunEnvironmentShape, + options: Options.StrykerOptions, +): Effect.Effect => + Option.match(Option.fromUndefinedOr(env.resolvedOptions), { + onNone: () => Effect.void, + onSome: (resolved) => Ref.set(resolved, options), + }) + const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( command: ReadProjectDone, ): Effect.fn.Return< @@ -158,6 +168,7 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( allowColor: env.allowConsoleColors, }, } + yield* publishResolvedOptions(env, options) const descriptors: readonly string[] = pluginUrlsFromOptions(options) const loaded = yield* loadPlugins(descriptors, env.basePath).pipe( Effect.tapError(() => diff --git a/packages/stryker-js/src/run/validate-options-admission.workflow.ts b/packages/stryker-js/src/run/validate-options-admission.workflow.ts index a7038137..efb52f83 100644 --- a/packages/stryker-js/src/run/validate-options-admission.workflow.ts +++ b/packages/stryker-js/src/run/validate-options-admission.workflow.ts @@ -194,9 +194,13 @@ const unknownOptionWarnings = ( schema: ValidationSchemaDocument, ): readonly string[] => { const excessNames = excessOptionNames(options, schema) - return excessNames.map((excess) => `Unknown stryker config option "${excess}".`).concat( - POSSIBLE_CAUSES_OF(pluginsJson(options)), - ) + return Boolean.match(excessNames.length > 0, { + onFalse: (): readonly string[] => [], + onTrue: () => [ + ...excessNames.map((excess) => `Unknown stryker config option "${excess}".`), + POSSIBLE_CAUSES_OF(pluginsJson(options)), + ], + }) } const excessOptionWarningsOf = ( diff --git a/packages/stryker-js/src/sarif-failure.schema.ts b/packages/stryker-js/src/sarif-failure.schema.ts new file mode 100644 index 00000000..a33774bd --- /dev/null +++ b/packages/stryker-js/src/sarif-failure.schema.ts @@ -0,0 +1,53 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import * as Arr from 'effect/Array' +import * as Option from 'effect/Option' +import * as Predicate from 'effect/Predicate' +import * as S from 'effect/Schema' + +const encodeUriSegment = encodeURIComponent + +const uriOf = (fileName: string): string => + Arr.join(Arr.map(fileName.split('/'), (segment) => encodeUriSegment(segment.toWellFormed())), '/') + +export const SarifFailureLocation = S.Struct({ + physicalLocation: S.Struct({ + artifactLocation: S.Struct({ uri: S.String }), + region: S.Struct({ startLine: S.Natural, startColumn: S.Natural }), + }), +}) + +export type SarifFailureLocation = typeof SarifFailureLocation.Type + +export const SarifFailureNotification = S.Struct({ + descriptor: S.Struct({ id: S.String }), + level: S.Literal('error'), + message: S.Struct({ text: S.String }), + locations: S.Array(SarifFailureLocation), +}) + +export type SarifFailureNotification = typeof SarifFailureNotification.Type + +export type SarifFailureNotifications = ReadonlyArray + +const locatedTestsOf = (record: FailureRecord.FailureRecord): ReadonlyArray => + Predicate.isTagged(record, 'BaselineTestsFailed') + ? Arr.flatMap(record.tests, (test) => Option.toArray(Option.fromNullishOr(test.location))) + : [] + +const locationOf = (location: FailureRecord.SourceLocation): SarifFailureLocation => ({ + physicalLocation: { + artifactLocation: { uri: uriOf(location.file) }, + region: { startLine: location.line, startColumn: location.column }, + }, +}) + +const notificationOf = (record: FailureRecord.FailureRecord): SarifFailureNotification => ({ + descriptor: { id: record._tag }, + level: 'error', + message: { text: FailureRecord.sarifTextOf(record) }, + locations: Arr.map(locatedTestsOf(record), locationOf), +}) + +export const failureNotificationsOf = ( + records: ReadonlyArray, +): SarifFailureNotifications => Arr.map(records, notificationOf) diff --git a/packages/stryker-js/src/sarif-report.workflow.ts b/packages/stryker-js/src/sarif-report.workflow.ts index db38e5d6..7f1dfc93 100644 --- a/packages/stryker-js/src/sarif-report.workflow.ts +++ b/packages/stryker-js/src/sarif-report.workflow.ts @@ -1,13 +1,16 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' -import { Report } from '@systemfsoftware/stryker-js-plugin-interface' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Plugin, Report } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Boolean from 'effect/Boolean' import * as HashMap from 'effect/HashMap' +import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Order from 'effect/Order' import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { failureNotificationsOf, SarifFailureNotification } from './sarif-failure.schema.js' import { SurvivorRef } from './surfacing.schema.js' const SARIF_SCHEMA_URI = 'https://json.schemastore.org/sarif-2.1.0.json' @@ -26,12 +29,17 @@ const SURVIVOR_LEVELS: Record = { Survived: 'warning const decodeMutantStatus = S.decodeOption(MUTANT_STATUSES) const encodeUriSegment = encodeURIComponent +const uriOf = (fileName: string): string => + Arr.join(Arr.map(fileName.split('/'), (segment) => encodeUriSegment(segment.toWellFormed())), '/') + const SarifToolSchema = S.Struct({ name: S.String, version: S.String, informationUri: S.String, }) +type SarifTool = typeof SarifToolSchema.Type + const SarifRuleSchema = S.Struct({ id: S.String, name: S.String, @@ -83,6 +91,47 @@ export type SarifLog = typeof SarifLogSchema.Type type SarifResult = typeof SarifResultSchema.Type +const SarifInvocationSchema = S.Struct({ + executionSuccessful: S.Literal(false), + exitCode: Plugin.ExitCode, + toolExecutionNotifications: S.Array(SarifFailureNotification), +}) + +const SarifFailureLogSchema = S.Struct({ + $schema: S.Literal(SARIF_SCHEMA_URI), + version: S.Literal('2.1.0'), + runs: S.Array( + S.Struct({ + tool: S.Struct({ driver: SarifDriverSchema }), + invocations: S.Array(SarifInvocationSchema), + results: S.Array(SarifResultSchema), + }), + ), +}) + +export type SarifFailureLog = typeof SarifFailureLogSchema.Type + +const failureLogOf = (tool: SarifTool, source: FailureReportSource): SarifFailureLog => ({ + $schema: SARIF_SCHEMA_URI, + version: '2.1.0', + runs: [{ + tool: { + driver: { + name: tool.name, + version: tool.version, + informationUri: tool.informationUri, + rules: [], + }, + }, + invocations: [{ + executionSuccessful: false, + exitCode: source.exitCode, + toolExecutionNotifications: failureNotificationsOf(source.records), + }], + results: [], + }], +}) + const SarifReportTypeId: unique symbol = Symbol.for('@systemfsoftware/stryker-js/SarifReportDecision') type SarifReportTypeId = typeof SarifReportTypeId @@ -99,14 +148,37 @@ export class SarifReportTruncated extends S.TaggedClass()( readonly [SarifReportTypeId] = SarifReportTypeId } -export const SarifReportDecision = S.Union([SarifReportRendered, SarifReportTruncated]) +export class SarifFailureReportRendered extends S.TaggedClass()( + 'SarifFailureReportRendered', + { log: SarifFailureLogSchema }, +) { + readonly [SarifReportTypeId] = SarifReportTypeId +} + +export const SarifReportDecision = S.Union([ + SarifReportRendered, + SarifReportTruncated, + SarifFailureReportRendered, +]) export type SarifReportDecision = typeof SarifReportDecision.Type -export class SarifReportCommand extends S.TaggedClass()('SarifReportCommand', { +export class SurvivorsReportSource extends S.TaggedClass()('SurvivorsReportSource', { report: Report.MutationTestResult, survivors: S.Array(SurvivorRef), - tool: SarifToolSchema, maxResults: S.Natural, +}) {} + +export class FailureReportSource extends S.TaggedClass()('FailureReportSource', { + records: S.NonEmptyArray(FailureRecord.FailureRecord), + exitCode: Plugin.ExitCode, +}) {} + +export const SarifReportSource = S.Union([SurvivorsReportSource, FailureReportSource]) +export type SarifReportSource = typeof SarifReportSource.Type + +export class SarifReportCommand extends S.TaggedClass()('SarifReportCommand', { + source: SarifReportSource, + tool: SarifToolSchema, }) { static readonly [Workflow.InstrumentationBrand] = {} as const } @@ -143,17 +215,15 @@ const entriesByIdOf = (report: Report.MutationTestResult): HashMap.HashMap, mutatorName: string): number => Option.getOrThrow(Arr.findFirstIndex(ruleNames, (name) => name === mutatorName)) -const uriOf = (fileName: string): string => Arr.join(Arr.map(fileName.split('/'), encodeUriSegment), '/') - interface Surfaced { readonly ref: SurvivorRef readonly entry: SurvivorEntry } -const surfacedOf = (command: SarifReportCommand): ReadonlyArray => { - const byId = entriesByIdOf(command.report) +const surfacedOf = (source: SurvivorsReportSource): ReadonlyArray => { + const byId = entriesByIdOf(source.report) return Arr.flatMap( - command.survivors, + source.survivors, (ref) => Option.toArray(Option.map(HashMap.get(byId, ref.id), (entry): Surfaced => ({ ref, entry }))), ) } @@ -186,7 +256,7 @@ const ruleOf = (mutatorName: string) => ({ shortDescription: { text: mutatorName }, }) -const logOf = (command: SarifReportCommand, results: ReadonlyArray): SarifLog => { +const logOf = (tool: SarifTool, results: ReadonlyArray): SarifLog => { const ruleNames = Arr.dedupe( Arr.sort(Arr.map(results, (surfaced) => surfaced.entry.mutant.mutatorName), Order.String), ) @@ -196,9 +266,9 @@ const logOf = (command: SarifReportCommand, results: ReadonlyArray): S runs: [{ tool: { driver: { - name: command.tool.name, - version: command.tool.version, - informationUri: command.tool.informationUri, + name: tool.name, + version: tool.version, + informationUri: tool.informationUri, rules: Arr.map(ruleNames, ruleOf), }, }, @@ -207,16 +277,24 @@ const logOf = (command: SarifReportCommand, results: ReadonlyArray): S } } -const decide = (command: SarifReportCommand): SarifReportDecision => { - const surfaced = surfacedOf(command) - const results = Arr.take(surfaced, command.maxResults) - const log = logOf(command, results) +const decideSurvivors = (tool: SarifTool, source: SurvivorsReportSource): SarifReportDecision => { + const surfaced = surfacedOf(source) + const results = Arr.take(surfaced, source.maxResults) + const log = logOf(tool, results) return Boolean.match(Arr.length(results) === Arr.length(surfaced), { onTrue: () => SarifReportRendered.make({ log }), onFalse: () => SarifReportTruncated.make({ log, omitted: Arr.length(surfaced) - Arr.length(results) }), }) } +const decide = (command: SarifReportCommand): SarifReportDecision => + Match.value(command.source).pipe( + Match.tag('SurvivorsReportSource', (source) => decideSurvivors(command.tool, source)), + Match.tag('FailureReportSource', (source) => + SarifFailureReportRendered.make({ log: failureLogOf(command.tool, source) })), + Match.exhaustive, + ) + export const sarifReport = Workflow.make({ command: SarifReportCommand, decision: SarifReportDecision, diff --git a/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts b/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts index e0b54478..39964848 100644 --- a/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts +++ b/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts @@ -9,7 +9,7 @@ export const ExitValue = S.declare>(( export const JsonRpcId = S.Union([S.String, S.Finite, S.Null]) export type JsonRpcId = typeof JsonRpcId.Type -export const JsonRpcError = S.Struct({ code: S.Int, message: S.String }) +export const JsonRpcError = S.Struct({ code: S.Int, message: S.String, data: S.optionalKey(S.Json) }) export const JsonRpcResponse = S.Struct({ jsonrpc: S.Literal('2.0'), diff --git a/packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts b/packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts new file mode 100644 index 00000000..3395a1d0 --- /dev/null +++ b/packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts @@ -0,0 +1,17 @@ +import * as S from 'effect/Schema' + +const AttributeValue = S.Struct({ + stringValue: S.optional(S.String), + intValue: S.optional(S.Union([S.String, S.Finite])), +}) + +const Attribute = S.Struct({ key: S.String, value: AttributeValue }) + +export const OtlpSpan = S.Struct({ name: S.String, traceId: S.String, attributes: S.Array(Attribute) }) +export type OtlpSpan = typeof OtlpSpan.Type + +const ScopeSpans = S.Struct({ spans: OtlpSpan.pipe(S.Array, S.optional) }) +const ResourceSpans = S.Struct({ scopeSpans: ScopeSpans.pipe(S.Array, S.optional) }) + +export const OtlpPayload = S.Struct({ resourceSpans: ResourceSpans.pipe(S.Array, S.optional) }) +export type OtlpPayload = typeof OtlpPayload.Type diff --git a/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts b/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts index 722c5759..a2c2a426 100644 --- a/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts +++ b/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts @@ -36,6 +36,30 @@ export const SarifDocument = S.Struct({ export type SarifDocument = typeof SarifDocument.Type +export const FailureSarifDocument = S.Struct({ + version: S.String, + runs: S.Array( + S.Struct({ + invocations: S.Array( + S.Struct({ + executionSuccessful: S.Boolean, + exitCode: S.Int, + toolExecutionNotifications: S.Array( + S.Struct({ + descriptor: S.Struct({ id: S.String }), + level: S.String, + message: S.Struct({ text: S.String }), + locations: S.Array(S.Unknown), + }), + ), + }), + ), + }), + ), +}) + +export type FailureSarifDocument = typeof FailureSarifDocument.Type + export const ReproducerList = S.Array( S.Struct({ id: S.String, fileName: S.String, diff: S.String, command: S.String }), ) diff --git a/packages/stryker-js/tests/mcp-server.integration.test.ts b/packages/stryker-js/tests/mcp-server.integration.test.ts index ea6669e0..65931e3d 100644 --- a/packages/stryker-js/tests/mcp-server.integration.test.ts +++ b/packages/stryker-js/tests/mcp-server.integration.test.ts @@ -1,8 +1,9 @@ import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Engine, Mcp } from '@systemfsoftware/stryker-js' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import * as Cause from 'effect/Cause' import * as Effect from 'effect/Effect' +import * as Equal from 'effect/Equal' import * as Fiber from 'effect/Fiber' import * as FileSystem from 'effect/FileSystem' import * as Option from 'effect/Option' @@ -45,6 +46,10 @@ const CONFIG = `export default { } ` +const FAILING_CONFIG = CONFIG.replace("command: 'true'", "command: 'false'") + +const FAILURE_FILE = 'reports/mutation/failure.json' + const UTF8 = new TextEncoder() type JsonRpcMessage = typeof JsonRpcResponse.Type @@ -88,18 +93,20 @@ interface Client { ) => Effect.Effect } -const writeWorkspace = (): Effect.Effect => +const writeWorkspaceWith = (config: string): Effect.Effect => Effect.gen(function*() { const fs = yield* FileSystem.FileSystem const path = yield* Path.Path const directory = yield* fs.realPath(yield* fs.makeTempDirectory({ prefix: 'stryker-mcp-' })) yield* fs.writeFileString(path.join(directory, 'package.json'), '{ "name": "mcp-consumer", "type": "module" }\n') - yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), CONFIG) + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), config) yield* fs.makeDirectory(path.join(directory, 'src'), { recursive: true }) yield* fs.writeFileString(path.join(directory, TARGET_FILE), TARGET_SOURCE) return { directory } }).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) +const writeWorkspace = (): Effect.Effect => writeWorkspaceWith(CONFIG) + const removeWorkspace = (directory: string): Effect.Effect => Effect.flatMap(FileSystem.FileSystem, (fs) => fs.remove(directory, { recursive: true, force: true })).pipe( Effect.orDie, @@ -289,6 +296,108 @@ const drive = (directory: string): Effect.Effect, }), ).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) +const rawToolCallOf = ( + response: JsonRpcMessage, +): { readonly failed: boolean; readonly structured: S.Json | undefined; readonly text: string } => + Option.match(toolEnvelopeOf(response), { + onNone: () => ({ failed: true, structured: undefined, text: '' }), + onSome: (call) => ({ + failed: call.isError === true, + structured: call.structuredContent, + text: call.content.map((part) => part.text ?? '').join('\n'), + }), + }) + +const withMcpClient = ( + directory: string, + use: (client: Client) => Effect.Effect, +): Effect.Effect => + Effect.scoped( + Effect.gen(function*() { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner + const handle = yield* spawner.spawn( + ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, 'mcp'], { + cwd: directory, + stdin: 'pipe', + stdout: 'pipe', + stderr: 'pipe', + env: { STRYKER_MODE: 'machine', NO_COLOR: '1' }, + extendEnv: true, + }), + ) + const client = yield* makeClient(handle) + yield* client.initialize + yield* client.notify('notifications/initialized') + return yield* use(client) + }), + ).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) + +const rewriteConfig = (directory: string, config: string): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), config) + }).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) + +interface FailureLookupObserved { + readonly runFailed: boolean + readonly failed: boolean + readonly recordTag: string | null + readonly recordMatchesFile: boolean +} + +const driveFailureLookup = (directory: string): Effect.Effect, never, never> => + withMcpClient(directory, (client) => + Effect.gen(function*() { + const call = rawToolCallOf(yield* client.call(2, 'tools/call', { name: 'get_failure', arguments: {} })) + const lookup = Option.getOrUndefined(S.decodeUnknownOption(Mcp.FailureLookup)(call.structured)) + const fileText = yield* readText(directory, FAILURE_FILE) + const fileRecord = Option.getOrNull(S.decodeOption(FailureRecord.FailureRecordFile)(fileText)) + return { + failed: call.failed, + recordTag: lookup?.failure == null ? null : lookup.failure._tag, + recordMatchesFile: lookup?.failure != null && fileRecord !== null && + Equal.equals(lookup.failure, fileRecord), + } + })) + +interface NoFailureObserved { + readonly failed: boolean + readonly absent: boolean +} + +const driveNoFailureLookup = (directory: string): Effect.Effect => + withMcpClient(directory, (client) => + Effect.gen(function*() { + const call = rawToolCallOf(yield* client.call(2, 'tools/call', { name: 'get_failure', arguments: {} })) + const lookup = Option.getOrUndefined(S.decodeUnknownOption(Mcp.FailureLookup)(call.structured)) + return { failed: call.failed, absent: lookup !== undefined && lookup.failure === null } + })) + +interface RerunFailureObserved { + readonly failed: boolean + readonly recordTag: string | null + readonly exitCode: number | null + readonly stillServing: boolean +} + +const driveRerunFailure = (directory: string): Effect.Effect => + withMcpClient(directory, (client) => + Effect.gen(function*() { + const surfaced = survivorsOf(yield* client.call(2, 'tools/call', { name: 'list_survivors', arguments: {} })) + const id = Option.getOrElse(Option.map(Option.fromNullishOr(surfaced[0]), (ref) => ref.id), () => '') + yield* rewriteConfig(directory, FAILING_CONFIG) + const rerun = rawToolCallOf(yield* client.call(3, 'tools/call', { name: 'rerun_mutant', arguments: { id } })) + const followed = rawToolCallOf(yield* client.call(4, 'tools/call', { name: 'show_mutant', arguments: { id } })) + const refusal = Option.getOrNull(S.decodeOption(S.fromJsonString(Mcp.RerunEngineUnusable))(rerun.text)) + return { + failed: rerun.failed, + recordTag: refusal === null ? null : refusal.record._tag, + exitCode: refusal === null ? null : refusal.code, + stillServing: followed.failed === false, + } + })) + Feature('An agent listing, inspecting and re-running mutants over MCP', { timeout: 180_000 }) .withLayer(Engine.nodePlatformLayer) .live('the built stryker mcp server speaks JSON-RPC over stdio against a project the engine mutated') @@ -346,7 +455,7 @@ Feature('An agent listing, inspecting and re-running mutants over MCP', { timeou }).toEqual({ runCompleted: true, initialized: true, - toolNames: ['list_survivors', 'report_usefulness', 'rerun_mutant', 'show_mutant'], + toolNames: ['get_failure', 'list_survivors', 'report_usefulness', 'rerun_mutant', 'show_mutant'], surfacedAtLeastOne: true, surfacedIdIsInTheReport: true, shownFailed: false, @@ -368,4 +477,84 @@ Feature('An agent listing, inspecting and re-running mutants over MCP', { timeou }), ), ) + + scenario( + 'A client reads the failure record the last failed run wrote', + Gherkin.Do.pipe( + Given('a consumer project whose test command fails')('workspace', () => writeWorkspaceWith(FAILING_CONFIG)), + When('the CLI runs and then an MCP client asks for the failure')( + 'observed', + (s: { readonly workspace: Workspace }) => + Effect.gen(function*() { + const seeded = yield* runStryker(s.workspace.directory, ['run']) + const driven = yield* driveFailureLookup(s.workspace.directory) + return { runFailed: seeded.exitCode === 5, ...driven } + }).pipe(Effect.ensuring(removeWorkspace(s.workspace.directory))), + ), + Then('get_failure returns the record written to reports/mutation/failure.json')( + (s: { readonly observed: FailureLookupObserved }, expect) => + expect({ + runFailed: s.observed.runFailed, + failed: s.observed.failed, + recordTag: s.observed.recordTag, + recordMatchesFile: s.observed.recordMatchesFile, + }).toEqual({ + runFailed: true, + failed: false, + recordTag: 'BaselineTestsFailed', + recordMatchesFile: true, + }), + ), + ), + ) + + scenario( + 'A client asks for the failure when no failed run left a record', + Gherkin.Do.pipe( + Given('a consumer project with no failed run')('workspace', () => writeWorkspaceWith(CONFIG)), + When('an MCP client asks for the failure')( + 'observed', + (s: { readonly workspace: Workspace }) => + driveNoFailureLookup(s.workspace.directory).pipe( + Effect.ensuring(removeWorkspace(s.workspace.directory)), + ), + ), + Then('get_failure answers that there is no failure')( + (s: { readonly observed: NoFailureObserved }, expect) => + expect({ failed: s.observed.failed, absent: s.observed.absent }).toEqual({ failed: false, absent: true }), + ), + ), + ) + + scenario( + 'A rerun whose engine fails returns the refusal with the record and the server keeps serving', + Gherkin.Do.pipe( + Given('a consumer project whose test command kills nothing')('workspace', () => writeWorkspace()), + When('the project is mutated, the test command is broken, and a client re-runs a survivor')( + 'observed', + (s: { readonly workspace: Workspace }) => + Effect.gen(function*() { + const seeded = yield* runStryker(s.workspace.directory, ['run']) + const driven = yield* driveRerunFailure(s.workspace.directory) + return { runCompleted: seeded.exitCode === 0, ...driven } + }).pipe(Effect.ensuring(removeWorkspace(s.workspace.directory))), + ), + Then('rerun_mutant refuses with the record and a later call is answered')( + (s: { readonly observed: RerunFailureObserved & { readonly runCompleted: boolean } }, expect) => + expect({ + runCompleted: s.observed.runCompleted, + failed: s.observed.failed, + recordTag: s.observed.recordTag, + exitCode: s.observed.exitCode, + stillServing: s.observed.stillServing, + }).toEqual({ + runCompleted: true, + failed: true, + recordTag: 'BaselineTestsFailed', + exitCode: 5, + stillServing: true, + }), + ), + ), + ) }) diff --git a/packages/stryker-js/tests/mutation-server.integration.test.ts b/packages/stryker-js/tests/mutation-server.integration.test.ts index dadf7e15..9f238868 100644 --- a/packages/stryker-js/tests/mutation-server.integration.test.ts +++ b/packages/stryker-js/tests/mutation-server.integration.test.ts @@ -1,6 +1,7 @@ import { NodeSocket } from '@effect/platform-node' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Engine, Serve } from '@systemfsoftware/stryker-js' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import * as Cause from 'effect/Cause' import * as Clock from 'effect/Clock' import * as Effect from 'effect/Effect' @@ -64,6 +65,10 @@ const CONFIG = `export default { } ` +const FAILING_CONFIG = CONFIG.replace("command: 'true'", "command: 'false'") + +const INTERNAL_ERROR_CODE = -32603 + const UTF8 = new TextEncoder() const DECODE = new TextDecoder() const CONTENT_LENGTH = 'Content-Length: ' @@ -387,18 +392,20 @@ const everyProgressIsNative = (notifications: ReadonlyArray 0 && notifications.every((notification) => Object.keys(notification.params.files).length > 0) -const writeWorkspace = (): Effect.Effect => +const writeWorkspaceWith = (config: string): Effect.Effect => Effect.gen(function*() { const fs = yield* FileSystem.FileSystem const path = yield* Path.Path const directory = yield* fs.realPath(yield* fs.makeTempDirectory({ prefix: 'stryker-msp-' })) yield* fs.writeFileString(path.join(directory, 'package.json'), '{ "name": "msp-consumer", "type": "module" }\n') - yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), CONFIG) + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), config) yield* fs.makeDirectory(path.join(directory, 'src'), { recursive: true }) yield* fs.writeFileString(path.join(directory, TARGET_FILE), TARGET_SOURCE) return { directory } }).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) +const writeWorkspace = (): Effect.Effect => writeWorkspaceWith(CONFIG) + const removeWorkspace = (directory: string): Effect.Effect => Effect.flatMap(FileSystem.FileSystem, (fs) => fs.remove(directory, { recursive: true, force: true })).pipe( Effect.orDie, @@ -590,6 +597,46 @@ const exposedAddressObserved = (): Effect.Effect => + Effect.scoped(Effect.gen(function*() { + const harness = yield* makeHarness() + const server = yield* Serve.serveMutationServer({ channel: 'stdio' }).pipe( + Effect.provideService(Stdio.Stdio, harness.stdio), + Effect.scoped, + Effect.forkScoped, + ) + const session = yield* sessionOf( + { write: (bytes) => Queue.offer(harness.input, bytes).pipe(Effect.asVoid) }, + yield* frameSourceOf(harness.output), + ) + yield* session.send(1, 'mutationTest', { files: [{ path: TARGET_FILE }] }) + const failed = yield* session.answer(1) + yield* session.send(2, 'configure', {}) + const reconnected = versionOf(yield* session.answer(2)) + yield* Queue.end(harness.input) + yield* Fiber.interrupt(server) + + const record = Option.getOrNull( + Option.flatMap( + Option.fromUndefinedOr(failed.error?.data), + (data) => S.decodeUnknownOption(FailureRecord.FailureRecord)(data), + ), + ) + return { + errorCode: failed.error?.code ?? 0, + failureTag: record === null ? null : record._tag, + failureExitCode: record === null ? null : FailureRecord.FailureCatalog[record._tag].exitCode, + stillServing: reconnected === '0.4.0', + } + })) + Feature('Serving the Mutation Server Protocol over stdio and sockets', { timeout: 180_000 }) .withLayer(Engine.nodePlatformLayer) .live('the mutation server drives the engine for a client that speaks the protocol') @@ -696,4 +743,34 @@ Feature('Serving the Mutation Server Protocol over stdio and sockets', { timeout ), ), ) + + scenario( + 'A mutationTest whose engine fails answers the failure record and the server keeps serving', + Gherkin.Do.pipe( + Given('a consumer project whose test command fails')('workspace', () => writeWorkspaceWith(FAILING_CONFIG)), + When('a client asks for a mutation test over stdio')( + 'observed', + (s: { readonly workspace: Workspace }) => + withDirectory(s.workspace.directory, failingEngineObserved()).pipe( + Effect.ensuring(removeWorkspace(s.workspace.directory)), + ), + ), + Then( + 'the JSON-RPC error carries the same failure record the stream would carry, and a later request is answered', + )( + (s: { readonly observed: FailureObserved }, expect) => + expect({ + errorCode: s.observed.errorCode, + failureTag: s.observed.failureTag, + failureExitCode: s.observed.failureExitCode, + stillServing: s.observed.stillServing, + }).toEqual({ + errorCode: INTERNAL_ERROR_CODE, + failureTag: 'BaselineTestsFailed', + failureExitCode: 5, + stillServing: true, + }), + ), + ), + ) }) diff --git a/packages/stryker-js/tests/sarif-reporter.integration.test.ts b/packages/stryker-js/tests/sarif-reporter.integration.test.ts index 1c05793d..c89fe02f 100644 --- a/packages/stryker-js/tests/sarif-reporter.integration.test.ts +++ b/packages/stryker-js/tests/sarif-reporter.integration.test.ts @@ -20,7 +20,7 @@ import * as S from 'effect/Schema' import * as Scope from 'effect/Scope' import * as Stream from 'effect/Stream' -import { ReproducerList, SarifDocument } from './__fixtures__/sarif-reporter.schema.js' +import { FailureSarifDocument, ReproducerList, SarifDocument } from './__fixtures__/sarif-reporter.schema.js' const Feature = makeFeature({ it }) @@ -235,15 +235,15 @@ interface AnnotateRun { readonly stderr: string } -const runAnnotate = ( +const spawnStryker = ( root: string, - args: ReadonlyArray, + argv: ReadonlyArray, ): Effect.Effect => Effect.scoped( Effect.gen(function*() { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner const handle = yield* spawner.spawn( - ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, 'annotate', ...args], { + ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, ...argv], { cwd: root, stdin: 'ignore', stdout: 'pipe', @@ -260,9 +260,42 @@ const runAnnotate = ( }), ).pipe(Effect.orDie) +const runAnnotate = ( + root: string, + args: ReadonlyArray, +): Effect.Effect => + spawnStryker(root, ['annotate', ...args]) + const annotationLinesOf = (stdout: string): ReadonlyArray => Arr.filter(stdout.split('\n'), (line) => line.startsWith('::')) +const FAILING_CONSUMER_CONFIG = `export default { + testRunner: 'command', + commandRunner: { command: 'false' }, + mutate: ['src/**/*.ts'], + coverageAnalysis: 'off', + concurrency: 1, + reporters: ['json', 'sarif'], +} +` + +const FAILING_TARGET_SOURCE = 'export const target = (value: number): number => value + 1\n' + +const writeFailingWorkspace = (): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const directory = yield* fs.makeTempDirectory({ prefix: 'stryker-sarif-fail-' }) + yield* fs.makeDirectory(path.join(directory, 'src'), { recursive: true }) + yield* fs.writeFileString(path.join(directory, 'package.json'), '{ "type": "commonjs" }\n') + yield* fs.writeFileString(path.join(directory, 'src', 'target.ts'), FAILING_TARGET_SOURCE) + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), FAILING_CONSUMER_CONFIG) + return { directory } + }).pipe(Effect.orDie) + +const parseFailureSarif = (text: string | undefined): Option.Option => + text === undefined ? Option.none() : Result.getSuccess(S.decodeResult(S.fromJsonString(FailureSarifDocument))(text)) + Feature('Survivors reaching code scanning, annotations, and reproducers', { timeout: 180_000 }) .withLayer(Engine.nodePlatformLayer) .live('the run drives the real engine in process and writes the report, the SARIF log, and the sidecar') @@ -378,4 +411,54 @@ Feature('Survivors reaching code scanning, annotations, and reproducers', { time ), ), ) + scenario( + 'A failed run with the sarif reporter writes a failed invocation with one notification for its record', + Gherkin.Do.pipe( + Given('a workspace whose test command fails the baseline')( + 'workspace', + () => writeFailingWorkspace().pipe(Effect.provide(filePorts)), + ), + When('stryker run executes with the json and sarif reporters')( + 'observation', + (s) => + Effect.gen(function*() { + const run = yield* spawnStryker(s.workspace.directory, ['run']) + const sarifText = yield* FileSystem.FileSystem.pipe( + Effect.flatMap((fs) => fs.readFileString(sarifFileOf(s.workspace.directory)).pipe(Effect.option)), + Effect.provide(filePorts), + ) + return { run, sarif: Option.getOrUndefined(Option.flatMap(sarifText, parseFailureSarif)) } + }).pipe( + Effect.ensuring(removeWorkspace(s.workspace.directory).pipe(Effect.provide(filePorts))), + ), + ), + Then('the SARIF invocation is failed at exit 5 and notifies the record')((s, expect) => { + const sarif = s.observation.sarif + const run = sarif === undefined ? undefined : Option.getOrUndefined(Arr.head(sarif.runs)) + const invocation = run === undefined ? undefined : Option.getOrUndefined(Arr.head(run.invocations)) + const notification = invocation === undefined + ? undefined + : Option.getOrUndefined(Arr.head(invocation.toolExecutionNotifications)) + return expect({ + exitCode: s.observation.run.exitCode, + version: sarif?.version, + executionSuccessful: invocation?.executionSuccessful, + invocationExitCode: invocation?.exitCode, + notificationCount: invocation?.toolExecutionNotifications.length, + descriptorId: notification?.descriptor.id, + level: notification?.level, + textNamesTheRecord: notification?.message.text.includes('BaselineTestsFailed'), + }).toEqual({ + exitCode: 5, + version: '2.1.0', + executionSuccessful: false, + invocationExitCode: 5, + notificationCount: 1, + descriptorId: 'BaselineTestsFailed', + level: 'error', + textNamesTheRecord: true, + }) + }), + ), + ) }) diff --git a/packages/stryker-js/tests/trace-propagation.integration.test.ts b/packages/stryker-js/tests/trace-propagation.integration.test.ts index f9b9b66c..094345b3 100644 --- a/packages/stryker-js/tests/trace-propagation.integration.test.ts +++ b/packages/stryker-js/tests/trace-propagation.integration.test.ts @@ -1,16 +1,32 @@ import * as NodeSdk from '@effect/opentelemetry/NodeSdk' +import { NodeHttpServer } from '@effect/platform-node' import { InMemorySpanExporter, type ReadableSpan, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' -import { Plugin } from '@systemfsoftware/stryker-js' +import { Engine, Plugin } from '@systemfsoftware/stryker-js' +import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { Options, Trace } from '@systemfsoftware/stryker-js-plugin-interface' import { Trace as RuntimeTrace } from '@systemfsoftware/stryker-js-plugin-runtime' +import * as Arr from 'effect/Array' import * as Effect from 'effect/Effect' +import * as Fiber from 'effect/Fiber' +import * as FileSystem from 'effect/FileSystem' import * as Headers from 'effect/http/Headers' -import * as Layer from 'effect/Layer' +import * as HttpServer from 'effect/http/HttpServer' +import * as HttpServerRequest from 'effect/http/HttpServerRequest' +import * as HttpServerResponse from 'effect/http/HttpServerResponse' +import * as Match from 'effect/Match' +import type * as NetAddress from 'effect/net/NetAddress' import * as Option from 'effect/Option' +import * as Path from 'effect/Path' +import * as Predicate from 'effect/Predicate' +import * as ChildProcess from 'effect/process/ChildProcess' +import * as ChildProcessSpawner from 'effect/process/ChildProcessSpawner' import * as Ref from 'effect/Ref' import * as S from 'effect/Schema' +import * as Scope from 'effect/Scope' +import * as Stream from 'effect/Stream' +import { OtlpPayload, type OtlpSpan } from './__fixtures__/otlp-spans.schema.js' import { makeTraceWorkerRecord, TRACE_WORKER_ENTRYPOINT, @@ -44,8 +60,10 @@ const PLAN: TraceWorkerPlan = { entrypoint: TRACE_WORKER_ENTRYPOINT, projectBase const FUTURE_TRACEPARENT = '01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01-extra' -const spanNamed = (spans: readonly ReadableSpan[], name: string): ReadableSpan | undefined => - spans.find((span) => span.name === name) +const spanNamed = ( + spans: readonly Span[], + name: string, +): Span | undefined => spans.find((span) => span.name === name) const makeClient = (plan: TraceWorkerPlan) => Effect.gen(function*() { @@ -114,10 +132,187 @@ const runCarriedCall = (plan: TraceWorkerPlan): Effect.Effect => { ).pipe(Effect.provide(telemetry), Effect.orDie) } -Feature('Linking a worker into the host run trace') - .withLayer(Layer.empty) +const PACKAGE_ROOT = decodeURIComponent(new URL('..', import.meta.url).pathname) +const STRYKER_BIN = decodeURIComponent(new URL('../dist/main.mjs', import.meta.url).pathname) + +const FAILURE_RECORD_FILE = 'reports/mutation/failure.json' +const FAILING_TEST_NAME = 'rejects a wrong sum' +const CONSUMER_PACKAGE = '{ "name": "trace-failure-consumer", "type": "module", "private": true }\n' +const SOURCE_CONTENT = 'export const add = (left: number, right: number): number => left + right\n' +const CONSUMER_CONFIG = [ + 'export default {', + " testRunner: 'vm',", + " testFiles: ['test/**/*.mjs'],", + " mutate: ['src/**/*.ts'],", + ' reporters: [],', + ' checkers: [],', + '}', + '', +].join('\n') +const TEST_CONTENT = [ + "import { expect, test } from 'vitest'", + "import { add } from '../src/math.ts'", + '', + `test('${FAILING_TEST_NAME}', () => {`, + ' expect(add(1, 1)).toBe(3)', + '})', + '', +].join('\n') + +const arrayOf = (value: readonly A[] | undefined): readonly A[] => + Option.getOrElse(Option.fromNullishOr(value), () => []) + +const spansOfPayload = (payload: OtlpPayload): readonly OtlpSpan[] => + Arr.flatMap( + arrayOf(payload.resourceSpans), + (resource) => Arr.flatMap(arrayOf(resource.scopeSpans), (scope) => arrayOf(scope.spans)), + ) + +const spansOfBody = (body: Option.Option): readonly OtlpSpan[] => + Option.match(body, { onNone: () => [], onSome: spansOfPayload }) + +const stringAttributeOf = (span: OtlpSpan, key: string): string | undefined => + Option.getOrUndefined( + Option.map( + Arr.findFirst(span.attributes, (attribute) => attribute.key === key), + (attribute) => attribute.value.stringValue, + ), + ) + +const failureCodeOf = (spans: readonly OtlpSpan[]): string | undefined => + Option.match(Option.fromNullishOr(spanNamed(spans, SpanTaxonomy.Spans.cliRun.name)), { + onNone: () => undefined, + onSome: (span) => stringAttributeOf(span, 'stryker.failure.code'), + }) + +const hasRunSpan = (spans: readonly OtlpSpan[]): boolean => + spanNamed(spans, SpanTaxonomy.Spans.cliRun.name) !== undefined + +const traceIdMatchesSpan = (spans: readonly OtlpSpan[], traceId: string | null): boolean => + Option.match(Option.fromNullishOr(spanNamed(spans, SpanTaxonomy.Spans.cliRun.name)), { + onNone: () => false, + onSome: (span) => traceId !== null && traceId === span.traceId, + }) + +const portOfAddress = (address: NetAddress.SocketAddress): number | undefined => + Match.value(address).pipe( + Match.tag('InetAddressV4', (inet) => inet.port), + Match.tag('InetAddressV6', (inet) => inet.port), + Match.tag('UnixPathAddress', () => undefined), + Match.exhaustive, + ) + +const endpointOf = (address: NetAddress.SocketAddress): Effect.Effect => + Option.match(Option.fromNullishOr(portOfAddress(address)), { + onNone: () => Effect.die(new Error('the OTLP collector must bind a TCP port')), + onSome: (port) => Effect.succeed(`http://127.0.0.1:${port}`), + }) + +interface SpanCollector { + readonly endpoint: string + readonly spans: Effect.Effect +} + +const startSpanCollector = (): Effect.Effect => + Effect.gen(function*() { + const server = yield* HttpServer.HttpServer + const collected: OtlpSpan[] = [] + const handler = Effect.gen(function*() { + const body = yield* OtlpPayload.pipe(HttpServerRequest.schemaBodyJson, Effect.option) + collected.push(...spansOfBody(body)) + return HttpServerResponse.empty() + }) + yield* server.serve(handler) + const endpoint = yield* endpointOf(server.address) + return { endpoint, spans: Effect.sync(() => [...collected]) } + }) + +const writeFixture = (): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const binaryPresent = yield* fs.exists(STRYKER_BIN) + yield* Effect.when( + Effect.die(new Error('dist/main.mjs is missing — build the package first')), + Effect.succeed(!binaryPresent), + ) + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped({ prefix: 'stryker-trace-failure-' })) + yield* fs.makeDirectory(path.join(root, 'node_modules', '@systemfsoftware'), { recursive: true }) + yield* fs.symlink(PACKAGE_ROOT, path.join(root, 'node_modules', '@systemfsoftware', 'stryker-js')) + yield* fs.makeDirectory(path.join(root, 'src'), { recursive: true }) + yield* fs.makeDirectory(path.join(root, 'test'), { recursive: true }) + yield* fs.writeFileString(path.join(root, 'package.json'), CONSUMER_PACKAGE) + yield* fs.writeFileString(path.join(root, 'src', 'math.ts'), SOURCE_CONTENT) + yield* fs.writeFileString(path.join(root, 'test', 'math.test.mjs'), TEST_CONTENT) + yield* fs.writeFileString(path.join(root, 'stryker.config.mjs'), CONSUMER_CONFIG) + return root + }).pipe(Effect.orDie) + +const runStrykerFrom = ( + root: string, + endpoint: string, + otelEnabled: boolean, +): Effect.Effect => + Effect.scoped( + Effect.gen(function*() { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner + const handle = yield* spawner.spawn( + ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, 'run'], { + cwd: root, + stdin: 'ignore', + stdout: 'pipe', + stderr: 'pipe', + env: { + STRYKER_MODE: 'machine', + NO_COLOR: '1', + OTEL_ENABLED: String(otelEnabled), + OTEL_EXPORTER_OTLP_ENDPOINT: endpoint, + }, + extendEnv: true, + }), + ) + const drainOut = yield* Effect.forkScoped(handle.stdout.pipe(Stream.decodeText, Stream.mkString)) + const drainErr = yield* Effect.forkScoped(handle.stderr.pipe(Stream.decodeText, Stream.mkString)) + const exitCode = yield* handle.exitCode + yield* Effect.all([Fiber.join(drainOut), Fiber.join(drainErr)], { discard: true }) + return Number(exitCode) + }), + ).pipe(Effect.orDie) + +const readFailureRecord = ( + root: string, +): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const text = yield* fs.readFileString(path.join(root, FAILURE_RECORD_FILE)) + return yield* S.decodeEffect(FailureRecord.FailureRecordFile)(text) + }).pipe(Effect.orDie) + +interface ObservedRun { + readonly exitCode: number + readonly record: FailureRecord.FailureRecord + readonly spans: readonly OtlpSpan[] +} + +const observeFailingRun = ( + root: string, + otelEnabled: boolean, +): Effect.Effect => + Effect.scoped( + Effect.gen(function*() { + const collector = yield* startSpanCollector() + const exitCode = yield* runStrykerFrom(root, collector.endpoint, otelEnabled) + const record = yield* readFailureRecord(root) + const spans = yield* collector.spans + return { exitCode, record, spans } + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ).pipe(Effect.orDie) + +Feature('Tracing a run, the work it starts, and the failure it ends with', { timeout: 180_000 }) + .withLayer(Engine.nodePlatformLayer) .live( - 'the OpenTelemetry span exporter schedules its export on a real setTimeout timer, which leaves the controlled schedule', + 'the OpenTelemetry exporter schedules its export on a real setTimeout timer, and the failing-run scenarios run the built stryker binary in a real Node process against a socket collector', ) .body(({ scenario }) => { scenario( @@ -170,4 +365,37 @@ Feature('Linking a worker into the host run trace') }), ), ) + + scenario( + 'A failing dry run stamps the run span with the record code and writes the span trace id into the record', + Gherkin.Do.pipe( + Given('a project whose test suite fails')('root', () => writeFixture()), + When('the run executes with OpenTelemetry enabled')('observed', (s) => observeFailingRun(s.root, true)), + Then( + 'the exported run span carries the failure code and the record trace id equals the span trace id', + )((s, expect) => + expect({ + exitCode: s.observed.exitCode, + failureCode: failureCodeOf(s.observed.spans), + traceIdMatchesExportedSpan: traceIdMatchesSpan(s.observed.spans, s.observed.record.traceId), + }).toEqual({ exitCode: 5, failureCode: 'BaselineTestsFailed', traceIdMatchesExportedSpan: true }) + ), + ), + ) + + scenario( + 'A failing dry run with OpenTelemetry disabled records no trace id and exports nothing', + Gherkin.Do.pipe( + Given('a project whose test suite fails')('root', () => writeFixture()), + When('the run executes with OpenTelemetry disabled')('observed', (s) => observeFailingRun(s.root, false)), + Then('the record has a null trace id and no run span was exported')((s, expect) => + expect({ + exitCode: s.observed.exitCode, + failureIsBaseline: Predicate.isTagged(s.observed.record, 'BaselineTestsFailed'), + hasRunSpan: hasRunSpan(s.observed.spans), + recordTraceId: s.observed.record.traceId, + }).toEqual({ exitCode: 5, failureIsBaseline: true, hasRunSpan: false, recordTraceId: null }) + ), + ), + ) }) From 8181b5651d5f551a0755b34c4e31375d43e83c7b Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:45 -0400 Subject: [PATCH 6/8] ci(repo): render mutation job failures from the failure record (#151) * ci(repo): render mutation job failures from the failure record The mutation job decodes the stream's terminal record, renders the summary and annotations through the contract renderers, builds RecordMissing, JobTimedOut and BinaryMissing records itself, reports a run that evaluated no mutants as exactly that, and no longer calls a failed run an infrastructure failure Verdict-Semantics: unchanged * ci(repo): keep a failed outcome visible when a run reused every verdict The evaluated-none label replaced the stryker outcome, so a verdict-failed run served entirely from cache read as evaluated no mutants instead of failure Verdict-Semantics: unchanged * docs(solutions): a failure record is the only rendering of a failure --- .changeset/proud-bananas-work.md | 5 + ...cord-is-the-only-rendering-of-a-failure.md | 79 ++++ .../src/render-annotations.workflow.ts | 20 +- packages/stryker-js/src/run-request.cell.ts | 20 + scripts/deno.json | 6 +- scripts/deno.lock | 366 +++++++++++++++++- scripts/lib/mutation-plan.test.ts | 92 ++++- scripts/lib/mutation-plan.ts | 153 +++++--- scripts/mutation-job.ts | 2 +- 9 files changed, 670 insertions(+), 73 deletions(-) create mode 100644 .changeset/proud-bananas-work.md create mode 100644 docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md diff --git a/.changeset/proud-bananas-work.md b/.changeset/proud-bananas-work.md new file mode 100644 index 00000000..826744cf --- /dev/null +++ b/.changeset/proud-bananas-work.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +stryker annotate also renders the GitHub annotations of reports/mutation/failure.json when a run failed. diff --git a/docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md b/docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md new file mode 100644 index 00000000..86ace7c3 --- /dev/null +++ b/docs/solutions/best-practices/failure-record-is-the-only-rendering-of-a-failure.md @@ -0,0 +1,79 @@ +--- +title: A failure record is the only rendering of a failure; nothing next to it may restate it +date: 2026-10-02 +category: best-practices +module: stryker-js failure reporting +problem_type: best_practice +component: tooling +severity: high +applies_when: + - adding a new failure path that raises a `RunFailure` or a tagged error with an `evidence` getter + - writing the `message` of an error class that also declares evidence + - adding stderr logging, CI summary text or annotations about a failed run + - asserting failure output in tests or e2e journeys +symptoms: + - "A gate refusal printed the new survivor id four times on stderr: a logError line, the record's evidence, the carrier's message and the carrier's stack header" + - "A failed dry run printed `Initial test run failed ...` from a logError before the record said the same thing" + - "A dry-run cause link dumped the whole `DryRunFailed` decision object, stack included, through the Formatter fallback" + - "The CI summary called a vacuous-property dry-run failure an infrastructure failure" +tags: + - failure-record + - diagnostics + - stderr + - cause-chain +--- + +# A failure record is the only rendering of a failure; nothing next to it may restate it + +## Context + +A failed run ends in one `FailureRecord.FailureRecord` from `@systemfsoftware/stryker-js-cli-contract`. That record is printed on stderr through `terminalTextOf`, emitted as the schema-3.0 `error` stream event, written to the run's `failure.json` under its mutation reports directory (`FAILURE_RECORD_FILE`), and turned into the CI summary and annotations by `markdownOf`/`annotationsOf` (`render-failure.ts`). Stack PRs #146-#151 introduced it after mutation run 36935602456 reported a vacuous in-source property in the dry run as an "infrastructure failure (missing binary, crashed run or timeout)". + +The cutover worked on the first pass but still produced noisy, contradictory output, because old renderings survived next to the record. + +## Guidance + +1. **No log line about a failure the record carries.** A cell that fails with a `RunFailure` must not also `Effect.logError` the same fact. The removed offenders were `explainGateRefusal` in `run-request.cell.ts` and the `Initial test run failed. N of M test(s) failed` log in `dry-run.cell.ts`. Integration tests that captured those logs were pinning a duplicate; assert on the evidence instead. +2. **An error that declares evidence keeps a one-line summary `message`.** The record renders the evidence. If the message lists the same data, every cause link repeats it. `GateRejected.message` is now `stryker gate: N new survivor(s) absent from the committed baseline`, and the survivor list lives only in `NewSurvivors.survivors`. +3. **`RunFailure.cause` holds a real error, never the decision data.** `writeDryRunFailed` used to pass the `DryRunFailed` decision as the cause. `messageOf` in `conclude-run.ts` falls back to `Formatter.format` for non-Error objects, so the whole decision, stacks included, landed in the cause chain. +4. **A cause link's stack keeps only its `at` frames.** A V8 stack starts with `Name: message`, so storing it whole prints the message twice. `linkOf` in `conclude-run.ts` drops the lines before the first `at` frame. +5. **Stacks show on the terminal only when nothing else locates the failure.** `terminalTextOf` and `markdownOf` print cause stacks only for `CatalogGap` records, and a failed test's stack only when it has no `location`. `failure.json` keeps every stack. +6. **Paths in evidence are project-relative.** The runner reports paths inside `.stryker-tmp/sandbox-*`, which is deleted after the run. `projectTestOf` in `dry-run.cell.ts` relativizes the test `file` and strips the sandbox prefix from the stack. The Vitest runner names file-level failures by their project-relative path. +7. **CI text comes only from the record.** The mutation job's `buildSummary` and `buildRequireError` (`mutation-plan.ts`) render the stream's terminal record, or build `RecordMissing`, `JobTimedOut` or `BinaryMissing` through `recordFor` when no record exists. A summary label must never replace the outcome: "evaluated no mutants" is appended to "failure", never shown instead of it. + +## Why This Matters + +Each duplicate is a second, drifting description of the same failure. An agent reading stderr cannot tell which line is authoritative. Duplicated or missing data also breaks the exact-count assertions that consumers write: the gate test counts survivor ids, and annotations must cover exactly the located evidence. And output that claims a cause the run never had sends the reader to fix the wrong thing. + +## When to Apply + +- Any new code path that ends a run in failure, or any new error class with an `evidence` getter. +- Any change to `render-failure.ts`, `conclude-run.ts` (`linkOf`, `messageOf`), or the CI scripts' summary. + +## Examples + +Gate refusal on stderr, before (one id, four times): + +```text +ERROR (#1): stryker gate: 1 new survivor(s) absent from the committed baseline: + src/sum.js:4 4d4d4d4d4d4d4d4d +... +GateRejected: stryker gate: 1 new survivor(s) absent from the committed baseline: + src/sum.js:4 4d4d4d4d4d4d4d4d + GateRejected: stryker gate: ... <- stack header repeating the message +``` + +After: + +```text +NewSurvivors: Mutants the change can affect survived every test and are absent from the accepted survivor baseline. +survivors: 1 listed, 2 unchecked +src/sum.js:4 4d4d4d4d4d4d4d4d +GateRejected: stryker gate: 1 new survivor(s) absent from the committed baseline +next: killSurvivor +``` + +## Related + +- `docs/plans/2026-10-01-2306-feat-agent-ready-failure-diagnostics-plan.md` +- `docs/solutions/test-failures/stream-schema-must-not-carry-json-rest-records.md` diff --git a/packages/stryker-js/src/render-annotations.workflow.ts b/packages/stryker-js/src/render-annotations.workflow.ts index cf2a199c..f51e3099 100644 --- a/packages/stryker-js/src/render-annotations.workflow.ts +++ b/packages/stryker-js/src/render-annotations.workflow.ts @@ -59,6 +59,7 @@ export class RenderAnnotationsCommand extends S.TaggedClass ].join(',') }::${escapeMessage(messageOf(entry))}` -const linesOf = (command: RenderAnnotationsCommand): ReadonlyArray => { - const byId = entriesByIdOf(command.report) - const baseline = HashSet.fromIterable(command.baseline) +const survivorLinesOf = ( + report: Report.MutationTestResult, + survivors: ReadonlyArray, + baseline: ReadonlyArray, +): ReadonlyArray => { + const byId = entriesByIdOf(report) + const baselineSet = HashSet.fromIterable(baseline) return Arr.flatMap( - command.survivors, + survivors, (ref) => - Boolean.match(HashSet.has(baseline, ref.id), { + Boolean.match(HashSet.has(baselineSet, ref.id), { onTrue: (): ReadonlyArray => [], onFalse: () => Arr.flatMap(Option.toArray(HashMap.get(byId, ref.id)), (entry) => [lineOf(entry)]), }), ) } +const linesOf = (command: RenderAnnotationsCommand): ReadonlyArray => [ + ...survivorLinesOf(command.report, command.survivors, command.baseline), + ...Option.fromUndefinedOr(command.failureAnnotations).pipe(Option.getOrElse((): ReadonlyArray => [])), +] + const decide = (command: RenderAnnotationsCommand): RenderAnnotationsDecision => { const lines = linesOf(command) return Boolean.match(Arr.length(lines) === 0, { diff --git a/packages/stryker-js/src/run-request.cell.ts b/packages/stryker-js/src/run-request.cell.ts index 037e6910..7056a545 100644 --- a/packages/stryker-js/src/run-request.cell.ts +++ b/packages/stryker-js/src/run-request.cell.ts @@ -1,6 +1,7 @@ import { Cell, Sandwich } from '@systemfsoftware/effect-cell-types' import { SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, Report } from '@systemfsoftware/stryker-js-plugin-interface' import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import * as CliError from 'effect/cli/CliError' @@ -61,6 +62,7 @@ import { mutationTestCell } from './run/run-stages.cell.js' import { RunEnvironment } from './run/RunEnvironment.service.js' import { serveMutationServer, type ServeRequest } from './Serve/Serve.cell.js' import { StrykerError } from './stryker-error.schema.js' +import { FAILURE_RECORD_FILE } from './stryker-outputs.js' import { annotationLinesOf, surfacedSurvivorsOf } from './surfacing.js' import { type SurfacingCaps, SurfacingFields } from './surfacing.schema.js' import type { SurvivorsAdmissionInput, SurvivorsSettlement } from './Survivors/mod.js' @@ -385,6 +387,7 @@ const surfacingCapsOf = (report: Report.MutationTestResult): SurfacingCaps => Option.getOrElse(Option.map(surfacingFieldsOf(report), capsOf), () => SURFACING_DEFAULTS) const decodeAnnotateReport = S.decodeUnknownResult(S.fromJsonString(Report.MutationTestResult)) +const decodeAnnotateFailure = S.decodeUnknownOption(FailureRecord.FailureRecordFile) const readAnnotateReport = ( file: string, @@ -409,6 +412,21 @@ const readAnnotateReport = ( ), )) +const readAnnotateFailureAnnotations = ( + file: string, +): Effect.Effect, never, FileSystem.FileSystem> => + Effect.flatMap(FileSystem.FileSystem, (fs) => + fs.readFileString(file).pipe( + Effect.asSome, + Effect.catchTag('PlatformError', () => Effect.succeed(Option.none())), + Effect.map((text) => + Option.flatMap(text, decodeAnnotateFailure).pipe( + Option.map(FailureRecord.annotationsOf), + Option.getOrElse((): ReadonlyArray => []), + ) + ), + )) + const readAnnotateBaseline = ( file: string, ): Effect.Effect, AnnotationsUnusable, FileSystem.FileSystem> => @@ -435,6 +453,7 @@ const annotateReport = ( const path = yield* Path.Path const basePath = channel.environment.basePath const report = yield* readAnnotateReport(path.resolve(basePath, GATE_REPORT_FILE)) + const failureAnnotations = yield* readAnnotateFailureAnnotations(path.resolve(basePath, FAILURE_RECORD_FILE)) const baseline = yield* Effect.forEach( Option.toArray(Option.fromUndefinedOr(annotate.baseline)), (file) => readAnnotateBaseline(path.resolve(basePath, file)), @@ -445,6 +464,7 @@ const annotateReport = ( report, survivors: surfacedSurvivorsOf(report, surfacingCapsOf(report)), baseline: baseline.flat(), + failureAnnotations, }), ), ) diff --git a/scripts/deno.json b/scripts/deno.json index a56f64f5..1543a704 100644 --- a/scripts/deno.json +++ b/scripts/deno.json @@ -1,8 +1,10 @@ { "compilerOptions": { "strict": true, - "noImplicitOverride": true + "noImplicitOverride": true, + "types": ["vitest/importMeta"] }, + "unstable": ["sloppy-imports"], "minimumDependencyAge": { "age": "P1D", "exclude": ["npm:effect"] @@ -19,6 +21,8 @@ "@std/fs/expand-glob": "jsr:@std/fs@1/expand-glob", "@std/path": "jsr:@std/path@1", "@std/yaml": "jsr:@std/yaml@1", + "@systemfsoftware/vitest": "npm:@systemfsoftware/vitest@^1.0.0", + "vitest": "npm:vitest@^5", "effect": "npm:effect@^4.0.0", "octokit": "npm:octokit@^4" } diff --git a/scripts/deno.lock b/scripts/deno.lock index 2ff9f8d3..dfdcd562 100644 --- a/scripts/deno.lock +++ b/scripts/deno.lock @@ -17,8 +17,10 @@ "jsr:@std/toml@^1.0.3": "1.0.11", "jsr:@std/yaml@1": "1.2.0", "jsr:@std/yaml@^1.0.5": "1.2.0", + "npm:@systemfsoftware/vitest@1": "1.0.0_effect@4.0.0_vitest@5.0.1", "npm:effect@4": "4.0.0", - "npm:octokit@4": "4.1.4" + "npm:octokit@4": "4.1.4", + "npm:vitest@5": "5.0.1_vite@8.3.1" }, "jsr": { "@std/assert@1.0.19": { @@ -73,6 +75,19 @@ } }, "npm": { + "@jridgewell/resolve-uri@3.1.2": { + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==" + }, + "@jridgewell/sourcemap-codec@1.6.0": { + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==" + }, + "@jridgewell/trace-mapping@0.3.31": { + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dependencies": [ + "@jridgewell/resolve-uri", + "@jridgewell/sourcemap-codec" + ] + }, "@octokit/app@15.1.6": { "integrity": "sha512-WELCamoCJo9SN0lf3SWZccf68CF0sBNPQuLYmZ/n87p5qvBJDe9aBtr5dHkh7T9nxWZ608pizwsUbypSzZAiUw==", "dependencies": [ @@ -265,21 +280,260 @@ "@octokit/webhooks-methods" ] }, + "@oxc-project/types@0.152.0": { + "integrity": "sha512-oM/5rLBm2tPkg0iBgkH/FOeR3PCDpY19GTgAZjMFM8h9WI9VW7cLgzp6nwtarYKmovavIQZ+Fe/RKX/8C8O/Rw==" + }, + "@rolldown/binding-android-arm-eabi@1.2.12": { + "integrity": "sha512-dB/a1214qKfHMXCpgqR4OZT+jS4kTyEXbQGJPqzobt5EwH5rX080pxE37alt3RzvR1bf1Yz/yGqRfrYAxuPw0A==", + "os": ["android"], + "cpu": ["arm"] + }, + "@rolldown/binding-android-arm64@1.2.12": { + "integrity": "sha512-7KHFgQ5VJxIHcLlrwrc3Xbds7oTNQT7Pgi9gQCJKrd2VGab/UksIOYp6VD8MzCstGxOKMgNamPwUCfxPdP1OHg==", + "os": ["android"], + "cpu": ["arm64"] + }, + "@rolldown/binding-darwin-arm64@1.2.12": { + "integrity": "sha512-3YIhqHD96nA5SaYNRBR16HnGv4oavZvXfD/ayHM+oYZ0WD/8lBAtf6zQua4kEyAvpqrluKXl0lnOBoiNby7x9w==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "@rolldown/binding-darwin-x64@1.2.12": { + "integrity": "sha512-UuuJ35MFw4gmFOrE9pEqIV+K3syIKveph+Qc1/ljHZVdoDW4pz/JHR/eMVom+TZGl/5OOvGJOWaOCVt3ZfqhxA==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "@rolldown/binding-freebsd-x64@1.2.12": { + "integrity": "sha512-uMvssit0a4W+/7D8CbHUvG719mH3R2jwXAlh/XcPvuHTE0g++LymF88DCGNX0HM2rBOn0xrzgXktIB6fLSJBTQ==", + "os": ["freebsd"], + "cpu": ["x64"] + }, + "@rolldown/binding-linux-arm-gnueabihf@1.2.12": { + "integrity": "sha512-XcFu0R0xWnwzSf4IQgFH1rJIckPN1pLy2R+4r9IDB7Yfu/ys9cVqfa4pBrMHj7a3gl8mIR4nRNPg0e5IvEVs6g==", + "os": ["linux"], + "cpu": ["arm"] + }, + "@rolldown/binding-linux-arm64-gnu@1.2.12": { + "integrity": "sha512-260UrKgn8tz39ak+SMDOirKzr7V04M9dWPw5llW00SwBivCZoWcRBKV1d8cXnRkUmSZA3BdiUmBHWk7734Ulpw==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@rolldown/binding-linux-arm64-musl@1.2.12": { + "integrity": "sha512-5YK1I9SqDkbPgc1IA8BgDl34suqUS2q0KWnBrirm0E51YjOs6eo6dV6jbQfNE/argHRSvd0QUGgtpIoYx+WWpw==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@rolldown/binding-linux-ppc64-gnu@1.2.12": { + "integrity": "sha512-Rkcrmp7eFRg74yL5fXEU91JEWbdEPLevWwGtXpmhbjlD1StScbWTmO94Bhly+Mo+ketKYkdmM1vNUKeWSlx8cQ==", + "os": ["linux"], + "cpu": ["ppc64"] + }, + "@rolldown/binding-linux-s390x-gnu@1.2.12": { + "integrity": "sha512-qvK4DuAsQc2BSjlx+Xr+IzOIvvxbGZqxFwdWfG6F518Erj0GGISyQbJ6pIappnOxlNPzNHvo/L0BwB30GZ+zVw==", + "os": ["linux"], + "cpu": ["s390x"] + }, + "@rolldown/binding-linux-x64-gnu@1.2.12": { + "integrity": "sha512-Q9uLBO53Xd4QIq1WOycVQyPP1O4HhraEV2qqb3uTrnVw6QZih9duY4vNXOivL1xoUS1/z+W8eF4NMfl2a8Sdjw==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@rolldown/binding-linux-x64-musl@1.2.12": { + "integrity": "sha512-3IBxWFMjbOZskDPKv8Lf9BCnahlKuHthWkYnyIxOH/QcJrFcS4EmcenthApkwr/5+nEqZlLzeYbxeMaX7A5u4g==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@rolldown/binding-openharmony-arm64@1.2.12": { + "integrity": "sha512-xtX61xg4LKPkPWilZU1ynKClz5Gj4bf74LML4r3eVLWumKnGjoEr1OSHQhMdbBDoYTi+yjrujvpZe2pUnqCrrA==", + "os": ["openharmony"], + "cpu": ["arm64"] + }, + "@rolldown/binding-win32-arm64-msvc@1.2.12": { + "integrity": "sha512-At7fPB6PCaIjzgIhEZFxuT+BBFqiQibJDT4d3PhiR3f4E7bbMZF4aKblbFfEM3sETRDd1YiQx/+U/g/B/ou5Ew==", + "os": ["win32"], + "cpu": ["arm64"] + }, + "@rolldown/binding-win32-x64-msvc@1.2.12": { + "integrity": "sha512-WIw2haVKwjuYdXkHaoC0mF8Le71TuCBxjrdKqLbJGctbBABj+ClfmNvtbOnzpq3RokNo5+V1qhtSzJyXorsklQ==", + "os": ["win32"], + "cpu": ["x64"] + }, + "@rolldown/pluginutils@1.0.1": { + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==" + }, + "@systemfsoftware/vitest@1.0.0_effect@4.0.0_vitest@5.0.1": { + "integrity": "sha512-B3v+wlfcAZwG6HdHuDC4pVMi9O9c5Zvrr8pcCBxtS9zVypvWDAvW8f0iXzlRMSQHaH04GOiX9uv0S4HkMPHnqA==", + "dependencies": [ + "effect", + "vitest" + ] + }, "@types/aws-lambda@8.10.163": { "integrity": "sha512-+4zuoEB3S8RIhimtOFT7zAEk2SbpwrKjjGl9CyYnQR6k08uynxfauIIB0pDU1ssr05F8oyGiETwpn+8eXZMqPw==" }, + "@types/chai@5.2.3": { + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dependencies": [ + "@types/deep-eql", + "assertion-error" + ] + }, + "@types/deep-eql@4.0.2": { + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==" + }, + "@types/estree@1.0.9": { + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==" + }, + "@vitest/mocker@5.0.1_vite@8.3.1": { + "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", + "dependencies": [ + "@jridgewell/trace-mapping", + "@vitest/spy", + "estree-walker", + "magic-string", + "vite" + ], + "optionalPeers": [ + "vite" + ] + }, + "@vitest/spy@5.0.1": { + "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==" + }, + "assertion-error@2.0.1": { + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==" + }, "before-after-hook@3.0.2": { "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==" }, "bottleneck@2.19.5": { "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, + "chai@6.3.0": { + "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==" + }, + "detect-libc@2.1.2": { + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==" + }, "effect@4.0.0": { "integrity": "sha512-ooc1TG5t+FfzgYnFz2ff6BBKyZ7EwBRVXC7c4RhQUAD6/TZ2gTXXMeb4WX7a19ozQo4J73/QW+S00YAIresoMQ==" }, + "es-module-lexer@2.3.2": { + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==" + }, + "estree-walker@3.0.3": { + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dependencies": [ + "@types/estree" + ] + }, + "expect-type@1.4.0": { + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==" + }, "fast-content-type-parse@2.0.1": { "integrity": "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q==" }, + "fdir@6.5.0_picomatch@4.0.7": { + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dependencies": [ + "picomatch" + ], + "optionalPeers": [ + "picomatch" + ] + }, + "fsevents@2.3.3": { + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "os": ["darwin"], + "scripts": true + }, + "lightningcss-android-arm64@1.33.0": { + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "os": ["android"], + "cpu": ["arm64"] + }, + "lightningcss-darwin-arm64@1.33.0": { + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "lightningcss-darwin-x64@1.33.0": { + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "lightningcss-freebsd-x64@1.33.0": { + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "os": ["freebsd"], + "cpu": ["x64"] + }, + "lightningcss-linux-arm-gnueabihf@1.33.0": { + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "os": ["linux"], + "cpu": ["arm"] + }, + "lightningcss-linux-arm64-gnu@1.33.0": { + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "lightningcss-linux-arm64-musl@1.33.0": { + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "lightningcss-linux-x64-gnu@1.33.0": { + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "os": ["linux"], + "cpu": ["x64"] + }, + "lightningcss-linux-x64-musl@1.33.0": { + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "os": ["linux"], + "cpu": ["x64"] + }, + "lightningcss-win32-arm64-msvc@1.33.0": { + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "os": ["win32"], + "cpu": ["arm64"] + }, + "lightningcss-win32-x64-msvc@1.33.0": { + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "os": ["win32"], + "cpu": ["x64"] + }, + "lightningcss@1.33.0": { + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dependencies": [ + "detect-libc" + ], + "optionalDependencies": [ + "lightningcss-android-arm64", + "lightningcss-darwin-arm64", + "lightningcss-darwin-x64", + "lightningcss-freebsd-x64", + "lightningcss-linux-arm-gnueabihf", + "lightningcss-linux-arm64-gnu", + "lightningcss-linux-arm64-musl", + "lightningcss-linux-x64-gnu", + "lightningcss-linux-x64-musl", + "lightningcss-win32-arm64-msvc", + "lightningcss-win32-x64-msvc" + ] + }, + "magic-string@1.4.2": { + "integrity": "sha512-vG+rjFRj1PqdIBozIxAGMjPlOhaVe+GXpbttY/iSK7rGcJRMlwNJO7dcUwmUqkymsFLJiNGI06t4D7Fr7yRC9g==", + "dependencies": [ + "@jridgewell/sourcemap-codec" + ] + }, + "nanoid@3.3.19": { + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "bin": true + }, + "obug@2.2.1": { + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==" + }, "octokit@4.1.4": { "integrity": "sha512-cRvxRte6FU3vAHRC9+PMSY3D+mRAs2Rd9emMoqp70UGRvJRM3sbAoim2IXRZNNsf8wVfn4sGxVBHRAP+JBVX/g==", "dependencies": [ @@ -296,6 +550,70 @@ "@octokit/webhooks" ] }, + "picocolors@1.1.1": { + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "picomatch@4.0.7": { + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==" + }, + "postcss@8.5.28": { + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dependencies": [ + "nanoid", + "picocolors", + "source-map-js" + ] + }, + "rolldown@1.2.12": { + "integrity": "sha512-8wafseiaG80xmXSfqidUNqZcylTlhmPZZt+za2m+js2sFZ8dTNlhIOV2WcbIPx2hgwPBJpEUGFAMZ9bgBBLTSQ==", + "dependencies": [ + "@oxc-project/types", + "@rolldown/pluginutils" + ], + "optionalDependencies": [ + "@rolldown/binding-android-arm-eabi", + "@rolldown/binding-android-arm64", + "@rolldown/binding-darwin-arm64", + "@rolldown/binding-darwin-x64", + "@rolldown/binding-freebsd-x64", + "@rolldown/binding-linux-arm-gnueabihf", + "@rolldown/binding-linux-arm64-gnu", + "@rolldown/binding-linux-arm64-musl", + "@rolldown/binding-linux-ppc64-gnu", + "@rolldown/binding-linux-s390x-gnu", + "@rolldown/binding-linux-x64-gnu", + "@rolldown/binding-linux-x64-musl", + "@rolldown/binding-openharmony-arm64", + "@rolldown/binding-win32-arm64-msvc", + "@rolldown/binding-win32-x64-msvc" + ], + "bin": true + }, + "siginfo@2.0.0": { + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==" + }, + "source-map-js@1.2.2": { + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==" + }, + "stackback@0.0.2": { + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==" + }, + "std-env@4.3.0": { + "integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==" + }, + "tinybench@6.1.4": { + "integrity": "sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==" + }, + "tinyexec@1.3.0": { + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==" + }, + "tinyglobby@0.2.17": { + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dependencies": [ + "fdir", + "picomatch" + ] + }, "toad-cache@3.7.4": { "integrity": "sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==" }, @@ -304,6 +622,48 @@ }, "universal-user-agent@7.0.3": { "integrity": "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==" + }, + "vite@8.3.1": { + "integrity": "sha512-/bvH9E9tmCXRGp2uXY3WbOldqpTwFkbha/8ANaEQ6VkxhH60KyqLwgZq6lG2y+4uT55x9+9eUHMpQ7uGnOCKjA==", + "dependencies": [ + "lightningcss", + "picomatch", + "postcss", + "rolldown", + "tinyglobby" + ], + "optionalDependencies": [ + "fsevents" + ], + "bin": true + }, + "vitest@5.0.1_vite@8.3.1": { + "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", + "dependencies": [ + "@types/chai", + "@vitest/mocker", + "chai", + "es-module-lexer", + "expect-type", + "magic-string", + "obug", + "picomatch", + "std-env", + "tinybench", + "tinyexec", + "tinyglobby", + "vite", + "why-is-node-running" + ], + "bin": true + }, + "why-is-node-running@2.3.0": { + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dependencies": [ + "siginfo", + "stackback" + ], + "bin": true } }, "workspace": { @@ -317,8 +677,10 @@ "jsr:@std/fs@1", "jsr:@std/path@1", "jsr:@std/yaml@1", + "npm:@systemfsoftware/vitest@1", "npm:effect@4", - "npm:octokit@4" + "npm:octokit@4", + "npm:vitest@5" ] } } diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index 9589bcb8..40732c9a 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -17,6 +17,7 @@ import { PnpmWorkspaceSchema, ReportSchema, type StagedPart, + type SummaryInput, type TimingRecord, } from './mutation-plan.ts' @@ -25,21 +26,26 @@ const readFileFor = (files: Record) => (path: string): Promise { - const base = { package: 'packages/x', outcome: 'failure', reportsDir: dir } as const - - const empty = readFileFor({}) - const zero = buildRequireError(base, await loadState(dir, empty)) - assertStringIncludes(zero ?? '', 'zero mutant results') +const inputOf = (overrides: Partial = {}): SummaryInput => ({ + package: 'packages/x', + outcome: 'failure', + reportsDir: dir, + exitCode: 3, + cwd: '/repo', + ...overrides, +}) - const streaming = readFileFor({ - [`${dir}/mutation-stream.jsonl`]: '{"_tag":"phase"}\n{"_tag":"mutant","id":"m1"}\n{"_tag":"mutant","id":"m2"}\n', - }) - const partial = buildRequireError(base, await loadState(dir, streaming)) - assertStringIncludes(partial ?? '', 'after 2 completed mutant(s)') +Deno.test('a package whose reports dir has no report requires one, naming the exit code', async () => { + const state = await loadState(dir, readFileFor({})) + assertStringIncludes(buildRequireError(inputOf({ exitCode: 7 }), state) ?? '', 'exit code: 7') +}) - const complete = readFileFor({ [`${dir}/mutation-report.json`]: '{"schemaVersion":"1.0","files":{}}' }) - assertEquals(buildRequireError(base, await loadState(dir, complete)), null) +Deno.test('a complete report needs no record and no gate failure', async () => { + const state = await loadState( + dir, + readFileFor({ [`${dir}/mutation-report.json`]: '{"schemaVersion":"1.0","files":{}}' }), + ) + assertEquals(buildRequireError(inputOf(), state), null) }) const shardPart = (index: number, files: string[]): StagedPart => ({ @@ -102,7 +108,28 @@ Deno.test('mergeRecord keeps the previous duration when a shard is missing, sums assertEquals(mergeRecord(previous, complete, 'new').packages['p'], { seconds: 200, sha: 'new' }) }) -const input = { package: 'packages/x', outcome: 'failure', reportsDir: dir } as const +const input = inputOf() + +const failureRecord = { + _tag: 'BaselineTestsFailed', + stage: 'dryRun', + testCount: 1, + tests: [{ + id: 'math.test.ts > adds numbers', + name: 'adds numbers', + file: 'src/math.test.ts', + location: { file: 'src/math.test.ts', line: 12, column: 3 }, + message: 'expected 3 to be 4', + stack: null, + reproduce: ['vitest', 'run', 'src/math.test.ts', '-t', 'adds numbers'], + }], + cause: [{ kind: 'AssertionError', message: 'expected 3 to be 4', stack: null }], + capsule: { _tag: 'DoesNotReplay', why: 'interrupted', standIn: 'npx vitest run src/math.test.ts' }, + nextAction: { primary: 'fixCode', otherwise: 'fixTest' }, + traceId: null, +} as const + +const failureStream = `${JSON.stringify({ _tag: 'error', schemaVersion: '3.0', code: 5, record: failureRecord })}\n` Deno.test('buildSummary marks a report with schemaVersion and files as complete', async () => { const state = await loadState( @@ -124,19 +151,44 @@ Deno.test('buildSummary flags a report that is not a valid Stryker report', asyn assertStringIncludes(buildSummary(input, state), 'not a valid Stryker report') }) -Deno.test('buildSummary reports zero mutants when there is no report and no stream', async () => { +Deno.test('a terminal BaselineTestsFailed record yields a summary with the code, test location and capsule', async () => { + const state = await loadState(dir, readFileFor({ [`${dir}/mutation-stream.jsonl`]: failureStream })) + const summary = buildSummary(input, state) + assertStringIncludes(summary, 'BaselineTestsFailed') + assertStringIncludes(summary, 'src/math.test.ts:12:3') + assertStringIncludes(summary, 'adds numbers') + assertStringIncludes(summary, '**Replay:**') + assertEquals(summary.includes('infrastructure'), false) +}) + +Deno.test('a run with neither report nor terminal record yields a RecordMissing summary naming the exit code', async () => { const state = await loadState(dir, readFileFor({})) - assertStringIncludes(buildSummary(input, state), 'zero completed mutants') + const summary = buildSummary(inputOf({ exitCode: 6 }), state) + assertStringIncludes(summary, 'RecordMissing') + assertStringIncludes(summary, 'exit code: 6') +}) + +Deno.test('a reuse line with ran: 0 reports the evaluated-none outcome and a non-zero gate', async () => { + const state = await loadState( + dir, + readFileFor({ [`${dir}/mutation-stream.jsonl`]: '{"_tag":"reuse","reused":4,"ran":0,"refused":{}}\n' }), + ) + assertStringIncludes(buildSummary(input, state), 'evaluated no mutants') + assertEquals(buildRequireError(input, state) !== null, true) }) -Deno.test('buildSummary counts completed mutants from a partial stream', async () => { +Deno.test('a failed run that reused every verdict keeps its failure outcome next to evaluated-none', async () => { const state = await loadState( dir, readFileFor({ - [`${dir}/mutation-stream.jsonl`]: '{"_tag":"mutant","id":"m1"}\n{"_tag":"phase"}\n{"_tag":"mutant","id":"m2"}\n', + [`${dir}/mutation-report.json`]: '{"schemaVersion":"1.0","files":{}}', + [`${dir}/mutation-stream.jsonl`]: '{"_tag":"reuse","reused":4,"ran":0,"refused":{}}\n', }), ) - assertStringIncludes(buildSummary(input, state), '2 completed mutant(s)') + assertStringIncludes( + buildSummary(inputOf({ outcome: 'failure', exitCode: 1 }), state), + '**failure, evaluated no mutants**', + ) }) Deno.test('a cleared reports dir cannot satisfy the no-report gate', async () => { @@ -149,7 +201,7 @@ Deno.test('a cleared reports dir cannot satisfy the no-report gate', async () => assertEquals(buildRequireError(input, stale), null) const cleared = await loadState(dir, readFileFor({})) - assertStringIncludes(buildRequireError(input, cleared) ?? '', 'zero mutant results') + assertEquals(buildRequireError(input, cleared) !== null, true) }) Deno.test('decodeJson rejects a malformed part meta, naming the source', () => { diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index 215c46a7..a9ab490d 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -1,7 +1,17 @@ +import * as Arr from 'effect/Array' +import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { type RecordContext, recordOf } from '../../packages/stryker-js-cli-contract/src/failure-catalog.ts' +import { + type Capsule, + type FailureEvidence, + FailureRecord, +} from '../../packages/stryker-js-cli-contract/src/failure-record.schema.ts' +import { annotationsOf, markdownOf } from '../../packages/stryker-js-cli-contract/src/render-failure.ts' + export const ShardSchema = S.Struct({ index: S.Int, count: S.Int }) export type Shard = S.Schema.Type @@ -205,27 +215,6 @@ export const summaryTable = (parts: readonly Part[], target: number, taskName = ].join('\n') } -const tagOf = (value: unknown): unknown => { - if (typeof value !== 'object' || value === null || !('_tag' in value)) return undefined - return value._tag -} - -const countMutantLines = (text: string): number => { - let n = 0 - for (const raw of text.split('\n')) { - const line = raw.trim() - if (line.length === 0) continue - let parsed: unknown - try { - parsed = JSON.parse(line) - } catch { - continue - } - if (tagOf(parsed) === 'mutant') n += 1 - } - return n -} - const isCompleteReport = (text: string): boolean => { let parsed: unknown try { @@ -240,7 +229,39 @@ const isCompleteReport = (text: string): boolean => { typeof files === 'object' && files !== null } -export type ReportState = { reportText: string | null; streamText: string | null } +type FailureRecordType = S.Schema.Type + +const RunFailedLineSchema = S.Struct({ + _tag: S.Literal('error'), + schemaVersion: S.Literal('3.0'), + code: S.Int, + record: FailureRecord, +}) + +const ReuseLineSchema = S.Struct({ + _tag: S.Literal('reuse'), + ran: S.Int.check(S.isGreaterThanOrEqualTo(0)), +}) + +const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder): ReadonlyArray => + (streamText ?? '').split('\n').flatMap((line) => + Option.toArray(S.decodeUnknownOption(S.fromJsonString(schema))(line)) + ) + +const terminalRecordOf = (streamText: string | null): Option.Option => + Option.map(Arr.last(decodedLines(streamText, RunFailedLineSchema)), (line) => line.record) + +export type ReuseState = { readonly ran: number } + +const reuseOf = (streamText: string | null): Option.Option => + Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran })) + +export type ReportState = { + reportText: string | null + streamText: string | null + failure: Option.Option + reuse: Option.Option +} export const loadState = async ( reportsDir: string, @@ -248,19 +269,80 @@ export const loadState = async ( ): Promise => { const reportText = await readFile(`${reportsDir}/mutation-report.json`).catch(() => null) const streamText = await readFile(`${reportsDir}/mutation-stream.jsonl`).catch(() => null) - return { reportText, streamText } + return { reportText, streamText, failure: terminalRecordOf(streamText), reuse: reuseOf(streamText) } } export interface SummaryInput { readonly package: string readonly outcome: Outcome readonly reportsDir: string + readonly exitCode: number | null + readonly cwd: string + readonly limitSeconds?: number } +const TIMEOUT_EXIT_CODE = 124 +const BINARY_MISSING_EXIT_CODE = 127 +const CI_BINARY = 'pnpm' + +type CiEvidence = Extract + +const ciEvidenceOf = (input: SummaryInput): CiEvidence => + Match.value(input.exitCode).pipe( + Match.when(TIMEOUT_EXIT_CODE, (): CiEvidence => ({ + _tag: 'JobTimedOut', + stage: 'ci', + limitSeconds: input.limitSeconds ?? 0, + })), + Match.when(BINARY_MISSING_EXIT_CODE, (): CiEvidence => ({ + _tag: 'BinaryMissing', + stage: 'ci', + binary: CI_BINARY, + })), + Match.orElse((): CiEvidence => ({ _tag: 'RecordMissing', stage: 'ci', exitCode: input.exitCode })), + ) + +const standInOf = (input: SummaryInput): string => + `stryker mutation --filter ${input.package} (exit ${input.exitCode === null ? 'unknown' : input.exitCode})` + +const capsuleOf = (evidence: CiEvidence, input: SummaryInput): Capsule => + Match.value(evidence).pipe( + Match.tagsExhaustive({ + RecordMissing: (): Capsule => ({ _tag: 'DoesNotReplay', why: 'recordMissing', standIn: standInOf(input) }), + JobTimedOut: (): Capsule => ({ _tag: 'DoesNotReplay', why: 'jobTimedOut', standIn: standInOf(input) }), + BinaryMissing: (): Capsule => ({ _tag: 'DoesNotReplay', why: 'binaryMissing', standIn: standInOf(input) }), + }), + ) + +const ciRecordOf = (input: SummaryInput): FailureRecordType => { + const evidence = ciEvidenceOf(input) + const context: RecordContext = { + cause: [], + cwd: input.cwd, + argv: ['stryker', 'mutation', '--filter', input.package], + env: [], + capsule: capsuleOf(evidence, input), + traceId: null, + } + return recordOf(evidence, context) +} + +export const recordFor = (input: SummaryInput, state: ReportState): FailureRecordType => + Option.getOrElse(state.failure, () => ciRecordOf(input)) + +const outcomeLabelOf = (input: SummaryInput, state: ReportState): string => + Option.match(state.reuse, { + onNone: () => input.outcome, + onSome: (reuse) => (reuse.ran === 0 ? `${input.outcome}, evaluated no mutants` : input.outcome), + }) + export const buildSummary = (input: SummaryInput, state: ReportState): string => { const reportPath = `${input.reportsDir}/mutation-report.json` - const streamPath = `${input.reportsDir}/mutation-stream.jsonl` - const lines = [`#### Mutation · **${input.package}**`, '', `- **Stryker outcome**: **${input.outcome}**`] + const lines = [ + `#### Mutation · **${input.package}**`, + '', + `- **Stryker outcome**: **${outcomeLabelOf(input, state)}**`, + ] if (state.reportText !== null) { if (isCompleteReport(state.reportText)) { @@ -273,30 +355,13 @@ export const buildSummary = (input: SummaryInput, state: ReportState): string => return `${lines.join('\n')}\n` } - const mutants = state.streamText === null ? 0 : countMutantLines(state.streamText) - if (mutants === 0) { - lines.push( - `- **Result**: no final report and zero completed mutants — infrastructure failure (missing binary, crashed run or timeout). Stream: **${streamPath}**`, - ) - } else { - lines.push( - `- **Result**: no final report (run interrupted) — ${mutants} completed mutant(s) recorded, marked incomplete in the merged report. Stream: **${streamPath}**`, - ) - } + lines.push(markdownOf(recordFor(input, state))) return `${lines.join('\n')}\n` } export const buildRequireError = (input: SummaryInput, state: ReportState): string | null => { if (state.reportText !== null) return null - const mutants = state.streamText === null ? 0 : countMutantLines(state.streamText) - if (mutants === 0) { - return [ - `::error title=Mutation produced no report::${input.package}: stryker exited '${input.outcome}' with zero mutant results — infrastructure failure (missing binary, crashed run or timeout), not a score outcome. Stream artifact: ${input.reportsDir}/mutation-stream.jsonl`, - ].join('') - } - return [ - `::error title=Mutation produced no report::${input.package}: stryker exited '${input.outcome}' after ${mutants} completed mutant(s) without a final report — infrastructure failure, not a score outcome. Partial stream: ${input.reportsDir}/mutation-stream.jsonl`, - ].join('') + return annotationsOf(recordFor(input, state)).join('\n') } export const combineParts = (parts: readonly StagedPart[]): Map => { diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index 2ff63a63..4114dfcc 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -114,7 +114,7 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom } const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' const reportsDir = join(dir, 'reports') - const input = { package: labelOf(dir, shard), outcome, reportsDir } + const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } const state = await loadState(reportsDir, readText) console.log(buildSummary(input, state)) const missing = buildRequireError(input, state) From eafe8f3a9c1defaec17a8feb75353292ed1f63d4 Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:45 -0400 Subject: [PATCH 7/8] ci(repo): dry-run the main mutation lane once and keep its evidence (#155) A preflight job dry-runs every planned package and publishes the coverage that shards reuse through incrementalSources. A dry-run-only run now writes that coverage to its incremental file, keeping any verdicts there, and the run-inputs digest no longer covers dryRunOnly. Each job exports its traces, a failed shard's parts are kept 30 days, the report job uploads one merged SARIF run and publishes main's survivor baseline Verdict-Semantics: unchanged --- .changeset/petite-insects-feel.md | 5 + .github/workflows/mutation.yml | 147 +++++++++++++++++- .../src/mutation-reporting.schema.ts | 2 + .../src/mutation-reporting.service.ts | 38 ++++- .../stryker-js/src/run/mutation-test.cell.ts | 13 +- packages/stryker-js/src/verdict-semantics.ts | 2 +- .../incremental-reuse.integration.test.ts | 100 ++++++++++++ packages/toolchain/stryker-config/lib/base.js | 2 +- scripts/lib/mutation-plan.test.ts | 89 +++++++++++ scripts/lib/mutation-plan.ts | 117 ++++++++++++++ scripts/mutation-job.ts | 120 ++++++++++++-- 11 files changed, 607 insertions(+), 28 deletions(-) create mode 100644 .changeset/petite-insects-feel.md diff --git a/.changeset/petite-insects-feel.md b/.changeset/petite-insects-feel.md new file mode 100644 index 00000000..44356c3a --- /dev/null +++ b/.changeset/petite-insects-feel.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +A dry-run-only run with incremental mode on publishes its dry-run coverage to the incremental file, keeping any verdicts already there, so a later run that reads the file through incrementalSources skips its own dry run. The run-inputs digest no longer covers dryRunOnly, so a preflight and the runs that reuse its coverage share one digest; existing caches are refused once with runInputsChanged. diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 7129bdeb..43817f08 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -51,12 +51,93 @@ jobs: if-no-files-found: ignore retention-days: 1 - mutation: - name: mutation · ${{ matrix.job.name }} + preflight: + name: preflight · one dry run per package needs: plan if: ${{ needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} runs-on: ubuntu-latest + timeout-minutes: 45 + env: + OTEL_ENABLED: "true" + OTEL_SERVICE_NAME: stryker-js-mutation + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4318 + OUT_DIR: mutation-telemetry + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: denoland/setup-deno@v2 + - name: Build + run: pnpm exec turbo run build --output-logs=errors-only --log-order=stream ${{ join(fromJSON(needs.plan.outputs.jobs).*.filters, ' ') }} + + - uses: actions/cache/restore@v6 + with: + path: .stryker-incremental + key: stryker-incremental-${{ github.run_id }} + restore-keys: | + stryker-incremental- + - name: Place incremental files + run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi + + - name: Install process-compose (pinned) + run: | + curl -sSfL -o /tmp/pc.tar.gz https://github.com/F1bonacc1/process-compose/releases/download/v1.122.0/process-compose_linux_amd64.tar.gz + echo '9b6dbc38324c0b0481f1cd1dd828ffdc78117129ec797678f4bf8c4023311281 /tmp/pc.tar.gz' | sha256sum -c - + sudo tar xzf /tmp/pc.tar.gz -C /usr/local/bin process-compose + - name: Start LGTM (blocks until probe-ready) + run: | + echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" + pnpm lgtm:up + + - name: Preflight + env: + JOBS: ${{ needs.plan.outputs.jobs }} + run: $MUTATION_JOB preflight --cap-seconds "$TARGET_SECONDS" + + - uses: actions/upload-artifact@v7 + with: + name: mutation-preflight + path: preflight/ + if-no-files-found: ignore + retention-days: 1 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: mutation-preflight-evidence + path: preflight-evidence/ + if-no-files-found: ignore + retention-days: 30 + + - name: Export telemetry + if: always() + continue-on-error: true + run: ./test/e2e/scripts/export-traces.ts + - name: Package telemetry + if: always() + run: if [ -d mutation-telemetry ]; then tar czf mutation-telemetry.tar.gz mutation-telemetry; fi + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mutation-telemetry-${{ github.run_id }}-preflight + path: mutation-telemetry.tar.gz + if-no-files-found: ignore + retention-days: 30 + + mutation: + name: mutation · ${{ matrix.job.name }} + needs: [plan, preflight] + if: ${{ needs.preflight.result == 'success' }} + runs-on: ubuntu-latest timeout-minutes: 75 + env: + OTEL_ENABLED: "true" + OTEL_SERVICE_NAME: stryker-js-mutation + OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4318 + OUT_DIR: mutation-telemetry strategy: fail-fast: false matrix: @@ -81,6 +162,21 @@ jobs: stryker-incremental- - name: Place incremental files run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi + - uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: mutation-preflight + path: . + + - name: Install process-compose (pinned) + run: | + curl -sSfL -o /tmp/pc.tar.gz https://github.com/F1bonacc1/process-compose/releases/download/v1.122.0/process-compose_linux_amd64.tar.gz + echo '9b6dbc38324c0b0481f1cd1dd828ffdc78117129ec797678f4bf8c4023311281 /tmp/pc.tar.gz' | sha256sum -c - + sudo tar xzf /tmp/pc.tar.gz -C /usr/local/bin process-compose + - name: Start LGTM (blocks until probe-ready) + run: | + echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" + pnpm lgtm:up - name: Mutation env: @@ -107,7 +203,22 @@ jobs: name: mutation-part-${{ matrix.job.id }} path: mutation-parts/ if-no-files-found: ignore - retention-days: 1 + retention-days: 30 + + - name: Export telemetry + if: always() + continue-on-error: true + run: ./test/e2e/scripts/export-traces.ts + - name: Package telemetry + if: always() + run: if [ -d mutation-telemetry ]; then tar czf mutation-telemetry.tar.gz mutation-telemetry; fi + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mutation-telemetry-${{ github.run_id }}-${{ matrix.job.id }} + path: mutation-telemetry.tar.gz + if-no-files-found: ignore + retention-days: 30 timings: name: record mutation timings @@ -138,13 +249,14 @@ jobs: report: name: Mutation report - needs: [plan, mutation] - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} + needs: [plan, preflight, mutation] + if: ${{ !cancelled() && needs.preflight.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 25 permissions: contents: read actions: write + security-events: write steps: - uses: actions/checkout@v7 - uses: pnpm/action-setup@v6 @@ -194,8 +306,31 @@ jobs: path: mutation-merged/* retention-days: 14 - - name: Delete per-job artifacts + - name: Merge SARIF if: always() + run: $MUTATION_JOB sarif --parts mutation-parts --out mutation-sarif/mutation.sarif + - uses: github/codeql-action/upload-sarif@v4 + if: ${{ always() && hashFiles('mutation-sarif/mutation.sarif') != '' }} + with: + sarif_file: mutation-sarif/mutation.sarif + category: mutation + + - name: Survivor baseline + if: ${{ env.MUTATION_REPORT_COMPLETE == 'true' && github.ref == 'refs/heads/main' }} + run: | + mkdir -p mutation-gate/reports/mutation + cp mutation-merged/mutation-report.json mutation-gate/reports/mutation/mutation.json + cd mutation-gate + "$GITHUB_WORKSPACE/packages/stryker-js/node_modules/.bin/stryker" gate --baseline mutation-survivor-baseline.json --update-baseline + - uses: actions/upload-artifact@v7 + if: ${{ env.MUTATION_REPORT_COMPLETE == 'true' && github.ref == 'refs/heads/main' }} + with: + name: mutation-survivor-baseline + path: mutation-gate/mutation-survivor-baseline.json + retention-days: 90 + + - name: Delete per-job artifacts + if: ${{ always() && needs.mutation.result == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ github.token }} diff --git a/packages/stryker-js/src/mutation-reporting.schema.ts b/packages/stryker-js/src/mutation-reporting.schema.ts index 229beec0..a4ab614a 100644 --- a/packages/stryker-js/src/mutation-reporting.schema.ts +++ b/packages/stryker-js/src/mutation-reporting.schema.ts @@ -10,3 +10,5 @@ export class ManifestUnreadable extends S.TaggedError()('Man } export const ManifestSchema = S.Struct({ version: S.optional(S.String) }) + +export const IncrementalReportObjectSchema = S.Record(S.String, S.Json) diff --git a/packages/stryker-js/src/mutation-reporting.service.ts b/packages/stryker-js/src/mutation-reporting.service.ts index a0d1180f..c044aba0 100644 --- a/packages/stryker-js/src/mutation-reporting.service.ts +++ b/packages/stryker-js/src/mutation-reporting.service.ts @@ -40,7 +40,7 @@ import { classifyExit, ClassifyExitCommand } from './classify-exit.workflow.js' import type { DryRunCoverage } from './dry-run-coverage.schema.js' import type { FormatIdentity, TimeoutEvidence, TimeoutKind } from './IncrementalDiff.schema.js' import { TimeoutEvidenceSchema } from './IncrementalDiff.schema.js' -import { ManifestSchema, ManifestUnreadable } from './mutation-reporting.schema.js' +import { IncrementalReportObjectSchema, ManifestSchema, ManifestUnreadable } from './mutation-reporting.schema.js' import type { ResolvedMode } from './output-mode.schema.js' import { ProjectFiles, type ProjectFilesShape } from './project-files.service.js' import type { Project, ProjectFile } from './Project.schema.js' @@ -135,6 +135,7 @@ export interface MutationReportingService { input: MutationReportingInput, plannedMutants: readonly Mutant.Mutant[], ) => Effect.Effect + readonly publishDryRunCoverage: (input: MutationReportingInput) => Effect.Effect } export class MutationReporting extends Context.Service()( @@ -158,6 +159,7 @@ export class MutationReporting extends Context.Service mapRunResult(mutant, result), reportAll: (input) => reportAll(deps, input), checkpoint: (input, plannedMutants) => checkpoint(deps, input, plannedMutants), + publishDryRunCoverage: (input) => publishDryRunCoverage(deps, input), }) }), ) @@ -928,6 +930,40 @@ const checkpoint = ( onFalse: () => Effect.void, }) +const priorIncrementalObjectOf = (text: string): Option.Option => + S.decodeOption(S.fromJsonString(IncrementalReportObjectSchema))(text) + +const dryRunCoverageReportOf = Effect.fnUntraced(function*( + deps: MutationReportingDeps, + input: MutationReportingInput, + coverage: DryRunCoverage, +) { + const prior = yield* deps.fs.readFileString(input.options.incrementalFile).pipe(Effect.option) + return yield* Option.match(Option.flatMap(prior, priorIncrementalObjectOf), { + onSome: (report) => Effect.succeed({ ...report, dryRunCoverage: coverage }), + onNone: () => slimIncrementalReport(deps, input, []), + }) +}) + +const writeDryRunCoverage = Effect.fnUntraced(function*( + deps: MutationReportingDeps, + input: MutationReportingInput, + coverage: DryRunCoverage, +) { + const report = yield* dryRunCoverageReportOf(deps, input, coverage) + const json = yield* S.encodeEffect(S.fromJsonString(S.Unknown))(report).pipe(Effect.orDie) + yield* writeAtomic(deps, input.options.incrementalFile, json) +}) + +const publishDryRunCoverage = (deps: MutationReportingDeps, input: MutationReportingInput) => + Option.match( + Option.filter(Option.fromUndefinedOr(input.testCoverage.dryRunCoverage), () => input.options.incremental), + { + onNone: () => Effect.void, + onSome: (coverage) => writeDryRunCoverage(deps, input, coverage), + }, + ) + if (import.meta.vitest !== void 0) { const { it } = await import('@systemfsoftware/vitest') const { Mutant: { Mutant, MutantStatusSchema }, TestRunner: { MutantRunResultSchema } } = await import( diff --git a/packages/stryker-js/src/run/mutation-test.cell.ts b/packages/stryker-js/src/run/mutation-test.cell.ts index ceb53165..6a8eea30 100644 --- a/packages/stryker-js/src/run/mutation-test.cell.ts +++ b/packages/stryker-js/src/run/mutation-test.cell.ts @@ -70,7 +70,15 @@ const writeMutationTestNoTests = Effect.fn(SpanTaxonomy.Spans.mutationTestNoTest return { results: [], verdict: null } }) -const writeMutationTestDryRunOnly = Effect.fn(SpanTaxonomy.Spans.mutationTestDryRunOnly.name)(function*() { +const writeMutationTestDryRunOnly = Effect.fn(SpanTaxonomy.Spans.mutationTestDryRunOnly.name)(function*( + raw: MutationTestRaw, +) { + const reporting = yield* MutationReporting + const env = yield* RunEnvironment + yield* reporting.publishDryRunCoverage(reportingInputOf({ prev: raw.prev, env, results: [] })).pipe( + Effect.tapCause((cause) => Effect.logWarning('Failed to publish the dry-run coverage', cause)), + Effect.ignoreCause, + ) yield* phaseEntered('mutation-test') yield* Effect.logInfo('The dry-run has been completed successfully. No mutations have been executed.') return { results: [], verdict: null } @@ -272,7 +280,8 @@ export const mutationTestCell = Sandwich.named( return raw }) ).decide(admitMutationTest).write({ - MutationTestDryRunOnly: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestDryRunOnly() }), + MutationTestDryRunOnly: (_decision, raw) => + writeMutationTestOutcome({ raw, outcome: writeMutationTestDryRunOnly(raw) }), MutationTestNoTests: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestNoTests() }), MutationTestProceed: (_decision, raw) => writeMutationTestOutcome({ raw, outcome: writeMutationTestProceed(raw) }), RunFailure: (failure) => Effect.fail(RunFailure.make(failure)), diff --git a/packages/stryker-js/src/verdict-semantics.ts b/packages/stryker-js/src/verdict-semantics.ts index 08e980b2..31ddefa0 100644 --- a/packages/stryker-js/src/verdict-semantics.ts +++ b/packages/stryker-js/src/verdict-semantics.ts @@ -72,7 +72,7 @@ const withoutVersionOf = (value: Json): Json => onNone: () => value, }) -const scopeOptionKeys: readonly string[] = ['mutate', 'since', 'mutantIds'] +const scopeOptionKeys: readonly string[] = ['mutate', 'since', 'mutantIds', 'dryRunOnly'] const presentationOptionKeys: readonly string[] = [ 'reporters', diff --git a/packages/stryker-js/tests/incremental-reuse.integration.test.ts b/packages/stryker-js/tests/incremental-reuse.integration.test.ts index c8cbf584..e628d8d3 100644 --- a/packages/stryker-js/tests/incremental-reuse.integration.test.ts +++ b/packages/stryker-js/tests/incremental-reuse.integration.test.ts @@ -440,6 +440,106 @@ Feature('Content-keyed reuse across incremental reports') ), ) + scenario( + 'A dry-run-only preflight publishes its coverage, and a run reading it through incrementalSources skips its own dry run', + Gherkin.Do.pipe( + Given('a workspace whose command runner appends every spawn to a log file')( + 'fixture', + () => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const root = yield* writeFixture([['src/math.ts', SOURCE]]) + const spawnLog = yield* fs.makeTempFile({ prefix: 'runner-spawns', suffix: '.txt' }) + const commandRunner = { command: `echo spawned >> ${spawnLog}` } + return yield* Effect.ensuring( + Effect.gen(function*() { + const cold = yield* runOnce( + root, + optionsOf(root, { commandRunner, incrementalFile: `${root}/reports/cold.json` }), + ) + const spawnsInColdRun = yield* lineCountOf(spawnLog) + yield* runOnce( + root, + optionsOf(root, { + commandRunner, + dryRunOnly: true, + incrementalFile: `${root}/reports/preflight.json`, + }), + ) + const spawnsAfterPreflight = yield* lineCountOf(spawnLog) + const preflighted = yield* runOnce( + root, + optionsOf(root, { commandRunner, incrementalSources: ['reports/preflight.json'] }), + ) + const spawnsAfterPreflighted = yield* lineCountOf(spawnLog) + return { + cold, + preflighted, + spawnsInColdRun, + spawnsInPreflight: spawnsAfterPreflight - spawnsInColdRun, + spawnsInPreflightedRun: spawnsAfterPreflighted - spawnsAfterPreflight, + } + }), + Effect.andThen(removeFixture(root), Effect.orDie(fs.remove(spawnLog, { force: true }))), + ) + }).pipe(Effect.orDie, Effect.provide(filePorts)), + ), + Then( + 'the preflight spawns exactly the dry run a cold run spawns, and the preflighted run spawns only its mutants', + )((s, expect) => { + const coldDryRunSpawns = s.fixture.spawnsInColdRun - (s.fixture.cold.reuse?.ran ?? 0) + return expect({ + coldRanMutants: (s.fixture.cold.reuse?.ran ?? 0) > 0, + coldDryRunSpawned: coldDryRunSpawns > 0, + preflightSpawns: s.fixture.spawnsInPreflight, + preflightedRan: s.fixture.preflighted.reuse?.ran, + preflightedDryRunSpawns: s.fixture.spawnsInPreflightedRun - (s.fixture.preflighted.reuse?.ran ?? 0), + }).toEqual({ + coldRanMutants: true, + coldDryRunSpawned: true, + preflightSpawns: coldDryRunSpawns, + preflightedRan: s.fixture.cold.reuse?.ran, + preflightedDryRunSpawns: 0, + }) + }), + ), + ) + + scenario( + 'A dry-run-only run into an existing incremental file keeps the verdicts it holds', + Gherkin.Do.pipe( + Given('a workspace whose incremental file already holds every verdict')( + 'fixture', + () => + Effect.gen(function*() { + const root = yield* writeFixture([['src/math.ts', SOURCE]]) + return yield* Effect.ensuring( + Effect.gen(function*() { + const first = yield* runOnce(root, optionsOf(root)) + yield* runOnce(root, optionsOf(root, { dryRunOnly: true })) + const after = yield* runOnce(root, optionsOf(root)) + return { first, after } + }), + removeFixture(root), + ) + }).pipe(Effect.orDie, Effect.provide(filePorts)), + ), + Then('the run after the dry-run-only run reuses every verdict the first run wrote')((s, expect) => + expect({ + plannedNonZero: s.fixture.first.mutants.length > 0, + after: { + reused: s.fixture.after.reuse?.reused, + ran: s.fixture.after.reuse?.ran, + refused: s.fixture.after.reuse?.refused, + }, + }).toEqual({ + plannedNonZero: true, + after: { reused: s.fixture.first.mutants.length, ran: 0, refused: ZERO_REFUSALS }, + }) + ), + ), + ) + scenario( 'A verdict written under another report path is reused through the incrementalSources globs', Gherkin.Do.pipe( diff --git a/packages/toolchain/stryker-config/lib/base.js b/packages/toolchain/stryker-config/lib/base.js index bc7b3d21..da8d5cce 100644 --- a/packages/toolchain/stryker-config/lib/base.js +++ b/packages/toolchain/stryker-config/lib/base.js @@ -13,7 +13,7 @@ const restoredShardReports = ['reports/stryker-incremental-*.json'] export const sharedConfig = { packageManager: 'pnpm', - reporters: isAgent || isCI ? ['json', 'html'] : ['progress', 'html', 'json'], + reporters: isAgent ? ['json', 'html'] : isCI ? ['json', 'html', 'sarif'] : ['progress', 'html', 'json'], htmlReporter: { fileName: 'reports/mutation-report.html' }, jsonReporter: { fileName: 'reports/mutation-report.json' }, coverageAnalysis: 'perTest', diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index 40732c9a..ed3d3abc 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -1,21 +1,29 @@ import { assertEquals, assertStringIncludes, assertThrows } from '@std/assert' +import * as Option from 'effect/Option' import { + buildPreflightError, + buildPreflightSummary, buildRequireError, buildSummary, combineParts, decodeInput, decodeJson, emptyRecord, + evaluatedOf, + evaluatedSummaryOf, JobOrNullSchema, JobsSchema, loadState, mergeRecord, + mergeSarif, type Part, PartMetaSchema, planJobs, PnpmWorkspaceSchema, + PREFLIGHT_FILE, ReportSchema, + SarifLogSchema, type StagedPart, type SummaryInput, type TimingRecord, @@ -240,3 +248,84 @@ Deno.test('decodeInput names the workspace file when packages is malformed', () 'pnpm-workspace.yaml', ) }) + +Deno.test('a passing preflight names where its coverage went, or that every shard repeats the dry run', async () => { + const state = await loadState(dir, readFileFor({})) + const passed = inputOf({ outcome: 'success', exitCode: 0 }) + assertStringIncludes(buildPreflightSummary(passed, state, true), `packages/x/${PREFLIGHT_FILE}`) + assertStringIncludes(buildPreflightSummary(passed, state, false), 'every shard runs its own dry run') + assertEquals(buildPreflightSummary(passed, state, true).includes('RecordMissing'), false) + assertEquals(buildPreflightError(passed, state), null) +}) + +Deno.test('a failing preflight renders its terminal record and annotates it', async () => { + const state = await loadState(dir, readFileFor({ [`${dir}/mutation-stream.jsonl`]: failureStream })) + const failed = inputOf({ exitCode: 5 }) + assertStringIncludes(buildPreflightSummary(failed, state, false), 'BaselineTestsFailed') + assertStringIncludes(buildPreflightError(failed, state) ?? '', 'BaselineTestsFailed') +}) + +const reuseLine = (ran: number): string => `${JSON.stringify({ _tag: 'reuse', reused: 4, ran, refused: {} })}\n` + +Deno.test('evaluatedOf sums the mutants every part ran and reports evaluated-none when that sum is zero', () => { + assertEquals(evaluatedOf([reuseLine(2), reuseLine(3), '{"_tag":"tick"}\n']), Option.some(5)) + assertStringIncludes(evaluatedSummaryOf(evaluatedOf([reuseLine(0), reuseLine(0)])), 'no mutants') + assertEquals(evaluatedOf(['{"_tag":"tick"}\n']), Option.none()) +}) + +const sarifOf = ( + rules: readonly string[], + results: readonly (readonly [string, string])[], + invocations?: unknown[], +) => ({ + $schema: 'https://json.schemastore.org/sarif-2.1.0.json', + version: '2.1.0' as const, + runs: [{ + tool: { + driver: { + name: 'StrykerJS', + version: '1', + informationUri: 'https://stryker-mutator.io', + rules: rules.map((id) => ({ id })), + }, + }, + results: results.map(([ruleId, uri]) => ({ + ruleId, + ruleIndex: rules.indexOf(ruleId), + locations: [{ physicalLocation: { artifactLocation: { uri }, region: { startLine: 1 } } }], + })), + ...(invocations === undefined ? {} : { invocations }), + }], +}) + +Deno.test('mergeSarif folds every package into one run with repository paths and re-indexed rules', () => { + const notification = { + descriptor: { id: 'BaselineTestsFailed' }, + locations: [{ physicalLocation: { artifactLocation: { uri: 'src/b.test.ts' }, region: { startLine: 3 } } }], + } + const merged = Option.getOrThrow(mergeSarif([ + { dir: 'packages/a', log: decodeInput(SarifLogSchema, sarifOf(['Z'], [['Z', 'src/a.ts']]), 'a') }, + { + dir: 'packages/b', + log: decodeInput( + SarifLogSchema, + sarifOf(['A', 'Z'], [['A', 'src/b.ts']], [{ toolExecutionNotifications: [notification] }]), + 'b', + ), + }, + ])) + const [run] = merged.runs + assertEquals(merged.runs.length, 1) + assertEquals(run?.tool.driver.rules.map((rule) => rule.id), ['A', 'Z']) + assertEquals( + run?.results.map(( + result, + ) => [result.ruleId, result['ruleIndex'], result.locations[0]?.physicalLocation.artifactLocation.uri]), + [['Z', 1, 'packages/a/src/a.ts'], ['A', 0, 'packages/b/src/b.ts']], + ) + assertEquals( + run?.invocations?.[0]?.toolExecutionNotifications?.[0]?.locations?.[0]?.physicalLocation.artifactLocation.uri, + 'packages/b/src/b.test.ts', + ) + assertEquals(mergeSarif([]), Option.none()) +}) diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index a9ab490d..24fe1ff3 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -364,6 +364,123 @@ export const buildRequireError = (input: SummaryInput, state: ReportState): stri return annotationsOf(recordFor(input, state)).join('\n') } +export const PREFLIGHT_FILE = 'reports/stryker-incremental-preflight.json' + +export const buildPreflightSummary = (input: SummaryInput, state: ReportState, published: boolean): string => { + const header = [`#### Mutation preflight · **${input.package}**`, ''] + if (input.outcome === 'failure') { + return `${[...header, '- **Dry run**: **failure**', markdownOf(recordFor(input, state))].join('\n')}\n` + } + const coverage = published + ? `- **Dry-run coverage**: published at **${input.package}/${PREFLIGHT_FILE}**; shards whose digests match reuse it` + : '- **Dry-run coverage**: not published by this CLI; every shard runs its own dry run' + return `${[...header, '- **Dry run**: **passed**', coverage].join('\n')}\n` +} + +export const buildPreflightError = (input: SummaryInput, state: ReportState): string | null => + input.outcome === 'success' ? null : annotationsOf(recordFor(input, state)).join('\n') + +export const evaluatedOf = (streams: readonly string[]): Option.Option => { + const reuses = streams.flatMap((stream) => Option.toArray(reuseOf(stream))) + return reuses.length === 0 ? Option.none() : Option.some(reuses.reduce((sum, reuse) => sum + reuse.ran, 0)) +} + +export const evaluatedSummaryOf = (evaluated: Option.Option): string => + Option.match(evaluated, { + onNone: () => '- **Evaluated**: no part reported which mutants it ran', + onSome: (ran) => + ran === 0 + ? '- **Evaluated**: **no mutants**; every verdict was reused, so this run tested nothing' + : `- **Evaluated**: **${ran}** mutant(s); every other verdict was reused`, + }) + +const SarifRest = [S.Record(S.String, S.Json)] as const +const SarifArtifactLocationSchema = S.StructWithRest(S.Struct({ uri: S.String }), SarifRest) +const SarifPhysicalLocationSchema = S.StructWithRest( + S.Struct({ artifactLocation: SarifArtifactLocationSchema }), + SarifRest, +) +const SarifLocationSchema = S.StructWithRest(S.Struct({ physicalLocation: SarifPhysicalLocationSchema }), SarifRest) +const SarifRuleSchema = S.StructWithRest(S.Struct({ id: S.String }), SarifRest) +const SarifResultSchema = S.StructWithRest( + S.Struct({ ruleId: S.String, locations: S.Array(SarifLocationSchema) }), + SarifRest, +) +const SarifNotificationSchema = S.StructWithRest( + S.Struct({ locations: S.optionalKey(S.Array(SarifLocationSchema)) }), + SarifRest, +) +const SarifInvocationSchema = S.StructWithRest( + S.Struct({ toolExecutionNotifications: S.optionalKey(S.Array(SarifNotificationSchema)) }), + SarifRest, +) +const SarifDriverSchema = S.StructWithRest(S.Struct({ name: S.String, rules: S.Array(SarifRuleSchema) }), SarifRest) +const SarifRunSchema = S.Struct({ + tool: S.Struct({ driver: SarifDriverSchema }), + results: S.Array(SarifResultSchema), + invocations: S.optionalKey(S.Array(SarifInvocationSchema)), +}) +export const SarifLogSchema = S.Struct({ + $schema: S.String, + version: S.Literal('2.1.0'), + runs: S.Array(SarifRunSchema), +}) +export type SarifLog = S.Schema.Type +type SarifLocation = S.Schema.Type +type SarifRun = S.Schema.Type + +export type PackageSarif = { readonly dir: string; readonly log: SarifLog } + +const underDir = (dir: string) => (location: SarifLocation): SarifLocation => ({ + ...location, + physicalLocation: { + ...location.physicalLocation, + artifactLocation: { + ...location.physicalLocation.artifactLocation, + uri: `${dir}/${location.physicalLocation.artifactLocation.uri}`, + }, + }, +}) + +const invocationsUnder = (dir: string, run: SarifRun) => + (run.invocations ?? []).map((invocation) => ({ + ...invocation, + ...(invocation.toolExecutionNotifications === undefined ? {} : { + toolExecutionNotifications: invocation.toolExecutionNotifications.map((notification) => ({ + ...notification, + ...(notification.locations === undefined ? {} : { locations: notification.locations.map(underDir(dir)) }), + })), + }), + })) + +export const mergeSarif = (packages: readonly PackageSarif[]): Option.Option => { + const runs = packages.flatMap(({ dir, log }) => log.runs.map((run) => ({ dir, run }))) + return Option.map(Arr.head(runs), ({ run: first }) => { + const rules = Arr.dedupeWith( + runs.flatMap(({ run }) => run.tool.driver.rules).sort((a, b) => a.id.localeCompare(b.id)), + (a, b) => a.id === b.id, + ) + const ruleIndex = new Map(rules.map((rule, index) => [rule.id, index] as const)) + const results = runs.flatMap(({ dir, run }) => + run.results.map((result) => ({ + ...result, + ruleIndex: ruleIndex.get(result.ruleId) ?? 0, + locations: result.locations.map(underDir(dir)), + })) + ) + const invocations = runs.flatMap(({ dir, run }) => invocationsUnder(dir, run)) + return { + $schema: packages[0]?.log.$schema ?? '', + version: '2.1.0', + runs: [{ + tool: { driver: { ...first.tool.driver, rules } }, + results, + ...(invocations.length === 0 ? {} : { invocations }), + }], + } + }) +} + export const combineParts = (parts: readonly StagedPart[]): Map => { const byPackage = Map.groupBy(parts, (part) => part.meta.package) const combined = new Map() diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index 4114dfcc..eddb9bee 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -3,28 +3,39 @@ import { parseArgs } from '@std/cli/parse-args' import { expandGlob } from '@std/fs/expand-glob' import { dirname, join } from '@std/path' +import * as Option from 'effect/Option' import { + buildPreflightError, + buildPreflightSummary, buildRequireError, buildSummary, type CombinedPart, combineParts, decodeJson, type Entry, + evaluatedOf, + evaluatedSummaryOf, type Job, JobOrNullSchema, JobsSchema, loadState, + mergeSarif, type Outcome, + type PackageSarif, type Part, type PartMeta, PartMetaSchema, + PREFLIGHT_FILE, ReportSchema, + SarifLogSchema, type Shard, slugOf, type StagedPart, } from './lib/mutation-plan.ts' +const PART_SARIF = 'mutation.sarif' + const incrementalFileOf = (shard: Shard | undefined): string => shard === undefined ? 'reports/stryker-incremental.json' @@ -59,18 +70,14 @@ const removeOutputsOfEarlierRun = async (reportsDir: string): Promise => { } } -const strykerUnderCap = async (name: string, shard: Shard | undefined, capSeconds: number): Promise => { +const strykerUnderCap = async ( + name: string, + shard: Shard | undefined, + capSeconds: number, + args: readonly string[], +): Promise => { const { code } = await new Deno.Command('timeout', { - args: [ - '--kill-after=60', - String(capSeconds), - 'pnpm', - '--filter', - name, - 'mutation', - '--incrementalFile', - incrementalFileOf(shard), - ], + args: ['--kill-after=60', String(capSeconds), 'pnpm', '--filter', name, 'mutation', ...args], env: { STRYKER_SHARD: shard === undefined ? '' : `${shard.index}/${shard.count}` }, stdout: 'inherit', stderr: 'inherit', @@ -93,7 +100,7 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom await removeOutputsOfEarlierRun(join(dir, 'reports')) if (cap > 0) { console.log(`::group::mutation ${labelOf(dir, shard)}`) - exitCode = await strykerUnderCap(name, shard, cap) + exitCode = await strykerUnderCap(name, shard, cap, ['--incrementalFile', incrementalFileOf(shard)]) console.log('::endgroup::') } else { console.log(`${name}: skipped, the job's ${budgetSeconds}s budget is spent`) @@ -131,6 +138,8 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom ) await copyIfPresent(join(reportsDir, 'mutation-report.json'), join(part, 'mutation-report.json')) await copyIfPresent(join(reportsDir, 'mutation-stream.jsonl'), join(part, 'mutation-stream.jsonl')) + await copyIfPresent(join(reportsDir, 'mutation-report.sarif'), join(part, PART_SARIF)) + await copyIfPresent(join(reportsDir, 'mutation', 'failure.json'), join(part, 'failure.json')) await copyIfPresent(join(dir, incrementalFile), join('incremental', dir, incrementalFile)) } return ok @@ -138,6 +147,60 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom const plannedPackages = (jobs: readonly Job[]): string[] => [...new Set(jobs.flatMap((job) => job.dirs))].sort() +const namedPackagesOf = (jobs: readonly Job[]): ReadonlyMap => + new Map(jobs.flatMap((job) => job.dirs.map((dir, position) => [dir, job.packages[position] ?? dir] as const))) + +const preflightPackage = async (name: string, dir: string, capSeconds: number): Promise => { + const reportsDir = join(dir, 'reports') + await removeOutputsOfEarlierRun(reportsDir) + console.log(`::group::preflight ${dir}`) + const exitCode = await strykerUnderCap(name, undefined, capSeconds, [ + '--dryRunOnly', + '--incrementalFile', + PREFLIGHT_FILE, + ]) + console.log('::endgroup::') + const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' + const input = { package: dir, outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: capSeconds } + const state = await loadState(reportsDir, readText) + const published = (await readIfPresent(join(dir, PREFLIGHT_FILE))) !== undefined + console.log(buildPreflightSummary(input, state, published)) + const error = buildPreflightError(input, state) + if (error !== null) console.log(error) + await copyIfPresent(join(dir, PREFLIGHT_FILE), join('preflight', dir, PREFLIGHT_FILE)) + await copyIfPresent( + join(reportsDir, 'mutation-stream.jsonl'), + join('preflight-evidence', dir, 'mutation-stream.jsonl'), + ) + await copyIfPresent(join(reportsDir, 'mutation', 'failure.json'), join('preflight-evidence', dir, 'failure.json')) + return outcome === 'success' +} + +const preflight = async (jobs: readonly Job[], capSeconds: number): Promise => { + let ok = true + for (const [dir, name] of [...namedPackagesOf(jobs)].sort(([a], [b]) => a.localeCompare(b))) { + if (!await preflightPackage(name, dir, capSeconds)) ok = false + } + return ok +} + +const readPackageSarifs = async (root: string): Promise => { + const sarifs: PackageSarif[] = [] + for await (const marker of expandGlob('**/mutation-part.json', { root })) { + const sarifPath = join(dirname(marker.path), PART_SARIF) + const text = await readIfPresent(sarifPath) + if (text === undefined) continue + const meta = decodeJson(PartMetaSchema, await Deno.readTextFile(marker.path), marker.path) + sarifs.push({ dir: meta.package, log: decodeJson(SarifLogSchema, text, sarifPath) }) + } + return sarifs +} + +const appendTo = async (variable: string, text: string): Promise => { + const file = Deno.env.get(variable) + if (file !== undefined && file !== '') await Deno.writeTextFile(file, text, { append: true }) +} + const readStagedParts = async (root: string): Promise => { const parts: StagedPart[] = [] for await (const marker of expandGlob('**/mutation-part.json', { root })) { @@ -178,19 +241,42 @@ const main = async (): Promise => { if (!await runJob(job, Number(args['cap-seconds']), Number(args['budget-seconds']))) Deno.exit(1) return } + if (command === 'preflight') { + if (args['cap-seconds'] === undefined) throw new Error('preflight needs --cap-seconds') + const jobs = decodeJson(JobsSchema, Deno.env.get('JOBS') ?? '[]', 'JOBS') + if (!await preflight(jobs, Number(args['cap-seconds']))) Deno.exit(1) + return + } if (command === 'combine') { if (args.parts === undefined || args.out === undefined) throw new Error('combine needs --parts and --out') const jobs = decodeJson(JobsSchema, Deno.env.get('JOBS') ?? '[]', 'JOBS') - await writeCombined(args.out, combineParts(await readStagedParts(args.parts))) - const envFile = Deno.env.get('GITHUB_ENV') + const staged = await readStagedParts(args.parts) + const combined = combineParts(staged) + await writeCombined(args.out, combined) const packages = JSON.stringify(plannedPackages(jobs)) - if (envFile !== undefined && envFile !== '') { - await Deno.writeTextFile(envFile, `PACKAGES=${packages}\n`, { append: true }) + const complete = combined.size > 0 && [...combined.values()].every((part) => part.report !== undefined) + await appendTo('GITHUB_ENV', `PACKAGES=${packages}\nMUTATION_REPORT_COMPLETE=${complete}\n`) + const evaluated = evaluatedOf(staged.flatMap((part) => (part.stream === undefined ? [] : [part.stream]))) + const evaluatedLine = evaluatedSummaryOf(evaluated) + await appendTo('GITHUB_STEP_SUMMARY', `${evaluatedLine}\n`) + if (Option.contains(evaluated, 0)) { + console.log('::notice title=Mutation evaluated no mutants::every verdict was reused, so this run tested nothing') } console.log(`planned packages: ${packages}`) + console.log(evaluatedLine) + return + } + if (command === 'sarif') { + if (args.parts === undefined || args.out === undefined) throw new Error('sarif needs --parts and --out') + const merged = Option.getOrUndefined(mergeSarif(await readPackageSarifs(args.parts))) + if (merged !== undefined) { + await Deno.mkdir(dirname(args.out), { recursive: true }) + await Deno.writeTextFile(args.out, JSON.stringify(merged)) + } + console.log(merged === undefined ? 'no part carried a SARIF log' : `wrote ${args.out}`) return } - throw new Error(`unknown command ${command ?? '(none)'}: expected run or combine`) + throw new Error(`unknown command ${command ?? '(none)'}: expected run, preflight, combine or sarif`) } if (import.meta.main) await main() From 950d2fb91866f48a2cf46eff7e6f087d7f32f35c Mon Sep 17 00:00:00 2001 From: systemfsoftware-maker Date: Fri, 2 Oct 2026 14:15:46 -0400 Subject: [PATCH 8/8] ci(repo): run the mutation lane on pull requests and gate new survivors (#156) * ci(repo): run the mutation lane on pull requests and gate new survivors A pull request runs the same preflight and sharded incremental run against main's restored verdict cache, saves no cache and records no timings, then gates its merged report against the survivor baseline main published. A run that evaluated no mutants says so in a notice instead of reading as a pass Verdict-Semantics: unchanged * ci(repo): plan mutation shards from the full mutant set's cost This stack's own pull request ran 7465 invalidated stryker-js mutants in one job planned at 9m43s and hit the 1800s cap: the timing record held main's reuse-shrunk duration. Each entry now carries the reuse line's ran and reused counts, the record scales a run to the full mutant set and keeps its previous value when nothing ran, and shards download the preflight coverage only when the preflight published some Verdict-Semantics: unchanged --- .github/workflows/mutation.yml | 34 +++++++++++++++++++++++++++---- scripts/lib/mutation-plan.test.ts | 9 ++++++++ scripts/lib/mutation-plan.ts | 32 ++++++++++++++++++++--------- scripts/mutation-job.ts | 15 +++++++++----- 4 files changed, 71 insertions(+), 19 deletions(-) diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 43817f08..f4f85317 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -3,6 +3,7 @@ name: Mutation on: push: branches: [main] + pull_request: workflow_dispatch: concurrency: @@ -57,6 +58,8 @@ jobs: if: ${{ needs.plan.result == 'success' && needs.plan.outputs.jobs != '[]' }} runs-on: ubuntu-latest timeout-minutes: 45 + outputs: + coverage: ${{ steps.preflight.outputs.coverage }} env: OTEL_ENABLED: "true" OTEL_SERVICE_NAME: stryker-js-mutation @@ -93,7 +96,8 @@ jobs: echo "TRACE_WINDOW_START=$(date +%s)" >> "$GITHUB_ENV" pnpm lgtm:up - - name: Preflight + - id: preflight + name: Preflight env: JOBS: ${{ needs.plan.outputs.jobs }} run: $MUTATION_JOB preflight --cap-seconds "$TARGET_SECONDS" @@ -163,7 +167,7 @@ jobs: - name: Place incremental files run: if [ -d .stryker-incremental ]; then cp -a .stryker-incremental/. .; fi - uses: actions/download-artifact@v8 - continue-on-error: true + if: ${{ needs.preflight.outputs.coverage == 'true' }} with: name: mutation-preflight path: . @@ -223,7 +227,7 @@ jobs: timings: name: record mutation timings needs: [plan, mutation] - if: ${{ always() && needs.plan.result == 'success' }} + if: ${{ always() && needs.plan.result == 'success' && github.event_name != 'pull_request' }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -280,7 +284,7 @@ jobs: path: .stryker-incremental merge-multiple: true - uses: actions/cache/save@v6 - if: ${{ hashFiles('.stryker-incremental/**') != '' }} + if: ${{ github.event_name != 'pull_request' && hashFiles('.stryker-incremental/**') != '' }} with: path: .stryker-incremental key: stryker-incremental-${{ github.run_id }} @@ -329,6 +333,28 @@ jobs: path: mutation-gate/mutation-survivor-baseline.json retention-days: 90 + - name: Survivor baseline from main + if: ${{ github.event_name == 'pull_request' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p mutation-gate/reports/mutation + id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=mutation-survivor-baseline&per_page=100" \ + --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_branch == "main")][0].id // empty') || id= + if [ -n "$id" ] && gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$id/zip" > mutation-gate/baseline.zip; then + unzip -o -q mutation-gate/baseline.zip -d mutation-gate + fi + - name: Gate new survivors + if: ${{ github.event_name == 'pull_request' }} + run: | + if [ ! -f mutation-gate/mutation-survivor-baseline.json ]; then + echo "::warning title=Mutation gate skipped::main has published no survivor baseline yet, so this run cannot tell a new survivor from an accepted one" + exit 0 + fi + cp mutation-merged/mutation-report.json mutation-gate/reports/mutation/mutation.json + cd mutation-gate + "$GITHUB_WORKSPACE/packages/stryker-js/node_modules/.bin/stryker" gate --baseline mutation-survivor-baseline.json + - name: Delete per-job artifacts if: ${{ always() && needs.mutation.result == 'success' }} continue-on-error: true diff --git a/scripts/lib/mutation-plan.test.ts b/scripts/lib/mutation-plan.test.ts index ed3d3abc..50fb525b 100644 --- a/scripts/lib/mutation-plan.test.ts +++ b/scripts/lib/mutation-plan.test.ts @@ -329,3 +329,12 @@ Deno.test('mergeSarif folds every package into one run with repository paths and ) assertEquals(mergeSarif([]), Option.none()) }) + +Deno.test('mergeRecord scales a reuse-shrunk duration to the full mutant set and keeps the record when nothing ran', () => { + const previous: TimingRecord = { version: 1, packages: { p: { seconds: 500, sha: 'old' } } } + const scaled: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 100, exitCode: 0, ran: 10, reused: 90 }] }] + assertEquals(mergeRecord(previous, scaled, 'new').packages['p'], { seconds: 1000, sha: 'new' }) + const nothing: Part[] = [{ job: 'p', entries: [{ package: 'p', seconds: 20, exitCode: 0, ran: 0, reused: 100 }] }] + assertEquals(mergeRecord(previous, nothing, 'new').packages['p'], { seconds: 500, sha: 'old' }) + assertEquals(mergeRecord(emptyRecord, nothing, 'new').packages['p'], { seconds: 20, sha: 'new' }) +}) diff --git a/scripts/lib/mutation-plan.ts b/scripts/lib/mutation-plan.ts index 24fe1ff3..86ddea67 100644 --- a/scripts/lib/mutation-plan.ts +++ b/scripts/lib/mutation-plan.ts @@ -29,6 +29,8 @@ export const EntrySchema = S.Struct({ seconds: S.Number, exitCode: S.Union([S.Int, S.Null]), shard: S.optional(ShardSchema), + ran: S.optional(S.Int), + reused: S.optional(S.Int), }) export type Entry = S.Schema.Type @@ -176,6 +178,19 @@ export const planJobs = ( return { jobs: [...shardJobs, ...wholeJobs] } } +const fullSetSecondsOf = (entry: Entry): number => + entry.ran === undefined || entry.reused === undefined || entry.ran === 0 + ? entry.seconds + : Math.round((entry.seconds * (entry.ran + entry.reused)) / entry.ran) + +const measuredOf = (entries: readonly Entry[]): number | undefined => { + const count = entries[0]?.shard?.count + if (count === undefined) return Math.max(...entries.map(fullSetSecondsOf)) + const indices = new Set(entries.map((entry) => entry.shard?.index)) + const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index)) + return complete ? entries.reduce((sum, entry) => sum + fullSetSecondsOf(entry), 0) : undefined +} + export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: string): TimingRecord => { const packages: Record = { ...previous.packages } const byPackage = new Map() @@ -183,14 +198,10 @@ export const mergeRecord = (previous: TimingRecord, parts: readonly Part[], sha: byPackage.set(entry.package, [...(byPackage.get(entry.package) ?? []), entry]) } for (const [name, entries] of byPackage) { - const count = entries[0]?.shard?.count - if (count === undefined) { - packages[name] = { seconds: Math.max(...entries.map((entry) => entry.seconds)), sha } - continue - } - const indices = new Set(entries.map((entry) => entry.shard?.index)) - const complete = Array.from({ length: count }, (_unused, i) => i + 1).every((index) => indices.has(index)) - if (complete) packages[name] = { seconds: entries.reduce((sum, entry) => sum + entry.seconds, 0), sha } + const evaluatedNothing = entries.some((entry) => entry.ran === 0) + const seconds = measuredOf(entries) + if (seconds === undefined || (evaluatedNothing && previous.packages[name] !== undefined)) continue + packages[name] = { seconds, sha } } return { version: 1, packages } } @@ -241,6 +252,7 @@ const RunFailedLineSchema = S.Struct({ const ReuseLineSchema = S.Struct({ _tag: S.Literal('reuse'), ran: S.Int.check(S.isGreaterThanOrEqualTo(0)), + reused: S.Int.check(S.isGreaterThanOrEqualTo(0)), }) const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder): ReadonlyArray => @@ -251,10 +263,10 @@ const decodedLines = (streamText: string | null, schema: S.ConstraintDecoder< const terminalRecordOf = (streamText: string | null): Option.Option => Option.map(Arr.last(decodedLines(streamText, RunFailedLineSchema)), (line) => line.record) -export type ReuseState = { readonly ran: number } +export type ReuseState = { readonly ran: number; readonly reused: number } const reuseOf = (streamText: string | null): Option.Option => - Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran })) + Option.map(Arr.last(decodedLines(streamText, ReuseLineSchema)), (line) => ({ ran: line.ran, reused: line.reused })) export type ReportState = { reportText: string | null diff --git a/scripts/mutation-job.ts b/scripts/mutation-job.ts index eddb9bee..fddd5716 100755 --- a/scripts/mutation-job.ts +++ b/scripts/mutation-job.ts @@ -106,12 +106,18 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom console.log(`${name}: skipped, the job's ${budgetSeconds}s budget is spent`) ok = false } + const seconds = Math.round((Date.now() - started) / 1000) + const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' + const reportsDir = join(dir, 'reports') + const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } + const state = await loadState(reportsDir, readText) if (exitCode !== null) { entries.push({ package: name, - seconds: Math.round((Date.now() - started) / 1000), + seconds, exitCode, ...(shard === undefined ? {} : { shard }), + ...Option.match(state.reuse, { onNone: () => ({}), onSome: ({ ran, reused }) => ({ ran, reused }) }), }) await Deno.mkdir('.timings', { recursive: true }) await Deno.writeTextFile( @@ -119,10 +125,6 @@ const runJob = async (job: Job, capSeconds: number, budgetSeconds: number): Prom JSON.stringify({ job: job.id, entries } satisfies Part), ) } - const outcome: Outcome = exitCode === 0 ? 'success' : 'failure' - const reportsDir = join(dir, 'reports') - const input = { package: labelOf(dir, shard), outcome, reportsDir, exitCode, cwd: Deno.cwd(), limitSeconds: cap } - const state = await loadState(reportsDir, readText) console.log(buildSummary(input, state)) const missing = buildRequireError(input, state) if (missing !== null) { @@ -178,9 +180,12 @@ const preflightPackage = async (name: string, dir: string, capSeconds: number): const preflight = async (jobs: readonly Job[], capSeconds: number): Promise => { let ok = true + let coverage = false for (const [dir, name] of [...namedPackagesOf(jobs)].sort(([a], [b]) => a.localeCompare(b))) { if (!await preflightPackage(name, dir, capSeconds)) ok = false + if ((await readIfPresent(join('preflight', dir, PREFLIGHT_FILE))) !== undefined) coverage = true } + await appendTo('GITHUB_OUTPUT', `coverage=${coverage}\n`) return ok }