diff --git a/.changeset/humble-coats-teach.md b/.changeset/humble-coats-teach.md new file mode 100644 index 000000000..7881d77cb --- /dev/null +++ b/.changeset/humble-coats-teach.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +A failed run's record reaches every surface: the Mutation Server Protocol answers engine failures with the record in error.data and keeps serving, MCP rerun_mutant returns a RerunEngineUnusable refusal carrying it and the new get_failure tool reads reports/mutation/failure.json, a configured sarif reporter writes a SARIF log whose invocation is executionSuccessful false with one notification per record, and the run log no longer prints an orphan 'Possible causes' block or the bundled parser's WASI ExperimentalWarning. diff --git a/packages/stryker-js/etc/stryker-js.api.md b/packages/stryker-js/etc/stryker-js.api.md index 1e8994a6c..08334102d 100644 --- a/packages/stryker-js/etc/stryker-js.api.md +++ b/packages/stryker-js/etc/stryker-js.api.md @@ -36,7 +36,7 @@ export namespace ImportClosure { // @public (undocumented) export namespace Mcp { - export { McpServerOptions, MutantDetail, MutantUnusable, RerunMutantFailure, ShowMutantFailure, mcpServerLayer, mcpToolkit }; + export { FailureLookup, GetFailure, McpServerOptions, MutantDetail, MutantUnusable, RerunEngineUnusable, RerunMutantFailure, ShowMutantFailure, mcpServerLayer, mcpToolkit }; } // @public (undocumented) diff --git a/packages/stryker-js/src/Mcp/mcp-server.cell.ts b/packages/stryker-js/src/Mcp/mcp-server.cell.ts index 2c25db384..b3b20dea7 100644 --- a/packages/stryker-js/src/Mcp/mcp-server.cell.ts +++ b/packages/stryker-js/src/Mcp/mcp-server.cell.ts @@ -1,39 +1,45 @@ -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, type Options, Report } from '@systemfsoftware/stryker-js-plugin-interface' import { McpProtocol, McpServer } from 'effect/ai' import * as Arr from 'effect/Array' import * as Cause from 'effect/Cause' import * as Effect from 'effect/Effect' +import * as Exit from 'effect/Exit' import * as FileSystem from 'effect/FileSystem' import * as Layer from 'effect/Layer' +import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Path from 'effect/Path' import * as Queue from 'effect/Queue' +import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { ReproducerSchema } from '../build-reproducers.workflow.js' +import { classifyRunOutcome, type RunOutcomeDecision } from '../classify-run-outcome.workflow.js' +import { runOutcomeCommandOf } from '../conclude-run.js' import type { ConfigReadError } from '../ConfigError.schema.js' import { recordFeedbackCell } from '../Feedback/Feedback.cell.js' import { FeedbackUnusable } from '../Feedback/Feedback.schema.js' import { readMutationReport, readSurfacedSurvivors } from '../Feedback/read-report.js' import { ResolvedMode } from '../output-mode.schema.js' import { MachineConsole } from '../reporting/machine-console.service.js' -import { mutantRerunAdmissionCell, RerunRefused } from '../Rerun/mod.js' +import { mutantRerunAdmissionCell, RerunEngineUnusable, RerunRefused } from '../Rerun/mod.js' import { mutantDetailEventsOf } from '../Rerun/rerun-selection.js' import type { MutationTestDone } from '../run/mutation-test.cell.js' import { mutationTestCell } from '../run/run-stages.cell.js' import { RunEnvironment, type RunEnvironmentShape } from '../run/RunEnvironment.service.js' import type { EnginePorts } from '../run/StageServices.service.js' +import { FAILURE_RECORD_FILE } from '../stryker-outputs.js' import { StrykerPackage } from '../stryker-package.schema.js' import { mcpToolkit } from './mcp-tools.js' -import { MutantDetail, MutantUnusable, RerunUnusable } from './mcp-tools.schema.js' +import { FailureLookup, MutantDetail, MutantUnusable, RerunMutantFailure, RerunUnusable } from './mcp-tools.schema.js' const MCP_RUN_ID = RunEvent.RunId.make('01ARZ3NDEKTSV4RRFFQ69G5FAM') const REPRODUCERS_FILE = 'reports/mutation/reproducers.json' const MCP_SERVER_INSTRUCTIONS = - 'Stryker mutation testing. Use list_survivors to read the surfaced survivors of the finished report, show_mutant to inspect one, rerun_mutant to re-run one through the run pipeline, and report_usefulness to record whether a survivor was worth acting on.' + 'Stryker mutation testing. Use list_survivors to read the surfaced survivors of the finished report, show_mutant to inspect one, rerun_mutant to re-run one through the run pipeline, report_usefulness to record whether a survivor was worth acting on, and get_failure to read the failure record the last failed run left behind.' const MCP_MODE = ResolvedMode.make({ mode: 'machine', signal: 'tool', stdoutIsTTY: false }) @@ -123,10 +129,30 @@ const restrictedOptionsOf = ({ incremental: true, }) +const engineDecisionOf = (exit: Exit.Exit, basePath: string): RunOutcomeDecision => + Result.getOrElse( + classifyRunOutcome(runOutcomeCommandOf({ exit, argv: [], cwd: basePath, traceId: null })), + (impossible) => impossible, + ) + +const engineRefusalOf = ( + exit: Exit.Exit, + basePath: string, +): Option.Option => + Match.value(engineDecisionOf(exit, basePath)).pipe( + Match.tag('RunFailed', (failed) => + Option.some(RerunEngineUnusable.make({ code: failed.code, record: failed.record }))), + Match.orElse(() => + Option.none() + ), + ) + +const NO_RESULTS: MutationTestDone = { results: [], verdict: null } + const runRestricted = ( basePath: string, options: Options.PartialStrykerOptions, -): Effect.Effect => +): Effect.Effect => Effect.gen(function*() { const queue = yield* Queue.bounded(RunEvent.RunEvent.QUEUE_BOUND) const environment: RunEnvironmentShape = { @@ -141,13 +167,23 @@ const runRestricted = ( RunEnvironment.stage(environment, queue), MachineConsole.captureLayer.pipe(Layer.provide(MachineConsole.layer)), ) - return yield* mutationTestCell - .run({ cliOptions: options, targetMutatePatterns: undefined }) - .pipe(Effect.provide(runLayer), Effect.scoped, Effect.orDie) + const exit = yield* Effect.exit( + mutationTestCell + .run({ cliOptions: options, targetMutatePatterns: undefined }) + .pipe(Effect.provide(runLayer), Effect.scoped), + ) + return yield* Exit.match(exit, { + onSuccess: (done) => Effect.succeed(done), + onFailure: (cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Option.match(engineRefusalOf(Exit.failCause(cause), basePath), { + onNone: () => Effect.die(new Error('an engine failure did not classify as a run failure')), + onSome: (refusal) => Effect.fail(refusal), + }), + }) }) -const NO_RESULTS: MutationTestDone = { results: [], verdict: null } - const resultsOf = (settled: void | MutationTestDone): ReadonlyArray => (settled ?? NO_RESULTS).results @@ -162,14 +198,20 @@ const reportedOf = ( const reasonOfRerun = (error: RerunRefused | ConfigReadError): string => S.is(RerunRefused)(error) ? error.reason : error.message -const rerunFailureOf = - (id: Mutant.MutantId) => (error: RerunRefused | ConfigReadError | MutantUnusable): MutantUnusable | RerunUnusable => - S.is(MutantUnusable)(error) ? error : RerunUnusable.make({ id, reason: reasonOfRerun(error) }) +const isRerunRefusal = (error: unknown): error is MutantUnusable | RerunEngineUnusable => + S.is(MutantUnusable)(error) || S.is(RerunEngineUnusable)(error) + +const rerunFailureOf = ( + id: Mutant.MutantId, +) => +( + error: RerunRefused | ConfigReadError | MutantUnusable | RerunEngineUnusable, +): RerunMutantFailure => isRerunRefusal(error) ? error : RerunUnusable.make({ id, reason: reasonOfRerun(error) }) const rerunMutant = ( basePath: string, id: Mutant.MutantId, -): Effect.Effect => +): Effect.Effect => Effect.gen(function*() { const settled = yield* mutantRerunAdmissionCell.run({ ids: [id], @@ -196,6 +238,16 @@ const rerunMutant = ( } }) +const readFailure = (basePath: string): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const text = yield* fs.readFileString(path.resolve(basePath, FAILURE_RECORD_FILE)).pipe( + Effect.orElseSucceed(() => ''), + ) + return { failure: Option.getOrNull(S.decodeOption(FailureRecord.FailureRecordFile)(text)) } + }) + const handlers = (basePath: string) => mcpToolkit.toLayer({ list_survivors: () => readSurfacedSurvivors(basePath), @@ -203,6 +255,7 @@ const handlers = (basePath: string) => rerun_mutant: ({ id }) => rerunMutant(basePath, id), report_usefulness: ({ id, judgment, reason }) => recordFeedbackCell({ basePath, id, judgment, reason: reason ?? null }), + get_failure: () => readFailure(basePath), }) export const mcpServerLayer = ({ basePath }: McpServerOptions) => diff --git a/packages/stryker-js/src/Mcp/mcp-tools.schema.ts b/packages/stryker-js/src/Mcp/mcp-tools.schema.ts index 33b372b48..4704fe94c 100644 --- a/packages/stryker-js/src/Mcp/mcp-tools.schema.ts +++ b/packages/stryker-js/src/Mcp/mcp-tools.schema.ts @@ -1,7 +1,9 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant } from '@systemfsoftware/stryker-js-plugin-interface' import * as S from 'effect/Schema' import { FeedbackUnusable } from '../Feedback/Feedback.schema.js' +import { RerunEngineUnusable } from '../Rerun/rerun.schema.js' export const MutantDetail = S.Struct({ id: Mutant.MutantId, @@ -31,6 +33,12 @@ export class RerunUnusable extends S.TaggedError()('RerunUnusable } } +export const FailureLookup = S.Struct({ failure: S.NullOr(FailureRecord.FailureRecord) }) + +export type FailureLookup = typeof FailureLookup.Type + export const ShowMutantFailure = S.Union([MutantUnusable, FeedbackUnusable]) -export const RerunMutantFailure = S.Union([MutantUnusable, RerunUnusable]) +export const RerunMutantFailure = S.Union([MutantUnusable, RerunUnusable, RerunEngineUnusable]) + +export type RerunMutantFailure = typeof RerunMutantFailure.Type diff --git a/packages/stryker-js/src/Mcp/mcp-tools.ts b/packages/stryker-js/src/Mcp/mcp-tools.ts index ea2fb3fd0..8aad737e5 100644 --- a/packages/stryker-js/src/Mcp/mcp-tools.ts +++ b/packages/stryker-js/src/Mcp/mcp-tools.ts @@ -11,7 +11,7 @@ import { FeedbackUnusable } from '../Feedback/Feedback.schema.js' import { GitDiff } from '../git-diff.service.js' import { SurvivorRef } from '../surfacing.schema.js' import { WorkerLauncher } from '../WorkerLauncher.service.js' -import { MutantDetail, RerunMutantFailure, ShowMutantFailure } from './mcp-tools.schema.js' +import { FailureLookup, MutantDetail, RerunMutantFailure, ShowMutantFailure } from './mcp-tools.schema.js' const reportDependencies = [FileSystem.FileSystem, Path.Path] @@ -63,4 +63,11 @@ export const ReportUsefulness = Tool.make('report_usefulness', { dependencies: reportDependencies, }) -export const mcpToolkit = Toolkit.make(ListSurvivors, ShowMutant, RerunMutant, ReportUsefulness) +export const GetFailure = Tool.make('get_failure', { + description: + 'Read the failure record the last finished run wrote to reports/mutation/failure.json: its stable code, evidence, reproduction and next action. Answers null when no failed run left a record.', + success: FailureLookup, + dependencies: reportDependencies, +}) + +export const mcpToolkit = Toolkit.make(ListSurvivors, ShowMutant, RerunMutant, ReportUsefulness, GetFailure) diff --git a/packages/stryker-js/src/Mcp/mod.ts b/packages/stryker-js/src/Mcp/mod.ts index 6bea3e78a..9a5874901 100644 --- a/packages/stryker-js/src/Mcp/mod.ts +++ b/packages/stryker-js/src/Mcp/mod.ts @@ -1,3 +1,10 @@ +export { RerunEngineUnusable } from '../Rerun/rerun.schema.js' export { mcpServerLayer, type McpServerOptions } from './mcp-server.cell.js' -export { mcpToolkit } from './mcp-tools.js' -export { MutantDetail, MutantUnusable, RerunMutantFailure, ShowMutantFailure } from './mcp-tools.schema.js' +export { GetFailure, mcpToolkit } from './mcp-tools.js' +export { + FailureLookup, + MutantDetail, + MutantUnusable, + RerunMutantFailure, + ShowMutantFailure, +} from './mcp-tools.schema.js' diff --git a/packages/stryker-js/src/Rerun/Rerun.cell.ts b/packages/stryker-js/src/Rerun/Rerun.cell.ts index a35a23064..d363b84c2 100644 --- a/packages/stryker-js/src/Rerun/Rerun.cell.ts +++ b/packages/stryker-js/src/Rerun/Rerun.cell.ts @@ -13,6 +13,7 @@ import type { MutationTestDone } from '../run/mutation-test.cell.js' import type { EnginePorts } from '../run/StageServices.service.js' import { PriorReportDocument } from '../Survivors/Survivors.schema.js' import { admitMutantRerun, type PriorMutantShape, RerunRefused } from './admit-mutant-rerun.workflow.js' +import type { RerunEngineUnusable } from './rerun.schema.js' export type MutantRerunSettled = void | MutationTestDone @@ -23,7 +24,7 @@ export interface MutantRerunRun { } export interface MutantRerunSettlement { - readonly runAdmitted: (run: MutantRerunRun) => Effect.Effect + readonly runAdmitted: (run: MutantRerunRun) => Effect.Effect } export interface MutantRerunInput { diff --git a/packages/stryker-js/src/Rerun/mod.ts b/packages/stryker-js/src/Rerun/mod.ts index a89677c57..438a4a2e9 100644 --- a/packages/stryker-js/src/Rerun/mod.ts +++ b/packages/stryker-js/src/Rerun/mod.ts @@ -6,3 +6,4 @@ export { type MutantRerunSettled, type MutantRerunSettlement, } from './Rerun.cell.js' +export { RerunEngineUnusable } from './rerun.schema.js' diff --git a/packages/stryker-js/src/Rerun/rerun.schema.ts b/packages/stryker-js/src/Rerun/rerun.schema.ts new file mode 100644 index 000000000..6a2f9ad3c --- /dev/null +++ b/packages/stryker-js/src/Rerun/rerun.schema.ts @@ -0,0 +1,8 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Plugin } from '@systemfsoftware/stryker-js-plugin-interface' +import * as S from 'effect/Schema' + +export class RerunEngineUnusable extends S.TaggedClass()('RerunEngineUnusable', { + code: Plugin.ExitCode, + record: FailureRecord.FailureRecord, +}) {} diff --git a/packages/stryker-js/src/Serve/Serve.cell.ts b/packages/stryker-js/src/Serve/Serve.cell.ts index 4dcc51f61..36f0265e2 100644 --- a/packages/stryker-js/src/Serve/Serve.cell.ts +++ b/packages/stryker-js/src/Serve/Serve.cell.ts @@ -14,14 +14,15 @@ */ import { NodeSocketServer } from '@effect/platform-node' import { Cell } from '@systemfsoftware/effect-cell-types' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import { HtmlReporter } from '@systemfsoftware/stryker-js-html-reporter' -import type { Mutant, Options } from '@systemfsoftware/stryker-js-plugin-interface' -import type * as Cause from 'effect/Cause' +import { type Mutant, type Options } from '@systemfsoftware/stryker-js-plugin-interface' +import * as Cause from 'effect/Cause' import * as Clock from 'effect/Clock' import * as Context from 'effect/Context' import * as DateTime from 'effect/DateTime' import * as Effect from 'effect/Effect' +import * as Exit from 'effect/Exit' import * as Fiber from 'effect/Fiber' import * as FiberSet from 'effect/FiberSet' import * as FileSystem from 'effect/FileSystem' @@ -39,6 +40,8 @@ import * as SocketServer from 'effect/socket/SocketServer' import * as Stdio from 'effect/Stdio' import * as Stream from 'effect/Stream' +import { classifyRunOutcome, type RunOutcomeDecision } from '../classify-run-outcome.workflow.js' +import { runOutcomeCommandOf } from '../conclude-run.js' import { concurrencyCell } from '../concurrency.cell.js' import type { ResolvedMode } from '../output-mode.schema.js' import { readProjectCell } from '../read-project.cell.js' @@ -108,6 +111,28 @@ interface EngineRun { readonly context: Context.Context } +const engineDecisionOf = (exit: Exit.Exit, basePath: string): RunOutcomeDecision => + Result.getOrElse( + classifyRunOutcome(runOutcomeCommandOf({ exit, argv: [], cwd: basePath, traceId: null })), + (impossible) => impossible, + ) + +const engineFailureOf = ( + exit: Exit.Exit, + basePath: string, + reason: string, +): Effect.Effect => + Exit.match(exit, { + onSuccess: (value) => Effect.succeed(value), + onFailure: (cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Match.value(engineDecisionOf(Exit.failCause(cause), basePath)).pipe( + Match.tag('RunFailed', (failed) => Effect.fail(ServeError.make({ reason, cause: failed.record }))), + Match.orElse(() => Effect.die(new Error(reason))), + ), + }) + const serveEnvironment = (basePath: string, startedAt: number): RunEnvironmentShape => ({ runId: generateRunId(DateTime.makeUnsafe(startedAt)), resolvedMode: HEADLESS_MODE, @@ -299,14 +324,15 @@ const runDiscover = ( withEngine(() => Effect.void, ({ env, context }) => Effect.gen(function*() { const path = yield* Path.Path - const done = yield* Cell.provideContext(discoverStageCell, context).run({ - cliOptions: options, - targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], - }) + const exit = yield* Effect.exit( + Cell.provideContext(discoverStageCell, context).run({ + cliOptions: options, + targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], + }), + ) + const done = yield* engineFailureOf(exit, env.basePath, 'discovery failed') return { files: discoveredFilesOf(done.mutants, (fileName) => path.relative(env.basePath, fileName)) } - }).pipe( - Effect.mapError((failure) => ServeError.make({ reason: describe(failure, 'discovery failed'), cause: failure })), - )) + })) const runMutationTest = ( options: ServeOptions, @@ -322,16 +348,15 @@ const runMutationTest = ( ({ env, context }) => Effect.gen(function*() { const path = yield* Path.Path - const done = yield* Cell.provideContext(mutationTestCell, context).run({ - cliOptions: options, - targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], - }) + const exit = yield* Effect.exit( + Cell.provideContext(mutationTestCell, context).run({ + cliOptions: options, + targetMutatePatterns: targetMutatePatterns === undefined ? undefined : [...targetMutatePatterns], + }), + ) + const done = yield* engineFailureOf(exit, env.basePath, 'the mutation test failed') return { files: mutationTestFilesOf(done.results, (fileName) => path.relative(env.basePath, fileName)) } - }).pipe( - Effect.mapError((failure) => - ServeError.make({ reason: describe(failure, 'the mutation test failed'), cause: failure }) - ), - ), + }), ) const respond = (connection: Connection, id: JsonRpcId, result: S.Json): Effect.Effect => @@ -345,12 +370,36 @@ const respondError = ( id: JsonRpcId | null, code: number, message: string, + data?: S.Json, ): Effect.Effect => Effect.flatMap( - encodedOr(encodeJsonRpcResponse({ jsonrpc: '2.0' as const, id, error: { code, message } }), 'an error response'), + encodedOr( + encodeJsonRpcResponse({ + jsonrpc: '2.0' as const, + id, + error: { code, message, ...(data === undefined ? {} : { data }) }, + }), + 'an error response', + ), (payload) => connection.write(payload), ) +const respondFailure = ( + connection: Connection, + id: JsonRpcId, + failure: ServeError, +): Effect.Effect => + Option.match( + Option.flatMap( + Option.fromUndefinedOr(failure.cause), + (cause) => S.decodeUnknownOption(FailureRecord.FailureRecord)(cause), + ), + { + onNone: () => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason), + onSome: (record) => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason, record), + }, + ) + const notifyProgress = ( connection: Connection, notification: ReportMutationTestProgress, @@ -382,7 +431,7 @@ const runRequested = ( runDiscover(optionsOf(invocation, yield* state), restrictPatternsOf(params.files)), ) yield* Result.match(result, { - onFailure: (failure) => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason), + onFailure: (failure) => respondFailure(connection, id, failure), onSuccess: (files) => respond(connection, id, files), }) })), @@ -396,7 +445,7 @@ const runRequested = ( ), ) yield* Result.match(result, { - onFailure: (failure) => respondError(connection, id, MSP_INTERNAL_ERROR, failure.reason), + onFailure: (failure) => respondFailure(connection, id, failure), onSuccess: (files) => respond(connection, id, files), }) })), diff --git a/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts b/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts index d38b67a39..36940f15d 100644 --- a/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/sarif-report.workflow.property.test.ts @@ -1,16 +1,27 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import { Mutant, type Report } from '@systemfsoftware/stryker-js-plugin-interface' import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' import * as Arr from 'effect/Array' import * as Match from 'effect/Match' import * as Option from 'effect/Option' +import * as Predicate from 'effect/Predicate' import * as Result from 'effect/Result' import * as S from 'effect/Schema' -import { sarifReport, SarifReportCommand, SarifReportRendered } from '../sarif-report.workflow.js' +import { + FailureReportSource, + SarifFailureReportRendered, + sarifReport, + SarifReportCommand, + type SarifReportDecision, + SarifReportRendered, + SurvivorsReportSource, +} from '../sarif-report.workflow.js' import type { SurvivorRef } from '../surfacing.schema.js' const GENEROUS_BUDGET = Number.MAX_SAFE_INTEGER +const TEST_TOOL = { name: 'StrykerJS', version: '0.0.0', informationUri: 'https://stryker-mutator.io' } as const type Subject = typeof sarifReport const isSurvivorStatus = (status: string): boolean => status === 'Survived' || status === 'NoCoverage' @@ -76,37 +87,69 @@ const refsOf = (report: Report.MutationTestResult): ReadonlyArray = const commandArb: Arbitrary.Arbitrary = Arbitrary.all({ report: reportArb, - tool: Arbitrary.all({ - name: Arbitrary.schema(S.String), - version: Arbitrary.schema(S.String), - informationUri: Arbitrary.schema(S.String), - }), maxResults: Arbitrary.schema(S.Natural), }).pipe( - Arbitrary.map(({ report, ...rest }) => SarifReportCommand.make({ report, survivors: refsOf(report), ...rest })), + Arbitrary.map(({ report, maxResults }) => + SarifReportCommand.make({ + source: SurvivorsReportSource.make({ report, survivors: refsOf(report), maxResults }), + tool: TEST_TOOL, + }) + ), ) const withBudget = ( command: SarifReportCommand, survivors: ReadonlyArray, maxResults: number, -): SarifReportCommand => SarifReportCommand.make({ report: command.report, survivors, tool: command.tool, maxResults }) +): SarifReportCommand => { + const source = command.source + return SarifReportCommand.make({ + source: S.is(SurvivorsReportSource)(source) + ? SurvivorsReportSource.make({ report: source.report, survivors, maxResults }) + : source, + tool: command.tool, + }) +} + +const survivorsOf = (command: SarifReportCommand): ReadonlyArray => + S.is(SurvivorsReportSource)(command.source) ? [...command.source.survivors] : [] const decisionOf = (subject: Subject, command: SarifReportCommand) => subject(command).pipe(Result.getOrThrow) const runOf = (subject: Subject, command: SarifReportCommand) => Option.getOrThrow(Arr.head(decisionOf(subject, command).log.runs)) +const failureDecisionOf = (source: FailureReportSource): SarifReportDecision => + sarifReport(SarifReportCommand.make({ source, tool: TEST_TOOL })).pipe(Result.getOrThrow) + +const failureRunOf = (decision: SarifReportDecision) => + Option.getOrThrow( + Arr.head(Option.getOrThrow(Option.liftPredicate(decision, S.is(SarifFailureReportRendered))).log.runs), + ) + +const failureCodeOf = (record: FailureRecord.FailureRecord): FailureRecord.FailureCode => + Option.getOrThrow(Arr.findFirst(FailureRecord.FailureCode.literals, (code) => Predicate.isTagged(record, code))) + +const recordLocationsOf = (record: FailureRecord.FailureRecord): ReadonlyArray => + Match.value(record).pipe( + Match.tag( + 'BaselineTestsFailed', + (evidence) => Arr.flatMap(evidence.tests, (test) => Option.toArray(Option.fromNullishOr(test.location))), + ), + Match.orElse((): ReadonlyArray => []), + ) + describe('sarifReport', () => { it.prop( '∀c_SarifReportCommand_≡AGenerousBudgetEmitsOneResultPerCommandedSurvivor', { of: [commandArb], subject: sarifReport }, (subject, [command]) => { - const results = runOf(subject, withBudget(command, command.survivors, GENEROUS_BUDGET)).results + const survivors = survivorsOf(command) + const results = runOf(subject, withBudget(command, survivors, GENEROUS_BUDGET)).results const fingerprints = Arr.map(results, (result) => result.partialFingerprints.primaryLocationLineHash) - return results.length === command.survivors.length && + return results.length === survivors.length && new Set(fingerprints).size === results.length && - fingerprints.every((id) => command.survivors.some((ref) => ref.id === id)) + fingerprints.every((id) => survivors.some((ref) => ref.id === id)) }, ) @@ -123,20 +166,60 @@ describe('sarifReport', () => { '∀c_SarifReportCommand_≡AGenerousBudgetNeverTruncates', { of: [commandArb], subject: sarifReport }, (subject, [command]) => - S.is(SarifReportRendered)(decisionOf(subject, withBudget(command, command.survivors, GENEROUS_BUDGET))), + S.is(SarifReportRendered)(decisionOf(subject, withBudget(command, survivorsOf(command), GENEROUS_BUDGET))), ) it.prop( '∀c_SarifReportCommand_≡AZeroResultBudgetEmitsNoResultAndDeclaresTheOmission', { of: [commandArb], subject: sarifReport }, (subject, [command]) => { - const decision = decisionOf(subject, withBudget(command, command.survivors, 0)) + const decision = decisionOf(subject, withBudget(command, survivorsOf(command), 0)) const results = Option.getOrThrow(Arr.head(decision.log.runs)).results return Match.value(decision).pipe( - Match.tag('SarifReportRendered', () => results.length === 0 && command.survivors.length === 0), + Match.tag('SarifReportRendered', () => results.length === 0 && survivorsOf(command).length === 0), Match.tag('SarifReportTruncated', (truncated) => results.length === 0 && truncated.omitted > 0), + Match.tag('SarifFailureReportRendered', () => false), Match.exhaustive, ) }, ) }) + +describe('sarifReport failure notifications', () => { + it.prop( + '∀s_FailureReportSource_≡AFailedInvocationNotifiesOncePerRecord', + { of: [FailureReportSource], subject: failureDecisionOf }, + (subject, [source]) => { + const invocation = Option.getOrThrow(Arr.head(failureRunOf(subject(source)).invocations)) + return invocation.exitCode === source.exitCode && + invocation.toolExecutionNotifications.length === source.records.length && + Arr.every( + Arr.zip(source.records, invocation.toolExecutionNotifications), + ([record, notification]) => + notification.descriptor.id === failureCodeOf(record) && + notification.message.text === FailureRecord.sarifTextOf(record), + ) + }, + ) + + it.prop( + '∀s_FailureReportSource_≡OnlyLocatedBaselineTestsCarryAPhysicalLocation', + { of: [FailureReportSource], subject: failureDecisionOf }, + (subject, [source]) => { + const notifications = Option.getOrThrow(Arr.head(failureRunOf(subject(source)).invocations)) + .toolExecutionNotifications + return notifications.length === source.records.length && + Arr.every(Arr.zip(source.records, notifications), ([record, notification]) => { + const expected = recordLocationsOf(record) + return notification.locations.length === expected.length && + Arr.every( + Arr.zip(expected, notification.locations), + ([location, projected]) => + decodeURIComponent(projected.physicalLocation.artifactLocation.uri) === location.file.toWellFormed() && + projected.physicalLocation.region.startLine === location.line && + projected.physicalLocation.region.startColumn === location.column, + ) + }) + }, + ) +}) diff --git a/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts b/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts index c98122cc5..0fe5d4a8b 100644 --- a/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts +++ b/packages/stryker-js/src/__tests__/validate-options-admission.workflow.property.test.ts @@ -1,25 +1,45 @@ import { describe, it } from '@systemfsoftware/vitest' import * as Arbitrary from 'effect/Arbitrary' +import * as Match from 'effect/Match' import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { forkOptionsSchema } from '../Config.schema.js' import { OptionsRefused, OptionsValidated, type OptionsValidationDecision, validateOptionsAdmission, ValidateOptionsCommand, + type ValidationSchemaDocument, } from '../run/validate-options-admission.workflow.js' -const commandOf = (options: Record): ValidateOptionsCommand => - ValidateOptionsCommand.make({ options, schema: {} }) +const CORE_SCHEMA_DOCUMENT: ValidationSchemaDocument = S.toJsonSchemaDocument(forkOptionsSchema).schema -const decide = (subject: typeof validateOptionsAdmission, options: Record) => - subject(commandOf(options)).pipe(Result.getOrElse((neverError) => neverError)) +const commandOf = ( + options: Record, + schema: ValidationSchemaDocument = {}, +): ValidateOptionsCommand => ValidateOptionsCommand.make({ options, schema }) + +const decide = ( + subject: typeof validateOptionsAdmission, + options: Record, + schema?: ValidationSchemaDocument, +) => subject(commandOf(options, schema)).pipe(Result.getOrElse((neverError) => neverError)) const refusedWith = (decision: OptionsValidationDecision, fragment: string): boolean => S.is(OptionsRefused)(decision) && decision.errors.some((error) => error.includes(fragment)) +const warningsIn = (decision: OptionsValidationDecision): readonly string[] => + Match.valueTags(decision, { + OptionsRefused: (refused) => refused.warnings, + OptionsUndecodable: (undecodable) => undecodable.warnings, + OptionsValidated: (validated) => validated.warnings, + }) + +const hasUnknownOptionWarning = (warnings: readonly string[]): boolean => + warnings.some((warning) => warning.includes('Unknown stryker config option') || warning.includes('Possible causes')) + const rangeArb = Arbitrary.schema(S.Struct({ startLine: S.Int.check(S.isBetween({ minimum: 0, maximum: 8 })), endLine: S.Int.check(S.isBetween({ minimum: 0, maximum: 8 })), @@ -30,6 +50,16 @@ const globArb = Arbitrary.schema(S.Struct({ name: S.String.check(S.isPattern(/^[a-z][a-z0-9]{0,4}$/)), })) +const unknownOptionArb = Arbitrary.schema(S.Struct({ + include: S.Boolean, + name: S.String.check(S.isPattern(/^k[a-z0-9]{0,4}$/)), +})) + +const optionsWith = (include: boolean, name: string): Record => { + const known: Record = { warnings: true } + return include ? { ...known, [name]: 42 } : known +} + describe('validateOptionsAdmission', () => { it.prop( '∀m_Mutate_≡RangeBoundsAreEnforced', @@ -84,4 +114,15 @@ describe('validateOptionsAdmission', () => { : S.is(OptionsValidated)(decision) }, ) + + it.prop( + '∀k_Included_≡UnknownOptionIsWarnedExactlyWhenTheSchemaOmitsIt', + { of: [unknownOptionArb], subject: validateOptionsAdmission }, + (subject, [{ include, name }]) => { + const decision = decide(subject, optionsWith(include, name), CORE_SCHEMA_DOCUMENT) + const warnings = warningsIn(decision) + const namesTheOption = warnings.some((warning) => warning.includes(`Unknown stryker config option "${name}".`)) + return namesTheOption === include && hasUnknownOptionWarning(warnings) === include + }, + ) }) diff --git a/packages/stryker-js/src/bin/main.ts b/packages/stryker-js/src/bin/main.ts index ee5bb4c08..584360fb0 100755 --- a/packages/stryker-js/src/bin/main.ts +++ b/packages/stryker-js/src/bin/main.ts @@ -9,6 +9,7 @@ import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics' import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { HtmlReporter } from '@systemfsoftware/stryker-js-html-reporter' +import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import cliPkgJson from '@systemfsoftware/stryker-js/package.json' with { type: 'json' } import * as Boolean from 'effect/Boolean' import * as Cause from 'effect/Cause' @@ -50,6 +51,25 @@ import { makeStrykerCommand } from './cli-command.js' globalThis.process.title = 'stryker' +const WASI_WARNING_MENTION = 'WASI' + +const isWasiExperimentalWarning = (warning: Error): boolean => + Boolean.and(warning.name === 'ExperimentalWarning', warning.message.includes(WASI_WARNING_MENTION)) + +const NODE_WARNING_PRINTERS = globalThis.process.listeners('warning') + +globalThis.process.removeAllListeners('warning') + +globalThis.process.on('warning', (warning) => + Boolean.match(isWasiExperimentalWarning(warning), { + onFalse: () => { + for (const print of NODE_WARNING_PRINTERS) { + print(warning) + } + }, + onTrue: () => undefined, + })) + const EXPORT_TIMEOUT_MILLIS = 5000 const SHUTDOWN_TIMEOUT = EffectDuration.millis(EXPORT_TIMEOUT_MILLIS + 1_000) @@ -266,11 +286,16 @@ const strykerProgram = Effect.gen(function*() { 'stryker.run.exit_code': runExitCodeFromOutcome(decision).code, ...failureCodeAttributeOf(decision), }) + const resolvedOptions = yield* Option.match(Option.fromUndefinedOr(host.resolvedOptions), { + onNone: () => Effect.succeed(null), + onSome: (resolved) => Ref.get(resolved), + }) return yield* concludeRunCell.run({ mode, stream, basePath: host.basePath, pathService, + resolvedOptions, fileSystem: yield* FileSystem.FileSystem, runEvents, decision, diff --git a/packages/stryker-js/src/conclude-run.cell.ts b/packages/stryker-js/src/conclude-run.cell.ts index f5c45cf8b..a180dc6f0 100644 --- a/packages/stryker-js/src/conclude-run.cell.ts +++ b/packages/stryker-js/src/conclude-run.cell.ts @@ -1,16 +1,21 @@ import { Sandwich } from '@systemfsoftware/effect-cell-types' import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' +import type { Options } from '@systemfsoftware/stryker-js-plugin-interface' import * as Effect from 'effect/Effect' import type * as FileSystem from 'effect/FileSystem' +import * as Option from 'effect/Option' import type * as Path from 'effect/Path' +import * as Result from 'effect/Result' import * as S from 'effect/Schema' import { RunExit, RunOutcomeDecision } from './classify-run-outcome.workflow.js' import type { ResolvedMode } from './output-mode.schema.js' import { planRunConclusion, type PlanRunConclusionCommand } from './plan-run-conclusion.workflow.js' +import { SARIF_TOOL } from './reporter-factories.js' import type { RunEventDrain, RunEventStream, RunEventStreamPort } from './run-event-stream.service.js' +import { FailureReportSource, sarifReport, SarifReportCommand } from './sarif-report.workflow.js' import { StrykerError } from './stryker-error.schema.js' -import { FAILURE_RECORD_FILE } from './stryker-outputs.js' +import { FAILURE_RECORD_FILE, sarifFileNameOf } from './stryker-outputs.js' export interface RunConclusionInput { readonly mode: ResolvedMode @@ -20,6 +25,7 @@ export interface RunConclusionInput { readonly fileSystem: FileSystem.FileSystem readonly runEvents: RunEventStreamPort readonly decision: RunOutcomeDecision + readonly resolvedOptions: Options.StrykerOptions | null } export type RunConclusionRaw = (typeof PlanRunConclusionCommand)['Encoded'] & { @@ -58,6 +64,52 @@ const removeStaleFailureRecord = (conclusion: RunConclusionInput) => Effect.ignore, ) +const SARIF_REPORTER_NAME = 'sarif' + +const sarifFailureFileOf = (conclusion: RunConclusionInput): string | null => + Option.getOrNull( + Option.map( + Option.filter( + Option.fromNullishOr(conclusion.resolvedOptions), + (options) => options.reporters.includes(SARIF_REPORTER_NAME), + ), + (options) => conclusion.pathService.resolve(conclusion.basePath, sarifFileNameOf(options.jsonReporter.fileName)), + ), + ) + +const writeSarifFailureLog = ( + conclusion: RunConclusionInput, + file: string, + exitCode: number, + record: FailureRecord.FailureRecord, +) => + Effect.gen(function*() { + const rendered = Result.getOrThrow( + sarifReport( + SarifReportCommand.make({ + source: FailureReportSource.make({ records: [record], exitCode }), + tool: SARIF_TOOL, + }), + ), + ) + const json = yield* S.encodeEffect(S.fromJsonString(S.Unknown, { space: 2 }))(rendered.log) + yield* conclusion.fileSystem.makeDirectory(conclusion.pathService.dirname(file), { recursive: true }) + yield* conclusion.fileSystem.writeFileString(file, json) + }).pipe( + Effect.tapError((error) => Effect.logWarning(`could not write the SARIF failure log: ${error.message}`)), + Effect.ignore, + ) + +const writeFailureSarif = ( + conclusion: RunConclusionInput, + exitCode: number, + record: FailureRecord.FailureRecord, +) => + Effect.forEach( + Option.toArray(Option.fromNullishOr(sarifFailureFileOf(conclusion))), + (file) => writeSarifFailureLog(conclusion, file, exitCode, record), + ).pipe(Effect.asVoid) + const closeAndExit = (conclusion: RunConclusionInput, exitCode: number) => Effect.andThen(conclusion.stream.closeAndDrain, Effect.fail(RunExit.make({ code: exitCode }))) @@ -89,6 +141,7 @@ export const concludeRunCell = Sandwich.named(SpanTaxonomy.Spans.runConclude.nam ), RunConclusionFailed: (decision, raw) => writeFailureRecord(raw.conclusion, decision.record).pipe( + Effect.andThen(writeFailureSarif(raw.conclusion, decision.exitCode, decision.record)), Effect.andThen( raw.conclusion.runEvents.emitFailureRecord(raw.conclusion.stream, decision.exitCode, decision.record), ), diff --git a/packages/stryker-js/src/reporter-factories.ts b/packages/stryker-js/src/reporter-factories.ts index 85ca6051d..2c67063cc 100644 --- a/packages/stryker-js/src/reporter-factories.ts +++ b/packages/stryker-js/src/reporter-factories.ts @@ -21,7 +21,7 @@ import { progressReportCell } from './progress-report.cell.js' import type { ProgressState, ProgressTally } from './render-progress-report.workflow.js' import { failAsProgress } from './reporter-failures.js' import { ReporterOutput } from './reporter-output.service.js' -import { sarifReport, SarifReportCommand } from './sarif-report.workflow.js' +import { sarifReport, SarifReportCommand, SurvivorsReportSource } from './sarif-report.workflow.js' import { sarifFileNameOf } from './stryker-outputs.js' import { StrykerPackage } from './stryker-package.schema.js' import { surfacedSurvivorsOf } from './surfacing.js' @@ -85,6 +85,12 @@ export const progressReporterFactory = ( const SARIF_MAX_RESULTS = 5000 const STRYKER_INFORMATION_URI = 'https://stryker-mutator.io' +export const SARIF_TOOL = { + name: 'StrykerJS', + version: StrykerPackage.version, + informationUri: STRYKER_INFORMATION_URI, +} as const + const failAsSarif = (cause: E): Reporter.ReporterFailed => Reporter.ReporterFailed.make({ reporterName: 'sarif', @@ -117,17 +123,15 @@ const writeSarifReport = (options: Options.StrykerOptions, report: Report.Mutati const rendered = Result.getOrThrow( sarifReport( SarifReportCommand.make({ - report, - survivors: surfacedSurvivorsOf(report, { - perLine: options.surfacing.perLine, - perFile: options.surfacing.perFile, + source: SurvivorsReportSource.make({ + report, + survivors: surfacedSurvivorsOf(report, { + perLine: options.surfacing.perLine, + perFile: options.surfacing.perFile, + }), + maxResults: SARIF_MAX_RESULTS, }), - tool: { - name: 'StrykerJS', - version: StrykerPackage.version, - informationUri: STRYKER_INFORMATION_URI, - }, - maxResults: SARIF_MAX_RESULTS, + tool: SARIF_TOOL, }), ), ) diff --git a/packages/stryker-js/src/run/RunEnvironment.service.ts b/packages/stryker-js/src/run/RunEnvironment.service.ts index b11bfb76f..5ac3e38c8 100644 --- a/packages/stryker-js/src/run/RunEnvironment.service.ts +++ b/packages/stryker-js/src/run/RunEnvironment.service.ts @@ -1,5 +1,5 @@ import { RunEvent, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' -import type { Reporter as InterfaceReporter } from '@systemfsoftware/stryker-js-plugin-interface' +import type { Options, Reporter as InterfaceReporter } from '@systemfsoftware/stryker-js-plugin-interface' import type * as Cause from 'effect/Cause' import * as Clock from 'effect/Clock' import * as Context from 'effect/Context' @@ -13,6 +13,7 @@ import * as Option from 'effect/Option' import type { PlatformError } from 'effect/PlatformError' import * as Predicate from 'effect/Predicate' import * as Queue from 'effect/Queue' +import * as Ref from 'effect/Ref' import * as S from 'effect/Schema' import * as Scope from 'effect/Scope' @@ -34,6 +35,7 @@ export interface RunEnvironmentShape { readonly basePath: string readonly builtinReporters: Readonly> readonly allowConsoleColors: boolean + readonly resolvedOptions?: Ref.Ref | undefined } export class RunEnvironment extends Context.Service()( @@ -81,9 +83,11 @@ export class RunEnvironment extends Context.Service> }, ): Effect.Effect => - Effect.map( - Effect.flatMap(FileSystem.FileSystem, (fs) => fs.realPath('.')), - (basePath) => ({ + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const basePath = yield* fs.realPath('.') + const resolvedOptions = yield* Ref.make(null) + return { runId: stream.runId, resolvedMode: mode, runStartedAt: stream.startedAt, @@ -91,8 +95,9 @@ export class RunEnvironment extends Context.Service( core, ) +const publishResolvedOptions = ( + env: RunEnvironmentShape, + options: Options.StrykerOptions, +): Effect.Effect => + Option.match(Option.fromUndefinedOr(env.resolvedOptions), { + onNone: () => Effect.void, + onSome: (resolved) => Ref.set(resolved, options), + }) + const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( command: ReadProjectDone, ): Effect.fn.Return< @@ -158,6 +168,7 @@ const readPrepare = Effect.fn(SpanTaxonomy.Spans.prepareGather.name)(function*( allowColor: env.allowConsoleColors, }, } + yield* publishResolvedOptions(env, options) const descriptors: readonly string[] = pluginUrlsFromOptions(options) const loaded = yield* loadPlugins(descriptors, env.basePath).pipe( Effect.tapError(() => diff --git a/packages/stryker-js/src/run/validate-options-admission.workflow.ts b/packages/stryker-js/src/run/validate-options-admission.workflow.ts index a7038137b..efb52f832 100644 --- a/packages/stryker-js/src/run/validate-options-admission.workflow.ts +++ b/packages/stryker-js/src/run/validate-options-admission.workflow.ts @@ -194,9 +194,13 @@ const unknownOptionWarnings = ( schema: ValidationSchemaDocument, ): readonly string[] => { const excessNames = excessOptionNames(options, schema) - return excessNames.map((excess) => `Unknown stryker config option "${excess}".`).concat( - POSSIBLE_CAUSES_OF(pluginsJson(options)), - ) + return Boolean.match(excessNames.length > 0, { + onFalse: (): readonly string[] => [], + onTrue: () => [ + ...excessNames.map((excess) => `Unknown stryker config option "${excess}".`), + POSSIBLE_CAUSES_OF(pluginsJson(options)), + ], + }) } const excessOptionWarningsOf = ( diff --git a/packages/stryker-js/src/sarif-failure.schema.ts b/packages/stryker-js/src/sarif-failure.schema.ts new file mode 100644 index 000000000..a33774bd9 --- /dev/null +++ b/packages/stryker-js/src/sarif-failure.schema.ts @@ -0,0 +1,53 @@ +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import * as Arr from 'effect/Array' +import * as Option from 'effect/Option' +import * as Predicate from 'effect/Predicate' +import * as S from 'effect/Schema' + +const encodeUriSegment = encodeURIComponent + +const uriOf = (fileName: string): string => + Arr.join(Arr.map(fileName.split('/'), (segment) => encodeUriSegment(segment.toWellFormed())), '/') + +export const SarifFailureLocation = S.Struct({ + physicalLocation: S.Struct({ + artifactLocation: S.Struct({ uri: S.String }), + region: S.Struct({ startLine: S.Natural, startColumn: S.Natural }), + }), +}) + +export type SarifFailureLocation = typeof SarifFailureLocation.Type + +export const SarifFailureNotification = S.Struct({ + descriptor: S.Struct({ id: S.String }), + level: S.Literal('error'), + message: S.Struct({ text: S.String }), + locations: S.Array(SarifFailureLocation), +}) + +export type SarifFailureNotification = typeof SarifFailureNotification.Type + +export type SarifFailureNotifications = ReadonlyArray + +const locatedTestsOf = (record: FailureRecord.FailureRecord): ReadonlyArray => + Predicate.isTagged(record, 'BaselineTestsFailed') + ? Arr.flatMap(record.tests, (test) => Option.toArray(Option.fromNullishOr(test.location))) + : [] + +const locationOf = (location: FailureRecord.SourceLocation): SarifFailureLocation => ({ + physicalLocation: { + artifactLocation: { uri: uriOf(location.file) }, + region: { startLine: location.line, startColumn: location.column }, + }, +}) + +const notificationOf = (record: FailureRecord.FailureRecord): SarifFailureNotification => ({ + descriptor: { id: record._tag }, + level: 'error', + message: { text: FailureRecord.sarifTextOf(record) }, + locations: Arr.map(locatedTestsOf(record), locationOf), +}) + +export const failureNotificationsOf = ( + records: ReadonlyArray, +): SarifFailureNotifications => Arr.map(records, notificationOf) diff --git a/packages/stryker-js/src/sarif-report.workflow.ts b/packages/stryker-js/src/sarif-report.workflow.ts index db38e5d64..7f1dfc937 100644 --- a/packages/stryker-js/src/sarif-report.workflow.ts +++ b/packages/stryker-js/src/sarif-report.workflow.ts @@ -1,13 +1,16 @@ import { Workflow } from '@systemfsoftware/effect-cell-types' -import { Report } from '@systemfsoftware/stryker-js-plugin-interface' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' +import { Plugin, Report } from '@systemfsoftware/stryker-js-plugin-interface' import * as Arr from 'effect/Array' import * as Boolean from 'effect/Boolean' import * as HashMap from 'effect/HashMap' +import * as Match from 'effect/Match' import * as Option from 'effect/Option' import * as Order from 'effect/Order' import * as Result from 'effect/Result' import * as S from 'effect/Schema' +import { failureNotificationsOf, SarifFailureNotification } from './sarif-failure.schema.js' import { SurvivorRef } from './surfacing.schema.js' const SARIF_SCHEMA_URI = 'https://json.schemastore.org/sarif-2.1.0.json' @@ -26,12 +29,17 @@ const SURVIVOR_LEVELS: Record = { Survived: 'warning const decodeMutantStatus = S.decodeOption(MUTANT_STATUSES) const encodeUriSegment = encodeURIComponent +const uriOf = (fileName: string): string => + Arr.join(Arr.map(fileName.split('/'), (segment) => encodeUriSegment(segment.toWellFormed())), '/') + const SarifToolSchema = S.Struct({ name: S.String, version: S.String, informationUri: S.String, }) +type SarifTool = typeof SarifToolSchema.Type + const SarifRuleSchema = S.Struct({ id: S.String, name: S.String, @@ -83,6 +91,47 @@ export type SarifLog = typeof SarifLogSchema.Type type SarifResult = typeof SarifResultSchema.Type +const SarifInvocationSchema = S.Struct({ + executionSuccessful: S.Literal(false), + exitCode: Plugin.ExitCode, + toolExecutionNotifications: S.Array(SarifFailureNotification), +}) + +const SarifFailureLogSchema = S.Struct({ + $schema: S.Literal(SARIF_SCHEMA_URI), + version: S.Literal('2.1.0'), + runs: S.Array( + S.Struct({ + tool: S.Struct({ driver: SarifDriverSchema }), + invocations: S.Array(SarifInvocationSchema), + results: S.Array(SarifResultSchema), + }), + ), +}) + +export type SarifFailureLog = typeof SarifFailureLogSchema.Type + +const failureLogOf = (tool: SarifTool, source: FailureReportSource): SarifFailureLog => ({ + $schema: SARIF_SCHEMA_URI, + version: '2.1.0', + runs: [{ + tool: { + driver: { + name: tool.name, + version: tool.version, + informationUri: tool.informationUri, + rules: [], + }, + }, + invocations: [{ + executionSuccessful: false, + exitCode: source.exitCode, + toolExecutionNotifications: failureNotificationsOf(source.records), + }], + results: [], + }], +}) + const SarifReportTypeId: unique symbol = Symbol.for('@systemfsoftware/stryker-js/SarifReportDecision') type SarifReportTypeId = typeof SarifReportTypeId @@ -99,14 +148,37 @@ export class SarifReportTruncated extends S.TaggedClass()( readonly [SarifReportTypeId] = SarifReportTypeId } -export const SarifReportDecision = S.Union([SarifReportRendered, SarifReportTruncated]) +export class SarifFailureReportRendered extends S.TaggedClass()( + 'SarifFailureReportRendered', + { log: SarifFailureLogSchema }, +) { + readonly [SarifReportTypeId] = SarifReportTypeId +} + +export const SarifReportDecision = S.Union([ + SarifReportRendered, + SarifReportTruncated, + SarifFailureReportRendered, +]) export type SarifReportDecision = typeof SarifReportDecision.Type -export class SarifReportCommand extends S.TaggedClass()('SarifReportCommand', { +export class SurvivorsReportSource extends S.TaggedClass()('SurvivorsReportSource', { report: Report.MutationTestResult, survivors: S.Array(SurvivorRef), - tool: SarifToolSchema, maxResults: S.Natural, +}) {} + +export class FailureReportSource extends S.TaggedClass()('FailureReportSource', { + records: S.NonEmptyArray(FailureRecord.FailureRecord), + exitCode: Plugin.ExitCode, +}) {} + +export const SarifReportSource = S.Union([SurvivorsReportSource, FailureReportSource]) +export type SarifReportSource = typeof SarifReportSource.Type + +export class SarifReportCommand extends S.TaggedClass()('SarifReportCommand', { + source: SarifReportSource, + tool: SarifToolSchema, }) { static readonly [Workflow.InstrumentationBrand] = {} as const } @@ -143,17 +215,15 @@ const entriesByIdOf = (report: Report.MutationTestResult): HashMap.HashMap, mutatorName: string): number => Option.getOrThrow(Arr.findFirstIndex(ruleNames, (name) => name === mutatorName)) -const uriOf = (fileName: string): string => Arr.join(Arr.map(fileName.split('/'), encodeUriSegment), '/') - interface Surfaced { readonly ref: SurvivorRef readonly entry: SurvivorEntry } -const surfacedOf = (command: SarifReportCommand): ReadonlyArray => { - const byId = entriesByIdOf(command.report) +const surfacedOf = (source: SurvivorsReportSource): ReadonlyArray => { + const byId = entriesByIdOf(source.report) return Arr.flatMap( - command.survivors, + source.survivors, (ref) => Option.toArray(Option.map(HashMap.get(byId, ref.id), (entry): Surfaced => ({ ref, entry }))), ) } @@ -186,7 +256,7 @@ const ruleOf = (mutatorName: string) => ({ shortDescription: { text: mutatorName }, }) -const logOf = (command: SarifReportCommand, results: ReadonlyArray): SarifLog => { +const logOf = (tool: SarifTool, results: ReadonlyArray): SarifLog => { const ruleNames = Arr.dedupe( Arr.sort(Arr.map(results, (surfaced) => surfaced.entry.mutant.mutatorName), Order.String), ) @@ -196,9 +266,9 @@ const logOf = (command: SarifReportCommand, results: ReadonlyArray): S runs: [{ tool: { driver: { - name: command.tool.name, - version: command.tool.version, - informationUri: command.tool.informationUri, + name: tool.name, + version: tool.version, + informationUri: tool.informationUri, rules: Arr.map(ruleNames, ruleOf), }, }, @@ -207,16 +277,24 @@ const logOf = (command: SarifReportCommand, results: ReadonlyArray): S } } -const decide = (command: SarifReportCommand): SarifReportDecision => { - const surfaced = surfacedOf(command) - const results = Arr.take(surfaced, command.maxResults) - const log = logOf(command, results) +const decideSurvivors = (tool: SarifTool, source: SurvivorsReportSource): SarifReportDecision => { + const surfaced = surfacedOf(source) + const results = Arr.take(surfaced, source.maxResults) + const log = logOf(tool, results) return Boolean.match(Arr.length(results) === Arr.length(surfaced), { onTrue: () => SarifReportRendered.make({ log }), onFalse: () => SarifReportTruncated.make({ log, omitted: Arr.length(surfaced) - Arr.length(results) }), }) } +const decide = (command: SarifReportCommand): SarifReportDecision => + Match.value(command.source).pipe( + Match.tag('SurvivorsReportSource', (source) => decideSurvivors(command.tool, source)), + Match.tag('FailureReportSource', (source) => + SarifFailureReportRendered.make({ log: failureLogOf(command.tool, source) })), + Match.exhaustive, + ) + export const sarifReport = Workflow.make({ command: SarifReportCommand, decision: SarifReportDecision, diff --git a/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts b/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts index e0b54478c..399648481 100644 --- a/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts +++ b/packages/stryker-js/tests/__fixtures__/mutation-server.schema.ts @@ -9,7 +9,7 @@ export const ExitValue = S.declare>(( export const JsonRpcId = S.Union([S.String, S.Finite, S.Null]) export type JsonRpcId = typeof JsonRpcId.Type -export const JsonRpcError = S.Struct({ code: S.Int, message: S.String }) +export const JsonRpcError = S.Struct({ code: S.Int, message: S.String, data: S.optionalKey(S.Json) }) export const JsonRpcResponse = S.Struct({ jsonrpc: S.Literal('2.0'), diff --git a/packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts b/packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts new file mode 100644 index 000000000..3395a1d00 --- /dev/null +++ b/packages/stryker-js/tests/__fixtures__/otlp-spans.schema.ts @@ -0,0 +1,17 @@ +import * as S from 'effect/Schema' + +const AttributeValue = S.Struct({ + stringValue: S.optional(S.String), + intValue: S.optional(S.Union([S.String, S.Finite])), +}) + +const Attribute = S.Struct({ key: S.String, value: AttributeValue }) + +export const OtlpSpan = S.Struct({ name: S.String, traceId: S.String, attributes: S.Array(Attribute) }) +export type OtlpSpan = typeof OtlpSpan.Type + +const ScopeSpans = S.Struct({ spans: OtlpSpan.pipe(S.Array, S.optional) }) +const ResourceSpans = S.Struct({ scopeSpans: ScopeSpans.pipe(S.Array, S.optional) }) + +export const OtlpPayload = S.Struct({ resourceSpans: ResourceSpans.pipe(S.Array, S.optional) }) +export type OtlpPayload = typeof OtlpPayload.Type diff --git a/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts b/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts index 722c5759a..a2c2a426b 100644 --- a/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts +++ b/packages/stryker-js/tests/__fixtures__/sarif-reporter.schema.ts @@ -36,6 +36,30 @@ export const SarifDocument = S.Struct({ export type SarifDocument = typeof SarifDocument.Type +export const FailureSarifDocument = S.Struct({ + version: S.String, + runs: S.Array( + S.Struct({ + invocations: S.Array( + S.Struct({ + executionSuccessful: S.Boolean, + exitCode: S.Int, + toolExecutionNotifications: S.Array( + S.Struct({ + descriptor: S.Struct({ id: S.String }), + level: S.String, + message: S.Struct({ text: S.String }), + locations: S.Array(S.Unknown), + }), + ), + }), + ), + }), + ), +}) + +export type FailureSarifDocument = typeof FailureSarifDocument.Type + export const ReproducerList = S.Array( S.Struct({ id: S.String, fileName: S.String, diff: S.String, command: S.String }), ) diff --git a/packages/stryker-js/tests/mcp-server.integration.test.ts b/packages/stryker-js/tests/mcp-server.integration.test.ts index ea6669e06..65931e3d8 100644 --- a/packages/stryker-js/tests/mcp-server.integration.test.ts +++ b/packages/stryker-js/tests/mcp-server.integration.test.ts @@ -1,8 +1,9 @@ import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Engine, Mcp } from '@systemfsoftware/stryker-js' -import { RunEvent } from '@systemfsoftware/stryker-js-cli-contract' +import { FailureRecord, RunEvent } from '@systemfsoftware/stryker-js-cli-contract' import * as Cause from 'effect/Cause' import * as Effect from 'effect/Effect' +import * as Equal from 'effect/Equal' import * as Fiber from 'effect/Fiber' import * as FileSystem from 'effect/FileSystem' import * as Option from 'effect/Option' @@ -45,6 +46,10 @@ const CONFIG = `export default { } ` +const FAILING_CONFIG = CONFIG.replace("command: 'true'", "command: 'false'") + +const FAILURE_FILE = 'reports/mutation/failure.json' + const UTF8 = new TextEncoder() type JsonRpcMessage = typeof JsonRpcResponse.Type @@ -88,18 +93,20 @@ interface Client { ) => Effect.Effect } -const writeWorkspace = (): Effect.Effect => +const writeWorkspaceWith = (config: string): Effect.Effect => Effect.gen(function*() { const fs = yield* FileSystem.FileSystem const path = yield* Path.Path const directory = yield* fs.realPath(yield* fs.makeTempDirectory({ prefix: 'stryker-mcp-' })) yield* fs.writeFileString(path.join(directory, 'package.json'), '{ "name": "mcp-consumer", "type": "module" }\n') - yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), CONFIG) + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), config) yield* fs.makeDirectory(path.join(directory, 'src'), { recursive: true }) yield* fs.writeFileString(path.join(directory, TARGET_FILE), TARGET_SOURCE) return { directory } }).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) +const writeWorkspace = (): Effect.Effect => writeWorkspaceWith(CONFIG) + const removeWorkspace = (directory: string): Effect.Effect => Effect.flatMap(FileSystem.FileSystem, (fs) => fs.remove(directory, { recursive: true, force: true })).pipe( Effect.orDie, @@ -289,6 +296,108 @@ const drive = (directory: string): Effect.Effect, }), ).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) +const rawToolCallOf = ( + response: JsonRpcMessage, +): { readonly failed: boolean; readonly structured: S.Json | undefined; readonly text: string } => + Option.match(toolEnvelopeOf(response), { + onNone: () => ({ failed: true, structured: undefined, text: '' }), + onSome: (call) => ({ + failed: call.isError === true, + structured: call.structuredContent, + text: call.content.map((part) => part.text ?? '').join('\n'), + }), + }) + +const withMcpClient = ( + directory: string, + use: (client: Client) => Effect.Effect, +): Effect.Effect => + Effect.scoped( + Effect.gen(function*() { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner + const handle = yield* spawner.spawn( + ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, 'mcp'], { + cwd: directory, + stdin: 'pipe', + stdout: 'pipe', + stderr: 'pipe', + env: { STRYKER_MODE: 'machine', NO_COLOR: '1' }, + extendEnv: true, + }), + ) + const client = yield* makeClient(handle) + yield* client.initialize + yield* client.notify('notifications/initialized') + return yield* use(client) + }), + ).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) + +const rewriteConfig = (directory: string, config: string): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), config) + }).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) + +interface FailureLookupObserved { + readonly runFailed: boolean + readonly failed: boolean + readonly recordTag: string | null + readonly recordMatchesFile: boolean +} + +const driveFailureLookup = (directory: string): Effect.Effect, never, never> => + withMcpClient(directory, (client) => + Effect.gen(function*() { + const call = rawToolCallOf(yield* client.call(2, 'tools/call', { name: 'get_failure', arguments: {} })) + const lookup = Option.getOrUndefined(S.decodeUnknownOption(Mcp.FailureLookup)(call.structured)) + const fileText = yield* readText(directory, FAILURE_FILE) + const fileRecord = Option.getOrNull(S.decodeOption(FailureRecord.FailureRecordFile)(fileText)) + return { + failed: call.failed, + recordTag: lookup?.failure == null ? null : lookup.failure._tag, + recordMatchesFile: lookup?.failure != null && fileRecord !== null && + Equal.equals(lookup.failure, fileRecord), + } + })) + +interface NoFailureObserved { + readonly failed: boolean + readonly absent: boolean +} + +const driveNoFailureLookup = (directory: string): Effect.Effect => + withMcpClient(directory, (client) => + Effect.gen(function*() { + const call = rawToolCallOf(yield* client.call(2, 'tools/call', { name: 'get_failure', arguments: {} })) + const lookup = Option.getOrUndefined(S.decodeUnknownOption(Mcp.FailureLookup)(call.structured)) + return { failed: call.failed, absent: lookup !== undefined && lookup.failure === null } + })) + +interface RerunFailureObserved { + readonly failed: boolean + readonly recordTag: string | null + readonly exitCode: number | null + readonly stillServing: boolean +} + +const driveRerunFailure = (directory: string): Effect.Effect => + withMcpClient(directory, (client) => + Effect.gen(function*() { + const surfaced = survivorsOf(yield* client.call(2, 'tools/call', { name: 'list_survivors', arguments: {} })) + const id = Option.getOrElse(Option.map(Option.fromNullishOr(surfaced[0]), (ref) => ref.id), () => '') + yield* rewriteConfig(directory, FAILING_CONFIG) + const rerun = rawToolCallOf(yield* client.call(3, 'tools/call', { name: 'rerun_mutant', arguments: { id } })) + const followed = rawToolCallOf(yield* client.call(4, 'tools/call', { name: 'show_mutant', arguments: { id } })) + const refusal = Option.getOrNull(S.decodeOption(S.fromJsonString(Mcp.RerunEngineUnusable))(rerun.text)) + return { + failed: rerun.failed, + recordTag: refusal === null ? null : refusal.record._tag, + exitCode: refusal === null ? null : refusal.code, + stillServing: followed.failed === false, + } + })) + Feature('An agent listing, inspecting and re-running mutants over MCP', { timeout: 180_000 }) .withLayer(Engine.nodePlatformLayer) .live('the built stryker mcp server speaks JSON-RPC over stdio against a project the engine mutated') @@ -346,7 +455,7 @@ Feature('An agent listing, inspecting and re-running mutants over MCP', { timeou }).toEqual({ runCompleted: true, initialized: true, - toolNames: ['list_survivors', 'report_usefulness', 'rerun_mutant', 'show_mutant'], + toolNames: ['get_failure', 'list_survivors', 'report_usefulness', 'rerun_mutant', 'show_mutant'], surfacedAtLeastOne: true, surfacedIdIsInTheReport: true, shownFailed: false, @@ -368,4 +477,84 @@ Feature('An agent listing, inspecting and re-running mutants over MCP', { timeou }), ), ) + + scenario( + 'A client reads the failure record the last failed run wrote', + Gherkin.Do.pipe( + Given('a consumer project whose test command fails')('workspace', () => writeWorkspaceWith(FAILING_CONFIG)), + When('the CLI runs and then an MCP client asks for the failure')( + 'observed', + (s: { readonly workspace: Workspace }) => + Effect.gen(function*() { + const seeded = yield* runStryker(s.workspace.directory, ['run']) + const driven = yield* driveFailureLookup(s.workspace.directory) + return { runFailed: seeded.exitCode === 5, ...driven } + }).pipe(Effect.ensuring(removeWorkspace(s.workspace.directory))), + ), + Then('get_failure returns the record written to reports/mutation/failure.json')( + (s: { readonly observed: FailureLookupObserved }, expect) => + expect({ + runFailed: s.observed.runFailed, + failed: s.observed.failed, + recordTag: s.observed.recordTag, + recordMatchesFile: s.observed.recordMatchesFile, + }).toEqual({ + runFailed: true, + failed: false, + recordTag: 'BaselineTestsFailed', + recordMatchesFile: true, + }), + ), + ), + ) + + scenario( + 'A client asks for the failure when no failed run left a record', + Gherkin.Do.pipe( + Given('a consumer project with no failed run')('workspace', () => writeWorkspaceWith(CONFIG)), + When('an MCP client asks for the failure')( + 'observed', + (s: { readonly workspace: Workspace }) => + driveNoFailureLookup(s.workspace.directory).pipe( + Effect.ensuring(removeWorkspace(s.workspace.directory)), + ), + ), + Then('get_failure answers that there is no failure')( + (s: { readonly observed: NoFailureObserved }, expect) => + expect({ failed: s.observed.failed, absent: s.observed.absent }).toEqual({ failed: false, absent: true }), + ), + ), + ) + + scenario( + 'A rerun whose engine fails returns the refusal with the record and the server keeps serving', + Gherkin.Do.pipe( + Given('a consumer project whose test command kills nothing')('workspace', () => writeWorkspace()), + When('the project is mutated, the test command is broken, and a client re-runs a survivor')( + 'observed', + (s: { readonly workspace: Workspace }) => + Effect.gen(function*() { + const seeded = yield* runStryker(s.workspace.directory, ['run']) + const driven = yield* driveRerunFailure(s.workspace.directory) + return { runCompleted: seeded.exitCode === 0, ...driven } + }).pipe(Effect.ensuring(removeWorkspace(s.workspace.directory))), + ), + Then('rerun_mutant refuses with the record and a later call is answered')( + (s: { readonly observed: RerunFailureObserved & { readonly runCompleted: boolean } }, expect) => + expect({ + runCompleted: s.observed.runCompleted, + failed: s.observed.failed, + recordTag: s.observed.recordTag, + exitCode: s.observed.exitCode, + stillServing: s.observed.stillServing, + }).toEqual({ + runCompleted: true, + failed: true, + recordTag: 'BaselineTestsFailed', + exitCode: 5, + stillServing: true, + }), + ), + ), + ) }) diff --git a/packages/stryker-js/tests/mutation-server.integration.test.ts b/packages/stryker-js/tests/mutation-server.integration.test.ts index dadf7e156..9f238868f 100644 --- a/packages/stryker-js/tests/mutation-server.integration.test.ts +++ b/packages/stryker-js/tests/mutation-server.integration.test.ts @@ -1,6 +1,7 @@ import { NodeSocket } from '@effect/platform-node' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Engine, Serve } from '@systemfsoftware/stryker-js' +import { FailureRecord } from '@systemfsoftware/stryker-js-cli-contract' import * as Cause from 'effect/Cause' import * as Clock from 'effect/Clock' import * as Effect from 'effect/Effect' @@ -64,6 +65,10 @@ const CONFIG = `export default { } ` +const FAILING_CONFIG = CONFIG.replace("command: 'true'", "command: 'false'") + +const INTERNAL_ERROR_CODE = -32603 + const UTF8 = new TextEncoder() const DECODE = new TextDecoder() const CONTENT_LENGTH = 'Content-Length: ' @@ -387,18 +392,20 @@ const everyProgressIsNative = (notifications: ReadonlyArray 0 && notifications.every((notification) => Object.keys(notification.params.files).length > 0) -const writeWorkspace = (): Effect.Effect => +const writeWorkspaceWith = (config: string): Effect.Effect => Effect.gen(function*() { const fs = yield* FileSystem.FileSystem const path = yield* Path.Path const directory = yield* fs.realPath(yield* fs.makeTempDirectory({ prefix: 'stryker-msp-' })) yield* fs.writeFileString(path.join(directory, 'package.json'), '{ "name": "msp-consumer", "type": "module" }\n') - yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), CONFIG) + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), config) yield* fs.makeDirectory(path.join(directory, 'src'), { recursive: true }) yield* fs.writeFileString(path.join(directory, TARGET_FILE), TARGET_SOURCE) return { directory } }).pipe(Effect.orDie, Effect.provide(Engine.nodePlatformLayer)) +const writeWorkspace = (): Effect.Effect => writeWorkspaceWith(CONFIG) + const removeWorkspace = (directory: string): Effect.Effect => Effect.flatMap(FileSystem.FileSystem, (fs) => fs.remove(directory, { recursive: true, force: true })).pipe( Effect.orDie, @@ -590,6 +597,46 @@ const exposedAddressObserved = (): Effect.Effect => + Effect.scoped(Effect.gen(function*() { + const harness = yield* makeHarness() + const server = yield* Serve.serveMutationServer({ channel: 'stdio' }).pipe( + Effect.provideService(Stdio.Stdio, harness.stdio), + Effect.scoped, + Effect.forkScoped, + ) + const session = yield* sessionOf( + { write: (bytes) => Queue.offer(harness.input, bytes).pipe(Effect.asVoid) }, + yield* frameSourceOf(harness.output), + ) + yield* session.send(1, 'mutationTest', { files: [{ path: TARGET_FILE }] }) + const failed = yield* session.answer(1) + yield* session.send(2, 'configure', {}) + const reconnected = versionOf(yield* session.answer(2)) + yield* Queue.end(harness.input) + yield* Fiber.interrupt(server) + + const record = Option.getOrNull( + Option.flatMap( + Option.fromUndefinedOr(failed.error?.data), + (data) => S.decodeUnknownOption(FailureRecord.FailureRecord)(data), + ), + ) + return { + errorCode: failed.error?.code ?? 0, + failureTag: record === null ? null : record._tag, + failureExitCode: record === null ? null : FailureRecord.FailureCatalog[record._tag].exitCode, + stillServing: reconnected === '0.4.0', + } + })) + Feature('Serving the Mutation Server Protocol over stdio and sockets', { timeout: 180_000 }) .withLayer(Engine.nodePlatformLayer) .live('the mutation server drives the engine for a client that speaks the protocol') @@ -696,4 +743,34 @@ Feature('Serving the Mutation Server Protocol over stdio and sockets', { timeout ), ), ) + + scenario( + 'A mutationTest whose engine fails answers the failure record and the server keeps serving', + Gherkin.Do.pipe( + Given('a consumer project whose test command fails')('workspace', () => writeWorkspaceWith(FAILING_CONFIG)), + When('a client asks for a mutation test over stdio')( + 'observed', + (s: { readonly workspace: Workspace }) => + withDirectory(s.workspace.directory, failingEngineObserved()).pipe( + Effect.ensuring(removeWorkspace(s.workspace.directory)), + ), + ), + Then( + 'the JSON-RPC error carries the same failure record the stream would carry, and a later request is answered', + )( + (s: { readonly observed: FailureObserved }, expect) => + expect({ + errorCode: s.observed.errorCode, + failureTag: s.observed.failureTag, + failureExitCode: s.observed.failureExitCode, + stillServing: s.observed.stillServing, + }).toEqual({ + errorCode: INTERNAL_ERROR_CODE, + failureTag: 'BaselineTestsFailed', + failureExitCode: 5, + stillServing: true, + }), + ), + ), + ) }) diff --git a/packages/stryker-js/tests/sarif-reporter.integration.test.ts b/packages/stryker-js/tests/sarif-reporter.integration.test.ts index 1c05793d4..c89fe02ff 100644 --- a/packages/stryker-js/tests/sarif-reporter.integration.test.ts +++ b/packages/stryker-js/tests/sarif-reporter.integration.test.ts @@ -20,7 +20,7 @@ import * as S from 'effect/Schema' import * as Scope from 'effect/Scope' import * as Stream from 'effect/Stream' -import { ReproducerList, SarifDocument } from './__fixtures__/sarif-reporter.schema.js' +import { FailureSarifDocument, ReproducerList, SarifDocument } from './__fixtures__/sarif-reporter.schema.js' const Feature = makeFeature({ it }) @@ -235,15 +235,15 @@ interface AnnotateRun { readonly stderr: string } -const runAnnotate = ( +const spawnStryker = ( root: string, - args: ReadonlyArray, + argv: ReadonlyArray, ): Effect.Effect => Effect.scoped( Effect.gen(function*() { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner const handle = yield* spawner.spawn( - ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, 'annotate', ...args], { + ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, ...argv], { cwd: root, stdin: 'ignore', stdout: 'pipe', @@ -260,9 +260,42 @@ const runAnnotate = ( }), ).pipe(Effect.orDie) +const runAnnotate = ( + root: string, + args: ReadonlyArray, +): Effect.Effect => + spawnStryker(root, ['annotate', ...args]) + const annotationLinesOf = (stdout: string): ReadonlyArray => Arr.filter(stdout.split('\n'), (line) => line.startsWith('::')) +const FAILING_CONSUMER_CONFIG = `export default { + testRunner: 'command', + commandRunner: { command: 'false' }, + mutate: ['src/**/*.ts'], + coverageAnalysis: 'off', + concurrency: 1, + reporters: ['json', 'sarif'], +} +` + +const FAILING_TARGET_SOURCE = 'export const target = (value: number): number => value + 1\n' + +const writeFailingWorkspace = (): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const directory = yield* fs.makeTempDirectory({ prefix: 'stryker-sarif-fail-' }) + yield* fs.makeDirectory(path.join(directory, 'src'), { recursive: true }) + yield* fs.writeFileString(path.join(directory, 'package.json'), '{ "type": "commonjs" }\n') + yield* fs.writeFileString(path.join(directory, 'src', 'target.ts'), FAILING_TARGET_SOURCE) + yield* fs.writeFileString(path.join(directory, 'stryker.config.mjs'), FAILING_CONSUMER_CONFIG) + return { directory } + }).pipe(Effect.orDie) + +const parseFailureSarif = (text: string | undefined): Option.Option => + text === undefined ? Option.none() : Result.getSuccess(S.decodeResult(S.fromJsonString(FailureSarifDocument))(text)) + Feature('Survivors reaching code scanning, annotations, and reproducers', { timeout: 180_000 }) .withLayer(Engine.nodePlatformLayer) .live('the run drives the real engine in process and writes the report, the SARIF log, and the sidecar') @@ -378,4 +411,54 @@ Feature('Survivors reaching code scanning, annotations, and reproducers', { time ), ), ) + scenario( + 'A failed run with the sarif reporter writes a failed invocation with one notification for its record', + Gherkin.Do.pipe( + Given('a workspace whose test command fails the baseline')( + 'workspace', + () => writeFailingWorkspace().pipe(Effect.provide(filePorts)), + ), + When('stryker run executes with the json and sarif reporters')( + 'observation', + (s) => + Effect.gen(function*() { + const run = yield* spawnStryker(s.workspace.directory, ['run']) + const sarifText = yield* FileSystem.FileSystem.pipe( + Effect.flatMap((fs) => fs.readFileString(sarifFileOf(s.workspace.directory)).pipe(Effect.option)), + Effect.provide(filePorts), + ) + return { run, sarif: Option.getOrUndefined(Option.flatMap(sarifText, parseFailureSarif)) } + }).pipe( + Effect.ensuring(removeWorkspace(s.workspace.directory).pipe(Effect.provide(filePorts))), + ), + ), + Then('the SARIF invocation is failed at exit 5 and notifies the record')((s, expect) => { + const sarif = s.observation.sarif + const run = sarif === undefined ? undefined : Option.getOrUndefined(Arr.head(sarif.runs)) + const invocation = run === undefined ? undefined : Option.getOrUndefined(Arr.head(run.invocations)) + const notification = invocation === undefined + ? undefined + : Option.getOrUndefined(Arr.head(invocation.toolExecutionNotifications)) + return expect({ + exitCode: s.observation.run.exitCode, + version: sarif?.version, + executionSuccessful: invocation?.executionSuccessful, + invocationExitCode: invocation?.exitCode, + notificationCount: invocation?.toolExecutionNotifications.length, + descriptorId: notification?.descriptor.id, + level: notification?.level, + textNamesTheRecord: notification?.message.text.includes('BaselineTestsFailed'), + }).toEqual({ + exitCode: 5, + version: '2.1.0', + executionSuccessful: false, + invocationExitCode: 5, + notificationCount: 1, + descriptorId: 'BaselineTestsFailed', + level: 'error', + textNamesTheRecord: true, + }) + }), + ), + ) }) diff --git a/packages/stryker-js/tests/trace-propagation.integration.test.ts b/packages/stryker-js/tests/trace-propagation.integration.test.ts index f9b9b66cd..094345b31 100644 --- a/packages/stryker-js/tests/trace-propagation.integration.test.ts +++ b/packages/stryker-js/tests/trace-propagation.integration.test.ts @@ -1,16 +1,32 @@ import * as NodeSdk from '@effect/opentelemetry/NodeSdk' +import { NodeHttpServer } from '@effect/platform-node' import { InMemorySpanExporter, type ReadableSpan, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' -import { Plugin } from '@systemfsoftware/stryker-js' +import { Engine, Plugin } from '@systemfsoftware/stryker-js' +import { FailureRecord, SpanTaxonomy } from '@systemfsoftware/stryker-js-cli-contract' import { Options, Trace } from '@systemfsoftware/stryker-js-plugin-interface' import { Trace as RuntimeTrace } from '@systemfsoftware/stryker-js-plugin-runtime' +import * as Arr from 'effect/Array' import * as Effect from 'effect/Effect' +import * as Fiber from 'effect/Fiber' +import * as FileSystem from 'effect/FileSystem' import * as Headers from 'effect/http/Headers' -import * as Layer from 'effect/Layer' +import * as HttpServer from 'effect/http/HttpServer' +import * as HttpServerRequest from 'effect/http/HttpServerRequest' +import * as HttpServerResponse from 'effect/http/HttpServerResponse' +import * as Match from 'effect/Match' +import type * as NetAddress from 'effect/net/NetAddress' import * as Option from 'effect/Option' +import * as Path from 'effect/Path' +import * as Predicate from 'effect/Predicate' +import * as ChildProcess from 'effect/process/ChildProcess' +import * as ChildProcessSpawner from 'effect/process/ChildProcessSpawner' import * as Ref from 'effect/Ref' import * as S from 'effect/Schema' +import * as Scope from 'effect/Scope' +import * as Stream from 'effect/Stream' +import { OtlpPayload, type OtlpSpan } from './__fixtures__/otlp-spans.schema.js' import { makeTraceWorkerRecord, TRACE_WORKER_ENTRYPOINT, @@ -44,8 +60,10 @@ const PLAN: TraceWorkerPlan = { entrypoint: TRACE_WORKER_ENTRYPOINT, projectBase const FUTURE_TRACEPARENT = '01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01-extra' -const spanNamed = (spans: readonly ReadableSpan[], name: string): ReadableSpan | undefined => - spans.find((span) => span.name === name) +const spanNamed = ( + spans: readonly Span[], + name: string, +): Span | undefined => spans.find((span) => span.name === name) const makeClient = (plan: TraceWorkerPlan) => Effect.gen(function*() { @@ -114,10 +132,187 @@ const runCarriedCall = (plan: TraceWorkerPlan): Effect.Effect => { ).pipe(Effect.provide(telemetry), Effect.orDie) } -Feature('Linking a worker into the host run trace') - .withLayer(Layer.empty) +const PACKAGE_ROOT = decodeURIComponent(new URL('..', import.meta.url).pathname) +const STRYKER_BIN = decodeURIComponent(new URL('../dist/main.mjs', import.meta.url).pathname) + +const FAILURE_RECORD_FILE = 'reports/mutation/failure.json' +const FAILING_TEST_NAME = 'rejects a wrong sum' +const CONSUMER_PACKAGE = '{ "name": "trace-failure-consumer", "type": "module", "private": true }\n' +const SOURCE_CONTENT = 'export const add = (left: number, right: number): number => left + right\n' +const CONSUMER_CONFIG = [ + 'export default {', + " testRunner: 'vm',", + " testFiles: ['test/**/*.mjs'],", + " mutate: ['src/**/*.ts'],", + ' reporters: [],', + ' checkers: [],', + '}', + '', +].join('\n') +const TEST_CONTENT = [ + "import { expect, test } from 'vitest'", + "import { add } from '../src/math.ts'", + '', + `test('${FAILING_TEST_NAME}', () => {`, + ' expect(add(1, 1)).toBe(3)', + '})', + '', +].join('\n') + +const arrayOf = (value: readonly A[] | undefined): readonly A[] => + Option.getOrElse(Option.fromNullishOr(value), () => []) + +const spansOfPayload = (payload: OtlpPayload): readonly OtlpSpan[] => + Arr.flatMap( + arrayOf(payload.resourceSpans), + (resource) => Arr.flatMap(arrayOf(resource.scopeSpans), (scope) => arrayOf(scope.spans)), + ) + +const spansOfBody = (body: Option.Option): readonly OtlpSpan[] => + Option.match(body, { onNone: () => [], onSome: spansOfPayload }) + +const stringAttributeOf = (span: OtlpSpan, key: string): string | undefined => + Option.getOrUndefined( + Option.map( + Arr.findFirst(span.attributes, (attribute) => attribute.key === key), + (attribute) => attribute.value.stringValue, + ), + ) + +const failureCodeOf = (spans: readonly OtlpSpan[]): string | undefined => + Option.match(Option.fromNullishOr(spanNamed(spans, SpanTaxonomy.Spans.cliRun.name)), { + onNone: () => undefined, + onSome: (span) => stringAttributeOf(span, 'stryker.failure.code'), + }) + +const hasRunSpan = (spans: readonly OtlpSpan[]): boolean => + spanNamed(spans, SpanTaxonomy.Spans.cliRun.name) !== undefined + +const traceIdMatchesSpan = (spans: readonly OtlpSpan[], traceId: string | null): boolean => + Option.match(Option.fromNullishOr(spanNamed(spans, SpanTaxonomy.Spans.cliRun.name)), { + onNone: () => false, + onSome: (span) => traceId !== null && traceId === span.traceId, + }) + +const portOfAddress = (address: NetAddress.SocketAddress): number | undefined => + Match.value(address).pipe( + Match.tag('InetAddressV4', (inet) => inet.port), + Match.tag('InetAddressV6', (inet) => inet.port), + Match.tag('UnixPathAddress', () => undefined), + Match.exhaustive, + ) + +const endpointOf = (address: NetAddress.SocketAddress): Effect.Effect => + Option.match(Option.fromNullishOr(portOfAddress(address)), { + onNone: () => Effect.die(new Error('the OTLP collector must bind a TCP port')), + onSome: (port) => Effect.succeed(`http://127.0.0.1:${port}`), + }) + +interface SpanCollector { + readonly endpoint: string + readonly spans: Effect.Effect +} + +const startSpanCollector = (): Effect.Effect => + Effect.gen(function*() { + const server = yield* HttpServer.HttpServer + const collected: OtlpSpan[] = [] + const handler = Effect.gen(function*() { + const body = yield* OtlpPayload.pipe(HttpServerRequest.schemaBodyJson, Effect.option) + collected.push(...spansOfBody(body)) + return HttpServerResponse.empty() + }) + yield* server.serve(handler) + const endpoint = yield* endpointOf(server.address) + return { endpoint, spans: Effect.sync(() => [...collected]) } + }) + +const writeFixture = (): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const binaryPresent = yield* fs.exists(STRYKER_BIN) + yield* Effect.when( + Effect.die(new Error('dist/main.mjs is missing — build the package first')), + Effect.succeed(!binaryPresent), + ) + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped({ prefix: 'stryker-trace-failure-' })) + yield* fs.makeDirectory(path.join(root, 'node_modules', '@systemfsoftware'), { recursive: true }) + yield* fs.symlink(PACKAGE_ROOT, path.join(root, 'node_modules', '@systemfsoftware', 'stryker-js')) + yield* fs.makeDirectory(path.join(root, 'src'), { recursive: true }) + yield* fs.makeDirectory(path.join(root, 'test'), { recursive: true }) + yield* fs.writeFileString(path.join(root, 'package.json'), CONSUMER_PACKAGE) + yield* fs.writeFileString(path.join(root, 'src', 'math.ts'), SOURCE_CONTENT) + yield* fs.writeFileString(path.join(root, 'test', 'math.test.mjs'), TEST_CONTENT) + yield* fs.writeFileString(path.join(root, 'stryker.config.mjs'), CONSUMER_CONFIG) + return root + }).pipe(Effect.orDie) + +const runStrykerFrom = ( + root: string, + endpoint: string, + otelEnabled: boolean, +): Effect.Effect => + Effect.scoped( + Effect.gen(function*() { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner + const handle = yield* spawner.spawn( + ChildProcess.make(globalThis.process.execPath, [STRYKER_BIN, 'run'], { + cwd: root, + stdin: 'ignore', + stdout: 'pipe', + stderr: 'pipe', + env: { + STRYKER_MODE: 'machine', + NO_COLOR: '1', + OTEL_ENABLED: String(otelEnabled), + OTEL_EXPORTER_OTLP_ENDPOINT: endpoint, + }, + extendEnv: true, + }), + ) + const drainOut = yield* Effect.forkScoped(handle.stdout.pipe(Stream.decodeText, Stream.mkString)) + const drainErr = yield* Effect.forkScoped(handle.stderr.pipe(Stream.decodeText, Stream.mkString)) + const exitCode = yield* handle.exitCode + yield* Effect.all([Fiber.join(drainOut), Fiber.join(drainErr)], { discard: true }) + return Number(exitCode) + }), + ).pipe(Effect.orDie) + +const readFailureRecord = ( + root: string, +): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const text = yield* fs.readFileString(path.join(root, FAILURE_RECORD_FILE)) + return yield* S.decodeEffect(FailureRecord.FailureRecordFile)(text) + }).pipe(Effect.orDie) + +interface ObservedRun { + readonly exitCode: number + readonly record: FailureRecord.FailureRecord + readonly spans: readonly OtlpSpan[] +} + +const observeFailingRun = ( + root: string, + otelEnabled: boolean, +): Effect.Effect => + Effect.scoped( + Effect.gen(function*() { + const collector = yield* startSpanCollector() + const exitCode = yield* runStrykerFrom(root, collector.endpoint, otelEnabled) + const record = yield* readFailureRecord(root) + const spans = yield* collector.spans + return { exitCode, record, spans } + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ).pipe(Effect.orDie) + +Feature('Tracing a run, the work it starts, and the failure it ends with', { timeout: 180_000 }) + .withLayer(Engine.nodePlatformLayer) .live( - 'the OpenTelemetry span exporter schedules its export on a real setTimeout timer, which leaves the controlled schedule', + 'the OpenTelemetry exporter schedules its export on a real setTimeout timer, and the failing-run scenarios run the built stryker binary in a real Node process against a socket collector', ) .body(({ scenario }) => { scenario( @@ -170,4 +365,37 @@ Feature('Linking a worker into the host run trace') }), ), ) + + scenario( + 'A failing dry run stamps the run span with the record code and writes the span trace id into the record', + Gherkin.Do.pipe( + Given('a project whose test suite fails')('root', () => writeFixture()), + When('the run executes with OpenTelemetry enabled')('observed', (s) => observeFailingRun(s.root, true)), + Then( + 'the exported run span carries the failure code and the record trace id equals the span trace id', + )((s, expect) => + expect({ + exitCode: s.observed.exitCode, + failureCode: failureCodeOf(s.observed.spans), + traceIdMatchesExportedSpan: traceIdMatchesSpan(s.observed.spans, s.observed.record.traceId), + }).toEqual({ exitCode: 5, failureCode: 'BaselineTestsFailed', traceIdMatchesExportedSpan: true }) + ), + ), + ) + + scenario( + 'A failing dry run with OpenTelemetry disabled records no trace id and exports nothing', + Gherkin.Do.pipe( + Given('a project whose test suite fails')('root', () => writeFixture()), + When('the run executes with OpenTelemetry disabled')('observed', (s) => observeFailingRun(s.root, false)), + Then('the record has a null trace id and no run span was exported')((s, expect) => + expect({ + exitCode: s.observed.exitCode, + failureIsBaseline: Predicate.isTagged(s.observed.record, 'BaselineTestsFailed'), + hasRunSpan: hasRunSpan(s.observed.spans), + recordTraceId: s.observed.record.traceId, + }).toEqual({ exitCode: 5, failureIsBaseline: true, hasRunSpan: false, recordTraceId: null }) + ), + ), + ) })