diff --git a/.changeset/closure-dry-run-module-evidence.md b/.changeset/closure-dry-run-module-evidence.md new file mode 100644 index 000000000..4e67410a4 --- /dev/null +++ b/.changeset/closure-dry-run-module-evidence.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +A dry run now records the modules each test file evaluated, and a later incremental run uses them. A test file that loads a module only through a computed dynamic `import()` is keyed to the modules it actually loaded: an edit to one of them re-runs that test file's mutants, while an edit to any file outside them keeps their verdicts. When the runtime target is not a project file, such as a `data:` URL, the test file stays keyed to the whole project and its mutants re-run on any change. diff --git a/.changeset/closure-evidence-modules.md b/.changeset/closure-evidence-modules.md new file mode 100644 index 000000000..3aea50098 --- /dev/null +++ b/.changeset/closure-evidence-modules.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-js": minor +--- + +`ImportClosure.analyzeImportClosure` accepts `observedModules`: for each test file, the modules a dry run saw it load. Each one that names an existing file joins that test file's closure. A test file that reaches a computed dynamic `import()` then closes around the files it actually loaded, instead of falling back to the whole project. A key that names no existing file keeps the closure open. diff --git a/packages/stryker-js/src/dry-run-coverage.schema.ts b/packages/stryker-js/src/dry-run-coverage.schema.ts index b90fee5e9..925357d76 100644 --- a/packages/stryker-js/src/dry-run-coverage.schema.ts +++ b/packages/stryker-js/src/dry-run-coverage.schema.ts @@ -5,6 +5,7 @@ export const DryRunCoverageSchema = S.Struct({ tests: S.Array(TestRunner.TestResultSchema), mutantCoverage: S.optional(Mutant.MutantCoverageSchema), globalTestInputs: S.Array(S.String), + testFileModules: S.optionalKey(S.Record(S.String, S.Array(S.String))), timeOverheadMs: S.Finite, flakyTestIds: S.Array(S.String), flakyMutantIds: S.Array(S.String), diff --git a/packages/stryker-js/src/import-closure.cell.ts b/packages/stryker-js/src/import-closure.cell.ts index 6f47f8d81..e27f1980c 100644 --- a/packages/stryker-js/src/import-closure.cell.ts +++ b/packages/stryker-js/src/import-closure.cell.ts @@ -28,6 +28,7 @@ export interface ImportClosureInput { readonly projectFiles: readonly string[] readonly testFiles: readonly string[] readonly globalInputs?: readonly string[] + readonly observedModules?: Readonly> } export interface TestFileClosureDigest { @@ -62,6 +63,14 @@ interface ModuleScan { readonly contentHash: string readonly dependencies: readonly string[] readonly open: boolean + readonly dynamicOpen: boolean +} + +type OpenMode = 'Combined' | 'Structural' + +interface ObservedEvidence { + readonly roots: readonly string[] + readonly invalid: boolean } interface Extraction { @@ -71,12 +80,12 @@ interface Extraction { interface Scan { readonly specifiers: readonly string[] - readonly open: boolean + readonly dynamicOpen: boolean } interface ScanState { readonly specifiers: HashSet.HashSet - readonly open: boolean + readonly hidden: boolean } interface Resolution { @@ -168,7 +177,7 @@ const MOCK_METHODS: Readonly> = { const LITERAL_TYPES: Readonly> = { Literal: true, StringLiteral: true } const NOTHING: Extraction = { specifier: undefined, hidden: false } -const EMPTY_STATE: ScanState = { specifiers: HashSet.empty(), open: false } +const EMPTY_STATE: ScanState = { specifiers: HashSet.empty(), hidden: false } const EXTERNAL_RESOLUTION: Resolution = { kind: 'External', file: '' } const UNRESOLVED_RESOLUTION: Resolution = { kind: 'Unresolved', file: '' } @@ -267,7 +276,7 @@ const extractionOf = (node: AstNode): Extraction => const mergeState = (state: ScanState, extraction: Extraction): ScanState => ({ specifiers: specifierState(state, extraction), - open: state.open || extraction.hidden, + hidden: state.hidden || extraction.hidden, }) const specifierState = (state: ScanState, extraction: Extraction): HashSet.HashSet => @@ -280,7 +289,7 @@ const walk = (state: ScanState, node: AstNode): ScanState => const scanProgram = (program: AstValue): Scan => { const state = isNode(program) ? walk(EMPTY_STATE, program) : EMPTY_STATE - return { specifiers: [...state.specifiers], open: state.open } + return { specifiers: [...state.specifiers], dynamicOpen: state.hidden } } const oxcModule = Effect.cached(Effect.promise(() => import('oxc-parser'))) @@ -289,7 +298,7 @@ const scanSource = Effect.fnUntraced(function*(content: string, absolute: string const oxc = yield* Effect.flatMap(oxcModule, (load) => load) const parsed = oxc.parseSync(absolute, content, { lang: language }) const scan = scanProgram(parsed.program) - return { specifiers: scan.specifiers, open: scan.open || parsed.errors.length > 0 } + return { specifiers: scan.specifiers, dynamicOpen: scan.dynamicOpen, parseFailed: parsed.errors.length > 0 } }) const extensionOf = (file: string): string => { @@ -588,6 +597,7 @@ const unparsedScan = (loaded: LoadedFile, verdictContent: string): ModuleScan => contentHash: hashOf(verdictContent), dependencies: [], open: INERT_EXTENSIONS[extensionOf(loaded.key)] !== true, + dynamicOpen: false, }) const parsedScan = Effect.fnUntraced(function*( @@ -601,7 +611,8 @@ const parsedScan = Effect.fnUntraced(function*( key: loaded.key, contentHash: hashOf(loaded.content), dependencies: dependencyFiles(resolutions), - open: scan.open || resolutions.some(opensClosure), + open: scan.parseFailed || resolutions.some(opensClosure), + dynamicOpen: scan.dynamicOpen, } }) @@ -624,13 +635,20 @@ const moduleScanOf = Effect.fnUntraced(function*( })) }) -const moduleEntry = (scan: ModuleScan): readonly [string, ImportClosureModule] => [ +const OPEN_COMBINERS: Readonly boolean>> = { + Combined: (scan) => scan.open || scan.dynamicOpen, + Structural: (scan) => scan.open, +} + +const openFor = (mode: OpenMode, scan: ModuleScan): boolean => OPEN_COMBINERS[mode](scan) + +const moduleEntry = (mode: OpenMode) => (scan: ModuleScan): readonly [string, ImportClosureModule] => [ scan.key, - { dependencies: scan.dependencies, open: scan.open }, + { dependencies: scan.dependencies, open: openFor(mode, scan) }, ] -const modulesOf = (scanned: readonly ModuleScan[]): Record => - Object.fromEntries(scanned.map(moduleEntry)) +const modulesOf = (mode: OpenMode, scanned: readonly ModuleScan[]): Record => + Object.fromEntries(scanned.map(moduleEntry(mode))) const digestLine = (scan: ModuleScan): string => `${scan.key}\u0000${scan.contentHash}` @@ -639,15 +657,20 @@ const projectDigestOf = (scanned: readonly ModuleScan[]): string => hashOf(scann const sortedKeys = (roots: Roots, files: readonly string[]): readonly string[] => [...HashSet.fromIterable(files.map((file) => keyOf(roots, file)))].sort() +const commandGlobalInputs = (roots: Roots, input: ImportClosureInput): readonly string[] => + sortedKeys(roots, input.globalInputs ?? []) + const commandOf = ( input: ImportClosureInput, roots: Roots, scanned: readonly ModuleScan[], + testFiles: readonly string[], + mode: OpenMode, ): ImportClosureCommand => ImportClosureCommand.make({ - modules: modulesOf(scanned), - globalInputs: sortedKeys(roots, input.globalInputs ?? []), - testFiles: sortedKeys(roots, input.testFiles), + modules: modulesOf(mode, scanned), + globalInputs: commandGlobalInputs(roots, input), + testFiles, }) const contentHashOf = ( @@ -734,6 +757,7 @@ const leafScanOf = Effect.fnUntraced(function*(roots: Roots, file: string) { contentHash: hashOf(loaded.content), dependencies: [], open: false, + dynamicOpen: false, } }) @@ -756,22 +780,130 @@ const hashGlobalInputsOf = Effect.fnUntraced(function*(roots: Roots, keys: reado return Arr.getSomes(scans) }) +const infoIsFile = (type: string | undefined): boolean => type === 'File' + +const statTypeOf = Effect.fnUntraced(function*(fs: FileSystem.FileSystem, path: Path.Path, value: string) { + if (!path.isAbsolute(value)) return undefined + return yield* fs.stat(value).pipe(Effect.map((info) => info.type), Effect.orElseSucceed(() => undefined)) +}) + +const observedRootOf = Effect.fnUntraced(function*(roots: Roots, value: string) { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const type = yield* statTypeOf(fs, path, value) + return infoIsFile(type) ? Option.some(keyOf(roots, value)) : Option.none() +}) + +const observedTestEvidenceOf = Effect.fnUntraced(function*(roots: Roots, keys: readonly string[]) { + const observed = yield* Effect.forEach(keys, (key) => observedRootOf(roots, key), { concurrency: CONCURRENCY }) + return { roots: Arr.getSomes(observed), invalid: observed.some(Option.isNone) } +}) + +const observedEntries = (input: ImportClosureInput): readonly (readonly [string, readonly string[]])[] => + Object.entries(input.observedModules ?? {}) + +const observedEvidenceOf = Effect.fnUntraced(function*(roots: Roots, input: ImportClosureInput) { + const resolved = yield* Effect.forEach( + observedEntries(input), + ([testFile, keys]) => + observedTestEvidenceOf(roots, keys).pipe( + Effect.map((entry) => [keyOf(roots, testFile), entry] as const), + ), + { concurrency: CONCURRENCY }, + ) + return new Map(resolved) +}) + +const unknownKeys = (known: HashSet.HashSet, keys: readonly string[]): readonly string[] => + [...HashSet.fromIterable(keys)].filter((key) => !HashSet.has(known, key)) + +const observedScansOf = Effect.fnUntraced(function*( + roots: Roots, + files: HashSet.HashSet, + memo: MutableHashMap.MutableHashMap, + known: readonly ModuleScan[], + keys: readonly string[], +) { + const pending = unknownKeys(scannedKeysOf(known), keys) + const sourceKeys = pending.filter((key) => !isNodeModulesKey(key)) + const nodeKeys = pending.filter(isNodeModulesKey) + const sourceScans = sourceKeys.length === 0 ? [] : yield* scanReachable(roots, files, memo, sourceKeys) + const leafScans = yield* Effect.forEach( + nodeKeys, + (key) => Effect.option(leafScanOf(roots, key)), + { concurrency: CONCURRENCY }, + ) + return [...sourceScans, ...Arr.getSomes(leafScans)] +}) + +const augmentModule = (member: ImportClosureModule, entry: ObservedEvidence): ImportClosureModule => ({ + dependencies: [...HashSet.fromIterable([...member.dependencies, ...entry.roots])].sort(), + open: member.open || entry.invalid, +}) + +const augmentedModules = ( + modules: Record, + evidence: ReadonlyMap, +): Record => + [...evidence].reduce((augmented, [testFile, entry]) => { + const member = augmented[testFile] + return member === undefined ? augmented : { ...augmented, [testFile]: augmentModule(member, entry) } + }, { ...modules }) + +const evidenceCommandOf = ( + input: ImportClosureInput, + roots: Roots, + scanned: readonly ModuleScan[], + evidence: ReadonlyMap, + testFiles: readonly string[], +): ImportClosureCommand => + ImportClosureCommand.make({ + modules: augmentedModules(modulesOf('Structural', scanned), evidence), + globalInputs: commandGlobalInputs(roots, input), + testFiles, + }) + +const closuresOf = (command: ImportClosureCommand): readonly TestFileClosure[] => + Result.getOrElse(importClosure(command), (unreachable: never) => unreachable) + export const analyzeImportClosure = Effect.fnUntraced(function*( input: ImportClosureInput, ): Effect.fn.Return { const roots = yield* readRoots(input) + const testKeys = sortedKeys(roots, input.testFiles) const seeds = [...HashSet.fromIterable(input.projectFiles.map((file) => keyOf(roots, file)))] const files = HashSet.fromIterable(seeds) const memo = MutableHashMap.empty() const reachable = yield* scanReachable(roots, files, memo, seeds) const globalScans = yield* hashGlobalInputsOf(roots, globalInputKeysOf(roots, input, reachable)) - const scanned = [...reachable, ...globalScans] - const projectDigest = projectDigestOf(scanned) + const baseScanned = [...reachable, ...globalScans] + const projectDigest = projectDigestOf(baseScanned) + const baseHashes = MutableHashMap.fromIterable(baseScanned.map((scan) => [scan.key, scan.contentHash] as const)) + const evidence = yield* observedEvidenceOf(roots, input) + const unevidenced = testKeys.filter((key) => !evidence.has(key)) + const evidenced = testKeys.filter((key) => evidence.has(key)) + const baseClosures = closuresOf(commandOf(input, roots, baseScanned, unevidenced, 'Combined')) + if (evidenced.length === 0) { + return { + closures: baseClosures.map((closure) => closureWithDigest(baseHashes, projectDigest, closure)), + projectDigest, + } + } + const baseKeys = scannedKeysOf(baseScanned) + const observedScans = (yield* observedScansOf( + roots, + files, + memo, + baseScanned, + Arr.dedupe([...evidence.values()].flatMap((entry) => entry.roots)), + )).filter((scan) => !HashSet.has(baseKeys, scan.key)) + const scanned = [...baseScanned, ...observedScans] const hashes = MutableHashMap.fromIterable(scanned.map((scan) => [scan.key, scan.contentHash] as const)) - const closures = Result.getOrElse( - importClosure(commandOf(input, roots, scanned)), - (unreachable: never) => unreachable, + const evidenceClosures = closuresOf(evidenceCommandOf(input, roots, scanned, evidence, evidenced)) + const byTestFile = new Map( + [...baseClosures, ...evidenceClosures].map((closure) => [closure.testFile, closure] as const), ) + const closures = testKeys.flatMap((key) => Option.toArray(Option.fromUndefinedOr(byTestFile.get(key)))) return { closures: closures.map((closure) => closureWithDigest(hashes, projectDigest, closure)), projectDigest, diff --git a/packages/stryker-js/src/plan-request.cell.ts b/packages/stryker-js/src/plan-request.cell.ts index 12897d2f2..12e4da840 100644 --- a/packages/stryker-js/src/plan-request.cell.ts +++ b/packages/stryker-js/src/plan-request.cell.ts @@ -186,6 +186,7 @@ const planProject = ( globalTestInputs: Option.map(coverage, (present) => [...present.globalTestInputs]).pipe( Option.getOrElse((): ReadonlyArray => []), ), + observedModules: Option.getOrUndefined(Option.map(coverage, (present) => present.testFileModules)), originalFileOf: (file) => path.resolve(file), }) const mutants = [ diff --git a/packages/stryker-js/src/run/dry-run-coverage.ts b/packages/stryker-js/src/run/dry-run-coverage.ts index a4356cf45..db170a8a0 100644 --- a/packages/stryker-js/src/run/dry-run-coverage.ts +++ b/packages/stryker-js/src/run/dry-run-coverage.ts @@ -6,6 +6,11 @@ import * as Option from 'effect/Option' import type { DryRunCoverage } from '../dry-run-coverage.schema.js' import type { TestCoverage } from '../test-coverage.schema.js' +export const testFileModulesFieldOf = ( + modules: Readonly> | undefined, +): { readonly testFileModules?: Readonly> } => + modules === undefined ? {} : { testFileModules: modules } + const testsByIdOf = (result: Readonly) => MutableHashMap.fromIterable(result.tests.map((test) => [test.id, test] as const)) @@ -85,6 +90,7 @@ export const reusedTestCoverage = (coverage: DryRunCoverage): TestCoverage => status: 'complete', tests: [...coverage.tests], globalTestInputs: [...coverage.globalTestInputs], + ...testFileModulesFieldOf(coverage.testFileModules), ...(coverage.mutantCoverage === undefined ? {} : { mutantCoverage: coverage.mutantCoverage }), }, dryRunCoverage: coverage, diff --git a/packages/stryker-js/src/run/dry-run.cell.ts b/packages/stryker-js/src/run/dry-run.cell.ts index d5e61c0d2..e624faa23 100644 --- a/packages/stryker-js/src/run/dry-run.cell.ts +++ b/packages/stryker-js/src/run/dry-run.cell.ts @@ -47,7 +47,7 @@ import { runInputsDigestOf } from '../verdict-semantics.js' import { testRunnerConfigOf } from '../vm-runner.js' import { IdGenerator, type IdGeneratorShape } from '../Worker.service.js' import { WorkerLauncher } from '../WorkerLauncher.service.js' -import { testCoverageOf } from './dry-run-coverage.js' +import { testCoverageOf, testFileModulesFieldOf } from './dry-run-coverage.js' import { incrementalReportTextsOf } from './incremental-reuse.js' import type { InstrumentDone } from './instrument.cell.js' import type { PhaseClock } from './phase-clock.service.js' @@ -70,6 +70,7 @@ export interface DryRunDone extends InstrumentDone { interface DryRunExtras { readonly reused?: DryRunCoverage | undefined readonly globalTestInputs: readonly string[] + readonly testFileModules: Readonly> | undefined readonly flakyTestIds: readonly string[] readonly flakyMutantIds: readonly string[] readonly testClosureDigest: string @@ -151,6 +152,7 @@ const closureAnalysisOf = ( command: InstrumentDone, rootDir: string, globalInputs: readonly string[], + observedModules: Readonly> | undefined, ) => Effect.option( analyzeImportClosure({ @@ -158,6 +160,7 @@ const closureAnalysisOf = ( projectFiles: Arr.dedupe([...MutableHashMap.keys(command.project.files), ...command.project.testFiles]), testFiles: [...command.project.testFiles], globalInputs: [...globalInputs], + ...(observedModules === undefined ? {} : { observedModules }), }), ) @@ -184,6 +187,73 @@ const originalGlobalInputsOf = (command: InstrumentDone, result: TestRunner.DryR () => [], ) +const sandboxOwnsPath = (command: InstrumentDone, value: string): boolean => + value.startsWith(`${command.sandbox.workingDirectory}/`) + +const originalModuleValuesOf = ( + command: InstrumentDone, + values: readonly string[], +): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const kept = yield* Effect.forEach( + values, + (value) => + Boolean.match(sandboxOwnsPath(command, value), { + onFalse: () => Effect.succeedSome(value), + onTrue: () => { + const original = originalFileFor(command.sandbox, value) + return Effect.map( + fs.exists(original).pipe(Effect.orElseSucceed(() => false)), + (exists) => (exists ? Option.some(original) : Option.none()), + ) + }, + }), + { concurrency: 16 }, + ) + return Arr.getSomes(kept) + }) + +const testFileModulesEntriesOf = ( + command: InstrumentDone, + modules: Readonly>, +): Effect.Effect>>, never, FileSystem.FileSystem> => + Effect.map( + Effect.forEach( + Object.entries(modules), + ([file, values]) => + Effect.map( + originalModuleValuesOf(command, values), + (kept) => [originalFileFor(command.sandbox, file), kept] as const, + ), + { concurrency: 8 }, + ), + (entries) => Option.some(Object.fromEntries(entries)), + ) + +const originalTestFileModulesOf = ( + command: InstrumentDone, + result: TestRunner.DryRunResult, +): Effect.Effect> | undefined, never, FileSystem.FileSystem> => + Effect.map( + Option.match( + Option.flatMap( + Option.liftPredicate(result, (value): value is TestRunner.CompleteDryRunResult => value.status === 'complete'), + (complete) => Option.fromUndefinedOr(complete.testFileModules), + ), + { + onNone: () => Effect.succeedNone, + onSome: (modules) => testFileModulesEntriesOf(command, modules), + }, + ), + Option.getOrUndefined, + ) + +const observedModulesOfCoverage = ( + coverage: Option.Option, +): Readonly> | undefined => + Option.getOrUndefined(Option.flatMap(coverage, (present) => Option.fromUndefinedOr(present.testFileModules))) + const decisionOfCoverage = ( prior: Option.Option, currentTestClosureDigest: Option.Option, @@ -264,6 +334,7 @@ const reusedRawOf = (command: InstrumentDone, coverage: DryRunCoverage): DryRunR status: 'complete', tests: [...coverage.tests], globalTestInputs: [...coverage.globalTestInputs], + ...testFileModulesFieldOf(coverage.testFileModules), ...(coverage.mutantCoverage === undefined ? {} : { mutantCoverage: coverage.mutantCoverage }), }, { reloadEnvironment: false }, @@ -271,6 +342,7 @@ const reusedRawOf = (command: InstrumentDone, coverage: DryRunCoverage): DryRunR { reused: coverage, globalTestInputs: [...coverage.globalTestInputs], + testFileModules: coverage.testFileModules, flakyTestIds: [...coverage.flakyTestIds], flakyMutantIds: [...coverage.flakyMutantIds], testClosureDigest: coverage.testClosureDigest, @@ -433,6 +505,7 @@ const freshCoverageOf = ( tests: [...dryRunResult.tests], ...(dryRunResult.mutantCoverage === undefined ? {} : { mutantCoverage: dryRunResult.mutantCoverage }), globalTestInputs: [...raw.globalTestInputs], + ...testFileModulesFieldOf(raw.testFileModules), timeOverheadMs: overheadMillis, flakyTestIds: [...raw.flakyTestIds], flakyMutantIds: [...raw.flakyMutantIds], @@ -466,7 +539,12 @@ const completeDryRunResultOf = Effect.fn(SpanTaxonomy.Spans.dryRunComplete.name) rawResult: TestRunner.CompleteDryRunResult, ) { const tests = withOriginalFileNames(rawResult.tests, raw.prev) - const dryRunResult = { ...rawResult, tests, status: 'complete' } as const + const dryRunResult = { + ...rawResult, + tests, + status: 'complete', + ...testFileModulesFieldOf(raw.testFileModules), + } as const return yield* Option.match(Option.fromUndefinedOr(raw.reused), { onNone: () => completeFreshDryRun(raw, dryRunResult), onSome: (coverage) => @@ -572,12 +650,14 @@ const runFreshDryRun = Effect.fnUntraced(function*( const flakes = flakesOf(first, second) const globalTestInputs = originalGlobalInputsOf(command, first) + const testFileModules = yield* originalTestFileModulesOf(command, first) const testClosureDigest = Option.getOrElse( - closureDigestOptionOf(yield* closureAnalysisOf(command, env.basePath, globalTestInputs)), + closureDigestOptionOf(yield* closureAnalysisOf(command, env.basePath, globalTestInputs, testFileModules)), () => Option.getOrElse(currentTestClosureDigest, () => ''), ) return rawOf(command, first, capabilities, gross, { globalTestInputs, + testFileModules, flakyTestIds: flakes.flakyTestIds, flakyMutantIds: flakes.flakyMutantIds, testClosureDigest, @@ -610,7 +690,12 @@ const readDryRun: (command: InstrumentDone) => Effect.Effect< const candidates = yield* priorCoveragesOf(command, env.basePath) const runInputsDigest = yield* runInputsDigestOf(fs, path, env.basePath, command.options) const currentTestClosureDigest = closureDigestOptionOf( - yield* closureAnalysisOf(command, env.basePath, globalInputsOfCoverage(Arr.head(candidates))), + yield* closureAnalysisOf( + command, + env.basePath, + globalInputsOfCoverage(Arr.head(candidates)), + observedModulesOfCoverage(Arr.head(candidates)), + ), ) const { prior, decision } = priorChoiceOf( candidates, diff --git a/packages/stryker-js/src/run/incremental-reuse.cell.ts b/packages/stryker-js/src/run/incremental-reuse.cell.ts index e0389dc9e..cd5dd0c56 100644 --- a/packages/stryker-js/src/run/incremental-reuse.cell.ts +++ b/packages/stryker-js/src/run/incremental-reuse.cell.ts @@ -302,6 +302,7 @@ const closureAnalysisOf = ( projectFiles: Arr.dedupe([...MutableHashMap.keys(input.project.files), ...input.project.testFiles]), testFiles: closureTestFilesOf(input), globalInputs: input.globalTestInputs.map((file) => input.originalFileOf(file)), + ...(input.observedModules === undefined ? {} : { observedModules: input.observedModules }), }).pipe( Effect.tapCause((cause: Cause.Cause) => Effect.logWarning( @@ -492,6 +493,7 @@ export interface IncrementalReuseInput { readonly force: boolean readonly options: Options.StrykerOptions readonly globalTestInputs: readonly string[] + readonly observedModules: Readonly> | undefined readonly originalFileOf: (file: string) => string } diff --git a/packages/stryker-js/src/run/mutation-test.cell.ts b/packages/stryker-js/src/run/mutation-test.cell.ts index 1f15acf59..844bd38b3 100644 --- a/packages/stryker-js/src/run/mutation-test.cell.ts +++ b/packages/stryker-js/src/run/mutation-test.cell.ts @@ -126,6 +126,7 @@ const proceedPipeline = Effect.fnUntraced(function*(raw: MutationTestRaw) { force: prev.options.force, options: prev.options, globalTestInputs: prev.dryRunResult.globalTestInputs ?? [], + observedModules: prev.dryRunResult.testFileModules, originalFileOf: (file) => originalFileFor(prev.sandbox, file), }) const rememberedResults = reuse.rememberedResults diff --git a/packages/stryker-js/tests/import-closure.integration.test.ts b/packages/stryker-js/tests/import-closure.integration.test.ts index 6c45cf5c3..5df938ead 100644 --- a/packages/stryker-js/tests/import-closure.integration.test.ts +++ b/packages/stryker-js/tests/import-closure.integration.test.ts @@ -23,6 +23,7 @@ interface FixtureSpec { readonly testFiles: readonly string[] readonly globalInputs?: readonly string[] readonly changed?: FixtureFiles + readonly observedModules?: Readonly> } interface Observation { @@ -76,6 +77,40 @@ const writeFixture = (spec: FixtureSpec): Effect.Effect => Effect.ignore(FileSystem.FileSystem.pipe(Effect.flatMap((fs) => fs.remove(root, { recursive: true })))) +const OBSERVED_SCHEME = /^[a-zA-Z][a-zA-Z0-9+.-]*:/ + +const observedModulesOf = ( + path: Path.Path, + root: string, + observed: Readonly> | undefined, +): { readonly observedModules?: Readonly> } => + observed === undefined + ? {} + : { + observedModules: Object.fromEntries( + Object.entries(observed).map(([testFile, keys]) => [ + path.join(root, testFile), + keys.map((key) => OBSERVED_SCHEME.test(key) ? key : path.join(root, key)), + ]), + ), + } + +interface ClosureInput { + readonly rootDir: string + readonly projectFiles: readonly string[] + readonly testFiles: readonly string[] + readonly globalInputs: readonly string[] + readonly observedModules?: Readonly> +} + +const inputOf = (path: Path.Path, root: string, spec: FixtureSpec): ClosureInput => ({ + rootDir: root, + projectFiles: Object.keys({ ...VITEST_STUB, ...spec.files }).filter((file) => !file.startsWith('node_modules/')), + testFiles: spec.testFiles.map((file) => path.join(root, file)), + globalInputs: (spec.globalInputs ?? []).map((file) => path.join(root, file)), + ...observedModulesOf(path, root, spec.observedModules), +}) + const observe = ( root: string, spec: FixtureSpec, @@ -83,15 +118,7 @@ const observe = ( Effect.gen(function*() { const fs = yield* FileSystem.FileSystem const path = yield* Path.Path - const projectFiles = Object.keys({ ...VITEST_STUB, ...spec.files }).filter( - (file) => !file.startsWith('node_modules/'), - ) - const input = { - rootDir: root, - projectFiles, - testFiles: spec.testFiles.map((file) => path.join(root, file)), - globalInputs: (spec.globalInputs ?? []).map((file) => path.join(root, file)), - } + const input = inputOf(path, root, spec) const before = yield* analyzeImportClosure(input) yield* Effect.forEach( Object.entries(spec.changed ?? {}), @@ -99,7 +126,29 @@ const observe = ( { discard: true }, ) const after = yield* analyzeImportClosure(input) - return { before, after, projectFiles } + return { before, after, projectFiles: input.projectFiles } + }) + +interface EvidenceObservation { + readonly before: ImportClosureAnalysis + readonly afterObserved: ImportClosureAnalysis + readonly afterUnreached: ImportClosureAnalysis +} + +const observeEvidence = ( + root: string, + spec: FixtureSpec, +): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const input = inputOf(path, root, spec) + const before = yield* analyzeImportClosure(input) + yield* fs.writeFileString(path.join(root, 'src/plugin.ts'), 'export const plugin = 2\n') + const afterObserved = yield* analyzeImportClosure(input) + yield* fs.writeFileString(path.join(root, 'src/other.ts'), 'export const other = 1\n') + const afterUnreached = yield* analyzeImportClosure(input) + return { before, afterObserved, afterUnreached } }) const closureOf = (analysis: ImportClosureAnalysis, testFile: string) => @@ -162,6 +211,31 @@ const NON_LITERAL: FixtureSpec = { changed: { 'src/unused.ts': 'export const unused = 1\n' }, } +const DYNAMIC_EVIDENCE: FixtureSpec = { + files: { + 'src/plugin.ts': 'export const plugin = 1\n', + 'src/other.ts': 'export const other = 0\n', + 'test/plugin.test.ts': + "import { test } from 'vitest'\nconst name = '../src/plugin.js'\nexport const load = () => import(name)\ntest('plugin', () => { load })\n", + }, + testFiles: ['test/plugin.test.ts'], +} + +const WITH_EVIDENCE: FixtureSpec = { + ...DYNAMIC_EVIDENCE, + observedModules: { 'test/plugin.test.ts': ['src/plugin.ts'] }, +} + +const WITH_DATA_EVIDENCE: FixtureSpec = { + ...DYNAMIC_EVIDENCE, + observedModules: { 'test/plugin.test.ts': ['data:text/javascript,export default 1'] }, +} + +const WITH_MISSING_EVIDENCE: FixtureSpec = { + ...DYNAMIC_EVIDENCE, + observedModules: { 'test/plugin.test.ts': ['src/gone.ts'] }, +} + const LITERALS: FixtureSpec = { files: { 'src/lazy.ts': 'export const lazy = 1\n', @@ -679,4 +753,82 @@ Feature('Mapping a test file to the import closure it can reach') ), ), ) + + scenario( + 'A computed dynamic import with no observed modules opens the closure', + Gherkin.Do.pipe( + Given('a project whose test file imports a module by a computed name')( + 'root', + () => writeFixture(DYNAMIC_EVIDENCE), + ), + When('the closure is analyzed without runtime evidence')( + 'observation', + (s) => observe(s.root, DYNAMIC_EVIDENCE).pipe(Effect.ensuring(removeDirectory(s.root))), + ), + Then('the closure is open')((s, expect) => + expect({ open: openOf(s.observation.before, 'test/plugin.test.ts') }).toEqual({ open: true }) + ), + ), + ) + + scenario( + 'Observed modules from the dry run close the closure around the file they name', + Gherkin.Do.pipe( + Given('a project whose dynamic import target a dry run observed')('root', () => writeFixture(WITH_EVIDENCE)), + When('the closure is analyzed, then the observed file changes, then an unrelated file changes')( + 'observation', + (s) => observeEvidence(s.root, WITH_EVIDENCE).pipe(Effect.ensuring(removeDirectory(s.root))), + ), + Then('the observed file joins a closed closure and only its change moves the digest')( + (s, expect) => + expect({ + files: filesOf(s.observation.before, 'test/plugin.test.ts'), + open: openOf(s.observation.before, 'test/plugin.test.ts'), + observedChangeMoved: digestOf(s.observation.before, 'test/plugin.test.ts') !== + digestOf(s.observation.afterObserved, 'test/plugin.test.ts'), + unreachedChangeMoved: digestOf(s.observation.afterObserved, 'test/plugin.test.ts') !== + digestOf(s.observation.afterUnreached, 'test/plugin.test.ts'), + }).toEqual({ + files: ['src/plugin.ts', 'test/plugin.test.ts'], + open: false, + observedChangeMoved: true, + unreachedChangeMoved: false, + }), + ), + ), + ) + + scenario( + 'An observed key that is not a file path keeps the closure open', + Gherkin.Do.pipe( + Given('a project whose dynamic import target a dry run reported as a data url')( + 'root', + () => writeFixture(WITH_DATA_EVIDENCE), + ), + When('the closure is analyzed with that evidence')( + 'observation', + (s) => observe(s.root, WITH_DATA_EVIDENCE).pipe(Effect.ensuring(removeDirectory(s.root))), + ), + Then('the closure is open')((s, expect) => + expect({ open: openOf(s.observation.before, 'test/plugin.test.ts') }).toEqual({ open: true }) + ), + ), + ) + + scenario( + 'An observed key naming a file no longer on disk keeps the closure open', + Gherkin.Do.pipe( + Given('a project whose dry run observed a file since removed')( + 'root', + () => writeFixture(WITH_MISSING_EVIDENCE), + ), + When('the closure is analyzed with that evidence')( + 'observation', + (s) => observe(s.root, WITH_MISSING_EVIDENCE).pipe(Effect.ensuring(removeDirectory(s.root))), + ), + Then('the closure is open')((s, expect) => + expect({ open: openOf(s.observation.before, 'test/plugin.test.ts') }).toEqual({ open: true }) + ), + ), + ) }) diff --git a/packages/stryker-js/tests/incremental-reuse.integration.test.ts b/packages/stryker-js/tests/incremental-reuse.integration.test.ts index 0787923ad..142072904 100644 --- a/packages/stryker-js/tests/incremental-reuse.integration.test.ts +++ b/packages/stryker-js/tests/incremental-reuse.integration.test.ts @@ -319,8 +319,49 @@ const VM_REACH_LOADER_TEST = [ '', 'const load = (name) => import(name)', '', - "test('exposes a loader whose specifier cannot be traced', () => {", - " expect(typeof load).toBe('function')", + "test('loads a module whose address only exists at runtime', async () => {", + " const loaded = await load('data:text/javascript,export default 1')", + ' expect(loaded.default).toBe(1)', + '})', + '', +].join('\n') + +const VM_WIRE_LEFT_SOURCE = [ + 'export const base = 1 + 1', + 'export const add = (left, right) => left + right', + '', +].join('\n') + +const VM_WIRE_PLUGIN_SOURCE = 'export const plugin = 1\n' + +const VM_WIRE_UNRELATED_SOURCE = 'export const untouched = () => 0\n' + +const VM_WIRE_LOADER_TEST = [ + "import { expect, test } from 'vitest'", + "import { add, base } from '../src/left.mjs'", + '', + 'const load = (name) => import(name)', + '', + "test('the module-level base is two and the runtime plugin loads', async () => {", + ' expect(base).toBe(2)', + ' expect(add(1, 2)).toBe(3)', + " const plugin = await load(new URL('../src/plugin.mjs', import.meta.url).href)", + ' expect(plugin.plugin).toBe(1)', + '})', + '', +].join('\n') + +const VM_WIRE_DATA_LOADER_TEST = [ + "import { expect, test } from 'vitest'", + "import { add, base } from '../src/left.mjs'", + '', + 'const load = (name) => import(name)', + '', + "test('the module-level base is two and the runtime module loads', async () => {", + ' expect(base).toBe(2)', + ' expect(add(1, 2)).toBe(3)', + " const loaded = await load('data:text/javascript,export default 1')", + ' expect(loaded.default).toBe(1)', '})', '', ].join('\n') @@ -1095,4 +1136,114 @@ Feature('Content-keyed reuse across incremental reports') ), ), ) + + const wireObservationOf = ( + loaderTest: string, + edit: 'plugin' | 'unrelated', + ): Effect.Effect => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const root = yield* writeVmFixture([ + ['package.json', '{ "type": "module" }\n'], + ['src/left.mjs', VM_WIRE_LEFT_SOURCE], + ['src/plugin.mjs', VM_WIRE_PLUGIN_SOURCE], + ['src/unrelated.mjs', VM_WIRE_UNRELATED_SOURCE], + ['test/loader.test.mjs', loaderTest], + ]).pipe(Effect.orDie) + return yield* Effect.ensuring( + Effect.gen(function*() { + const options = vmOptionsOf(root, { mutate: ['src/**/*.mjs'] }) + const first = yield* runOnce(root, options) + if (edit === 'plugin') { + yield* fs.writeFileString( + path.join(root, 'src', 'plugin.mjs'), + `${VM_WIRE_PLUGIN_SOURCE}// a changed plugin\n`, + ) + } else { + yield* fs.writeFileString( + path.join(root, 'src', 'unrelated.mjs'), + `${VM_WIRE_UNRELATED_SOURCE}// an edit no closure reaches\n`, + ) + } + const second = yield* runOnce(root, options) + return { first, second } + }), + removeFixture(root), + ) + }).pipe(Effect.orDie, Effect.provide(filePorts)) + + const leftMutantVerdictOf = (variation: ReuseVariation) => { + const firstReport = reusableReportOf(variation.first.incrementalText) + const leftIds = firstReport === undefined ? [] : mutantIdsInOf(firstReport, 'src/left.mjs') + const reusedIds = new Set( + variation.second.mutants + .filter((mutant) => mutant.statusReason === 'Remembered') + .map((mutant) => mutant.id), + ) + const firstStatuses = statusMapOf(variation.first.mutants) + const secondStatuses = statusMapOf(variation.second.mutants) + return { + runSucceeded: Exit.isSuccess(variation.first.exit) && Exit.isSuccess(variation.second.exit), + leftMutantsPositive: leftIds.length > 0, + kept: leftIds.length > 0 && + leftIds.every((id) => reusedIds.has(id) && secondStatuses.get(id) === firstStatuses.get(id)), + reran: leftIds.length > 0 && + leftIds.every((id) => !reusedIds.has(id) && secondStatuses.get(id) === firstStatuses.get(id)), + } + } + + scenario( + 'A runtime module a test file loads keeps the closure closed around an unreached edit', + Gherkin.Do.pipe( + Given('a workspace whose loader test imports its source statically and a plugin at runtime')( + 'variation', + () => wireObservationOf(VM_WIRE_LOADER_TEST, 'unrelated'), + ), + Then('the mutants covered by that test file are remembered across the unreached edit')((s, expect) => + expect(leftMutantVerdictOf(s.variation)).toEqual({ + runSucceeded: true, + leftMutantsPositive: true, + kept: true, + reran: false, + }) + ), + ), + ) + + scenario( + 'A change to the runtime module a test file loads re-runs the mutants it covers', + Gherkin.Do.pipe( + Given('a workspace whose loader test imports its source statically and a plugin at runtime')( + 'variation', + () => wireObservationOf(VM_WIRE_LOADER_TEST, 'plugin'), + ), + Then('the mutants covered by that test file re-run when its runtime target changes')((s, expect) => + expect(leftMutantVerdictOf(s.variation)).toEqual({ + runSucceeded: true, + leftMutantsPositive: true, + kept: false, + reran: true, + }) + ), + ), + ) + + scenario( + 'A runtime module behind a data url keeps the closure open around an unreached edit', + Gherkin.Do.pipe( + Given('a workspace whose loader test loads a module addressed by a data url')( + 'variation', + () => wireObservationOf(VM_WIRE_DATA_LOADER_TEST, 'unrelated'), + ), + Then('the mutants covered by that test file re-run because the closure stayed open')((s, expect) => + expect(leftMutantVerdictOf(s.variation)).toEqual({ + runSucceeded: true, + leftMutantsPositive: true, + kept: false, + reran: true, + }) + ), + ), + ) })