diff --git a/.github/workflows/rust-tops.yml b/.github/workflows/rust-tops.yml index c3ba44d..38e3505 100644 --- a/.github/workflows/rust-tops.yml +++ b/.github/workflows/rust-tops.yml @@ -386,6 +386,10 @@ jobs: tool: cargo-mutants@27.1.0 - run: cargo install --path crates/cargo-tops --locked - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" + env: + # The gate's own tests include a run of the real cargo-mutants: it + # must run here, never skip. + RUST_TOPS_REQUIRE_MUTANTS: "1" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default # features usually need no FFI sysdeps. Do not mark required until one diff --git a/PROTOCOL.md b/PROTOCOL.md index 6fdcde7..f9f8952 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -497,17 +497,15 @@ The first source that is set wins. A ref that is named by the flag or the key an `CI` counts as set when it is not empty and is not `0` or `false` (in any case). When no source gives a base, the layer is `fail` if `CI` is set. The report's `detail` and the gate's problem line name `fetch-depth: 0` and `--base`: a checkout that never fetched the base must not read as a clean diff. When `CI` is not set, the layer is `vacuous`, and the detail lists each source tried. A base that resolves but shares no history with `HEAD`, as in a shallow checkout, fails the diff and names `fetch-depth: 0` too. CI workflows pass `--base "origin/${{ github.base_ref || github.event.repository.default_branch }}"` from a checkout with `fetch-depth: 0`, because `actions/checkout` may not create `origin/HEAD`. +How the gate judges the layer: it removes any `mutants.out/outcomes.json` from an earlier run, runs `cargo mutants --output `, and reads the `outcomes.json` that run wrote there, whatever `output` an adopter's own `.cargo/mutants.toml` sets. A diff with nothing to mutate (documentation, comments, tests, CI) makes cargo-mutants 27.1.0 exit 0 and write no `mutants.out`; it says `Diff changes no Rust source files` or `No mutants to filter` on stderr. Exit 0 with no file passes as zero mutants only when the run printed one of those lines; any other missing file fails. The file must be in the shape cargo-mutants 27.1.0 writes; another release, a missing field, an outcome the gate does not know, a failed baseline, or totals that disagree with the listed outcomes (`total_mutants` included) fail the layer and name `outcomes.json`. Exit 2 with no missed outcome listed, and exit 3 with no timeout listed, fail too. cargo-mutants exits 2 when mutants were missed and 3 when some timed out; the gate treats 0, 2 and 3 as a finished run and judges it from the file, and any other exit is a failed command. Survivors are explained per function (§19): a missed mutant inside a function is explained only by a live `layer: mutation` exemption whose `symbol` names that function and whose glob covers its file, and one in no function (`function: null`) only by a live record with no `symbol` whose glob covers its file. Timeouts and unviable mutants are counted and reported; they are not survivors. The layer fails when the survivors nothing explains exceed `gates.mutation.unexplained_survivors_max`, and the problem names each of them. `kill_rate_min_pct` is a floor `check` compares with the class profile; the gate does not judge it. The mutation row's `judged` carries `caught`, `missed`, `explained`, `unexplained`, `timeouts`, `unviable` and `unexplained_survivors_max`, beside `base`. + Pass criteria: - baseline green -- **0 unexplained survivors** in the diff -- kill rate on examined mutants ≥ class minimum (see §4) - -A survivor is explained only if *all* of the following hold: +- **0 unexplained survivors** in the diff (`unexplained_survivors_max`) +- kill rate on examined mutants ≥ class minimum (see §4; declarative) -1. The mutant is equivalent (`x + 0`, `x | 0` on a proven-zero, debug-only formatting). -2. A one-line reason is written in the PR and in `exemptions` if it will persist. -3. `#[mutants::skip]` is applied at the tightest scope (expression, then function, never file) with the same reason. +A survivor is explained when a live `layer: mutation` exemption names its function (§19), and the record gives the reason: the mutant is equivalent (`x + 0`, `x | 0` on a proven-zero, debug-only formatting), or a named test covers the behaviour. `#[mutants::skip]` at the tightest scope (expression, then function, never file) with the same reason takes an equivalent mutant out of the run, so it is never a survivor to explain. ```rust #[mutants::skip] // equivalent: trailing comma in this macro has no runtime effect @@ -981,7 +979,9 @@ exemptions: Expired exemptions fail CI. Reasons shorter than one clause are rejected. -How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. +`cargo tops gate` checks every record, whatever its `layer`, whether that layer is on or off, and at any `--max-tier`: a record that has expired, or whose `expires` is not a real day, is a problem line in the report and the gate exits 1 (never 2; the config is still valid). `cargo tops check` prints the same lines as warnings and exits 0. The day is UTC: a record expires at 00:00 UTC on its `expires` date, whatever the local time zone. + +How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains survivors per function. A missed mutant inside a function is explained only by a record whose `symbol` names that function (matched as for `crap`) and whose `path` covers its file. A missed mutant in no function (cargo-mutants writes `function: null` for a `const` or `static` initialiser, a type alias or a field type) is explained only by a record with no `symbol` whose `path` covers its file. No record, of any shape or path, explains every survivor, so `gates.mutation.unexplained_survivors_max` always decides. A record with no `symbol` and a whole-crate `path` (`**`, `./**`, `**/*`, `.` or `./`) still lifts the class floor on `unexplained_survivors_max` in `cargo tops check` (§4), as for the other floors; it explains no survivor inside a function. A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. --- diff --git a/crates/cargo-tops/src/exemptions.rs b/crates/cargo-tops/src/exemptions.rs index 1697bd6..6b59565 100644 --- a/crates/cargo-tops/src/exemptions.rs +++ b/crates/cargo-tops/src/exemptions.rs @@ -58,24 +58,27 @@ impl Exemption { } } -/// The records in `exemptions` whose `layer` is `layer`. A record without a -/// string `path` or `expires` is skipped; the schema requires both. +/// One record, when it has a string `path` and `expires`; the schema +/// requires both. +fn record_of(entry: &Value) -> Option { + Some(Exemption { + path: entry.get("path")?.as_str()?.to_string(), + symbol: entry + .get("symbol") + .and_then(Value::as_str) + .map(str::to_string), + expires: entry.get("expires")?.as_str()?.to_string(), + }) +} + +/// The records in `exemptions` whose `layer` is `layer`. pub(crate) fn for_layer(exemptions: Option<&Value>, layer: &str) -> Vec { exemptions .and_then(Value::as_array) .into_iter() .flatten() .filter(|entry| entry.get("layer").and_then(Value::as_str) == Some(layer)) - .filter_map(|entry| { - Some(Exemption { - path: entry.get("path")?.as_str()?.to_string(), - symbol: entry - .get("symbol") - .and_then(Value::as_str) - .map(str::to_string), - expires: entry.get("expires")?.as_str()?.to_string(), - }) - }) + .filter_map(record_of) .collect() } diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index 285f4bb..f9f36f9 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -1,8 +1,9 @@ //! Run a plan: each command in order, timed. A dry run lists the commands //! and runs none. use std::collections::HashMap; +use std::io::{BufRead, BufReader}; use std::path::Path; -use std::process::Command; +use std::process::{ChildStderr, Command, ExitStatus, Stdio}; use std::time::{Instant, SystemTime}; use anyhow::{Context, Result}; @@ -12,6 +13,7 @@ use super::{Fault, Mode}; use crate::config::Layer; use crate::evidence; use crate::facts::{Outcome, Run}; +use crate::mutation::{self, Outcomes}; use crate::steps::{self, Base, Kind, MutationPlan, Step}; use crate::{laws, TOOL_LOCK}; @@ -56,6 +58,9 @@ pub(crate) struct Executed<'a> { pub(crate) not_run: Vec, /// Commands named, whether or not they ran. pub(crate) listed: usize, + /// What `cargo mutants` left in this run's `outcomes.json`: `None` when it + /// did not finish, `Err` when the file is not the shape [`mutation`] reads. + pub(crate) found: Option>, } /// The pinned Bend is fetched first: a missing Bend fails before any work. @@ -136,6 +141,7 @@ pub(crate) fn run<'a>(dir: &Path, plan: &GatePlan<'a>, mode: Mode) -> Result Runner<'_, 'a> { self.done.listed += 1; if self.live { let clock = Clock::start(); - let run = run_mutation(self.dir, &layer.id, mutation).map_err(Fault::env)?; + let Mutated { run, found } = + run_mutation(self.dir, &layer.id, mutation).map_err(Fault::env)?; + self.done.found = found; // The tool is asked for its version only when it ran: a vacuous // or failed diff runs no `cargo mutants`. let version = (run.outcome != Outcome::ToolMissing @@ -411,33 +419,92 @@ pub(crate) fn run_pinned_crap(dir: &Path, layer: &str, chain: &[Step], pin: &str Ok(run) } +/// A mutation layer's run, and what its `outcomes.json` held. +pub(crate) struct Mutated { + pub(crate) run: Run, + pub(crate) found: Option>, +} + +impl Mutated { + fn without_outcomes(run: Run) -> Self { + Self { run, found: None } + } +} + /// `cargo-mutants` is looked up before the diff, so an empty diff cannot /// hide a missing tool. A base that was named and is not a commit fails the /// layer before any command runs. A failed diff is `Failed`; nothing waives /// it. An empty one is `Vacuous` and `cargo mutants` does not run, except /// that no base at all fails under CI. [`steps::stop_note`] words every one -/// of these stops. -pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Result { +/// of these stops. When `cargo mutants` runs, an `outcomes.json` from an +/// earlier run is removed first, and exit 0, 2 (missed) and 3 (timeouts) +/// are a finished run whose ledger the judge reads. +pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Result { if let Err(missing) = steps::check_tool(&plan.mutants.argv) { println!("{missing}"); - return Ok(finish( + return Ok(Mutated::without_outcomes(finish( layer, &[plan.mutants.display()], Outcome::ToolMissing, - )); + ))); } if matches!(plan.base, Base::Unresolved { .. }) { - return Ok(stopped(layer, &[], &plan.base, Outcome::Failed)); + return Ok(Mutated::without_outcomes(stopped( + layer, + &[], + &plan.base, + Outcome::Failed, + ))); } let mut ran = vec![plan.diff.display()]; let diff_ok = run_step(dir, &plan.diff)? == Outcome::Passed; let diff = steps::judge_diff(diff_ok, &plan.diff_path); let Some(mutants) = plan.after_diff(diff) else { - return Ok(stopped(layer, &ran, &plan.base, diff)); + return Ok(Mutated::without_outcomes(stopped( + layer, &ran, &plan.base, diff, + ))); }; ran.push(mutants.display()); - let outcome = run_step(dir, mutants)?; - Ok(finish(layer, &ran, outcome)) + mutation::clear(&plan.output)?; + let ended = run_mutants(dir, mutants)?; + Ok(settled( + finish(layer, &ran, ended.outcome), + &ended, + &plan.output, + )) +} + +/// The run of `cargo mutants` once it ended. A command that failed says what +/// its exit means; one that finished (exit 0, 2 or 3) is judged from the +/// outcomes it wrote. +fn settled(mut run: Run, ended: &Ended, output: &Path) -> Mutated { + if ended.outcome != Outcome::Passed { + // A missing tool has its own line; a failed run says what its exit means. + run.note = (ended.outcome == Outcome::Failed) + .then(|| format!("`{}` {}", run.step, exit_meaning(ended.code))); + return Mutated::without_outcomes(run); + } + let found = mutation::read(output, ended.code.unwrap_or_default(), &ended.stderr); + if let Err(why) = &found { + run.outcome = Outcome::Failed; + run.note = Some(why.clone()); + } + Mutated { + run, + found: Some(found), + } +} + +/// What a failed `cargo mutants` exit means, in cargo-mutants' own terms. +fn exit_meaning(code: Option) -> String { + match code { + Some(1) => "exited 1: it was given arguments it does not accept".to_string(), + Some(4) => { + "exited 4: the tests fail in the unmutated tree, so no mutant was tested".to_string() + } + Some(code) => format!("exited {code}"), + None => "was ended by a signal".to_string(), + } } /// The run of a layer that stopped on its base or its diff: prints the @@ -474,12 +541,103 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { } Kind::Command => {} } + Ok(match spawn(dir, step)? { + Spawned::Exited(status) if status.success() => Outcome::Passed, + Spawned::Exited(_) | Spawned::NoProgram => Outcome::Failed, + Spawned::Missing => Outcome::ToolMissing, + }) +} + +/// How `cargo mutants` ended: its outcome, exit code and what it printed. +struct Ended { + outcome: Outcome, + code: Option, + stderr: String, +} + +impl Ended { + fn without_output(outcome: Outcome) -> Self { + Self { + outcome, + code: None, + stderr: String::new(), + } + } +} + +/// `cargo mutants` exits 2 when mutants were missed and 3 when some timed +/// out: those and 0 are a finished run, judged by the ledger. Any other exit +/// is a failed command. What it prints to stderr is shown and kept, because +/// a run that found nothing to mutate says so there and writes no file. +fn run_mutants(dir: &Path, step: &Step) -> Result { + let Some((program, args)) = step.program() else { + return Ok(Ended::without_output(Outcome::Failed)); + }; + let mut child = match Command::new(program) + .args(args) + .current_dir(dir) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + println!( + "{}", + steps::MissingTool { + binary: program.to_string() + } + ); + return Ok(Ended::without_output(Outcome::ToolMissing)); + } + Err(err) => return Err(err).with_context(|| format!("spawn: {}", step.display())), + }; + let stderr = tee(child.stderr.take()); + let status = child + .wait() + .with_context(|| format!("wait: {}", step.display()))?; + let code = status.code(); + let outcome = if matches!(code, Some(0 | 2 | 3)) { + Outcome::Passed + } else { + Outcome::Failed + }; + Ok(Ended { + outcome, + code, + stderr, + }) +} + +/// Everything `pipe` yields, passed on to our own stderr as it comes. +fn tee(pipe: Option) -> String { + let Some(pipe) = pipe else { + return String::new(); + }; + let mut kept = String::new(); + for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) { + let line = String::from_utf8_lossy(&line).into_owned(); + eprintln!("{line}"); + kept.push_str(&line); + kept.push('\n'); + } + kept +} + +/// How a spawned command ended. +enum Spawned { + Exited(ExitStatus), + /// The program is not there; the missing-tool line is already printed. + Missing, + /// The step names no program. + NoProgram, +} + +fn spawn(dir: &Path, step: &Step) -> Result { let Some((program, args)) = step.program() else { - return Ok(Outcome::Failed); + return Ok(Spawned::NoProgram); }; match Command::new(program).args(args).current_dir(dir).status() { - Ok(status) if status.success() => Ok(Outcome::Passed), - Ok(_) => Ok(Outcome::Failed), + Ok(status) => Ok(Spawned::Exited(status)), Err(err) if err.kind() == std::io::ErrorKind::NotFound => { println!( "{}", @@ -487,7 +645,7 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { binary: program.to_string() } ); - Ok(Outcome::ToolMissing) + Ok(Spawned::Missing) } Err(err) => Err(err).with_context(|| format!("spawn: {}", step.display())), } @@ -495,7 +653,7 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { #[cfg(test)] mod tests { - use super::{run_step, version_ask, Executed, Timing}; + use super::{exit_meaning, run_mutants, run_step, version_ask, Executed, Timing}; use crate::facts::Outcome; use crate::steps::Step; use std::fs; @@ -541,6 +699,7 @@ mod tests { shared: Vec::new(), not_run: Vec::new(), listed: 0, + found: None, }; assert!((done.seconds("a") - 3.5).abs() < 1e-9); assert!((done.seconds("b") - 4.0).abs() < 1e-9); @@ -566,4 +725,59 @@ mod tests { assert!(format!("{err:#}").contains("spawn"), "{err:#}"); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn cargo_mutants_exits_0_2_and_3_are_a_finished_run() { + let wants = [ + (0, Outcome::Passed), + (1, Outcome::Failed), + (2, Outcome::Passed), + (3, Outcome::Passed), + (4, Outcome::Failed), + (101, Outcome::Failed), + ]; + for (code, want) in wants { + let step = Step::command(["sh".to_string(), "-c".to_string(), format!("exit {code}")]); + let got = run_mutants(Path::new("."), &step).expect("it ran"); + assert_eq!(got.outcome, want, "exit {code}"); + assert_eq!(got.code, Some(code), "exit {code}"); + // Any other command still fails on every nonzero exit. + let plain = run_step(Path::new("."), &step).expect("it ran"); + let plain_want = if code == 0 { + Outcome::Passed + } else { + Outcome::Failed + }; + assert_eq!(plain, plain_want, "exit {code}"); + } + let nowhere = Step::command(["/no/such/program/here"]); + assert_eq!( + run_mutants(Path::new("."), &nowhere) + .expect("an outcome") + .outcome, + Outcome::ToolMissing + ); + } + + #[test] + fn a_mutants_spawn_error_is_an_error_and_a_signal_is_named() { + let dir = std::env::temp_dir().join(format!("cargo-tops-mutexec-{}", std::process::id())); + fs::remove_dir_all(&dir).ok(); + fs::create_dir_all(&dir).expect("mkdir"); + let plain = dir.join("plain"); + fs::write(&plain, "x").expect("write"); + let blocked = Step::command([plain.display().to_string()]); + let err = run_mutants(Path::new("."), &blocked) + .err() + .expect("not spawnable"); + assert!(format!("{err:#}").contains("spawn"), "{err:#}"); + fs::remove_dir_all(&dir).ok(); + let killed = Step::command(["sh".to_string(), "-c".to_string(), "kill -9 $$".to_string()]); + let ended = run_mutants(Path::new("."), &killed).expect("it ran"); + assert_eq!((ended.outcome, ended.code), (Outcome::Failed, None)); + assert!(exit_meaning(None).contains("signal")); + assert!(exit_meaning(Some(4)).contains("unmutated tree")); + assert!(exit_meaning(Some(1)).contains("exited 1")); + assert_eq!(exit_meaning(Some(77)), "exited 77"); + } } diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 53313c2..7dcaaad 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -11,7 +11,9 @@ use super::plan::GatePlan; use super::{Fault, Mode}; use crate::config::{self, TopsConfig}; use crate::facts::{self, Outcome, Run}; -use crate::{config_path, floors, laws, profiles, steps}; +use crate::mutation::Ledger; +use crate::scope::crate_root; +use crate::{config_path, exemptions, floors, laws, mutation, profiles, steps}; /// What the judging found. pub(crate) struct Judgement { @@ -21,8 +23,10 @@ pub(crate) struct Judgement { pub(crate) evaluated: String, /// The `count:` line. pub(crate) count: String, - /// Commands that ran, shared layers excluded. + /// Runs, shared layers excluded. pub(crate) ran: usize, + /// Commands that ran: a missing tool ran none, `bend` is one. + pub(crate) commands: usize, pub(crate) vacuous: usize, /// The numbers the coverage floors and the CRAP ratchet judged. pub(crate) measured: floors::Judged, @@ -33,6 +37,30 @@ pub(crate) struct Judgement { pub(crate) environment: bool, /// The laws in the pack, checked or not. pub(crate) law_total: usize, + /// The survivors `cargo mutants` left, sorted by the records that cover + /// them, when it finished. + pub(crate) survivors: Option, +} + +impl Judgement { + /// Whether a floor, or the survivor maximum, of layer `id` was missed. + pub(crate) fn missed(&self, id: &str) -> bool { + match id { + "coverage" => self.measured.coverage_problems > 0, + "crap" => self.measured.crap_problems > 0, + "mutation" => self + .survivors + .as_ref() + .is_some_and(|found| !found.ledger.passes(found.max)), + _ => false, + } + } +} + +/// The mutation ledger and the maximum it was judged against. +pub(crate) struct Survivors { + pub(crate) ledger: Ledger, + pub(crate) max: i64, } /// What `judge` reads. @@ -69,7 +97,7 @@ pub(crate) fn judge( ) -> Result { let total = law_total(dir); if mode == Mode::DryRun { - return Ok(dry_judgement(plan, done, total)); + return Ok(dry_judgement(cfg, plan, done, total)); } let inputs = Inputs { dir, @@ -78,24 +106,102 @@ pub(crate) fn judge( done, }; let mut failed = run_problems(done); + let survivors = judge_survivors(&inputs, &mut failed); // The coverage floors and the CRAP ratchet, on the reports just written. let measured = floors::judge(dir, cfg, &done.runs, &plan.report); failed.extend(measured.problems.iter().cloned()); + add_expired(cfg, &mut failed); let laws = judge_laws(&inputs, prepared, total, &mut failed)?; - Ok(finish(&inputs, failed, measured, laws, total)) + Ok(finish(&inputs, failed, measured, laws, total, survivors)) +} + +/// One problem for every exemption record that has expired, or whose date +/// is not a day, whatever its layer and whether or not the layer ran (§19). +pub(crate) fn expired_problems(cfg: &TopsConfig) -> Vec { + let today = exemptions::today(); + let records = cfg.exemptions.as_ref(); + let mut layers: Vec<&str> = Vec::new(); + for layer in records + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|entry| entry.get("layer")?.as_str()) + { + if !layers.contains(&layer) { + layers.push(layer); + } + } + layers + .into_iter() + .flat_map(|layer| { + exemptions::for_layer(records, layer) + .into_iter() + .filter(|record| !record.live(&today)) + .map(|record| record.expired_problem(layer)) + .collect::>() + }) + .collect() +} + +/// Add each expired record to `failed`. A record a floor already named is +/// not named twice. +fn add_expired(cfg: &TopsConfig, failed: &mut Vec) { + for problem in expired_problems(cfg) { + if !failed.contains(&problem) { + failed.push(problem); + } + } +} + +/// The crate's directory under the workspace root, which is what the +/// `file` of a mutant is relative to. Empty for a crate at the root. +fn crate_prefix(dir: &Path, name: &str) -> String { + let Some(root) = laws::workspace_root(dir) else { + return String::new(); + }; + crate_root(dir, name) + .strip_prefix(&root) + .map(|rest| rest.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// The survivors of this run's `cargo mutants`, matched to the live +/// `layer: mutation` records, and the problem when more than +/// `unexplained_survivors_max` are left. An outcomes file that could not be +/// read is already the layer's failure. +fn judge_survivors(inputs: &Inputs<'_, '_>, failed: &mut Vec) -> Option { + let Some(Ok(found)) = &inputs.done.found else { + return None; + }; + let cfg = inputs.cfg; + let today = exemptions::today(); + let mut live = exemptions::for_layer(cfg.exemptions.as_ref(), "mutation"); + live.retain(|record| record.live(&today)); + let prefix = crate_prefix(inputs.dir, &cfg.crate_.name); + let max = cfg.gates.mutation.unexplained_survivors_max; + let ledger = mutation::explain(found, &live, &prefix); + failed.extend(ledger.problem(max)); + Some(Survivors { ledger, max }) } /// Nothing ran, so the count is short for every layer: a dry run lists the /// commands and is never a pass. -fn dry_judgement(plan: &GatePlan<'_>, done: &Executed<'_>, total: usize) -> Judgement { +fn dry_judgement( + cfg: &TopsConfig, + plan: &GatePlan<'_>, + done: &Executed<'_>, + total: usize, +) -> Judgement { let (count, _) = command_count(0, &plan.enabled, &[], false, plan.max_tier, &done.shared); let mut failed = done.not_run.clone(); failed.push(count.clone()); + add_expired(cfg, &mut failed); Judgement { failed, evaluated: format!("laws evaluated: 0/{total} (dry run: bend did not run)"), count, ran: 0, + commands: 0, vacuous: 0, measured: floors::Judged::default(), laws_seconds: 0.0, @@ -103,6 +209,7 @@ fn dry_judgement(plan: &GatePlan<'_>, done: &Executed<'_>, total: usize) -> Judg laws_failed: false, environment: false, law_total: total, + survivors: None, } } @@ -130,17 +237,21 @@ fn finish( measured: floors::Judged, laws: LawsOutcome, law_total: usize, + survivors: Option, ) -> Judgement { let (plan, done) = (inputs.plan, inputs.done); // A layer that took another's run ran no command of its own. let ran = done.runs.len() - done.shared.len(); + // The `count:` line and the closing `fail:` line say the same number: + // commands that ran. + let commands = ran_commands(&done.runs, laws.ran, &done.shared); let vacuous = done .runs .iter() .filter(|run| run.outcome == Outcome::Vacuous) .count(); let (count, short) = command_count( - ran_commands(&done.runs, laws.ran, &done.shared), + commands, &plan.enabled, &done.runs, laws.ran, @@ -155,6 +266,7 @@ fn finish( evaluated: laws.evaluated, count, ran, + commands, vacuous, measured, laws_seconds: laws.seconds, @@ -165,6 +277,59 @@ fn finish( .iter() .any(|run| run.outcome == Outcome::ToolMissing), law_total, + survivors, + } +} + +/// The verdict of a layer's runs: a failure outranks a missing tool, which +/// outranks a vacuous run. +fn runs_verdict(runs: &[&Run]) -> &'static str { + let any = |outcome: Outcome| runs.iter().any(|run| run.outcome == outcome); + if any(Outcome::Failed) { + "fail" + } else if any(Outcome::ToolMissing) { + "tool-missing" + } else if any(Outcome::Vacuous) { + "vacuous" + } else { + "pass" + } +} + +/// The verdict of a layer enabled above `--max-tier`: it is not judged. +pub(crate) fn above_range_verdict() -> &'static str { + "not-evaluated" +} + +/// The verdict of a row the run never reached (a fault before it ran). +pub(crate) fn unreached_verdict() -> &'static str { + "not-run" +} + +/// The verdict of the laws layer: `not-run` when `bend` did not run. +pub(crate) fn laws_verdict(judged: Option<&Judgement>) -> &'static str { + match judged.filter(|judged| judged.bend_ran) { + None => "not-run", + Some(judged) if judged.laws_failed => "fail", + Some(_) => "pass", + } +} + +/// The row verdict of layer `id`: `not-run` when nothing ran for it, +/// otherwise its worst run. A run that passed still fails the row when its +/// floors, or its survivors, did not. +pub(crate) fn layer_verdict( + done: &Executed<'_>, + judged: Option<&Judgement>, + id: &str, +) -> &'static str { + let runs: Vec<&Run> = done.runs.iter().filter(|run| run.layer == id).collect(); + if runs.is_empty() { + return "not-run"; + } + match runs_verdict(&runs) { + "pass" if judged.is_some_and(|judged| judged.missed(id)) => "fail", + verdict => verdict, } } @@ -336,7 +501,8 @@ pub(crate) fn run_failure(run: &Run) -> Option { #[cfg(test)] mod tests { - use super::clear_cases; + use super::{clear_cases, crate_prefix, laws_verdict, Judgement}; + use crate::floors; use std::fs; use std::path::PathBuf; @@ -381,4 +547,59 @@ mod tests { ); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn a_crate_under_the_workspace_root_is_named_by_its_directory() { + let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .expect("workspace"); + assert_eq!(crate_prefix(&workspace, "cargo-tops"), "crates/cargo-tops"); + // No such package, or no cargo project here: the gate directory. + assert_eq!(crate_prefix(&workspace, "no-such-package"), ""); + let empty = scratch("prefix-none"); + assert_eq!(crate_prefix(&empty, "cargo-tops"), ""); + fs::remove_dir_all(&empty).ok(); + } + + /// A judgement of a run in which the laws layer did not run. + fn quiet() -> Judgement { + Judgement { + failed: Vec::new(), + evaluated: String::new(), + count: String::new(), + ran: 0, + commands: 0, + vacuous: 0, + measured: floors::Judged::default(), + laws_seconds: 0.0, + bend_ran: false, + laws_failed: false, + environment: false, + law_total: 0, + survivors: None, + } + } + + #[test] + fn the_laws_row_is_not_run_unless_bend_ran_and_then_pass_or_fail() { + assert_eq!(laws_verdict(None), "not-run"); + assert_eq!(laws_verdict(Some(&quiet())), "not-run"); + let failed_without_bend = Judgement { + laws_failed: true, + ..quiet() + }; + assert_eq!(laws_verdict(Some(&failed_without_bend)), "not-run"); + let passed = Judgement { + bend_ran: true, + ..quiet() + }; + assert_eq!(laws_verdict(Some(&passed)), "pass"); + let failed = Judgement { + bend_ran: true, + laws_failed: true, + ..quiet() + }; + assert_eq!(laws_verdict(Some(&failed)), "fail"); + } } diff --git a/crates/cargo-tops/src/gate/mod.rs b/crates/cargo-tops/src/gate/mod.rs index 69bc0fa..ebd89a6 100644 --- a/crates/cargo-tops/src/gate/mod.rs +++ b/crates/cargo-tops/src/gate/mod.rs @@ -6,10 +6,13 @@ //! - [`report`] prints the summary, writes `report.json` and picks the exit //! code. //! -//! Exit codes: 0 pass; 1 the gate ran and failed (a dry run included); 2 the -//! config is invalid; 3 the environment (a required tool or `cargo metadata` -//! is missing). Where more than one applies, 2 wins over 3, and 3 over 1; -//! every problem is still listed in the report. +//! Exit codes: 0 pass; 1 the gate ran and failed (a dry run included, and an +//! expired exemption); 2 the config or the request cannot be gated +//! (`rust-tops.yaml` missing or failing check, `rust-tops.lock` unreadable, +//! no enabled layer at or below `--max-tier`); 3 the environment (a required +//! tool or `cargo metadata` is missing, or `report.json` cannot be written). +//! Where more than one applies, 2 wins over 3, and 3 over 1; every problem is +//! still listed in the report. use std::path::Path; use std::process::ExitCode; diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index 279cde0..12f885e 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -10,11 +10,10 @@ use std::process::{Command, ExitCode}; use serde_json::{json, Map, Value}; use super::execute::Executed; -use super::judge::Judgement; +use super::judge::{self, Judgement}; use super::plan::{GatePlan, What}; use super::{Fault, Mode, Options}; use crate::config::TopsConfig; -use crate::facts::{Outcome, Run}; use crate::floors::{CoverageNumbers, CrapNumbers}; use crate::steps::{self, Base}; use crate::{laws, PROTOCOL_VERSION}; @@ -80,7 +79,6 @@ pub(crate) fn finish( done: &Executed<'_>, judged: &Judgement, ) -> ExitCode { - print_summary(opts, done, judged); let kind = if judged.failed.is_empty() { Kind::Pass } else if judged.environment { @@ -88,7 +86,9 @@ pub(crate) fn finish( } else { Kind::Failed }; - emit(&Parts { + // The report is written before the summary, so the summary's last line + // is true: a report that could not be written ends it with `fail:`. + let (code, unwritten) = write_report(&Parts { dir, cfg: Some(cfg), opts, @@ -98,48 +98,67 @@ pub(crate) fn finish( done: Some(done), judged: Some(judged), report_dir: plan.report.clone(), - }) + }); + print_summary(opts, done, judged, unwritten.as_deref()); + code } -fn print_summary(opts: Options, done: &Executed<'_>, judged: &Judgement) { +/// `unwritten` is the path of a report that could not be written. +fn print_summary(opts: Options, done: &Executed<'_>, judged: &Judgement, unwritten: Option<&Path>) { println!(); println!("{}", judged.evaluated); println!("{}", judged.count); if opts.mode == Mode::DryRun { - if done.not_run.is_empty() { - println!( - "dry-run: {} command(s) listed, 0 run; a dry run is not a pass", - done.listed - ); - } else { - println!( - "dry-run: {} command(s) listed, 0 run; {} layer(s) NotRun, so the gate would fail", - done.listed, - done.not_run.len() - ); - } - for problem in &judged.failed { - println!(" - {problem}"); - } + print_dry_run(done, judged); } else if judged.failed.is_empty() { - if judged.vacuous == 0 { - println!("ok: {} command(s) passed", judged.ran); - } else { - println!( - "ok: {} command(s) passed, {} vacuous (counted for no law)", - judged.ran - judged.vacuous, - judged.vacuous - ); - } + print_pass(judged, unwritten); } else { println!( "fail: {} problem(s), {} command(s) run", judged.failed.len(), - judged.ran + judged.commands + ); + print_problems(judged); + } +} + +fn print_problems(judged: &Judgement) { + for problem in &judged.failed { + println!(" - {problem}"); + } +} + +fn print_dry_run(done: &Executed<'_>, judged: &Judgement) { + if done.not_run.is_empty() { + println!( + "dry-run: {} command(s) listed, 0 run; a dry run is not a pass", + done.listed + ); + } else { + println!( + "dry-run: {} command(s) listed, 0 run; {} layer(s) NotRun, so the gate would fail", + done.listed, + done.not_run.len() + ); + } + print_problems(judged); +} + +/// No problem: `ok`, unless the report that says so could not be written. +fn print_pass(judged: &Judgement, unwritten: Option<&Path>) { + if let Some(path) = unwritten { + println!( + "fail: {} could not be written; the gate has no verdict on record", + path.display() + ); + } else if judged.vacuous == 0 { + println!("ok: {} command(s) passed", judged.ran); + } else { + println!( + "ok: {} command(s) passed, {} vacuous (counted for no law)", + judged.ran - judged.vacuous, + judged.vacuous ); - for problem in &judged.failed { - println!(" - {problem}"); - } } } @@ -210,9 +229,15 @@ struct Parts<'a> { report_dir: PathBuf, } -/// Write the report. A report that cannot be written is an environment -/// fault, unless the config fault is already the exit code. +/// Write the report and return the exit code. fn emit(parts: &Parts<'_>) -> ExitCode { + write_report(parts).0 +} + +/// Write the report. A report that cannot be written is an environment +/// fault, unless the config fault is already the exit code; its path is +/// returned so the summary can say so. +fn write_report(parts: &Parts<'_>) -> (ExitCode, Option) { let mut kind = parts.kind; let path = parts.report_dir.join(REPORT); let wrote = fs::create_dir_all(&parts.report_dir).and_then(|()| { @@ -220,13 +245,15 @@ fn emit(parts: &Parts<'_>) -> ExitCode { serde_json::to_string_pretty(&document(parts, kind)).map_err(std::io::Error::other)?; fs::write(&path, format!("{text}\n")) }); + let mut unwritten = None; if let Err(err) = wrote { eprintln!("error: write {}: {err}", path.display()); if matches!(kind, Kind::Pass | Kind::Failed) { kind = Kind::Env; } + unwritten = Some(path); } - ExitCode::from(kind.code()) + (ExitCode::from(kind.code()), unwritten) } fn git_sha(dir: &Path) -> Option { @@ -310,6 +337,16 @@ fn row( row } +/// A layer enabled above `--max-tier`: named, not judged. +fn above_range_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { + let mut entry = row(id, tier, judge::above_range_verdict(), &[], 0.0, None); + entry.insert( + "detail".into(), + json!(format!("above --max-tier {}", parts.opts.max_tier)), + ); + entry +} + /// One row per enabled layer, in `rust-tops.yaml` order, and the ids of the /// layers that are off. fn layer_rows(parts: &Parts<'_>) -> (Vec, Vec) { @@ -325,12 +362,7 @@ fn layer_rows(parts: &Parts<'_>) -> (Vec, Vec) { } let tier = crate::tiers::tier_for(layer); let mut entry = if tier > parts.opts.max_tier { - let mut entry = row(&layer.id, tier, "not-evaluated", &[], 0.0, None); - entry.insert( - "detail".into(), - json!(format!("above --max-tier {}", parts.opts.max_tier)), - ); - entry + above_range_row(parts, &layer.id, tier) } else if layer.id == "laws" { laws_row(parts, tier) } else { @@ -349,58 +381,30 @@ fn layer_rows(parts: &Parts<'_>) -> (Vec, Vec) { (rows, disabled) } -/// The verdict of a layer's runs: a failure outranks a missing tool, which -/// outranks a vacuous run. -fn runs_verdict(runs: &[&Run]) -> &'static str { - let any = |outcome: Outcome| runs.iter().any(|run| run.outcome == outcome); - if any(Outcome::Failed) { - "fail" - } else if any(Outcome::ToolMissing) { - "tool-missing" - } else if any(Outcome::Vacuous) { - "vacuous" - } else { - "pass" - } -} - -/// The numbers a layer's floors were judged on, and whether any floor missed. -fn judged_numbers(parts: &Parts<'_>, id: &str) -> (Option, bool) { - let Some(measured) = parts.judged.map(|judged| &judged.measured) else { - return (None, false); - }; +/// The numbers a layer's floors, or its survivors, were judged on. +fn judged_numbers(parts: &Parts<'_>, id: &str) -> Option { + let judged = parts.judged?; match id { - "coverage" => ( - measured.coverage.as_ref().map(CoverageNumbers::to_json), - measured.coverage_problems > 0, - ), - "crap" => ( - measured.crap.as_ref().map(CrapNumbers::to_json), - measured.crap_problems > 0, - ), - _ => (None, false), + "coverage" => judged + .measured + .coverage + .as_ref() + .map(CoverageNumbers::to_json), + "crap" => judged.measured.crap.as_ref().map(CrapNumbers::to_json), + "mutation" => judged + .survivors + .as_ref() + .map(|found| found.ledger.to_json(found.max)), + _ => None, } } -/// `not-run` when nothing ran for the layer; otherwise its worst run. -fn unit_verdict(done: &Executed<'_>, id: &str) -> &'static str { - let runs: Vec<&Run> = done.runs.iter().filter(|run| run.layer == id).collect(); - if runs.is_empty() { - return "not-run"; - } - runs_verdict(&runs) -} - fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { let (Some(plan), Some(done)) = (parts.plan, parts.done) else { - return row(id, tier, "not-run", &[], 0.0, None); - }; - let (numbers, missed) = judged_numbers(parts, id); - let verdict = match unit_verdict(done, id) { - // The run passed and its floors did not. - "pass" if missed => "fail", - verdict => verdict, + return row(id, tier, judge::unreached_verdict(), &[], 0.0, None); }; + let numbers = judged_numbers(parts, id); + let verdict = judge::layer_verdict(done, parts.judged, id); let version = done.tool_version(id); let mut entry = row( id, @@ -444,11 +448,10 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map json!({ "ref": null, "source": null, "tried": tried }), }; put_judged(entry, "base", base); - let ran = run_note(done, "mutation"); - let planned = (!matches!(mutation.base, Base::Found { .. })) - .then(|| steps::stop_note(&mutation.base, Outcome::Vacuous).map(|(_, note)| note)) - .flatten(); - if let Some(detail) = ran.or(planned) { + // A run that stopped says why in its own words. A row with no such note + // (a missing tool, a dry run) names the base's fault and no outcome. + if let Some(detail) = run_note(done, "mutation").or_else(|| steps::base_finding(&mutation.base)) + { entry.insert("detail".into(), json!(detail)); } } @@ -490,10 +493,10 @@ fn laws_row(parts: &Parts<'_>, tier: u8) -> Map { "bend".to_string(), parts.report_dir.join("cases.bend").display().to_string(), ]]; + let verdict = judge::laws_verdict(parts.judged); let Some(judged) = parts.judged.filter(|judged| judged.bend_ran) else { - return row("laws", tier, "not-run", &argv, 0.0, None); + return row("laws", tier, verdict, &argv, 0.0, None); }; - let verdict = if judged.laws_failed { "fail" } else { "pass" }; let cases = read_cases(&parts.report_dir); let version = cases.as_ref().and_then(|cases| cases.get("bend")?.as_str()); row("laws", tier, verdict, &argv, judged.laws_seconds, version) diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 4ad0711..2c0217c 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -31,6 +31,7 @@ mod floors; mod gate; mod init_meta; mod laws; +mod mutation; mod profile_data; mod profiles; mod schema; @@ -92,8 +93,12 @@ enum Commands { /// Run enabled Tier 0–1 tools in protocol order, then the Bend laws layer #[command(after_help = "Writes /rust-tops/report.json on every run.\n\ Exit codes:\n 0 pass\n 1 the gate ran and failed (a dry run included)\n \ -2 the config is invalid (rust-tops.yaml unreadable or failing check)\n \ -3 the environment: a required tool or cargo metadata is missing\n\ +2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable \ +or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable \ +when a layer in range needs a pin (crap today); a command-line usage error is 2 too and \ +writes no report. An expired exemption is not 2: it is 1\n \ +3 the environment: a required tool or cargo metadata is missing, or report.json \ +cannot be written\n\ Where more than one applies, 2 wins over 3, and 3 over 1.")] Gate { #[arg(long, default_value = ".")] @@ -181,15 +186,7 @@ fn run() -> Result { init(&args)?; Ok(ExitCode::SUCCESS) } - Commands::Check { path } => { - let cfg = load_config(&path)?; - check(&cfg)?; - println!( - "ok: {} ({}) matches rust-tops {}", - cfg.crate_.name, cfg.crate_.class, cfg.protocol.version - ); - Ok(ExitCode::SUCCESS) - } + Commands::Check { path } => check_command(&path), Commands::Gate { path, max_tier, @@ -215,6 +212,21 @@ fn run() -> Result { } } +/// `cargo tops check`: an expired exemption is a warning here and a failure +/// of the gate; the config is still valid. +fn check_command(path: &Path) -> Result { + let cfg = load_config(path)?; + check(&cfg)?; + for problem in gate::judge::expired_problems(&cfg) { + println!("warning: {problem}"); + } + println!( + "ok: {} ({}) matches rust-tops {}", + cfg.crate_.name, cfg.crate_.class, cfg.protocol.version + ); + Ok(ExitCode::SUCCESS) +} + /// `cargo tops gate`: a config that does not load, or fails `check`, is exit /// 2 with a report; anything else is the gate's own verdict. fn run_gate(path: &Path, opts: gate::Options, base: Option<&str>) -> ExitCode { diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs new file mode 100644 index 0000000..d9af1d3 --- /dev/null +++ b/crates/cargo-tops/src/mutation.rs @@ -0,0 +1,822 @@ +//! The survivors ledger: what `cargo mutants` left in `mutants.out/outcomes.json`. +//! +//! [`parse`] reads the file in the shape cargo-mutants [`PINNED`] writes and +//! refuses any other: a version it was not written for, a field it cannot +//! read, an outcome it does not know, or totals that disagree with the +//! outcomes. [`explain`] sorts the missed mutants into those a live +//! `layer: mutation` exemption covers (the rule of [`Exemption::covers`]) +//! and those nothing covers. The gate passes the layer when the unexplained +//! count is at most `gates.mutation.unexplained_survivors_max`. Timeouts and +//! unviable mutants are counted and reported; they are not survivors. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + +use std::fs; +use std::path::Path; + +use anyhow::{Context, Result}; +use serde_json::{json, Value}; + +use crate::exemptions::Exemption; + +/// Where cargo-mutants writes the run's outcomes, under the directory it ran in. +const OUTCOMES: &str = "mutants.out/outcomes.json"; + +/// What cargo-mutants 27.1.0 prints (to stderr) when the diff leaves nothing +/// to mutate, and then exits 0 without writing `mutants.out`: no Rust source +/// file changed, or the changed lines are in no function it mutates (a +/// comment, a test, a documentation or CI file). Recorded from real runs. +const NO_MUTANTS: [&str; 2] = ["Diff changes no Rust source files", "No mutants to filter"]; + +/// The cargo-mutants release whose `outcomes.json` this reads. +pub(crate) const PINNED: &str = "27.1.0"; + +/// One mutant the suite did not catch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Missed { + /// The file, as cargo-mutants names it: relative to the workspace root. + pub file: String, + /// The function the mutant is in, `module::name` for a nested one. + pub function: Option, + pub line: u64, + /// cargo-mutants' own name for the mutant, such as + /// `src/lib.rs:8:5: replace double -> i32 with 0`. + pub name: String, +} + +/// What a run found, by kind. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Outcomes { + pub caught: usize, + pub missed: Vec, + pub timeouts: usize, + pub unviable: usize, +} + +/// The missed mutants of a run, sorted by the records that cover them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Ledger { + pub caught: usize, + pub explained: Vec, + pub unexplained: Vec, + pub timeouts: usize, + pub unviable: usize, +} + +/// Remove the `outcomes.json` an earlier run left, so one found after +/// `cargo mutants` is the one it just wrote. +pub(crate) fn clear(dir: &Path) -> Result<()> { + let path = dir.join(OUTCOMES); + match fs::remove_file(&path) { + Err(err) if err.kind() != std::io::ErrorKind::NotFound => { + Err(err).with_context(|| format!("remove {}", path.display())) + } + _ => Ok(()), + } +} + +/// The outcomes `cargo mutants` just wrote under `dir`, after it ended with +/// `code` and printed `stderr`. Call [`clear`] before the run: a file that is +/// there afterwards is this run's. No file is the zero mutants of a diff with +/// nothing to mutate only when the run exited 0 and said so itself; any other +/// missing file is a failure. +pub(crate) fn read(dir: &Path, code: i32, stderr: &str) -> Result { + let text = match fs::read_to_string(dir.join(OUTCOMES)) { + Ok(text) => text, + Err(_) if code == 0 && says_no_mutants(stderr) => { + return Ok(Outcomes::default()); + } + Err(err) => { + return Err(format!( + "no {OUTCOMES} after cargo mutants exited {code}, and its output does not \ + say it found no mutants: {err}" + )); + } + }; + let found = parse(&text)?; + check_exit(&found, code)?; + Ok(found) +} + +/// Whether a whole line of `stderr` is cargo-mutants' `INFO` line for a diff +/// with nothing to mutate (colour codes and surrounding space aside). +fn says_no_mutants(stderr: &str) -> bool { + stderr.lines().any(|line| { + let plain = without_colour(line); + let text = plain.trim(); + NO_MUTANTS.iter().any(|said| { + text.strip_prefix("INFO") + .is_some_and(|rest| rest.trim() == *said) + }) + }) +} + +/// `line` without ANSI colour sequences. +fn without_colour(line: &str) -> String { + let mut plain = String::new(); + let mut chars = line.chars(); + while let Some(c) = chars.next() { + if c == '\u{1b}' { + // `ESC [ ... letter` + for skipped in chars.by_ref() { + if skipped.is_ascii_alphabetic() { + break; + } + } + } else { + plain.push(c); + } + } + plain +} + +/// The exit code and the file must agree: 2 says mutants were missed and 3 +/// says some timed out. +fn check_exit(found: &Outcomes, code: i32) -> Result<(), String> { + match code { + 2 if found.missed.is_empty() => Err( + "cargo mutants exited 2 (mutants were missed) but outcomes.json lists no missed mutant" + .to_string(), + ), + 3 if found.timeouts == 0 => Err( + "cargo mutants exited 3 (mutants timed out) but outcomes.json lists no timeout" + .to_string(), + ), + _ => Ok(()), + } +} + +fn field<'v>(value: &'v Value, key: &str) -> Result<&'v Value, String> { + value.get(key).ok_or_else(|| { + format!("outcomes.json has no `{key}` where cargo-mutants {PINNED} writes one") + }) +} + +fn count(value: &Value, key: &str) -> Result { + field(value, key)? + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| format!("outcomes.json `{key}` is not a count")) +} + +/// The missed mutant a `Mutant` scenario describes. +fn missed(scenario: &Value) -> Result { + let mutant = field(scenario, "Mutant")?; + let function = match field(mutant, "function")? { + Value::Null => None, + named => Some( + field(named, "function_name")? + .as_str() + .ok_or("outcomes.json `function_name` is not a string")? + .to_string(), + ), + }; + let line = field(field(field(mutant, "span")?, "start")?, "line")? + .as_u64() + .ok_or("outcomes.json span `line` is not a number")?; + let text = |key: &str| -> Result { + Ok(field(mutant, key)? + .as_str() + .ok_or_else(|| format!("outcomes.json `{key}` is not a string"))? + .to_string()) + }; + Ok(Missed { + file: text("file")?, + function, + line, + name: text("name")?, + }) +} + +/// What one outcome says happened to its mutant. +#[derive(Clone, Copy)] +enum Kind { + Caught, + Missed, + Timeout, + Unviable, +} + +/// The scenario and the summary of one outcome. +fn parts(outcome: &Value) -> Result<(&Value, &str), String> { + let scenario = field(outcome, "scenario")?; + let summary = field(outcome, "summary")? + .as_str() + .ok_or("outcomes.json `summary` is not a string")?; + Ok((scenario, summary)) +} + +/// The baseline run is not a mutant: it must have succeeded. +fn baseline(summary: &str) -> Result<(), String> { + match summary { + "Success" => Ok(()), + other => Err(format!("the baseline run ended as `{other}`")), + } +} + +fn kind(summary: &str) -> Result { + match summary { + "CaughtMutant" => Ok(Kind::Caught), + "MissedMutant" => Ok(Kind::Missed), + "Timeout" => Ok(Kind::Timeout), + "Unviable" => Ok(Kind::Unviable), + other => Err(format!( + "outcomes.json has an outcome `{other}` this gate does not know" + )), + } +} + +impl Outcomes { + fn add(&mut self, kind: Kind, scenario: &Value) -> Result<(), String> { + match kind { + Kind::Caught => self.caught += 1, + Kind::Missed => self.missed.push(missed(scenario)?), + Kind::Timeout => self.timeouts += 1, + Kind::Unviable => self.unviable += 1, + } + Ok(()) + } +} + +/// Add one outcome to `found`. +fn take(found: &mut Outcomes, outcome: &Value) -> Result<(), String> { + let (scenario, summary) = parts(outcome)?; + if scenario.as_str() == Some("Baseline") { + return baseline(summary); + } + found.add(kind(summary)?, scenario) +} + +/// Read `outcomes.json`. Any shape but the pinned one is an `Err` saying +/// what was not as expected, never a pass. +pub(crate) fn parse(text: &str) -> Result { + let doc: Value = + serde_json::from_str(text).map_err(|err| format!("outcomes.json is not JSON: {err}"))?; + let version = field(&doc, "cargo_mutants_version")? + .as_str() + .unwrap_or_default(); + if version != PINNED { + return Err(format!( + "outcomes.json is from cargo-mutants {version:?}; this gate reads {PINNED}" + )); + } + let list = field(&doc, "outcomes")? + .as_array() + .ok_or("outcomes.json `outcomes` is not a list")?; + let mut found = Outcomes::default(); + for outcome in list { + take(&mut found, outcome)?; + } + let totals = [ + ("caught", found.caught), + ("missed", found.missed.len()), + ("timeout", found.timeouts), + ("unviable", found.unviable), + ( + "total_mutants", + found.caught + found.missed.len() + found.timeouts + found.unviable, + ), + ]; + for (key, seen) in totals { + let said = count(&doc, key)?; + if said != seen { + return Err(format!("outcomes.json says {said} {key} but lists {seen}")); + } + } + Ok(found) +} + +/// `file` relative to the crate root, given the crate's path under the +/// workspace root (`""` for a crate at the root). +fn relative<'f>(file: &'f str, crate_prefix: &str) -> &'f str { + let prefix = crate_prefix.trim_matches('/'); + if prefix.is_empty() { + return file; + } + file.strip_prefix(prefix) + .and_then(|rest| rest.strip_prefix('/')) + .unwrap_or(file) +} + +/// Whether `record` explains a survivor in `function` of the file `rel`, +/// per function: a survivor inside a function is explained only by a record +/// whose `symbol` names that function and whose glob covers its file. A +/// survivor in no function (`function: null`: a `const` or `static` +/// initialiser, a type alias, a field type) is explained only by a record +/// with no symbol whose glob covers its file. No record explains every +/// survivor, so `unexplained_survivors_max` always decides. +fn explains(record: &Exemption, rel: &str, function: Option<&str>) -> bool { + record.symbol.is_some() == function.is_some() && record.covers(rel, function) +} + +/// Sort the missed mutants. `live` are the layer's records that have not +/// expired; `crate_prefix` is the crate's directory under the workspace root, +/// which the records' globs are relative to. +pub(crate) fn explain(found: &Outcomes, live: &[Exemption], crate_prefix: &str) -> Ledger { + let (explained, unexplained) = found.missed.iter().cloned().partition(|mutant: &Missed| { + let rel = relative(&mutant.file, crate_prefix); + live.iter() + .any(|record| explains(record, rel, mutant.function.as_deref())) + }); + Ledger { + caught: found.caught, + explained, + unexplained, + timeouts: found.timeouts, + unviable: found.unviable, + } +} + +impl Ledger { + /// Whether the layer passes: at most `max` survivors nothing explains. + pub(crate) fn passes(&self, max: i64) -> bool { + i64::try_from(self.unexplained.len()).is_ok_and(|n| n <= max) + } + + /// The problem that names each unexplained survivor, or `None` when the + /// layer passes. Explained survivors are never named. + pub(crate) fn problem(&self, max: i64) -> Option { + if self.passes(max) { + return None; + } + let named: Vec = self + .unexplained + .iter() + .map(|mutant| mutant.name.clone()) + .collect(); + Some(format!( + "mutation: {} unexplained survivor(s), above unexplained_survivors_max {max}: {}", + named.len(), + named.join("; ") + )) + } + + /// The numbers the mutation row's `judged` carries. + pub(crate) fn to_json(&self, max: i64) -> Value { + json!({ + "caught": self.caught, + "missed": self.explained.len() + self.unexplained.len(), + "explained": self.explained.len(), + "unexplained": self.unexplained.len(), + "timeouts": self.timeouts, + "unviable": self.unviable, + "unexplained_survivors_max": max, + }) + } +} + +#[cfg(test)] +#[expect( + clippy::indexing_slicing, + reason = "the tests edit and read JSON they built from the recorded fixture" +)] +mod tests { + use super::{check_exit, clear, explain, parse, read, relative, Outcomes, PINNED}; + use crate::exemptions::Exemption; + use serde_json::Value; + use std::fs; + + /// A real run of cargo-mutants 27.1.0 over a small crate: 11 caught, 10 + /// missed (5 in `double`, 5 in `inner::triple`), 1 timeout, 1 unviable. + type Edit = fn(&mut Value); + + const FIXTURE: &str = include_str!("../tests/fixtures/mutants-27.1.0/outcomes.json"); + + fn record(path: &str, symbol: Option<&str>) -> Exemption { + Exemption { + path: path.to_string(), + symbol: symbol.map(str::to_string), + expires: "2099-01-01".to_string(), + } + } + + fn edited(change: impl FnOnce(&mut Value)) -> String { + let mut doc: Value = serde_json::from_str(FIXTURE).expect("fixture is JSON"); + change(&mut doc); + doc.to_string() + } + + #[test] + fn the_recorded_run_has_the_counts_cargo_mutants_reported() { + let found = parse(FIXTURE).expect("the pinned shape parses"); + assert_eq!(found.caught, 11); + assert_eq!(found.missed.len(), 10); + assert_eq!(found.timeouts, 1); + assert_eq!(found.unviable, 1); + let first = found.missed.first().expect("a survivor"); + assert_eq!(first.file, "src/lib.rs"); + assert_eq!(first.function.as_deref(), Some("double")); + assert_eq!(first.line, 8); + assert_eq!(first.name, "src/lib.rs:8:5: replace double -> i32 with 0"); + let nested = found + .missed + .iter() + .filter(|m| m.function.as_deref() == Some("inner::triple")) + .count(); + assert_eq!(nested, 5); + } + + #[test] + fn another_release_is_refused_naming_both() { + let text = edited(|doc| doc["cargo_mutants_version"] = "26.0.0".into()); + let err = parse(&text).expect_err("not the pinned release"); + assert!(err.contains("26.0.0") && err.contains(PINNED), "{err}"); + let text = edited(|doc| { + doc.as_object_mut() + .expect("object") + .remove("cargo_mutants_version"); + }); + assert!(parse(&text).is_err()); + } + + #[test] + fn an_unknown_shape_is_an_error_and_never_a_pass() { + assert!(parse("").is_err()); + assert!(parse("[]").is_err()); + assert!(parse("{}").is_err()); + let cases: [(&str, Edit); 7] = [ + ("outcomes is not a list", |d| d["outcomes"] = "x".into()), + ("an unknown summary", |d| { + d["outcomes"][1]["summary"] = "Skipped".into() + }), + ("no scenario", |d| { + d["outcomes"][1] + .as_object_mut() + .expect("object") + .remove("scenario"); + }), + ("a baseline that failed", |d| { + d["outcomes"][0]["summary"] = "Failure".into() + }), + ("no span", |d| { + d["outcomes"][6]["scenario"]["Mutant"] + .as_object_mut() + .expect("object") + .remove("span"); + }), + ("a missed total that disagrees", |d| d["missed"] = 3.into()), + ("no caught total", |d| { + d.as_object_mut().expect("object").remove("caught"); + }), + ]; + for (what, change) in cases { + let err = parse(&edited(change)).expect_err(what); + assert!(!err.is_empty(), "{what}"); + } + } + + #[test] + fn a_missed_mutant_outside_a_function_has_no_function_name() { + let text = edited(|doc| { + let at = doc["outcomes"] + .as_array() + .expect("list") + .iter() + .position(|o| o["summary"] == "MissedMutant") + .expect("a survivor"); + doc["outcomes"][at]["scenario"]["Mutant"]["function"] = Value::Null; + }); + let found = parse(&text).expect("a null function is allowed"); + assert_eq!(found.missed.first().and_then(|m| m.function.clone()), None); + } + + #[test] + fn a_record_for_one_function_explains_only_its_survivors() { + let found = parse(FIXTURE).expect("parses"); + let ledger = explain(&found, &[record("src/**", Some("double"))], ""); + assert_eq!(ledger.explained.len(), 5); + assert_eq!(ledger.unexplained.len(), 5); + assert!(ledger + .unexplained + .iter() + .all(|m| m.function.as_deref() == Some("inner::triple"))); + assert!(!ledger.passes(0)); + assert!(ledger.passes(5)); + let problem = ledger.problem(0).expect("fails"); + assert!( + problem.contains("5 unexplained") && problem.contains("inner::triple"), + "{problem}" + ); + assert!(!problem.contains(" double "), "{problem}"); + } + + #[test] + fn a_symbol_matches_the_last_path_segment_and_a_symbol_free_record_explains_no_function() { + let found = parse(FIXTURE).expect("parses"); + let nested = explain(&found, &[record("src/lib.rs", Some("triple"))], ""); + assert_eq!(nested.explained.len(), 5); + let both = [ + record("src/lib.rs", Some("double")), + record("src/lib.rs", Some("inner::triple")), + ]; + assert!(explain(&found, &both, "").unexplained.is_empty()); + // No record explains every survivor: one with no symbol explains + // none of those inside a function. + let all = explain(&found, &[record("src/**", None)], ""); + assert_eq!(all.explained.len(), 0); + assert_eq!(all.unexplained.len(), 10); + assert!(!all.passes(0) && all.problem(0).is_some()); + let elsewhere = explain(&found, &[record("other/**", None)], ""); + assert_eq!(elsewhere.unexplained.len(), 10); + } + + /// One record per function of the recorded run, over `path`. + fn both(path: &str) -> [Exemption; 2] { + [ + record(path, Some("double")), + record(path, Some("inner::triple")), + ] + } + + #[test] + fn globs_are_relative_to_the_crate_not_the_workspace() { + let text = edited(|doc| { + for outcome in doc["outcomes"].as_array_mut().expect("list") { + if let Some(file) = outcome["scenario"]["Mutant"]["file"].as_str() { + let moved = format!("crates/sample/{file}"); + outcome["scenario"]["Mutant"]["file"] = moved.into(); + } + } + }); + let found = parse(&text).expect("parses"); + // The record is relative to the crate: it names `src/lib.rs`, and the + // workspace-relative file is `crates/sample/src/lib.rs`. + let inside = both("src/lib.rs"); + assert_eq!( + explain(&found, &inside, "crates/sample").explained.len(), + 10 + ); + assert_eq!( + explain(&found, &inside, "crates/sample/").explained.len(), + 10 + ); + // A record that spells the workspace path matches only when no + // prefix is stripped. + let spelled = both("crates/sample/src/lib.rs"); + assert_eq!(explain(&found, &spelled, "").explained.len(), 10); + assert_eq!( + explain(&found, &spelled, "crates/sample").explained.len(), + 0 + ); + // A record for another crate's file explains nothing. + let other = both("crates/other/src/lib.rs"); + assert_eq!(explain(&found, &other, "crates/sample").explained.len(), 0); + } + + #[test] + fn the_prefix_is_stripped_whole_segments_only() { + assert_eq!(relative("crates/a/src/x.rs", "crates/a"), "src/x.rs"); + assert_eq!( + relative("crates/ab/src/x.rs", "crates/a"), + "crates/ab/src/x.rs" + ); + assert_eq!(relative("src/x.rs", ""), "src/x.rs"); + assert_eq!(relative("src/x.rs", "/"), "src/x.rs"); + } + + #[test] + fn the_judged_numbers_count_every_kind_and_the_max() { + let found = parse(FIXTURE).expect("parses"); + let ledger = explain(&found, &[record("src/lib.rs", Some("double"))], ""); + let judged = ledger.to_json(2); + assert_eq!(judged["caught"], 11); + assert_eq!(judged["missed"], 10); + assert_eq!(judged["explained"], 5); + assert_eq!(judged["unexplained"], 5); + assert_eq!(judged["timeouts"], 1); + assert_eq!(judged["unviable"], 1); + assert_eq!(judged["unexplained_survivors_max"], 2); + } + + #[test] + fn timeouts_and_unviable_mutants_are_not_survivors() { + let only = Outcomes { + caught: 0, + missed: Vec::new(), + timeouts: 4, + unviable: 9, + }; + let ledger = explain(&only, &[], ""); + assert!(ledger.passes(0) && ledger.problem(0).is_none()); + assert_eq!(ledger.timeouts, 4); + assert_eq!(ledger.unviable, 9); + } + + #[test] + fn a_negative_max_is_never_met() { + let ledger = explain(&Outcomes::default(), &[], ""); + assert!(!ledger.passes(-1)); + } + + /// The QA H4 shape, recorded in a workspace: `qa_h4_survivor_miss` has no + /// test, so its two mutants survive, and `file` is relative to the + /// workspace root, not to the crate. + const PLANT: &str = + include_str!("../tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json"); + + #[test] + fn the_h4_plant_has_two_survivors_in_a_crate_under_the_workspace() { + let found = parse(PLANT).expect("parses"); + assert_eq!((found.caught, found.timeouts, found.unviable), (5, 0, 1)); + assert_eq!(found.missed.len(), 2); + assert!(found + .missed + .iter() + .all(|m| m.file == "crates/plant/src/qa_h4_plant.rs" + && m.function.as_deref() == Some("qa_h4_survivor_miss"))); + let by_symbol = [record("src/qa_h4_plant.rs", Some("qa_h4_survivor_miss"))]; + assert_eq!( + explain(&found, &by_symbol, "crates/plant").explained.len(), + 2 + ); + assert_eq!(explain(&found, &by_symbol, "").explained.len(), 2); + let other = [record("src/qa_h4_plant.rs", Some("another"))]; + assert_eq!(explain(&found, &other, "crates/plant").unexplained.len(), 2); + let wrong_file = [record("src/lib.rs", Some("qa_h4_survivor_miss"))]; + assert_eq!( + explain(&found, &wrong_file, "crates/plant") + .unexplained + .len(), + 2 + ); + } + + fn scratch(name: &str) -> std::path::PathBuf { + let dir = + std::env::temp_dir().join(format!("cargo-tops-mutation-{name}-{}", std::process::id())); + fs::remove_dir_all(&dir).ok(); + fs::create_dir_all(&dir).expect("mkdir"); + dir + } + + #[test] + fn an_outcomes_file_from_an_earlier_run_is_removed_and_none_is_nothing() { + let dir = scratch("clear"); + assert!(clear(&dir).is_ok()); + fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); + let file = dir.join("mutants.out/outcomes.json"); + fs::write(&file, FIXTURE).expect("write"); + assert!(clear(&dir).is_ok()); + assert!(!file.exists()); + // A directory where the file should be cannot be removed as a file. + fs::create_dir_all(file.join("inside")).expect("mkdir"); + let err = clear(&dir).expect_err("a directory is not a report"); + assert!(format!("{err:#}").contains("remove"), "{err:#}"); + fs::remove_dir_all(&dir).ok(); + } + + const NOTHING: &str = " INFO No mutants to filter\n"; + + #[test] + fn what_the_run_wrote_is_read_and_a_missing_file_is_an_error() { + let dir = scratch("read"); + let err = read(&dir, 0, "").expect_err("no file"); + assert!(err.contains("mutants.out/outcomes.json"), "{err}"); + assert!(err.contains("exited 0"), "{err}"); + fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); + fs::write(dir.join("mutants.out/outcomes.json"), "not json").expect("write"); + assert!(read(&dir, 2, "") + .expect_err("not json") + .contains("not JSON")); + fs::write(dir.join("mutants.out/outcomes.json"), FIXTURE).expect("write"); + assert_eq!(read(&dir, 2, "").expect("parses").missed.len(), 10); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn no_file_is_zero_mutants_only_for_exit_0_and_a_line_the_run_printed() { + let dir = scratch("nothing"); + for line in super::NO_MUTANTS { + let said = format!(" INFO {line}\n"); + assert_eq!( + read(&dir, 0, &said).expect("zero mutants"), + Outcomes::default() + ); + } + let said = format!("found 3 mutants\n{NOTHING}more\n"); + assert_eq!( + read(&dir, 0, &said).expect("zero mutants"), + Outcomes::default() + ); + // With colour on, only the level is coloured. + let coloured = "\u{1b}[32m INFO\u{1b}[0m No mutants to filter\n"; + assert_eq!( + read(&dir, 0, coloured).expect("zero mutants"), + Outcomes::default() + ); + // Not a whole line of its own: the phrase inside another line is not it. + for said in [ + "WARN path src/No mutants to filter.rs ignored\n", + " INFO No mutants to filter.rs was skipped\n", + " WARN No mutants to filter\n", + "No mutants to filter\n", + "", + "Diff file is empty\n", + ] { + assert!(read(&dir, 0, said).is_err(), "{said:?}"); + } + for code in [2, 3, 1] { + assert!(read(&dir, code, NOTHING).is_err(), "exit {code}"); + } + // A file that is there is read, whatever the run printed. + fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); + fs::write(dir.join("mutants.out/outcomes.json"), FIXTURE).expect("write"); + assert_eq!(read(&dir, 0, NOTHING).expect("parses").caught, 11); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn the_exit_code_and_the_file_must_agree() { + let found = parse(FIXTURE).expect("parses"); + assert!(check_exit(&found, 2).is_ok() && check_exit(&found, 3).is_ok()); + assert!(check_exit(&found, 0).is_ok()); + let none = Outcomes { + missed: Vec::new(), + timeouts: 0, + ..found + }; + let two = check_exit(&none, 2).expect_err("nothing missed"); + assert!( + two.contains("exited 2") && two.contains("no missed"), + "{two}" + ); + let three = check_exit(&none, 3).expect_err("no timeout"); + assert!( + three.contains("exited 3") && three.contains("no timeout"), + "{three}" + ); + // Missed but no timeout is fine for 2; a timeout but none missed for 3. + let only_missed = Outcomes { + timeouts: 0, + ..found + }; + assert!(check_exit(&only_missed, 2).is_ok()); + assert!(check_exit(&only_missed, 3).is_err()); + let only_timeout = Outcomes { + missed: Vec::new(), + ..found + }; + assert!(check_exit(&only_timeout, 3).is_ok()); + assert!(check_exit(&only_timeout, 2).is_err()); + } + + #[test] + fn the_total_must_be_there_and_agree() { + let wrong = edited(|doc| doc["total_mutants"] = Value::from(999)); + let err = parse(&wrong).expect_err("999 is not 23"); + assert!(err.contains("says 999 total_mutants but lists 23"), "{err}"); + let gone = edited(|doc| { + doc.as_object_mut().expect("object").remove("total_mutants"); + }); + assert!(parse(&gone) + .expect_err("no total") + .contains("total_mutants")); + } + + const NULL_FUNCTION: &str = + include_str!("../tests/fixtures/mutants-27.1.0/null-function/outcomes.json"); + + #[test] + fn a_survivor_in_no_function_takes_a_symbol_free_record_and_nothing_explains_them_all() { + let found = parse(NULL_FUNCTION).expect("parses"); + assert_eq!(found.missed.len(), 12); + assert_eq!( + found.missed.iter().filter(|m| m.function.is_none()).count(), + 7 + ); + let count = |records: &[Exemption]| explain(&found, records, "").explained.len(); + // A symbol explains the function it names: 5 in `get`, none outside. + assert_eq!(count(&[record("src/lib.rs", Some("get"))]), 5); + assert_eq!(count(&[record("src/lib.rs", Some("no_such_fn"))]), 0); + // No symbol: the 7 in no function, in the files the glob covers. + for path in ["src/lib.rs", "src/**", "**", "*", "**/*.rs"] { + assert_eq!(count(&[record(path, None)]), 7, "{path}"); + } + assert_eq!(count(&[record("other/**", None)]), 0); + // Both kinds of record: every survivor, each counted once. + let both = [ + record("src/lib.rs", Some("get")), + record("src/lib.rs", None), + ]; + assert_eq!(count(&both), 12); + // A symbol with a whole-tree path still names only its function. + assert_eq!(count(&[record("**", Some("get"))]), 5); + // In the recorded run every survivor is in a function: a symbol-free + // record explains none, whatever its glob, so the max decides. + let in_functions = parse(FIXTURE).expect("parses"); + for path in ["**", "/**", "*", "*.rs", "src/**", ".", "src/lib.rs"] { + let ledger = explain(&in_functions, &[record(path, None)], ""); + assert_eq!(ledger.explained.len(), 0, "{path}"); + assert!(!ledger.passes(0), "{path}"); + } + } + + #[test] + fn a_survivor_is_named_once_by_its_own_name() { + let found = parse(FIXTURE).expect("parses"); + let ledger = explain(&found, &[], ""); + let problem = ledger.problem(0).expect("fails"); + let first = &found.missed[0]; + assert_eq!(problem.matches(&first.name).count(), 1, "{problem}"); + assert!(!problem.contains(&format!("{}:{} {}", first.file, first.line, first.name))); + } +} diff --git a/crates/cargo-tops/src/steps.rs b/crates/cargo-tops/src/steps.rs index c27c90e..1dbe27d 100644 --- a/crates/cargo-tops/src/steps.rs +++ b/crates/cargo-tops/src/steps.rs @@ -449,12 +449,25 @@ fn origin_head(dir: &Path) -> Result { Some(name) => Err(format!( "origin/HEAD: points at {name}, which is not a commit" )), - // Not a symbolic ref. A direct ref that names a commit is a base. - None if names_a_commit(dir, "origin/HEAD") => Ok("origin/HEAD".to_string()), - None => Err("origin/HEAD: not set".to_string()), + // Not a symbolic ref: a direct one, or none. + None => direct_head(dir), } } +/// `origin/HEAD` as a direct ref: a base when it names a commit. +fn direct_head(dir: &Path) -> Result { + if names_a_commit(dir, "origin/HEAD") { + return Ok("origin/HEAD".to_string()); + } + let exists = git_line(dir, &["rev-parse", "--verify", "--quiet", "origin/HEAD"]).is_some(); + Err(if exists { + "origin/HEAD: not a commit" + } else { + "origin/HEAD: not set" + } + .to_string()) +} + /// Why the mutation layer stopped before `cargo mutants`, and its outcome. /// This is the one place the words live: stdout, the report's `detail` and /// its `problems` all print what this returns. `diff` is the outcome of the @@ -527,13 +540,30 @@ fn no_base_outside_ci(tried: &[String]) -> (Outcome, String) { ( Outcome::Vacuous, format!( - "no mutation base ({}); the mutation diff is empty; \ + "{}; the mutation diff is empty; \ cargo mutants did not run and no law counts this layer", - tried.join("; ") + no_base(tried) ), ) } +/// What was looked for and not found, with no outcome after it. +fn no_base(tried: &[String]) -> String { + format!("no mutation base ({})", tried.join("; ")) +} + +/// What is wrong with the base, worded with no outcome: a row that never +/// reached the diff (a missing tool, a dry run) says only this. `None` for +/// a base that is a commit. +pub(crate) fn base_finding(base: &Base) -> Option { + match base { + Base::Found { .. } => None, + Base::Missing { tried, ci: false } => Some(no_base(tried)), + Base::Missing { tried, ci: true } => Some(no_base_in_ci(tried).1), + Base::Unresolved { .. } => stop_note(base, Outcome::Failed).map(|(_, note)| note), + } +} + /// The first line git prints for `args`, when it succeeds with one. fn git_line(dir: &Path, args: &[&str]) -> Option { let out = Command::new("git") @@ -585,6 +615,10 @@ pub(crate) struct MutationPlan { pub diff_path: PathBuf, pub mutants: Step, pub base: Base, + /// Where `cargo mutants` writes `mutants.out`: the report directory, so + /// the gate reads the files this run wrote and no setting of the + /// adopter's moves them. + pub output: PathBuf, } impl MutationPlan { @@ -598,6 +632,7 @@ impl MutationPlan { }; let script = mutation_diff_script(report_dir, &diff_path, from); let diff = Step::command(["sh", "-c", script.as_str()]); + let output_arg = report_dir.display().to_string(); let mutants = Step::tool([ "cargo", "mutants", @@ -605,12 +640,15 @@ impl MutationPlan { diff_arg.as_str(), "-j", "2", + "--output", + output_arg.as_str(), ]); Self { diff, diff_path, mutants, base, + output: report_dir.to_path_buf(), } } @@ -985,8 +1023,8 @@ mod expiry { #[cfg(test)] mod base { use super::{ - ci_active, git_line, mutation_diff_script, shell_quote, stop_note, Base, BaseSource, - MutationPlan, Outcome, + base_finding, ci_active, git_line, mutation_diff_script, shell_quote, stop_note, Base, + BaseSource, MutationPlan, Outcome, }; use std::path::Path; @@ -1163,4 +1201,38 @@ mod base { assert_eq!(stop_note(&base, Outcome::Passed), None); } } + + #[test] + fn a_base_finding_names_the_fault_and_no_outcome() { + assert_eq!(base_finding(&found()), None); + let tried = vec![ + "--base: not given".to_string(), + "origin/HEAD: not set".to_string(), + ]; + let outside = Base::Missing { + tried: tried.clone(), + ci: false, + }; + assert_eq!( + base_finding(&outside).as_deref(), + Some("no mutation base (--base: not given; origin/HEAD: not set)") + ); + let inside = Base::Missing { tried, ci: true }; + let finding = base_finding(&inside).expect("a finding"); + assert!(finding.starts_with("no mutation base ("), "{finding}"); + assert!(finding.contains("fetch-depth: 0"), "{finding}"); + let unresolved = Base::Unresolved { + source: BaseSource::Flag, + name: "nope".to_string(), + }; + let finding = base_finding(&unresolved).expect("a finding"); + assert!( + finding.contains("`nope`") && finding.contains("not a commit"), + "{finding}" + ); + for base in [&outside, &inside, &unresolved] { + let finding = base_finding(base).expect("a finding"); + assert!(!finding.contains("is empty") && !finding.contains("did not run")); + } + } } diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md new file mode 100644 index 0000000..5d50a68 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md @@ -0,0 +1,45 @@ +# mutants-27.1.0 + +`outcomes.json` is the file `cargo mutants --timeout 4 -j 1` 27.1.0 wrote for the +crate below, with the toolchain path in each `argv` shortened to `cargo`. +Nothing else was edited. It holds 11 caught, 10 missed, 1 timeout and 1 +unviable mutant, and the baseline run. + +```rust +pub struct Opaque(u8); + +pub fn add(a: i32, b: i32) -> i32 { a + b } +pub fn double(x: i32) -> i32 { x * 2 } +pub fn opaque() -> Opaque { Opaque(1) } +pub fn spin(n: u32) -> u32 { let mut i = 0; while i < n { i += 1; } i } + +pub mod inner { + pub fn triple(x: i32) -> i32 { x * 3 } +} +``` + +The tests call `add` (three asserts), `spin(3)` and `opaque()`. `double` and +`inner::triple` have no test, so every mutant in them is missed. `opaque` has no +`Default`, so its mutant is unviable. `+=` to `*=` in `spin` never ends, so it +times out. + +The file is real output, not a hand-built shape. To re-record it, run the same +command over the same crate with cargo-mutants 27.1.0 and shorten the paths. + +## null-function/ + +`outcomes.json` from `cargo mutants -j 1 --output ` over a crate whose +`src/lib.rs` is a `const SIZE: usize = 2 + 3`, a `static LIMIT: u32 = 10 * 4`, +a `type Buf = [u8; 4 + 4]`, a struct field `[u8; 1 + 1]` and one untested +`fn get() -> i32 { 2 + 2 }`, with the toolchain path shortened to `cargo`. It +holds 12 missed mutants and 1 unviable one. The 7 missed in the `const`, the +`static`, the alias and the field have `"function": null`; the 5 in `get` name +it. The file sits in `/mutants.out/outcomes.json`. + +## A diff with nothing to mutate + +Recorded from real 27.1.0 runs, `cargo mutants --in-diff --output +`: for a diff that changes only `README.md`, stderr is ` INFO Diff +changes no Rust source files`; for one that adds a doc comment, ` INFO No +mutants to filter`. Both exit 0, print nothing to stdout, and leave `` +empty: there is no `mutants.out` and no `outcomes.json`. diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json new file mode 100644 index 0000000..720f3b7 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json @@ -0,0 +1,750 @@ +{ + "outcomes": [ + { + "scenario": "Baseline", + "summary": "Success", + "log_path": "log/baseline.log", + "diff_path": null, + "phase_results": [ + { + "phase": "Build", + "duration": 0.346787875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.405252542, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:1:27: replace + with -", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 1, + "column": 27 + }, + "end": { + "line": 1, + "column": 28 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "Unviable", + "log_path": "log/src__lib.rs_line_1_col_27.log", + "diff_path": "diff/src__lib.rs_line_1_col_27.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.060888917, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:1:27: replace + with *", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 1, + "column": 27 + }, + "end": { + "line": 1, + "column": 28 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_1_col_27_001.log", + "diff_path": "diff/src__lib.rs_line_1_col_27_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.111410833, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.392369083, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:3:28: replace * with +", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 3, + "column": 28 + }, + "end": { + "line": 3, + "column": 29 + } + }, + "replacement": "+", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_3_col_28.log", + "diff_path": "diff/src__lib.rs_line_3_col_28.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113310791, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.460165542, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:3:28: replace * with /", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 3, + "column": 28 + }, + "end": { + "line": 3, + "column": 29 + } + }, + "replacement": "/", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_3_col_28_001.log", + "diff_path": "diff/src__lib.rs_line_3_col_28_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.118249292, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.298558834, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:5:23: replace + with -", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 5, + "column": 23 + }, + "end": { + "line": 5, + "column": 24 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_5_col_23.log", + "diff_path": "diff/src__lib.rs_line_5_col_23.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121582667, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.404460041, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:5:23: replace + with *", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 5, + "column": 23 + }, + "end": { + "line": 5, + "column": 24 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_5_col_23_001.log", + "diff_path": "diff/src__lib.rs_line_5_col_23_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113811834, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.363051042, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:19: replace + with -", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 8, + "column": 19 + }, + "end": { + "line": 8, + "column": 20 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_19.log", + "diff_path": "diff/src__lib.rs_line_8_col_19.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.12164225, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.408842875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:19: replace + with *", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 8, + "column": 19 + }, + "end": { + "line": 8, + "column": 20 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_19_001.log", + "diff_path": "diff/src__lib.rs_line_8_col_19_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120965875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.068054125, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace get -> i32 with 0", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_5.log", + "diff_path": "diff/src__lib.rs_line_12_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.118840291, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.08889725, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace get -> i32 with 1", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_12_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121670917, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.732568292, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace get -> i32 with -1", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_5_002.log", + "diff_path": "diff/src__lib.rs_line_12_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120978583, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.233461208, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:7: replace + with - in get", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 7 + }, + "end": { + "line": 12, + "column": 8 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_7.log", + "diff_path": "diff/src__lib.rs_line_12_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.115573167, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.274348375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:7: replace + with * in get", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 7 + }, + "end": { + "line": 12, + "column": 8 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_7_001.log", + "diff_path": "diff/src__lib.rs_line_12_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.111984334, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.232997167, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + } + ], + "total_mutants": 13, + "missed": 12, + "caught": 0, + "timeout": 0, + "unviable": 1, + "success": 0, + "start_time": "2026-11-20T06:57:08.422784Z", + "end_time": "2026-11-20T06:57:23.625455Z", + "cargo_mutants_version": "27.1.0" +} \ No newline at end of file diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json new file mode 100644 index 0000000..48a212d --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json @@ -0,0 +1,1506 @@ +{ + "outcomes": [ + { + "scenario": "Baseline", + "summary": "Success", + "log_path": "log/baseline.log", + "diff_path": null, + "phase_results": [ + { + "phase": "Build", + "duration": 0.819982375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.513537209, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:5: replace add -> i32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 5 + }, + "end": { + "line": 4, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_5.log", + "diff_path": "diff/src__lib.rs_line_4_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120934916, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181101834, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:5: replace add -> i32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 5 + }, + "end": { + "line": 4, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_4_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.107022792, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181147208, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:5: replace add -> i32 with -1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 5 + }, + "end": { + "line": 4, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_5_002.log", + "diff_path": "diff/src__lib.rs_line_4_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.106414417, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.23059925, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:7: replace + with - in add", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 7 + }, + "end": { + "line": 4, + "column": 8 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_7.log", + "diff_path": "diff/src__lib.rs_line_4_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113024167, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181143791, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:7: replace + with * in add", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 7 + }, + "end": { + "line": 4, + "column": 8 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_7_001.log", + "diff_path": "diff/src__lib.rs_line_4_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121543958, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.18111875, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:5: replace double -> i32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 5 + }, + "end": { + "line": 8, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_5.log", + "diff_path": "diff/src__lib.rs_line_8_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.118955125, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.232848, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:5: replace double -> i32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 5 + }, + "end": { + "line": 8, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_8_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.119879292, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.241087875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:5: replace double -> i32 with -1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 5 + }, + "end": { + "line": 8, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_5_002.log", + "diff_path": "diff/src__lib.rs_line_8_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.1075925, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.230802417, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:7: replace * with + in double", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 7 + }, + "end": { + "line": 8, + "column": 8 + } + }, + "replacement": "+", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_7.log", + "diff_path": "diff/src__lib.rs_line_8_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.170658209, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.229900042, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:7: replace * with / in double", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 7 + }, + "end": { + "line": 8, + "column": 8 + } + }, + "replacement": "/", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_7_001.log", + "diff_path": "diff/src__lib.rs_line_8_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.167602084, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.219947583, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace opaque -> Opaque with Default::default()", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "opaque", + "return_type": "-> Opaque", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 14 + } + }, + "replacement": "Default::default()", + "genre": "FnValue" + } + }, + "summary": "Unviable", + "log_path": "log/src__lib.rs_line_12_col_5.log", + "diff_path": "diff/src__lib.rs_line_12_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.106317833, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:16:5: replace spin -> u32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 16, + "column": 5 + }, + "end": { + "line": 20, + "column": 6 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_16_col_5.log", + "diff_path": "diff/src__lib.rs_line_16_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113336542, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.239245958, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:16:5: replace spin -> u32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 16, + "column": 5 + }, + "end": { + "line": 20, + "column": 6 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_16_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_16_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.110367875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181226375, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:17:13: replace < with == in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 17, + "column": 13 + }, + "end": { + "line": 17, + "column": 14 + } + }, + "replacement": "==", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_17_col_13.log", + "diff_path": "diff/src__lib.rs_line_17_col_13.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.115069583, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181267458, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:17:13: replace < with > in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 17, + "column": 13 + }, + "end": { + "line": 17, + "column": 14 + } + }, + "replacement": ">", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_17_col_13_001.log", + "diff_path": "diff/src__lib.rs_line_17_col_13_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.11942975, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.172611042, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:17:13: replace < with <= in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 17, + "column": 13 + }, + "end": { + "line": 17, + "column": 14 + } + }, + "replacement": "<=", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_17_col_13_002.log", + "diff_path": "diff/src__lib.rs_line_17_col_13_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.107558, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 2.168281208, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:18:11: replace += with -= in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 18, + "column": 11 + }, + "end": { + "line": 18, + "column": 13 + } + }, + "replacement": "-=", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_18_col_11.log", + "diff_path": "diff/src__lib.rs_line_18_col_11.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.111740458, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.493100667, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:18:11: replace += with *= in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 18, + "column": 11 + }, + "end": { + "line": 18, + "column": 13 + } + }, + "replacement": "*=", + "genre": "BinaryOperator" + } + }, + "summary": "Timeout", + "log_path": "log/src__lib.rs_line_18_col_11_001.log", + "diff_path": "diff/src__lib.rs_line_18_col_11_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121307375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 4.031318541, + "process_status": "Timeout", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:9: replace inner::triple -> i32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 9 + }, + "end": { + "line": 25, + "column": 14 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_9.log", + "diff_path": "diff/src__lib.rs_line_25_col_9.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.109103375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.415611625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:9: replace inner::triple -> i32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 9 + }, + "end": { + "line": 25, + "column": 14 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_9_001.log", + "diff_path": "diff/src__lib.rs_line_25_col_9_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.12097825, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.612745625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:9: replace inner::triple -> i32 with -1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 9 + }, + "end": { + "line": 25, + "column": 14 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_9_002.log", + "diff_path": "diff/src__lib.rs_line_25_col_9_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.106036375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 2.853253708, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:11: replace * with + in inner::triple", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 11 + }, + "end": { + "line": 25, + "column": 12 + } + }, + "replacement": "+", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_11.log", + "diff_path": "diff/src__lib.rs_line_25_col_11.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.112429208, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.871245709, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:11: replace * with / in inner::triple", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 11 + }, + "end": { + "line": 25, + "column": 12 + } + }, + "replacement": "/", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_11_001.log", + "diff_path": "diff/src__lib.rs_line_25_col_11_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121888083, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.7411889170000001, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + } + ], + "total_mutants": 23, + "missed": 10, + "caught": 11, + "timeout": 1, + "unviable": 1, + "success": 0, + "start_time": "2026-11-20T03:59:09.188302Z", + "end_time": "2026-11-20T03:59:33.389279Z", + "cargo_mutants_version": "27.1.0" +} \ No newline at end of file diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json new file mode 100644 index 0000000..02bc6df --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json @@ -0,0 +1,549 @@ +{ + "outcomes": [ + { + "scenario": "Baseline", + "summary": "Success", + "log_path": "log/baseline.log", + "diff_path": null, + "phase_results": [ + { + "phase": "Build", + "duration": 0.467251625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.450718083, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:5: replace add -> i32 with 0", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 5 + }, + "end": { + "line": 6, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_5.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121359417, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.238109916, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:5: replace add -> i32 with 1", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 5 + }, + "end": { + "line": 6, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_5_001.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.179454666, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.24125775, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:5: replace add -> i32 with -1", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 5 + }, + "end": { + "line": 6, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_5_002.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.171344166, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.231705667, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:7: replace + with - in add", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 7 + }, + "end": { + "line": 6, + "column": 8 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_7.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.173006334, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.240732125, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:7: replace + with * in add", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 7 + }, + "end": { + "line": 6, + "column": 8 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_7_001.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.175716625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.241108292, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:10:5: replace opaque -> Opaque with Default::default()", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "opaque", + "return_type": "-> Opaque", + "span": { + "start": { + "line": 9, + "column": 1 + }, + "end": { + "line": 11, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 10, + "column": 5 + }, + "end": { + "line": 10, + "column": 14 + } + }, + "replacement": "Default::default()", + "genre": "FnValue" + } + }, + "summary": "Unviable", + "log_path": "log/crates__plant__src__lib.rs_line_10_col_5.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_10_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.115931709, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/qa_h4_plant.rs:2:5: replace qa_h4_survivor_miss -> u32 with 0", + "package": "plant", + "file": "crates/plant/src/qa_h4_plant.rs", + "function": { + "function_name": "qa_h4_survivor_miss", + "return_type": "-> u32", + "span": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 3, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 28 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/crates__plant__src__qa_h4_plant.rs_line_2_col_5.log", + "diff_path": "diff/crates__plant__src__qa_h4_plant.rs_line_2_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120998125, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.288781958, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/qa_h4_plant.rs:2:5: replace qa_h4_survivor_miss -> u32 with 1", + "package": "plant", + "file": "crates/plant/src/qa_h4_plant.rs", + "function": { + "function_name": "qa_h4_survivor_miss", + "return_type": "-> u32", + "span": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 3, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 28 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/crates__plant__src__qa_h4_plant.rs_line_2_col_5_001.log", + "diff_path": "diff/crates__plant__src__qa_h4_plant.rs_line_2_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.122019708, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.241327625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + } + ], + "total_mutants": 8, + "missed": 2, + "caught": 5, + "timeout": 0, + "unviable": 1, + "success": 0, + "start_time": "2026-11-20T04:08:58.142114Z", + "end_time": "2026-11-20T04:09:02.002208Z", + "cargo_mutants_version": "27.1.0" +} \ No newline at end of file diff --git a/crates/cargo-tops/tests/gate_base.rs b/crates/cargo-tops/tests/gate_base.rs index 3707ded..6f7f563 100644 --- a/crates/cargo-tops/tests/gate_base.rs +++ b/crates/cargo-tops/tests/gate_base.rs @@ -25,11 +25,17 @@ mod profiles { const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); /// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// `cargo mutants` does what 27.1.0 does for a diff with nothing to mutate; /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +# 27.1.0 finds nothing to mutate in a diff of comments, tests, docs or CI: +# it says so on stderr, exits 0 and writes no `mutants.out`. +if [ "$1" = "mutants" ]; then + echo ' INFO No mutants to filter' >&2 +fi exit 0 "#; @@ -740,3 +746,114 @@ fn a_base_with_no_common_history_says_why_and_names_fetch_depth() { "{problems:?}" ); } + +/// The number that follows `after` in `text`. +fn number_after(text: &str, after: &str) -> usize { + let tail = text.split(after).nth(1).unwrap_or_default(); + let digits: String = tail.chars().take_while(char::is_ascii_digit).collect(); + digits.parse().unwrap_or(usize::MAX) +} + +#[test] +fn an_origin_head_that_names_a_tree_reads_not_a_commit() { + let fixture = fixture("tree-head", "main"); + let tree = git(&fixture, &fixture.repo, &["rev-parse", "HEAD^{tree}"]); + git( + &fixture, + &fixture.repo, + &[ + "update-ref", + "--no-deref", + "refs/remotes/origin/HEAD", + &tree, + ], + ); + let (_, text) = gate(&fixture, &[], None); + assert!(text.contains("origin/HEAD: not a commit"), "{text}"); + assert!(!text.contains("origin/HEAD: not set"), "{text}"); +} + +#[test] +fn a_mutation_row_that_stopped_on_a_missing_tool_does_not_say_the_diff_is_empty() { + let fixture = fixture("tool-missing-words", "main"); + fs::remove_file(fixture.bin.join("cargo-mutants")).expect("remove the stub"); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let doc = report(&fixture); + assert_eq!(at(mutation(&doc), "/verdict"), "tool-missing", "{doc}"); + let detail = at(mutation(&doc), "/detail").as_str().unwrap_or_default(); + assert!(detail.contains("no mutation base"), "{detail}"); + assert!( + !detail.contains("is empty") && !detail.contains("did not run"), + "{detail}" + ); +} + +#[test] +fn a_dry_run_row_words_the_base_without_an_outcome() { + let fixture = fixture("dry-words", "main"); + let (_, _) = gate(&fixture, &["--dry-run"], None); + let doc = report(&fixture); + let detail = at(mutation(&doc), "/detail").as_str().unwrap_or_default(); + assert!(detail.contains("no mutation base"), "{detail}"); + assert!( + !detail.contains("is empty") && !detail.contains("did not run"), + "{detail}" + ); +} + +#[test] +fn the_summary_counts_commands_that_ran_as_the_count_line_does() { + // A missing tool ran no command: it is in neither number. + let fixture = fixture("count-words", "main"); + fs::remove_file(fixture.bin.join("cargo-deny")).expect("remove the stub"); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let said = number_after(&text, "count: "); + let closing = number_after(&text, "problem(s), "); + assert!(said != usize::MAX && closing != usize::MAX, "{text}"); + assert_eq!(said, closing, "{text}"); +} + +#[test] +fn a_report_that_cannot_be_written_ends_the_output_with_fail() { + let fixture = fixture("unwritable", "main"); + fs::create_dir_all(fixture.repo.join("target/rust-tops/report.json")).expect("mkdir"); + // At HEAD the diff is empty: without the write error this is a pass. + let (out, text) = gate(&fixture, &["--base", "HEAD"], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let stdout = String::from_utf8_lossy(&out.stdout); + let last = stdout.lines().rev().find(|line| !line.trim().is_empty()); + assert!( + last.is_some_and(|line| line.starts_with("fail:") && line.contains("report.json")), + "{stdout}" + ); + assert!( + !stdout.lines().any(|line| line.starts_with("ok:")), + "{stdout}" + ); +} + +#[test] +fn the_help_and_the_schema_say_what_exit_2_covers() { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["gate", "--help"]) + .output() + .expect("spawn help"); + let help = String::from_utf8_lossy(&out.stdout).replace("\n ", " "); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let described = at(&schema, "/properties/exit_code/description") + .as_str() + .unwrap_or_default(); + for (name, text) in [("help", help.as_str()), ("schema", described)] { + for phrase in [ + "rust-tops.yaml", + "rust-tops.lock", + "no enabled layer at or below --max-tier", + "usage error", + "unreadable when a layer in range needs a pin", + ] { + assert!(text.contains(phrase), "{name} lacks `{phrase}`: {text}"); + } + } +} diff --git a/crates/cargo-tops/tests/gate_exemptions.rs b/crates/cargo-tops/tests/gate_exemptions.rs new file mode 100644 index 0000000..aba772e --- /dev/null +++ b/crates/cargo-tops/tests/gate_exemptions.rs @@ -0,0 +1,373 @@ +//! Every exemption record is checked, whatever its layer, whether that layer +//! is on or off, and whatever `--max-tier` is: one that has expired, or whose +//! date is not a real day, is a gate problem and the gate exits 1. A date +//! passing does not make the config invalid, so it is never exit 2. `check` +//! prints the same lines as warnings and still exits 0. +//! +//! Cargo and its plugins are stubs. Every report is validated against the +//! schema. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::{json, Value}; + +/// The validator `cargo tops check` uses, and the schema it reads. +#[path = "../src/schema.rs"] +mod schema; + +mod profiles { + pub(crate) const SCHEMA_JSON: &str = include_str!("../templates/rust-tops.schema.json"); +} + +const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); + +/// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// passes everything else. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + home: PathBuf, +} + +fn at<'v>(value: &'v Value, pointer: &str) -> &'v Value { + static NULL: Value = Value::Null; + value.pointer(pointer).unwrap_or(&NULL) +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +fn set(doc: &mut Value, key: &str, value: Value) { + doc.as_object_mut() + .expect("an object") + .insert(key.to_string(), value); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A library-core instance named `fixture`, in a directory of its own with +/// a `Cargo.toml`, one source file and stub tools, not yet a git repository. +fn scaffold(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-base-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + home: root.join("home"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir bin"); + fs::create_dir_all(&fixture.home).expect("mkdir home"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); + fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); + stub(&fixture.bin, "cargo", CARGO_STUB); + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); + } + fixture +} + +/// The scaffold with `static-lint`, `security-advisory`, `property` and +/// `mutation` on, in a git repository whose one branch is `branch` and whose +/// one commit is everything `init` wrote. There is no remote. +/// A commit that adds `file` under `src/`. +/// What a clone leaves: a remote `origin` holding `branch`, with +/// `refs/remotes/origin/HEAD` pointing at it. +/// `refs/remotes/origin/` at `commit`. +fn gate(fixture: &Fixture, args: &[&str], ci: Option<&str>) -> (Output, String) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.arg("gate") + .args(args) + .arg("--path") + .arg(&fixture.repo) + .env( + "PATH", + std::env::join_paths([ + fixture.bin.as_path(), + Path::new("/usr/bin"), + Path::new("/bin"), + ]) + .expect("PATH"), + ) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("CARGO_STUB_LOG", fixture.root.join("cargo.log")) + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI"); + if let Some(value) = ci { + cmd.env("CI", value); + } + let out = cmd.output().expect("spawn cargo-tops gate"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +fn report(fixture: &Fixture) -> Value { + let raw = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("report.json was written"); + let doc: Value = serde_json::from_str(&raw).expect("report.json is JSON"); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let errors = schema::validate(&schema, &doc); + assert!( + errors.is_empty(), + "report.json breaks the schema: {errors:?}\n{raw}" + ); + doc +} + +/// The scaffold with `static-lint`, `security-advisory` and `property` on. +fn fixture(name: &str) -> Fixture { + let fixture = scaffold(name); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!(id, "static-lint" | "security-advisory" | "property"); + set(layer, "enabled", Value::Bool(on)); + } + }); + fixture +} + +/// One record for `layer` over `src/**`, expiring on `expires`. +fn exempt(fixture: &Fixture, layer: &str, expires: &str) { + edit(fixture, |doc| { + let record = json!({ + "path": "src/**", + "layer": layer, + "owner": "fixture", + "reason": "planted for the expiry rule", + "expires": expires, + }); + set(doc, "exemptions", json!([record])); + }); +} + +fn check(fixture: &Fixture) -> (Output, String) { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["check", "--path"]) + .arg(&fixture.repo) + .output() + .expect("spawn check"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +/// The report's problems that mention an exemption. +fn exemption_problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|problem| problem.contains("exemption")) + .map(str::to_string) + .collect() +} + +#[test] +fn an_expired_record_for_a_layer_that_is_off_fails_the_gate() { + // `miri` is off in this instance and above `--max-tier 1` besides. + let fixture = fixture("expired-miri"); + exempt(&fixture, "miri", "2020-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + let report = report(&fixture); + assert_eq!(at(&report, "/exit_code"), 1, "{report}"); + assert_eq!(at(&report, "/kind"), "fail", "{report}"); + let problems = exemption_problems(&report); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!( + problems.iter().all(|p| p.starts_with("miri:") + && p.contains("src/**") + && p.contains("expired on 2020-01-01")), + "{problems:?}" + ); + assert!(text.contains("expired on 2020-01-01"), "{text}"); +} + +/// Every layer, with `tier` as the range, has an expired record fail the gate. +/// `name` keeps the scratch directories of tests that run together apart. +fn expired_record_fails_each_of(name: &str, layers: &[&str], tier: &str) { + for layer in layers { + let fixture = fixture(&format!("{name}-{layer}")); + exempt(&fixture, layer, "2020-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", tier], None); + assert_eq!(out.status.code(), Some(1), "{layer} at {tier}: {text}"); + let problems = exemption_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{layer} at {tier}: {problems:?}"); + } +} + +// On and in range, on and above the range, and off: every layer at tiers 0 +// and 1, in four tests so each stays well inside the CI time limit. +#[test] +fn an_expired_record_fails_for_the_first_layers_at_tier_0() { + expired_record_fails_each_of( + "a0", + &["static-lint", "security-advisory", "miri", "fuzz"], + "0", + ); +} + +#[test] +fn an_expired_record_fails_for_the_last_layers_at_tier_0() { + expired_record_fails_each_of("b0", &["mutation", "coverage", "crap", "acceptance"], "0"); +} + +#[test] +fn an_expired_record_fails_for_the_first_layers_at_tier_1() { + expired_record_fails_each_of( + "a1", + &["static-lint", "security-advisory", "miri", "fuzz"], + "1", + ); +} + +#[test] +fn an_expired_record_fails_for_the_last_layers_at_tier_1() { + expired_record_fails_each_of("b1", &["mutation", "coverage", "crap", "acceptance"], "1"); +} + +#[test] +fn a_record_whose_date_is_not_a_day_fails() { + for date in ["2027-02-30", "not-a-date", "2024-13-01", "2099-1-1"] { + let fixture = fixture("not-a-day"); + exempt(&fixture, "miri", date); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(1), "{date}: {text}"); + let problems = exemption_problems(&report(&fixture)); + assert!( + problems + .iter() + .all(|p| p.contains(date) && p.contains("not a day")), + "{date}: {problems:?}" + ); + assert_eq!(problems.len(), 1, "{date}: {problems:?}"); + } +} + +#[test] +fn a_live_record_passes() { + let fixture = fixture("live"); + for layer in ["miri", "mutation", "static-lint"] { + exempt(&fixture, layer, "2099-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(0), "{layer}: {text}"); + assert!(exemption_problems(&report(&fixture)).is_empty(), "{layer}"); + } +} + +#[test] +fn an_expired_record_is_not_a_config_error() { + // Exit 1 with a report of kind `fail`, never 2. + let fixture = fixture("not-config"); + exempt(&fixture, "miri", "2020-01-01"); + let (out, _) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(1)); + assert_eq!(at(&report(&fixture), "/kind"), "fail"); +} + +#[test] +fn every_expired_record_is_named_not_only_the_first() { + let fixture = fixture("two-expired"); + edit(&fixture, |doc| { + let record = |layer: &str, expires: &str| { + json!({ + "path": "src/**", "layer": layer, "owner": "fixture", + "reason": "planted for this test", "expires": expires, + }) + }; + let records = json!([ + record("miri", "2020-01-01"), + record("fuzz", "2021-06-30"), + record("crap", "2099-01-01"), + ]); + set(doc, "exemptions", records); + }); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + let problems = exemption_problems(&report(&fixture)); + assert_eq!(problems.len(), 2, "{problems:?}"); + assert!( + problems.iter().any(|p| p.starts_with("miri:")), + "{problems:?}" + ); + assert!( + problems.iter().any(|p| p.starts_with("fuzz:")), + "{problems:?}" + ); +} + +#[test] +fn check_warns_about_an_expired_record_and_still_exits_0() { + let fixture = fixture("check-warns"); + exempt(&fixture, "miri", "2020-01-01"); + let (out, text) = check(&fixture); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(text.contains("expired on 2020-01-01"), "{text}"); + assert!(text.to_lowercase().contains("warn"), "{text}"); + // The same file, with a live record, has no warning. + exempt(&fixture, "miri", "2099-01-01"); + let (out, text) = check(&fixture); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(!text.contains("expired"), "{text}"); +} + +#[test] +fn a_not_a_day_date_is_a_check_warning_too() { + let fixture = fixture("check-not-a-day"); + exempt(&fixture, "miri", "2027-02-30"); + let (out, text) = check(&fixture); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(text.contains("2027-02-30"), "{text}"); +} diff --git a/crates/cargo-tops/tests/gate_real_mutants.rs b/crates/cargo-tops/tests/gate_real_mutants.rs new file mode 100644 index 0000000..124c207 --- /dev/null +++ b/crates/cargo-tops/tests/gate_real_mutants.rs @@ -0,0 +1,307 @@ +//! A diff with nothing to mutate, through the real cargo-mutants. Real 27.1.0 +//! exits 0 for a diff of a README or of a comment, writes no `mutants.out` +//! and says so on stderr; the layer passes as zero mutants. Only `cargo-deny` +//! is a stub. The test needs `cargo-mutants` 27.1.0 on `PATH`: without it +//! the test stops, unless `RUST_TOPS_REQUIRE_MUTANTS` is set, which fails it. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::Value; + +/// The validator `cargo tops check` uses, and the schema it reads. +#[path = "../src/schema.rs"] +mod schema; + +mod profiles { + pub(crate) const SCHEMA_JSON: &str = include_str!("../templates/rust-tops.schema.json"); +} + +const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); + +/// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// passes everything else. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +if [ "$1" = "mutants" ]; then + out= + prev= + for arg in "$@"; do + if [ "$prev" = "--output" ]; then out=$arg; fi + prev=$arg + done + if [ -z "$out" ]; then echo "no --output given" >&2; exit 64; fi + if [ -n "${MUTANTS_SAYS:-}" ]; then echo "$MUTANTS_SAYS" >&2; fi + if [ -n "${MUTANTS_OUTCOMES:-}" ]; then + mkdir -p "$out/mutants.out" && cp "$MUTANTS_OUTCOMES" "$out/mutants.out/outcomes.json" + fi + exit "${MUTANTS_EXIT:-0}" +fi +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + home: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn at<'v>(value: &'v Value, pointer: &str) -> &'v Value { + static NULL: Value = Value::Null; + value.pointer(pointer).unwrap_or(&NULL) +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +fn git(fixture: &Fixture, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(dir) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_AUTHOR_NAME", "fixture") + .env("GIT_AUTHOR_EMAIL", "fixture@example.invalid") + .env("GIT_COMMITTER_NAME", "fixture") + .env("GIT_COMMITTER_EMAIL", "fixture@example.invalid") + .output() + .expect("spawn git"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn set(doc: &mut Value, key: &str, value: Value) { + doc.as_object_mut() + .expect("an object") + .insert(key.to_string(), value); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A library-core instance named `fixture`, in a directory of its own with +/// a `Cargo.toml`, one source file and stub tools, not yet a git repository. +fn scaffold(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-base-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + home: root.join("home"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir bin"); + fs::create_dir_all(&fixture.home).expect("mkdir home"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-util", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); + fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); + stub(&fixture.bin, "cargo", CARGO_STUB); + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); + } + fixture +} + +/// The scaffold with `static-lint`, `security-advisory`, `property` and +/// `mutation` on, in a git repository whose one branch is `branch` and whose +/// one commit is everything `init` wrote. There is no remote. +fn fixture(name: &str, branch: &str) -> Fixture { + let fixture = scaffold(name); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!( + id, + "static-lint" | "security-advisory" | "property" | "mutation" + ); + set(layer, "enabled", Value::Bool(on)); + } + }); + git(&fixture, &fixture.repo, &["init", "-q", "-b", branch]); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "adopt"]); + fixture +} + +fn report(fixture: &Fixture) -> Value { + let raw = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("report.json was written"); + let doc: Value = serde_json::from_str(&raw).expect("report.json is JSON"); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let errors = schema::validate(&schema, &doc); + assert!( + errors.is_empty(), + "report.json breaks the schema: {errors:?}\n{raw}" + ); + doc +} + +fn mutation(report: &Value) -> &Value { + at(report, "/layers") + .as_array() + .and_then(|layers| layers.iter().find(|row| at(row, "/id") == "mutation")) + .unwrap_or_else(|| at(report, "/missing")) +} + +/// The entries of `problems` that start with `mutation:`. +fn mutation_problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|problem| problem.starts_with("mutation:")) + .map(str::to_string) + .collect() +} + +/// The real `cargo-mutants`, or `None` when it is not installed. Without it +/// the tests stop, unless `RUST_TOPS_REQUIRE_MUTANTS` is set: then they fail. +fn real_mutants() -> Option { + let path = std::env::var_os("PATH").unwrap_or_default(); + let version = Command::new("cargo") + .args(["mutants", "--version"]) + .env("PATH", &path) + .output(); + let said = version + .ok() + .filter(|out| out.status.success()) + .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string()); + if said.as_deref() == Some("cargo-mutants 27.1.0") { + return Some(path); + } + assert!( + std::env::var_os("RUST_TOPS_REQUIRE_MUTANTS").is_none(), + "RUST_TOPS_REQUIRE_MUTANTS is set and `cargo mutants --version` printed {said:?}, not cargo-mutants 27.1.0" + ); + eprintln!("cargo-mutants 27.1.0 is not installed: this test did not run"); + None +} + +/// A fixture with only `mutation` and `security-advisory` on, `cargo-deny` +/// a stub and everything else real, whose `feature` branch holds one more +/// commit that writes `file` as `text`. Returns the gate's output. +fn gate_after_writing(name: &str, file: &str, text: &str) -> Option<(Fixture, Output)> { + let path = real_mutants()?; + let fixture = fixture(name, "main"); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!(id, "security-advisory" | "mutation"); + set(layer, "enabled", Value::Bool(on)); + } + }); + // Only `cargo-deny` stays a stub: the real cargo and cargo-mutants run. + for real in ["cargo", "cargo-mutants", "cargo-nextest"] { + fs::remove_file(fixture.bin.join(real)).expect("remove the stub"); + } + fs::write( + fixture.repo.join("src/lib.rs"), + "pub fn one() -> u8 {\n 1\n}\n", + ) + .expect("lib"); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "lib"]); + git( + &fixture, + &fixture.repo, + &["checkout", "-q", "-b", "feature"], + ); + fs::write(fixture.repo.join(file), text).expect("write the change"); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "change"]); + let mut dirs = vec![fixture.bin.clone()]; + dirs.extend(std::env::split_paths(&path)); + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["gate", "--base", "main", "--path"]) + .arg(&fixture.repo) + .env("PATH", std::env::join_paths(dirs).expect("PATH")) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI") + .output() + .expect("spawn cargo-tops gate"); + Some((fixture, out)) +} + +fn expect_zero_mutants(fixture: &Fixture, out: &Output, said: &str) { + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(0), "{stdout}{stderr}"); + assert!( + stderr.contains(said), + "cargo mutants said {said:?}: {stderr}" + ); + let report = report(fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{row}"); + assert_eq!(at(row, "/judged/caught"), 0, "{row}"); + assert_eq!(at(row, "/judged/missed"), 0, "{row}"); + assert!(mutation_problems(&report).is_empty(), "{report}"); + // Real 27.1.0 writes no `mutants.out` for a diff with nothing to mutate. + let moved = fixture.repo.join("target/rust-tops/mutants.out"); + assert!(!moved.exists(), "27.1.0 wrote {}", moved.display()); + let here = fixture.repo.join("mutants.out"); + assert!(!here.exists(), "27.1.0 wrote {}", here.display()); +} + +#[test] +fn a_readme_only_diff_passes_through_the_real_cargo_mutants() { + if let Some((fixture, out)) = gate_after_writing("readme-only", "README.md", "# fixture\n") { + expect_zero_mutants(&fixture, &out, "Diff changes no Rust source files"); + } +} + +#[test] +fn a_comment_only_diff_passes_through_the_real_cargo_mutants() { + let changed = "/// One.\npub fn one() -> u8 {\n 1\n}\n"; + if let Some((fixture, out)) = gate_after_writing("comment-only", "src/lib.rs", changed) { + expect_zero_mutants(&fixture, &out, "No mutants to filter"); + } +} diff --git a/crates/cargo-tops/tests/gate_survivors.rs b/crates/cargo-tops/tests/gate_survivors.rs new file mode 100644 index 0000000..7a31820 --- /dev/null +++ b/crates/cargo-tops/tests/gate_survivors.rs @@ -0,0 +1,867 @@ +//! The survivors ledger. After `cargo mutants` runs, the gate reads +//! `mutants.out/outcomes.json` (pinned to cargo-mutants 27.1.0) and judges +//! the missed mutants: one that a live `layer: mutation` exemption covers +//! (same rule as coverage and crap) is explained, the rest are not, and the +//! layer passes when the unexplained count is at most +//! `gates.mutation.unexplained_survivors_max`. Timeouts and unviable mutants +//! are counted, not survivors. A file that is missing, unreadable or of an +//! unknown shape fails the layer. +//! +//! `cargo` is a stub that, for `mutants`, copies a planted `outcomes.json` +//! into `<--output>/mutants.out/` and exits as cargo-mutants does (2 when +//! mutants were missed, 3 when some timed out). It writes nothing, and prints +//! what a run prints, when a test says so. The planted files come from the fixture +//! recorded from a real 27.1.0 run. Every report is validated against the +//! schema. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::{json, Value}; + +/// The validator `cargo tops check` uses, and the schema it reads. +#[path = "../src/schema.rs"] +mod schema; + +mod profiles { + pub(crate) const SCHEMA_JSON: &str = include_str!("../templates/rust-tops.schema.json"); +} + +const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); + +/// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// passes everything else. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +if [ "$1" = "mutants" ]; then + out= + prev= + for arg in "$@"; do + if [ "$prev" = "--output" ]; then out=$arg; fi + prev=$arg + done + if [ -z "$out" ]; then echo "no --output given" >&2; exit 64; fi + if [ -n "${MUTANTS_SAYS:-}" ]; then echo "$MUTANTS_SAYS" >&2; fi + if [ -n "${MUTANTS_OUTCOMES:-}" ]; then + mkdir -p "$out/mutants.out" && cp "$MUTANTS_OUTCOMES" "$out/mutants.out/outcomes.json" + fi + exit "${MUTANTS_EXIT:-0}" +fi +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + home: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn at<'v>(value: &'v Value, pointer: &str) -> &'v Value { + static NULL: Value = Value::Null; + value.pointer(pointer).unwrap_or(&NULL) +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +fn git(fixture: &Fixture, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(dir) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_AUTHOR_NAME", "fixture") + .env("GIT_AUTHOR_EMAIL", "fixture@example.invalid") + .env("GIT_COMMITTER_NAME", "fixture") + .env("GIT_COMMITTER_EMAIL", "fixture@example.invalid") + .output() + .expect("spawn git"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn set(doc: &mut Value, key: &str, value: Value) { + doc.as_object_mut() + .expect("an object") + .insert(key.to_string(), value); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A library-core instance named `fixture`, in a directory of its own with +/// a `Cargo.toml`, one source file and stub tools, not yet a git repository. +fn scaffold(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-base-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + home: root.join("home"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir bin"); + fs::create_dir_all(&fixture.home).expect("mkdir home"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); + fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); + stub(&fixture.bin, "cargo", CARGO_STUB); + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); + } + fixture +} + +/// The scaffold with `static-lint`, `security-advisory`, `property` and +/// `mutation` on, in a git repository whose one branch is `branch` and whose +/// one commit is everything `init` wrote. There is no remote. +fn fixture(name: &str, branch: &str) -> Fixture { + let fixture = scaffold(name); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!( + id, + "static-lint" | "security-advisory" | "property" | "mutation" + ); + set(layer, "enabled", Value::Bool(on)); + } + }); + git(&fixture, &fixture.repo, &["init", "-q", "-b", branch]); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "adopt"]); + fixture +} + +/// A commit that adds `file` under `src/`. +fn commit_file(fixture: &Fixture, file: &str) -> String { + fs::write(fixture.repo.join("src").join(file), format!("// {file}\n")).expect("write"); + git(fixture, &fixture.repo, &["add", "-A"]); + git(fixture, &fixture.repo, &["commit", "-q", "-m", file]); + git(fixture, &fixture.repo, &["rev-parse", "HEAD"]) +} + +/// What a clone leaves: a remote `origin` holding `branch`, with +/// `refs/remotes/origin/HEAD` pointing at it. +/// `refs/remotes/origin/` at `commit`. +fn gate(fixture: &Fixture, args: &[&str], ci: Option<&str>) -> (Output, String) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.arg("gate") + .args(args) + .arg("--path") + .arg(&fixture.repo) + .env( + "PATH", + std::env::join_paths([ + fixture.bin.as_path(), + Path::new("/usr/bin"), + Path::new("/bin"), + ]) + .expect("PATH"), + ) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("CARGO_STUB_LOG", fixture.root.join("cargo.log")) + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI"); + if let Some(value) = ci { + cmd.env("CI", value); + } + let out = cmd.output().expect("spawn cargo-tops gate"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +fn report(fixture: &Fixture) -> Value { + let raw = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("report.json was written"); + let doc: Value = serde_json::from_str(&raw).expect("report.json is JSON"); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let errors = schema::validate(&schema, &doc); + assert!( + errors.is_empty(), + "report.json breaks the schema: {errors:?}\n{raw}" + ); + doc +} + +fn mutation(report: &Value) -> &Value { + at(report, "/layers") + .as_array() + .and_then(|layers| layers.iter().find(|row| at(row, "/id") == "mutation")) + .unwrap_or_else(|| at(report, "/missing")) +} + +/// The entries of `problems` that start with `mutation:`. +fn mutation_problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|problem| problem.starts_with("mutation:")) + .map(str::to_string) + .collect() +} + +/// The real 27.1.0 run: 11 caught, 10 missed (5 in `double`, 5 in +/// `inner::triple`), 1 timeout, 1 unviable. +const RECORDED: &str = include_str!("fixtures/mutants-27.1.0/outcomes.json"); + +/// A real 27.1.0 run over a crate with a `const`, a `static`, a type alias, a +/// struct field type and one untested `fn get`: 12 missed (7 with `function: +/// null`, 5 in `get`) and 1 unviable. +const NULL_FUNCTION: &str = include_str!("fixtures/mutants-27.1.0/null-function/outcomes.json"); + +/// `text` planted under the fixture. +fn planted_text(fixture: &Fixture, text: &str) -> PathBuf { + let path = fixture.root.join("outcomes.json"); + fs::write(&path, text).expect("plant outcomes.json"); + path +} + +/// The recorded run with `change` applied, planted under the fixture. +fn planted(fixture: &Fixture, change: impl FnOnce(&mut Value)) -> PathBuf { + let mut doc: Value = serde_json::from_str(RECORDED).expect("fixture is JSON"); + change(&mut doc); + let path = fixture.root.join("outcomes.json"); + fs::write(&path, doc.to_string()).expect("plant outcomes.json"); + path +} + +type Edit = fn(&mut Value); + +fn outcomes_mut(doc: &mut Value) -> &mut Vec { + doc.get_mut("outcomes") + .and_then(Value::as_array_mut) + .expect("outcomes") +} + +/// Set each total to what the outcomes list, so an edit keeps the file whole. +fn recount(doc: &mut Value) { + let summaries: Vec = outcomes_mut(doc) + .iter() + .skip(1) + .map(|outcome| { + at(outcome, "/summary") + .as_str() + .unwrap_or_default() + .to_string() + }) + .collect(); + let of = |summary: &str| summaries.iter().filter(|s| *s == summary).count(); + let totals = [ + ("caught", of("CaughtMutant")), + ("missed", of("MissedMutant")), + ("timeout", of("Timeout")), + ("unviable", of("Unviable")), + ]; + let all: usize = totals.iter().map(|(_, n)| n).sum(); + for (key, n) in totals { + set(doc, key, json!(n)); + } + set(doc, "total_mutants", json!(all)); +} + +/// Keep only the missed mutants for which `keep(function)` holds, and +/// correct the totals. +fn keep_missed(doc: &mut Value, keep: impl Fn(&str) -> bool) { + outcomes_mut(doc).retain(|outcome| { + let function = at(outcome, "/scenario/Mutant/function/function_name") + .as_str() + .unwrap_or_default(); + at(outcome, "/summary") != "MissedMutant" || keep(function) + }); + recount(doc); +} + +/// One survivor in `double` and one in `inner::triple`. +fn one_each(doc: &mut Value) { + let mut seen = Vec::new(); + outcomes_mut(doc).retain(|outcome| { + let function = at(outcome, "/scenario/Mutant/function/function_name") + .as_str() + .unwrap_or_default() + .to_string(); + if at(outcome, "/summary") != "MissedMutant" { + return true; + } + let first = !seen.contains(&function); + seen.push(function); + first + }); + recount(doc); +} + +/// Run the gate with `outcomes` planted, the way cargo-mutants exits for it. +fn gate_with(fixture: &Fixture, outcomes: Option<&Path>, exit: &str) -> (Output, String) { + gate_saying(fixture, outcomes, exit, "") +} + +/// As [`gate_with`], and `says` is what the run prints to stderr. +fn gate_saying( + fixture: &Fixture, + outcomes: Option<&Path>, + exit: &str, + says: &str, +) -> (Output, String) { + git(fixture, &fixture.repo, &["checkout", "-q", "-b", "feature"]); + commit_file(fixture, "b.rs"); + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.args(["gate", "--base", "HEAD~1", "--path"]) + .arg(&fixture.repo) + .env("PATH", stub_path(fixture)) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("MUTANTS_EXIT", exit) + .env("MUTANTS_SAYS", says) + .env_remove("MUTANTS_OUTCOMES") + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI"); + if let Some(path) = outcomes { + cmd.env("MUTANTS_OUTCOMES", path); + } + let out = cmd.output().expect("spawn cargo-tops gate"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +fn stub_path(fixture: &Fixture) -> std::ffi::OsString { + std::env::join_paths([ + fixture.bin.as_path(), + Path::new("/usr/bin"), + Path::new("/bin"), + ]) + .expect("PATH") +} + +/// A live `layer: mutation` record for `path`, and `symbol` when given. +fn exempt(fixture: &Fixture, path: &str, symbol: Option<&str>, expires: &str) { + edit(fixture, |doc| { + let mut record = json!({ + "path": path, + "layer": "mutation", + "owner": "fixture", + "reason": "a planted survivor", + "expires": expires, + }); + if let Some(symbol) = symbol { + set(&mut record, "symbol", json!(symbol)); + } + set(doc, "exemptions", json!([record])); + }); +} + +fn set_max(fixture: &Fixture, max: u64) { + edit(fixture, |doc| { + let mutation = doc.pointer_mut("/gates/mutation").expect("gates.mutation"); + set(mutation, "unexplained_survivors_max", json!(max)); + }); +} + +/// The row's `judged` numbers, in the order the SPEC names them. +fn counts(row: &Value) -> [u64; 6] { + [ + "caught", + "missed", + "explained", + "unexplained", + "timeouts", + "unviable", + ] + .map(|key| { + at(row, &format!("/judged/{key}")) + .as_u64() + .unwrap_or(u64::MAX) + }) +} + +fn expect_failed_naming(text: &str, report: &Value, named: &str, not_named: &str) { + let problems = mutation_problems(report); + assert_eq!(problems.len(), 1, "{problems:?}\n{text}"); + assert!(problems.iter().all(|p| p.contains(named)), "{problems:?}"); + assert!( + problems.iter().all(|p| !p.contains(not_named)), + "{problems:?}" + ); +} + +#[test] +fn one_explained_and_one_unexplained_survivor_fails_naming_only_the_second() { + let fixture = fixture("one-each", "main"); + exempt(&fixture, "src/**", Some("double"), "2099-01-01"); + let file = planted(&fixture, one_each); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "fail", "{row}"); + expect_failed_naming(&text, &report, "inner::triple", "double"); + // caught 11, missed 2 (1 explained, 1 not), 1 timeout, 1 unviable. + assert_eq!(counts(row), [11, 2, 1, 1, 1, 1], "{row}"); + assert_eq!(at(row, "/judged/unexplained_survivors_max"), 0, "{row}"); +} + +#[test] +fn survivors_that_every_record_explains_pass() { + let fixture = fixture("both-explained", "main"); + edit(&fixture, |doc| { + let record = |symbol: &str| { + json!({ + "path": "src/lib.rs", + "symbol": symbol, + "layer": "mutation", + "owner": "fixture", + "reason": "planted for this test", + "expires": "2099-01-01", + }) + }; + set( + doc, + "exemptions", + json!([record("double"), record("inner::triple")]), + ); + }); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(0), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{row}"); + assert_eq!(counts(row), [11, 10, 10, 0, 1, 1], "{row}"); + assert!(mutation_problems(&report).is_empty()); +} + +#[test] +fn a_record_with_no_symbol_explains_no_survivor_inside_a_function() { + // Whatever its glob: only a symbol names a function. + for path in ["src/lib.rs", "src/**", "**", "*", "."] { + let fixture = fixture("no-symbol", "main"); + exempt(&fixture, path, None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{path}: {text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(counts(row), [11, 10, 0, 10, 1, 1], "{path}: {row}"); + // Every survivor is named: the max of 0 decides. + let problems = mutation_problems(&report); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!( + problems + .iter() + .all(|p| p.contains("double") && p.contains("triple")), + "{problems:?}" + ); + } +} + +#[test] +fn a_record_for_another_file_symbol_or_layer_explains_nothing() { + for (path, symbol, layer) in [ + ("src/other.rs", Some("double"), "mutation"), + ("src/lib.rs", Some("nothing"), "mutation"), + ("src/lib.rs", Some("double"), "crap"), + ] { + let fixture = fixture("explains-nothing", "main"); + edit(&fixture, |doc| { + let mut record = json!({ + "path": path, "layer": layer, "owner": "fixture", + "reason": "planted for this test", "expires": "2099-01-01", + }); + set(&mut record, "symbol", json!(symbol)); + set(doc, "exemptions", json!([record])); + }); + let file = planted(&fixture, |doc| keep_missed(doc, |f| f == "double")); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!( + out.status.code(), + Some(1), + "{path} {symbol:?} {layer}: {text}" + ); + let report = report(&fixture); + assert_eq!(counts(mutation(&report))[2], 0, "{path} {symbol:?} {layer}"); + } +} + +#[test] +fn an_expired_mutation_record_explains_nothing() { + let fixture = fixture("expired-record", "main"); + exempt(&fixture, "src/lib.rs", None, "2020-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert_eq!(counts(mutation(&report(&fixture)))[2], 0); +} + +#[test] +fn a_maximum_of_one_passes_one_unexplained_survivor_and_fails_two() { + // A crate-wide record lifts the class floor for the max in `check` and, + // naming no file a mutant is in, explains nothing. + let first = fixture("max-one", "main"); + set_max(&first, 1); + exempt(&first, ".", None, "2099-01-01"); + let one = planted(&first, |doc| { + keep_missed(doc, |f| f == "double"); + outcomes_mut(doc).retain(|o| { + at(o, "/summary") != "MissedMutant" || at(o, "/scenario/Mutant/replacement") == "0" + }); + recount(doc); + }); + let (out, text) = gate_with(&first, Some(&one), "2"); + assert_eq!(out.status.code(), Some(0), "{text}"); + let report = report(&first); + let row = mutation(&report); + assert_eq!(counts(row), [11, 1, 0, 1, 1, 1], "{row}"); + assert_eq!(at(row, "/judged/unexplained_survivors_max"), 1); + + let second = fixture("max-one-two", "main"); + set_max(&second, 1); + exempt(&second, ".", None, "2099-01-01"); + let two = planted(&second, one_each); + let (out, text) = gate_with(&second, Some(&two), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); +} + +#[test] +fn timeouts_and_unviable_mutants_are_counted_and_are_not_survivors() { + let fixture = fixture("not-survivors", "main"); + let file = planted(&fixture, |doc| keep_missed(doc, |_| false)); + let (out, text) = gate_with(&fixture, Some(&file), "3"); + assert_eq!(out.status.code(), Some(0), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{row}"); + assert_eq!(counts(row), [11, 0, 0, 0, 1, 1], "{row}"); +} + +#[test] +fn an_unknown_shape_fails_with_a_message_and_never_passes() { + let cases: [(&str, Edit); 3] = [ + ("another release", |d| { + set(d, "cargo_mutants_version", json!("99.0.0")) + }), + ("an unknown outcome", |d| { + if let Some(first) = outcomes_mut(d).get_mut(1) { + set(first, "summary", json!("Skipped")); + } + }), + ("totals that disagree", |d| set(d, "missed", json!(0))), + ]; + for (what, change) in cases { + let fixture = fixture("unknown-shape", "main"); + let file = planted(&fixture, change); + let (out, text) = gate_with(&fixture, Some(&file), "0"); + assert_eq!(out.status.code(), Some(1), "{what}: {text}"); + let report = report(&fixture); + assert_eq!(at(mutation(&report), "/verdict"), "fail", "{what}"); + let problems = mutation_problems(&report); + assert!( + problems.iter().all(|p| p.contains("outcomes.json")), + "{what}: {problems:?}" + ); + assert_eq!(problems.len(), 1, "{what}: {problems:?}"); + } +} + +#[test] +fn a_file_that_is_not_json_fails() { + let fixture = fixture("not-json", "main"); + let path = fixture.root.join("garbage.json"); + fs::write(&path, "not json").expect("plant"); + let (out, text) = gate_with(&fixture, Some(&path), "0"); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!(text.contains("outcomes.json"), "{text}"); +} + +#[test] +fn no_outcomes_file_is_a_failure_and_a_stale_one_is_never_read() { + let fixture = fixture("no-outcomes", "main"); + // An earlier run left a clean file behind. This run wrote none. + let stale = fixture.repo.join("target/rust-tops/mutants.out"); + fs::create_dir_all(&stale).expect("mkdir"); + let clean = planted(&fixture, |doc| keep_missed(doc, |_| false)); + fs::copy(&clean, stale.join("outcomes.json")).expect("plant stale outcomes"); + let (out, text) = gate_with(&fixture, None, "0"); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!(text.contains("outcomes.json"), "{text}"); +} + +#[test] +fn a_dry_run_reads_no_outcomes() { + let fixture = fixture("no-run", "main"); + let (out, text) = gate(&fixture, &["--dry-run"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!(!fixture.repo.join("mutants.out").exists()); + assert!(!fixture.repo.join("target/rust-tops/mutants.out").exists()); +} + +/// What 27.1.0 prints when the diff leaves nothing to mutate. +const NO_MUTANTS: [&str; 2] = [ + " INFO Diff changes no Rust source files", + " INFO No mutants to filter", +]; + +#[test] +fn a_diff_with_nothing_to_mutate_passes_as_zero_mutants() { + for says in NO_MUTANTS { + let fixture = fixture("nothing-to-mutate", "main"); + let (out, text) = gate_saying(&fixture, None, "0", says); + assert_eq!(out.status.code(), Some(0), "{says}: {text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{says}: {row}"); + assert_eq!(counts(row), [0; 6], "{says}: {row}"); + assert!(mutation_problems(&report).is_empty()); + } +} + +#[test] +fn no_file_is_zero_mutants_only_when_the_run_said_so_and_exited_0() { + // Said nothing, or something else: the file went missing. + for says in [ + "", + " INFO Found 3 mutants to test", + " INFO Diff file is empty", + ] { + let fixture = fixture("went-missing", "main"); + let (out, text) = gate_saying(&fixture, None, "0", says); + assert_eq!(out.status.code(), Some(1), "{says:?}: {text}"); + let problems = mutation_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!( + problems + .iter() + .all(|p| p.contains("outcomes.json") && p.contains("exited 0")), + "{problems:?}" + ); + } + // The line, and an exit that says mutants were missed, or any failure. + for exit in ["2", "3", "1", "4"] { + let fixture = fixture("line-and-exit", "main"); + let (out, text) = gate_saying(&fixture, None, exit, NO_MUTANTS[1]); + assert_eq!(out.status.code(), Some(1), "exit {exit}: {text}"); + } +} + +#[test] +fn a_failed_cargo_mutants_says_what_its_exit_means() { + for (exit, words) in [ + ("1", "exited 1"), + ("4", "unmutated tree"), + ("9", "exited 9"), + ] { + let fixture = fixture("exit-meaning", "main"); + let (out, text) = gate_with(&fixture, None, exit); + assert_eq!(out.status.code(), Some(1), "exit {exit}: {text}"); + let problems = mutation_problems(&report(&fixture)); + assert!( + problems + .iter() + .all(|p| p.contains("cargo mutants") && p.contains(words)), + "exit {exit}: {problems:?}" + ); + assert_eq!(problems.len(), 1, "{problems:?}"); + } +} + +#[test] +fn the_gate_names_the_output_directory_so_an_adopters_setting_cannot_move_it() { + let fixture = fixture("output-dir", "main"); + fs::create_dir_all(fixture.repo.join(".cargo")).expect("mkdir"); + fs::write( + fixture.repo.join(".cargo/mutants.toml"), + "output = \"elsewhere\"\n", + ) + .expect("write mutants.toml"); + let file = planted(&fixture, |doc| keep_missed(doc, |_| false)); + let (out, text) = gate_with(&fixture, Some(&file), "3"); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(fixture + .repo + .join("target/rust-tops/mutants.out/outcomes.json") + .exists()); + assert!(!fixture.repo.join("elsewhere").exists()); +} + +type Disagreement = (&'static str, Edit, &'static str, &'static [&'static str]); + +/// Each planted file, with the exit its run had, fails naming `words`. +fn expect_disagreements(name: &str, cases: &[Disagreement]) { + for (what, change, exit, words) in cases { + let fixture = fixture(name, "main"); + let file = planted(&fixture, *change); + let (out, text) = gate_with(&fixture, Some(&file), exit); + assert_eq!(out.status.code(), Some(1), "{what}: {text}"); + let problems = mutation_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{what}: {problems:?}"); + for word in *words { + assert!( + problems.iter().all(|p| p.contains(word)), + "{what} lacks {word}: {problems:?}" + ); + } + } +} + +#[test] +fn a_total_that_is_wrong_or_missing_fails_naming_both_numbers() { + expect_disagreements( + "bad-total", + &[ + ( + "a wrong total", + |d| set(d, "total_mutants", json!(999)), + "2", + &["says 999", "lists 23"], + ), + ( + "no total", + |d| { + d.as_object_mut().expect("object").remove("total_mutants"); + }, + "2", + &["total_mutants"], + ), + ], + ); +} + +#[test] +fn an_exit_code_the_file_does_not_bear_out_fails_naming_both() { + expect_disagreements( + "bad-exit", + &[ + ( + "exit 2, nothing missed", + |d| keep_missed(d, |_| false), + "2", + &["exited 2", "no missed"], + ), + ( + "exit 3, no timeout", + |d| { + outcomes_mut(d).retain(|o| at(o, "/summary") != "Timeout"); + set(d, "timeout", json!(0)); + set(d, "total_mutants", json!(22)); + }, + "3", + &["exited 3", "no timeout"], + ), + ], + ); +} + +#[test] +fn a_survivor_in_no_function_is_explained_only_by_a_record_with_no_symbol() { + // 7 of the 12 survivors are in a const, a static, a type alias and a + // field type: cargo-mutants 27.1.0 writes `function: null` for them. + let unexplained = |symbol: Option<&str>, path: &str| -> u64 { + let fixture = fixture("null-function", "main"); + exempt(&fixture, path, symbol, "2099-01-01"); + let file = planted_text(&fixture, NULL_FUNCTION); + let (_, text) = gate_with(&fixture, Some(&file), "2"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(counts(row)[1], 12, "{text}"); + counts(row)[3] + }; + assert_eq!(unexplained(Some("get"), "src/lib.rs"), 7); + assert_eq!(unexplained(Some("no_such_fn"), "src/lib.rs"), 12); + // A record with no symbol explains the 7 in no function, never the 5 + // in `get`, whatever its glob covers. + assert_eq!(unexplained(None, "src/lib.rs"), 5); + assert_eq!(unexplained(None, "src/**"), 5); + assert_eq!(unexplained(None, "**"), 5); +} + +#[test] +fn a_whole_crate_record_lifts_the_floor_and_explains_no_survivor() { + // The spellings `check` accepts to lift the floor of the max; the max + // then decides, because none of them explains a survivor. + for path in ["**", "./**", "**/*", ".", "./"] { + let fixture = fixture("whole-crate", "main"); + set_max(&fixture, 10); + exempt(&fixture, path, None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(0), "{path}: {text}"); + let report = report(&fixture); + let row = mutation(&report); + // 10 missed, none explained: the max decides. + assert_eq!(counts(row), [11, 10, 0, 10, 1, 1], "{path}: {row}"); + } + // At a max of 9 the same record changes nothing: 10 survivors fail. + let fixture = fixture("whole-crate-nine", "main"); + set_max(&fixture, 9); + exempt(&fixture, "**", None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); +} + +#[test] +fn a_symbol_record_explains_only_its_own_function() { + let fixture = fixture("own-function", "main"); + exempt(&fixture, "**", Some("double"), "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(counts(row), [11, 10, 5, 5, 1, 1], "{row}"); + expect_failed_naming(&text, &report, "triple", " double "); +} diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 41c0144..7c98d53 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -50,6 +50,11 @@ if [ "$*" = "crap --version" ]; then echo 'cargo-crap 0.9.0' exit 0 fi +# 27.1.0 finds nothing to mutate in a diff of comments, tests, docs or CI: +# it says so on stderr, exits 0 and writes no `mutants.out`. +if [ "$1" = "mutants" ]; then + echo ' INFO No mutants to filter' >&2 +fi prev= for arg in "$@"; do case $prev in @@ -1856,7 +1861,12 @@ fn a_real_diff_runs_mutants_on_the_report_dir_diff() { let body = fs::read_to_string(&diff).expect("diff under the report dir"); assert!(body.contains("src/lib.rs"), "{body}"); let log = cargo_log(&fixture); - let want = format!("mutants --in-diff {} -j 2", diff.display()); + let report_dir = diff.parent().expect("the diff is in the report dir"); + let want = format!( + "mutants --in-diff {} -j 2 --output {}", + diff.display(), + report_dir.display() + ); assert!(log.lines().any(|line| line == want), "{log}"); assert!(!log.contains("/tmp/rust-tops.diff"), "{log}"); assert!(!text.contains("/tmp/rust-tops.diff"), "{text}"); diff --git a/schema/gate-report.schema.json b/schema/gate-report.schema.json index b591ee5..e65feb8 100644 --- a/schema/gate-report.schema.json +++ b/schema/gate-report.schema.json @@ -39,7 +39,7 @@ "enum": ["pass", "fail", "config-invalid", "environment"] }, "exit_code": { - "description": "0 pass; 1 the gate ran and failed (a dry run included); 2 the config is invalid; 3 the environment (a required tool or cargo metadata is missing).", + "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable when a layer in range needs a pin (crap today); a command-line usage error is 2 too and writes no report; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", "type": "integer", "enum": [0, 1, 2, 3] },