From e1a68ec880ad3fbaa6b86aa1eaf2e656ca6e897e Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:02:28 -0500 Subject: [PATCH 01/10] Read the mutation survivors from outcomes.json mutation.rs parses cargo-mutants 27.1.0's outcomes.json and refuses any other version or shape, then sorts the missed mutants by the live mutation exemptions that cover them (Exemption::covers). Timeouts and unviable mutants are counted, not survivors. The fixture is a real run. Not wired into the gate yet. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/main.rs | 1 + crates/cargo-tops/src/mutation.rs | 454 +++++ .../tests/fixtures/mutants-27.1.0/README.md | 27 + .../fixtures/mutants-27.1.0/outcomes.json | 1506 +++++++++++++++++ 4 files changed, 1988 insertions(+) create mode 100644 crates/cargo-tops/src/mutation.rs create mode 100644 crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md create mode 100644 crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 5e261bd..cd91edc 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -30,6 +30,7 @@ mod floors; mod gate; mod init_meta; mod laws; +mod mutation; mod profile_data; mod profiles; mod schema; diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs new file mode 100644 index 0000000..3cd0ba8 --- /dev/null +++ b/crates/cargo-tops/src/mutation.rs @@ -0,0 +1,454 @@ +//! The survivors ledger: what `cargo mutants` left in `mutants.out/outcomes.json`. +//! +//! [`parse`] reads the file in the shape cargo-mutants [`PINNED`] writes and +//! refuses any other: a version it was not written for, a field it cannot +//! read, an outcome it does not know, or totals that disagree with the +//! outcomes. [`explain`] sorts the missed mutants into those a live +//! `layer: mutation` exemption covers (the rule of [`Exemption::covers`]) +//! and those nothing covers. The gate passes the layer when the unexplained +//! count is at most `gates.mutation.unexplained_survivors_max`. Timeouts and +//! unviable mutants are counted and reported; they are not survivors. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] +#![allow(dead_code, reason = "the gate wires these items in G2")] + +use serde_json::{json, Value}; + +use crate::exemptions::Exemption; + +/// The cargo-mutants release whose `outcomes.json` this reads. +pub(crate) const PINNED: &str = "27.1.0"; + +/// One mutant the suite did not catch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Missed { + /// The file, as cargo-mutants names it: relative to the workspace root. + pub file: String, + /// The function the mutant is in, `module::name` for a nested one. + pub function: Option, + pub line: u64, + /// cargo-mutants' own name for the mutant, such as + /// `src/lib.rs:8:5: replace double -> i32 with 0`. + pub name: String, +} + +/// What a run found, by kind. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Outcomes { + pub caught: usize, + pub missed: Vec, + pub timeouts: usize, + pub unviable: usize, +} + +/// The missed mutants of a run, sorted by the records that cover them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Ledger { + pub caught: usize, + pub explained: Vec, + pub unexplained: Vec, + pub timeouts: usize, + pub unviable: usize, +} + +fn field<'v>(value: &'v Value, key: &str) -> Result<&'v Value, String> { + value.get(key).ok_or_else(|| { + format!("outcomes.json has no `{key}` where cargo-mutants {PINNED} writes one") + }) +} + +fn count(value: &Value, key: &str) -> Result { + field(value, key)? + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| format!("outcomes.json `{key}` is not a count")) +} + +/// The missed mutant a `Mutant` scenario describes. +fn missed(scenario: &Value) -> Result { + let mutant = field(scenario, "Mutant")?; + let function = match field(mutant, "function")? { + Value::Null => None, + named => Some( + field(named, "function_name")? + .as_str() + .ok_or("outcomes.json `function_name` is not a string")? + .to_string(), + ), + }; + let line = field(field(field(mutant, "span")?, "start")?, "line")? + .as_u64() + .ok_or("outcomes.json span `line` is not a number")?; + let text = |key: &str| -> Result { + Ok(field(mutant, key)? + .as_str() + .ok_or_else(|| format!("outcomes.json `{key}` is not a string"))? + .to_string()) + }; + Ok(Missed { + file: text("file")?, + function, + line, + name: text("name")?, + }) +} + +/// Add one outcome to `found`. The baseline run is not a mutant. +fn take(found: &mut Outcomes, outcome: &Value) -> Result<(), String> { + let scenario = field(outcome, "scenario")?; + let summary = field(outcome, "summary")? + .as_str() + .ok_or("outcomes.json `summary` is not a string")?; + if scenario.as_str() == Some("Baseline") { + return match summary { + "Success" => Ok(()), + other => Err(format!("the baseline run ended as `{other}`")), + }; + } + match summary { + "CaughtMutant" => found.caught += 1, + "MissedMutant" => found.missed.push(missed(scenario)?), + "Timeout" => found.timeouts += 1, + "Unviable" => found.unviable += 1, + other => { + return Err(format!( + "outcomes.json has an outcome `{other}` this gate does not know" + )) + } + } + Ok(()) +} + +/// Read `outcomes.json`. Any shape but the pinned one is an `Err` saying +/// what was not as expected, never a pass. +pub(crate) fn parse(text: &str) -> Result { + let doc: Value = + serde_json::from_str(text).map_err(|err| format!("outcomes.json is not JSON: {err}"))?; + let version = field(&doc, "cargo_mutants_version")? + .as_str() + .unwrap_or_default(); + if version != PINNED { + return Err(format!( + "outcomes.json is from cargo-mutants {version:?}; this gate reads {PINNED}" + )); + } + let list = field(&doc, "outcomes")? + .as_array() + .ok_or("outcomes.json `outcomes` is not a list")?; + let mut found = Outcomes::default(); + for outcome in list { + take(&mut found, outcome)?; + } + let totals = [ + ("caught", found.caught), + ("missed", found.missed.len()), + ("timeout", found.timeouts), + ("unviable", found.unviable), + ]; + for (key, seen) in totals { + let said = count(&doc, key)?; + if said != seen { + return Err(format!("outcomes.json says {said} {key} but lists {seen}")); + } + } + Ok(found) +} + +/// `file` relative to the crate root, given the crate's path under the +/// workspace root (`""` for a crate at the root). +fn relative<'f>(file: &'f str, crate_prefix: &str) -> &'f str { + let prefix = crate_prefix.trim_matches('/'); + if prefix.is_empty() { + return file; + } + file.strip_prefix(prefix) + .and_then(|rest| rest.strip_prefix('/')) + .unwrap_or(file) +} + +/// Sort the missed mutants. `live` are the layer's records that have not +/// expired; `crate_prefix` is the crate's directory under the workspace root, +/// which the records' globs are relative to. +pub(crate) fn explain(found: &Outcomes, live: &[Exemption], crate_prefix: &str) -> Ledger { + let (explained, unexplained) = found.missed.iter().cloned().partition(|mutant: &Missed| { + let rel = relative(&mutant.file, crate_prefix); + live.iter() + .any(|record| record.covers(rel, mutant.function.as_deref())) + }); + Ledger { + caught: found.caught, + explained, + unexplained, + timeouts: found.timeouts, + unviable: found.unviable, + } +} + +impl Ledger { + /// Whether the layer passes: at most `max` survivors nothing explains. + pub(crate) fn passes(&self, max: i64) -> bool { + i64::try_from(self.unexplained.len()).is_ok_and(|n| n <= max) + } + + /// The problem that names each unexplained survivor, or `None` when the + /// layer passes. Explained survivors are never named. + pub(crate) fn problem(&self, max: i64) -> Option { + if self.passes(max) { + return None; + } + let named: Vec = self + .unexplained + .iter() + .map(|mutant| format!("{}:{} {}", mutant.file, mutant.line, mutant.name)) + .collect(); + Some(format!( + "mutation: {} unexplained survivor(s), above unexplained_survivors_max {max}: {}", + named.len(), + named.join("; ") + )) + } + + /// The numbers the mutation row's `judged` carries. + pub(crate) fn to_json(&self, max: i64) -> Value { + json!({ + "caught": self.caught, + "missed": self.explained.len() + self.unexplained.len(), + "explained": self.explained.len(), + "unexplained": self.unexplained.len(), + "timeouts": self.timeouts, + "unviable": self.unviable, + "unexplained_survivors_max": max, + }) + } +} + +#[cfg(test)] +#[expect( + clippy::indexing_slicing, + reason = "the tests edit and read JSON they built from the recorded fixture" +)] +mod tests { + use super::{explain, parse, relative, Outcomes, PINNED}; + use crate::exemptions::Exemption; + use serde_json::Value; + + /// A real run of cargo-mutants 27.1.0 over a small crate: 11 caught, 10 + /// missed (5 in `double`, 5 in `inner::triple`), 1 timeout, 1 unviable. + type Edit = fn(&mut Value); + + const FIXTURE: &str = include_str!("../tests/fixtures/mutants-27.1.0/outcomes.json"); + + fn record(path: &str, symbol: Option<&str>) -> Exemption { + Exemption { + path: path.to_string(), + symbol: symbol.map(str::to_string), + expires: "2099-01-01".to_string(), + } + } + + fn edited(change: impl FnOnce(&mut Value)) -> String { + let mut doc: Value = serde_json::from_str(FIXTURE).expect("fixture is JSON"); + change(&mut doc); + doc.to_string() + } + + #[test] + fn the_recorded_run_has_the_counts_cargo_mutants_reported() { + let found = parse(FIXTURE).expect("the pinned shape parses"); + assert_eq!(found.caught, 11); + assert_eq!(found.missed.len(), 10); + assert_eq!(found.timeouts, 1); + assert_eq!(found.unviable, 1); + let first = found.missed.first().expect("a survivor"); + assert_eq!(first.file, "src/lib.rs"); + assert_eq!(first.function.as_deref(), Some("double")); + assert_eq!(first.line, 8); + assert_eq!(first.name, "src/lib.rs:8:5: replace double -> i32 with 0"); + let nested = found + .missed + .iter() + .filter(|m| m.function.as_deref() == Some("inner::triple")) + .count(); + assert_eq!(nested, 5); + } + + #[test] + fn another_release_is_refused_naming_both() { + let text = edited(|doc| doc["cargo_mutants_version"] = "26.0.0".into()); + let err = parse(&text).expect_err("not the pinned release"); + assert!(err.contains("26.0.0") && err.contains(PINNED), "{err}"); + let text = edited(|doc| { + doc.as_object_mut() + .expect("object") + .remove("cargo_mutants_version"); + }); + assert!(parse(&text).is_err()); + } + + #[test] + fn an_unknown_shape_is_an_error_and_never_a_pass() { + assert!(parse("").is_err()); + assert!(parse("[]").is_err()); + assert!(parse("{}").is_err()); + let cases: [(&str, Edit); 7] = [ + ("outcomes is not a list", |d| d["outcomes"] = "x".into()), + ("an unknown summary", |d| { + d["outcomes"][1]["summary"] = "Skipped".into() + }), + ("no scenario", |d| { + d["outcomes"][1] + .as_object_mut() + .expect("object") + .remove("scenario"); + }), + ("a baseline that failed", |d| { + d["outcomes"][0]["summary"] = "Failure".into() + }), + ("no span", |d| { + d["outcomes"][6]["scenario"]["Mutant"] + .as_object_mut() + .expect("object") + .remove("span"); + }), + ("a missed total that disagrees", |d| d["missed"] = 3.into()), + ("no caught total", |d| { + d.as_object_mut().expect("object").remove("caught"); + }), + ]; + for (what, change) in cases { + let err = parse(&edited(change)).expect_err(what); + assert!(!err.is_empty(), "{what}"); + } + } + + #[test] + fn a_missed_mutant_outside_a_function_has_no_function_name() { + let text = edited(|doc| { + let at = doc["outcomes"] + .as_array() + .expect("list") + .iter() + .position(|o| o["summary"] == "MissedMutant") + .expect("a survivor"); + doc["outcomes"][at]["scenario"]["Mutant"]["function"] = Value::Null; + }); + let found = parse(&text).expect("a null function is allowed"); + assert_eq!(found.missed.first().and_then(|m| m.function.clone()), None); + } + + #[test] + fn a_record_for_one_function_explains_only_its_survivors() { + let found = parse(FIXTURE).expect("parses"); + let ledger = explain(&found, &[record("src/**", Some("double"))], ""); + assert_eq!(ledger.explained.len(), 5); + assert_eq!(ledger.unexplained.len(), 5); + assert!(ledger + .unexplained + .iter() + .all(|m| m.function.as_deref() == Some("inner::triple"))); + assert!(!ledger.passes(0)); + assert!(ledger.passes(5)); + let problem = ledger.problem(0).expect("fails"); + assert!( + problem.contains("5 unexplained") && problem.contains("inner::triple"), + "{problem}" + ); + assert!(!problem.contains(" double "), "{problem}"); + } + + #[test] + fn a_symbol_matches_the_last_path_segment_and_a_symbol_free_record_covers_its_glob() { + let found = parse(FIXTURE).expect("parses"); + let nested = explain(&found, &[record("src/lib.rs", Some("triple"))], ""); + assert_eq!(nested.explained.len(), 5); + let both = [ + record("src/lib.rs", Some("double")), + record("src/lib.rs", Some("inner::triple")), + ]; + assert!(explain(&found, &both, "").unexplained.is_empty()); + let all = explain(&found, &[record("src/**", None)], ""); + assert_eq!(all.explained.len(), 10); + assert!(all.passes(0) && all.problem(0).is_none()); + let elsewhere = explain(&found, &[record("other/**", None)], ""); + assert_eq!(elsewhere.unexplained.len(), 10); + } + + #[test] + fn globs_are_relative_to_the_crate_not_the_workspace() { + let text = edited(|doc| { + for outcome in doc["outcomes"].as_array_mut().expect("list") { + if let Some(file) = outcome["scenario"]["Mutant"]["file"].as_str() { + let moved = format!("crates/sample/{file}"); + outcome["scenario"]["Mutant"]["file"] = moved.into(); + } + } + }); + let found = parse(&text).expect("parses"); + // The record is relative to the crate: it names `src/lib.rs`, and the + // workspace-relative file is `crates/sample/src/lib.rs`. + let inside = [record("src/lib.rs", None)]; + assert_eq!( + explain(&found, &inside, "crates/sample").explained.len(), + 10 + ); + assert_eq!( + explain(&found, &inside, "crates/sample/").explained.len(), + 10 + ); + // A record that spells the workspace path matches only when no + // prefix is stripped. + let spelled = [record("crates/sample/src/lib.rs", None)]; + assert_eq!(explain(&found, &spelled, "").explained.len(), 10); + assert_eq!( + explain(&found, &spelled, "crates/sample").explained.len(), + 0 + ); + // A record for another crate's file explains nothing. + let other = [record("crates/other/src/lib.rs", None)]; + assert_eq!(explain(&found, &other, "crates/sample").explained.len(), 0); + } + + #[test] + fn the_prefix_is_stripped_whole_segments_only() { + assert_eq!(relative("crates/a/src/x.rs", "crates/a"), "src/x.rs"); + assert_eq!( + relative("crates/ab/src/x.rs", "crates/a"), + "crates/ab/src/x.rs" + ); + assert_eq!(relative("src/x.rs", ""), "src/x.rs"); + assert_eq!(relative("src/x.rs", "/"), "src/x.rs"); + } + + #[test] + fn the_judged_numbers_count_every_kind_and_the_max() { + let found = parse(FIXTURE).expect("parses"); + let ledger = explain(&found, &[record("src/lib.rs", Some("double"))], ""); + let judged = ledger.to_json(2); + assert_eq!(judged["caught"], 11); + assert_eq!(judged["missed"], 10); + assert_eq!(judged["explained"], 5); + assert_eq!(judged["unexplained"], 5); + assert_eq!(judged["timeouts"], 1); + assert_eq!(judged["unviable"], 1); + assert_eq!(judged["unexplained_survivors_max"], 2); + } + + #[test] + fn timeouts_and_unviable_mutants_are_not_survivors() { + let only = Outcomes { + caught: 0, + missed: Vec::new(), + timeouts: 4, + unviable: 9, + }; + let ledger = explain(&only, &[], ""); + assert!(ledger.passes(0) && ledger.problem(0).is_none()); + assert_eq!(ledger.timeouts, 4); + assert_eq!(ledger.unviable, 9); + } + + #[test] + fn a_negative_max_is_never_met() { + let ledger = explain(&Outcomes::default(), &[], ""); + assert!(!ledger.passes(-1)); + } +} diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md new file mode 100644 index 0000000..e26ddcd --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md @@ -0,0 +1,27 @@ +# mutants-27.1.0 + +`outcomes.json` is the file `cargo mutants --timeout 4 -j 1` 27.1.0 wrote for the +crate below, with the toolchain path in each `argv` shortened to `cargo`. +Nothing else was edited. It holds 11 caught, 10 missed, 1 timeout and 1 +unviable mutant, and the baseline run. + +```rust +pub struct Opaque(u8); + +pub fn add(a: i32, b: i32) -> i32 { a + b } +pub fn double(x: i32) -> i32 { x * 2 } +pub fn opaque() -> Opaque { Opaque(1) } +pub fn spin(n: u32) -> u32 { let mut i = 0; while i < n { i += 1; } i } + +pub mod inner { + pub fn triple(x: i32) -> i32 { x * 3 } +} +``` + +The tests call `add` (three asserts), `spin(3)` and `opaque()`. `double` and +`inner::triple` have no test, so every mutant in them is missed. `opaque` has no +`Default`, so its mutant is unviable. `+=` to `*=` in `spin` never ends, so it +times out. + +The file is real output, not a hand-built shape. To re-record it, run the same +command over the same crate with cargo-mutants 27.1.0 and shorten the paths. diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json new file mode 100644 index 0000000..48a212d --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/outcomes.json @@ -0,0 +1,1506 @@ +{ + "outcomes": [ + { + "scenario": "Baseline", + "summary": "Success", + "log_path": "log/baseline.log", + "diff_path": null, + "phase_results": [ + { + "phase": "Build", + "duration": 0.819982375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.513537209, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:5: replace add -> i32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 5 + }, + "end": { + "line": 4, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_5.log", + "diff_path": "diff/src__lib.rs_line_4_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120934916, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181101834, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:5: replace add -> i32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 5 + }, + "end": { + "line": 4, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_4_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.107022792, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181147208, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:5: replace add -> i32 with -1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 5 + }, + "end": { + "line": 4, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_5_002.log", + "diff_path": "diff/src__lib.rs_line_4_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.106414417, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.23059925, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:7: replace + with - in add", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 7 + }, + "end": { + "line": 4, + "column": 8 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_7.log", + "diff_path": "diff/src__lib.rs_line_4_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113024167, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181143791, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:4:7: replace + with * in add", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 3, + "column": 1 + }, + "end": { + "line": 5, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 4, + "column": 7 + }, + "end": { + "line": 4, + "column": 8 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_4_col_7_001.log", + "diff_path": "diff/src__lib.rs_line_4_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121543958, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.18111875, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:5: replace double -> i32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 5 + }, + "end": { + "line": 8, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_5.log", + "diff_path": "diff/src__lib.rs_line_8_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.118955125, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.232848, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:5: replace double -> i32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 5 + }, + "end": { + "line": 8, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_8_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.119879292, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.241087875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:5: replace double -> i32 with -1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 5 + }, + "end": { + "line": 8, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_5_002.log", + "diff_path": "diff/src__lib.rs_line_8_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.1075925, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.230802417, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:7: replace * with + in double", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 7 + }, + "end": { + "line": 8, + "column": 8 + } + }, + "replacement": "+", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_7.log", + "diff_path": "diff/src__lib.rs_line_8_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.170658209, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.229900042, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:7: replace * with / in double", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "double", + "return_type": "-> i32", + "span": { + "start": { + "line": 7, + "column": 1 + }, + "end": { + "line": 9, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 8, + "column": 7 + }, + "end": { + "line": 8, + "column": 8 + } + }, + "replacement": "/", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_7_001.log", + "diff_path": "diff/src__lib.rs_line_8_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.167602084, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.219947583, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace opaque -> Opaque with Default::default()", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "opaque", + "return_type": "-> Opaque", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 14 + } + }, + "replacement": "Default::default()", + "genre": "FnValue" + } + }, + "summary": "Unviable", + "log_path": "log/src__lib.rs_line_12_col_5.log", + "diff_path": "diff/src__lib.rs_line_12_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.106317833, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:16:5: replace spin -> u32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 16, + "column": 5 + }, + "end": { + "line": 20, + "column": 6 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_16_col_5.log", + "diff_path": "diff/src__lib.rs_line_16_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113336542, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.239245958, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:16:5: replace spin -> u32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 16, + "column": 5 + }, + "end": { + "line": 20, + "column": 6 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_16_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_16_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.110367875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181226375, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:17:13: replace < with == in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 17, + "column": 13 + }, + "end": { + "line": 17, + "column": 14 + } + }, + "replacement": "==", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_17_col_13.log", + "diff_path": "diff/src__lib.rs_line_17_col_13.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.115069583, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.181267458, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:17:13: replace < with > in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 17, + "column": 13 + }, + "end": { + "line": 17, + "column": 14 + } + }, + "replacement": ">", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_17_col_13_001.log", + "diff_path": "diff/src__lib.rs_line_17_col_13_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.11942975, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.172611042, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:17:13: replace < with <= in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 17, + "column": 13 + }, + "end": { + "line": 17, + "column": 14 + } + }, + "replacement": "<=", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_17_col_13_002.log", + "diff_path": "diff/src__lib.rs_line_17_col_13_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.107558, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 2.168281208, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:18:11: replace += with -= in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 18, + "column": 11 + }, + "end": { + "line": 18, + "column": 13 + } + }, + "replacement": "-=", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/src__lib.rs_line_18_col_11.log", + "diff_path": "diff/src__lib.rs_line_18_col_11.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.111740458, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.493100667, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:18:11: replace += with *= in spin", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "spin", + "return_type": "-> u32", + "span": { + "start": { + "line": 15, + "column": 1 + }, + "end": { + "line": 21, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 18, + "column": 11 + }, + "end": { + "line": 18, + "column": 13 + } + }, + "replacement": "*=", + "genre": "BinaryOperator" + } + }, + "summary": "Timeout", + "log_path": "log/src__lib.rs_line_18_col_11_001.log", + "diff_path": "diff/src__lib.rs_line_18_col_11_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121307375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 4.031318541, + "process_status": "Timeout", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:9: replace inner::triple -> i32 with 0", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 9 + }, + "end": { + "line": 25, + "column": 14 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_9.log", + "diff_path": "diff/src__lib.rs_line_25_col_9.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.109103375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.415611625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:9: replace inner::triple -> i32 with 1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 9 + }, + "end": { + "line": 25, + "column": 14 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_9_001.log", + "diff_path": "diff/src__lib.rs_line_25_col_9_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.12097825, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.612745625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:9: replace inner::triple -> i32 with -1", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 9 + }, + "end": { + "line": 25, + "column": 14 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_9_002.log", + "diff_path": "diff/src__lib.rs_line_25_col_9_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.106036375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 2.853253708, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:11: replace * with + in inner::triple", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 11 + }, + "end": { + "line": 25, + "column": 12 + } + }, + "replacement": "+", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_11.log", + "diff_path": "diff/src__lib.rs_line_25_col_11.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.112429208, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.871245709, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:25:11: replace * with / in inner::triple", + "package": "sample", + "file": "src/lib.rs", + "function": { + "function_name": "inner::triple", + "return_type": "-> i32", + "span": { + "start": { + "line": 24, + "column": 5 + }, + "end": { + "line": 26, + "column": 6 + } + } + }, + "span": { + "start": { + "line": 25, + "column": 11 + }, + "end": { + "line": 25, + "column": 12 + } + }, + "replacement": "/", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_25_col_11_001.log", + "diff_path": "diff/src__lib.rs_line_25_col_11_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121888083, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=sample@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.7411889170000001, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=sample@0.1.0" + ] + } + ] + } + ], + "total_mutants": 23, + "missed": 10, + "caught": 11, + "timeout": 1, + "unviable": 1, + "success": 0, + "start_time": "2026-11-20T03:59:09.188302Z", + "end_time": "2026-11-20T03:59:33.389279Z", + "cargo_mutants_version": "27.1.0" +} \ No newline at end of file From 4c85b9d10c2ecfa3e7a2eba897d6a4f523cf6eeb Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:12:18 -0500 Subject: [PATCH 02/10] Oracle tests for the survivors ledger and expired exemptions gate_survivors.rs plants a recorded cargo-mutants 27.1.0 outcomes.json through a cargo stub and judges explained and unexplained survivors, the max, timeouts, unviable mutants and an unknown shape. gate_exemptions.rs checks that an expired or not-a-day record fails the gate for any layer, on or off, at any max-tier, and that check only warns. A second recorded run holds the shape of a crate under a workspace. Most of these fail until the gate is wired to read outcomes.json and every record. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/mutation.rs | 33 + .../plant-workspace/outcomes.json | 549 +++++++++++++++++ crates/cargo-tops/tests/gate_exemptions.rs | 354 +++++++++++ crates/cargo-tops/tests/gate_survivors.rs | 581 ++++++++++++++++++ 4 files changed, 1517 insertions(+) create mode 100644 crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json create mode 100644 crates/cargo-tops/tests/gate_exemptions.rs create mode 100644 crates/cargo-tops/tests/gate_survivors.rs diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs index 3cd0ba8..2af6e7a 100644 --- a/crates/cargo-tops/src/mutation.rs +++ b/crates/cargo-tops/src/mutation.rs @@ -451,4 +451,37 @@ mod tests { let ledger = explain(&Outcomes::default(), &[], ""); assert!(!ledger.passes(-1)); } + + /// The QA H4 shape, recorded in a workspace: `qa_h4_survivor_miss` has no + /// test, so its two mutants survive, and `file` is relative to the + /// workspace root, not to the crate. + const PLANT: &str = + include_str!("../tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json"); + + #[test] + fn the_h4_plant_has_two_survivors_in_a_crate_under_the_workspace() { + let found = parse(PLANT).expect("parses"); + assert_eq!((found.caught, found.timeouts, found.unviable), (5, 0, 1)); + assert_eq!(found.missed.len(), 2); + assert!(found + .missed + .iter() + .all(|m| m.file == "crates/plant/src/qa_h4_plant.rs" + && m.function.as_deref() == Some("qa_h4_survivor_miss"))); + let by_symbol = [record("src/qa_h4_plant.rs", Some("qa_h4_survivor_miss"))]; + assert_eq!( + explain(&found, &by_symbol, "crates/plant").explained.len(), + 2 + ); + assert_eq!(explain(&found, &by_symbol, "").explained.len(), 2); + let other = [record("src/qa_h4_plant.rs", Some("another"))]; + assert_eq!(explain(&found, &other, "crates/plant").unexplained.len(), 2); + let wrong_file = [record("src/lib.rs", Some("qa_h4_survivor_miss"))]; + assert_eq!( + explain(&found, &wrong_file, "crates/plant") + .unexplained + .len(), + 2 + ); + } } diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json new file mode 100644 index 0000000..02bc6df --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/plant-workspace/outcomes.json @@ -0,0 +1,549 @@ +{ + "outcomes": [ + { + "scenario": "Baseline", + "summary": "Success", + "log_path": "log/baseline.log", + "diff_path": null, + "phase_results": [ + { + "phase": "Build", + "duration": 0.467251625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.450718083, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:5: replace add -> i32 with 0", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 5 + }, + "end": { + "line": 6, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_5.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121359417, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.238109916, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:5: replace add -> i32 with 1", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 5 + }, + "end": { + "line": 6, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_5_001.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.179454666, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.24125775, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:5: replace add -> i32 with -1", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 5 + }, + "end": { + "line": 6, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_5_002.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.171344166, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.231705667, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:7: replace + with - in add", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 7 + }, + "end": { + "line": 6, + "column": 8 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_7.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.173006334, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.240732125, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:6:7: replace + with * in add", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "add", + "return_type": "-> i32", + "span": { + "start": { + "line": 5, + "column": 1 + }, + "end": { + "line": 7, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 6, + "column": 7 + }, + "end": { + "line": 6, + "column": 8 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "CaughtMutant", + "log_path": "log/crates__plant__src__lib.rs_line_6_col_7_001.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_6_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.175716625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.241108292, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/lib.rs:10:5: replace opaque -> Opaque with Default::default()", + "package": "plant", + "file": "crates/plant/src/lib.rs", + "function": { + "function_name": "opaque", + "return_type": "-> Opaque", + "span": { + "start": { + "line": 9, + "column": 1 + }, + "end": { + "line": 11, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 10, + "column": 5 + }, + "end": { + "line": 10, + "column": 14 + } + }, + "replacement": "Default::default()", + "genre": "FnValue" + } + }, + "summary": "Unviable", + "log_path": "log/crates__plant__src__lib.rs_line_10_col_5.log", + "diff_path": "diff/crates__plant__src__lib.rs_line_10_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.115931709, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/qa_h4_plant.rs:2:5: replace qa_h4_survivor_miss -> u32 with 0", + "package": "plant", + "file": "crates/plant/src/qa_h4_plant.rs", + "function": { + "function_name": "qa_h4_survivor_miss", + "return_type": "-> u32", + "span": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 3, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 28 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/crates__plant__src__qa_h4_plant.rs_line_2_col_5.log", + "diff_path": "diff/crates__plant__src__qa_h4_plant.rs_line_2_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120998125, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.288781958, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "crates/plant/src/qa_h4_plant.rs:2:5: replace qa_h4_survivor_miss -> u32 with 1", + "package": "plant", + "file": "crates/plant/src/qa_h4_plant.rs", + "function": { + "function_name": "qa_h4_survivor_miss", + "return_type": "-> u32", + "span": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 3, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 28 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/crates__plant__src__qa_h4_plant.rs_line_2_col_5_001.log", + "diff_path": "diff/crates__plant__src__qa_h4_plant.rs_line_2_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.122019708, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=plant@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.241327625, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=plant@0.1.0" + ] + } + ] + } + ], + "total_mutants": 8, + "missed": 2, + "caught": 5, + "timeout": 0, + "unviable": 1, + "success": 0, + "start_time": "2026-11-20T04:08:58.142114Z", + "end_time": "2026-11-20T04:09:02.002208Z", + "cargo_mutants_version": "27.1.0" +} \ No newline at end of file diff --git a/crates/cargo-tops/tests/gate_exemptions.rs b/crates/cargo-tops/tests/gate_exemptions.rs new file mode 100644 index 0000000..4b104e7 --- /dev/null +++ b/crates/cargo-tops/tests/gate_exemptions.rs @@ -0,0 +1,354 @@ +//! Every exemption record is checked, whatever its layer, whether that layer +//! is on or off, and whatever `--max-tier` is: one that has expired, or whose +//! date is not a real day, is a gate problem and the gate exits 1. A date +//! passing does not make the config invalid, so it is never exit 2. `check` +//! prints the same lines as warnings and still exits 0. +//! +//! Cargo and its plugins are stubs. Every report is validated against the +//! schema. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::{json, Value}; + +/// The validator `cargo tops check` uses, and the schema it reads. +#[path = "../src/schema.rs"] +mod schema; + +mod profiles { + pub(crate) const SCHEMA_JSON: &str = include_str!("../templates/rust-tops.schema.json"); +} + +const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); + +/// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// passes everything else. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + home: PathBuf, +} + +fn at<'v>(value: &'v Value, pointer: &str) -> &'v Value { + static NULL: Value = Value::Null; + value.pointer(pointer).unwrap_or(&NULL) +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +fn set(doc: &mut Value, key: &str, value: Value) { + doc.as_object_mut() + .expect("an object") + .insert(key.to_string(), value); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A library-core instance named `fixture`, in a directory of its own with +/// a `Cargo.toml`, one source file and stub tools, not yet a git repository. +fn scaffold(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-base-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + home: root.join("home"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir bin"); + fs::create_dir_all(&fixture.home).expect("mkdir home"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); + fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); + stub(&fixture.bin, "cargo", CARGO_STUB); + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); + } + fixture +} + +/// The scaffold with `static-lint`, `security-advisory`, `property` and +/// `mutation` on, in a git repository whose one branch is `branch` and whose +/// one commit is everything `init` wrote. There is no remote. +/// A commit that adds `file` under `src/`. +/// What a clone leaves: a remote `origin` holding `branch`, with +/// `refs/remotes/origin/HEAD` pointing at it. +/// `refs/remotes/origin/` at `commit`. +fn gate(fixture: &Fixture, args: &[&str], ci: Option<&str>) -> (Output, String) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.arg("gate") + .args(args) + .arg("--path") + .arg(&fixture.repo) + .env( + "PATH", + std::env::join_paths([ + fixture.bin.as_path(), + Path::new("/usr/bin"), + Path::new("/bin"), + ]) + .expect("PATH"), + ) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("CARGO_STUB_LOG", fixture.root.join("cargo.log")) + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI"); + if let Some(value) = ci { + cmd.env("CI", value); + } + let out = cmd.output().expect("spawn cargo-tops gate"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +fn report(fixture: &Fixture) -> Value { + let raw = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("report.json was written"); + let doc: Value = serde_json::from_str(&raw).expect("report.json is JSON"); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let errors = schema::validate(&schema, &doc); + assert!( + errors.is_empty(), + "report.json breaks the schema: {errors:?}\n{raw}" + ); + doc +} + +/// The scaffold with `static-lint`, `security-advisory` and `property` on. +fn fixture(name: &str) -> Fixture { + let fixture = scaffold(name); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!(id, "static-lint" | "security-advisory" | "property"); + set(layer, "enabled", Value::Bool(on)); + } + }); + fixture +} + +/// One record for `layer` over `src/**`, expiring on `expires`. +fn exempt(fixture: &Fixture, layer: &str, expires: &str) { + edit(fixture, |doc| { + let record = json!({ + "path": "src/**", + "layer": layer, + "owner": "fixture", + "reason": "planted for the expiry rule", + "expires": expires, + }); + set(doc, "exemptions", json!([record])); + }); +} + +fn check(fixture: &Fixture) -> (Output, String) { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["check", "--path"]) + .arg(&fixture.repo) + .output() + .expect("spawn check"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +/// The report's problems that mention an exemption. +fn exemption_problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|problem| problem.contains("exemption")) + .map(str::to_string) + .collect() +} + +#[test] +fn an_expired_record_for_a_layer_that_is_off_fails_the_gate() { + // `miri` is off in this instance and above `--max-tier 1` besides. + let fixture = fixture("expired-miri"); + exempt(&fixture, "miri", "2020-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + let report = report(&fixture); + assert_eq!(at(&report, "/exit_code"), 1, "{report}"); + assert_eq!(at(&report, "/kind"), "fail", "{report}"); + let problems = exemption_problems(&report); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!( + problems.iter().all(|p| p.starts_with("miri:") + && p.contains("src/**") + && p.contains("expired on 2020-01-01")), + "{problems:?}" + ); + assert!(text.contains("expired on 2020-01-01"), "{text}"); +} + +#[test] +fn an_expired_record_fails_for_every_layer_and_every_state_of_it() { + // On and in range, on and above the range, and off. + for layer in [ + "static-lint", + "security-advisory", + "miri", + "fuzz", + "mutation", + "coverage", + "crap", + "acceptance", + ] { + for tier in ["0", "1"] { + let fixture = fixture("expired-any"); + exempt(&fixture, layer, "2020-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", tier], None); + assert_eq!(out.status.code(), Some(1), "{layer} at {tier}: {text}"); + let problems = exemption_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{layer} at {tier}: {problems:?}"); + } + } +} + +#[test] +fn a_record_whose_date_is_not_a_day_fails() { + for date in ["2027-02-30", "not-a-date", "2024-13-01", "2099-1-1"] { + let fixture = fixture("not-a-day"); + exempt(&fixture, "miri", date); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(1), "{date}: {text}"); + let problems = exemption_problems(&report(&fixture)); + assert!( + problems + .iter() + .all(|p| p.contains(date) && p.contains("not a day")), + "{date}: {problems:?}" + ); + assert_eq!(problems.len(), 1, "{date}: {problems:?}"); + } +} + +#[test] +fn a_live_record_passes() { + let fixture = fixture("live"); + for layer in ["miri", "mutation", "static-lint"] { + exempt(&fixture, layer, "2099-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(0), "{layer}: {text}"); + assert!(exemption_problems(&report(&fixture)).is_empty(), "{layer}"); + } +} + +#[test] +fn an_expired_record_is_not_a_config_error() { + // Exit 1 with a report of kind `fail`, never 2. + let fixture = fixture("not-config"); + exempt(&fixture, "miri", "2020-01-01"); + let (out, _) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(1)); + assert_eq!(at(&report(&fixture), "/kind"), "fail"); +} + +#[test] +fn every_expired_record_is_named_not_only_the_first() { + let fixture = fixture("two-expired"); + edit(&fixture, |doc| { + let record = |layer: &str, expires: &str| { + json!({ + "path": "src/**", "layer": layer, "owner": "fixture", + "reason": "planted for this test", "expires": expires, + }) + }; + let records = json!([ + record("miri", "2020-01-01"), + record("fuzz", "2021-06-30"), + record("crap", "2099-01-01"), + ]); + set(doc, "exemptions", records); + }); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + let problems = exemption_problems(&report(&fixture)); + assert_eq!(problems.len(), 2, "{problems:?}"); + assert!( + problems.iter().any(|p| p.starts_with("miri:")), + "{problems:?}" + ); + assert!( + problems.iter().any(|p| p.starts_with("fuzz:")), + "{problems:?}" + ); +} + +#[test] +fn check_warns_about_an_expired_record_and_still_exits_0() { + let fixture = fixture("check-warns"); + exempt(&fixture, "miri", "2020-01-01"); + let (out, text) = check(&fixture); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(text.contains("expired on 2020-01-01"), "{text}"); + assert!(text.to_lowercase().contains("warn"), "{text}"); + // The same file, with a live record, has no warning. + exempt(&fixture, "miri", "2099-01-01"); + let (out, text) = check(&fixture); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(!text.contains("expired"), "{text}"); +} + +#[test] +fn a_not_a_day_date_is_a_check_warning_too() { + let fixture = fixture("check-not-a-day"); + exempt(&fixture, "miri", "2027-02-30"); + let (out, text) = check(&fixture); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(text.contains("2027-02-30"), "{text}"); +} diff --git a/crates/cargo-tops/tests/gate_survivors.rs b/crates/cargo-tops/tests/gate_survivors.rs new file mode 100644 index 0000000..bf08be1 --- /dev/null +++ b/crates/cargo-tops/tests/gate_survivors.rs @@ -0,0 +1,581 @@ +//! The survivors ledger. After `cargo mutants` runs, the gate reads +//! `mutants.out/outcomes.json` (pinned to cargo-mutants 27.1.0) and judges +//! the missed mutants: one that a live `layer: mutation` exemption covers +//! (same rule as coverage and crap) is explained, the rest are not, and the +//! layer passes when the unexplained count is at most +//! `gates.mutation.unexplained_survivors_max`. Timeouts and unviable mutants +//! are counted, not survivors. A file that is missing, unreadable or of an +//! unknown shape fails the layer. +//! +//! `cargo` is a stub that, for `mutants`, copies a planted `outcomes.json` +//! into `mutants.out/` and exits as cargo-mutants does (2 when mutants were +//! missed, 3 when some timed out). The planted files come from the fixture +//! recorded from a real 27.1.0 run. Every report is validated against the +//! schema. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::{json, Value}; + +/// The validator `cargo tops check` uses, and the schema it reads. +#[path = "../src/schema.rs"] +mod schema; + +mod profiles { + pub(crate) const SCHEMA_JSON: &str = include_str!("../templates/rust-tops.schema.json"); +} + +const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); + +/// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// passes everything else. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +if [ "$1" = "mutants" ]; then + if [ -n "${MUTANTS_OUTCOMES:-}" ]; then + mkdir -p mutants.out && cp "$MUTANTS_OUTCOMES" mutants.out/outcomes.json + fi + exit "${MUTANTS_EXIT:-0}" +fi +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + home: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn at<'v>(value: &'v Value, pointer: &str) -> &'v Value { + static NULL: Value = Value::Null; + value.pointer(pointer).unwrap_or(&NULL) +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +fn git(fixture: &Fixture, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(dir) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_AUTHOR_NAME", "fixture") + .env("GIT_AUTHOR_EMAIL", "fixture@example.invalid") + .env("GIT_COMMITTER_NAME", "fixture") + .env("GIT_COMMITTER_EMAIL", "fixture@example.invalid") + .output() + .expect("spawn git"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn set(doc: &mut Value, key: &str, value: Value) { + doc.as_object_mut() + .expect("an object") + .insert(key.to_string(), value); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A library-core instance named `fixture`, in a directory of its own with +/// a `Cargo.toml`, one source file and stub tools, not yet a git repository. +fn scaffold(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-base-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + home: root.join("home"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir bin"); + fs::create_dir_all(&fixture.home).expect("mkdir home"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); + fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); + stub(&fixture.bin, "cargo", CARGO_STUB); + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); + } + fixture +} + +/// The scaffold with `static-lint`, `security-advisory`, `property` and +/// `mutation` on, in a git repository whose one branch is `branch` and whose +/// one commit is everything `init` wrote. There is no remote. +fn fixture(name: &str, branch: &str) -> Fixture { + let fixture = scaffold(name); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!( + id, + "static-lint" | "security-advisory" | "property" | "mutation" + ); + set(layer, "enabled", Value::Bool(on)); + } + }); + git(&fixture, &fixture.repo, &["init", "-q", "-b", branch]); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "adopt"]); + fixture +} + +/// A commit that adds `file` under `src/`. +fn commit_file(fixture: &Fixture, file: &str) -> String { + fs::write(fixture.repo.join("src").join(file), format!("// {file}\n")).expect("write"); + git(fixture, &fixture.repo, &["add", "-A"]); + git(fixture, &fixture.repo, &["commit", "-q", "-m", file]); + git(fixture, &fixture.repo, &["rev-parse", "HEAD"]) +} + +/// What a clone leaves: a remote `origin` holding `branch`, with +/// `refs/remotes/origin/HEAD` pointing at it. +/// `refs/remotes/origin/` at `commit`. +fn gate(fixture: &Fixture, args: &[&str], ci: Option<&str>) -> (Output, String) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.arg("gate") + .args(args) + .arg("--path") + .arg(&fixture.repo) + .env( + "PATH", + std::env::join_paths([ + fixture.bin.as_path(), + Path::new("/usr/bin"), + Path::new("/bin"), + ]) + .expect("PATH"), + ) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("CARGO_STUB_LOG", fixture.root.join("cargo.log")) + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI"); + if let Some(value) = ci { + cmd.env("CI", value); + } + let out = cmd.output().expect("spawn cargo-tops gate"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +fn report(fixture: &Fixture) -> Value { + let raw = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("report.json was written"); + let doc: Value = serde_json::from_str(&raw).expect("report.json is JSON"); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let errors = schema::validate(&schema, &doc); + assert!( + errors.is_empty(), + "report.json breaks the schema: {errors:?}\n{raw}" + ); + doc +} + +fn mutation(report: &Value) -> &Value { + at(report, "/layers") + .as_array() + .and_then(|layers| layers.iter().find(|row| at(row, "/id") == "mutation")) + .unwrap_or_else(|| at(report, "/missing")) +} + +/// The entries of `problems` that start with `mutation:`. +fn mutation_problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|problem| problem.starts_with("mutation:")) + .map(str::to_string) + .collect() +} + +/// The real 27.1.0 run: 11 caught, 10 missed (5 in `double`, 5 in +/// `inner::triple`), 1 timeout, 1 unviable. +const RECORDED: &str = include_str!("fixtures/mutants-27.1.0/outcomes.json"); + +/// The recorded run with `change` applied, planted under the fixture. +fn planted(fixture: &Fixture, change: impl FnOnce(&mut Value)) -> PathBuf { + let mut doc: Value = serde_json::from_str(RECORDED).expect("fixture is JSON"); + change(&mut doc); + let path = fixture.root.join("outcomes.json"); + fs::write(&path, doc.to_string()).expect("plant outcomes.json"); + path +} + +type Edit = fn(&mut Value); + +fn outcomes_mut(doc: &mut Value) -> &mut Vec { + doc.get_mut("outcomes") + .and_then(Value::as_array_mut) + .expect("outcomes") +} + +/// Keep only the missed mutants for which `keep(function)` holds, and +/// correct the totals. +fn keep_missed(doc: &mut Value, keep: impl Fn(&str) -> bool) { + outcomes_mut(doc).retain(|outcome| { + let function = at(outcome, "/scenario/Mutant/function/function_name") + .as_str() + .unwrap_or_default(); + at(outcome, "/summary") != "MissedMutant" || keep(function) + }); + let missed = outcomes_mut(doc) + .iter() + .filter(|outcome| at(outcome, "/summary") == "MissedMutant") + .count(); + set(doc, "missed", json!(missed)); +} + +/// One survivor in `double` and one in `inner::triple`. +fn one_each(doc: &mut Value) { + let mut seen = Vec::new(); + outcomes_mut(doc).retain(|outcome| { + let function = at(outcome, "/scenario/Mutant/function/function_name") + .as_str() + .unwrap_or_default() + .to_string(); + if at(outcome, "/summary") != "MissedMutant" { + return true; + } + let first = !seen.contains(&function); + seen.push(function); + first + }); + set(doc, "missed", json!(2)); +} + +/// Run the gate with `outcomes` planted, the way cargo-mutants exits for it. +fn gate_with(fixture: &Fixture, outcomes: Option<&Path>, exit: &str) -> (Output, String) { + git(fixture, &fixture.repo, &["checkout", "-q", "-b", "feature"]); + commit_file(fixture, "b.rs"); + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.args(["gate", "--base", "HEAD~1", "--path"]) + .arg(&fixture.repo) + .env("PATH", stub_path(fixture)) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("MUTANTS_EXIT", exit) + .env_remove("MUTANTS_OUTCOMES") + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI"); + if let Some(path) = outcomes { + cmd.env("MUTANTS_OUTCOMES", path); + } + let out = cmd.output().expect("spawn cargo-tops gate"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out, text) +} + +fn stub_path(fixture: &Fixture) -> std::ffi::OsString { + std::env::join_paths([ + fixture.bin.as_path(), + Path::new("/usr/bin"), + Path::new("/bin"), + ]) + .expect("PATH") +} + +/// A live `layer: mutation` record for `path`, and `symbol` when given. +fn exempt(fixture: &Fixture, path: &str, symbol: Option<&str>, expires: &str) { + edit(fixture, |doc| { + let mut record = json!({ + "path": path, + "layer": "mutation", + "owner": "fixture", + "reason": "a planted survivor", + "expires": expires, + }); + if let Some(symbol) = symbol { + set(&mut record, "symbol", json!(symbol)); + } + set(doc, "exemptions", json!([record])); + }); +} + +fn set_max(fixture: &Fixture, max: u64) { + edit(fixture, |doc| { + let mutation = doc.pointer_mut("/gates/mutation").expect("gates.mutation"); + set(mutation, "unexplained_survivors_max", json!(max)); + }); +} + +/// The row's `judged` numbers, in the order the SPEC names them. +fn counts(row: &Value) -> [u64; 6] { + [ + "caught", + "missed", + "explained", + "unexplained", + "timeouts", + "unviable", + ] + .map(|key| { + at(row, &format!("/judged/{key}")) + .as_u64() + .unwrap_or(u64::MAX) + }) +} + +fn expect_failed_naming(text: &str, report: &Value, named: &str, not_named: &str) { + let problems = mutation_problems(report); + assert_eq!(problems.len(), 1, "{problems:?}\n{text}"); + assert!(problems.iter().all(|p| p.contains(named)), "{problems:?}"); + assert!( + problems.iter().all(|p| !p.contains(not_named)), + "{problems:?}" + ); +} + +#[test] +fn one_explained_and_one_unexplained_survivor_fails_naming_only_the_second() { + let fixture = fixture("one-each", "main"); + exempt(&fixture, "src/**", Some("double"), "2099-01-01"); + let file = planted(&fixture, one_each); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "fail", "{row}"); + expect_failed_naming(&text, &report, "inner::triple", "double"); + // caught 11, missed 2 (1 explained, 1 not), 1 timeout, 1 unviable. + assert_eq!(counts(row), [11, 2, 1, 1, 1, 1], "{row}"); + assert_eq!(at(row, "/judged/unexplained_survivors_max"), 0, "{row}"); +} + +#[test] +fn survivors_that_every_record_explains_pass() { + let fixture = fixture("both-explained", "main"); + edit(&fixture, |doc| { + let record = |symbol: &str| { + json!({ + "path": "src/lib.rs", + "symbol": symbol, + "layer": "mutation", + "owner": "fixture", + "reason": "planted for this test", + "expires": "2099-01-01", + }) + }; + set( + doc, + "exemptions", + json!([record("double"), record("inner::triple")]), + ); + }); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(0), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{row}"); + assert_eq!(counts(row), [11, 10, 10, 0, 1, 1], "{row}"); + assert!(mutation_problems(&report).is_empty()); +} + +#[test] +fn a_record_with_no_symbol_covers_every_function_in_its_glob() { + let fixture = fixture("no-symbol", "main"); + exempt(&fixture, "src/lib.rs", None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert_eq!(counts(mutation(&report(&fixture))), [11, 10, 10, 0, 1, 1]); +} + +#[test] +fn a_record_for_another_file_symbol_or_layer_explains_nothing() { + for (path, symbol, layer) in [ + ("src/other.rs", Some("double"), "mutation"), + ("src/lib.rs", Some("nothing"), "mutation"), + ("src/lib.rs", Some("double"), "crap"), + ] { + let fixture = fixture("explains-nothing", "main"); + edit(&fixture, |doc| { + let mut record = json!({ + "path": path, "layer": layer, "owner": "fixture", + "reason": "planted for this test", "expires": "2099-01-01", + }); + set(&mut record, "symbol", json!(symbol)); + set(doc, "exemptions", json!([record])); + }); + let file = planted(&fixture, |doc| keep_missed(doc, |f| f == "double")); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!( + out.status.code(), + Some(1), + "{path} {symbol:?} {layer}: {text}" + ); + let report = report(&fixture); + assert_eq!(counts(mutation(&report))[2], 0, "{path} {symbol:?} {layer}"); + } +} + +#[test] +fn an_expired_mutation_record_explains_nothing() { + let fixture = fixture("expired-record", "main"); + exempt(&fixture, "src/lib.rs", None, "2020-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert_eq!(counts(mutation(&report(&fixture)))[2], 0); +} + +#[test] +fn a_maximum_of_one_passes_one_unexplained_survivor_and_fails_two() { + // A crate-wide record lifts the class floor for the max in `check` and, + // naming no file a mutant is in, explains nothing. + let first = fixture("max-one", "main"); + set_max(&first, 1); + exempt(&first, ".", None, "2099-01-01"); + let one = planted(&first, |doc| { + keep_missed(doc, |f| f == "double"); + outcomes_mut(doc).retain(|o| { + at(o, "/summary") != "MissedMutant" || at(o, "/scenario/Mutant/replacement") == "0" + }); + set(doc, "missed", json!(1)); + }); + let (out, text) = gate_with(&first, Some(&one), "2"); + assert_eq!(out.status.code(), Some(0), "{text}"); + let report = report(&first); + let row = mutation(&report); + assert_eq!(counts(row), [11, 1, 0, 1, 1, 1], "{row}"); + assert_eq!(at(row, "/judged/unexplained_survivors_max"), 1); + + let second = fixture("max-one-two", "main"); + set_max(&second, 1); + exempt(&second, ".", None, "2099-01-01"); + let two = planted(&second, one_each); + let (out, text) = gate_with(&second, Some(&two), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); +} + +#[test] +fn timeouts_and_unviable_mutants_are_counted_and_are_not_survivors() { + let fixture = fixture("not-survivors", "main"); + let file = planted(&fixture, |doc| keep_missed(doc, |_| false)); + let (out, text) = gate_with(&fixture, Some(&file), "3"); + assert_eq!(out.status.code(), Some(0), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{row}"); + assert_eq!(counts(row), [11, 0, 0, 0, 1, 1], "{row}"); +} + +#[test] +fn an_unknown_shape_fails_with_a_message_and_never_passes() { + let cases: [(&str, Edit); 3] = [ + ("another release", |d| { + set(d, "cargo_mutants_version", json!("99.0.0")) + }), + ("an unknown outcome", |d| { + if let Some(first) = outcomes_mut(d).get_mut(1) { + set(first, "summary", json!("Skipped")); + } + }), + ("totals that disagree", |d| set(d, "missed", json!(0))), + ]; + for (what, change) in cases { + let fixture = fixture("unknown-shape", "main"); + let file = planted(&fixture, change); + let (out, text) = gate_with(&fixture, Some(&file), "0"); + assert_eq!(out.status.code(), Some(1), "{what}: {text}"); + let report = report(&fixture); + assert_eq!(at(mutation(&report), "/verdict"), "fail", "{what}"); + let problems = mutation_problems(&report); + assert!( + problems.iter().all(|p| p.contains("outcomes.json")), + "{what}: {problems:?}" + ); + assert_eq!(problems.len(), 1, "{what}: {problems:?}"); + } +} + +#[test] +fn a_file_that_is_not_json_fails() { + let fixture = fixture("not-json", "main"); + let path = fixture.root.join("garbage.json"); + fs::write(&path, "not json").expect("plant"); + let (out, text) = gate_with(&fixture, Some(&path), "0"); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!(text.contains("outcomes.json"), "{text}"); +} + +#[test] +fn no_outcomes_file_is_a_failure_and_a_stale_one_is_never_read() { + let fixture = fixture("no-outcomes", "main"); + // An earlier run left a clean file behind. This run wrote none. + let stale = fixture.repo.join("mutants.out"); + fs::create_dir_all(&stale).expect("mkdir"); + let clean = planted(&fixture, |doc| keep_missed(doc, |_| false)); + fs::copy(&clean, stale.join("outcomes.json")).expect("plant stale outcomes"); + let (out, text) = gate_with(&fixture, None, "0"); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!(text.contains("outcomes.json"), "{text}"); +} + +#[test] +fn a_dry_run_reads_no_outcomes() { + let fixture = fixture("no-run", "main"); + let (out, text) = gate(&fixture, &["--dry-run"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!(!fixture.repo.join("mutants.out").exists()); +} From 06beedb0d2a839b04f7bb17e93afb861dd2de817 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 00:43:19 -0500 Subject: [PATCH 03/10] Judge the survivors ledger and every exemption record The mutation layer now removes any mutants.out/outcomes.json before cargo mutants runs, treats exit 0, 2 and 3 as a finished run, and reads the file that run wrote. Survivors are matched to live layer: mutation records and judged against unexplained_survivors_max; the row's judged carries the counts. An unreadable or other-release file fails the layer. Every exemption record is checked whatever its layer, on or off, at any max-tier: an expired or not-a-day date is a problem and exit 1. check prints the same lines as warnings and exits 0. Carried from the base review: a row with a missing tool or a dry run words the base without an outcome; the closing fail line counts commands that ran as the count line does; origin/HEAD naming a tree reads "not a commit"; the report is written before the summary, and a failed write ends the output with fail:; the per-layer verdict moves to judge.rs; the help and the report schema say what exit 2 covers. PROTOCOL 8.3 and 19 say how the ledger and the records are read. The cargo stubs of gate_base and laws_gate write what 27.1.0 writes for a run with no mutants, as the real tool always does. Co-Authored-By: Claude Sonnet 5.5 --- PROTOCOL.md | 10 +- crates/cargo-tops/src/exemptions.rs | 27 ++-- crates/cargo-tops/src/gate/execute.rs | 129 ++++++++++++++++--- crates/cargo-tops/src/gate/judge.rs | 158 +++++++++++++++++++++++- crates/cargo-tops/src/gate/mod.rs | 11 +- crates/cargo-tops/src/gate/report.rs | 170 +++++++++++++------------- crates/cargo-tops/src/main.rs | 32 +++-- crates/cargo-tops/src/mutation.rs | 68 ++++++++++- crates/cargo-tops/src/steps.rs | 78 ++++++++++-- crates/cargo-tops/tests/gate_base.rs | 114 +++++++++++++++++ crates/cargo-tops/tests/laws_gate.rs | 5 + schema/gate-report.schema.json | 2 +- 12 files changed, 657 insertions(+), 147 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index 6fdcde7..b12ed5d 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -497,11 +497,13 @@ The first source that is set wins. A ref that is named by the flag or the key an `CI` counts as set when it is not empty and is not `0` or `false` (in any case). When no source gives a base, the layer is `fail` if `CI` is set. The report's `detail` and the gate's problem line name `fetch-depth: 0` and `--base`: a checkout that never fetched the base must not read as a clean diff. When `CI` is not set, the layer is `vacuous`, and the detail lists each source tried. A base that resolves but shares no history with `HEAD`, as in a shallow checkout, fails the diff and names `fetch-depth: 0` too. CI workflows pass `--base "origin/${{ github.base_ref || github.event.repository.default_branch }}"` from a checkout with `fetch-depth: 0`, because `actions/checkout` may not create `origin/HEAD`. +How the gate judges the layer: it removes any `mutants.out/outcomes.json` from an earlier run, runs `cargo mutants`, and reads the `outcomes.json` that run wrote. The file must be in the shape cargo-mutants 27.1.0 writes; another release, a missing field, an outcome the gate does not know, a failed baseline, or totals that disagree with the listed outcomes fail the layer and name `outcomes.json`. cargo-mutants exits 2 when mutants were missed and 3 when some timed out; the gate treats 0, 2 and 3 as a finished run and judges it from the file, and any other exit is a failed command. Each missed mutant is explained when a live `layer: mutation` exemption (§19) covers its file and function. Timeouts and unviable mutants are counted and reported; they are not survivors. The layer fails when the survivors nothing explains exceed `gates.mutation.unexplained_survivors_max`, and the problem names each of them. `kill_rate_min_pct` is a floor `check` compares with the class profile; the gate does not judge it. The mutation row's `judged` carries `caught`, `missed`, `explained`, `unexplained`, `timeouts`, `unviable` and `unexplained_survivors_max`, beside `base`. + Pass criteria: - baseline green -- **0 unexplained survivors** in the diff -- kill rate on examined mutants ≥ class minimum (see §4) +- **0 unexplained survivors** in the diff (`unexplained_survivors_max`) +- kill rate on examined mutants ≥ class minimum (see §4; declarative) A survivor is explained only if *all* of the following hold: @@ -981,7 +983,9 @@ exemptions: Expired exemptions fail CI. Reasons shorter than one clause are rejected. -How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. +`cargo tops gate` checks every record, whatever its `layer`, whether that layer is on or off, and at any `--max-tier`: a record that has expired, or whose `expires` is not a real day, is a problem line in the report and the gate exits 1 (never 2; the config is still valid). `cargo tops check` prints the same lines as warnings and exits 0. + +How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains the survivors in the matching files, or, with `symbol`, in the function of that name; a record with no `symbol` covers every function in its glob. A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. --- diff --git a/crates/cargo-tops/src/exemptions.rs b/crates/cargo-tops/src/exemptions.rs index 1697bd6..6b59565 100644 --- a/crates/cargo-tops/src/exemptions.rs +++ b/crates/cargo-tops/src/exemptions.rs @@ -58,24 +58,27 @@ impl Exemption { } } -/// The records in `exemptions` whose `layer` is `layer`. A record without a -/// string `path` or `expires` is skipped; the schema requires both. +/// One record, when it has a string `path` and `expires`; the schema +/// requires both. +fn record_of(entry: &Value) -> Option { + Some(Exemption { + path: entry.get("path")?.as_str()?.to_string(), + symbol: entry + .get("symbol") + .and_then(Value::as_str) + .map(str::to_string), + expires: entry.get("expires")?.as_str()?.to_string(), + }) +} + +/// The records in `exemptions` whose `layer` is `layer`. pub(crate) fn for_layer(exemptions: Option<&Value>, layer: &str) -> Vec { exemptions .and_then(Value::as_array) .into_iter() .flatten() .filter(|entry| entry.get("layer").and_then(Value::as_str) == Some(layer)) - .filter_map(|entry| { - Some(Exemption { - path: entry.get("path")?.as_str()?.to_string(), - symbol: entry - .get("symbol") - .and_then(Value::as_str) - .map(str::to_string), - expires: entry.get("expires")?.as_str()?.to_string(), - }) - }) + .filter_map(record_of) .collect() } diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index 285f4bb..a3163a3 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -2,7 +2,7 @@ //! and runs none. use std::collections::HashMap; use std::path::Path; -use std::process::Command; +use std::process::{Command, ExitStatus}; use std::time::{Instant, SystemTime}; use anyhow::{Context, Result}; @@ -12,6 +12,7 @@ use super::{Fault, Mode}; use crate::config::Layer; use crate::evidence; use crate::facts::{Outcome, Run}; +use crate::mutation::{self, Outcomes}; use crate::steps::{self, Base, Kind, MutationPlan, Step}; use crate::{laws, TOOL_LOCK}; @@ -56,6 +57,9 @@ pub(crate) struct Executed<'a> { pub(crate) not_run: Vec, /// Commands named, whether or not they ran. pub(crate) listed: usize, + /// What `cargo mutants` left in this run's `outcomes.json`: `None` when it + /// did not finish, `Err` when the file is not the shape [`mutation`] reads. + pub(crate) found: Option>, } /// The pinned Bend is fetched first: a missing Bend fails before any work. @@ -136,6 +140,7 @@ pub(crate) fn run<'a>(dir: &Path, plan: &GatePlan<'a>, mode: Mode) -> Result Runner<'_, 'a> { self.done.listed += 1; if self.live { let clock = Clock::start(); - let run = run_mutation(self.dir, &layer.id, mutation).map_err(Fault::env)?; + let Mutated { run, found } = + run_mutation(self.dir, &layer.id, mutation).map_err(Fault::env)?; + self.done.found = found; // The tool is asked for its version only when it ran: a vacuous // or failed diff runs no `cargo mutants`. let version = (run.outcome != Outcome::ToolMissing @@ -411,33 +418,67 @@ pub(crate) fn run_pinned_crap(dir: &Path, layer: &str, chain: &[Step], pin: &str Ok(run) } +/// A mutation layer's run, and what its `outcomes.json` held. +pub(crate) struct Mutated { + pub(crate) run: Run, + pub(crate) found: Option>, +} + +impl Mutated { + fn without_outcomes(run: Run) -> Self { + Self { run, found: None } + } +} + /// `cargo-mutants` is looked up before the diff, so an empty diff cannot /// hide a missing tool. A base that was named and is not a commit fails the /// layer before any command runs. A failed diff is `Failed`; nothing waives /// it. An empty one is `Vacuous` and `cargo mutants` does not run, except /// that no base at all fails under CI. [`steps::stop_note`] words every one -/// of these stops. -pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Result { +/// of these stops. When `cargo mutants` runs, an `outcomes.json` from an +/// earlier run is removed first, and exit 0, 2 (missed) and 3 (timeouts) +/// are a finished run whose ledger the judge reads. +pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Result { if let Err(missing) = steps::check_tool(&plan.mutants.argv) { println!("{missing}"); - return Ok(finish( + return Ok(Mutated::without_outcomes(finish( layer, &[plan.mutants.display()], Outcome::ToolMissing, - )); + ))); } if matches!(plan.base, Base::Unresolved { .. }) { - return Ok(stopped(layer, &[], &plan.base, Outcome::Failed)); + return Ok(Mutated::without_outcomes(stopped( + layer, + &[], + &plan.base, + Outcome::Failed, + ))); } let mut ran = vec![plan.diff.display()]; let diff_ok = run_step(dir, &plan.diff)? == Outcome::Passed; let diff = steps::judge_diff(diff_ok, &plan.diff_path); let Some(mutants) = plan.after_diff(diff) else { - return Ok(stopped(layer, &ran, &plan.base, diff)); + return Ok(Mutated::without_outcomes(stopped( + layer, &ran, &plan.base, diff, + ))); }; ran.push(mutants.display()); - let outcome = run_step(dir, mutants)?; - Ok(finish(layer, &ran, outcome)) + mutation::clear(dir)?; + let outcome = run_mutants(dir, mutants)?; + let mut run = finish(layer, &ran, outcome); + if outcome != Outcome::Passed { + return Ok(Mutated::without_outcomes(run)); + } + let found = mutation::read(dir); + if let Err(why) = &found { + run.outcome = Outcome::Failed; + run.note = Some(why.clone()); + } + Ok(Mutated { + run, + found: Some(found), + }) } /// The run of a layer that stopped on its base or its diff: prints the @@ -474,12 +515,39 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { } Kind::Command => {} } + Ok(match spawn(dir, step)? { + Spawned::Exited(status) if status.success() => Outcome::Passed, + Spawned::Exited(_) | Spawned::NoProgram => Outcome::Failed, + Spawned::Missing => Outcome::ToolMissing, + }) +} + +/// `cargo mutants` exits 2 when mutants were missed and 3 when some timed +/// out: those and 0 are a finished run, judged by the ledger. Any other exit +/// is a failed command. +fn run_mutants(dir: &Path, step: &Step) -> Result { + Ok(match spawn(dir, step)? { + Spawned::Exited(status) if matches!(status.code(), Some(0 | 2 | 3)) => Outcome::Passed, + Spawned::Exited(_) | Spawned::NoProgram => Outcome::Failed, + Spawned::Missing => Outcome::ToolMissing, + }) +} + +/// How a spawned command ended. +enum Spawned { + Exited(ExitStatus), + /// The program is not there; the missing-tool line is already printed. + Missing, + /// The step names no program. + NoProgram, +} + +fn spawn(dir: &Path, step: &Step) -> Result { let Some((program, args)) = step.program() else { - return Ok(Outcome::Failed); + return Ok(Spawned::NoProgram); }; match Command::new(program).args(args).current_dir(dir).status() { - Ok(status) if status.success() => Ok(Outcome::Passed), - Ok(_) => Ok(Outcome::Failed), + Ok(status) => Ok(Spawned::Exited(status)), Err(err) if err.kind() == std::io::ErrorKind::NotFound => { println!( "{}", @@ -487,7 +555,7 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { binary: program.to_string() } ); - Ok(Outcome::ToolMissing) + Ok(Spawned::Missing) } Err(err) => Err(err).with_context(|| format!("spawn: {}", step.display())), } @@ -495,7 +563,7 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { #[cfg(test)] mod tests { - use super::{run_step, version_ask, Executed, Timing}; + use super::{run_mutants, run_step, version_ask, Executed, Timing}; use crate::facts::Outcome; use crate::steps::Step; use std::fs; @@ -541,6 +609,7 @@ mod tests { shared: Vec::new(), not_run: Vec::new(), listed: 0, + found: None, }; assert!((done.seconds("a") - 3.5).abs() < 1e-9); assert!((done.seconds("b") - 4.0).abs() < 1e-9); @@ -566,4 +635,34 @@ mod tests { assert!(format!("{err:#}").contains("spawn"), "{err:#}"); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn cargo_mutants_exits_0_2_and_3_are_a_finished_run() { + let wants = [ + (0, Outcome::Passed), + (1, Outcome::Failed), + (2, Outcome::Passed), + (3, Outcome::Passed), + (4, Outcome::Failed), + (101, Outcome::Failed), + ]; + for (code, want) in wants { + let step = Step::command(["sh".to_string(), "-c".to_string(), format!("exit {code}")]); + let got = run_mutants(Path::new("."), &step).expect("it ran"); + assert_eq!(got, want, "exit {code}"); + // Any other command still fails on every nonzero exit. + let plain = run_step(Path::new("."), &step).expect("it ran"); + let plain_want = if code == 0 { + Outcome::Passed + } else { + Outcome::Failed + }; + assert_eq!(plain, plain_want, "exit {code}"); + } + let nowhere = Step::command(["/no/such/program/here"]); + assert_eq!( + run_mutants(Path::new("."), &nowhere).expect("an outcome"), + Outcome::ToolMissing + ); + } } diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 53313c2..034e666 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -11,7 +11,9 @@ use super::plan::GatePlan; use super::{Fault, Mode}; use crate::config::{self, TopsConfig}; use crate::facts::{self, Outcome, Run}; -use crate::{config_path, floors, laws, profiles, steps}; +use crate::mutation::Ledger; +use crate::scope::crate_root; +use crate::{config_path, exemptions, floors, laws, mutation, profiles, steps}; /// What the judging found. pub(crate) struct Judgement { @@ -21,8 +23,10 @@ pub(crate) struct Judgement { pub(crate) evaluated: String, /// The `count:` line. pub(crate) count: String, - /// Commands that ran, shared layers excluded. + /// Runs, shared layers excluded. pub(crate) ran: usize, + /// Commands that ran: a missing tool ran none, `bend` is one. + pub(crate) commands: usize, pub(crate) vacuous: usize, /// The numbers the coverage floors and the CRAP ratchet judged. pub(crate) measured: floors::Judged, @@ -33,6 +37,30 @@ pub(crate) struct Judgement { pub(crate) environment: bool, /// The laws in the pack, checked or not. pub(crate) law_total: usize, + /// The survivors `cargo mutants` left, sorted by the records that cover + /// them, when it finished. + pub(crate) survivors: Option, +} + +impl Judgement { + /// Whether a floor, or the survivor maximum, of layer `id` was missed. + pub(crate) fn missed(&self, id: &str) -> bool { + match id { + "coverage" => self.measured.coverage_problems > 0, + "crap" => self.measured.crap_problems > 0, + "mutation" => self + .survivors + .as_ref() + .is_some_and(|found| !found.ledger.passes(found.max)), + _ => false, + } + } +} + +/// The mutation ledger and the maximum it was judged against. +pub(crate) struct Survivors { + pub(crate) ledger: Ledger, + pub(crate) max: i64, } /// What `judge` reads. @@ -69,7 +97,7 @@ pub(crate) fn judge( ) -> Result { let total = law_total(dir); if mode == Mode::DryRun { - return Ok(dry_judgement(plan, done, total)); + return Ok(dry_judgement(cfg, plan, done, total)); } let inputs = Inputs { dir, @@ -78,24 +106,102 @@ pub(crate) fn judge( done, }; let mut failed = run_problems(done); + let survivors = judge_survivors(&inputs, &mut failed); // The coverage floors and the CRAP ratchet, on the reports just written. let measured = floors::judge(dir, cfg, &done.runs, &plan.report); failed.extend(measured.problems.iter().cloned()); + add_expired(cfg, &mut failed); let laws = judge_laws(&inputs, prepared, total, &mut failed)?; - Ok(finish(&inputs, failed, measured, laws, total)) + Ok(finish(&inputs, failed, measured, laws, total, survivors)) +} + +/// One problem for every exemption record that has expired, or whose date +/// is not a day, whatever its layer and whether or not the layer ran (§19). +pub(crate) fn expired_problems(cfg: &TopsConfig) -> Vec { + let today = exemptions::today(); + let records = cfg.exemptions.as_ref(); + let mut layers: Vec<&str> = Vec::new(); + for layer in records + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|entry| entry.get("layer")?.as_str()) + { + if !layers.contains(&layer) { + layers.push(layer); + } + } + layers + .into_iter() + .flat_map(|layer| { + exemptions::for_layer(records, layer) + .into_iter() + .filter(|record| !record.live(&today)) + .map(|record| record.expired_problem(layer)) + .collect::>() + }) + .collect() +} + +/// Add each expired record to `failed`. A record a floor already named is +/// not named twice. +fn add_expired(cfg: &TopsConfig, failed: &mut Vec) { + for problem in expired_problems(cfg) { + if !failed.contains(&problem) { + failed.push(problem); + } + } +} + +/// The crate's directory under the workspace root, which is what the +/// `file` of a mutant is relative to. Empty for a crate at the root. +fn crate_prefix(dir: &Path, name: &str) -> String { + let Some(root) = laws::workspace_root(dir) else { + return String::new(); + }; + crate_root(dir, name) + .strip_prefix(&root) + .map(|rest| rest.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// The survivors of this run's `cargo mutants`, matched to the live +/// `layer: mutation` records, and the problem when more than +/// `unexplained_survivors_max` are left. An outcomes file that could not be +/// read is already the layer's failure. +fn judge_survivors(inputs: &Inputs<'_, '_>, failed: &mut Vec) -> Option { + let Some(Ok(found)) = &inputs.done.found else { + return None; + }; + let cfg = inputs.cfg; + let today = exemptions::today(); + let mut live = exemptions::for_layer(cfg.exemptions.as_ref(), "mutation"); + live.retain(|record| record.live(&today)); + let prefix = crate_prefix(inputs.dir, &cfg.crate_.name); + let max = cfg.gates.mutation.unexplained_survivors_max; + let ledger = mutation::explain(found, &live, &prefix); + failed.extend(ledger.problem(max)); + Some(Survivors { ledger, max }) } /// Nothing ran, so the count is short for every layer: a dry run lists the /// commands and is never a pass. -fn dry_judgement(plan: &GatePlan<'_>, done: &Executed<'_>, total: usize) -> Judgement { +fn dry_judgement( + cfg: &TopsConfig, + plan: &GatePlan<'_>, + done: &Executed<'_>, + total: usize, +) -> Judgement { let (count, _) = command_count(0, &plan.enabled, &[], false, plan.max_tier, &done.shared); let mut failed = done.not_run.clone(); failed.push(count.clone()); + add_expired(cfg, &mut failed); Judgement { failed, evaluated: format!("laws evaluated: 0/{total} (dry run: bend did not run)"), count, ran: 0, + commands: 0, vacuous: 0, measured: floors::Judged::default(), laws_seconds: 0.0, @@ -103,6 +209,7 @@ fn dry_judgement(plan: &GatePlan<'_>, done: &Executed<'_>, total: usize) -> Judg laws_failed: false, environment: false, law_total: total, + survivors: None, } } @@ -130,17 +237,21 @@ fn finish( measured: floors::Judged, laws: LawsOutcome, law_total: usize, + survivors: Option, ) -> Judgement { let (plan, done) = (inputs.plan, inputs.done); // A layer that took another's run ran no command of its own. let ran = done.runs.len() - done.shared.len(); + // The `count:` line and the closing `fail:` line say the same number: + // commands that ran. + let commands = ran_commands(&done.runs, laws.ran, &done.shared); let vacuous = done .runs .iter() .filter(|run| run.outcome == Outcome::Vacuous) .count(); let (count, short) = command_count( - ran_commands(&done.runs, laws.ran, &done.shared), + commands, &plan.enabled, &done.runs, laws.ran, @@ -155,6 +266,7 @@ fn finish( evaluated: laws.evaluated, count, ran, + commands, vacuous, measured, laws_seconds: laws.seconds, @@ -165,6 +277,40 @@ fn finish( .iter() .any(|run| run.outcome == Outcome::ToolMissing), law_total, + survivors, + } +} + +/// The verdict of a layer's runs: a failure outranks a missing tool, which +/// outranks a vacuous run. +fn runs_verdict(runs: &[&Run]) -> &'static str { + let any = |outcome: Outcome| runs.iter().any(|run| run.outcome == outcome); + if any(Outcome::Failed) { + "fail" + } else if any(Outcome::ToolMissing) { + "tool-missing" + } else if any(Outcome::Vacuous) { + "vacuous" + } else { + "pass" + } +} + +/// The row verdict of layer `id`: `not-run` when nothing ran for it, +/// otherwise its worst run. A run that passed still fails the row when its +/// floors, or its survivors, did not. +pub(crate) fn layer_verdict( + done: &Executed<'_>, + judged: Option<&Judgement>, + id: &str, +) -> &'static str { + let runs: Vec<&Run> = done.runs.iter().filter(|run| run.layer == id).collect(); + if runs.is_empty() { + return "not-run"; + } + match runs_verdict(&runs) { + "pass" if judged.is_some_and(|judged| judged.missed(id)) => "fail", + verdict => verdict, } } diff --git a/crates/cargo-tops/src/gate/mod.rs b/crates/cargo-tops/src/gate/mod.rs index 69bc0fa..ebd89a6 100644 --- a/crates/cargo-tops/src/gate/mod.rs +++ b/crates/cargo-tops/src/gate/mod.rs @@ -6,10 +6,13 @@ //! - [`report`] prints the summary, writes `report.json` and picks the exit //! code. //! -//! Exit codes: 0 pass; 1 the gate ran and failed (a dry run included); 2 the -//! config is invalid; 3 the environment (a required tool or `cargo metadata` -//! is missing). Where more than one applies, 2 wins over 3, and 3 over 1; -//! every problem is still listed in the report. +//! Exit codes: 0 pass; 1 the gate ran and failed (a dry run included, and an +//! expired exemption); 2 the config or the request cannot be gated +//! (`rust-tops.yaml` missing or failing check, `rust-tops.lock` unreadable, +//! no enabled layer at or below `--max-tier`); 3 the environment (a required +//! tool or `cargo metadata` is missing, or `report.json` cannot be written). +//! Where more than one applies, 2 wins over 3, and 3 over 1; every problem is +//! still listed in the report. use std::path::Path; use std::process::ExitCode; diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index 279cde0..dea0a11 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -10,11 +10,10 @@ use std::process::{Command, ExitCode}; use serde_json::{json, Map, Value}; use super::execute::Executed; -use super::judge::Judgement; +use super::judge::{self, Judgement}; use super::plan::{GatePlan, What}; use super::{Fault, Mode, Options}; use crate::config::TopsConfig; -use crate::facts::{Outcome, Run}; use crate::floors::{CoverageNumbers, CrapNumbers}; use crate::steps::{self, Base}; use crate::{laws, PROTOCOL_VERSION}; @@ -80,7 +79,6 @@ pub(crate) fn finish( done: &Executed<'_>, judged: &Judgement, ) -> ExitCode { - print_summary(opts, done, judged); let kind = if judged.failed.is_empty() { Kind::Pass } else if judged.environment { @@ -88,7 +86,9 @@ pub(crate) fn finish( } else { Kind::Failed }; - emit(&Parts { + // The report is written before the summary, so the summary's last line + // is true: a report that could not be written ends it with `fail:`. + let (code, unwritten) = write_report(&Parts { dir, cfg: Some(cfg), opts, @@ -98,48 +98,67 @@ pub(crate) fn finish( done: Some(done), judged: Some(judged), report_dir: plan.report.clone(), - }) + }); + print_summary(opts, done, judged, unwritten.as_deref()); + code } -fn print_summary(opts: Options, done: &Executed<'_>, judged: &Judgement) { +/// `unwritten` is the path of a report that could not be written. +fn print_summary(opts: Options, done: &Executed<'_>, judged: &Judgement, unwritten: Option<&Path>) { println!(); println!("{}", judged.evaluated); println!("{}", judged.count); if opts.mode == Mode::DryRun { - if done.not_run.is_empty() { - println!( - "dry-run: {} command(s) listed, 0 run; a dry run is not a pass", - done.listed - ); - } else { - println!( - "dry-run: {} command(s) listed, 0 run; {} layer(s) NotRun, so the gate would fail", - done.listed, - done.not_run.len() - ); - } - for problem in &judged.failed { - println!(" - {problem}"); - } + print_dry_run(done, judged); } else if judged.failed.is_empty() { - if judged.vacuous == 0 { - println!("ok: {} command(s) passed", judged.ran); - } else { - println!( - "ok: {} command(s) passed, {} vacuous (counted for no law)", - judged.ran - judged.vacuous, - judged.vacuous - ); - } + print_pass(judged, unwritten); } else { println!( "fail: {} problem(s), {} command(s) run", judged.failed.len(), - judged.ran + judged.commands + ); + print_problems(judged); + } +} + +fn print_problems(judged: &Judgement) { + for problem in &judged.failed { + println!(" - {problem}"); + } +} + +fn print_dry_run(done: &Executed<'_>, judged: &Judgement) { + if done.not_run.is_empty() { + println!( + "dry-run: {} command(s) listed, 0 run; a dry run is not a pass", + done.listed + ); + } else { + println!( + "dry-run: {} command(s) listed, 0 run; {} layer(s) NotRun, so the gate would fail", + done.listed, + done.not_run.len() + ); + } + print_problems(judged); +} + +/// No problem: `ok`, unless the report that says so could not be written. +fn print_pass(judged: &Judgement, unwritten: Option<&Path>) { + if let Some(path) = unwritten { + println!( + "fail: {} could not be written; the gate has no verdict on record", + path.display() + ); + } else if judged.vacuous == 0 { + println!("ok: {} command(s) passed", judged.ran); + } else { + println!( + "ok: {} command(s) passed, {} vacuous (counted for no law)", + judged.ran - judged.vacuous, + judged.vacuous ); - for problem in &judged.failed { - println!(" - {problem}"); - } } } @@ -210,9 +229,15 @@ struct Parts<'a> { report_dir: PathBuf, } -/// Write the report. A report that cannot be written is an environment -/// fault, unless the config fault is already the exit code. +/// Write the report and return the exit code. fn emit(parts: &Parts<'_>) -> ExitCode { + write_report(parts).0 +} + +/// Write the report. A report that cannot be written is an environment +/// fault, unless the config fault is already the exit code; its path is +/// returned so the summary can say so. +fn write_report(parts: &Parts<'_>) -> (ExitCode, Option) { let mut kind = parts.kind; let path = parts.report_dir.join(REPORT); let wrote = fs::create_dir_all(&parts.report_dir).and_then(|()| { @@ -220,13 +245,15 @@ fn emit(parts: &Parts<'_>) -> ExitCode { serde_json::to_string_pretty(&document(parts, kind)).map_err(std::io::Error::other)?; fs::write(&path, format!("{text}\n")) }); + let mut unwritten = None; if let Err(err) = wrote { eprintln!("error: write {}: {err}", path.display()); if matches!(kind, Kind::Pass | Kind::Failed) { kind = Kind::Env; } + unwritten = Some(path); } - ExitCode::from(kind.code()) + (ExitCode::from(kind.code()), unwritten) } fn git_sha(dir: &Path) -> Option { @@ -349,58 +376,30 @@ fn layer_rows(parts: &Parts<'_>) -> (Vec, Vec) { (rows, disabled) } -/// The verdict of a layer's runs: a failure outranks a missing tool, which -/// outranks a vacuous run. -fn runs_verdict(runs: &[&Run]) -> &'static str { - let any = |outcome: Outcome| runs.iter().any(|run| run.outcome == outcome); - if any(Outcome::Failed) { - "fail" - } else if any(Outcome::ToolMissing) { - "tool-missing" - } else if any(Outcome::Vacuous) { - "vacuous" - } else { - "pass" - } -} - -/// The numbers a layer's floors were judged on, and whether any floor missed. -fn judged_numbers(parts: &Parts<'_>, id: &str) -> (Option, bool) { - let Some(measured) = parts.judged.map(|judged| &judged.measured) else { - return (None, false); - }; +/// The numbers a layer's floors, or its survivors, were judged on. +fn judged_numbers(parts: &Parts<'_>, id: &str) -> Option { + let judged = parts.judged?; match id { - "coverage" => ( - measured.coverage.as_ref().map(CoverageNumbers::to_json), - measured.coverage_problems > 0, - ), - "crap" => ( - measured.crap.as_ref().map(CrapNumbers::to_json), - measured.crap_problems > 0, - ), - _ => (None, false), + "coverage" => judged + .measured + .coverage + .as_ref() + .map(CoverageNumbers::to_json), + "crap" => judged.measured.crap.as_ref().map(CrapNumbers::to_json), + "mutation" => judged + .survivors + .as_ref() + .map(|found| found.ledger.to_json(found.max)), + _ => None, } } -/// `not-run` when nothing ran for the layer; otherwise its worst run. -fn unit_verdict(done: &Executed<'_>, id: &str) -> &'static str { - let runs: Vec<&Run> = done.runs.iter().filter(|run| run.layer == id).collect(); - if runs.is_empty() { - return "not-run"; - } - runs_verdict(&runs) -} - fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { let (Some(plan), Some(done)) = (parts.plan, parts.done) else { return row(id, tier, "not-run", &[], 0.0, None); }; - let (numbers, missed) = judged_numbers(parts, id); - let verdict = match unit_verdict(done, id) { - // The run passed and its floors did not. - "pass" if missed => "fail", - verdict => verdict, - }; + let numbers = judged_numbers(parts, id); + let verdict = judge::layer_verdict(done, parts.judged, id); let version = done.tool_version(id); let mut entry = row( id, @@ -444,11 +443,10 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map json!({ "ref": null, "source": null, "tried": tried }), }; put_judged(entry, "base", base); - let ran = run_note(done, "mutation"); - let planned = (!matches!(mutation.base, Base::Found { .. })) - .then(|| steps::stop_note(&mutation.base, Outcome::Vacuous).map(|(_, note)| note)) - .flatten(); - if let Some(detail) = ran.or(planned) { + // A run that stopped says why in its own words. A row with no such note + // (a missing tool, a dry run) names the base's fault and no outcome. + if let Some(detail) = run_note(done, "mutation").or_else(|| steps::base_finding(&mutation.base)) + { entry.insert("detail".into(), json!(detail)); } } diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 0af9e43..9f061bb 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -93,8 +93,11 @@ enum Commands { /// Run enabled Tier 0–1 tools in protocol order, then the Bend laws layer #[command(after_help = "Writes /rust-tops/report.json on every run.\n\ Exit codes:\n 0 pass\n 1 the gate ran and failed (a dry run included)\n \ -2 the config is invalid (rust-tops.yaml unreadable or failing check)\n \ -3 the environment: a required tool or cargo metadata is missing\n\ +2 the config or the request cannot be gated: rust-tops.yaml missing, unreadable \ +or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; \ +a command-line usage error is 2 too. An expired exemption is not 2: it is 1\n \ +3 the environment: a required tool or cargo metadata is missing, or report.json \ +cannot be written\n\ Where more than one applies, 2 wins over 3, and 3 over 1.")] Gate { #[arg(long, default_value = ".")] @@ -182,15 +185,7 @@ fn run() -> Result { init(&args)?; Ok(ExitCode::SUCCESS) } - Commands::Check { path } => { - let cfg = load_config(&path)?; - check(&cfg)?; - println!( - "ok: {} ({}) matches rust-tops {}", - cfg.crate_.name, cfg.crate_.class, cfg.protocol.version - ); - Ok(ExitCode::SUCCESS) - } + Commands::Check { path } => check_command(&path), Commands::Gate { path, max_tier, @@ -216,6 +211,21 @@ fn run() -> Result { } } +/// `cargo tops check`: an expired exemption is a warning here and a failure +/// of the gate; the config is still valid. +fn check_command(path: &Path) -> Result { + let cfg = load_config(path)?; + check(&cfg)?; + for problem in gate::judge::expired_problems(&cfg) { + println!("warning: {problem}"); + } + println!( + "ok: {} ({}) matches rust-tops {}", + cfg.crate_.name, cfg.crate_.class, cfg.protocol.version + ); + Ok(ExitCode::SUCCESS) +} + /// `cargo tops gate`: a config that does not load, or fails `check`, is exit /// 2 with a report; anything else is the gate's own verdict. fn run_gate(path: &Path, opts: gate::Options, base: Option<&str>) -> ExitCode { diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs index 2af6e7a..e174e8e 100644 --- a/crates/cargo-tops/src/mutation.rs +++ b/crates/cargo-tops/src/mutation.rs @@ -9,12 +9,18 @@ //! count is at most `gates.mutation.unexplained_survivors_max`. Timeouts and //! unviable mutants are counted and reported; they are not survivors. #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] -#![allow(dead_code, reason = "the gate wires these items in G2")] +use std::fs; +use std::path::Path; + +use anyhow::{Context, Result}; use serde_json::{json, Value}; use crate::exemptions::Exemption; +/// Where cargo-mutants writes the run's outcomes, under the directory it ran in. +const OUTCOMES: &str = "mutants.out/outcomes.json"; + /// The cargo-mutants release whose `outcomes.json` this reads. pub(crate) const PINNED: &str = "27.1.0"; @@ -50,6 +56,26 @@ pub(crate) struct Ledger { pub unviable: usize, } +/// Remove the `outcomes.json` an earlier run left, so one found after +/// `cargo mutants` is the one it just wrote. +pub(crate) fn clear(dir: &Path) -> Result<()> { + let path = dir.join(OUTCOMES); + match fs::remove_file(&path) { + Err(err) if err.kind() != std::io::ErrorKind::NotFound => { + Err(err).with_context(|| format!("remove {}", path.display())) + } + _ => Ok(()), + } +} + +/// The outcomes `cargo mutants` just wrote under `dir`. Call [`clear`] before +/// the run: a file that is there afterwards is this run's. +pub(crate) fn read(dir: &Path) -> Result { + let text = fs::read_to_string(dir.join(OUTCOMES)) + .map_err(|err| format!("no {OUTCOMES} after cargo mutants ran: {err}"))?; + parse(&text) +} + fn field<'v>(value: &'v Value, key: &str) -> Result<&'v Value, String> { value.get(key).ok_or_else(|| { format!("outcomes.json has no `{key}` where cargo-mutants {PINNED} writes one") @@ -227,9 +253,10 @@ impl Ledger { reason = "the tests edit and read JSON they built from the recorded fixture" )] mod tests { - use super::{explain, parse, relative, Outcomes, PINNED}; + use super::{clear, explain, parse, read, relative, Outcomes, PINNED}; use crate::exemptions::Exemption; use serde_json::Value; + use std::fs; /// A real run of cargo-mutants 27.1.0 over a small crate: 11 caught, 10 /// missed (5 in `double`, 5 in `inner::triple`), 1 timeout, 1 unviable. @@ -484,4 +511,41 @@ mod tests { 2 ); } + + fn scratch(name: &str) -> std::path::PathBuf { + let dir = + std::env::temp_dir().join(format!("cargo-tops-mutation-{name}-{}", std::process::id())); + fs::remove_dir_all(&dir).ok(); + fs::create_dir_all(&dir).expect("mkdir"); + dir + } + + #[test] + fn an_outcomes_file_from_an_earlier_run_is_removed_and_none_is_nothing() { + let dir = scratch("clear"); + assert!(clear(&dir).is_ok()); + fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); + let file = dir.join("mutants.out/outcomes.json"); + fs::write(&file, FIXTURE).expect("write"); + assert!(clear(&dir).is_ok()); + assert!(!file.exists()); + // A directory where the file should be cannot be removed as a file. + fs::create_dir_all(file.join("inside")).expect("mkdir"); + let err = clear(&dir).expect_err("a directory is not a report"); + assert!(format!("{err:#}").contains("remove"), "{err:#}"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn what_the_run_wrote_is_read_and_a_missing_file_is_an_error() { + let dir = scratch("read"); + let err = read(&dir).expect_err("no file"); + assert!(err.contains("mutants.out/outcomes.json"), "{err}"); + fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); + fs::write(dir.join("mutants.out/outcomes.json"), "not json").expect("write"); + assert!(read(&dir).expect_err("not json").contains("not JSON")); + fs::write(dir.join("mutants.out/outcomes.json"), FIXTURE).expect("write"); + assert_eq!(read(&dir).expect("parses").missed.len(), 10); + fs::remove_dir_all(&dir).ok(); + } } diff --git a/crates/cargo-tops/src/steps.rs b/crates/cargo-tops/src/steps.rs index c27c90e..f3e8d00 100644 --- a/crates/cargo-tops/src/steps.rs +++ b/crates/cargo-tops/src/steps.rs @@ -449,12 +449,25 @@ fn origin_head(dir: &Path) -> Result { Some(name) => Err(format!( "origin/HEAD: points at {name}, which is not a commit" )), - // Not a symbolic ref. A direct ref that names a commit is a base. - None if names_a_commit(dir, "origin/HEAD") => Ok("origin/HEAD".to_string()), - None => Err("origin/HEAD: not set".to_string()), + // Not a symbolic ref: a direct one, or none. + None => direct_head(dir), } } +/// `origin/HEAD` as a direct ref: a base when it names a commit. +fn direct_head(dir: &Path) -> Result { + if names_a_commit(dir, "origin/HEAD") { + return Ok("origin/HEAD".to_string()); + } + let exists = git_line(dir, &["rev-parse", "--verify", "--quiet", "origin/HEAD"]).is_some(); + Err(if exists { + "origin/HEAD: not a commit" + } else { + "origin/HEAD: not set" + } + .to_string()) +} + /// Why the mutation layer stopped before `cargo mutants`, and its outcome. /// This is the one place the words live: stdout, the report's `detail` and /// its `problems` all print what this returns. `diff` is the outcome of the @@ -527,13 +540,30 @@ fn no_base_outside_ci(tried: &[String]) -> (Outcome, String) { ( Outcome::Vacuous, format!( - "no mutation base ({}); the mutation diff is empty; \ + "{}; the mutation diff is empty; \ cargo mutants did not run and no law counts this layer", - tried.join("; ") + no_base(tried) ), ) } +/// What was looked for and not found, with no outcome after it. +fn no_base(tried: &[String]) -> String { + format!("no mutation base ({})", tried.join("; ")) +} + +/// What is wrong with the base, worded with no outcome: a row that never +/// reached the diff (a missing tool, a dry run) says only this. `None` for +/// a base that is a commit. +pub(crate) fn base_finding(base: &Base) -> Option { + match base { + Base::Found { .. } => None, + Base::Missing { tried, ci: false } => Some(no_base(tried)), + Base::Missing { tried, ci: true } => Some(no_base_in_ci(tried).1), + Base::Unresolved { .. } => stop_note(base, Outcome::Failed).map(|(_, note)| note), + } +} + /// The first line git prints for `args`, when it succeeds with one. fn git_line(dir: &Path, args: &[&str]) -> Option { let out = Command::new("git") @@ -985,8 +1015,8 @@ mod expiry { #[cfg(test)] mod base { use super::{ - ci_active, git_line, mutation_diff_script, shell_quote, stop_note, Base, BaseSource, - MutationPlan, Outcome, + base_finding, ci_active, git_line, mutation_diff_script, shell_quote, stop_note, Base, + BaseSource, MutationPlan, Outcome, }; use std::path::Path; @@ -1163,4 +1193,38 @@ mod base { assert_eq!(stop_note(&base, Outcome::Passed), None); } } + + #[test] + fn a_base_finding_names_the_fault_and_no_outcome() { + assert_eq!(base_finding(&found()), None); + let tried = vec![ + "--base: not given".to_string(), + "origin/HEAD: not set".to_string(), + ]; + let outside = Base::Missing { + tried: tried.clone(), + ci: false, + }; + assert_eq!( + base_finding(&outside).as_deref(), + Some("no mutation base (--base: not given; origin/HEAD: not set)") + ); + let inside = Base::Missing { tried, ci: true }; + let finding = base_finding(&inside).expect("a finding"); + assert!(finding.starts_with("no mutation base ("), "{finding}"); + assert!(finding.contains("fetch-depth: 0"), "{finding}"); + let unresolved = Base::Unresolved { + source: BaseSource::Flag, + name: "nope".to_string(), + }; + let finding = base_finding(&unresolved).expect("a finding"); + assert!( + finding.contains("`nope`") && finding.contains("not a commit"), + "{finding}" + ); + for base in [&outside, &inside, &unresolved] { + let finding = base_finding(base).expect("a finding"); + assert!(!finding.contains("is empty") && !finding.contains("did not run")); + } + } } diff --git a/crates/cargo-tops/tests/gate_base.rs b/crates/cargo-tops/tests/gate_base.rs index 3707ded..d26d1f3 100644 --- a/crates/cargo-tops/tests/gate_base.rs +++ b/crates/cargo-tops/tests/gate_base.rs @@ -25,11 +25,16 @@ mod profiles { const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); /// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// `cargo mutants` writes what 27.1.0 writes for a run with no mutants; /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +if [ "$1" = "mutants" ]; then + mkdir -p mutants.out + echo '{"outcomes":[{"scenario":"Baseline","summary":"Success"}],"total_mutants":0,"missed":0,"caught":0,"timeout":0,"unviable":0,"success":0,"cargo_mutants_version":"27.1.0"}' > mutants.out/outcomes.json +fi exit 0 "#; @@ -740,3 +745,112 @@ fn a_base_with_no_common_history_says_why_and_names_fetch_depth() { "{problems:?}" ); } + +/// The number that follows `after` in `text`. +fn number_after(text: &str, after: &str) -> usize { + let tail = text.split(after).nth(1).unwrap_or_default(); + let digits: String = tail.chars().take_while(char::is_ascii_digit).collect(); + digits.parse().unwrap_or(usize::MAX) +} + +#[test] +fn an_origin_head_that_names_a_tree_reads_not_a_commit() { + let fixture = fixture("tree-head", "main"); + let tree = git(&fixture, &fixture.repo, &["rev-parse", "HEAD^{tree}"]); + git( + &fixture, + &fixture.repo, + &[ + "update-ref", + "--no-deref", + "refs/remotes/origin/HEAD", + &tree, + ], + ); + let (_, text) = gate(&fixture, &[], None); + assert!(text.contains("origin/HEAD: not a commit"), "{text}"); + assert!(!text.contains("origin/HEAD: not set"), "{text}"); +} + +#[test] +fn a_mutation_row_that_stopped_on_a_missing_tool_does_not_say_the_diff_is_empty() { + let fixture = fixture("tool-missing-words", "main"); + fs::remove_file(fixture.bin.join("cargo-mutants")).expect("remove the stub"); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let doc = report(&fixture); + assert_eq!(at(mutation(&doc), "/verdict"), "tool-missing", "{doc}"); + let detail = at(mutation(&doc), "/detail").as_str().unwrap_or_default(); + assert!(detail.contains("no mutation base"), "{detail}"); + assert!( + !detail.contains("is empty") && !detail.contains("did not run"), + "{detail}" + ); +} + +#[test] +fn a_dry_run_row_words_the_base_without_an_outcome() { + let fixture = fixture("dry-words", "main"); + let (_, _) = gate(&fixture, &["--dry-run"], None); + let doc = report(&fixture); + let detail = at(mutation(&doc), "/detail").as_str().unwrap_or_default(); + assert!(detail.contains("no mutation base"), "{detail}"); + assert!( + !detail.contains("is empty") && !detail.contains("did not run"), + "{detail}" + ); +} + +#[test] +fn the_summary_counts_commands_that_ran_as_the_count_line_does() { + // A missing tool ran no command: it is in neither number. + let fixture = fixture("count-words", "main"); + fs::remove_file(fixture.bin.join("cargo-deny")).expect("remove the stub"); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let said = number_after(&text, "count: "); + let closing = number_after(&text, "problem(s), "); + assert!(said != usize::MAX && closing != usize::MAX, "{text}"); + assert_eq!(said, closing, "{text}"); +} + +#[test] +fn a_report_that_cannot_be_written_ends_the_output_with_fail() { + let fixture = fixture("unwritable", "main"); + fs::create_dir_all(fixture.repo.join("target/rust-tops/report.json")).expect("mkdir"); + // At HEAD the diff is empty: without the write error this is a pass. + let (out, text) = gate(&fixture, &["--base", "HEAD"], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let stdout = String::from_utf8_lossy(&out.stdout); + let last = stdout.lines().rev().find(|line| !line.trim().is_empty()); + assert!( + last.is_some_and(|line| line.starts_with("fail:") && line.contains("report.json")), + "{stdout}" + ); + assert!( + !stdout.lines().any(|line| line.starts_with("ok:")), + "{stdout}" + ); +} + +#[test] +fn the_help_and_the_schema_say_what_exit_2_covers() { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["gate", "--help"]) + .output() + .expect("spawn help"); + let help = String::from_utf8_lossy(&out.stdout).replace("\n ", " "); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let described = at(&schema, "/properties/exit_code/description") + .as_str() + .unwrap_or_default(); + for (name, text) in [("help", help.as_str()), ("schema", described)] { + for phrase in [ + "rust-tops.yaml", + "rust-tops.lock", + "no enabled layer at or below --max-tier", + ] { + assert!(text.contains(phrase), "{name} lacks `{phrase}`: {text}"); + } + } +} diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 41c0144..9d63b42 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -50,6 +50,11 @@ if [ "$*" = "crap --version" ]; then echo 'cargo-crap 0.9.0' exit 0 fi +# `cargo mutants` leaves what 27.1.0 leaves for a run with no mutants. +if [ "$1" = "mutants" ]; then + mkdir -p mutants.out + echo '{"outcomes":[{"scenario":"Baseline","summary":"Success"}],"total_mutants":0,"missed":0,"caught":0,"timeout":0,"unviable":0,"success":0,"cargo_mutants_version":"27.1.0"}' > mutants.out/outcomes.json +fi prev= for arg in "$@"; do case $prev in diff --git a/schema/gate-report.schema.json b/schema/gate-report.schema.json index b591ee5..62a56c2 100644 --- a/schema/gate-report.schema.json +++ b/schema/gate-report.schema.json @@ -39,7 +39,7 @@ "enum": ["pass", "fail", "config-invalid", "environment"] }, "exit_code": { - "description": "0 pass; 1 the gate ran and failed (a dry run included); 2 the config is invalid; 3 the environment (a required tool or cargo metadata is missing).", + "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", "type": "integer", "enum": [0, 1, 2, 3] }, From 9cd1e53023234ecb731f91cefc1d97b1a6e4a810 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 00:47:02 -0500 Subject: [PATCH 04/10] Keep the exit 2 wording and split the all-layers expiry test The help and the report schema keep their "the config is invalid" opening and then say what else exit 2 covers. The loop over every layer and both tiers took longer than the 20 second CI limit under nextest; it is four tests now, each over four layers at one tier. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/main.rs | 2 +- crates/cargo-tops/tests/gate_exemptions.rs | 52 +++++++++++++--------- schema/gate-report.schema.json | 2 +- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 9f061bb..937f3db 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -93,7 +93,7 @@ enum Commands { /// Run enabled Tier 0–1 tools in protocol order, then the Bend laws layer #[command(after_help = "Writes /rust-tops/report.json on every run.\n\ Exit codes:\n 0 pass\n 1 the gate ran and failed (a dry run included)\n \ -2 the config or the request cannot be gated: rust-tops.yaml missing, unreadable \ +2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable \ or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; \ a command-line usage error is 2 too. An expired exemption is not 2: it is 1\n \ 3 the environment: a required tool or cargo metadata is missing, or report.json \ diff --git a/crates/cargo-tops/tests/gate_exemptions.rs b/crates/cargo-tops/tests/gate_exemptions.rs index 4b104e7..54ba853 100644 --- a/crates/cargo-tops/tests/gate_exemptions.rs +++ b/crates/cargo-tops/tests/gate_exemptions.rs @@ -235,30 +235,40 @@ fn an_expired_record_for_a_layer_that_is_off_fails_the_gate() { assert!(text.contains("expired on 2020-01-01"), "{text}"); } -#[test] -fn an_expired_record_fails_for_every_layer_and_every_state_of_it() { - // On and in range, on and above the range, and off. - for layer in [ - "static-lint", - "security-advisory", - "miri", - "fuzz", - "mutation", - "coverage", - "crap", - "acceptance", - ] { - for tier in ["0", "1"] { - let fixture = fixture("expired-any"); - exempt(&fixture, layer, "2020-01-01"); - let (out, text) = gate(&fixture, &["--max-tier", tier], None); - assert_eq!(out.status.code(), Some(1), "{layer} at {tier}: {text}"); - let problems = exemption_problems(&report(&fixture)); - assert_eq!(problems.len(), 1, "{layer} at {tier}: {problems:?}"); - } +/// Every layer, with `tier` as the range, has an expired record fail the gate. +fn expired_record_fails_each_of(layers: &[&str], tier: &str) { + for layer in layers { + let fixture = fixture("expired-any"); + exempt(&fixture, layer, "2020-01-01"); + let (out, text) = gate(&fixture, &["--max-tier", tier], None); + assert_eq!(out.status.code(), Some(1), "{layer} at {tier}: {text}"); + let problems = exemption_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{layer} at {tier}: {problems:?}"); } } +// On and in range, on and above the range, and off: every layer at tiers 0 +// and 1, in four tests so each stays well inside the CI time limit. +#[test] +fn an_expired_record_fails_for_the_first_layers_at_tier_0() { + expired_record_fails_each_of(&["static-lint", "security-advisory", "miri", "fuzz"], "0"); +} + +#[test] +fn an_expired_record_fails_for_the_last_layers_at_tier_0() { + expired_record_fails_each_of(&["mutation", "coverage", "crap", "acceptance"], "0"); +} + +#[test] +fn an_expired_record_fails_for_the_first_layers_at_tier_1() { + expired_record_fails_each_of(&["static-lint", "security-advisory", "miri", "fuzz"], "1"); +} + +#[test] +fn an_expired_record_fails_for_the_last_layers_at_tier_1() { + expired_record_fails_each_of(&["mutation", "coverage", "crap", "acceptance"], "1"); +} + #[test] fn a_record_whose_date_is_not_a_day_fails() { for date in ["2027-02-30", "not-a-date", "2024-13-01", "2099-1-1"] { diff --git a/schema/gate-report.schema.json b/schema/gate-report.schema.json index 62a56c2..27aeb9c 100644 --- a/schema/gate-report.schema.json +++ b/schema/gate-report.schema.json @@ -39,7 +39,7 @@ "enum": ["pass", "fail", "config-invalid", "environment"] }, "exit_code": { - "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", + "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", "type": "integer", "enum": [0, 1, 2, 3] }, From 3901683c494b5050db108e7f87a919750c2a7841 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 00:49:25 -0500 Subject: [PATCH 05/10] Give each split expiry test its own scratch directory The four tests shared one scratch name, so threaded cargo test had them remove each other's directories. Each now names its own. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/tests/gate_exemptions.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/cargo-tops/tests/gate_exemptions.rs b/crates/cargo-tops/tests/gate_exemptions.rs index 54ba853..aba772e 100644 --- a/crates/cargo-tops/tests/gate_exemptions.rs +++ b/crates/cargo-tops/tests/gate_exemptions.rs @@ -236,9 +236,10 @@ fn an_expired_record_for_a_layer_that_is_off_fails_the_gate() { } /// Every layer, with `tier` as the range, has an expired record fail the gate. -fn expired_record_fails_each_of(layers: &[&str], tier: &str) { +/// `name` keeps the scratch directories of tests that run together apart. +fn expired_record_fails_each_of(name: &str, layers: &[&str], tier: &str) { for layer in layers { - let fixture = fixture("expired-any"); + let fixture = fixture(&format!("{name}-{layer}")); exempt(&fixture, layer, "2020-01-01"); let (out, text) = gate(&fixture, &["--max-tier", tier], None); assert_eq!(out.status.code(), Some(1), "{layer} at {tier}: {text}"); @@ -251,22 +252,30 @@ fn expired_record_fails_each_of(layers: &[&str], tier: &str) { // and 1, in four tests so each stays well inside the CI time limit. #[test] fn an_expired_record_fails_for_the_first_layers_at_tier_0() { - expired_record_fails_each_of(&["static-lint", "security-advisory", "miri", "fuzz"], "0"); + expired_record_fails_each_of( + "a0", + &["static-lint", "security-advisory", "miri", "fuzz"], + "0", + ); } #[test] fn an_expired_record_fails_for_the_last_layers_at_tier_0() { - expired_record_fails_each_of(&["mutation", "coverage", "crap", "acceptance"], "0"); + expired_record_fails_each_of("b0", &["mutation", "coverage", "crap", "acceptance"], "0"); } #[test] fn an_expired_record_fails_for_the_first_layers_at_tier_1() { - expired_record_fails_each_of(&["static-lint", "security-advisory", "miri", "fuzz"], "1"); + expired_record_fails_each_of( + "a1", + &["static-lint", "security-advisory", "miri", "fuzz"], + "1", + ); } #[test] fn an_expired_record_fails_for_the_last_layers_at_tier_1() { - expired_record_fails_each_of(&["mutation", "coverage", "crap", "acceptance"], "1"); + expired_record_fails_each_of("b1", &["mutation", "coverage", "crap", "acceptance"], "1"); } #[test] From 254061e045d0fa0e841f1d54b3b5f21eb5ea0600 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 01:01:31 -0500 Subject: [PATCH 06/10] Split the outcome reader by what each part decides, and test the crate prefix take scored a CRAP above the new-function threshold; the baseline check, the outcome kind and the counting are separate decisions now. The gate's own run found that nothing tested crate_prefix against a real workspace: a test names a crate under crates/ by its directory and a crate at the root by none. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/gate/judge.rs | 16 ++++++- crates/cargo-tops/src/mutation.rs | 65 ++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 034e666..dcf366d 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -482,7 +482,7 @@ pub(crate) fn run_failure(run: &Run) -> Option { #[cfg(test)] mod tests { - use super::clear_cases; + use super::{clear_cases, crate_prefix}; use std::fs; use std::path::PathBuf; @@ -527,4 +527,18 @@ mod tests { ); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn a_crate_under_the_workspace_root_is_named_by_its_directory() { + let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .expect("workspace"); + assert_eq!(crate_prefix(&workspace, "cargo-tops"), "crates/cargo-tops"); + // No such package, or no cargo project here: the gate directory. + assert_eq!(crate_prefix(&workspace, "no-such-package"), ""); + let empty = scratch("prefix-none"); + assert_eq!(crate_prefix(&empty, "cargo-tops"), ""); + fs::remove_dir_all(&empty).ok(); + } } diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs index e174e8e..c3b4812 100644 --- a/crates/cargo-tops/src/mutation.rs +++ b/crates/cargo-tops/src/mutation.rs @@ -118,30 +118,63 @@ fn missed(scenario: &Value) -> Result { }) } -/// Add one outcome to `found`. The baseline run is not a mutant. -fn take(found: &mut Outcomes, outcome: &Value) -> Result<(), String> { +/// What one outcome says happened to its mutant. +#[derive(Clone, Copy)] +enum Kind { + Caught, + Missed, + Timeout, + Unviable, +} + +/// The scenario and the summary of one outcome. +fn parts(outcome: &Value) -> Result<(&Value, &str), String> { let scenario = field(outcome, "scenario")?; let summary = field(outcome, "summary")? .as_str() .ok_or("outcomes.json `summary` is not a string")?; - if scenario.as_str() == Some("Baseline") { - return match summary { - "Success" => Ok(()), - other => Err(format!("the baseline run ended as `{other}`")), - }; + Ok((scenario, summary)) +} + +/// The baseline run is not a mutant: it must have succeeded. +fn baseline(summary: &str) -> Result<(), String> { + match summary { + "Success" => Ok(()), + other => Err(format!("the baseline run ended as `{other}`")), } +} + +fn kind(summary: &str) -> Result { match summary { - "CaughtMutant" => found.caught += 1, - "MissedMutant" => found.missed.push(missed(scenario)?), - "Timeout" => found.timeouts += 1, - "Unviable" => found.unviable += 1, - other => { - return Err(format!( - "outcomes.json has an outcome `{other}` this gate does not know" - )) + "CaughtMutant" => Ok(Kind::Caught), + "MissedMutant" => Ok(Kind::Missed), + "Timeout" => Ok(Kind::Timeout), + "Unviable" => Ok(Kind::Unviable), + other => Err(format!( + "outcomes.json has an outcome `{other}` this gate does not know" + )), + } +} + +impl Outcomes { + fn add(&mut self, kind: Kind, scenario: &Value) -> Result<(), String> { + match kind { + Kind::Caught => self.caught += 1, + Kind::Missed => self.missed.push(missed(scenario)?), + Kind::Timeout => self.timeouts += 1, + Kind::Unviable => self.unviable += 1, } + Ok(()) + } +} + +/// Add one outcome to `found`. +fn take(found: &mut Outcomes, outcome: &Value) -> Result<(), String> { + let (scenario, summary) = parts(outcome)?; + if scenario.as_str() == Some("Baseline") { + return baseline(summary); } - Ok(()) + found.add(kind(summary)?, scenario) } /// Read `outcomes.json`. Any shape but the pinned one is an `Err` saying From b13471a913c251304c54e2ebd0df886cee641f99 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 02:06:32 -0500 Subject: [PATCH 07/10] Pass a diff with nothing to mutate, and hold a survivor to its own record B1: real cargo-mutants 27.1.0 exits 0 and writes no mutants.out for a diff of docs, comments, tests or CI. The gate now passes --output , so the file is where the gate looks whatever an adopter sets, and reads exit 0 with no file as zero mutants only when the run printed the line 27.1.0 prints for it (recorded from real runs and pinned). Any other missing file fails. The test stubs write nothing, as the real tool does, and a new test runs the real cargo-mutants over a README-only and a comment-only diff. B2: a survivor with function: null (a const, static, type alias or field type) is explained only by a record with no symbol. A real 27.1.0 run over such a crate is the new fixture. S2: a symbol-free whole-tree mutation record lifts the floor and explains no survivor; section 19 says so. S1: the laws and above-range verdicts are decided in judge.rs. S3: the exit 2 text names usage errors and the lock only when a layer in range needs a pin. S4: a wrong or missing total_mutants, exit 2 with nothing missed and exit 3 with no timeout each fail naming both facts. N1: a survivor is named once. N2: a failed cargo mutants says what its exit means. N4: the expiry day is UTC. N5: the explained-survivor list is rewritten to the one rule. Co-Authored-By: Claude Sonnet 5.5 --- PROTOCOL.md | 12 +- crates/cargo-tops/src/gate/execute.rs | 127 ++- crates/cargo-tops/src/gate/judge.rs | 19 + crates/cargo-tops/src/gate/report.rs | 23 +- crates/cargo-tops/src/main.rs | 5 +- crates/cargo-tops/src/mutation.rs | 207 ++++- crates/cargo-tops/src/steps.rs | 8 + crates/cargo-tops/src/tiers.rs | 4 +- .../tests/fixtures/mutants-27.1.0/README.md | 18 + .../null-function/outcomes.json | 750 ++++++++++++++++++ crates/cargo-tops/tests/gate_base.rs | 9 +- crates/cargo-tops/tests/gate_real_mutants.rs | 307 +++++++ crates/cargo-tops/tests/gate_survivors.rs | 276 ++++++- crates/cargo-tops/tests/laws_gate.rs | 13 +- schema/gate-report.schema.json | 2 +- 15 files changed, 1711 insertions(+), 69 deletions(-) create mode 100644 crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json create mode 100644 crates/cargo-tops/tests/gate_real_mutants.rs diff --git a/PROTOCOL.md b/PROTOCOL.md index b12ed5d..01770be 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -497,7 +497,7 @@ The first source that is set wins. A ref that is named by the flag or the key an `CI` counts as set when it is not empty and is not `0` or `false` (in any case). When no source gives a base, the layer is `fail` if `CI` is set. The report's `detail` and the gate's problem line name `fetch-depth: 0` and `--base`: a checkout that never fetched the base must not read as a clean diff. When `CI` is not set, the layer is `vacuous`, and the detail lists each source tried. A base that resolves but shares no history with `HEAD`, as in a shallow checkout, fails the diff and names `fetch-depth: 0` too. CI workflows pass `--base "origin/${{ github.base_ref || github.event.repository.default_branch }}"` from a checkout with `fetch-depth: 0`, because `actions/checkout` may not create `origin/HEAD`. -How the gate judges the layer: it removes any `mutants.out/outcomes.json` from an earlier run, runs `cargo mutants`, and reads the `outcomes.json` that run wrote. The file must be in the shape cargo-mutants 27.1.0 writes; another release, a missing field, an outcome the gate does not know, a failed baseline, or totals that disagree with the listed outcomes fail the layer and name `outcomes.json`. cargo-mutants exits 2 when mutants were missed and 3 when some timed out; the gate treats 0, 2 and 3 as a finished run and judges it from the file, and any other exit is a failed command. Each missed mutant is explained when a live `layer: mutation` exemption (§19) covers its file and function. Timeouts and unviable mutants are counted and reported; they are not survivors. The layer fails when the survivors nothing explains exceed `gates.mutation.unexplained_survivors_max`, and the problem names each of them. `kill_rate_min_pct` is a floor `check` compares with the class profile; the gate does not judge it. The mutation row's `judged` carries `caught`, `missed`, `explained`, `unexplained`, `timeouts`, `unviable` and `unexplained_survivors_max`, beside `base`. +How the gate judges the layer: it removes any `mutants.out/outcomes.json` from an earlier run, runs `cargo mutants --output `, and reads the `outcomes.json` that run wrote there, whatever `output` an adopter's own `.cargo/mutants.toml` sets. A diff with nothing to mutate (documentation, comments, tests, CI) makes cargo-mutants 27.1.0 exit 0 and write no `mutants.out`; it says `Diff changes no Rust source files` or `No mutants to filter` on stderr. Exit 0 with no file passes as zero mutants only when the run printed one of those lines; any other missing file fails. The file must be in the shape cargo-mutants 27.1.0 writes; another release, a missing field, an outcome the gate does not know, a failed baseline, or totals that disagree with the listed outcomes (`total_mutants` included) fail the layer and name `outcomes.json`. Exit 2 with no missed outcome listed, and exit 3 with no timeout listed, fail too. cargo-mutants exits 2 when mutants were missed and 3 when some timed out; the gate treats 0, 2 and 3 as a finished run and judges it from the file, and any other exit is a failed command. Each missed mutant is explained when a live `layer: mutation` exemption (§19) covers its file and function. Timeouts and unviable mutants are counted and reported; they are not survivors. The layer fails when the survivors nothing explains exceed `gates.mutation.unexplained_survivors_max`, and the problem names each of them. `kill_rate_min_pct` is a floor `check` compares with the class profile; the gate does not judge it. The mutation row's `judged` carries `caught`, `missed`, `explained`, `unexplained`, `timeouts`, `unviable` and `unexplained_survivors_max`, beside `base`. Pass criteria: @@ -505,11 +505,7 @@ Pass criteria: - **0 unexplained survivors** in the diff (`unexplained_survivors_max`) - kill rate on examined mutants ≥ class minimum (see §4; declarative) -A survivor is explained only if *all* of the following hold: - -1. The mutant is equivalent (`x + 0`, `x | 0` on a proven-zero, debug-only formatting). -2. A one-line reason is written in the PR and in `exemptions` if it will persist. -3. `#[mutants::skip]` is applied at the tightest scope (expression, then function, never file) with the same reason. +A survivor is explained when a live `layer: mutation` exemption covers it (§19), and the record gives the reason: the mutant is equivalent (`x + 0`, `x | 0` on a proven-zero, debug-only formatting), or a named test covers the behaviour. `#[mutants::skip]` at the tightest scope (expression, then function, never file) with the same reason takes an equivalent mutant out of the run, so it is never a survivor to explain. ```rust #[mutants::skip] // equivalent: trailing comma in this macro has no runtime effect @@ -983,9 +979,9 @@ exemptions: Expired exemptions fail CI. Reasons shorter than one clause are rejected. -`cargo tops gate` checks every record, whatever its `layer`, whether that layer is on or off, and at any `--max-tier`: a record that has expired, or whose `expires` is not a real day, is a problem line in the report and the gate exits 1 (never 2; the config is still valid). `cargo tops check` prints the same lines as warnings and exits 0. +`cargo tops gate` checks every record, whatever its `layer`, whether that layer is on or off, and at any `--max-tier`: a record that has expired, or whose `expires` is not a real day, is a problem line in the report and the gate exits 1 (never 2; the config is still valid). `cargo tops check` prints the same lines as warnings and exits 0. The day is UTC: a record expires at 00:00 UTC on its `expires` date, whatever the local time zone. -How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains the survivors in the matching files, or, with `symbol`, in the function of that name; a record with no `symbol` covers every function in its glob. A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. +How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains the survivors in the matching files, or, with `symbol`, in the function of that name; a record with no `symbol` covers every function in its glob, and the survivors that are in no function (a `const` or `static` initialiser, a type alias, a field type: cargo-mutants writes `function: null` for them). A record with a `symbol` never explains a survivor in no function. A record with no `symbol` whose path is the whole tree (`**`, `./**`, `**/*`, `*`, `.`, `./`) lifts the class floor on `unexplained_survivors_max` (§4) and explains no survivor; to explain survivors, a record needs a `symbol` or a narrower glob. A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. --- diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index a3163a3..0f02d96 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -1,8 +1,9 @@ //! Run a plan: each command in order, timed. A dry run lists the commands //! and runs none. use std::collections::HashMap; +use std::io::{BufRead, BufReader}; use std::path::Path; -use std::process::{Command, ExitStatus}; +use std::process::{ChildStderr, Command, ExitStatus, Stdio}; use std::time::{Instant, SystemTime}; use anyhow::{Context, Result}; @@ -464,21 +465,46 @@ pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Resu ))); }; ran.push(mutants.display()); - mutation::clear(dir)?; - let outcome = run_mutants(dir, mutants)?; - let mut run = finish(layer, &ran, outcome); - if outcome != Outcome::Passed { - return Ok(Mutated::without_outcomes(run)); + mutation::clear(&plan.output)?; + let ended = run_mutants(dir, mutants)?; + Ok(settled( + finish(layer, &ran, ended.outcome), + &ended, + &plan.output, + )) +} + +/// The run of `cargo mutants` once it ended. A command that failed says what +/// its exit means; one that finished (exit 0, 2 or 3) is judged from the +/// outcomes it wrote. +fn settled(mut run: Run, ended: &Ended, output: &Path) -> Mutated { + if ended.outcome != Outcome::Passed { + run.note = exit_note(&run.step, ended); + return Mutated::without_outcomes(run); } - let found = mutation::read(dir); + let found = mutation::read(output, ended.code.unwrap_or_default(), &ended.stderr); if let Err(why) = &found { run.outcome = Outcome::Failed; run.note = Some(why.clone()); } - Ok(Mutated { + Mutated { run, found: Some(found), - }) + } +} + +/// What a failed `cargo mutants` exit means, in cargo-mutants' own terms. +fn exit_note(step: &str, ended: &Ended) -> Option { + if ended.outcome != Outcome::Failed { + return None; + } + let what = match ended.code { + Some(1) => "exited 1: it was given arguments it does not accept", + Some(4) => "exited 4: the tests fail in the unmutated tree, so no mutant was tested", + Some(code) => return Some(format!("`{step}` exited {code}")), + None => "was ended by a signal", + }; + Some(format!("`{step}` {what}")) } /// The run of a layer that stopped on its base or its diff: prints the @@ -522,17 +548,81 @@ pub(crate) fn run_step(dir: &Path, step: &Step) -> Result { }) } +/// How `cargo mutants` ended: its outcome, exit code and what it printed. +struct Ended { + outcome: Outcome, + code: Option, + stderr: String, +} + +impl Ended { + fn without_output(outcome: Outcome) -> Self { + Self { + outcome, + code: None, + stderr: String::new(), + } + } +} + /// `cargo mutants` exits 2 when mutants were missed and 3 when some timed /// out: those and 0 are a finished run, judged by the ledger. Any other exit -/// is a failed command. -fn run_mutants(dir: &Path, step: &Step) -> Result { - Ok(match spawn(dir, step)? { - Spawned::Exited(status) if matches!(status.code(), Some(0 | 2 | 3)) => Outcome::Passed, - Spawned::Exited(_) | Spawned::NoProgram => Outcome::Failed, - Spawned::Missing => Outcome::ToolMissing, +/// is a failed command. What it prints to stderr is shown and kept, because +/// a run that found nothing to mutate says so there and writes no file. +fn run_mutants(dir: &Path, step: &Step) -> Result { + let Some((program, args)) = step.program() else { + return Ok(Ended::without_output(Outcome::Failed)); + }; + let mut child = match Command::new(program) + .args(args) + .current_dir(dir) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + println!( + "{}", + steps::MissingTool { + binary: program.to_string() + } + ); + return Ok(Ended::without_output(Outcome::ToolMissing)); + } + Err(err) => return Err(err).with_context(|| format!("spawn: {}", step.display())), + }; + let stderr = tee(child.stderr.take()); + let status = child + .wait() + .with_context(|| format!("wait: {}", step.display()))?; + let code = status.code(); + let outcome = if matches!(code, Some(0 | 2 | 3)) { + Outcome::Passed + } else { + Outcome::Failed + }; + Ok(Ended { + outcome, + code, + stderr, }) } +/// Everything `pipe` yields, passed on to our own stderr as it comes. +fn tee(pipe: Option) -> String { + let Some(pipe) = pipe else { + return String::new(); + }; + let mut kept = String::new(); + for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) { + let line = String::from_utf8_lossy(&line).into_owned(); + eprintln!("{line}"); + kept.push_str(&line); + kept.push('\n'); + } + kept +} + /// How a spawned command ended. enum Spawned { Exited(ExitStatus), @@ -649,7 +739,8 @@ mod tests { for (code, want) in wants { let step = Step::command(["sh".to_string(), "-c".to_string(), format!("exit {code}")]); let got = run_mutants(Path::new("."), &step).expect("it ran"); - assert_eq!(got, want, "exit {code}"); + assert_eq!(got.outcome, want, "exit {code}"); + assert_eq!(got.code, Some(code), "exit {code}"); // Any other command still fails on every nonzero exit. let plain = run_step(Path::new("."), &step).expect("it ran"); let plain_want = if code == 0 { @@ -661,7 +752,9 @@ mod tests { } let nowhere = Step::command(["/no/such/program/here"]); assert_eq!( - run_mutants(Path::new("."), &nowhere).expect("an outcome"), + run_mutants(Path::new("."), &nowhere) + .expect("an outcome") + .outcome, Outcome::ToolMissing ); } diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index dcf366d..98f4340 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -296,6 +296,25 @@ fn runs_verdict(runs: &[&Run]) -> &'static str { } } +/// The verdict of a layer enabled above `--max-tier`: it is not judged. +pub(crate) fn above_range_verdict() -> &'static str { + "not-evaluated" +} + +/// The verdict of a row the run never reached (a fault before it ran). +pub(crate) fn unreached_verdict() -> &'static str { + "not-run" +} + +/// The verdict of the laws layer: `not-run` when `bend` did not run. +pub(crate) fn laws_verdict(judged: Option<&Judgement>) -> &'static str { + match judged.filter(|judged| judged.bend_ran) { + None => "not-run", + Some(judged) if judged.laws_failed => "fail", + Some(_) => "pass", + } +} + /// The row verdict of layer `id`: `not-run` when nothing ran for it, /// otherwise its worst run. A run that passed still fails the row when its /// floors, or its survivors, did not. diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index dea0a11..12f885e 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -337,6 +337,16 @@ fn row( row } +/// A layer enabled above `--max-tier`: named, not judged. +fn above_range_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { + let mut entry = row(id, tier, judge::above_range_verdict(), &[], 0.0, None); + entry.insert( + "detail".into(), + json!(format!("above --max-tier {}", parts.opts.max_tier)), + ); + entry +} + /// One row per enabled layer, in `rust-tops.yaml` order, and the ids of the /// layers that are off. fn layer_rows(parts: &Parts<'_>) -> (Vec, Vec) { @@ -352,12 +362,7 @@ fn layer_rows(parts: &Parts<'_>) -> (Vec, Vec) { } let tier = crate::tiers::tier_for(layer); let mut entry = if tier > parts.opts.max_tier { - let mut entry = row(&layer.id, tier, "not-evaluated", &[], 0.0, None); - entry.insert( - "detail".into(), - json!(format!("above --max-tier {}", parts.opts.max_tier)), - ); - entry + above_range_row(parts, &layer.id, tier) } else if layer.id == "laws" { laws_row(parts, tier) } else { @@ -396,7 +401,7 @@ fn judged_numbers(parts: &Parts<'_>, id: &str) -> Option { fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { let (Some(plan), Some(done)) = (parts.plan, parts.done) else { - return row(id, tier, "not-run", &[], 0.0, None); + return row(id, tier, judge::unreached_verdict(), &[], 0.0, None); }; let numbers = judged_numbers(parts, id); let verdict = judge::layer_verdict(done, parts.judged, id); @@ -488,10 +493,10 @@ fn laws_row(parts: &Parts<'_>, tier: u8) -> Map { "bend".to_string(), parts.report_dir.join("cases.bend").display().to_string(), ]]; + let verdict = judge::laws_verdict(parts.judged); let Some(judged) = parts.judged.filter(|judged| judged.bend_ran) else { - return row("laws", tier, "not-run", &argv, 0.0, None); + return row("laws", tier, verdict, &argv, 0.0, None); }; - let verdict = if judged.laws_failed { "fail" } else { "pass" }; let cases = read_cases(&parts.report_dir); let version = cases.as_ref().and_then(|cases| cases.get("bend")?.as_str()); row("laws", tier, verdict, &argv, judged.laws_seconds, version) diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 937f3db..2c0217c 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -94,8 +94,9 @@ enum Commands { #[command(after_help = "Writes /rust-tops/report.json on every run.\n\ Exit codes:\n 0 pass\n 1 the gate ran and failed (a dry run included)\n \ 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable \ -or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; \ -a command-line usage error is 2 too. An expired exemption is not 2: it is 1\n \ +or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable \ +when a layer in range needs a pin (crap today); a command-line usage error is 2 too and \ +writes no report. An expired exemption is not 2: it is 1\n \ 3 the environment: a required tool or cargo metadata is missing, or report.json \ cannot be written\n\ Where more than one applies, 2 wins over 3, and 3 over 1.")] diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs index c3b4812..439c65c 100644 --- a/crates/cargo-tops/src/mutation.rs +++ b/crates/cargo-tops/src/mutation.rs @@ -17,10 +17,17 @@ use anyhow::{Context, Result}; use serde_json::{json, Value}; use crate::exemptions::Exemption; +use crate::tiers::covers_crate; /// Where cargo-mutants writes the run's outcomes, under the directory it ran in. const OUTCOMES: &str = "mutants.out/outcomes.json"; +/// What cargo-mutants 27.1.0 prints (to stderr) when the diff leaves nothing +/// to mutate, and then exits 0 without writing `mutants.out`: no Rust source +/// file changed, or the changed lines are in no function it mutates (a +/// comment, a test, a documentation or CI file). Recorded from real runs. +const NO_MUTANTS: [&str; 2] = ["Diff changes no Rust source files", "No mutants to filter"]; + /// The cargo-mutants release whose `outcomes.json` this reads. pub(crate) const PINNED: &str = "27.1.0"; @@ -68,12 +75,43 @@ pub(crate) fn clear(dir: &Path) -> Result<()> { } } -/// The outcomes `cargo mutants` just wrote under `dir`. Call [`clear`] before -/// the run: a file that is there afterwards is this run's. -pub(crate) fn read(dir: &Path) -> Result { - let text = fs::read_to_string(dir.join(OUTCOMES)) - .map_err(|err| format!("no {OUTCOMES} after cargo mutants ran: {err}"))?; - parse(&text) +/// The outcomes `cargo mutants` just wrote under `dir`, after it ended with +/// `code` and printed `stderr`. Call [`clear`] before the run: a file that is +/// there afterwards is this run's. No file is the zero mutants of a diff with +/// nothing to mutate only when the run exited 0 and said so itself; any other +/// missing file is a failure. +pub(crate) fn read(dir: &Path, code: i32, stderr: &str) -> Result { + let text = match fs::read_to_string(dir.join(OUTCOMES)) { + Ok(text) => text, + Err(_) if code == 0 && NO_MUTANTS.iter().any(|line| stderr.contains(line)) => { + return Ok(Outcomes::default()); + } + Err(err) => { + return Err(format!( + "no {OUTCOMES} after cargo mutants exited {code}, and its output does not \ + say it found no mutants: {err}" + )); + } + }; + let found = parse(&text)?; + check_exit(&found, code)?; + Ok(found) +} + +/// The exit code and the file must agree: 2 says mutants were missed and 3 +/// says some timed out. +fn check_exit(found: &Outcomes, code: i32) -> Result<(), String> { + match code { + 2 if found.missed.is_empty() => Err( + "cargo mutants exited 2 (mutants were missed) but outcomes.json lists no missed mutant" + .to_string(), + ), + 3 if found.timeouts == 0 => Err( + "cargo mutants exited 3 (mutants timed out) but outcomes.json lists no timeout" + .to_string(), + ), + _ => Ok(()), + } } fn field<'v>(value: &'v Value, key: &str) -> Result<&'v Value, String> { @@ -202,6 +240,10 @@ pub(crate) fn parse(text: &str) -> Result { ("missed", found.missed.len()), ("timeout", found.timeouts), ("unviable", found.unviable), + ( + "total_mutants", + found.caught + found.missed.len() + found.timeouts + found.unviable, + ), ]; for (key, seen) in totals { let said = count(&doc, key)?; @@ -224,6 +266,22 @@ fn relative<'f>(file: &'f str, crate_prefix: &str) -> &'f str { .unwrap_or(file) } +/// Whether `record` explains a survivor in `function` of the file `rel` +/// (`None`: in no function, as in a `const`, a `static`, a type alias or a +/// field type). A record with no symbol and a whole-tree path lifts a floor +/// and explains nothing. A symbol names a function, so a survivor in none is +/// explained only by a record with no symbol. +fn explains(record: &Exemption, rel: &str, function: Option<&str>) -> bool { + let named = record.symbol.is_some(); + if !named && covers_crate(&record.path) { + return false; + } + if named && function.is_none() { + return false; + } + record.covers(rel, function) +} + /// Sort the missed mutants. `live` are the layer's records that have not /// expired; `crate_prefix` is the crate's directory under the workspace root, /// which the records' globs are relative to. @@ -231,7 +289,7 @@ pub(crate) fn explain(found: &Outcomes, live: &[Exemption], crate_prefix: &str) let (explained, unexplained) = found.missed.iter().cloned().partition(|mutant: &Missed| { let rel = relative(&mutant.file, crate_prefix); live.iter() - .any(|record| record.covers(rel, mutant.function.as_deref())) + .any(|record| explains(record, rel, mutant.function.as_deref())) }); Ledger { caught: found.caught, @@ -257,7 +315,7 @@ impl Ledger { let named: Vec = self .unexplained .iter() - .map(|mutant| format!("{}:{} {}", mutant.file, mutant.line, mutant.name)) + .map(|mutant| mutant.name.clone()) .collect(); Some(format!( "mutation: {} unexplained survivor(s), above unexplained_survivors_max {max}: {}", @@ -286,7 +344,7 @@ impl Ledger { reason = "the tests edit and read JSON they built from the recorded fixture" )] mod tests { - use super::{clear, explain, parse, read, relative, Outcomes, PINNED}; + use super::{check_exit, clear, explain, parse, read, relative, Outcomes, PINNED}; use crate::exemptions::Exemption; use serde_json::Value; use std::fs; @@ -569,16 +627,141 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + const NOTHING: &str = " INFO No mutants to filter\n"; + #[test] fn what_the_run_wrote_is_read_and_a_missing_file_is_an_error() { let dir = scratch("read"); - let err = read(&dir).expect_err("no file"); + let err = read(&dir, 0, "").expect_err("no file"); assert!(err.contains("mutants.out/outcomes.json"), "{err}"); + assert!(err.contains("exited 0"), "{err}"); fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); fs::write(dir.join("mutants.out/outcomes.json"), "not json").expect("write"); - assert!(read(&dir).expect_err("not json").contains("not JSON")); + assert!(read(&dir, 2, "") + .expect_err("not json") + .contains("not JSON")); fs::write(dir.join("mutants.out/outcomes.json"), FIXTURE).expect("write"); - assert_eq!(read(&dir).expect("parses").missed.len(), 10); + assert_eq!(read(&dir, 2, "").expect("parses").missed.len(), 10); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn no_file_is_zero_mutants_only_for_exit_0_and_a_line_the_run_printed() { + let dir = scratch("nothing"); + for line in super::NO_MUTANTS { + assert_eq!( + read(&dir, 0, line).expect("zero mutants"), + Outcomes::default() + ); + } + let said = format!("found 3 mutants\n{NOTHING}more\n"); + assert_eq!( + read(&dir, 0, &said).expect("zero mutants"), + Outcomes::default() + ); + for (code, said) in [ + (0, ""), + (0, "Diff file is empty"), + (2, NOTHING), + (3, NOTHING), + (1, NOTHING), + ] { + assert!(read(&dir, code, said).is_err(), "exit {code} {said:?}"); + } + // A file that is there is read, whatever the run printed. + fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); + fs::write(dir.join("mutants.out/outcomes.json"), FIXTURE).expect("write"); + assert_eq!(read(&dir, 0, NOTHING).expect("parses").caught, 11); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn the_exit_code_and_the_file_must_agree() { + let found = parse(FIXTURE).expect("parses"); + assert!(check_exit(&found, 2).is_ok() && check_exit(&found, 3).is_ok()); + assert!(check_exit(&found, 0).is_ok()); + let none = Outcomes { + missed: Vec::new(), + timeouts: 0, + ..found + }; + let two = check_exit(&none, 2).expect_err("nothing missed"); + assert!( + two.contains("exited 2") && two.contains("no missed"), + "{two}" + ); + let three = check_exit(&none, 3).expect_err("no timeout"); + assert!( + three.contains("exited 3") && three.contains("no timeout"), + "{three}" + ); + // Missed but no timeout is fine for 2; a timeout but none missed for 3. + let only_missed = Outcomes { + timeouts: 0, + ..found + }; + assert!(check_exit(&only_missed, 2).is_ok()); + assert!(check_exit(&only_missed, 3).is_err()); + let only_timeout = Outcomes { + missed: Vec::new(), + ..found + }; + assert!(check_exit(&only_timeout, 3).is_ok()); + assert!(check_exit(&only_timeout, 2).is_err()); + } + + #[test] + fn the_total_must_be_there_and_agree() { + let wrong = edited(|doc| doc["total_mutants"] = Value::from(999)); + let err = parse(&wrong).expect_err("999 is not 23"); + assert!(err.contains("says 999 total_mutants but lists 23"), "{err}"); + let gone = edited(|doc| { + doc.as_object_mut().expect("object").remove("total_mutants"); + }); + assert!(parse(&gone) + .expect_err("no total") + .contains("total_mutants")); + } + + const NULL_FUNCTION: &str = + include_str!("../tests/fixtures/mutants-27.1.0/null-function/outcomes.json"); + + #[test] + fn a_survivor_in_no_function_takes_no_symbol_and_a_whole_tree_record_explains_none() { + let found = parse(NULL_FUNCTION).expect("parses"); + assert_eq!(found.missed.len(), 12); + assert_eq!( + found.missed.iter().filter(|m| m.function.is_none()).count(), + 7 + ); + let count = |records: &[Exemption]| explain(&found, records, "").explained.len(); + assert_eq!(count(&[record("src/lib.rs", Some("get"))]), 5); + assert_eq!(count(&[record("src/lib.rs", Some("no_such_fn"))]), 0); + assert_eq!(count(&[record("src/lib.rs", None)]), 12); + assert_eq!(count(&[record("src/**", None)]), 12); + // Symbol and no symbol: every survivor once. + assert_eq!( + count(&[ + record("src/lib.rs", Some("get")), + record("src/lib.rs", None) + ]), + 12 + ); + for whole in ["**", "./**", "**/*", "*", ".", "./"] { + assert_eq!(count(&[record(whole, None)]), 0, "{whole}"); + } + // A symbol never lifts a floor, so a symbol record with a whole-tree + // path still names its function. + assert_eq!(count(&[record("**", Some("get"))]), 5); + } + + #[test] + fn a_survivor_is_named_once_by_its_own_name() { + let found = parse(FIXTURE).expect("parses"); + let ledger = explain(&found, &[], ""); + let problem = ledger.problem(0).expect("fails"); + let first = &found.missed[0]; + assert_eq!(problem.matches(&first.name).count(), 1, "{problem}"); + assert!(!problem.contains(&format!("{}:{} {}", first.file, first.line, first.name))); + } } diff --git a/crates/cargo-tops/src/steps.rs b/crates/cargo-tops/src/steps.rs index f3e8d00..1dbe27d 100644 --- a/crates/cargo-tops/src/steps.rs +++ b/crates/cargo-tops/src/steps.rs @@ -615,6 +615,10 @@ pub(crate) struct MutationPlan { pub diff_path: PathBuf, pub mutants: Step, pub base: Base, + /// Where `cargo mutants` writes `mutants.out`: the report directory, so + /// the gate reads the files this run wrote and no setting of the + /// adopter's moves them. + pub output: PathBuf, } impl MutationPlan { @@ -628,6 +632,7 @@ impl MutationPlan { }; let script = mutation_diff_script(report_dir, &diff_path, from); let diff = Step::command(["sh", "-c", script.as_str()]); + let output_arg = report_dir.display().to_string(); let mutants = Step::tool([ "cargo", "mutants", @@ -635,12 +640,15 @@ impl MutationPlan { diff_arg.as_str(), "-j", "2", + "--output", + output_arg.as_str(), ]); Self { diff, diff_path, mutants, base, + output: report_dir.to_path_buf(), } } diff --git a/crates/cargo-tops/src/tiers.rs b/crates/cargo-tops/src/tiers.rs index 42a1097..343d76f 100644 --- a/crates/cargo-tops/src/tiers.rs +++ b/crates/cargo-tops/src/tiers.rs @@ -234,8 +234,8 @@ fn live_exemption(doc: &Value, layer: &str, today: &str) -> bool { /// Whether an exemption path covers the whole crate, the only reach that may /// lift a floor (VD-R7). A narrower glob exempts files, not the floor. A record /// that names a symbol exempts one function, so it never lifts a floor. -fn covers_crate(path: &str) -> bool { - matches!(path, "**" | "./**" | "**/*" | "." | "./") +pub(crate) fn covers_crate(path: &str) -> bool { + matches!(path, "**" | "./**" | "**/*" | "*" | "." | "./") } fn required_problems( diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md index e26ddcd..5d50a68 100644 --- a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/README.md @@ -25,3 +25,21 @@ times out. The file is real output, not a hand-built shape. To re-record it, run the same command over the same crate with cargo-mutants 27.1.0 and shorten the paths. + +## null-function/ + +`outcomes.json` from `cargo mutants -j 1 --output ` over a crate whose +`src/lib.rs` is a `const SIZE: usize = 2 + 3`, a `static LIMIT: u32 = 10 * 4`, +a `type Buf = [u8; 4 + 4]`, a struct field `[u8; 1 + 1]` and one untested +`fn get() -> i32 { 2 + 2 }`, with the toolchain path shortened to `cargo`. It +holds 12 missed mutants and 1 unviable one. The 7 missed in the `const`, the +`static`, the alias and the field have `"function": null`; the 5 in `get` name +it. The file sits in `/mutants.out/outcomes.json`. + +## A diff with nothing to mutate + +Recorded from real 27.1.0 runs, `cargo mutants --in-diff --output +`: for a diff that changes only `README.md`, stderr is ` INFO Diff +changes no Rust source files`; for one that adds a doc comment, ` INFO No +mutants to filter`. Both exit 0, print nothing to stdout, and leave `` +empty: there is no `mutants.out` and no `outcomes.json`. diff --git a/crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json new file mode 100644 index 0000000..720f3b7 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/mutants-27.1.0/null-function/outcomes.json @@ -0,0 +1,750 @@ +{ + "outcomes": [ + { + "scenario": "Baseline", + "summary": "Success", + "log_path": "log/baseline.log", + "diff_path": null, + "phase_results": [ + { + "phase": "Build", + "duration": 0.346787875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.405252542, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:1:27: replace + with -", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 1, + "column": 27 + }, + "end": { + "line": 1, + "column": 28 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "Unviable", + "log_path": "log/src__lib.rs_line_1_col_27.log", + "diff_path": "diff/src__lib.rs_line_1_col_27.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.060888917, + "process_status": { + "Failure": 101 + }, + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:1:27: replace + with *", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 1, + "column": 27 + }, + "end": { + "line": 1, + "column": 28 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_1_col_27_001.log", + "diff_path": "diff/src__lib.rs_line_1_col_27_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.111410833, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.392369083, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:3:28: replace * with +", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 3, + "column": 28 + }, + "end": { + "line": 3, + "column": 29 + } + }, + "replacement": "+", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_3_col_28.log", + "diff_path": "diff/src__lib.rs_line_3_col_28.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113310791, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.460165542, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:3:28: replace * with /", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 3, + "column": 28 + }, + "end": { + "line": 3, + "column": 29 + } + }, + "replacement": "/", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_3_col_28_001.log", + "diff_path": "diff/src__lib.rs_line_3_col_28_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.118249292, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.298558834, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:5:23: replace + with -", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 5, + "column": 23 + }, + "end": { + "line": 5, + "column": 24 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_5_col_23.log", + "diff_path": "diff/src__lib.rs_line_5_col_23.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121582667, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.404460041, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:5:23: replace + with *", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 5, + "column": 23 + }, + "end": { + "line": 5, + "column": 24 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_5_col_23_001.log", + "diff_path": "diff/src__lib.rs_line_5_col_23_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.113811834, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.363051042, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:19: replace + with -", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 8, + "column": 19 + }, + "end": { + "line": 8, + "column": 20 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_19.log", + "diff_path": "diff/src__lib.rs_line_8_col_19.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.12164225, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.408842875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:8:19: replace + with *", + "package": "nofn", + "file": "src/lib.rs", + "function": null, + "span": { + "start": { + "line": 8, + "column": 19 + }, + "end": { + "line": 8, + "column": 20 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_8_col_19_001.log", + "diff_path": "diff/src__lib.rs_line_8_col_19_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120965875, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.068054125, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace get -> i32 with 0", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 10 + } + }, + "replacement": "0", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_5.log", + "diff_path": "diff/src__lib.rs_line_12_col_5.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.118840291, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 1.08889725, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace get -> i32 with 1", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 10 + } + }, + "replacement": "1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_5_001.log", + "diff_path": "diff/src__lib.rs_line_12_col_5_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.121670917, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.732568292, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:5: replace get -> i32 with -1", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 5 + }, + "end": { + "line": 12, + "column": 10 + } + }, + "replacement": "-1", + "genre": "FnValue" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_5_002.log", + "diff_path": "diff/src__lib.rs_line_12_col_5_002.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.120978583, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.233461208, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:7: replace + with - in get", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 7 + }, + "end": { + "line": 12, + "column": 8 + } + }, + "replacement": "-", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_7.log", + "diff_path": "diff/src__lib.rs_line_12_col_7.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.115573167, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.274348375, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + }, + { + "scenario": { + "Mutant": { + "name": "src/lib.rs:12:7: replace + with * in get", + "package": "nofn", + "file": "src/lib.rs", + "function": { + "function_name": "get", + "return_type": "-> i32", + "span": { + "start": { + "line": 11, + "column": 1 + }, + "end": { + "line": 13, + "column": 2 + } + } + }, + "span": { + "start": { + "line": 12, + "column": 7 + }, + "end": { + "line": 12, + "column": 8 + } + }, + "replacement": "*", + "genre": "BinaryOperator" + } + }, + "summary": "MissedMutant", + "log_path": "log/src__lib.rs_line_12_col_7_001.log", + "diff_path": "diff/src__lib.rs_line_12_col_7_001.diff", + "phase_results": [ + { + "phase": "Build", + "duration": 0.111984334, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--no-run", + "--verbose", + "--package=nofn@0.1.0" + ] + }, + { + "phase": "Test", + "duration": 0.232997167, + "process_status": "Success", + "argv": [ + "cargo", + "test", + "--verbose", + "--package=nofn@0.1.0" + ] + } + ] + } + ], + "total_mutants": 13, + "missed": 12, + "caught": 0, + "timeout": 0, + "unviable": 1, + "success": 0, + "start_time": "2026-11-20T06:57:08.422784Z", + "end_time": "2026-11-20T06:57:23.625455Z", + "cargo_mutants_version": "27.1.0" +} \ No newline at end of file diff --git a/crates/cargo-tops/tests/gate_base.rs b/crates/cargo-tops/tests/gate_base.rs index d26d1f3..6f7f563 100644 --- a/crates/cargo-tops/tests/gate_base.rs +++ b/crates/cargo-tops/tests/gate_base.rs @@ -25,15 +25,16 @@ mod profiles { const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); /// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; -/// `cargo mutants` writes what 27.1.0 writes for a run with no mutants; +/// `cargo mutants` does what 27.1.0 does for a diff with nothing to mutate; /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +# 27.1.0 finds nothing to mutate in a diff of comments, tests, docs or CI: +# it says so on stderr, exits 0 and writes no `mutants.out`. if [ "$1" = "mutants" ]; then - mkdir -p mutants.out - echo '{"outcomes":[{"scenario":"Baseline","summary":"Success"}],"total_mutants":0,"missed":0,"caught":0,"timeout":0,"unviable":0,"success":0,"cargo_mutants_version":"27.1.0"}' > mutants.out/outcomes.json + echo ' INFO No mutants to filter' >&2 fi exit 0 "#; @@ -849,6 +850,8 @@ fn the_help_and_the_schema_say_what_exit_2_covers() { "rust-tops.yaml", "rust-tops.lock", "no enabled layer at or below --max-tier", + "usage error", + "unreadable when a layer in range needs a pin", ] { assert!(text.contains(phrase), "{name} lacks `{phrase}`: {text}"); } diff --git a/crates/cargo-tops/tests/gate_real_mutants.rs b/crates/cargo-tops/tests/gate_real_mutants.rs new file mode 100644 index 0000000..124c207 --- /dev/null +++ b/crates/cargo-tops/tests/gate_real_mutants.rs @@ -0,0 +1,307 @@ +//! A diff with nothing to mutate, through the real cargo-mutants. Real 27.1.0 +//! exits 0 for a diff of a README or of a comment, writes no `mutants.out` +//! and says so on stderr; the layer passes as zero mutants. Only `cargo-deny` +//! is a stub. The test needs `cargo-mutants` 27.1.0 on `PATH`: without it +//! the test stops, unless `RUST_TOPS_REQUIRE_MUTANTS` is set, which fails it. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use serde_json::Value; + +/// The validator `cargo tops check` uses, and the schema it reads. +#[path = "../src/schema.rs"] +mod schema; + +mod profiles { + pub(crate) const SCHEMA_JSON: &str = include_str!("../templates/rust-tops.schema.json"); +} + +const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.json"); + +/// Logs its argv when `$CARGO_STUB_LOG` is set; answers every `--version`; +/// passes everything else. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac +if [ "$1" = "mutants" ]; then + out= + prev= + for arg in "$@"; do + if [ "$prev" = "--output" ]; then out=$arg; fi + prev=$arg + done + if [ -z "$out" ]; then echo "no --output given" >&2; exit 64; fi + if [ -n "${MUTANTS_SAYS:-}" ]; then echo "$MUTANTS_SAYS" >&2; fi + if [ -n "${MUTANTS_OUTCOMES:-}" ]; then + mkdir -p "$out/mutants.out" && cp "$MUTANTS_OUTCOMES" "$out/mutants.out/outcomes.json" + fi + exit "${MUTANTS_EXIT:-0}" +fi +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + home: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn at<'v>(value: &'v Value, pointer: &str) -> &'v Value { + static NULL: Value = Value::Null; + value.pointer(pointer).unwrap_or(&NULL) +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +fn git(fixture: &Fixture, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(dir) + .env("HOME", &fixture.home) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_AUTHOR_NAME", "fixture") + .env("GIT_AUTHOR_EMAIL", "fixture@example.invalid") + .env("GIT_COMMITTER_NAME", "fixture") + .env("GIT_COMMITTER_EMAIL", "fixture@example.invalid") + .output() + .expect("spawn git"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn set(doc: &mut Value, key: &str, value: Value) { + doc.as_object_mut() + .expect("an object") + .insert(key.to_string(), value); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A library-core instance named `fixture`, in a directory of its own with +/// a `Cargo.toml`, one source file and stub tools, not yet a git repository. +fn scaffold(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-base-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + home: root.join("home"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir bin"); + fs::create_dir_all(&fixture.home).expect("mkdir home"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-util", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); + fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); + stub(&fixture.bin, "cargo", CARGO_STUB); + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); + } + fixture +} + +/// The scaffold with `static-lint`, `security-advisory`, `property` and +/// `mutation` on, in a git repository whose one branch is `branch` and whose +/// one commit is everything `init` wrote. There is no remote. +fn fixture(name: &str, branch: &str) -> Fixture { + let fixture = scaffold(name); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!( + id, + "static-lint" | "security-advisory" | "property" | "mutation" + ); + set(layer, "enabled", Value::Bool(on)); + } + }); + git(&fixture, &fixture.repo, &["init", "-q", "-b", branch]); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "adopt"]); + fixture +} + +fn report(fixture: &Fixture) -> Value { + let raw = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("report.json was written"); + let doc: Value = serde_json::from_str(&raw).expect("report.json is JSON"); + let schema: Value = serde_json::from_str(REPORT_SCHEMA).expect("schema is JSON"); + let errors = schema::validate(&schema, &doc); + assert!( + errors.is_empty(), + "report.json breaks the schema: {errors:?}\n{raw}" + ); + doc +} + +fn mutation(report: &Value) -> &Value { + at(report, "/layers") + .as_array() + .and_then(|layers| layers.iter().find(|row| at(row, "/id") == "mutation")) + .unwrap_or_else(|| at(report, "/missing")) +} + +/// The entries of `problems` that start with `mutation:`. +fn mutation_problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|problem| problem.starts_with("mutation:")) + .map(str::to_string) + .collect() +} + +/// The real `cargo-mutants`, or `None` when it is not installed. Without it +/// the tests stop, unless `RUST_TOPS_REQUIRE_MUTANTS` is set: then they fail. +fn real_mutants() -> Option { + let path = std::env::var_os("PATH").unwrap_or_default(); + let version = Command::new("cargo") + .args(["mutants", "--version"]) + .env("PATH", &path) + .output(); + let said = version + .ok() + .filter(|out| out.status.success()) + .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string()); + if said.as_deref() == Some("cargo-mutants 27.1.0") { + return Some(path); + } + assert!( + std::env::var_os("RUST_TOPS_REQUIRE_MUTANTS").is_none(), + "RUST_TOPS_REQUIRE_MUTANTS is set and `cargo mutants --version` printed {said:?}, not cargo-mutants 27.1.0" + ); + eprintln!("cargo-mutants 27.1.0 is not installed: this test did not run"); + None +} + +/// A fixture with only `mutation` and `security-advisory` on, `cargo-deny` +/// a stub and everything else real, whose `feature` branch holds one more +/// commit that writes `file` as `text`. Returns the gate's output. +fn gate_after_writing(name: &str, file: &str, text: &str) -> Option<(Fixture, Output)> { + let path = real_mutants()?; + let fixture = fixture(name, "main"); + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!(id, "security-advisory" | "mutation"); + set(layer, "enabled", Value::Bool(on)); + } + }); + // Only `cargo-deny` stays a stub: the real cargo and cargo-mutants run. + for real in ["cargo", "cargo-mutants", "cargo-nextest"] { + fs::remove_file(fixture.bin.join(real)).expect("remove the stub"); + } + fs::write( + fixture.repo.join("src/lib.rs"), + "pub fn one() -> u8 {\n 1\n}\n", + ) + .expect("lib"); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "lib"]); + git( + &fixture, + &fixture.repo, + &["checkout", "-q", "-b", "feature"], + ); + fs::write(fixture.repo.join(file), text).expect("write the change"); + git(&fixture, &fixture.repo, &["add", "-A"]); + git(&fixture, &fixture.repo, &["commit", "-q", "-m", "change"]); + let mut dirs = vec![fixture.bin.clone()]; + dirs.extend(std::env::split_paths(&path)); + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["gate", "--base", "main", "--path"]) + .arg(&fixture.repo) + .env("PATH", std::env::join_paths(dirs).expect("PATH")) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env_remove("CARGO_TARGET_DIR") + .env_remove("CI") + .output() + .expect("spawn cargo-tops gate"); + Some((fixture, out)) +} + +fn expect_zero_mutants(fixture: &Fixture, out: &Output, said: &str) { + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(0), "{stdout}{stderr}"); + assert!( + stderr.contains(said), + "cargo mutants said {said:?}: {stderr}" + ); + let report = report(fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{row}"); + assert_eq!(at(row, "/judged/caught"), 0, "{row}"); + assert_eq!(at(row, "/judged/missed"), 0, "{row}"); + assert!(mutation_problems(&report).is_empty(), "{report}"); + // Real 27.1.0 writes no `mutants.out` for a diff with nothing to mutate. + let moved = fixture.repo.join("target/rust-tops/mutants.out"); + assert!(!moved.exists(), "27.1.0 wrote {}", moved.display()); + let here = fixture.repo.join("mutants.out"); + assert!(!here.exists(), "27.1.0 wrote {}", here.display()); +} + +#[test] +fn a_readme_only_diff_passes_through_the_real_cargo_mutants() { + if let Some((fixture, out)) = gate_after_writing("readme-only", "README.md", "# fixture\n") { + expect_zero_mutants(&fixture, &out, "Diff changes no Rust source files"); + } +} + +#[test] +fn a_comment_only_diff_passes_through_the_real_cargo_mutants() { + let changed = "/// One.\npub fn one() -> u8 {\n 1\n}\n"; + if let Some((fixture, out)) = gate_after_writing("comment-only", "src/lib.rs", changed) { + expect_zero_mutants(&fixture, &out, "No mutants to filter"); + } +} diff --git a/crates/cargo-tops/tests/gate_survivors.rs b/crates/cargo-tops/tests/gate_survivors.rs index bf08be1..b782e1b 100644 --- a/crates/cargo-tops/tests/gate_survivors.rs +++ b/crates/cargo-tops/tests/gate_survivors.rs @@ -8,8 +8,9 @@ //! unknown shape fails the layer. //! //! `cargo` is a stub that, for `mutants`, copies a planted `outcomes.json` -//! into `mutants.out/` and exits as cargo-mutants does (2 when mutants were -//! missed, 3 when some timed out). The planted files come from the fixture +//! into `<--output>/mutants.out/` and exits as cargo-mutants does (2 when +//! mutants were missed, 3 when some timed out). It writes nothing, and prints +//! what a run prints, when a test says so. The planted files come from the fixture //! recorded from a real 27.1.0 run. Every report is validated against the //! schema. use std::fs; @@ -35,8 +36,16 @@ if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac if [ "$1" = "mutants" ]; then + out= + prev= + for arg in "$@"; do + if [ "$prev" = "--output" ]; then out=$arg; fi + prev=$arg + done + if [ -z "$out" ]; then echo "no --output given" >&2; exit 64; fi + if [ -n "${MUTANTS_SAYS:-}" ]; then echo "$MUTANTS_SAYS" >&2; fi if [ -n "${MUTANTS_OUTCOMES:-}" ]; then - mkdir -p mutants.out && cp "$MUTANTS_OUTCOMES" mutants.out/outcomes.json + mkdir -p "$out/mutants.out" && cp "$MUTANTS_OUTCOMES" "$out/mutants.out/outcomes.json" fi exit "${MUTANTS_EXIT:-0}" fi @@ -248,6 +257,18 @@ fn mutation_problems(report: &Value) -> Vec { /// `inner::triple`), 1 timeout, 1 unviable. const RECORDED: &str = include_str!("fixtures/mutants-27.1.0/outcomes.json"); +/// A real 27.1.0 run over a crate with a `const`, a `static`, a type alias, a +/// struct field type and one untested `fn get`: 12 missed (7 with `function: +/// null`, 5 in `get`) and 1 unviable. +const NULL_FUNCTION: &str = include_str!("fixtures/mutants-27.1.0/null-function/outcomes.json"); + +/// `text` planted under the fixture. +fn planted_text(fixture: &Fixture, text: &str) -> PathBuf { + let path = fixture.root.join("outcomes.json"); + fs::write(&path, text).expect("plant outcomes.json"); + path +} + /// The recorded run with `change` applied, planted under the fixture. fn planted(fixture: &Fixture, change: impl FnOnce(&mut Value)) -> PathBuf { let mut doc: Value = serde_json::from_str(RECORDED).expect("fixture is JSON"); @@ -265,6 +286,32 @@ fn outcomes_mut(doc: &mut Value) -> &mut Vec { .expect("outcomes") } +/// Set each total to what the outcomes list, so an edit keeps the file whole. +fn recount(doc: &mut Value) { + let summaries: Vec = outcomes_mut(doc) + .iter() + .skip(1) + .map(|outcome| { + at(outcome, "/summary") + .as_str() + .unwrap_or_default() + .to_string() + }) + .collect(); + let of = |summary: &str| summaries.iter().filter(|s| *s == summary).count(); + let totals = [ + ("caught", of("CaughtMutant")), + ("missed", of("MissedMutant")), + ("timeout", of("Timeout")), + ("unviable", of("Unviable")), + ]; + let all: usize = totals.iter().map(|(_, n)| n).sum(); + for (key, n) in totals { + set(doc, key, json!(n)); + } + set(doc, "total_mutants", json!(all)); +} + /// Keep only the missed mutants for which `keep(function)` holds, and /// correct the totals. fn keep_missed(doc: &mut Value, keep: impl Fn(&str) -> bool) { @@ -274,11 +321,7 @@ fn keep_missed(doc: &mut Value, keep: impl Fn(&str) -> bool) { .unwrap_or_default(); at(outcome, "/summary") != "MissedMutant" || keep(function) }); - let missed = outcomes_mut(doc) - .iter() - .filter(|outcome| at(outcome, "/summary") == "MissedMutant") - .count(); - set(doc, "missed", json!(missed)); + recount(doc); } /// One survivor in `double` and one in `inner::triple`. @@ -296,11 +339,21 @@ fn one_each(doc: &mut Value) { seen.push(function); first }); - set(doc, "missed", json!(2)); + recount(doc); } /// Run the gate with `outcomes` planted, the way cargo-mutants exits for it. fn gate_with(fixture: &Fixture, outcomes: Option<&Path>, exit: &str) -> (Output, String) { + gate_saying(fixture, outcomes, exit, "") +} + +/// As [`gate_with`], and `says` is what the run prints to stderr. +fn gate_saying( + fixture: &Fixture, + outcomes: Option<&Path>, + exit: &str, + says: &str, +) -> (Output, String) { git(fixture, &fixture.repo, &["checkout", "-q", "-b", "feature"]); commit_file(fixture, "b.rs"); let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); @@ -310,6 +363,7 @@ fn gate_with(fixture: &Fixture, outcomes: Option<&Path>, exit: &str) -> (Output, .env("HOME", &fixture.home) .env("GIT_CONFIG_NOSYSTEM", "1") .env("MUTANTS_EXIT", exit) + .env("MUTANTS_SAYS", says) .env_remove("MUTANTS_OUTCOMES") .env_remove("CARGO_TARGET_DIR") .env_remove("CI"); @@ -491,7 +545,7 @@ fn a_maximum_of_one_passes_one_unexplained_survivor_and_fails_two() { outcomes_mut(doc).retain(|o| { at(o, "/summary") != "MissedMutant" || at(o, "/scenario/Mutant/replacement") == "0" }); - set(doc, "missed", json!(1)); + recount(doc); }); let (out, text) = gate_with(&first, Some(&one), "2"); assert_eq!(out.status.code(), Some(0), "{text}"); @@ -563,7 +617,7 @@ fn a_file_that_is_not_json_fails() { fn no_outcomes_file_is_a_failure_and_a_stale_one_is_never_read() { let fixture = fixture("no-outcomes", "main"); // An earlier run left a clean file behind. This run wrote none. - let stale = fixture.repo.join("mutants.out"); + let stale = fixture.repo.join("target/rust-tops/mutants.out"); fs::create_dir_all(&stale).expect("mkdir"); let clean = planted(&fixture, |doc| keep_missed(doc, |_| false)); fs::copy(&clean, stale.join("outcomes.json")).expect("plant stale outcomes"); @@ -578,4 +632,204 @@ fn a_dry_run_reads_no_outcomes() { let (out, text) = gate(&fixture, &["--dry-run"], None); assert_eq!(out.status.code(), Some(1), "{text}"); assert!(!fixture.repo.join("mutants.out").exists()); + assert!(!fixture.repo.join("target/rust-tops/mutants.out").exists()); +} + +/// What 27.1.0 prints when the diff leaves nothing to mutate. +const NO_MUTANTS: [&str; 2] = [ + " INFO Diff changes no Rust source files", + " INFO No mutants to filter", +]; + +#[test] +fn a_diff_with_nothing_to_mutate_passes_as_zero_mutants() { + for says in NO_MUTANTS { + let fixture = fixture("nothing-to-mutate", "main"); + let (out, text) = gate_saying(&fixture, None, "0", says); + assert_eq!(out.status.code(), Some(0), "{says}: {text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(at(row, "/verdict"), "pass", "{says}: {row}"); + assert_eq!(counts(row), [0; 6], "{says}: {row}"); + assert!(mutation_problems(&report).is_empty()); + } +} + +#[test] +fn no_file_is_zero_mutants_only_when_the_run_said_so_and_exited_0() { + // Said nothing, or something else: the file went missing. + for says in [ + "", + " INFO Found 3 mutants to test", + " INFO Diff file is empty", + ] { + let fixture = fixture("went-missing", "main"); + let (out, text) = gate_saying(&fixture, None, "0", says); + assert_eq!(out.status.code(), Some(1), "{says:?}: {text}"); + let problems = mutation_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!( + problems + .iter() + .all(|p| p.contains("outcomes.json") && p.contains("exited 0")), + "{problems:?}" + ); + } + // The line, and an exit that says mutants were missed, or any failure. + for exit in ["2", "3", "1", "4"] { + let fixture = fixture("line-and-exit", "main"); + let (out, text) = gate_saying(&fixture, None, exit, NO_MUTANTS[1]); + assert_eq!(out.status.code(), Some(1), "exit {exit}: {text}"); + } +} + +#[test] +fn a_failed_cargo_mutants_says_what_its_exit_means() { + for (exit, words) in [ + ("1", "exited 1"), + ("4", "unmutated tree"), + ("9", "exited 9"), + ] { + let fixture = fixture("exit-meaning", "main"); + let (out, text) = gate_with(&fixture, None, exit); + assert_eq!(out.status.code(), Some(1), "exit {exit}: {text}"); + let problems = mutation_problems(&report(&fixture)); + assert!( + problems + .iter() + .all(|p| p.contains("cargo mutants") && p.contains(words)), + "exit {exit}: {problems:?}" + ); + assert_eq!(problems.len(), 1, "{problems:?}"); + } +} + +#[test] +fn the_gate_names_the_output_directory_so_an_adopters_setting_cannot_move_it() { + let fixture = fixture("output-dir", "main"); + fs::create_dir_all(fixture.repo.join(".cargo")).expect("mkdir"); + fs::write( + fixture.repo.join(".cargo/mutants.toml"), + "output = \"elsewhere\"\n", + ) + .expect("write mutants.toml"); + let file = planted(&fixture, |doc| keep_missed(doc, |_| false)); + let (out, text) = gate_with(&fixture, Some(&file), "3"); + assert_eq!(out.status.code(), Some(0), "{text}"); + assert!(fixture + .repo + .join("target/rust-tops/mutants.out/outcomes.json") + .exists()); + assert!(!fixture.repo.join("elsewhere").exists()); +} + +type Disagreement = (&'static str, Edit, &'static str, &'static [&'static str]); + +/// Each planted file, with the exit its run had, fails naming `words`. +fn expect_disagreements(name: &str, cases: &[Disagreement]) { + for (what, change, exit, words) in cases { + let fixture = fixture(name, "main"); + let file = planted(&fixture, *change); + let (out, text) = gate_with(&fixture, Some(&file), exit); + assert_eq!(out.status.code(), Some(1), "{what}: {text}"); + let problems = mutation_problems(&report(&fixture)); + assert_eq!(problems.len(), 1, "{what}: {problems:?}"); + for word in *words { + assert!( + problems.iter().all(|p| p.contains(word)), + "{what} lacks {word}: {problems:?}" + ); + } + } +} + +#[test] +fn a_total_that_is_wrong_or_missing_fails_naming_both_numbers() { + expect_disagreements( + "bad-total", + &[ + ( + "a wrong total", + |d| set(d, "total_mutants", json!(999)), + "2", + &["says 999", "lists 23"], + ), + ( + "no total", + |d| { + d.as_object_mut().expect("object").remove("total_mutants"); + }, + "2", + &["total_mutants"], + ), + ], + ); +} + +#[test] +fn an_exit_code_the_file_does_not_bear_out_fails_naming_both() { + expect_disagreements( + "bad-exit", + &[ + ( + "exit 2, nothing missed", + |d| keep_missed(d, |_| false), + "2", + &["exited 2", "no missed"], + ), + ( + "exit 3, no timeout", + |d| { + outcomes_mut(d).retain(|o| at(o, "/summary") != "Timeout"); + set(d, "timeout", json!(0)); + set(d, "total_mutants", json!(22)); + }, + "3", + &["exited 3", "no timeout"], + ), + ], + ); +} + +#[test] +fn a_survivor_in_no_function_is_explained_only_by_a_record_with_no_symbol() { + // 7 of the 12 survivors are in a const, a static, a type alias and a + // field type: cargo-mutants 27.1.0 writes `function: null` for them. + let unexplained = |symbol: Option<&str>, path: &str| -> u64 { + let fixture = fixture("null-function", "main"); + exempt(&fixture, path, symbol, "2099-01-01"); + let file = planted_text(&fixture, NULL_FUNCTION); + let (_, text) = gate_with(&fixture, Some(&file), "2"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(counts(row)[1], 12, "{text}"); + counts(row)[3] + }; + assert_eq!(unexplained(Some("get"), "src/lib.rs"), 7); + assert_eq!(unexplained(Some("no_such_fn"), "src/lib.rs"), 12); + assert_eq!(unexplained(None, "src/lib.rs"), 0); + assert_eq!(unexplained(None, "src/**"), 0); +} + +#[test] +fn a_whole_tree_record_lifts_the_floor_and_explains_no_survivor() { + for path in ["**", "./**", "**/*", "*", ".", "./"] { + let fixture = fixture("whole-tree", "main"); + set_max(&fixture, 10); + exempt(&fixture, path, None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(0), "{path}: {text}"); + let report = report(&fixture); + let row = mutation(&report); + // 10 missed, none explained: the max decides. + assert_eq!(counts(row), [11, 10, 0, 10, 1, 1], "{path}: {row}"); + } + // At a max of 9 the same record changes nothing: 10 survivors fail. + let fixture = fixture("whole-tree-nine", "main"); + set_max(&fixture, 9); + exempt(&fixture, "**", None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); } diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 9d63b42..7c98d53 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -50,10 +50,10 @@ if [ "$*" = "crap --version" ]; then echo 'cargo-crap 0.9.0' exit 0 fi -# `cargo mutants` leaves what 27.1.0 leaves for a run with no mutants. +# 27.1.0 finds nothing to mutate in a diff of comments, tests, docs or CI: +# it says so on stderr, exits 0 and writes no `mutants.out`. if [ "$1" = "mutants" ]; then - mkdir -p mutants.out - echo '{"outcomes":[{"scenario":"Baseline","summary":"Success"}],"total_mutants":0,"missed":0,"caught":0,"timeout":0,"unviable":0,"success":0,"cargo_mutants_version":"27.1.0"}' > mutants.out/outcomes.json + echo ' INFO No mutants to filter' >&2 fi prev= for arg in "$@"; do @@ -1861,7 +1861,12 @@ fn a_real_diff_runs_mutants_on_the_report_dir_diff() { let body = fs::read_to_string(&diff).expect("diff under the report dir"); assert!(body.contains("src/lib.rs"), "{body}"); let log = cargo_log(&fixture); - let want = format!("mutants --in-diff {} -j 2", diff.display()); + let report_dir = diff.parent().expect("the diff is in the report dir"); + let want = format!( + "mutants --in-diff {} -j 2 --output {}", + diff.display(), + report_dir.display() + ); assert!(log.lines().any(|line| line == want), "{log}"); assert!(!log.contains("/tmp/rust-tops.diff"), "{log}"); assert!(!text.contains("/tmp/rust-tops.diff"), "{text}"); diff --git a/schema/gate-report.schema.json b/schema/gate-report.schema.json index 27aeb9c..e65feb8 100644 --- a/schema/gate-report.schema.json +++ b/schema/gate-report.schema.json @@ -39,7 +39,7 @@ "enum": ["pass", "fail", "config-invalid", "environment"] }, "exit_code": { - "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, rust-tops.lock unreadable, or no enabled layer at or below --max-tier; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", + "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable when a layer in range needs a pin (crap today); a command-line usage error is 2 too and writes no report; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", "type": "integer", "enum": [0, 1, 2, 3] }, From b56e7293298bba90c6bd6e9bf49363eaad39f712 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 02:23:49 -0500 Subject: [PATCH 08/10] Test the laws verdict and the mutants spawn path, and simplify the exit note The gate's own mutants run found two survivors: the guard that tells a missing program from another spawn error in run_mutants, and the laws row verdict now decided in judge.rs. Tests kill both. exit_note scored a CRAP just over the threshold; the exit meaning is its own small function. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/gate/execute.rs | 48 +++++++++++++++++++-------- crates/cargo-tops/src/gate/judge.rs | 44 +++++++++++++++++++++++- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index 0f02d96..f9f36f9 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -479,7 +479,9 @@ pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Resu /// outcomes it wrote. fn settled(mut run: Run, ended: &Ended, output: &Path) -> Mutated { if ended.outcome != Outcome::Passed { - run.note = exit_note(&run.step, ended); + // A missing tool has its own line; a failed run says what its exit means. + run.note = (ended.outcome == Outcome::Failed) + .then(|| format!("`{}` {}", run.step, exit_meaning(ended.code))); return Mutated::without_outcomes(run); } let found = mutation::read(output, ended.code.unwrap_or_default(), &ended.stderr); @@ -494,17 +496,15 @@ fn settled(mut run: Run, ended: &Ended, output: &Path) -> Mutated { } /// What a failed `cargo mutants` exit means, in cargo-mutants' own terms. -fn exit_note(step: &str, ended: &Ended) -> Option { - if ended.outcome != Outcome::Failed { - return None; - } - let what = match ended.code { - Some(1) => "exited 1: it was given arguments it does not accept", - Some(4) => "exited 4: the tests fail in the unmutated tree, so no mutant was tested", - Some(code) => return Some(format!("`{step}` exited {code}")), - None => "was ended by a signal", - }; - Some(format!("`{step}` {what}")) +fn exit_meaning(code: Option) -> String { + match code { + Some(1) => "exited 1: it was given arguments it does not accept".to_string(), + Some(4) => { + "exited 4: the tests fail in the unmutated tree, so no mutant was tested".to_string() + } + Some(code) => format!("exited {code}"), + None => "was ended by a signal".to_string(), + } } /// The run of a layer that stopped on its base or its diff: prints the @@ -653,7 +653,7 @@ fn spawn(dir: &Path, step: &Step) -> Result { #[cfg(test)] mod tests { - use super::{run_mutants, run_step, version_ask, Executed, Timing}; + use super::{exit_meaning, run_mutants, run_step, version_ask, Executed, Timing}; use crate::facts::Outcome; use crate::steps::Step; use std::fs; @@ -758,4 +758,26 @@ mod tests { Outcome::ToolMissing ); } + + #[test] + fn a_mutants_spawn_error_is_an_error_and_a_signal_is_named() { + let dir = std::env::temp_dir().join(format!("cargo-tops-mutexec-{}", std::process::id())); + fs::remove_dir_all(&dir).ok(); + fs::create_dir_all(&dir).expect("mkdir"); + let plain = dir.join("plain"); + fs::write(&plain, "x").expect("write"); + let blocked = Step::command([plain.display().to_string()]); + let err = run_mutants(Path::new("."), &blocked) + .err() + .expect("not spawnable"); + assert!(format!("{err:#}").contains("spawn"), "{err:#}"); + fs::remove_dir_all(&dir).ok(); + let killed = Step::command(["sh".to_string(), "-c".to_string(), "kill -9 $$".to_string()]); + let ended = run_mutants(Path::new("."), &killed).expect("it ran"); + assert_eq!((ended.outcome, ended.code), (Outcome::Failed, None)); + assert!(exit_meaning(None).contains("signal")); + assert!(exit_meaning(Some(4)).contains("unmutated tree")); + assert!(exit_meaning(Some(1)).contains("exited 1")); + assert_eq!(exit_meaning(Some(77)), "exited 77"); + } } diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 98f4340..7dcaaad 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -501,7 +501,8 @@ pub(crate) fn run_failure(run: &Run) -> Option { #[cfg(test)] mod tests { - use super::{clear_cases, crate_prefix}; + use super::{clear_cases, crate_prefix, laws_verdict, Judgement}; + use crate::floors; use std::fs; use std::path::PathBuf; @@ -560,4 +561,45 @@ mod tests { assert_eq!(crate_prefix(&empty, "cargo-tops"), ""); fs::remove_dir_all(&empty).ok(); } + + /// A judgement of a run in which the laws layer did not run. + fn quiet() -> Judgement { + Judgement { + failed: Vec::new(), + evaluated: String::new(), + count: String::new(), + ran: 0, + commands: 0, + vacuous: 0, + measured: floors::Judged::default(), + laws_seconds: 0.0, + bend_ran: false, + laws_failed: false, + environment: false, + law_total: 0, + survivors: None, + } + } + + #[test] + fn the_laws_row_is_not_run_unless_bend_ran_and_then_pass_or_fail() { + assert_eq!(laws_verdict(None), "not-run"); + assert_eq!(laws_verdict(Some(&quiet())), "not-run"); + let failed_without_bend = Judgement { + laws_failed: true, + ..quiet() + }; + assert_eq!(laws_verdict(Some(&failed_without_bend)), "not-run"); + let passed = Judgement { + bend_ran: true, + ..quiet() + }; + assert_eq!(laws_verdict(Some(&passed)), "pass"); + let failed = Judgement { + bend_ran: true, + laws_failed: true, + ..quiet() + }; + assert_eq!(laws_verdict(Some(&failed)), "fail"); + } } From f4bf0868717548c476d6331e75108d61fd62a969 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 03:13:32 -0500 Subject: [PATCH 09/10] Decide the whole tree from the files, and keep the real-mutants test from skipping B3, S6: a symbol-free layer: mutation record is the whole tree when its glob matches every Rust source file the mutation scope examines (the .rs files under src/, and build.rs), or is made only of * and ** components, or is the crate root. One function, tiers::whole_tree, answers that for both check (the floor lift) and the gate (a record that explains no survivor), so /**, **/, **/**, *.rs, **/*.rs, */** and a single-file crate's only file are held to the same rule as **. Every spelling is tested at both ends. S7: the coverage and CRAP floors keep their own list of whole-crate paths, as before; * is not on it. S8: the tops-gate step sets RUST_TOPS_REQUIRE_MUTANTS, so the real cargo-mutants test cannot skip in CI. N6: the no-mutants line must be a whole stderr line (colour codes aside), not a phrase inside another line. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/rust-tops.yml | 4 + PROTOCOL.md | 2 +- crates/cargo-tops/src/gate/judge.rs | 5 +- crates/cargo-tops/src/main.rs | 8 +- crates/cargo-tops/src/mutation.rs | 137 +++++++++++++----- crates/cargo-tops/src/scope.rs | 52 +++++++ crates/cargo-tops/src/tiers.rs | 163 ++++++++++++++++++++-- crates/cargo-tops/tests/gate_survivors.rs | 9 +- 8 files changed, 328 insertions(+), 52 deletions(-) diff --git a/.github/workflows/rust-tops.yml b/.github/workflows/rust-tops.yml index c3ba44d..38e3505 100644 --- a/.github/workflows/rust-tops.yml +++ b/.github/workflows/rust-tops.yml @@ -386,6 +386,10 @@ jobs: tool: cargo-mutants@27.1.0 - run: cargo install --path crates/cargo-tops --locked - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" + env: + # The gate's own tests include a run of the real cargo-mutants: it + # must run here, never skip. + RUST_TOPS_REQUIRE_MUTANTS: "1" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default # features usually need no FFI sysdeps. Do not mark required until one diff --git a/PROTOCOL.md b/PROTOCOL.md index 01770be..20537d5 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -981,7 +981,7 @@ Expired exemptions fail CI. Reasons shorter than one clause are rejected. `cargo tops gate` checks every record, whatever its `layer`, whether that layer is on or off, and at any `--max-tier`: a record that has expired, or whose `expires` is not a real day, is a problem line in the report and the gate exits 1 (never 2; the config is still valid). `cargo tops check` prints the same lines as warnings and exits 0. The day is UTC: a record expires at 00:00 UTC on its `expires` date, whatever the local time zone. -How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains the survivors in the matching files, or, with `symbol`, in the function of that name; a record with no `symbol` covers every function in its glob, and the survivors that are in no function (a `const` or `static` initialiser, a type alias, a field type: cargo-mutants writes `function: null` for them). A record with a `symbol` never explains a survivor in no function. A record with no `symbol` whose path is the whole tree (`**`, `./**`, `**/*`, `*`, `.`, `./`) lifts the class floor on `unexplained_survivors_max` (§4) and explains no survivor; to explain survivors, a record needs a `symbol` or a narrower glob. A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. +How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains the survivors in the matching files, or, with `symbol`, in the function of that name; a record with no `symbol` covers every function in its glob, and the survivors that are in no function (a `const` or `static` initialiser, a type alias, a field type: cargo-mutants writes `function: null` for them). A record with a `symbol` never explains a survivor in no function. A record with no `symbol` that is the whole tree lifts the class floor on `unexplained_survivors_max` (§4) and explains no survivor; to explain survivors, a record needs a `symbol` or a narrower glob. "The whole tree" is decided from the files: the glob matches every Rust source file the mutation scope examines (the `.rs` files under `src/`, and `build.rs`), or is made of `*` and `**` components only, or is `.`. So `**`, `/**`, `**/`, `**/**`, `*`, `*.rs`, `**/*.rs`, `*/**` and `.` are the whole tree, and so is a glob such as `src/**` in a crate whose files are all under `src/`, or a single-file crate's only file. `cargo tops check` and the gate ask the same question. The coverage and CRAP floors keep their own list of whole-crate paths (`**`, `./**`, `**/*`, `.`, `./`). A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. --- diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 7dcaaad..b60c3a6 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -12,7 +12,7 @@ use super::{Fault, Mode}; use crate::config::{self, TopsConfig}; use crate::facts::{self, Outcome, Run}; use crate::mutation::Ledger; -use crate::scope::crate_root; +use crate::scope::{crate_root, examined_files}; use crate::{config_path, exemptions, floors, laws, mutation, profiles, steps}; /// What the judging found. @@ -179,7 +179,8 @@ fn judge_survivors(inputs: &Inputs<'_, '_>, failed: &mut Vec) -> Option< live.retain(|record| record.live(&today)); let prefix = crate_prefix(inputs.dir, &cfg.crate_.name); let max = cfg.gates.mutation.unexplained_survivors_max; - let ledger = mutation::explain(found, &live, &prefix); + let examined = examined_files(&crate_root(inputs.dir, &cfg.crate_.name)); + let ledger = mutation::explain(found, &live, &prefix, &examined); failed.extend(ledger.problem(max)); Some(Survivors { ledger, max }) } diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 2c0217c..b7507af 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -258,7 +258,13 @@ fn load_config(path: &Path) -> Result { errors.join("\n ") )); } - let problems = tiers::check(&file.display().to_string(), &doc, &exemptions::today()); + let examined = scope::examined_files(path); + let problems = tiers::check_with( + &file.display().to_string(), + &doc, + &exemptions::today(), + &examined, + ); if !problems.is_empty() { return Err(anyhow!("{}", problems.join("\n"))); } diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs index 439c65c..1080ae1 100644 --- a/crates/cargo-tops/src/mutation.rs +++ b/crates/cargo-tops/src/mutation.rs @@ -17,7 +17,7 @@ use anyhow::{Context, Result}; use serde_json::{json, Value}; use crate::exemptions::Exemption; -use crate::tiers::covers_crate; +use crate::tiers::whole_tree; /// Where cargo-mutants writes the run's outcomes, under the directory it ran in. const OUTCOMES: &str = "mutants.out/outcomes.json"; @@ -83,7 +83,7 @@ pub(crate) fn clear(dir: &Path) -> Result<()> { pub(crate) fn read(dir: &Path, code: i32, stderr: &str) -> Result { let text = match fs::read_to_string(dir.join(OUTCOMES)) { Ok(text) => text, - Err(_) if code == 0 && NO_MUTANTS.iter().any(|line| stderr.contains(line)) => { + Err(_) if code == 0 && says_no_mutants(stderr) => { return Ok(Outcomes::default()); } Err(err) => { @@ -98,6 +98,38 @@ pub(crate) fn read(dir: &Path, code: i32, stderr: &str) -> Result bool { + stderr.lines().any(|line| { + let plain = without_colour(line); + let text = plain.trim(); + NO_MUTANTS.iter().any(|said| { + text.strip_prefix("INFO") + .is_some_and(|rest| rest.trim() == *said) + }) + }) +} + +/// `line` without ANSI colour sequences. +fn without_colour(line: &str) -> String { + let mut plain = String::new(); + let mut chars = line.chars(); + while let Some(c) = chars.next() { + if c == '\u{1b}' { + // `ESC [ ... letter` + for skipped in chars.by_ref() { + if skipped.is_ascii_alphabetic() { + break; + } + } + } else { + plain.push(c); + } + } + plain +} + /// The exit code and the file must agree: 2 says mutants were missed and 3 /// says some timed out. fn check_exit(found: &Outcomes, code: i32) -> Result<(), String> { @@ -268,12 +300,12 @@ fn relative<'f>(file: &'f str, crate_prefix: &str) -> &'f str { /// Whether `record` explains a survivor in `function` of the file `rel` /// (`None`: in no function, as in a `const`, a `static`, a type alias or a -/// field type). A record with no symbol and a whole-tree path lifts a floor -/// and explains nothing. A symbol names a function, so a survivor in none is +/// field type). A record with no symbol that matches every Rust file the +/// scope examines ([`whole_tree`]) lifts a floor and explains nothing. A symbol names a function, so a survivor in none is /// explained only by a record with no symbol. -fn explains(record: &Exemption, rel: &str, function: Option<&str>) -> bool { +fn explains(record: &Exemption, rel: &str, function: Option<&str>, examined: &[String]) -> bool { let named = record.symbol.is_some(); - if !named && covers_crate(&record.path) { + if !named && whole_tree(&record.path, examined) { return false; } if named && function.is_none() { @@ -285,11 +317,16 @@ fn explains(record: &Exemption, rel: &str, function: Option<&str>) -> bool { /// Sort the missed mutants. `live` are the layer's records that have not /// expired; `crate_prefix` is the crate's directory under the workspace root, /// which the records' globs are relative to. -pub(crate) fn explain(found: &Outcomes, live: &[Exemption], crate_prefix: &str) -> Ledger { +pub(crate) fn explain( + found: &Outcomes, + live: &[Exemption], + crate_prefix: &str, + examined: &[String], +) -> Ledger { let (explained, unexplained) = found.missed.iter().cloned().partition(|mutant: &Missed| { let rel = relative(&mutant.file, crate_prefix); live.iter() - .any(|record| explains(record, rel, mutant.function.as_deref())) + .any(|record| explains(record, rel, mutant.function.as_deref(), examined)) }); Ledger { caught: found.caught, @@ -456,7 +493,7 @@ mod tests { #[test] fn a_record_for_one_function_explains_only_its_survivors() { let found = parse(FIXTURE).expect("parses"); - let ledger = explain(&found, &[record("src/**", Some("double"))], ""); + let ledger = explain(&found, &[record("src/**", Some("double"))], "", &[]); assert_eq!(ledger.explained.len(), 5); assert_eq!(ledger.unexplained.len(), 5); assert!(ledger @@ -476,17 +513,17 @@ mod tests { #[test] fn a_symbol_matches_the_last_path_segment_and_a_symbol_free_record_covers_its_glob() { let found = parse(FIXTURE).expect("parses"); - let nested = explain(&found, &[record("src/lib.rs", Some("triple"))], ""); + let nested = explain(&found, &[record("src/lib.rs", Some("triple"))], "", &[]); assert_eq!(nested.explained.len(), 5); let both = [ record("src/lib.rs", Some("double")), record("src/lib.rs", Some("inner::triple")), ]; - assert!(explain(&found, &both, "").unexplained.is_empty()); - let all = explain(&found, &[record("src/**", None)], ""); + assert!(explain(&found, &both, "", &[]).unexplained.is_empty()); + let all = explain(&found, &[record("src/**", None)], "", &[]); assert_eq!(all.explained.len(), 10); assert!(all.passes(0) && all.problem(0).is_none()); - let elsewhere = explain(&found, &[record("other/**", None)], ""); + let elsewhere = explain(&found, &[record("other/**", None)], "", &[]); assert_eq!(elsewhere.unexplained.len(), 10); } @@ -505,24 +542,35 @@ mod tests { // workspace-relative file is `crates/sample/src/lib.rs`. let inside = [record("src/lib.rs", None)]; assert_eq!( - explain(&found, &inside, "crates/sample").explained.len(), + explain(&found, &inside, "crates/sample", &[]) + .explained + .len(), 10 ); assert_eq!( - explain(&found, &inside, "crates/sample/").explained.len(), + explain(&found, &inside, "crates/sample/", &[]) + .explained + .len(), 10 ); // A record that spells the workspace path matches only when no // prefix is stripped. let spelled = [record("crates/sample/src/lib.rs", None)]; - assert_eq!(explain(&found, &spelled, "").explained.len(), 10); + assert_eq!(explain(&found, &spelled, "", &[]).explained.len(), 10); assert_eq!( - explain(&found, &spelled, "crates/sample").explained.len(), + explain(&found, &spelled, "crates/sample", &[]) + .explained + .len(), 0 ); // A record for another crate's file explains nothing. let other = [record("crates/other/src/lib.rs", None)]; - assert_eq!(explain(&found, &other, "crates/sample").explained.len(), 0); + assert_eq!( + explain(&found, &other, "crates/sample", &[]) + .explained + .len(), + 0 + ); } #[test] @@ -539,7 +587,7 @@ mod tests { #[test] fn the_judged_numbers_count_every_kind_and_the_max() { let found = parse(FIXTURE).expect("parses"); - let ledger = explain(&found, &[record("src/lib.rs", Some("double"))], ""); + let ledger = explain(&found, &[record("src/lib.rs", Some("double"))], "", &[]); let judged = ledger.to_json(2); assert_eq!(judged["caught"], 11); assert_eq!(judged["missed"], 10); @@ -558,7 +606,7 @@ mod tests { timeouts: 4, unviable: 9, }; - let ledger = explain(&only, &[], ""); + let ledger = explain(&only, &[], "", &[]); assert!(ledger.passes(0) && ledger.problem(0).is_none()); assert_eq!(ledger.timeouts, 4); assert_eq!(ledger.unviable, 9); @@ -566,7 +614,7 @@ mod tests { #[test] fn a_negative_max_is_never_met() { - let ledger = explain(&Outcomes::default(), &[], ""); + let ledger = explain(&Outcomes::default(), &[], "", &[]); assert!(!ledger.passes(-1)); } @@ -588,15 +636,22 @@ mod tests { && m.function.as_deref() == Some("qa_h4_survivor_miss"))); let by_symbol = [record("src/qa_h4_plant.rs", Some("qa_h4_survivor_miss"))]; assert_eq!( - explain(&found, &by_symbol, "crates/plant").explained.len(), + explain(&found, &by_symbol, "crates/plant", &[]) + .explained + .len(), 2 ); - assert_eq!(explain(&found, &by_symbol, "").explained.len(), 2); + assert_eq!(explain(&found, &by_symbol, "", &[]).explained.len(), 2); let other = [record("src/qa_h4_plant.rs", Some("another"))]; - assert_eq!(explain(&found, &other, "crates/plant").unexplained.len(), 2); + assert_eq!( + explain(&found, &other, "crates/plant", &[]) + .unexplained + .len(), + 2 + ); let wrong_file = [record("src/lib.rs", Some("qa_h4_survivor_miss"))]; assert_eq!( - explain(&found, &wrong_file, "crates/plant") + explain(&found, &wrong_file, "crates/plant", &[]) .unexplained .len(), 2 @@ -649,8 +704,9 @@ mod tests { fn no_file_is_zero_mutants_only_for_exit_0_and_a_line_the_run_printed() { let dir = scratch("nothing"); for line in super::NO_MUTANTS { + let said = format!(" INFO {line}\n"); assert_eq!( - read(&dir, 0, line).expect("zero mutants"), + read(&dir, 0, &said).expect("zero mutants"), Outcomes::default() ); } @@ -659,14 +715,25 @@ mod tests { read(&dir, 0, &said).expect("zero mutants"), Outcomes::default() ); - for (code, said) in [ - (0, ""), - (0, "Diff file is empty"), - (2, NOTHING), - (3, NOTHING), - (1, NOTHING), + // With colour on, only the level is coloured. + let coloured = "\u{1b}[32m INFO\u{1b}[0m No mutants to filter\n"; + assert_eq!( + read(&dir, 0, coloured).expect("zero mutants"), + Outcomes::default() + ); + // Not a whole line of its own: the phrase inside another line is not it. + for said in [ + "WARN path src/No mutants to filter.rs ignored\n", + " INFO No mutants to filter.rs was skipped\n", + " WARN No mutants to filter\n", + "No mutants to filter\n", + "", + "Diff file is empty\n", ] { - assert!(read(&dir, code, said).is_err(), "exit {code} {said:?}"); + assert!(read(&dir, 0, said).is_err(), "{said:?}"); + } + for code in [2, 3, 1] { + assert!(read(&dir, code, NOTHING).is_err(), "exit {code}"); } // A file that is there is read, whatever the run printed. fs::create_dir_all(dir.join("mutants.out")).expect("mkdir"); @@ -734,7 +801,7 @@ mod tests { found.missed.iter().filter(|m| m.function.is_none()).count(), 7 ); - let count = |records: &[Exemption]| explain(&found, records, "").explained.len(); + let count = |records: &[Exemption]| explain(&found, records, "", &[]).explained.len(); assert_eq!(count(&[record("src/lib.rs", Some("get"))]), 5); assert_eq!(count(&[record("src/lib.rs", Some("no_such_fn"))]), 0); assert_eq!(count(&[record("src/lib.rs", None)]), 12); @@ -758,7 +825,7 @@ mod tests { #[test] fn a_survivor_is_named_once_by_its_own_name() { let found = parse(FIXTURE).expect("parses"); - let ledger = explain(&found, &[], ""); + let ledger = explain(&found, &[], "", &[]); let problem = ledger.problem(0).expect("fails"); let first = &found.missed[0]; assert_eq!(problem.matches(&first.name).count(), 1, "{problem}"); diff --git a/crates/cargo-tops/src/scope.rs b/crates/cargo-tops/src/scope.rs index 135f738..1b0812d 100644 --- a/crates/cargo-tops/src/scope.rs +++ b/crates/cargo-tops/src/scope.rs @@ -127,6 +127,35 @@ pub(crate) fn package_dir(metadata: &str, name: &str) -> Option { Path::new(manifest).parent().map(Path::to_path_buf) } +/// The Rust source files cargo-mutants examines in the crate at `root`, +/// relative to it with `/` separators: every `.rs` file under `src/`, and +/// `build.rs`. Sorted. +pub(crate) fn examined_files(root: &Path) -> Vec { + let mut found = Vec::new(); + walk_rs(&root.join("src"), "src", &mut found); + if root.join("build.rs").is_file() { + found.push("build.rs".to_string()); + } + found.sort(); + found +} + +fn walk_rs(dir: &Path, rel: &str, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.filter_map(Result::ok) { + let name = entry.file_name().to_string_lossy().into_owned(); + let path = entry.path(); + let below = format!("{rel}/{name}"); + if path.is_dir() { + walk_rs(&path, &below, found); + } else if name.ends_with(".rs") { + found.push(below); + } + } +} + /// Whether `glob` matches the `/`-separated `rel`, or a trailing part of it /// that starts at a directory. pub(crate) fn glob_matches(glob: &str, rel: &str) -> bool { @@ -269,4 +298,27 @@ mod tests { assert_eq!(crate_root(&empty, "cargo-tops"), empty); fs::remove_dir_all(&empty).expect("cleanup"); } + + #[test] + fn the_examined_files_are_the_rust_files_under_src_and_build_rs() { + let root = std::env::temp_dir().join(format!("cargo-tops-examined-{}", std::process::id())); + std::fs::remove_dir_all(&root).ok(); + for file in [ + "src/lib.rs", + "src/a/b.rs", + "src/a/data.txt", + "tests/t.rs", + "build.rs", + ] { + let path = root.join(file); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(&path, "").expect("write"); + } + assert_eq!( + super::examined_files(&root), + ["build.rs", "src/a/b.rs", "src/lib.rs"] + ); + assert!(super::examined_files(&root.join("nowhere")).is_empty()); + std::fs::remove_dir_all(&root).ok(); + } } diff --git a/crates/cargo-tops/src/tiers.rs b/crates/cargo-tops/src/tiers.rs index 343d76f..5d48981 100644 --- a/crates/cargo-tops/src/tiers.rs +++ b/crates/cargo-tops/src/tiers.rs @@ -11,6 +11,7 @@ use serde_json::Value; use crate::exemptions; use crate::profile_data::class_profile; +use crate::scope::glob_matches; /// The tier a layer runs at: the table's, else the instance's own for an id /// the table lacks, else 0. @@ -136,17 +137,30 @@ const REQUIRED_BY_PROFILE: [(&str, &str); 5] = [ /// Every problem in the instance `doc` (read from `file`): a tier that is not /// the table's, a floor looser than its class profile with no live exemption /// on `today` (`YYYY-MM-DD`), and a required layer that is off. -pub(crate) fn check(file: &str, doc: &Value, today: &str) -> Vec { +pub(crate) fn check_with(file: &str, doc: &Value, today: &str, examined: &[String]) -> Vec { let mut problems = tier_problems(file, doc); let class = doc.pointer("/crate/class").and_then(Value::as_str); let profile = class.and_then(class_profile); if let (Some(class), Some(profile)) = (class, profile.as_ref()) { - problems.extend(floor_problems(file, doc, class, profile, today)); + let when = When { today, examined }; + problems.extend(floor_problems(file, doc, class, profile, &when)); } problems.extend(required_problems(file, doc, class, profile.as_ref())); problems } +/// `check` with no Rust source files known: the tests of the floors. +#[cfg(test)] +fn check(file: &str, doc: &Value, today: &str) -> Vec { + check_with(file, doc, today, &[]) +} + +/// The day, and the Rust files of the crate, that a floor's lift is judged on. +struct When<'a> { + today: &'a str, + examined: &'a [String], +} + fn layers(doc: &Value) -> &[Value] { doc.get("layers") .and_then(Value::as_array) @@ -182,11 +196,11 @@ fn floor_problems( doc: &Value, class: &str, profile: &Value, - today: &str, + when: &When<'_>, ) -> Vec { FLOORS .iter() - .filter_map(|floor| floor_problem(file, doc, class, profile, today, floor)) + .filter_map(|floor| floor_problem(file, doc, class, profile, when, floor)) .collect() } @@ -195,7 +209,7 @@ fn floor_problem( doc: &Value, class: &str, profile: &Value, - today: &str, + when: &When<'_>, floor: &Floor, ) -> Option { let wanted = profile.pointer(floor.profile).and_then(Value::as_f64)?; @@ -208,7 +222,7 @@ fn floor_problem( Bound::Min => have < wanted, Bound::Max => have > wanted, }); - if !loose || live_exemption(doc, floor.layer, today) { + if !loose || live_exemption(doc, floor.layer, when) { return None; } let key = found.map_or(floor.key, |(key, _)| key); @@ -225,17 +239,49 @@ fn floor_problem( )) } -fn live_exemption(doc: &Value, layer: &str, today: &str) -> bool { +fn live_exemption(doc: &Value, layer: &str, when: &When<'_>) -> bool { exemptions::for_layer(doc.get("exemptions"), layer) .iter() - .any(|record| record.live(today) && record.symbol.is_none() && covers_crate(&record.path)) + .any(|record| { + record.live(when.today) + && record.symbol.is_none() + && lifts(layer, &record.path, when.examined) + }) +} + +/// Whether a record for `layer` at `path` covers the whole crate. The +/// mutation layer decides it from the crate's Rust files ([`whole_tree`]); +/// the coverage and CRAP floors keep the spellings of [`covers_crate`]. +fn lifts(layer: &str, path: &str, examined: &[String]) -> bool { + if layer == "mutation" { + whole_tree(path, examined) + } else { + covers_crate(path) + } +} + +/// Whether `path`, a glob of a `layer: mutation` record with no symbol, is +/// the whole tree: it matches every Rust source file the mutation scope +/// examines (`examined`, relative to the crate root), or, read as a pattern, +/// is made of `*` and `**` components only, or is `.`. Such a record lifts the +/// floor on `unexplained_survivors_max` and explains no survivor; `check` and +/// the gate both ask this one question. +pub(crate) fn whole_tree(path: &str, examined: &[String]) -> bool { + let parts: Vec<&str> = path + .split('/') + .filter(|part| !part.is_empty() && *part != ".") + .collect(); + let wild = parts.iter().all(|part| matches!(*part, "*" | "**")); + let every_file = !examined.is_empty() && examined.iter().all(|file| glob_matches(path, file)); + // No component left (`.`, `./`) names the crate root: `wild` holds then. + wild || every_file } /// Whether an exemption path covers the whole crate, the only reach that may /// lift a floor (VD-R7). A narrower glob exempts files, not the floor. A record /// that names a symbol exempts one function, so it never lifts a floor. -pub(crate) fn covers_crate(path: &str) -> bool { - matches!(path, "**" | "./**" | "**/*" | "*" | "." | "./") +fn covers_crate(path: &str) -> bool { + matches!(path, "**" | "./**" | "**/*" | "." | "./") } fn required_problems( @@ -281,7 +327,7 @@ fn required_problem(file: &str, doc: &Value, layer: &str, who: &str) -> Option Vec { + FILES.iter().map(|file| (*file).to_string()).collect() + } + + #[test] + fn every_spelling_that_reaches_every_file_is_the_whole_tree() { + let files = examined(); + for path in [ + "**", "/**", "**/", "**/**", "*", "*.rs", "**/*.rs", "*/**", "**/*", "./**", ".", "./", + "./**/*", "src/**", + ] { + assert!(whole_tree(path, &files), "{path}"); + } + // Spelling alone is enough for the wild ones, even with no file known. + for path in [ + "**", "/**", "**/", "**/**", "*", "*/**", "**/*", "./**", ".", "./", + ] { + assert!(whole_tree(path, &[]), "{path} with no files"); + } + // Narrower than the tree: one file, one directory, one other crate. + for path in [ + "src/lib.rs", + "src/gate/**", + "gate/*.rs", + "tests/**", + "src/other.rs", + ] { + assert!(!whole_tree(path, &files), "{path}"); + } + assert!(!whole_tree("src/lib.rs", &[])); + // A crate's only file is the whole tree. + assert!(whole_tree("src/lib.rs", &["src/lib.rs".to_string()])); + // A build script outside src/ is also examined. + let with_build = [files, vec!["build.rs".to_string()]].concat(); + assert!(!whole_tree("src/**", &with_build)); + assert!(whole_tree("**/*.rs", &with_build)); + } + + #[test] + fn a_mutation_record_lifts_the_floor_by_the_files_and_check_says_so() { + let files = examined(); + for path in [ + "**", "/**", "**/", "**/**", "*", "*.rs", "**/*.rs", "*/**", "src/**", + ] { + let mut doc = golden("library-core"); + set( + &mut doc, + "/gates/mutation/unexplained_survivors_max", + json!(3), + ); + set( + &mut doc, + "/exemptions", + exemption_at(path, "mutation", "2026-12-31"), + ); + assert!( + check_with("f.yaml", &doc, TODAY, &files).is_empty(), + "{path}" + ); + } + let mut doc = golden("library-core"); + set( + &mut doc, + "/gates/mutation/unexplained_survivors_max", + json!(3), + ); + set( + &mut doc, + "/exemptions", + exemption_at("src/lib.rs", "mutation", "2026-12-31"), + ); + assert!(!check_with("f.yaml", &doc, TODAY, &files).is_empty()); + } + + #[test] + fn a_star_record_does_not_relax_the_coverage_or_crap_floor() { + // As at base: `*` is not one of the spellings that lift those floors. + let mut doc = golden("library-core"); + set(&mut doc, "/gates/coverage/line_min_pct", json!(50)); + set( + &mut doc, + "/exemptions", + exemption_at("*", "coverage", "2026-12-31"), + ); + assert!(one(&doc).contains("gates.coverage.line_min_pct is 50")); + set( + &mut doc, + "/exemptions", + exemption_at("**", "coverage", "2026-12-31"), + ); + assert!(check("f.yaml", &doc, TODAY).is_empty()); + } } diff --git a/crates/cargo-tops/tests/gate_survivors.rs b/crates/cargo-tops/tests/gate_survivors.rs index b782e1b..b03e702 100644 --- a/crates/cargo-tops/tests/gate_survivors.rs +++ b/crates/cargo-tops/tests/gate_survivors.rs @@ -808,12 +808,17 @@ fn a_survivor_in_no_function_is_explained_only_by_a_record_with_no_symbol() { assert_eq!(unexplained(Some("get"), "src/lib.rs"), 7); assert_eq!(unexplained(Some("no_such_fn"), "src/lib.rs"), 12); assert_eq!(unexplained(None, "src/lib.rs"), 0); - assert_eq!(unexplained(None, "src/**"), 0); + // In this crate every file is under src/, so `src/**` is the whole tree. + assert_eq!(unexplained(None, "src/**"), 12); } #[test] fn a_whole_tree_record_lifts_the_floor_and_explains_no_survivor() { - for path in ["**", "./**", "**/*", "*", ".", "./"] { + // The listed spellings reach every Rust file, however they are written. + for path in [ + "**", "/**", "**/", "**/**", "./**", "**/*", "*", "*.rs", "**/*.rs", "*/**", ".", "./", + "src/**", + ] { let fixture = fixture("whole-tree", "main"); set_max(&fixture, 10); exempt(&fixture, path, None, "2099-01-01"); From 296b78006bae9d9a086e29173e1485bd8c6070c6 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 03:55:13 -0500 Subject: [PATCH 10/10] Explain a survivor per function, and drop the file walk B4, S9, N8: telling the whole tree from a part of it depended on how cargo-mutants walks files, and three reviews each found another shape that let one record silence the ledger. A survivor is now explained per function. One inside a function is explained only by a live mutation record whose symbol names that function and whose glob covers its file; one in no function (function: null) only by a record with no symbol whose glob covers its file. No record explains every survivor, so unexplained_survivors_max always decides. scope::examined_files, tiers::whole_tree and check_with's file list are gone: tiers.rs and scope.rs are as at main, so check's floor lift for the max, and the coverage and CRAP lists, are unchanged. The expectation that a symbol-free record covers every function in its glob is replaced by the per-function rule. PROTOCOL 8.3 and 19 say it. Co-Authored-By: Claude Sonnet 5.5 --- PROTOCOL.md | 6 +- crates/cargo-tops/src/gate/judge.rs | 5 +- crates/cargo-tops/src/main.rs | 8 +- crates/cargo-tops/src/mutation.rs | 148 +++++++++----------- crates/cargo-tops/src/scope.rs | 52 ------- crates/cargo-tops/src/tiers.rs | 159 ++-------------------- crates/cargo-tops/tests/gate_survivors.rs | 63 ++++++--- 7 files changed, 128 insertions(+), 313 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index 20537d5..f9f8952 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -497,7 +497,7 @@ The first source that is set wins. A ref that is named by the flag or the key an `CI` counts as set when it is not empty and is not `0` or `false` (in any case). When no source gives a base, the layer is `fail` if `CI` is set. The report's `detail` and the gate's problem line name `fetch-depth: 0` and `--base`: a checkout that never fetched the base must not read as a clean diff. When `CI` is not set, the layer is `vacuous`, and the detail lists each source tried. A base that resolves but shares no history with `HEAD`, as in a shallow checkout, fails the diff and names `fetch-depth: 0` too. CI workflows pass `--base "origin/${{ github.base_ref || github.event.repository.default_branch }}"` from a checkout with `fetch-depth: 0`, because `actions/checkout` may not create `origin/HEAD`. -How the gate judges the layer: it removes any `mutants.out/outcomes.json` from an earlier run, runs `cargo mutants --output `, and reads the `outcomes.json` that run wrote there, whatever `output` an adopter's own `.cargo/mutants.toml` sets. A diff with nothing to mutate (documentation, comments, tests, CI) makes cargo-mutants 27.1.0 exit 0 and write no `mutants.out`; it says `Diff changes no Rust source files` or `No mutants to filter` on stderr. Exit 0 with no file passes as zero mutants only when the run printed one of those lines; any other missing file fails. The file must be in the shape cargo-mutants 27.1.0 writes; another release, a missing field, an outcome the gate does not know, a failed baseline, or totals that disagree with the listed outcomes (`total_mutants` included) fail the layer and name `outcomes.json`. Exit 2 with no missed outcome listed, and exit 3 with no timeout listed, fail too. cargo-mutants exits 2 when mutants were missed and 3 when some timed out; the gate treats 0, 2 and 3 as a finished run and judges it from the file, and any other exit is a failed command. Each missed mutant is explained when a live `layer: mutation` exemption (§19) covers its file and function. Timeouts and unviable mutants are counted and reported; they are not survivors. The layer fails when the survivors nothing explains exceed `gates.mutation.unexplained_survivors_max`, and the problem names each of them. `kill_rate_min_pct` is a floor `check` compares with the class profile; the gate does not judge it. The mutation row's `judged` carries `caught`, `missed`, `explained`, `unexplained`, `timeouts`, `unviable` and `unexplained_survivors_max`, beside `base`. +How the gate judges the layer: it removes any `mutants.out/outcomes.json` from an earlier run, runs `cargo mutants --output `, and reads the `outcomes.json` that run wrote there, whatever `output` an adopter's own `.cargo/mutants.toml` sets. A diff with nothing to mutate (documentation, comments, tests, CI) makes cargo-mutants 27.1.0 exit 0 and write no `mutants.out`; it says `Diff changes no Rust source files` or `No mutants to filter` on stderr. Exit 0 with no file passes as zero mutants only when the run printed one of those lines; any other missing file fails. The file must be in the shape cargo-mutants 27.1.0 writes; another release, a missing field, an outcome the gate does not know, a failed baseline, or totals that disagree with the listed outcomes (`total_mutants` included) fail the layer and name `outcomes.json`. Exit 2 with no missed outcome listed, and exit 3 with no timeout listed, fail too. cargo-mutants exits 2 when mutants were missed and 3 when some timed out; the gate treats 0, 2 and 3 as a finished run and judges it from the file, and any other exit is a failed command. Survivors are explained per function (§19): a missed mutant inside a function is explained only by a live `layer: mutation` exemption whose `symbol` names that function and whose glob covers its file, and one in no function (`function: null`) only by a live record with no `symbol` whose glob covers its file. Timeouts and unviable mutants are counted and reported; they are not survivors. The layer fails when the survivors nothing explains exceed `gates.mutation.unexplained_survivors_max`, and the problem names each of them. `kill_rate_min_pct` is a floor `check` compares with the class profile; the gate does not judge it. The mutation row's `judged` carries `caught`, `missed`, `explained`, `unexplained`, `timeouts`, `unviable` and `unexplained_survivors_max`, beside `base`. Pass criteria: @@ -505,7 +505,7 @@ Pass criteria: - **0 unexplained survivors** in the diff (`unexplained_survivors_max`) - kill rate on examined mutants ≥ class minimum (see §4; declarative) -A survivor is explained when a live `layer: mutation` exemption covers it (§19), and the record gives the reason: the mutant is equivalent (`x + 0`, `x | 0` on a proven-zero, debug-only formatting), or a named test covers the behaviour. `#[mutants::skip]` at the tightest scope (expression, then function, never file) with the same reason takes an equivalent mutant out of the run, so it is never a survivor to explain. +A survivor is explained when a live `layer: mutation` exemption names its function (§19), and the record gives the reason: the mutant is equivalent (`x + 0`, `x | 0` on a proven-zero, debug-only formatting), or a named test covers the behaviour. `#[mutants::skip]` at the tightest scope (expression, then function, never file) with the same reason takes an equivalent mutant out of the run, so it is never a survivor to explain. ```rust #[mutants::skip] // equivalent: trailing comma in this macro has no runtime effect @@ -981,7 +981,7 @@ Expired exemptions fail CI. Reasons shorter than one clause are rejected. `cargo tops gate` checks every record, whatever its `layer`, whether that layer is on or off, and at any `--max-tier`: a record that has expired, or whose `expires` is not a real day, is a problem line in the report and the gate exits 1 (never 2; the config is still valid). `cargo tops check` prints the same lines as warnings and exits 0. The day is UTC: a record expires at 00:00 UTC on its `expires` date, whatever the local time zone. -How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains the survivors in the matching files, or, with `symbol`, in the function of that name; a record with no `symbol` covers every function in its glob, and the survivors that are in no function (a `const` or `static` initialiser, a type alias, a field type: cargo-mutants writes `function: null` for them). A record with a `symbol` never explains a survivor in no function. A record with no `symbol` that is the whole tree lifts the class floor on `unexplained_survivors_max` (§4) and explains no survivor; to explain survivors, a record needs a `symbol` or a narrower glob. "The whole tree" is decided from the files: the glob matches every Rust source file the mutation scope examines (the `.rs` files under `src/`, and `build.rs`), or is made of `*` and `**` components only, or is `.`. So `**`, `/**`, `**/`, `**/**`, `*`, `*.rs`, `**/*.rs`, `*/**` and `.` are the whole tree, and so is a glob such as `src/**` in a crate whose files are all under `src/`, or a single-file crate's only file. `cargo tops check` and the gate ask the same question. The coverage and CRAP floors keep their own list of whole-crate paths (`**`, `./**`, `**/*`, `.`, `./`). A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. +How the gate reads them (since 1.2.0): `path` is a glob matched like the coverage exclude globs (§7.5), relative to the crate root. A `crap` exemption covers the functions of the matching files, or, with `symbol`, only the function of that name (`run` also matches a method `Cli::run`). A `mutation` exemption explains survivors per function. A missed mutant inside a function is explained only by a record whose `symbol` names that function (matched as for `crap`) and whose `path` covers its file. A missed mutant in no function (cargo-mutants writes `function: null` for a `const` or `static` initialiser, a type alias or a field type) is explained only by a record with no `symbol` whose `path` covers its file. No record, of any shape or path, explains every survivor, so `gates.mutation.unexplained_survivors_max` always decides. A record with no `symbol` and a whole-crate `path` (`**`, `./**`, `**/*`, `.` or `./`) still lifts the class floor on `unexplained_survivors_max` in `cargo tops check` (§4), as for the other floors; it explains no survivor inside a function. A `coverage` exemption takes a file out of the per-file floor; its `symbol` is not read, and the file still counts toward the crate total. An exemption counts until its `expires` date. On that date and after, or when `expires` is not a real day, it exempts nothing and the gate reports it as a failure. --- diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index b60c3a6..7dcaaad 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -12,7 +12,7 @@ use super::{Fault, Mode}; use crate::config::{self, TopsConfig}; use crate::facts::{self, Outcome, Run}; use crate::mutation::Ledger; -use crate::scope::{crate_root, examined_files}; +use crate::scope::crate_root; use crate::{config_path, exemptions, floors, laws, mutation, profiles, steps}; /// What the judging found. @@ -179,8 +179,7 @@ fn judge_survivors(inputs: &Inputs<'_, '_>, failed: &mut Vec) -> Option< live.retain(|record| record.live(&today)); let prefix = crate_prefix(inputs.dir, &cfg.crate_.name); let max = cfg.gates.mutation.unexplained_survivors_max; - let examined = examined_files(&crate_root(inputs.dir, &cfg.crate_.name)); - let ledger = mutation::explain(found, &live, &prefix, &examined); + let ledger = mutation::explain(found, &live, &prefix); failed.extend(ledger.problem(max)); Some(Survivors { ledger, max }) } diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index b7507af..2c0217c 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -258,13 +258,7 @@ fn load_config(path: &Path) -> Result { errors.join("\n ") )); } - let examined = scope::examined_files(path); - let problems = tiers::check_with( - &file.display().to_string(), - &doc, - &exemptions::today(), - &examined, - ); + let problems = tiers::check(&file.display().to_string(), &doc, &exemptions::today()); if !problems.is_empty() { return Err(anyhow!("{}", problems.join("\n"))); } diff --git a/crates/cargo-tops/src/mutation.rs b/crates/cargo-tops/src/mutation.rs index 1080ae1..d9af1d3 100644 --- a/crates/cargo-tops/src/mutation.rs +++ b/crates/cargo-tops/src/mutation.rs @@ -17,7 +17,6 @@ use anyhow::{Context, Result}; use serde_json::{json, Value}; use crate::exemptions::Exemption; -use crate::tiers::whole_tree; /// Where cargo-mutants writes the run's outcomes, under the directory it ran in. const OUTCOMES: &str = "mutants.out/outcomes.json"; @@ -298,35 +297,25 @@ fn relative<'f>(file: &'f str, crate_prefix: &str) -> &'f str { .unwrap_or(file) } -/// Whether `record` explains a survivor in `function` of the file `rel` -/// (`None`: in no function, as in a `const`, a `static`, a type alias or a -/// field type). A record with no symbol that matches every Rust file the -/// scope examines ([`whole_tree`]) lifts a floor and explains nothing. A symbol names a function, so a survivor in none is -/// explained only by a record with no symbol. -fn explains(record: &Exemption, rel: &str, function: Option<&str>, examined: &[String]) -> bool { - let named = record.symbol.is_some(); - if !named && whole_tree(&record.path, examined) { - return false; - } - if named && function.is_none() { - return false; - } - record.covers(rel, function) +/// Whether `record` explains a survivor in `function` of the file `rel`, +/// per function: a survivor inside a function is explained only by a record +/// whose `symbol` names that function and whose glob covers its file. A +/// survivor in no function (`function: null`: a `const` or `static` +/// initialiser, a type alias, a field type) is explained only by a record +/// with no symbol whose glob covers its file. No record explains every +/// survivor, so `unexplained_survivors_max` always decides. +fn explains(record: &Exemption, rel: &str, function: Option<&str>) -> bool { + record.symbol.is_some() == function.is_some() && record.covers(rel, function) } /// Sort the missed mutants. `live` are the layer's records that have not /// expired; `crate_prefix` is the crate's directory under the workspace root, /// which the records' globs are relative to. -pub(crate) fn explain( - found: &Outcomes, - live: &[Exemption], - crate_prefix: &str, - examined: &[String], -) -> Ledger { +pub(crate) fn explain(found: &Outcomes, live: &[Exemption], crate_prefix: &str) -> Ledger { let (explained, unexplained) = found.missed.iter().cloned().partition(|mutant: &Missed| { let rel = relative(&mutant.file, crate_prefix); live.iter() - .any(|record| explains(record, rel, mutant.function.as_deref(), examined)) + .any(|record| explains(record, rel, mutant.function.as_deref())) }); Ledger { caught: found.caught, @@ -493,7 +482,7 @@ mod tests { #[test] fn a_record_for_one_function_explains_only_its_survivors() { let found = parse(FIXTURE).expect("parses"); - let ledger = explain(&found, &[record("src/**", Some("double"))], "", &[]); + let ledger = explain(&found, &[record("src/**", Some("double"))], ""); assert_eq!(ledger.explained.len(), 5); assert_eq!(ledger.unexplained.len(), 5); assert!(ledger @@ -511,22 +500,33 @@ mod tests { } #[test] - fn a_symbol_matches_the_last_path_segment_and_a_symbol_free_record_covers_its_glob() { + fn a_symbol_matches_the_last_path_segment_and_a_symbol_free_record_explains_no_function() { let found = parse(FIXTURE).expect("parses"); - let nested = explain(&found, &[record("src/lib.rs", Some("triple"))], "", &[]); + let nested = explain(&found, &[record("src/lib.rs", Some("triple"))], ""); assert_eq!(nested.explained.len(), 5); let both = [ record("src/lib.rs", Some("double")), record("src/lib.rs", Some("inner::triple")), ]; - assert!(explain(&found, &both, "", &[]).unexplained.is_empty()); - let all = explain(&found, &[record("src/**", None)], "", &[]); - assert_eq!(all.explained.len(), 10); - assert!(all.passes(0) && all.problem(0).is_none()); - let elsewhere = explain(&found, &[record("other/**", None)], "", &[]); + assert!(explain(&found, &both, "").unexplained.is_empty()); + // No record explains every survivor: one with no symbol explains + // none of those inside a function. + let all = explain(&found, &[record("src/**", None)], ""); + assert_eq!(all.explained.len(), 0); + assert_eq!(all.unexplained.len(), 10); + assert!(!all.passes(0) && all.problem(0).is_some()); + let elsewhere = explain(&found, &[record("other/**", None)], ""); assert_eq!(elsewhere.unexplained.len(), 10); } + /// One record per function of the recorded run, over `path`. + fn both(path: &str) -> [Exemption; 2] { + [ + record(path, Some("double")), + record(path, Some("inner::triple")), + ] + } + #[test] fn globs_are_relative_to_the_crate_not_the_workspace() { let text = edited(|doc| { @@ -540,37 +540,26 @@ mod tests { let found = parse(&text).expect("parses"); // The record is relative to the crate: it names `src/lib.rs`, and the // workspace-relative file is `crates/sample/src/lib.rs`. - let inside = [record("src/lib.rs", None)]; + let inside = both("src/lib.rs"); assert_eq!( - explain(&found, &inside, "crates/sample", &[]) - .explained - .len(), + explain(&found, &inside, "crates/sample").explained.len(), 10 ); assert_eq!( - explain(&found, &inside, "crates/sample/", &[]) - .explained - .len(), + explain(&found, &inside, "crates/sample/").explained.len(), 10 ); // A record that spells the workspace path matches only when no // prefix is stripped. - let spelled = [record("crates/sample/src/lib.rs", None)]; - assert_eq!(explain(&found, &spelled, "", &[]).explained.len(), 10); + let spelled = both("crates/sample/src/lib.rs"); + assert_eq!(explain(&found, &spelled, "").explained.len(), 10); assert_eq!( - explain(&found, &spelled, "crates/sample", &[]) - .explained - .len(), + explain(&found, &spelled, "crates/sample").explained.len(), 0 ); // A record for another crate's file explains nothing. - let other = [record("crates/other/src/lib.rs", None)]; - assert_eq!( - explain(&found, &other, "crates/sample", &[]) - .explained - .len(), - 0 - ); + let other = both("crates/other/src/lib.rs"); + assert_eq!(explain(&found, &other, "crates/sample").explained.len(), 0); } #[test] @@ -587,7 +576,7 @@ mod tests { #[test] fn the_judged_numbers_count_every_kind_and_the_max() { let found = parse(FIXTURE).expect("parses"); - let ledger = explain(&found, &[record("src/lib.rs", Some("double"))], "", &[]); + let ledger = explain(&found, &[record("src/lib.rs", Some("double"))], ""); let judged = ledger.to_json(2); assert_eq!(judged["caught"], 11); assert_eq!(judged["missed"], 10); @@ -606,7 +595,7 @@ mod tests { timeouts: 4, unviable: 9, }; - let ledger = explain(&only, &[], "", &[]); + let ledger = explain(&only, &[], ""); assert!(ledger.passes(0) && ledger.problem(0).is_none()); assert_eq!(ledger.timeouts, 4); assert_eq!(ledger.unviable, 9); @@ -614,7 +603,7 @@ mod tests { #[test] fn a_negative_max_is_never_met() { - let ledger = explain(&Outcomes::default(), &[], "", &[]); + let ledger = explain(&Outcomes::default(), &[], ""); assert!(!ledger.passes(-1)); } @@ -636,22 +625,15 @@ mod tests { && m.function.as_deref() == Some("qa_h4_survivor_miss"))); let by_symbol = [record("src/qa_h4_plant.rs", Some("qa_h4_survivor_miss"))]; assert_eq!( - explain(&found, &by_symbol, "crates/plant", &[]) - .explained - .len(), + explain(&found, &by_symbol, "crates/plant").explained.len(), 2 ); - assert_eq!(explain(&found, &by_symbol, "", &[]).explained.len(), 2); + assert_eq!(explain(&found, &by_symbol, "").explained.len(), 2); let other = [record("src/qa_h4_plant.rs", Some("another"))]; - assert_eq!( - explain(&found, &other, "crates/plant", &[]) - .unexplained - .len(), - 2 - ); + assert_eq!(explain(&found, &other, "crates/plant").unexplained.len(), 2); let wrong_file = [record("src/lib.rs", Some("qa_h4_survivor_miss"))]; assert_eq!( - explain(&found, &wrong_file, "crates/plant", &[]) + explain(&found, &wrong_file, "crates/plant") .unexplained .len(), 2 @@ -794,38 +776,44 @@ mod tests { include_str!("../tests/fixtures/mutants-27.1.0/null-function/outcomes.json"); #[test] - fn a_survivor_in_no_function_takes_no_symbol_and_a_whole_tree_record_explains_none() { + fn a_survivor_in_no_function_takes_a_symbol_free_record_and_nothing_explains_them_all() { let found = parse(NULL_FUNCTION).expect("parses"); assert_eq!(found.missed.len(), 12); assert_eq!( found.missed.iter().filter(|m| m.function.is_none()).count(), 7 ); - let count = |records: &[Exemption]| explain(&found, records, "", &[]).explained.len(); + let count = |records: &[Exemption]| explain(&found, records, "").explained.len(); + // A symbol explains the function it names: 5 in `get`, none outside. assert_eq!(count(&[record("src/lib.rs", Some("get"))]), 5); assert_eq!(count(&[record("src/lib.rs", Some("no_such_fn"))]), 0); - assert_eq!(count(&[record("src/lib.rs", None)]), 12); - assert_eq!(count(&[record("src/**", None)]), 12); - // Symbol and no symbol: every survivor once. - assert_eq!( - count(&[ - record("src/lib.rs", Some("get")), - record("src/lib.rs", None) - ]), - 12 - ); - for whole in ["**", "./**", "**/*", "*", ".", "./"] { - assert_eq!(count(&[record(whole, None)]), 0, "{whole}"); + // No symbol: the 7 in no function, in the files the glob covers. + for path in ["src/lib.rs", "src/**", "**", "*", "**/*.rs"] { + assert_eq!(count(&[record(path, None)]), 7, "{path}"); } - // A symbol never lifts a floor, so a symbol record with a whole-tree - // path still names its function. + assert_eq!(count(&[record("other/**", None)]), 0); + // Both kinds of record: every survivor, each counted once. + let both = [ + record("src/lib.rs", Some("get")), + record("src/lib.rs", None), + ]; + assert_eq!(count(&both), 12); + // A symbol with a whole-tree path still names only its function. assert_eq!(count(&[record("**", Some("get"))]), 5); + // In the recorded run every survivor is in a function: a symbol-free + // record explains none, whatever its glob, so the max decides. + let in_functions = parse(FIXTURE).expect("parses"); + for path in ["**", "/**", "*", "*.rs", "src/**", ".", "src/lib.rs"] { + let ledger = explain(&in_functions, &[record(path, None)], ""); + assert_eq!(ledger.explained.len(), 0, "{path}"); + assert!(!ledger.passes(0), "{path}"); + } } #[test] fn a_survivor_is_named_once_by_its_own_name() { let found = parse(FIXTURE).expect("parses"); - let ledger = explain(&found, &[], "", &[]); + let ledger = explain(&found, &[], ""); let problem = ledger.problem(0).expect("fails"); let first = &found.missed[0]; assert_eq!(problem.matches(&first.name).count(), 1, "{problem}"); diff --git a/crates/cargo-tops/src/scope.rs b/crates/cargo-tops/src/scope.rs index 1b0812d..135f738 100644 --- a/crates/cargo-tops/src/scope.rs +++ b/crates/cargo-tops/src/scope.rs @@ -127,35 +127,6 @@ pub(crate) fn package_dir(metadata: &str, name: &str) -> Option { Path::new(manifest).parent().map(Path::to_path_buf) } -/// The Rust source files cargo-mutants examines in the crate at `root`, -/// relative to it with `/` separators: every `.rs` file under `src/`, and -/// `build.rs`. Sorted. -pub(crate) fn examined_files(root: &Path) -> Vec { - let mut found = Vec::new(); - walk_rs(&root.join("src"), "src", &mut found); - if root.join("build.rs").is_file() { - found.push("build.rs".to_string()); - } - found.sort(); - found -} - -fn walk_rs(dir: &Path, rel: &str, found: &mut Vec) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.filter_map(Result::ok) { - let name = entry.file_name().to_string_lossy().into_owned(); - let path = entry.path(); - let below = format!("{rel}/{name}"); - if path.is_dir() { - walk_rs(&path, &below, found); - } else if name.ends_with(".rs") { - found.push(below); - } - } -} - /// Whether `glob` matches the `/`-separated `rel`, or a trailing part of it /// that starts at a directory. pub(crate) fn glob_matches(glob: &str, rel: &str) -> bool { @@ -298,27 +269,4 @@ mod tests { assert_eq!(crate_root(&empty, "cargo-tops"), empty); fs::remove_dir_all(&empty).expect("cleanup"); } - - #[test] - fn the_examined_files_are_the_rust_files_under_src_and_build_rs() { - let root = std::env::temp_dir().join(format!("cargo-tops-examined-{}", std::process::id())); - std::fs::remove_dir_all(&root).ok(); - for file in [ - "src/lib.rs", - "src/a/b.rs", - "src/a/data.txt", - "tests/t.rs", - "build.rs", - ] { - let path = root.join(file); - std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); - std::fs::write(&path, "").expect("write"); - } - assert_eq!( - super::examined_files(&root), - ["build.rs", "src/a/b.rs", "src/lib.rs"] - ); - assert!(super::examined_files(&root.join("nowhere")).is_empty()); - std::fs::remove_dir_all(&root).ok(); - } } diff --git a/crates/cargo-tops/src/tiers.rs b/crates/cargo-tops/src/tiers.rs index 5d48981..42a1097 100644 --- a/crates/cargo-tops/src/tiers.rs +++ b/crates/cargo-tops/src/tiers.rs @@ -11,7 +11,6 @@ use serde_json::Value; use crate::exemptions; use crate::profile_data::class_profile; -use crate::scope::glob_matches; /// The tier a layer runs at: the table's, else the instance's own for an id /// the table lacks, else 0. @@ -137,30 +136,17 @@ const REQUIRED_BY_PROFILE: [(&str, &str); 5] = [ /// Every problem in the instance `doc` (read from `file`): a tier that is not /// the table's, a floor looser than its class profile with no live exemption /// on `today` (`YYYY-MM-DD`), and a required layer that is off. -pub(crate) fn check_with(file: &str, doc: &Value, today: &str, examined: &[String]) -> Vec { +pub(crate) fn check(file: &str, doc: &Value, today: &str) -> Vec { let mut problems = tier_problems(file, doc); let class = doc.pointer("/crate/class").and_then(Value::as_str); let profile = class.and_then(class_profile); if let (Some(class), Some(profile)) = (class, profile.as_ref()) { - let when = When { today, examined }; - problems.extend(floor_problems(file, doc, class, profile, &when)); + problems.extend(floor_problems(file, doc, class, profile, today)); } problems.extend(required_problems(file, doc, class, profile.as_ref())); problems } -/// `check` with no Rust source files known: the tests of the floors. -#[cfg(test)] -fn check(file: &str, doc: &Value, today: &str) -> Vec { - check_with(file, doc, today, &[]) -} - -/// The day, and the Rust files of the crate, that a floor's lift is judged on. -struct When<'a> { - today: &'a str, - examined: &'a [String], -} - fn layers(doc: &Value) -> &[Value] { doc.get("layers") .and_then(Value::as_array) @@ -196,11 +182,11 @@ fn floor_problems( doc: &Value, class: &str, profile: &Value, - when: &When<'_>, + today: &str, ) -> Vec { FLOORS .iter() - .filter_map(|floor| floor_problem(file, doc, class, profile, when, floor)) + .filter_map(|floor| floor_problem(file, doc, class, profile, today, floor)) .collect() } @@ -209,7 +195,7 @@ fn floor_problem( doc: &Value, class: &str, profile: &Value, - when: &When<'_>, + today: &str, floor: &Floor, ) -> Option { let wanted = profile.pointer(floor.profile).and_then(Value::as_f64)?; @@ -222,7 +208,7 @@ fn floor_problem( Bound::Min => have < wanted, Bound::Max => have > wanted, }); - if !loose || live_exemption(doc, floor.layer, when) { + if !loose || live_exemption(doc, floor.layer, today) { return None; } let key = found.map_or(floor.key, |(key, _)| key); @@ -239,42 +225,10 @@ fn floor_problem( )) } -fn live_exemption(doc: &Value, layer: &str, when: &When<'_>) -> bool { +fn live_exemption(doc: &Value, layer: &str, today: &str) -> bool { exemptions::for_layer(doc.get("exemptions"), layer) .iter() - .any(|record| { - record.live(when.today) - && record.symbol.is_none() - && lifts(layer, &record.path, when.examined) - }) -} - -/// Whether a record for `layer` at `path` covers the whole crate. The -/// mutation layer decides it from the crate's Rust files ([`whole_tree`]); -/// the coverage and CRAP floors keep the spellings of [`covers_crate`]. -fn lifts(layer: &str, path: &str, examined: &[String]) -> bool { - if layer == "mutation" { - whole_tree(path, examined) - } else { - covers_crate(path) - } -} - -/// Whether `path`, a glob of a `layer: mutation` record with no symbol, is -/// the whole tree: it matches every Rust source file the mutation scope -/// examines (`examined`, relative to the crate root), or, read as a pattern, -/// is made of `*` and `**` components only, or is `.`. Such a record lifts the -/// floor on `unexplained_survivors_max` and explains no survivor; `check` and -/// the gate both ask this one question. -pub(crate) fn whole_tree(path: &str, examined: &[String]) -> bool { - let parts: Vec<&str> = path - .split('/') - .filter(|part| !part.is_empty() && *part != ".") - .collect(); - let wild = parts.iter().all(|part| matches!(*part, "*" | "**")); - let every_file = !examined.is_empty() && examined.iter().all(|file| glob_matches(path, file)); - // No component left (`.`, `./`) names the crate root: `wild` holds then. - wild || every_file + .any(|record| record.live(today) && record.symbol.is_none() && covers_crate(&record.path)) } /// Whether an exemption path covers the whole crate, the only reach that may @@ -327,7 +281,7 @@ fn required_problem(file: &str, doc: &Value, layer: &str, who: &str) -> Option Vec { - FILES.iter().map(|file| (*file).to_string()).collect() - } - - #[test] - fn every_spelling_that_reaches_every_file_is_the_whole_tree() { - let files = examined(); - for path in [ - "**", "/**", "**/", "**/**", "*", "*.rs", "**/*.rs", "*/**", "**/*", "./**", ".", "./", - "./**/*", "src/**", - ] { - assert!(whole_tree(path, &files), "{path}"); - } - // Spelling alone is enough for the wild ones, even with no file known. - for path in [ - "**", "/**", "**/", "**/**", "*", "*/**", "**/*", "./**", ".", "./", - ] { - assert!(whole_tree(path, &[]), "{path} with no files"); - } - // Narrower than the tree: one file, one directory, one other crate. - for path in [ - "src/lib.rs", - "src/gate/**", - "gate/*.rs", - "tests/**", - "src/other.rs", - ] { - assert!(!whole_tree(path, &files), "{path}"); - } - assert!(!whole_tree("src/lib.rs", &[])); - // A crate's only file is the whole tree. - assert!(whole_tree("src/lib.rs", &["src/lib.rs".to_string()])); - // A build script outside src/ is also examined. - let with_build = [files, vec!["build.rs".to_string()]].concat(); - assert!(!whole_tree("src/**", &with_build)); - assert!(whole_tree("**/*.rs", &with_build)); - } - - #[test] - fn a_mutation_record_lifts_the_floor_by_the_files_and_check_says_so() { - let files = examined(); - for path in [ - "**", "/**", "**/", "**/**", "*", "*.rs", "**/*.rs", "*/**", "src/**", - ] { - let mut doc = golden("library-core"); - set( - &mut doc, - "/gates/mutation/unexplained_survivors_max", - json!(3), - ); - set( - &mut doc, - "/exemptions", - exemption_at(path, "mutation", "2026-12-31"), - ); - assert!( - check_with("f.yaml", &doc, TODAY, &files).is_empty(), - "{path}" - ); - } - let mut doc = golden("library-core"); - set( - &mut doc, - "/gates/mutation/unexplained_survivors_max", - json!(3), - ); - set( - &mut doc, - "/exemptions", - exemption_at("src/lib.rs", "mutation", "2026-12-31"), - ); - assert!(!check_with("f.yaml", &doc, TODAY, &files).is_empty()); - } - - #[test] - fn a_star_record_does_not_relax_the_coverage_or_crap_floor() { - // As at base: `*` is not one of the spellings that lift those floors. - let mut doc = golden("library-core"); - set(&mut doc, "/gates/coverage/line_min_pct", json!(50)); - set( - &mut doc, - "/exemptions", - exemption_at("*", "coverage", "2026-12-31"), - ); - assert!(one(&doc).contains("gates.coverage.line_min_pct is 50")); - set( - &mut doc, - "/exemptions", - exemption_at("**", "coverage", "2026-12-31"), - ); - assert!(check("f.yaml", &doc, TODAY).is_empty()); - } } diff --git a/crates/cargo-tops/tests/gate_survivors.rs b/crates/cargo-tops/tests/gate_survivors.rs index b03e702..7a31820 100644 --- a/crates/cargo-tops/tests/gate_survivors.rs +++ b/crates/cargo-tops/tests/gate_survivors.rs @@ -486,13 +486,27 @@ fn survivors_that_every_record_explains_pass() { } #[test] -fn a_record_with_no_symbol_covers_every_function_in_its_glob() { - let fixture = fixture("no-symbol", "main"); - exempt(&fixture, "src/lib.rs", None, "2099-01-01"); - let file = planted(&fixture, |_| {}); - let (out, text) = gate_with(&fixture, Some(&file), "2"); - assert_eq!(out.status.code(), Some(0), "{text}"); - assert_eq!(counts(mutation(&report(&fixture))), [11, 10, 10, 0, 1, 1]); +fn a_record_with_no_symbol_explains_no_survivor_inside_a_function() { + // Whatever its glob: only a symbol names a function. + for path in ["src/lib.rs", "src/**", "**", "*", "."] { + let fixture = fixture("no-symbol", "main"); + exempt(&fixture, path, None, "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{path}: {text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(counts(row), [11, 10, 0, 10, 1, 1], "{path}: {row}"); + // Every survivor is named: the max of 0 decides. + let problems = mutation_problems(&report); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!( + problems + .iter() + .all(|p| p.contains("double") && p.contains("triple")), + "{problems:?}" + ); + } } #[test] @@ -807,19 +821,19 @@ fn a_survivor_in_no_function_is_explained_only_by_a_record_with_no_symbol() { }; assert_eq!(unexplained(Some("get"), "src/lib.rs"), 7); assert_eq!(unexplained(Some("no_such_fn"), "src/lib.rs"), 12); - assert_eq!(unexplained(None, "src/lib.rs"), 0); - // In this crate every file is under src/, so `src/**` is the whole tree. - assert_eq!(unexplained(None, "src/**"), 12); + // A record with no symbol explains the 7 in no function, never the 5 + // in `get`, whatever its glob covers. + assert_eq!(unexplained(None, "src/lib.rs"), 5); + assert_eq!(unexplained(None, "src/**"), 5); + assert_eq!(unexplained(None, "**"), 5); } #[test] -fn a_whole_tree_record_lifts_the_floor_and_explains_no_survivor() { - // The listed spellings reach every Rust file, however they are written. - for path in [ - "**", "/**", "**/", "**/**", "./**", "**/*", "*", "*.rs", "**/*.rs", "*/**", ".", "./", - "src/**", - ] { - let fixture = fixture("whole-tree", "main"); +fn a_whole_crate_record_lifts_the_floor_and_explains_no_survivor() { + // The spellings `check` accepts to lift the floor of the max; the max + // then decides, because none of them explains a survivor. + for path in ["**", "./**", "**/*", ".", "./"] { + let fixture = fixture("whole-crate", "main"); set_max(&fixture, 10); exempt(&fixture, path, None, "2099-01-01"); let file = planted(&fixture, |_| {}); @@ -831,10 +845,23 @@ fn a_whole_tree_record_lifts_the_floor_and_explains_no_survivor() { assert_eq!(counts(row), [11, 10, 0, 10, 1, 1], "{path}: {row}"); } // At a max of 9 the same record changes nothing: 10 survivors fail. - let fixture = fixture("whole-tree-nine", "main"); + let fixture = fixture("whole-crate-nine", "main"); set_max(&fixture, 9); exempt(&fixture, "**", None, "2099-01-01"); let file = planted(&fixture, |_| {}); let (out, text) = gate_with(&fixture, Some(&file), "2"); assert_eq!(out.status.code(), Some(1), "{text}"); } + +#[test] +fn a_symbol_record_explains_only_its_own_function() { + let fixture = fixture("own-function", "main"); + exempt(&fixture, "**", Some("double"), "2099-01-01"); + let file = planted(&fixture, |_| {}); + let (out, text) = gate_with(&fixture, Some(&file), "2"); + assert_eq!(out.status.code(), Some(1), "{text}"); + let report = report(&fixture); + let row = mutation(&report); + assert_eq!(counts(row), [11, 10, 5, 5, 1, 1], "{row}"); + expect_failed_naming(&text, &report, "triple", " double "); +}