Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/rust-tops.yml
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,8 @@ jobs:

# Runs this repository's own gate: every enabled layer at tier <= 1 in
# rust-tops.yaml, laws included. Not a --dry-run. fetch-depth: 0 because
# the mutation layer diffs against origin/main. cargo-tops is built from
# the mutation layer diffs against the base branch, which --base names
# (a gate with no base fails under CI). cargo-tops is built from
# this checkout, so the gate under test is the one on this commit. Bend is
# not preinstalled: the laws layer fetches it by laws/bend.lock, as it
# does on an adopter.
Expand Down Expand Up @@ -384,7 +385,7 @@ jobs:
with:
tool: cargo-mutants@27.1.0
- run: cargo install --path crates/cargo-tops --locked
- run: cargo tops gate --max-tier 1
- run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}"

# mutants-diff: class overlay. Diff-scoped; no --baseline skip; default
# features usually need no FFI sysdeps. Do not mark required until one
Expand Down
14 changes: 12 additions & 2 deletions PROTOCOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -479,10 +479,20 @@ Coverage says a line ran. A survivor says no test *cared*.
On every PR that touches `src/`:

```bash
git diff origin/main...HEAD --unified=0 > /tmp/pr.diff
git diff "$(git merge-base "$BASE" HEAD)" HEAD --unified=0 > /tmp/pr.diff
cargo mutants --in-diff /tmp/pr.diff -j 2 -- --skip ignored
```

`$BASE` is the PR's base ref, such as `origin/main`. `cargo tops gate` takes it, in this order:

1. `--base <ref>`.
2. `gates.base_ref` in `rust-tops.yaml`.
3. `origin/HEAD`, the remote's default branch.

The first source that is set wins. A ref that is named by the flag or the key and does not name a commit fails the layer, and the failure names the ref. The diff is from `git merge-base <base> HEAD` to `HEAD`. The report names the base and its source (`flag`, `key` or `origin/HEAD`).

`CI` counts as set when it is not empty and is not `0` or `false` (in any case). When no source gives a base, the layer is `fail` if `CI` is set. The report's `detail` and the gate's problem line name `fetch-depth: 0` and `--base`: a checkout that never fetched the base must not read as a clean diff. When `CI` is not set, the layer is `vacuous`, and the detail lists each source tried. A base that resolves but shares no history with `HEAD`, as in a shallow checkout, fails the diff and names `fetch-depth: 0` too. CI workflows pass `--base "origin/${{ github.base_ref || github.event.repository.default_branch }}"` from a checkout with `fetch-depth: 0`, because `actions/checkout` may not create `origin/HEAD`.

Pass criteria:

- baseline green
Expand Down Expand Up @@ -1011,7 +1021,7 @@ cargo test --doc --workspace
cargo llvm-cov nextest --workspace --all-features \
--lcov --output-path target/coverage/lcov.info --fail-under-lines 90
cargo crap
git diff origin/main...HEAD --unified=0 > /tmp/pr.diff
git diff "$(git merge-base "$BASE" HEAD)" HEAD --unified=0 > /tmp/pr.diff # BASE: section 8.3
cargo mutants --in-diff /tmp/pr.diff

# extra by class
Expand Down
7 changes: 4 additions & 3 deletions config/drop-in/rust-tops.yml
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,9 @@ jobs:

# Runs the gate rust-tops.yaml declares: every enabled layer at tier <= 1,
# laws included when they are on. Not a --dry-run. fetch-depth: 0 because
# the mutation layer diffs against origin/main, which a depth-1 checkout
# does not have. cargo-tops builds with Rust 1.99.1 (its rust-version),
# the mutation layer diffs against the base branch, which a depth-1 checkout
# does not have. --base names that branch: actions/checkout may not create
# origin/HEAD, and a gate with no base fails under CI. cargo-tops builds with Rust 1.99.1 (its rust-version),
# which can be newer than crate.msrv; the layers run on the pinned channel.
# The cargo-tops version is pinned; bump it when the kit publishes.
# Since 0.4.0 the gate enforces the coverage floors and the CRAP threshold
Expand Down Expand Up @@ -220,7 +221,7 @@ jobs:
with:
tool: cargo-mutants@27.1.0
fallback: none
- run: cargo tops gate --max-tier 1
- run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}"

# mutants-diff: class overlay. Diff-scoped; no --baseline skip; default
# features usually need no FFI sysdeps. Do not mark required until one
Expand Down
4 changes: 4 additions & 0 deletions crates/cargo-tops/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ pub(crate) struct Gates {
pub coverage: CoverageGate,
pub crap: CrapGate,
pub mutation: MutationGate,
/// The ref the mutation diff starts from, when `--base` is not given.
/// Absent means the remote's default branch (`origin/HEAD`).
#[serde(skip_serializing_if = "Option::is_none")]
pub base_ref: Option<String>,
}

#[derive(Debug, Deserialize, Serialize, PartialEq)]
Expand Down
24 changes: 24 additions & 0 deletions crates/cargo-tops/src/facts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,18 @@ pub(crate) struct Run {
/// The last step that ran: the one that decided `outcome`.
pub step: String,
pub outcome: Outcome,
/// Why the layer stopped, when it stopped on something no command
/// reported: the mutation base. The report prints it as the layer's
/// detail and its problem, in place of "`step` failed".
pub note: Option<String>,
}

impl Run {
/// Whether a command ran. A layer that stopped on its note before any
/// command has an empty `cmd` and a note.
pub(crate) fn ran(&self) -> bool {
!self.cmd.is_empty() || self.note.is_none()
}
}

/// A run some law may count: `Passed`, `Failed` or `ToolMissing`, never
Expand Down Expand Up @@ -1047,9 +1059,21 @@ mod tests {
cmd: cmd.into(),
step: cmd.into(),
outcome,
note: None,
}
}

#[test]
fn a_run_with_a_note_and_no_command_ran_nothing() {
let mut stopped = run("mutation", "", Outcome::Failed);
assert!(stopped.ran());
stopped.note = Some("why".to_string());
assert!(!stopped.ran());
stopped.cmd = "git x".to_string();
assert!(stopped.ran());
assert!(run("unit", "cargo nextest", Outcome::Passed).ran());
}

fn ctx_with<'a>(cfg: &'a TopsConfig, runs: &'a [Run]) -> Ctx<'a> {
Ctx {
dir: Path::new("."),
Expand Down
59 changes: 28 additions & 31 deletions crates/cargo-tops/src/gate/execute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use super::plan::{GatePlan, Unit, What, CRAP_VERSION};
use super::{Fault, Mode};
use crate::config::Layer;
use crate::facts::{Outcome, Run};
use crate::steps::{self, Kind, MutationPlan, Step};
use crate::steps::{self, Base, Kind, MutationPlan, Step};
use crate::{laws, TOOL_LOCK};

/// What the laws layer needs before any work starts.
Expand Down Expand Up @@ -313,6 +313,7 @@ pub(crate) fn finish(layer: &str, ran: &[String], outcome: Outcome) -> Run {
cmd: ran.join(" && "),
step: ran.last().cloned().unwrap_or_default(),
outcome,
note: None,
}
}

Expand Down Expand Up @@ -360,8 +361,11 @@ pub(crate) fn run_pinned_crap(dir: &Path, layer: &str, chain: &[Step], pin: &str
}

/// `cargo-mutants` is looked up before the diff, so an empty diff cannot
/// hide a missing tool. A failed diff is `Failed`; nothing waives it. An
/// empty one is `Vacuous` and `cargo mutants` does not run.
/// hide a missing tool. A base that was named and is not a commit fails the
/// layer before any command runs. A failed diff is `Failed`; nothing waives
/// it. An empty one is `Vacuous` and `cargo mutants` does not run, except
/// that no base at all fails under CI. [`steps::stop_note`] words every one
/// of these stops.
pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Result<Run> {
if let Err(missing) = steps::check_tool(&plan.mutants.argv) {
println!("{missing}");
Expand All @@ -371,44 +375,37 @@ pub(crate) fn run_mutation(dir: &Path, layer: &str, plan: &MutationPlan) -> Resu
Outcome::ToolMissing,
));
}
if matches!(plan.base, Base::Unresolved { .. }) {
return Ok(stopped(layer, &[], &plan.base, Outcome::Failed));
}
let mut ran = vec![plan.diff.display()];
let diff_ok = run_step(dir, &plan.diff)? == Outcome::Passed;
let diff = steps::judge_diff(diff_ok, &plan.diff_path);
let Some(mutants) = plan.after_diff(diff) else {
if diff == Outcome::Vacuous && has_origin_main(dir) {
println!(
"Vacuous: the diff from `git merge-base origin/main HEAD` is empty; \
cargo mutants did not run and no law counts this layer"
);
} else if diff == Outcome::Vacuous {
println!(
"Vacuous: no origin/main; the mutation diff is empty; \
cargo mutants did not run and no law counts this layer"
);
} else {
println!("Failed: the mutation diff did not succeed; a failed diff is never waived");
}
return Ok(finish(layer, &ran, diff));
return Ok(stopped(layer, &ran, &plan.base, diff));
};
ran.push(mutants.display());
let outcome = run_step(dir, mutants)?;
Ok(finish(layer, &ran, outcome))
}

/// Whether `refs/remotes/origin/main` exists, for the Vacuous wording only.
pub(crate) fn has_origin_main(dir: &Path) -> bool {
Command::new("git")
.args([
"rev-parse",
"--verify",
"--quiet",
"refs/remotes/origin/main",
])
.current_dir(dir)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.is_ok_and(|status| status.success())
/// The run of a layer that stopped on its base or its diff: prints the
/// note and keeps it with the run, so the report says the same. A diff
/// command that ran and failed is named in the kept note, since it did fail;
/// a base that was never tried, or an empty diff, names no command.
fn stopped(layer: &str, ran: &[String], base: &Base, diff: Outcome) -> Run {
let (outcome, note) = steps::stop_note(base, diff)
.unwrap_or_else(|| (diff, format!("the mutation diff ended as {diff}")));
println!("{outcome}: {note}");
let command_failed = diff == Outcome::Failed && !matches!(base, Base::Unresolved { .. });
let kept = match ran.last() {
Some(step) if command_failed => format!("`{step}` failed: {note}"),
_ => note,
};
Run {
note: Some(kept),
..finish(layer, ran, outcome)
}
}

pub(crate) fn run_step(dir: &Path, step: &Step) -> Result<Outcome> {
Expand Down
9 changes: 6 additions & 3 deletions crates/cargo-tops/src/gate/judge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ fn judge_laws(
/// judged the laws layer. A layer that took another layer's run ran none.
pub(crate) fn ran_commands(runs: &[Run], bend_ran: bool, shared: &[(&str, &str)]) -> usize {
runs.iter()
.filter(|run| run.outcome != Outcome::ToolMissing)
.filter(|run| run.outcome != Outcome::ToolMissing && run.ran())
.filter(|run| !shared.iter().any(|(layer, _)| *layer == run.layer))
.count()
+ usize::from(bend_ran)
Expand Down Expand Up @@ -272,7 +272,7 @@ pub(crate) fn command_count(
"laws" => !bend_ran,
_ => !runs
.iter()
.any(|run| run.layer == *id && run.outcome != Outcome::ToolMissing),
.any(|run| run.layer == *id && run.outcome != Outcome::ToolMissing && run.ran()),
})
.collect();
let mut line = format!(
Expand Down Expand Up @@ -320,7 +320,10 @@ pub(crate) fn laws_evaluated(cases: &Path) -> Option<(usize, usize)> {
pub(crate) fn run_failure(run: &Run) -> Option<String> {
match run.outcome {
Outcome::Passed | Outcome::Vacuous => None,
Outcome::Failed => Some(format!("{}: `{}` failed", run.layer, run.step)),
Outcome::Failed => Some(match &run.note {
Some(note) => format!("{}: {note}", run.layer),
None => format!("{}: `{}` failed", run.layer, run.step),
}),
Outcome::ToolMissing => {
let missing = steps::MissingTool {
binary: steps::tool_binary(std::slice::from_ref(&run.step))
Expand Down
32 changes: 23 additions & 9 deletions crates/cargo-tops/src/gate/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,16 @@ impl Fault {
}

/// Run the gate for `cfg` under `dir`: always writes `report.json`.
pub(crate) fn run(dir: &Path, cfg: &TopsConfig, opts: Options) -> ExitCode {
/// `base_flag` is `--base`, the first source of the mutation base.
pub(crate) fn run(
dir: &Path,
cfg: &TopsConfig,
opts: Options,
base_flag: Option<&str>,
) -> ExitCode {
// A gate resolves `cargo metadata` afresh, once per project.
laws::begin_gate();
let plan = match plan::build(dir, cfg, opts.max_tier) {
let plan = match plan::build(dir, cfg, opts.max_tier, base_flag) {
Ok(plan) => plan,
Err(fault) => return report::fault(dir, Some(cfg), opts, None, &fault),
};
Expand Down Expand Up @@ -123,7 +129,7 @@ mod tests {
use super::judge::{command_count, ran_commands};
use super::plan::{plan_for, Plan, TestCommands};
use crate::facts::{Outcome, Run};
use crate::steps::{self, Kind, Step};
use crate::steps::{self, Base, Kind, Step};
use crate::{config, credit};
use std::fs;
use std::path::Path;
Expand All @@ -136,7 +142,7 @@ mod tests {
}

fn chains(id: &str) -> Vec<Vec<Step>> {
match plan_for(&layer(id), Path::new("/r")) {
match plan_for(&layer(id), Path::new("/r"), &Base::unused()) {
Some(Plan::Chains(chains)) => chains,
_ => panic!("{id} has no chains"),
}
Expand All @@ -152,7 +158,7 @@ mod tests {
#[test]
fn the_g0_strings_name_the_tool_they_run() {
let coverage = "mkdir -p target/coverage && cargo llvm-cov nextest --workspace --all-features --profile ci --lcov --output-path target/coverage/lcov.info && test -s target/coverage/lcov.info";
let mutation = "sh -c 'git diff origin/main...HEAD --unified=0 > /tmp/rust-tops.diff; cargo mutants --in-diff /tmp/rust-tops.diff -j 2'";
let mutation = "sh -c 'git diff base...HEAD --unified=0 > /tmp/rust-tops.diff; cargo mutants --in-diff /tmp/rust-tops.diff -j 2'";
assert_eq!(
steps::tool_binary(&[coverage.to_string()]).as_deref(),
Some("cargo-llvm-cov")
Expand Down Expand Up @@ -263,6 +269,7 @@ mod tests {
cmd: format!("{layer} cmd"),
step: format!("{layer} step"),
outcome,
note: None,
}
}

Expand Down Expand Up @@ -391,16 +398,20 @@ mod tests {
#[test]
fn the_mutation_diff_uses_merge_base_and_the_report_dir() {
let report = Path::new("/work/target/rust-tops");
let Some(Plan::Mutation(plan)) = plan_for(&layer("mutation"), report) else {
let base = Base::Found {
source: steps::BaseSource::Flag,
name: "origin/trunk".to_string(),
};
let Some(Plan::Mutation(plan)) = plan_for(&layer("mutation"), report, &base) else {
panic!("mutation has no plan");
};
assert_eq!(plan.diff_path, report.join("rust-tops.diff"));
let script = plan.diff.argv.join(" ");
assert!(
script.contains("git merge-base origin/main HEAD"),
script.contains("git merge-base 'origin/trunk' HEAD"),
"{script}"
);
assert!(script.contains("refs/remotes/origin/main"), "{script}");
assert!(!script.contains("origin/main"), "{script}");
assert!(script.contains("&& git diff"), "{script}");
assert!(!script.contains("mutants"), "{script}");
assert!(!script.contains("/tmp"), "{script}");
Expand Down Expand Up @@ -446,7 +457,10 @@ mod tests {
"density-audit",
"laws",
] {
assert!(plan_for(&layer(id), Path::new("/r")).is_none(), "{id}");
assert!(
plan_for(&layer(id), Path::new("/r"), &Base::unused()).is_none(),
"{id}"
);
}
}

Expand Down
Loading
Loading