From 93a7a960ebb8ccac8d7d7d22b5cfb46d3d1b8c72 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 06:16:26 -0500 Subject: [PATCH 1/4] Check every pinned tool against the lock, and add cargo tops lock Each pinned tool a running layer needs is asked once before the layer runs. A mismatch is the tool-mismatch verdict: kind environment, exit 3, naming the tool, the pin and the version found. The cargo-crap-only check is gone. cargo tops lock --verify exits 0, 2 or 3, and --write rewrites values only. The kit and drop-in workflows run lock --verify before the gate. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rust-tops.yml | 9 + PROTOCOL.md | 40 + config/drop-in/rust-tops.yml | 2 + crates/cargo-tops/src/gate/execute.rs | 177 +++-- crates/cargo-tops/src/gate/judge.rs | 17 +- crates/cargo-tops/src/gate/mod.rs | 3 +- crates/cargo-tops/src/gate/plan.rs | 138 +++- crates/cargo-tops/src/lock.rs | 744 ++++++++++++++++++ crates/cargo-tops/src/main.rs | 81 +- crates/cargo-tops/src/steps.rs | 61 +- .../templates/drop-in/rust-tops.yml | 2 + .../tests/fixtures/versions/cargo-crap.txt | 1 + .../tests/fixtures/versions/cargo-deny.txt | 1 + .../tests/fixtures/versions/cargo-fuzz.txt | 1 + .../fixtures/versions/cargo-llvm-cov.txt | 1 + .../tests/fixtures/versions/cargo-mutants.txt | 1 + .../tests/fixtures/versions/cargo-nextest.txt | 5 + .../tests/fixtures/versions/kani-verifier.txt | 2 + .../tests/fixtures/versions/nushell.txt | 1 + .../tests/fixtures/versions/rustc.txt | 1 + crates/cargo-tops/tests/gate_base.rs | 14 +- crates/cargo-tops/tests/gate_credit.rs | 14 +- .../cargo-tops/tests/gate_credit_nextest.rs | 13 + crates/cargo-tops/tests/gate_exemptions.rs | 14 +- crates/cargo-tops/tests/gate_floors.rs | 18 +- crates/cargo-tops/tests/gate_real_mutants.rs | 32 +- crates/cargo-tops/tests/gate_report.rs | 188 ++++- crates/cargo-tops/tests/gate_survivors.rs | 14 +- .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + .../.github/workflows/rust-tops.yml | 2 + crates/cargo-tops/tests/laws_gate.rs | 129 +-- crates/cargo-tops/tests/lock_verify.rs | 322 ++++++++ crates/cargo-tops/tests/support/mod.rs | 17 + schema/gate-report.schema.json | 3 +- 40 files changed, 1895 insertions(+), 187 deletions(-) create mode 100644 crates/cargo-tops/src/lock.rs create mode 100644 crates/cargo-tops/tests/fixtures/versions/cargo-crap.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/cargo-deny.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/cargo-fuzz.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/cargo-llvm-cov.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/cargo-mutants.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/cargo-nextest.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/kani-verifier.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/nushell.txt create mode 100644 crates/cargo-tops/tests/fixtures/versions/rustc.txt create mode 100644 crates/cargo-tops/tests/lock_verify.rs create mode 100644 crates/cargo-tops/tests/support/mod.rs diff --git a/.github/workflows/rust-tops.yml b/.github/workflows/rust-tops.yml index 38e3505..2d6f107 100644 --- a/.github/workflows/rust-tops.yml +++ b/.github/workflows/rust-tops.yml @@ -385,6 +385,15 @@ jobs: with: tool: cargo-mutants@27.1.0 - run: cargo install --path crates/cargo-tops --locked + # The tools just installed are the pinned ones: checked on a fresh + # `init` of a throwaway crate (the lock an adopter gets), at the tiers + # the gate below runs. + - name: Lock verify on a fresh init + run: | + probe="$(mktemp -d)" + cargo new --lib --vcs none "$probe/probe" + cargo-tops init --path "$probe/probe" --class library-core --name probe + cargo-tops lock --verify --max-tier 1 --path "$probe/probe" - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" env: # The gate's own tests include a run of the real cargo-mutants: it diff --git a/PROTOCOL.md b/PROTOCOL.md index f9f8952..5e79ae4 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -849,6 +849,43 @@ Every adopting repository keeps a `RUST-TOPS.md` at its root. It is the one page The template is [`config/drop-in/RUST-TOPS.md`](config/drop-in/RUST-TOPS.md). `cargo tops init` writes it beside the instance, with items 2–4 and 6 filled from that instance. When `init` runs in one crate's directory of a larger repo, move its row into the root record and delete the copy. Keep it current when an instance, an exemption, or the baseline path changes. A record that disagrees with a `rust-tops.yaml` is wrong; the instance wins. +### 14.5 The tool lock (since 1.4.0) + +`rust-tops.lock` pins the tools the gate runs, under `[tools]`. A pin the gate does not check is only a comment, so the gate asks each tool for its version and compares the answer with the pin. + +**What is asked.** One row per tool. The first line of the output is read, and the version is the first word after the tool's name, cut at the first character that is not a digit or a dot. + +| Lock key | Asked | Line starts with | +|---|---|---| +| `rustc` | `rustc --version` | `rustc ` | +| `clippy`, `rustfmt` | judged by the toolchain: `rustc --version`; `cargo clippy --version` and `cargo fmt --version` must also exit 0 | `rustc ` | +| `cargo-nextest` | `cargo nextest --version` | `cargo-nextest ` | +| `cargo-llvm-cov` | `cargo llvm-cov --version` | `cargo-llvm-cov ` | +| `cargo-mutants` | `cargo mutants --version` | `cargo-mutants ` | +| `cargo-crap` | `cargo crap --version` | `cargo-crap ` | +| `cargo-deny` | `cargo deny --version` | `cargo-deny ` | +| `cargo-fuzz` | `cargo fuzz --version` | `cargo-fuzz ` | +| `kani-verifier` | `cargo kani --version` | `Kani Rust Verifier ` | +| `nushell` | `nu --version` | (the version alone) | + +Keys that name a library or a runner (`proptest`, `criterion`, `callgrind-runner`) are not tools on `PATH`. They are skipped. + +**Matching.** A pin matches when its dot-separated parts are the first parts of the version found. `0.12` matches `0.12.0` and `0.12.3`, and does not match `0.120.0`. `0.12.3` does not match `0.12`. A line that does not have the shape above is a mismatch, and the line is named as found. + +**In the gate.** +- Before the commands of a layer that runs, the gate asks each pinned tool that layer needs, once per gate. The layers and tools are: `static-lint` (`rustc`, `clippy`, `rustfmt`, `cargo-deny`), `unit`, `integration`, `property` and `acceptance` (`cargo-nextest`), `doc` (`rustc`), `coverage` (`cargo-llvm-cov`, `cargo-nextest`), `crap` (`cargo-crap`), `security-advisory` (`cargo-deny`), `mutation` (`cargo-mutants`), `fuzz` (`cargo-fuzz`) and `formal-kani` (`kani-verifier`). +- A tool that prints another version fails its layer with the verdict `tool-mismatch`. The layer's `detail`, its problem and the line on stdout carry the same words: `mismatch : pin , found `. The kind is `environment` and the exit code is 3. None of the layer's commands run. +- A tool that cannot be run is the `tool-missing` outcome, as before. +- A layer that takes another layer's run takes its verdict, `tool-mismatch` included. +- A layer above `--max-tier`, or one that is off, asks nothing. A tool with no pin in the lock is not asked. With no lock file the gate asks nothing. +- A `rust-tops.lock` that cannot be read or parsed, when a layer in range needs a pinned tool, is a config error: exit 2. + +**`cargo tops lock --verify [--max-tier N] [--path P]`.** Asks every tool in the lock, or, with `--max-tier`, only the tools that the enabled layers at or below `N` need, and prints one line per tool: `ok`, `mismatch`, `missing` or `skipped`. Exit 0 when none is a mismatch or missing, 2 when the lock cannot be read (or the config cannot be loaded for `--max-tier`), 3 otherwise. It never writes a file. + +**`cargo tops lock --write [--path P]`.** Replaces each `[tools]` value with the version the tool prints now. Comments, order, spacing and other tables are kept. A tool that cannot be run keeps its value. + +The reference workflow runs `cargo tops lock --verify` after the tools are installed and before the gate (section 17). + --- ## 15. Repository layout @@ -937,6 +974,8 @@ Jobs are separated so a 90-minute Miri run cannot hide a 20-second clippy fail. | `criterion` | dedicated runner optional | no | | `sanitizers` | schedule | release for ffi/unsafe | +A job that runs `cargo tops gate` runs `cargo tops lock --verify` first, with the same `--max-tier`, after the pinned tools are installed (section 14.5). + --- ## 18. Scoring model (optional dashboard) @@ -1035,6 +1074,7 @@ cargo test --doc --workspace cargo llvm-cov nextest --workspace --all-features \ --lcov --output-path target/coverage/lcov.info --fail-under-lines 90 cargo crap +cargo tops lock --verify # the installed tools are the pinned ones (section 14.5) git diff "$(git merge-base "$BASE" HEAD)" HEAD --unified=0 > /tmp/pr.diff # BASE: section 8.3 cargo mutants --in-diff /tmp/pr.diff diff --git a/config/drop-in/rust-tops.yml b/config/drop-in/rust-tops.yml index 0088f14..e0a1aae 100644 --- a/config/drop-in/rust-tops.yml +++ b/config/drop-in/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index f9f36f9..07d4cb4 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -8,11 +8,12 @@ use std::time::{Instant, SystemTime}; use anyhow::{Context, Result}; -use super::plan::{GatePlan, Unit, What, CRAP_VERSION}; +use super::plan::{GatePlan, Unit, What}; use super::{Fault, Mode}; use crate::config::Layer; use crate::evidence; use crate::facts::{Outcome, Run}; +use crate::lock::{self, Line, Verdict}; use crate::mutation::{self, Outcomes}; use crate::steps::{self, Base, Kind, MutationPlan, Step}; use crate::{laws, TOOL_LOCK}; @@ -88,7 +89,21 @@ fn version_of( cache: &mut HashMap>, ) -> Option { let (program, rest) = argv.split_first()?; - let asks = version_ask(program, rest.first().map(String::as_str)); + ask_cached( + dir, + &version_ask(program, rest.first().map(String::as_str)), + cache, + ) +} + +/// The first line `asks` prints, or `None` when it cannot run or fails. Each +/// distinct argv is run once per gate, so a pinned tool the gate checked and +/// the version the report names come from the same answer. +fn ask_cached( + dir: &Path, + asks: &[&str], + cache: &mut HashMap>, +) -> Option { cache .entry(asks.join(" ")) .or_insert_with(|| { @@ -144,9 +159,13 @@ pub(crate) fn run<'a>(dir: &Path, plan: &GatePlan<'a>, mode: Mode) -> Result { live: bool, done: Executed<'a>, versions: HashMap>, + /// What each pinned tool answered, asked once per gate. + asked: HashMap<&'static str, Line>, } impl<'a> Runner<'_, 'a> { @@ -182,6 +203,63 @@ impl<'a> Runner<'_, 'a> { Ok(()) } + /// Asks each pinned tool `layer` needs for its version, before any of the + /// layer's commands ([`Runner::ask_pins`]). `true` when the layer stopped + /// on one. A dry run only names the pins. + fn pinned_tools_stop(&mut self, layer: &Layer) -> bool { + let pins = self.plan.pins_of(&layer.id); + if pins.is_empty() { + return false; + } + let named: Vec = pins + .iter() + .map(|(row, pin)| format!("{} {pin}", row.key)) + .collect(); + println!( + "\n==> [{}] {TOOL_LOCK} pins, asked first: {}", + layer.id, + named.join(", ") + ); + if !self.live { + return false; + } + let clock = Clock::start(); + let Some(run) = self.ask_pins(layer) else { + return false; + }; + self.done.record(run, &clock, None); + true + } + + /// The run that stands for `layer` when a pinned tool it needs is at + /// another version ([`Outcome::ToolMismatch`], the note naming tool, pin + /// and found) or will not run ([`Outcome::ToolMissing`]); `None` when + /// every tool answers with its pin. Each tool is asked once per gate. + fn ask_pins(&mut self, layer: &Layer) -> Option { + let dir = self.dir; + let mut mismatched = Vec::new(); + let mut missing: Option = None; + for (row, pin) in self.plan.pins_of(&layer.id) { + let line = self.asked.entry(row.key).or_insert_with(|| { + lock::check_row(row, pin, &mut |argv| { + ask_cached(dir, argv, &mut self.versions) + }) + }); + match &line.verdict { + Verdict::Mismatch { .. } => mismatched.push((row.argv.join(" "), line.text())), + Verdict::Missing { asked } => { + missing.get_or_insert_with(|| asked.clone()); + } + Verdict::Ok { .. } | Verdict::Skipped { .. } => {} + } + } + if let Some((argv, _)) = mismatched.first() { + let notes: Vec<&str> = mismatched.iter().map(|(_, text)| text.as_str()).collect(); + return Some(mismatch_run(&layer.id, argv, notes.join("; "))); + } + missing.map(|asked| missing_run(&layer.id, asked)) + } + fn shared(&mut self, layer: &'a Layer, from: &'static str) { println!( "\n==> [{}] no command of its own: it takes the {from} run's outcome", @@ -227,38 +305,27 @@ impl<'a> Runner<'_, 'a> { } fn chain(&mut self, layer: &Layer, chain: &[Step]) -> Result<(), Fault> { - let pin = self.plan.crap_pin.as_deref().filter(|_| layer.id == "crap"); - let mut shown: Vec = pin.map(|_| CRAP_VERSION.join(" ")).into_iter().collect(); - shown.extend(chain.iter().map(Step::display)); - println!("\n==> [{}] {}", layer.id, shown.join(" && ")); - if let Some(pin) = pin { - println!( - " `{}` must print `cargo-crap {pin}`, the {TOOL_LOCK} pin", - CRAP_VERSION.join(" ") - ); - } + println!( + "\n==> [{}] {}", + layer.id, + chain + .iter() + .map(Step::display) + .collect::>() + .join(" && ") + ); self.done.listed += 1; if !self.live { return Ok(()); } let clock = Clock::start(); - let run = match pin { - Some(pin) => run_pinned_crap(self.dir, &layer.id, chain, pin), - None => run_chain(self.dir, &layer.id, chain), - } - .map_err(Fault::env)?; - let version = match pin { - // The version the gate already verified against the lock. - Some(pin) => run - .cmd - .starts_with(&format!("{} && ", CRAP_VERSION.join(" "))) - .then(|| format!("cargo-crap {pin}")), - None if layer.id == "crap" => None, - // A tool that is not there has no version to ask for. - None => first_tool(chain) - .filter(|_| run.outcome != Outcome::ToolMissing) - .and_then(|argv| version_of(self.dir, argv, &mut self.versions)), - }; + let run = run_chain(self.dir, &layer.id, chain).map_err(Fault::env)?; + let version = first_tool(chain) + // A tool that is not there has no version to ask for, and crap's + // is named only when the lock pinned it. + .filter(|_| run.outcome != Outcome::ToolMissing) + .filter(|_| layer.id != "crap" || !self.plan.pins_of("crap").is_empty()) + .and_then(|argv| version_of(self.dir, argv, &mut self.versions)); self.done.record(run, &clock, version); Ok(()) } @@ -389,34 +456,32 @@ pub(crate) fn run_chain(dir: &Path, layer: &str, chain: &[Step]) -> Result Ok(finish(layer, &ran, outcome)) } -/// The crap chain, run only when `cargo crap --version` prints -/// `cargo-crap `. It asks the binary that `cargo crap` runs, so another -/// version earlier on `PATH` fails the layer before a report is written. A -/// cargo-crap that is not there is left to the chain, which reports it -/// missing. -pub(crate) fn run_pinned_crap(dir: &Path, layer: &str, chain: &[Step], pin: &str) -> Result { - let version = CRAP_VERSION.join(" "); - let argv = CRAP_VERSION.map(String::from); - if steps::check_tool(&argv).is_err() { - return run_chain(dir, layer, chain); +/// A layer that stopped because a pinned tool prints another version than its +/// pin. The same `note` is the stdout line, the layer's detail and its problem. +fn mismatch_run(layer: &str, argv: &str, note: String) -> Run { + println!("tool-mismatch [{layer}]: {note}"); + Run { + layer: layer.to_string(), + cmd: String::new(), + step: argv.to_string(), + outcome: Outcome::ToolMismatch, + note: Some(note), } - let output = Command::new(CRAP_VERSION[0]) - .args(&CRAP_VERSION[1..]) - .current_dir(dir) - .output() - .with_context(|| format!("spawn: {version}"))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let printed = stdout.lines().next().unwrap_or_default().trim(); - let want = format!("cargo-crap {pin}"); - if !output.status.success() || printed != want { - println!( - "Failed [{layer}]: `{version}` printed `{printed}`, not `{want}` from {TOOL_LOCK}" - ); - return Ok(finish(layer, &[version], Outcome::Failed)); +} + +/// A layer that stopped because a pinned tool will not run: the same outcome +/// and words as a command whose tool is not on `PATH`. +fn missing_run(layer: &str, asked: String) -> Run { + let words: Vec = asked.split_whitespace().map(str::to_string).collect(); + let binary = steps::tool_binary(&words).unwrap_or_else(|| asked.clone()); + println!("{}", steps::MissingTool { binary }); + Run { + layer: layer.to_string(), + cmd: String::new(), + step: asked, + outcome: Outcome::ToolMissing, + note: None, } - let mut run = run_chain(dir, layer, chain)?; - run.cmd = format!("{version} && {}", run.cmd); - Ok(run) } /// A mutation layer's run, and what its `outcomes.json` held. diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 7dcaaad..c1ef03c 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -272,23 +272,23 @@ fn finish( laws_seconds: laws.seconds, bend_ran: laws.ran, laws_failed: laws.failed, - environment: done - .runs - .iter() - .any(|run| run.outcome == Outcome::ToolMissing), + environment: done.runs.iter().any(|run| run.outcome.lacks_tool()), law_total, survivors, } } /// The verdict of a layer's runs: a failure outranks a missing tool, which -/// outranks a vacuous run. +/// outranks a tool at another version than its pin, which outranks a vacuous +/// run. fn runs_verdict(runs: &[&Run]) -> &'static str { let any = |outcome: Outcome| runs.iter().any(|run| run.outcome == outcome); if any(Outcome::Failed) { "fail" } else if any(Outcome::ToolMissing) { "tool-missing" + } else if any(Outcome::ToolMismatch) { + "tool-mismatch" } else if any(Outcome::Vacuous) { "vacuous" } else { @@ -408,7 +408,7 @@ fn judge_laws( /// judged the laws layer. A layer that took another layer's run ran none. pub(crate) fn ran_commands(runs: &[Run], bend_ran: bool, shared: &[(&str, &str)]) -> usize { runs.iter() - .filter(|run| run.outcome != Outcome::ToolMissing && run.ran()) + .filter(|run| !run.outcome.lacks_tool() && run.ran()) .filter(|run| !shared.iter().any(|(layer, _)| *layer == run.layer)) .count() + usize::from(bend_ran) @@ -437,7 +437,7 @@ pub(crate) fn command_count( "laws" => !bend_ran, _ => !runs .iter() - .any(|run| run.layer == *id && run.outcome != Outcome::ToolMissing && run.ran()), + .any(|run| run.layer == *id && !run.outcome.lacks_tool() && run.ran()), }) .collect(); let mut line = format!( @@ -485,7 +485,8 @@ pub(crate) fn laws_evaluated(cases: &Path) -> Option<(usize, usize)> { pub(crate) fn run_failure(run: &Run) -> Option { match run.outcome { Outcome::Passed | Outcome::Vacuous => None, - Outcome::Failed => Some(match &run.note { + // A tool at another version always carries its note: tool, pin, found. + Outcome::Failed | Outcome::ToolMismatch => Some(match &run.note { Some(note) => format!("{}: {note}", run.layer), None => format!("{}: `{}` failed", run.layer, run.step), }), diff --git a/crates/cargo-tops/src/gate/mod.rs b/crates/cargo-tops/src/gate/mod.rs index ebd89a6..49f49ed 100644 --- a/crates/cargo-tops/src/gate/mod.rs +++ b/crates/cargo-tops/src/gate/mod.rs @@ -10,7 +10,8 @@ //! expired exemption); 2 the config or the request cannot be gated //! (`rust-tops.yaml` missing or failing check, `rust-tops.lock` unreadable, //! no enabled layer at or below `--max-tier`); 3 the environment (a required -//! tool or `cargo metadata` is missing, or `report.json` cannot be written). +//! tool or `cargo metadata` is missing, a pinned tool prints another version +//! than its pin in `rust-tops.lock`, or `report.json` cannot be written). //! Where more than one applies, 2 wins over 3, and 3 over 1; every problem is //! still listed in the report. use std::path::Path; diff --git a/crates/cargo-tops/src/gate/plan.rs b/crates/cargo-tops/src/gate/plan.rs index a5c9a57..b4c1639 100644 --- a/crates/cargo-tops/src/gate/plan.rs +++ b/crates/cargo-tops/src/gate/plan.rs @@ -7,13 +7,14 @@ use anyhow::anyhow; use super::Fault; use crate::config::{self, TopsConfig}; use crate::facts::{Outcome, Run}; +use crate::lock::{self, Row}; use crate::{credit, laws, steps, TOOL_LOCK}; use anyhow::{Context, Result}; use std::fs; use steps::{Base, Kind, MutationPlan, Step}; -/// The command whose first line must name the pinned cargo-crap. -pub(crate) const CRAP_VERSION: [&str; 3] = ["cargo", "crap", "--version"]; +/// The pinned tools each layer needs, with the pin of each. +pub(crate) type LayerPins = Vec<(String, Vec<(&'static Row, String)>)>; /// What a layer runs. pub(crate) enum Plan { @@ -65,7 +66,7 @@ impl TestCommands { .find(|run| run.layer == id) .map(|run| match run.outcome { Outcome::Passed => credit::Outcome::Passed, - Outcome::ToolMissing => credit::Outcome::ToolMissing, + Outcome::ToolMissing | Outcome::ToolMismatch => credit::Outcome::ToolMissing, // A test run is never Vacuous; if one were, it is no pass. Outcome::Failed | Outcome::Vacuous => credit::Outcome::Failed, }) @@ -104,32 +105,29 @@ impl TestCommands { Some(credit::Outcome::Failed) | None => Outcome::Failed, }; let source = runs.iter().find(|run| run.layer == from); + // A source that stopped on a pinned tool's version says so in the + // words it used, not as a missing tool. + let mismatch = source.filter(|run| run.outcome == Outcome::ToolMismatch); Run { layer: layer.to_string(), cmd: source.map(|run| run.cmd.clone()).unwrap_or_default(), step: source.map(|run| run.step.clone()).unwrap_or_default(), - outcome, - note: None, + outcome: mismatch.map_or(outcome, |run| run.outcome), + note: mismatch.and_then(|run| run.note.clone()), } } } -/// The `[tools]` version that `dir/rust-tops.lock` pins for `tool`. No lock, -/// or no line for `tool`, is `None`. A lock that does not parse is an error. -pub(crate) fn tool_pin(dir: &Path, tool: &str) -> Result> { +/// The `[tools]` pins of `dir/rust-tops.lock`. No lock is no pins. A lock +/// that cannot be read or parsed is an error. +fn lock_pins(dir: &Path) -> Result> { let path = dir.join(TOOL_LOCK); let raw = match fs::read_to_string(&path) { Ok(raw) => raw, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Default::default()), Err(err) => return Err(err).with_context(|| format!("read {}", path.display())), }; - let lock: toml::Table = - toml::from_str(&raw).with_context(|| format!("parse {}", path.display()))?; - Ok(lock - .get("tools") - .and_then(|tools| tools.get(tool)) - .and_then(toml::Value::as_str) - .map(str::to_string)) + lock::read_pins(&raw).with_context(|| format!("parse {}", path.display())) } pub(crate) fn tool(argv: &[&str]) -> Vec { @@ -314,25 +312,50 @@ pub(crate) struct GatePlan<'a> { pub(crate) tests: TestCommands, /// The name prefixes of `property` and `acceptance`. pub(crate) filters: Filters, - pub(crate) crap_pin: Option, + /// The pinned tools each running layer needs, asked before the layer's + /// commands run. A layer with none is absent. + pub(crate) pins: LayerPins, pub(crate) units: Vec>, } impl GatePlan<'_> { + /// The pinned tools `layer` needs, as the argv that asks each for its + /// version, once per argv. + pub(crate) fn asked(&self, layer: &str) -> Vec> { + let mut asked: Vec> = Vec::new(); + for (row, _) in self.pins_of(layer) { + let argv: Vec = row.argv.iter().map(|word| (*word).to_string()).collect(); + if !asked.contains(&argv) { + asked.push(argv); + } + } + asked + } + + /// The pinned tools `layer` needs, each with its pin. + pub(crate) fn pins_of(&self, layer: &str) -> &[(&'static Row, String)] { + self.pins + .iter() + .find(|(id, _)| id == layer) + .map_or(&[], |(_, pins)| pins.as_slice()) + } + /// The commands of `layer` as argv lists, in the order they run. pub(crate) fn argv_of(&self, layer: &str) -> Vec> { let Some(unit) = self.units.iter().find(|unit| unit.layer.id == layer) else { return Vec::new(); }; match &unit.what { - What::Chains(chains) => { - let pin = (layer == "crap" && self.crap_pin.is_some()) - .then(|| CRAP_VERSION.map(String::from).to_vec()); - pin.into_iter() - .chain(chains.iter().flatten().map(|step| step.argv.clone())) - .collect() - } - What::Mutation(plan) => vec![plan.diff.argv.clone(), plan.mutants.argv.clone()], + What::Chains(chains) => self + .asked(layer) + .into_iter() + .chain(chains.iter().flatten().map(|step| step.argv.clone())) + .collect(), + What::Mutation(plan) => self + .asked(layer) + .into_iter() + .chain([plan.diff.argv.clone(), plan.mutants.argv.clone()]) + .collect(), What::Shared(from) => self.argv_of(from), What::NotRun => Vec::new(), } @@ -411,6 +434,27 @@ fn units_of<'a>( .collect() } +/// The pinned tools each unit that runs a command needs. The lock is read +/// only when some such layer needs a tool; one that cannot be read is a +/// [`Fault::config`]. +fn pins_of_units(dir: &Path, units: &[Unit<'_>]) -> Result { + let running: Vec<&str> = units + .iter() + .filter(|unit| matches!(unit.what, What::Chains(_) | What::Mutation(_))) + .map(|unit| unit.layer.id.as_str()) + .filter(|id| !lock::tools_for([*id]).is_empty()) + .collect(); + if running.is_empty() { + return Ok(Vec::new()); + } + let pins = lock_pins(dir).map_err(Fault::config)?; + Ok(running + .into_iter() + .map(|id| (id.to_string(), lock::pins_for([id], &pins))) + .filter(|(_, needed)| !needed.is_empty()) + .collect()) +} + /// The plan for `cfg` at `max_tier`. `base` is the `--base` flag; with a /// mutation layer in range it and `gates.base_ref` decide where the diff /// starts. A config the gate cannot plan from is @@ -432,11 +476,8 @@ pub(crate) fn build<'a>( let filters = Filters::of(&cfg.gates); let tests = TestCommands::new(&enabled, &report, &filters); let base = base_for(&enabled, dir, base_flag, cfg.gates.base_ref.as_deref()); - let crap_pin = if enabled.iter().any(|layer| layer.id == "crap") { - tool_pin(dir, "cargo-crap").map_err(Fault::config)? - } else { - None - }; + let units = units_of(&enabled, &tests, &report, &base, &filters); + let pins = pins_of_units(dir, &units)?; Ok(GatePlan { max_tier, laws_on: enabled.iter().any(|l| l.id == "laws"), @@ -445,18 +486,20 @@ pub(crate) fn build<'a>( above: ids(cfg, |l| l.enabled && crate::tiers::tier_for(l) > max_tier), // A layer turned off still shows in the run: it is not run, not omitted. disabled: ids(cfg, |l| !l.enabled), - units: units_of(&enabled, &tests, &report, &base, &filters), + units, enabled, report, tests, filters, - crap_pin, + pins, }) } #[cfg(test)] mod tests { - use super::{base_for, plan_for, Filters, GatePlan, Plan, TestCommands}; + use super::{ + base_for, pins_of_units, plan_for, Filters, GatePlan, Plan, TestCommands, Unit, What, + }; use crate::config::Layer; use crate::steps::{Base, BaseSource}; use std::path::PathBuf; @@ -554,9 +597,36 @@ mod tests { integration: Vec::new(), }, filters: super::Filters::default(), - crap_pin: None, + pins: Vec::new(), units: Vec::new(), }; assert!(plan.argv_of("static-lint").is_empty()); } + + #[test] + fn the_lock_is_read_only_when_a_running_layer_needs_a_pinned_tool() { + let dir = std::env::temp_dir().join(format!( + "cargo-tops-plan-lock-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join("rust-tops.lock"), "[tools\n").expect("write lock"); + // No tool of a layer that needs none, and a layer that runs nothing, + // read nothing: a lock that does not parse is not in the way. + let miri = layer("miri"); + let none = vec![Unit { + layer: &miri, + what: What::NotRun, + }]; + assert!(pins_of_units(&dir, &none).is_ok_and(|pins| pins.is_empty())); + // A running layer that needs a pinned tool reads it. + let lint = layer("static-lint"); + let some = vec![Unit { + layer: &lint, + what: What::Chains(Vec::new()), + }]; + assert!(pins_of_units(&dir, &some).is_err()); + std::fs::remove_dir_all(&dir).ok(); + } } diff --git a/crates/cargo-tops/src/lock.rs b/crates/cargo-tops/src/lock.rs new file mode 100644 index 0000000..881c0de --- /dev/null +++ b/crates/cargo-tops/src/lock.rs @@ -0,0 +1,744 @@ +//! The tool lock (§ tool pins): one table of the eleven pinned tools, how each +//! is asked for its version, and how that line is read. +//! +//! A pin is the `[tools]` value in `rust-tops.lock`. The gate and +//! `cargo tops lock --verify` ask the tool that will run, never a file, so a +//! different binary earlier on `PATH` is a mismatch. `clippy` and `rustfmt` +//! ship with the toolchain, so both are judged by `rustc --version`. Library +//! and name rows (`proptest`, `criterion`, `callgrind-runner`) are not tools +//! on `PATH`; they are skipped. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::process::{Command, ExitCode}; + +use anyhow::{Context, Result}; + +/// One pinned tool: its lock key, the argv that prints its version, and the +/// text the first line starts with before the version. +pub(crate) struct Row { + pub(crate) key: &'static str, + pub(crate) argv: &'static [&'static str], + pub(crate) prefix: &'static str, + /// Other argvs that must exit 0 for the tool to count as present. + pub(crate) present: &'static [&'static [&'static str]], +} + +const RUSTC: &[&str] = &["rustc", "--version"]; + +/// The table. Order is the order `--verify` prints. +pub(crate) const ROWS: [Row; 11] = [ + Row { + key: "rustc", + argv: RUSTC, + prefix: "rustc ", + present: &[], + }, + Row { + key: "clippy", + argv: RUSTC, + prefix: "rustc ", + present: &[&["cargo", "clippy", "--version"]], + }, + Row { + key: "rustfmt", + argv: RUSTC, + prefix: "rustc ", + present: &[&["cargo", "fmt", "--version"]], + }, + Row { + key: "cargo-nextest", + argv: &["cargo", "nextest", "--version"], + prefix: "cargo-nextest ", + present: &[], + }, + Row { + key: "cargo-llvm-cov", + argv: &["cargo", "llvm-cov", "--version"], + prefix: "cargo-llvm-cov ", + present: &[], + }, + Row { + key: "cargo-mutants", + argv: &["cargo", "mutants", "--version"], + prefix: "cargo-mutants ", + present: &[], + }, + Row { + key: "cargo-crap", + argv: &["cargo", "crap", "--version"], + prefix: "cargo-crap ", + present: &[], + }, + Row { + key: "cargo-deny", + argv: &["cargo", "deny", "--version"], + prefix: "cargo-deny ", + present: &[], + }, + Row { + key: "cargo-fuzz", + argv: &["cargo", "fuzz", "--version"], + prefix: "cargo-fuzz ", + present: &[], + }, + Row { + key: "kani-verifier", + argv: &["cargo", "kani", "--version"], + prefix: "Kani Rust Verifier ", + present: &[], + }, + Row { + key: "nushell", + argv: &["nu", "--version"], + prefix: "", + present: &[], + }, +]; + +/// The lock keys a layer's command needs. A layer not listed needs none. +const LAYER_TOOLS: &[(&str, &[&str])] = &[ + ("static-lint", &["rustc", "clippy", "rustfmt", "cargo-deny"]), + ("unit", &["cargo-nextest"]), + ("integration", &["cargo-nextest"]), + ("property", &["cargo-nextest"]), + ("acceptance", &["cargo-nextest"]), + ("doc", &["rustc"]), + ("coverage", &["cargo-llvm-cov", "cargo-nextest"]), + ("crap", &["cargo-crap"]), + ("security-advisory", &["cargo-deny"]), + ("mutation", &["cargo-mutants"]), + ("fuzz", &["cargo-fuzz"]), + ("formal-kani", &["kani-verifier"]), +]; + +/// The tools the given layer ids need, each once, in table order. +pub(crate) fn tools_for<'a>(layers: impl IntoIterator) -> Vec<&'static str> { + let wanted: BTreeSet<&str> = layers + .into_iter() + .filter_map(|id| LAYER_TOOLS.iter().find(|(layer, _)| *layer == id)) + .flat_map(|(_, tools)| tools.iter().copied()) + .collect(); + ROWS.iter() + .map(|row| row.key) + .filter(|key| wanted.contains(key)) + .collect() +} + +/// The version on the first line of a tool's output, read by the row's rule: +/// the first word after the prefix, cut at the first character that is not a +/// digit or a dot. `None` when the line does not have that shape. +pub(crate) fn parse_version(row: &Row, output: &str) -> Option { + let line = output.lines().next()?.trim(); + let word = line.strip_prefix(row.prefix)?.split_whitespace().next()?; + let version: String = word + .chars() + .take_while(|c| c.is_ascii_digit() || *c == '.') + .collect(); + let version = version.trim_end_matches('.'); + // Only digits and dots are left, so a malformed version has an empty part: + // `1..2`, `.5`, or nothing at all. + version + .split('.') + .all(|part| !part.is_empty()) + .then(|| version.to_string()) +} + +/// Whether `found` satisfies `pin`. Whole dot-separated parts only: `0.12` +/// matches `0.12.3` and not `0.120.0`; `0.12.3` does not match `0.12`. +pub(crate) fn pin_matches(pin: &str, found: &str) -> bool { + let pin_parts: Vec<&str> = pin.split('.').collect(); + let found_parts: Vec<&str> = found.split('.').collect(); + found_parts.starts_with(&pin_parts) +} + +/// What asking one tool came to. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Verdict { + Ok { + found: String, + }, + /// The tool ran and printed another version, or a line with no version. + Mismatch { + found: String, + }, + /// The tool could not be run, or exited non-zero; `asked` is the command. + Missing { + asked: String, + }, + /// Nothing to ask: a library or name row, or a tool with no pin. + Skipped { + why: &'static str, + }, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Line { + pub(crate) key: String, + pub(crate) pin: String, + pub(crate) verdict: Verdict, +} + +impl Line { + pub(crate) fn passes(&self) -> bool { + matches!(self.verdict, Verdict::Ok { .. } | Verdict::Skipped { .. }) + } + + /// The line `--verify` prints, and the detail of a `tool-mismatch`. + pub(crate) fn text(&self) -> String { + let Line { key, pin, verdict } = self; + match verdict { + Verdict::Ok { found } => format!("ok {key} {found} (pin {pin})"), + Verdict::Mismatch { found } => { + format!("mismatch {key}: pin {pin}, found {found}") + } + Verdict::Missing { .. } => format!("missing {key}: pin {pin}, not found"), + Verdict::Skipped { why } => format!("skipped {key}: {why}"), + } + } +} + +/// Reads `[tools]` from a lock text. Values that are not strings are dropped. +pub(crate) fn read_pins(text: &str) -> Result> { + let lock: toml::Table = toml::from_str(text).context("parse rust-tops.lock")?; + Ok(lock + .get("tools") + .and_then(toml::Value::as_table) + .map(|tools| { + tools + .iter() + .filter_map(|(key, value)| Some((key.clone(), value.as_str()?.to_string()))) + .collect() + }) + .unwrap_or_default()) +} + +/// A probe that runs each argv in `dir` and returns its stdout, or `None` +/// when it could not run or failed. +pub(crate) fn probe_in(dir: &Path) -> impl FnMut(&[&str]) -> Option + '_ { + move |argv| { + let (program, args) = argv.split_first()?; + let output = Command::new(program) + .args(args) + .current_dir(dir) + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).into_owned()) + } +} + +/// The row for a lock key. +pub(crate) fn row(key: &str) -> Option<&'static Row> { + ROWS.iter().find(|row| row.key == key) +} + +/// The pinned tools the given layers need, each with its pin, in table order. +/// A tool with no pin in the lock is left out: nothing to check. +pub(crate) fn pins_for<'a>( + layers: impl IntoIterator, + pins: &BTreeMap, +) -> Vec<(&'static Row, String)> { + tools_for(layers) + .into_iter() + .filter_map(|key| Some((row(key)?, pins.get(key)?.clone()))) + .collect() +} + +/// Asks one pinned tool. `probe` runs an argv and returns its stdout. +pub(crate) fn check_row( + row: &Row, + pin: &str, + probe: &mut dyn FnMut(&[&str]) -> Option, +) -> Line { + let verdict = match probe(row.argv) { + None => Verdict::Missing { + asked: row.argv.join(" "), + }, + Some(out) => match row.present.iter().find(|argv| probe(argv).is_none()) { + Some(argv) => Verdict::Missing { + asked: argv.join(" "), + }, + None => version_verdict(row, pin, &out), + }, + }; + Line { + key: row.key.to_string(), + pin: pin.to_string(), + verdict, + } +} + +/// What the line a tool printed says about its pin. A line with no version in +/// the shape of the row is a mismatch, and the line is named as found. +fn version_verdict(row: &Row, pin: &str, out: &str) -> Verdict { + let Some(found) = parse_version(row, out) else { + return Verdict::Mismatch { + found: out.lines().next().unwrap_or_default().trim().to_string(), + }; + }; + if pin_matches(pin, &found) { + Verdict::Ok { found } + } else { + Verdict::Mismatch { found } + } +} + +/// Checks the pins. With `only`, the tools in it (a tool with no pin is +/// skipped); without, every pin in the lock, and a lock key with no row is +/// skipped. +pub(crate) fn verify( + pins: &BTreeMap, + only: Option<&[&str]>, + probe: &mut dyn FnMut(&[&str]) -> Option, +) -> Vec { + let mut lines = tool_lines(pins, only, probe); + if only.is_none() { + lines.extend(unmatched_keys(pins)); + } + lines +} + +/// One line for each table row the check is about, in table order. +fn tool_lines( + pins: &BTreeMap, + only: Option<&[&str]>, + probe: &mut dyn FnMut(&[&str]) -> Option, +) -> Vec { + ROWS.iter() + .filter(|row| only.is_none_or(|only| only.contains(&row.key))) + .filter_map(|row| match pins.get(row.key) { + Some(pin) => Some(check_row(row, pin, probe)), + None => only.map(|_| Line { + key: row.key.to_string(), + pin: String::new(), + verdict: Verdict::Skipped { + why: "no pin in rust-tops.lock", + }, + }), + }) + .collect() +} + +/// The lock keys that name no tool: a library or a runner. Skipped. +fn unmatched_keys(pins: &BTreeMap) -> Vec { + pins.iter() + .filter(|(key, _)| row(key).is_none()) + .map(|(key, pin)| Line { + key: key.clone(), + pin: pin.clone(), + verdict: Verdict::Skipped { + why: "a library or name, not a tool on PATH", + }, + }) + .collect() +} + +/// The version each pinned tool prints now, for `--write`. A tool that is +/// missing or prints no version is left out. +pub(crate) fn found_versions( + pins: &BTreeMap, + probe: &mut dyn FnMut(&[&str]) -> Option, +) -> BTreeMap { + ROWS.iter() + .filter(|row| pins.contains_key(row.key)) + .filter_map(|row| { + let found = parse_version(row, &probe(row.argv)?)?; + Some((row.key.to_string(), found)) + }) + .collect() +} + +/// `text` with the `[tools]` values in `found` replaced. Only the quoted value +/// changes; comments, order, spacing and other tables are kept byte for byte. +pub(crate) fn write_values(text: &str, found: &BTreeMap) -> String { + let mut in_tools = false; + let mut out = String::with_capacity(text.len()); + for line in text.split_inclusive('\n') { + let trimmed = line.trim_start(); + if trimmed.starts_with('[') { + in_tools = trimmed.trim_end().starts_with("[tools]"); + } + out.push_str(&if in_tools { + replace_value(line, found) + } else { + line.to_string() + }); + } + out +} + +fn replace_value(line: &str, found: &BTreeMap) -> String { + let Some((key, rest)) = line.split_once('=') else { + return line.to_string(); + }; + let Some(new) = found.get(key.trim()) else { + return line.to_string(); + }; + let Some(open) = rest.find('"') else { + return line.to_string(); + }; + let Some(len) = rest[open + 1..].find('"') else { + return line.to_string(); + }; + let close = open + 1 + len; + format!("{key}={}{new}{}", &rest[..=open], &rest[close..]) +} + +/// What `cargo tops lock` was asked to do. +pub(crate) enum Action { + /// Check the pins; with a tier, only the tools the enabled layers at or + /// below it need. + Verify { max_tier: Option }, + /// Write the versions the tools print into `[tools]`. + Write, +} + +/// `cargo tops lock`. An unreadable lock, or a config that will not load for +/// `--max-tier`, is an error (exit 2); a missing or mismatched tool is exit 3. +pub(crate) fn command(dir: &Path, action: &Action) -> anyhow::Result { + let path = dir.join(crate::TOOL_LOCK); + let text = + std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; + let pins = read_pins(&text).with_context(|| format!("read {}", path.display()))?; + match action { + Action::Verify { max_tier } => verify_command(dir, &pins, *max_tier), + Action::Write => write_command(dir, &path, &text, &pins), + } +} + +/// The tools the enabled layers at or below `tier` need, from the config. +fn tools_in_range(dir: &Path, tier: u8) -> anyhow::Result> { + let cfg = crate::load_config(dir)?; + Ok(tools_for( + cfg.layers + .iter() + .filter(|layer| layer.enabled && crate::tiers::tier_for(layer) <= tier) + .map(|layer| layer.id.as_str()), + )) +} + +fn verify_command( + dir: &Path, + pins: &BTreeMap, + max_tier: Option, +) -> anyhow::Result { + let needed = max_tier.map(|tier| tools_in_range(dir, tier)).transpose()?; + let lines = verify(pins, needed.as_deref(), &mut probe_in(dir)); + for line in &lines { + println!("{}", line.text()); + } + Ok(if lines.iter().all(Line::passes) { + ExitCode::SUCCESS + } else { + ExitCode::from(3) + }) +} + +/// One line for each pinned tool `--write` is about: the old and new value +/// when they differ, and that a tool that could not be run keeps its value. +fn write_lines(pins: &BTreeMap, found: &BTreeMap) -> Vec { + pins.iter() + .filter(|(key, _)| row(key).is_some()) + .map(|(key, pin)| match found.get(key) { + Some(now) if now != pin => format!("wrote {key}: {pin} -> {now}"), + Some(_) => format!("kept {key} {pin}"), + None => format!("kept {key} {pin}: not found"), + }) + .collect() +} + +fn write_command( + dir: &Path, + path: &Path, + text: &str, + pins: &BTreeMap, +) -> anyhow::Result { + let found = found_versions(pins, &mut probe_in(dir)); + for line in write_lines(pins, &found) { + println!("{line}"); + } + let written = write_values(text, &found); + if written != text { + std::fs::write(path, written).with_context(|| format!("write {}", path.display()))?; + } + Ok(ExitCode::SUCCESS) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/versions"); + + fn row(key: &str) -> &'static Row { + ROWS.iter().find(|row| row.key == key).expect("row") + } + + fn fixture(name: &str) -> String { + std::fs::read_to_string(format!("{FIXTURES}/{name}.txt")).expect("fixture") + } + + #[test] + fn the_table_has_the_eleven_tools_once_each() { + let keys: BTreeSet<&str> = ROWS.iter().map(|row| row.key).collect(); + assert_eq!(keys.len(), 11); + assert!([ + "rustc", + "clippy", + "rustfmt", + "cargo-nextest", + "cargo-llvm-cov", + "cargo-mutants", + "cargo-crap", + "cargo-deny", + "cargo-fuzz", + "kani-verifier", + "nushell" + ] + .iter() + .all(|key| keys.contains(key))); + } + + #[test] + fn each_row_reads_the_line_a_real_tool_printed() { + for (key, file, want) in [ + ("rustc", "rustc", "1.99.1"), + ("cargo-nextest", "cargo-nextest", "0.9.145"), + ("cargo-llvm-cov", "cargo-llvm-cov", "0.9.1"), + ("cargo-mutants", "cargo-mutants", "27.1.0"), + ("cargo-crap", "cargo-crap", "0.9.0"), + ("cargo-deny", "cargo-deny", "0.18.9"), + ("cargo-fuzz", "cargo-fuzz", "0.12.0"), + ("kani-verifier", "kani-verifier", "0.68.0"), + ("nushell", "nushell", "0.108.0"), + ] { + assert_eq!( + parse_version(row(key), &fixture(file)).as_deref(), + Some(want), + "{key}" + ); + } + } + + #[test] + fn clippy_and_rustfmt_are_judged_by_the_toolchain_line() { + let line = fixture("rustc"); + for key in ["clippy", "rustfmt"] { + assert_eq!(row(key).argv, ["rustc", "--version"]); + assert_eq!( + parse_version(row(key), &line).as_deref(), + Some("1.99.1"), + "{key}" + ); + } + } + + #[test] + fn clippy_or_rustfmt_that_will_not_run_is_missing_though_rustc_answers() { + let pins = BTreeMap::from([ + ("clippy".to_string(), "1.99.1".to_string()), + ("rustfmt".to_string(), "1.99.1".to_string()), + ]); + let mut probe = probe_for(&[ + ("rustc", Some("rustc 1.99.1 (x 2026-10-13)\n")), + ("clippy", None), + ("fmt", Some("rustfmt 1.10.0-stable (x)\n")), + ]); + let texts: Vec = verify(&pins, None, &mut probe) + .iter() + .map(Line::text) + .collect(); + assert_eq!( + texts, + [ + "missing clippy: pin 1.99.1, not found", + "ok rustfmt 1.99.1 (pin 1.99.1)", + ] + ); + } + + #[test] + fn a_version_with_an_empty_part_reads_as_none() { + for out in [ + "cargo-nextest 1..2\n", + "cargo-nextest .5\n", + "cargo-nextest \n", + ] { + assert_eq!(parse_version(row("cargo-nextest"), out), None, "{out:?}"); + } + // A suffix after the digits and dots is cut, and a trailing dot goes. + for (out, want) in [ + ("cargo-nextest 1.2-rc\n", "1.2"), + ("cargo-nextest 1.\n", "1"), + ] { + assert_eq!( + parse_version(row("cargo-nextest"), out).as_deref(), + Some(want), + "{out:?}" + ); + } + } + + #[test] + fn write_lines_say_what_changed_and_leave_library_rows_out() { + let pins = BTreeMap::from([ + ("cargo-crap".to_string(), "0.9.0".to_string()), + ("cargo-deny".to_string(), "0.18.9".to_string()), + ("cargo-nextest".to_string(), "0.9.145".to_string()), + ("proptest".to_string(), "1".to_string()), + ]); + let found = BTreeMap::from([ + ("cargo-deny".to_string(), "0.18.9".to_string()), + ("cargo-nextest".to_string(), "0.9.150".to_string()), + ]); + assert_eq!( + write_lines(&pins, &found), + [ + "kept cargo-crap 0.9.0: not found", + "kept cargo-deny 0.18.9", + "wrote cargo-nextest: 0.9.145 -> 0.9.150", + ] + ); + } + + #[test] + fn a_line_without_the_prefix_or_a_version_reads_as_none() { + for out in [ + "", + "error: no such command: `nextest`", + "cargo-nextest\n", + "cargo-nextest next\n", + "cargo-crap 0.9.0\n", + ] { + assert_eq!(parse_version(row("cargo-nextest"), out), None, "{out:?}"); + } + } + + #[test] + fn only_the_first_line_is_read() { + let out = "cargo-nextest 0.9.145 (x)\nrelease: 0.9.99\n"; + assert_eq!( + parse_version(row("cargo-nextest"), out).as_deref(), + Some("0.9.145") + ); + } + + #[test] + fn a_partial_pin_matches_whole_parts_only() { + assert!(pin_matches("0.12", "0.12.0")); + assert!(pin_matches("0.12", "0.12.3")); + assert!(!pin_matches("0.12", "0.120.0")); + assert!(!pin_matches("0.12", "0.1.2")); + assert!(pin_matches("0.9.145", "0.9.145")); + assert!(!pin_matches("0.9.145", "0.9.14")); + assert!(!pin_matches("0.12.3", "0.12")); + assert!(!pin_matches("1", "10.0.0")); + assert!(pin_matches("1", "1.4.2")); + } + + fn probe_for<'a>( + table: &'a [(&'a str, Option<&'a str>)], + ) -> impl FnMut(&[&str]) -> Option + 'a { + move |argv| { + table + .iter() + .find(|(tool, _)| argv.contains(tool)) + .and_then(|(_, out)| out.map(str::to_string)) + } + } + + #[test] + fn verify_names_tool_pin_and_found_for_a_mismatch() { + let pins = BTreeMap::from([("cargo-nextest".to_string(), "0.9.145".to_string())]); + let mut probe = probe_for(&[("nextest", Some("cargo-nextest 0.9.144\n"))]); + let lines = verify(&pins, None, &mut probe); + let [line] = lines.as_slice() else { + panic!("one line: {lines:?}") + }; + assert!(!line.passes()); + assert_eq!( + line.text(), + "mismatch cargo-nextest: pin 0.9.145, found 0.9.144" + ); + } + + #[test] + fn a_tool_that_cannot_run_is_missing_and_a_garbled_line_is_a_mismatch() { + let pins = BTreeMap::from([ + ("cargo-crap".to_string(), "0.9.0".to_string()), + ("cargo-deny".to_string(), "0.18.9".to_string()), + ]); + let mut probe = probe_for(&[("crap", None), ("deny", Some("what\n"))]); + let lines = verify(&pins, None, &mut probe); + let texts: Vec = lines.iter().map(Line::text).collect(); + assert_eq!( + texts, + [ + "missing cargo-crap: pin 0.9.0, not found", + "mismatch cargo-deny: pin 0.18.9, found what", + ] + ); + } + + #[test] + fn library_and_name_rows_are_skipped_and_never_asked() { + let pins = BTreeMap::from([ + ("proptest".to_string(), "1".to_string()), + ( + "callgrind-runner".to_string(), + "gungraun-runner".to_string(), + ), + ]); + let mut probe = |_: &[&str]| -> Option { panic!("asked a library row") }; + let lines = verify(&pins, None, &mut probe); + assert_eq!(lines.len(), 2); + assert!(lines + .iter() + .all(|line| line.passes() && matches!(line.verdict, Verdict::Skipped { .. }))); + } + + #[test] + fn only_the_tools_a_layer_needs_are_asked_and_an_unpinned_one_is_skipped() { + let pins = BTreeMap::from([("cargo-mutants".to_string(), "27.1.0".to_string())]); + let needed = tools_for(["unit", "mutation"]); + assert_eq!(needed, ["cargo-nextest", "cargo-mutants"]); + let mut probe = probe_for(&[("mutants", Some("cargo-mutants 27.1.0\n"))]); + let lines = verify(&pins, Some(&needed), &mut probe); + let texts: Vec = lines.iter().map(Line::text).collect(); + assert_eq!( + texts, + [ + "skipped cargo-nextest: no pin in rust-tops.lock", + "ok cargo-mutants 27.1.0 (pin 27.1.0)", + ] + ); + } + + #[test] + fn write_changes_only_the_values_it_found() { + let text = "# keep me\n[tools]\nrustc = \"1.99.1\" # trailing\n# cargo-nextest note\ncargo-nextest = \"0.9.145\"\nproptest = \"1\"\n\n[notes]\nrustc = \"untouched\"\n"; + let found = BTreeMap::from([ + ("cargo-nextest".to_string(), "0.9.150".to_string()), + ("rustc".to_string(), "1.99.2".to_string()), + ]); + assert_eq!( + write_values(text, &found), + "# keep me\n[tools]\nrustc = \"1.99.2\" # trailing\n# cargo-nextest note\ncargo-nextest = \"0.9.150\"\nproptest = \"1\"\n\n[notes]\nrustc = \"untouched\"\n" + ); + } + + #[test] + fn read_pins_reads_tools_only_and_rejects_bad_toml() { + let pins = + read_pins("[tools]\nrustc = \"1.99.1\"\nn = 3\n[notes]\nx = \"y\"\n").expect("pins"); + assert_eq!( + pins, + BTreeMap::from([("rustc".to_string(), "1.99.1".to_string())]) + ); + assert!(read_pins("[tools\n").is_err()); + } +} diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index 2c0217c..d25da4c 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -31,6 +31,7 @@ mod floors; mod gate; mod init_meta; mod laws; +mod lock; mod mutation; mod profile_data; mod profiles; @@ -81,10 +82,38 @@ struct InitArgs { laws: bool, } +#[derive(clap::Args)] +struct LockArgs { + /// Ask each pinned tool for its version and compare it with the pin + #[arg(long, conflicts_with = "write", required_unless_present = "write")] + verify: bool, + /// Write the versions the tools print into the [tools] values, keeping comments and order + #[arg(long)] + write: bool, + /// With --verify: only the tools the enabled layers at or below this tier need + #[arg(long, requires = "verify")] + max_tier: Option, + #[arg(long, default_value = ".")] + path: PathBuf, +} + +impl LockArgs { + /// `--verify` (with its tier) or, without it, `--write`. + fn action(&self) -> lock::Action { + if self.verify { + lock::Action::Verify { + max_tier: self.max_tier, + } + } else { + lock::Action::Write + } + } +} + +/// The commands that act on a project: the directory `--path` names, with its +/// `rust-tops.yaml` and `rust-tops.lock`. #[derive(Subcommand)] -enum Commands { - /// Write rust-tops.yaml, rust-tops.lock, AGENTS.md, RUST-TOPS.md and the drop-in tool configs, read from the crate's Cargo.toml - Init(InitArgs), +enum ProjectCommand { /// Validate rust-tops.yaml against the protocol schema (typed check) Check { #[arg(long, default_value = ".")] @@ -95,10 +124,11 @@ enum Commands { Exit codes:\n 0 pass\n 1 the gate ran and failed (a dry run included)\n \ 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable \ or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable \ -when a layer in range needs a pin (crap today); a command-line usage error is 2 too and \ -writes no report. An expired exemption is not 2: it is 1\n \ -3 the environment: a required tool or cargo metadata is missing, or report.json \ -cannot be written\n\ +when a layer in range needs a pinned tool (every pinned tool a running layer needs reads \ +the lock); a command-line usage error is 2 too and writes no report. An expired \ +exemption is not 2: it is 1\n \ +3 the environment: a required tool or cargo metadata is missing, a pinned tool prints \ +another version than its pin in rust-tops.lock, or report.json cannot be written\n\ Where more than one applies, 2 wins over 3, and 3 over 1.")] Gate { #[arg(long, default_value = ".")] @@ -115,11 +145,26 @@ Where more than one applies, 2 wins over 3, and 3 over 1.")] #[arg(long, value_name = "REF")] base: Option, }, + /// Check or rewrite the tool pins in rust-tops.lock against the tools on PATH + #[command( + after_help = "Exit codes:\n 0 every pin matches (verify) or the lock is written\n \ +2 rust-tops.lock is unreadable, or the config does not load for --max-tier, or a usage error\n \ +3 a pinned tool is missing or prints another version than its pin" + )] + Lock(LockArgs), /// Record the last gate's CRAP report as the baseline the crap ratchet reads CrapBaseline { #[arg(long, default_value = ".")] path: PathBuf, }, +} + +#[derive(Subcommand)] +enum Commands { + /// Write rust-tops.yaml, rust-tops.lock, AGENTS.md, RUST-TOPS.md and the drop-in tool configs, read from the crate's Cargo.toml + Init(InitArgs), + #[command(flatten)] + Project(ProjectCommand), /// Print the package version, the protocol it implements, and the pins in ./rust-tops.lock Version, } @@ -186,8 +231,19 @@ fn run() -> Result { init(&args)?; Ok(ExitCode::SUCCESS) } - Commands::Check { path } => check_command(&path), - Commands::Gate { + Commands::Project(command) => run_project(command), + Commands::Version => { + print_version(); + Ok(ExitCode::SUCCESS) + } + } +} + +/// A command that acts on a project. +fn run_project(command: ProjectCommand) -> Result { + match command { + ProjectCommand::Check { path } => check_command(&path), + ProjectCommand::Gate { path, max_tier, dry_run, @@ -200,15 +256,12 @@ fn run() -> Result { }, base.as_deref(), )), - Commands::CrapBaseline { path } => { + ProjectCommand::Lock(args) => lock::command(&args.path, &args.action()), + ProjectCommand::CrapBaseline { path } => { let cfg = load_config(&path)?; floors::record_baseline(&path, &cfg)?; Ok(ExitCode::SUCCESS) } - Commands::Version => { - print_version(); - Ok(ExitCode::SUCCESS) - } } } diff --git a/crates/cargo-tops/src/steps.rs b/crates/cargo-tops/src/steps.rs index 1dbe27d..c23100b 100644 --- a/crates/cargo-tops/src/steps.rs +++ b/crates/cargo-tops/src/steps.rs @@ -109,17 +109,28 @@ pub(crate) enum Outcome { Passed, Failed, ToolMissing, + /// A pinned tool ran and printed a version other than its pin in + /// `rust-tops.lock`. Like `ToolMissing`, the verdict rests on the + /// environment. + ToolMismatch, /// The diff command succeeded and the diff is empty. Vacuous, } impl Outcome { + /// The tool the layer needs is absent or not the pinned one: the layer ran + /// no command and the verdict rests on the environment. + #[must_use] + pub(crate) fn lacks_tool(self) -> bool { + matches!(self, Self::ToolMissing | Self::ToolMismatch) + } + /// `Some(true)` pass, `Some(false)` fail, `None` when no law may count it. #[must_use] pub(crate) fn law_verdict(self) -> Option { match self { Self::Passed => Some(true), - Self::Failed | Self::ToolMissing => Some(false), + Self::Failed | Self::ToolMissing | Self::ToolMismatch => Some(false), Self::Vacuous => None, } } @@ -133,18 +144,26 @@ impl Outcome { /// Gate failure. Vacuous does not fail the gate. #[must_use] pub(crate) fn fails_the_gate(self) -> bool { - matches!(self, Self::Failed | Self::ToolMissing) + matches!(self, Self::Failed | Self::ToolMissing | Self::ToolMismatch) } } -impl fmt::Display for Outcome { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { +impl Outcome { + /// The words a run's outcome is named by in a problem line. + fn label(self) -> &'static str { + match self { Self::Passed => "passed", Self::Failed => "Failed", Self::ToolMissing => "not on PATH", + Self::ToolMismatch => "not the pinned version", Self::Vacuous => "Vacuous", - }) + } + } +} + +impl fmt::Display for Outcome { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.label()) } } @@ -989,6 +1008,36 @@ fn is_env_assign(token: &str) -> bool { && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') } +#[cfg(test)] +mod outcome_words { + use super::Outcome; + + #[test] + fn each_outcome_is_named_in_its_own_words() { + for (outcome, words) in [ + (Outcome::Passed, "passed"), + (Outcome::Failed, "Failed"), + (Outcome::ToolMissing, "not on PATH"), + (Outcome::ToolMismatch, "not the pinned version"), + (Outcome::Vacuous, "Vacuous"), + ] { + assert_eq!(outcome.to_string(), words); + } + } + + #[test] + fn a_missing_or_mismatched_tool_fails_the_gate_and_is_a_lacking_tool() { + for outcome in [Outcome::ToolMissing, Outcome::ToolMismatch] { + assert!(outcome.lacks_tool(), "{outcome}"); + assert!(outcome.fails_the_gate(), "{outcome}"); + assert_eq!(outcome.law_verdict(), Some(false), "{outcome}"); + } + for outcome in [Outcome::Passed, Outcome::Failed, Outcome::Vacuous] { + assert!(!outcome.lacks_tool(), "{outcome}"); + } + } +} + #[cfg(test)] mod expiry { use super::{Exemption, ExemptionError, Outcome}; diff --git a/crates/cargo-tops/templates/drop-in/rust-tops.yml b/crates/cargo-tops/templates/drop-in/rust-tops.yml index 0088f14..e0a1aae 100644 --- a/crates/cargo-tops/templates/drop-in/rust-tops.yml +++ b/crates/cargo-tops/templates/drop-in/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/fixtures/versions/cargo-crap.txt b/crates/cargo-tops/tests/fixtures/versions/cargo-crap.txt new file mode 100644 index 0000000..a554b83 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/cargo-crap.txt @@ -0,0 +1 @@ +cargo-crap 0.9.0 diff --git a/crates/cargo-tops/tests/fixtures/versions/cargo-deny.txt b/crates/cargo-tops/tests/fixtures/versions/cargo-deny.txt new file mode 100644 index 0000000..471f1fb --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/cargo-deny.txt @@ -0,0 +1 @@ +cargo-deny 0.18.9 diff --git a/crates/cargo-tops/tests/fixtures/versions/cargo-fuzz.txt b/crates/cargo-tops/tests/fixtures/versions/cargo-fuzz.txt new file mode 100644 index 0000000..7914536 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/cargo-fuzz.txt @@ -0,0 +1 @@ +cargo-fuzz 0.12.0 diff --git a/crates/cargo-tops/tests/fixtures/versions/cargo-llvm-cov.txt b/crates/cargo-tops/tests/fixtures/versions/cargo-llvm-cov.txt new file mode 100644 index 0000000..0eb7ca3 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/cargo-llvm-cov.txt @@ -0,0 +1 @@ +cargo-llvm-cov 0.9.1 diff --git a/crates/cargo-tops/tests/fixtures/versions/cargo-mutants.txt b/crates/cargo-tops/tests/fixtures/versions/cargo-mutants.txt new file mode 100644 index 0000000..c5cc563 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/cargo-mutants.txt @@ -0,0 +1 @@ +cargo-mutants 27.1.0 diff --git a/crates/cargo-tops/tests/fixtures/versions/cargo-nextest.txt b/crates/cargo-tops/tests/fixtures/versions/cargo-nextest.txt new file mode 100644 index 0000000..4e94404 --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/cargo-nextest.txt @@ -0,0 +1,5 @@ +cargo-nextest 0.9.145 (00af4550e 2026-09-16) +release: 0.9.145 +commit-hash: 00af4550ec3b3b9f0e574b897b06acb95d325ba2 +commit-date: 2026-09-16 +host: aarch64-apple-darwin diff --git a/crates/cargo-tops/tests/fixtures/versions/kani-verifier.txt b/crates/cargo-tops/tests/fixtures/versions/kani-verifier.txt new file mode 100644 index 0000000..c079b0d --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/kani-verifier.txt @@ -0,0 +1,2 @@ +Kani Rust Verifier 0.68.0 (cargo plugin) +CBMC 6.11.0 diff --git a/crates/cargo-tops/tests/fixtures/versions/nushell.txt b/crates/cargo-tops/tests/fixtures/versions/nushell.txt new file mode 100644 index 0000000..3f667db --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/nushell.txt @@ -0,0 +1 @@ +0.108.0 diff --git a/crates/cargo-tops/tests/fixtures/versions/rustc.txt b/crates/cargo-tops/tests/fixtures/versions/rustc.txt new file mode 100644 index 0000000..6f8ebab --- /dev/null +++ b/crates/cargo-tops/tests/fixtures/versions/rustc.txt @@ -0,0 +1 @@ +rustc 1.99.1 (58cf8f9c3 2026-10-13) diff --git a/crates/cargo-tops/tests/gate_base.rs b/crates/cargo-tops/tests/gate_base.rs index 6f7f563..dac9fd0 100644 --- a/crates/cargo-tops/tests/gate_base.rs +++ b/crates/cargo-tops/tests/gate_base.rs @@ -8,6 +8,7 @@ //! Each case is a real git repository. Cargo and its plugins are stubs that //! log what they are asked, so a test sees whether `cargo mutants` ran and //! on which diff. Every report is validated against the schema. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -29,7 +30,17 @@ const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.jso /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi -if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac # 27.1.0 finds nothing to mutate in a diff of comments, tests, docs or CI: # it says so on stderr, exits 0 and writes no `mutants.out`. @@ -137,6 +148,7 @@ fn scaffold(name: &str) -> Fixture { fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); } diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs index 5ee6b15..886dbaa 100644 --- a/crates/cargo-tops/tests/gate_credit.rs +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -7,6 +7,7 @@ //! `target/nextest/ci/junit.xml` in the repo, the way nextest does, and //! writes no report when the test left none. Only the layers the protocol //! requires of a library-core instance, plus the one under test, are on. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -17,7 +18,17 @@ use serde_json::{json, Value}; /// way the floors tests do, and, for `llvm-cov nextest`, plays nextest: the /// JUnit report in `$REPORTS/junit.xml` goes to the path nextest uses. const CARGO_STUB: &str = r#"#!/bin/sh -if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json"; exit 0; fi if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/coverage-fails" ]; then exit 1; fi if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/junit.xml" ]; then @@ -103,6 +114,7 @@ fn fixture(name: &str, extra: &[&str], junit: Option<&str>) -> Fixture { /// The tool stubs, the coverage summary, and coverage's JUnit report. fn stage_tools_and_reports(fixture: &Fixture, junit: Option<&str>) { stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); for plugin in [ "cargo-llvm-cov", "cargo-crap", diff --git a/crates/cargo-tops/tests/gate_credit_nextest.rs b/crates/cargo-tops/tests/gate_credit_nextest.rs index 231548b..b261963 100644 --- a/crates/cargo-tops/tests/gate_credit_nextest.rs +++ b/crates/cargo-tops/tests/gate_credit_nextest.rs @@ -4,6 +4,7 @@ //! looks. These tests cover a `CARGO_TARGET_DIR` that moves the build, a //! `[store] dir` that moves nextest's reports, and a report left behind by an //! earlier run (PROTOCOL.md §9.5). +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -14,6 +15,17 @@ use serde_json::Value; /// `$REAL_NEXTEST` is set. `llvm-cov report` writes a fully covered summary. /// `cargo metadata` is the real cargo; any other subcommand passes. const CARGO_STUB: &str = r#"#!/bin/sh +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac if [ "$1 $2" = "llvm-cov nextest" ]; then prev= for arg in "$@"; do @@ -98,6 +110,7 @@ fn fixture(name: &str) -> Fixture { fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); set_tests(&fixture, WITH_TEST); stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); for plugin in ["cargo-llvm-cov", "cargo-deny"] { stub(&fixture.bin, plugin, "#!/bin/sh\nexit 0\n"); } diff --git a/crates/cargo-tops/tests/gate_exemptions.rs b/crates/cargo-tops/tests/gate_exemptions.rs index aba772e..bb92c7f 100644 --- a/crates/cargo-tops/tests/gate_exemptions.rs +++ b/crates/cargo-tops/tests/gate_exemptions.rs @@ -6,6 +6,7 @@ //! //! Cargo and its plugins are stubs. Every report is validated against the //! schema. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -26,7 +27,17 @@ const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.jso /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi -if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac exit 0 "#; @@ -102,6 +113,7 @@ fn scaffold(name: &str) -> Fixture { fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); } diff --git a/crates/cargo-tops/tests/gate_floors.rs b/crates/cargo-tops/tests/gate_floors.rs index 2dacac0..b5f9b0e 100644 --- a/crates/cargo-tops/tests/gate_floors.rs +++ b/crates/cargo-tops/tests/gate_floors.rs @@ -6,6 +6,7 @@ //! summary and the CRAP report the test put in the fixture's `reports/`, so //! each test states the numbers the gate judges. Only the coverage and crap //! layers are on. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -21,11 +22,17 @@ if [ -n "${CARGO_STUB_FAIL:-}" ]; then case "$*" in *"$CARGO_STUB_FAIL"*) exit 1 ;; esac fi if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi -if [ "$1 $2" = "crap --version" ]; then - # The lock `init` writes pins cargo-crap 0.9.0; the gate compares this line. - echo "cargo-crap 0.9.0" - exit 0 -fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json" exit 0 @@ -76,6 +83,7 @@ fn stub(bin: &Path, name: &str, body: &str) { /// is among them, so no test here depends on the host having it. fn stub_tools(bin: &Path) { stub(bin, "cargo", CARGO_STUB); + support::stub_rustc(bin); for plugin in [ "cargo-llvm-cov", "cargo-crap", diff --git a/crates/cargo-tops/tests/gate_real_mutants.rs b/crates/cargo-tops/tests/gate_real_mutants.rs index 124c207..709ca55 100644 --- a/crates/cargo-tops/tests/gate_real_mutants.rs +++ b/crates/cargo-tops/tests/gate_real_mutants.rs @@ -3,6 +3,7 @@ //! and says so on stderr; the layer passes as zero mutants. Only `cargo-deny` //! is a stub. The test needs `cargo-mutants` 27.1.0 on `PATH`: without it //! the test stops, unless `RUST_TOPS_REQUIRE_MUTANTS` is set, which fails it. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -23,7 +24,17 @@ const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.jso /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi -if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac if [ "$1" = "mutants" ]; then out= @@ -105,6 +116,20 @@ fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); } +/// The plugin files `check_tool` looks for. Real cargo hands `cargo deny +/// --version` to the `cargo-deny` file, so it answers with the pinned line, as +/// the real tool does. +fn stub_plugins(bin: &Path) { + for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { + let body = if tool == "cargo-deny" { + "#!/bin/sh\necho 'cargo-deny 0.18.9'\n" + } else { + "#!/bin/sh\nexit 0\n" + }; + stub(bin, tool, body); + } +} + /// A library-core instance named `fixture`, in a directory of its own with /// a `Cargo.toml`, one source file and stub tools, not yet a git repository. fn scaffold(name: &str) -> Fixture { @@ -140,9 +165,8 @@ fn scaffold(name: &str) -> Fixture { fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); stub(&fixture.bin, "cargo", CARGO_STUB); - for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { - stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); - } + support::stub_rustc(&fixture.bin); + stub_plugins(&fixture.bin); fixture } diff --git a/crates/cargo-tops/tests/gate_report.rs b/crates/cargo-tops/tests/gate_report.rs index 0752d9c..476f993 100644 --- a/crates/cargo-tops/tests/gate_report.rs +++ b/crates/cargo-tops/tests/gate_report.rs @@ -5,6 +5,7 @@ //! //! Cargo and its plugins are stubs. Every report is validated against //! `schema/gate-report.schema.json`. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -31,7 +32,18 @@ const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.jso /// Fails for an argv that matches `$CARGO_STUB_FAIL`; answers every /// `--version` ask; passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh -if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac if [ -n "${CARGO_STUB_FAIL:-}" ]; then case "$*" in *"$CARGO_STUB_FAIL"*) exit 1 ;; esac @@ -93,6 +105,7 @@ fn fixture(name: &str) -> Fixture { ) .expect("write Cargo.toml"); stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); stub(&fixture.bin, "cargo-deny", "#!/bin/sh\nexit 0\n"); stub(&fixture.bin, "cargo-nextest", "#!/bin/sh\nexit 0\n"); edit(&fixture, |doc| { @@ -147,6 +160,7 @@ fn gate(fixture: &Fixture, args: &[&str], fail: Option<&str>) -> (Output, String .arg("--path") .arg(&fixture.repo) .env("PATH", path(fixture)) + .env("CARGO_STUB_LOG", fixture.root.join("cargo.log")) .env_remove("CARGO_TARGET_DIR") .env_remove("CARGO_STUB_FAIL"); if let Some(pattern) = fail { @@ -218,14 +232,20 @@ fn a_pass_exits_0_and_the_report_says_so() { assert_eq!(at(lint, "/verdict"), "pass"); assert_eq!( at(lint, "/argv").as_array().map(Vec::len), - Some(3), + Some(5), "{lint}" ); + // The pinned tools are asked first: the toolchain, then cargo-deny. + assert_eq!(at(lint, "/argv/0"), &json!(["rustc", "--version"])); + assert_eq!(at(lint, "/argv/1"), &json!(["cargo", "deny", "--version"])); assert_eq!( - at(lint, "/argv/0"), + at(lint, "/argv/2"), &json!(["cargo", "fmt", "--all", "--", "--check"]) ); - assert_eq!(at(lint, "/tool_version"), "stub-tool 1.2.3"); + assert_eq!( + at(lint, "/tool_version"), + "rustfmt 1.10.0-stable (stub); clippy 0.1.99 (stub); cargo-deny 0.18.9" + ); assert!(at(&report, "/disabled") .as_array() .is_some_and(|d| !d.is_empty())); @@ -294,7 +314,7 @@ fn a_dry_run_exits_1_and_writes_a_report_with_nothing_run() { assert_eq!(at(lint, "/verdict"), "not-run"); assert_eq!( at(lint, "/argv").as_array().map(Vec::len), - Some(3), + Some(5), "{lint}" ); assert_eq!(at(lint, "/seconds").as_f64(), Some(0.0)); @@ -565,6 +585,8 @@ fn a_directory_with_no_manifest_is_an_unavailable_cargo_metadata() { #[test] fn a_tool_version_is_asked_from_the_crate_directory() { let fixture = fixture("version-dir"); + // No pins: the check asks nothing, so only the report's own ask is here. + fs::remove_file(fixture.repo.join("rust-tops.lock")).expect("init wrote a lock"); stub( &fixture.bin, "cargo", @@ -607,3 +629,159 @@ fn the_law_total_is_left_out_when_no_judgement_counted_it() { assert_eq!(out.status.code(), Some(2), "{text}"); assert!(report(&fixture).get("laws_total").is_none()); } + +/// The cargo stub, except that `cargo --version` answers `answer` +/// (`exit 101` with no output when `answer` is `None`), as the tool would. +fn answer_version(fixture: &Fixture, sub: &str, answer: Option<&str>) { + let reply = answer.map_or_else( + || "exit 101".to_string(), + |line| format!("echo '{line}'; exit 0"), + ); + let log = "if [ -n \"${CARGO_STUB_LOG:-}\" ]; then echo \"$*\" >> \"$CARGO_STUB_LOG\"; fi\n"; + let body = CARGO_STUB.replacen( + log, + &format!("{log}if [ \"$1 $2\" = \"{sub} --version\" ]; then {reply}; fi\n"), + 1, + ); + stub(&fixture.bin, "cargo", &body); +} + +fn problems(report: &Value) -> Vec { + at(report, "/problems") + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() +} + +#[test] +fn a_pinned_tool_at_another_version_is_a_tool_mismatch_worded_once() { + let fixture = fixture("mismatch"); + answer_version(&fixture, "deny", Some("cargo-deny 0.18.8")); + let (out, text) = gate(&fixture, &[], None); + let said = "mismatch cargo-deny: pin 0.18.9, found 0.18.8"; + assert_eq!(out.status.code(), Some(3), "{text}"); + let report = report(&fixture); + expect_consistent(&report, 3); + // The same words on stdout, in the layer's detail and in the problems. + assert!( + text.contains(&format!("tool-mismatch [static-lint]: {said}")), + "{text}" + ); + assert_eq!( + at(layer(&report, "static-lint"), "/verdict"), + "tool-mismatch" + ); + assert_eq!(at(layer(&report, "static-lint"), "/detail"), said); + assert!( + problems(&report).contains(&format!("static-lint: {said}")), + "{report}" + ); + assert_eq!( + at(layer(&report, "security-advisory"), "/verdict"), + "tool-mismatch" + ); + // Nothing of the layer ran: no fmt, no clippy, no deny check. + let log = cargo_log(&fixture); + assert!( + !log.lines() + .filter(|line| !line.ends_with("--version")) + .any(|line| line.starts_with("fmt ") || line.starts_with("clippy ")), + "{log}" + ); +} + +#[test] +fn the_toolchain_pin_judges_the_layers_that_need_rustc() { + let fixture = fixture("rustc-mismatch"); + stub( + &fixture.bin, + "rustc", + "#!/bin/sh\necho 'rustc 1.98.0 (stub 2025-01-01)'\n", + ); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + let report = report(&fixture); + assert_eq!( + at(layer(&report, "static-lint"), "/verdict"), + "tool-mismatch" + ); + assert_eq!( + at(layer(&report, "static-lint"), "/detail"), + "mismatch rustc: pin 1.99.1, found 1.98.0; mismatch clippy: pin 1.99.1, found 1.98.0; mismatch rustfmt: pin 1.99.1, found 1.98.0" + ); +} + +#[test] +fn clippy_that_will_not_run_is_a_missing_tool_not_a_version_miss() { + let fixture = fixture("no-clippy"); + answer_version(&fixture, "clippy", None); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + assert!(text.contains("static-lint: `cargo` not on PATH"), "{text}"); + let report = report(&fixture); + assert_eq!( + at(layer(&report, "static-lint"), "/verdict"), + "tool-missing" + ); +} + +#[test] +fn a_tool_only_a_layer_above_the_range_needs_is_not_asked() { + let fixture = fixture("above-range"); + // `property` is tier 1 and needs nextest; at --max-tier 0 it is not in range. + answer_version(&fixture, "nextest", Some("cargo-nextest 0.0.1")); + let (out, text) = gate(&fixture, &["--max-tier", "0"], None); + assert_eq!(out.status.code(), Some(0), "{text}"); + let log = cargo_log(&fixture); + assert!( + !log.lines().any(|line| line.starts_with("nextest ")), + "{log}" + ); + let (out, text) = gate(&fixture, &["--max-tier", "1"], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + assert!( + text.contains("mismatch cargo-nextest: pin 0.9.145, found 0.0.1"), + "{text}" + ); +} + +#[test] +fn a_dry_run_names_the_pins_and_asks_nothing() { + let fixture = fixture("dry-pins"); + answer_version(&fixture, "deny", Some("cargo-deny 0.0.1")); + let (out, text) = gate(&fixture, &["--dry-run"], None); + assert_eq!(out.status.code(), Some(1), "{text}"); + assert!( + text.contains("[static-lint] rust-tops.lock pins, asked first: rustc 1.99.1, clippy 1.99.1, rustfmt 1.99.1, cargo-deny 0.18.9"), + "{text}" + ); + assert!(!text.contains("tool-mismatch"), "{text}"); +} + +#[test] +fn a_layer_that_takes_another_layers_run_keeps_that_runs_mismatch() { + let fixture = fixture("takes-mismatch"); + // `property` runs itself here, so turn coverage on: it takes coverage's run. + edit(&fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let on = stays_on(layer) || layer.get("id").and_then(Value::as_str) == Some("coverage"); + set(layer, "enabled", Value::Bool(on)); + } + }); + answer_version(&fixture, "llvm-cov", Some("cargo-llvm-cov 0.8.0")); + let (out, text) = gate(&fixture, &[], None); + let said = "mismatch cargo-llvm-cov: pin 0.9.1, found 0.8.0"; + assert_eq!(out.status.code(), Some(3), "{text}"); + let report = report(&fixture); + assert_eq!(at(layer(&report, "coverage"), "/verdict"), "tool-mismatch"); + assert_eq!(at(layer(&report, "property"), "/verdict"), "tool-mismatch"); + assert_eq!(at(layer(&report, "property"), "/detail"), said); +} + +fn cargo_log(fixture: &Fixture) -> String { + fs::read_to_string(fixture.root.join("cargo.log")).unwrap_or_default() +} diff --git a/crates/cargo-tops/tests/gate_survivors.rs b/crates/cargo-tops/tests/gate_survivors.rs index 7a31820..425cca9 100644 --- a/crates/cargo-tops/tests/gate_survivors.rs +++ b/crates/cargo-tops/tests/gate_survivors.rs @@ -13,6 +13,7 @@ //! what a run prints, when a test says so. The planted files come from the fixture //! recorded from a real 27.1.0 run. Every report is validated against the //! schema. +mod support; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -33,7 +34,17 @@ const REPORT_SCHEMA: &str = include_str!("../../../schema/gate-report.schema.jso /// passes everything else. const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then echo "$*" >> "$CARGO_STUB_LOG"; fi -if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac case "$*" in *--version) echo "stub-tool 1.2.3"; exit 0 ;; esac if [ "$1" = "mutants" ]; then out= @@ -150,6 +161,7 @@ fn scaffold(name: &str) -> Fixture { fs::write(fixture.repo.join("src/a.rs"), "pub fn a() {}\n").expect("write a.rs"); fs::write(fixture.repo.join(".gitignore"), "/target\n").expect("write .gitignore"); stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); for tool in ["cargo-deny", "cargo-nextest", "cargo-mutants"] { stub(&fixture.bin, tool, "#!/bin/sh\nexit 0\n"); } diff --git a/crates/cargo-tops/tests/golden/init/binary-cli/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/binary-cli/.github/workflows/rust-tops.yml index 0088f14..e0a1aae 100644 --- a/crates/cargo-tops/tests/golden/init/binary-cli/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/binary-cli/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/ffi-bridge/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/ffi-bridge/.github/workflows/rust-tops.yml index c2d18d4..3a26cb5 100644 --- a/crates/cargo-tops/tests/golden/init/ffi-bridge/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/ffi-bridge/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/library-core/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/library-core/.github/workflows/rust-tops.yml index 0088f14..e0a1aae 100644 --- a/crates/cargo-tops/tests/golden/init/library-core/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/library-core/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/library-util/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/library-util/.github/workflows/rust-tops.yml index 0088f14..e0a1aae 100644 --- a/crates/cargo-tops/tests/golden/init/library-util/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/library-util/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/no-std-embedded/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/no-std-embedded/.github/workflows/rust-tops.yml index ec2b6f2..58f4061 100644 --- a/crates/cargo-tops/tests/golden/init/no-std-embedded/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/no-std-embedded/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/parser-codec/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/parser-codec/.github/workflows/rust-tops.yml index c4b6146..979eef7 100644 --- a/crates/cargo-tops/tests/golden/init/parser-codec/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/parser-codec/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/unsafe-kernel/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/unsafe-kernel/.github/workflows/rust-tops.yml index eadedb1..094599c 100644 --- a/crates/cargo-tops/tests/golden/init/unsafe-kernel/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/unsafe-kernel/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/golden/init/workspace-root/.github/workflows/rust-tops.yml b/crates/cargo-tops/tests/golden/init/workspace-root/.github/workflows/rust-tops.yml index 0088f14..e0a1aae 100644 --- a/crates/cargo-tops/tests/golden/init/workspace-root/.github/workflows/rust-tops.yml +++ b/crates/cargo-tops/tests/golden/init/workspace-root/.github/workflows/rust-tops.yml @@ -221,6 +221,8 @@ jobs: with: tool: cargo-mutants@27.1.0 fallback: none + # The installed tools are the pinned ones, at the tiers the gate runs. + - run: cargo tops lock --verify --max-tier 1 - run: cargo tops gate --max-tier 1 --base "origin/${{ github.base_ref || github.event.repository.default_branch }}" # mutants-diff: class overlay. Diff-scoped; no --baseline skip; default diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 7c98d53..7186487 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -16,6 +16,7 @@ reason = "this crate already violates the strict table" )] #![expect(clippy::panic, reason = "this crate already violates the strict table")] +mod support; use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; @@ -46,10 +47,17 @@ if [ -n "${CARGO_STUB_LOG:-}" ]; then printf '%s\n' "$*" >> "$CARGO_STUB_LOG" fi # `init` writes rust-tops.lock, which pins cargo-crap 0.9.0. -if [ "$*" = "crap --version" ]; then - echo 'cargo-crap 0.9.0' - exit 0 -fi +case "$1 $2" in + "nextest --version") echo "cargo-nextest 0.9.145 (stub)"; exit 0 ;; + "llvm-cov --version") echo "cargo-llvm-cov 0.9.1"; exit 0 ;; + "mutants --version") echo "cargo-mutants 27.1.0"; exit 0 ;; + "crap --version") echo "cargo-crap 0.9.0"; exit 0 ;; + "deny --version") echo "cargo-deny 0.18.9"; exit 0 ;; + "fuzz --version") echo "cargo-fuzz 0.12.0"; exit 0 ;; + "kani --version") echo "Kani Rust Verifier 0.68.0 (cargo plugin)"; exit 0 ;; + "clippy --version") echo "clippy 0.1.99 (stub)"; exit 0 ;; + "fmt --version") echo "rustfmt 1.10.0-stable (stub)"; exit 0 ;; +esac # 27.1.0 finds nothing to mutate in a diff of comments, tests, docs or CI: # it says so on stderr, exits 0 and writes no `mutants.out`. if [ "$1" = "mutants" ]; then @@ -242,6 +250,7 @@ fn init_with(name: &str, class: &str, extra: &[&str]) -> Fixture { ) .expect("write Cargo.toml"); stub(&fixture.bin, "cargo", CARGO_STUB); + support::stub_rustc(&fixture.bin); for plugin in PLUGINS { stub(&fixture.bin, plugin, "#!/bin/sh\nexit 0\n"); } @@ -448,10 +457,12 @@ fn cargo_log(fixture: &Fixture) -> String { fs::read_to_string(fixture.root.join("cargo.log")).unwrap_or_default() } -/// Whether the cargo stub ran subcommand `sub`. +/// Whether the cargo stub ran subcommand `sub`. A `--version` ask is the +/// gate checking a pin or naming a version, not the subcommand running. fn cargo_ran(fixture: &Fixture, sub: &str) -> bool { cargo_log(fixture) .lines() + .filter(|line| !line.ends_with("--version")) .any(|line| line.split_whitespace().next() == Some(sub)) } @@ -476,6 +487,22 @@ fn nextest_runs(fixture: &Fixture) -> Vec { /// The cargo stub, except that it exits `code` (after logging) when its /// argv matches the shell `case` pattern `pattern`. +/// The gate's `report.json`, parsed. +fn report_json(fixture: &Fixture) -> serde_json::Value { + let raw = + fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")).expect("report.json"); + serde_json::from_str(&raw).expect("report.json is json") +} + +/// The report row of layer `id`. +fn layer_of(report: &serde_json::Value, id: &str) -> serde_json::Value { + at(report, "/layers") + .as_array() + .and_then(|rows| rows.iter().find(|row| at(row, "/id") == id)) + .cloned() + .unwrap_or_default() +} + fn stub_cargo_failing(fixture: &Fixture, pattern: &str, code: u8) { let body = CARGO_STUB.replacen( "#!/bin/sh\n", @@ -1367,7 +1394,8 @@ fn a_missing_or_failed_coverage_run_is_no_pass_for_the_layers_that_took_it() { // Failed: the coverage run exits 1, and that is what the three take. let failed = fixture_with("one-run-failed", &TEST_LAYERS); - stub_cargo_failing(&failed, "llvm-cov*", 1); + // The run fails; asking the tool its version (the pin check) still works. + stub_cargo_failing(&failed, "\"llvm-cov nextest\"*", 1); let (out, text) = gate(&failed, path_with_only_stubs(&failed.bin)); assert_eq!(out.status.code(), Some(1), "{text}"); assert_eq!(nextest_runs(&failed), [coverage_run(&failed)], "{text}"); @@ -2034,23 +2062,34 @@ fn pin_crap(fixture: &Fixture, version: &str) { } #[test] -fn a_cargo_crap_that_is_not_the_lock_pin_fails_the_crap_layer() { +fn a_cargo_crap_that_is_not_the_lock_pin_is_a_tool_mismatch_for_the_crap_layer() { // The second pin is not 0.9.0: the gate reads the pin, it has none built in. - for (pin, printed) in [("0.9.0", "cargo-crap 0.5.0"), ("1.2.3", "cargo-crap 0.9.0")] { + for (pin, printed, found) in [ + ("0.9.0", "cargo-crap 0.5.0", "0.5.0"), + ("1.2.3", "cargo-crap 0.9.0", "0.9.0"), + ] { let fixture = fixture_with(&format!("crap-pin-miss-{pin}"), &["coverage", "crap"]); pin_crap(&fixture, pin); stub_crap_version(&fixture, printed); let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); - assert_eq!(out.status.code(), Some(1), "{text}"); + assert_eq!(out.status.code(), Some(3), "{text}"); + let said = format!("mismatch cargo-crap: pin {pin}, found {found}"); + // One wording: the stdout line, the problem and the layer's detail. assert!( - text.contains(&format!( - "Failed [crap]: `cargo crap --version` printed `{printed}`, not `cargo-crap {pin}` from rust-tops.lock" - )), + text.contains(&format!("tool-mismatch [crap]: {said}")), "{text}" ); + assert!(text.contains(&format!("crap: {said}")), "{text}"); + let report = report_json(&fixture); + let crap = layer_of(&report, "crap"); + assert_eq!(at(&crap, "/verdict"), "tool-mismatch"); + assert_eq!(at(&crap, "/detail"), said); + assert_eq!(at(&report, "/exit_code"), 3); assert!( - text.contains("crap: `cargo crap --version` failed"), - "{text}" + at(&report, "/problems") + .as_array() + .is_some_and(|all| all.iter().any(|p| p == &format!("crap: {said}"))), + "{report}" ); let log = cargo_log(&fixture); assert!(log.lines().any(|line| line == "crap --version"), "{log}"); @@ -2072,19 +2111,15 @@ fn the_crap_layer_runs_when_cargo_crap_reports_the_lock_pin() { let fixture = fixture_with(&format!("crap-pin-hit-{pin}"), &["coverage", "crap"]); pin_crap(&fixture, pin); stub_crap_version(&fixture, &format!("cargo-crap {pin}")); - let must = - format!("`cargo crap --version` must print `cargo-crap {pin}`, the rust-tops.lock pin"); + let asked = format!("[crap] rust-tops.lock pins, asked first: cargo-crap {pin}"); let (_, dry) = gate_with( &fixture, path_without_bend(&fixture.bin), None, &["--dry-run"], ); - assert!( - dry.contains("\n==> [crap] cargo crap --version && cargo crap --lcov "), - "{dry}" - ); - assert!(dry.contains(&must), "{dry}"); + assert!(dry.contains(&asked), "{dry}"); + assert!(dry.contains("\n==> [crap] cargo crap --lcov "), "{dry}"); assert!( cargo_log(&fixture) .lines() @@ -2093,29 +2128,29 @@ fn the_crap_layer_runs_when_cargo_crap_reports_the_lock_pin() { ); let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); assert_eq!(out.status.code(), Some(0), "{text}"); - assert!(text.contains(&must), "{text}"); + assert!(text.contains(&asked), "{text}"); let log = cargo_log(&fixture); let crap: Vec<&str> = log .lines() .filter(|line| line.starts_with("crap ")) .collect(); + // Asked once: the check and the report's tool version share the answer. assert_eq!(crap.len(), 2, "{log}"); - #[expect( - clippy::indexing_slicing, - reason = "already in this crate when the strict table was turned on" - )] - let tops_idx_1962_20 = crap[0]; - assert_eq!(tops_idx_1962_20, "crap --version", "{log}"); - #[expect( - clippy::indexing_slicing, - reason = "already in this crate when the strict table was turned on" - )] - let tops_idx_1967_17 = crap[1]; - assert!(tops_idx_1967_17.starts_with("crap --lcov "), "{log}"); + assert_eq!(crap.first().copied(), Some("crap --version"), "{log}"); + assert!( + crap.get(1) + .is_some_and(|line| line.starts_with("crap --lcov ")), + "{log}" + ); assert!( fixture.repo.join("target/rust-tops/crap.json").is_file(), "{text}" ); + let report = report_json(&fixture); + assert_eq!( + at(&layer_of(&report, "crap"), "/tool_version"), + format!("cargo-crap {pin}") + ); } } @@ -2126,7 +2161,7 @@ fn without_a_rust_tops_lock_the_crap_layer_asks_no_version() { stub_crap_version(&fixture, "cargo-crap 0.5.0"); let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); assert_eq!(out.status.code(), Some(0), "{text}"); - assert!(!text.contains("must print"), "{text}"); + assert!(!text.contains("asked first"), "{text}"); let log = cargo_log(&fixture); assert!(!log.lines().any(|line| line == "crap --version"), "{log}"); assert!( @@ -2139,13 +2174,19 @@ fn without_a_rust_tops_lock_the_crap_layer_asks_no_version() { fn a_pinned_cargo_crap_that_is_not_installed_is_missing_not_a_version_miss() { let fixture = fixture_with("crap-pin-missing", &["coverage", "crap"]); pin_crap(&fixture, "0.9.0"); - fs::remove_file(fixture.bin.join("cargo-crap")).expect("remove stub"); - let (out, text) = gate(&fixture, path_with_only_stubs(&fixture.bin)); + // `cargo crap` with no cargo-crap on PATH: cargo itself says no such command. + stub_cargo_failing(&fixture, "\"crap --version\"", 101); + let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); assert_eq!(out.status.code(), Some(3), "{text}"); assert!(text.contains("crap: `cargo-crap` not on PATH"), "{text}"); - assert!(!text.contains("Failed [crap]"), "{text}"); + assert!(!text.contains("tool-mismatch"), "{text}"); + let report = report_json(&fixture); + assert_eq!(at(&layer_of(&report, "crap"), "/verdict"), "tool-missing"); let log = cargo_log(&fixture); - assert!(!log.lines().any(|line| line.starts_with("crap ")), "{log}"); + assert!( + !log.lines().any(|line| line.starts_with("crap --lcov")), + "{log}" + ); } #[test] @@ -2169,16 +2210,6 @@ fn a_rust_tops_lock_that_cannot_be_read_or_parsed_stops_the_gate() { } } -/// Only the crap layer reads the pin, so a gate without it runs on past a -/// `rust-tops.lock` that does not parse. -#[test] -fn without_the_crap_layer_the_gate_does_not_read_the_pin() { - let fixture = fixture_with("crap-off-bad-lock", &["coverage"]); - fs::write(fixture.repo.join("rust-tops.lock"), "[tools\n").expect("write lock"); - let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); - assert_eq!(out.status.code(), Some(0), "{text}"); - assert!(!text.contains("rust-tops.lock"), "{text}"); -} #[expect(clippy::too_many_lines, reason = "over the kit line threshold")] #[test] fn lock_honesty_holds_a_rust_tops_lock_to_head_and_a_missing_one_is_honest() { diff --git a/crates/cargo-tops/tests/lock_verify.rs b/crates/cargo-tops/tests/lock_verify.rs new file mode 100644 index 0000000..c8a6ae5 --- /dev/null +++ b/crates/cargo-tops/tests/lock_verify.rs @@ -0,0 +1,322 @@ +//! `cargo tops lock --verify` and `--write`, against shims on `PATH`. +//! +//! Each test makes its own scratch directory under the system temp dir and +//! removes only that. The `PATH` of the command under test is the shim +//! directory alone, so a real tool on the host never answers for a missing +//! one. The `cargo` shim hands `cargo …` to a `cargo-` shim in the +//! same directory, as the real cargo does. +use std::collections::BTreeMap; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct Scratch { + root: PathBuf, +} + +impl Scratch { + fn new(name: &str) -> Self { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let root = std::env::temp_dir().join(format!( + "cargo-tops-lock-{name}-{}-{}-{nanos}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + // A fresh name per test and per process: create_dir_all on the new + // path, and Drop removes only that path. + fs::create_dir_all(root.join("bin")).expect("mkdir bin"); + fs::create_dir_all(root.join("repo")).expect("mkdir repo"); + let scratch = Scratch { root }; + scratch.script( + "cargo", + "#!/bin/sh\nsub=\"$1\"; shift\nexec \"${0%/*}/cargo-$sub\" \"$@\"\n", + ); + scratch + } + + fn bin(&self) -> PathBuf { + self.root.join("bin") + } + + fn repo(&self) -> PathBuf { + self.root.join("repo") + } + + fn script(&self, name: &str, body: &str) { + let path = self.bin().join(name); + fs::write(&path, body).expect("write shim"); + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod shim"); + } + + /// A tool that prints `line` and, for a nextest-shaped shim, more lines. + fn tool(&self, name: &str, line: &str) { + self.script(name, &format!("#!/bin/sh\nprintf '%s\\n' '{line}'\n")); + } + + fn lock(&self, text: &str) { + fs::write(self.repo().join("rust-tops.lock"), text).expect("write lock"); + } + + fn read_lock(&self) -> String { + fs::read_to_string(self.repo().join("rust-tops.lock")).expect("read lock") + } + + fn run(&self, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .arg("lock") + .args(args) + .arg("--path") + .arg(self.repo()) + .env("PATH", self.bin()) + .output() + .expect("spawn cargo-tops lock") + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn text(out: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +/// A shim for every tool in the lock's `[tools]` that prints a line in the +/// tool's own shape. `fuzz` and `mutants` keep the pin as the version, with a +/// patch part added when the pin has two parts, as the real tool prints. +fn shim_all(scratch: &Scratch, pins: &BTreeMap) { + let full = |pin: &str| { + if pin.matches('.').count() == 1 { + format!("{pin}.0") + } else { + pin.to_string() + } + }; + let lines: [(&str, &str, &str); 8] = [ + ("rustc", "rustc", "rustc {} (58cf8f9c3 2026-10-13)"), + ( + "cargo-nextest", + "cargo-nextest", + "cargo-nextest {} (00af4550e 2026-09-16)", + ), + ("cargo-llvm-cov", "cargo-llvm-cov", "cargo-llvm-cov {}"), + ("cargo-mutants", "cargo-mutants", "cargo-mutants {}"), + ("cargo-crap", "cargo-crap", "cargo-crap {}"), + ("cargo-deny", "cargo-deny", "cargo-deny {}"), + ("cargo-fuzz", "cargo-fuzz", "cargo-fuzz {}"), + ("nushell", "nu", "{}"), + ]; + for (key, shim, shape) in lines { + if let Some(pin) = pins.get(key) { + scratch.tool(shim, &shape.replace("{}", &full(pin))); + } + } + if let Some(pin) = pins.get("kani-verifier") { + scratch.tool( + "cargo-kani", + &format!("Kani Rust Verifier {} (cargo plugin)", full(pin)), + ); + } + if pins.contains_key("clippy") { + scratch.tool("cargo-clippy", "clippy 0.1.99 (58cf8f9c35 2026-10-13)"); + } + if pins.contains_key("rustfmt") { + scratch.tool("cargo-fmt", "rustfmt 1.10.0-stable (58cf8f9c35 2026-10-13)"); + } +} + +fn pins_of(lock: &str) -> BTreeMap { + let table: toml::Table = lock.parse().expect("lock is toml"); + table + .get("tools") + .and_then(toml::Value::as_table) + .expect("[tools]") + .iter() + .filter_map(|(key, value)| Some((key.clone(), value.as_str()?.to_string()))) + .collect() +} + +fn init(scratch: &Scratch, class: &str) { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["init", "--class", class, "--name", "fixture", "--path"]) + .arg(scratch.repo()) + .output() + .expect("spawn cargo-tops init"); + assert_eq!(out.status.code(), Some(0), "{class}: {}", text(&out)); +} + +#[test] +fn a_nextest_that_prints_another_version_exits_3_and_names_tool_pin_and_found() { + let scratch = Scratch::new("nextest"); + scratch.lock("[tools]\ncargo-nextest = \"0.9.145\"\n"); + scratch.tool( + "cargo-nextest", + "cargo-nextest 0.9.144 (00af4550e 2026-09-16)", + ); + let out = scratch.run(&["--verify"]); + let said = text(&out); + assert_eq!(out.status.code(), Some(3), "{said}"); + assert!( + said.contains("mismatch cargo-nextest: pin 0.9.145, found 0.9.144"), + "{said}" + ); +} + +#[test] +fn a_partial_pin_matches_whole_parts_only() { + let scratch = Scratch::new("partial"); + scratch.lock("[tools]\ncargo-fuzz = \"0.12\"\n"); + scratch.tool("cargo-fuzz", "cargo-fuzz 0.120.0"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(3), "{}", text(&out)); + assert!(text(&out).contains("found 0.120.0"), "{}", text(&out)); + scratch.tool("cargo-fuzz", "cargo-fuzz 0.12.3"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); +} + +#[test] +fn a_pinned_tool_that_is_not_on_path_is_missing_and_exits_3() { + let scratch = Scratch::new("missing"); + scratch.lock("[tools]\ncargo-crap = \"0.9.0\"\n"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(3), "{}", text(&out)); + assert!( + text(&out).contains("missing cargo-crap: pin 0.9.0, not found"), + "{}", + text(&out) + ); +} + +#[test] +fn library_and_name_rows_are_skipped_and_the_run_passes() { + let scratch = Scratch::new("skipped"); + scratch.lock( + "[tools]\nproptest = \"1\"\ncriterion = \"0.8\"\ncallgrind-runner = \"gungraun-runner\"\n", + ); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + assert!(text(&out).contains("skipped proptest"), "{}", text(&out)); +} + +#[test] +fn a_missing_or_unparsable_lock_exits_2() { + let scratch = Scratch::new("unreadable"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(2), "no lock: {}", text(&out)); + scratch.lock("[tools\nrustc = "); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(2), "bad toml: {}", text(&out)); +} + +#[test] +fn write_replaces_values_and_keeps_comments_order_and_other_tables() { + let scratch = Scratch::new("write"); + let before = "# head comment\n[tools]\nrustc = \"1.99.1\" # the toolchain\n# nextest note\ncargo-nextest = \"0.9.145\"\ncargo-crap = \"0.9.0\"\nproptest = \"1\"\n\n[notes]\nschema = \"1.3.0\"\n"; + scratch.lock(before); + scratch.tool("rustc", "rustc 1.99.2 (aaaaaaaaa 2026-11-01)"); + scratch.tool( + "cargo-nextest", + "cargo-nextest 0.9.150 (bbbbbbbbb 2026-11-02)", + ); + // cargo-crap is not on PATH: its value is left alone. + let out = scratch.run(&["--write"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + assert_eq!( + scratch.read_lock(), + "# head comment\n[tools]\nrustc = \"1.99.2\" # the toolchain\n# nextest note\ncargo-nextest = \"0.9.150\"\ncargo-crap = \"0.9.0\"\nproptest = \"1\"\n\n[notes]\nschema = \"1.3.0\"\n" + ); +} + +#[test] +fn verify_never_writes_the_lock() { + let scratch = Scratch::new("readonly"); + let before = "[tools]\ncargo-crap = \"0.9.0\"\n"; + scratch.lock(before); + scratch.tool("cargo-crap", "cargo-crap 0.9.1"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(3), "{}", text(&out)); + assert_eq!(scratch.read_lock(), before); +} + +#[test] +fn max_tier_asks_only_for_the_tools_of_layers_in_range() { + let scratch = Scratch::new("tier"); + init(&scratch, "library-core"); + let pins = pins_of(&scratch.read_lock()); + shim_all(&scratch, &pins); + fs::remove_file(scratch.bin().join("cargo-mutants")).expect("drop mutants"); + fs::remove_file(scratch.bin().join("cargo-fuzz")).expect("drop fuzz"); + // The mutation layer is tier 1 and fuzz is tier 3. + let tier0 = scratch.run(&["--verify", "--max-tier", "0"]); + assert_eq!(tier0.status.code(), Some(0), "{}", text(&tier0)); + let tier1 = scratch.run(&["--verify", "--max-tier", "1"]); + assert_eq!(tier1.status.code(), Some(3), "{}", text(&tier1)); + assert!( + text(&tier1).contains("missing cargo-mutants"), + "{}", + text(&tier1) + ); + assert!(!text(&tier1).contains("cargo-fuzz"), "{}", text(&tier1)); +} + +#[test] +fn the_lock_every_init_class_writes_passes_verify_against_shims_at_the_pins() { + // `init_golden` holds every class to these bytes, so this is the lock each + // of the eight classes gets. + let golden = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/golden/init/shared/rust-tops.lock"); + let lock_text = fs::read_to_string(golden).expect("the golden lock"); + let scratch = Scratch::new("golden-lock"); + scratch.lock(&lock_text); + shim_all(&scratch, &pins_of(&lock_text)); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + // One line per tool the lock names, none of them a mismatch. + let said = text(&out); + assert!( + !said.contains("mismatch") && !said.contains("missing"), + "{said}" + ); + for tool in [ + "rustc", + "cargo-nextest", + "cargo-llvm-cov", + "cargo-mutants", + "cargo-crap", + "cargo-deny", + "cargo-fuzz", + "kani-verifier", + "nushell", + ] { + assert!( + said.contains(&format!("ok {tool} ")), + "{tool}: {said}" + ); + } +} +#[test] +fn clippy_and_rustfmt_must_run_even_when_rustc_matches_the_pin() { + let scratch = Scratch::new("toolchain"); + scratch.lock("[tools]\nrustc = \"1.99.1\"\nclippy = \"1.99.1\"\nrustfmt = \"1.99.1\"\n"); + scratch.tool("rustc", "rustc 1.99.1 (58cf8f9c3 2026-10-13)"); + scratch.tool("cargo-fmt", "rustfmt 1.10.0-stable (58cf8f9c35 2026-10-13)"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(3), "{}", text(&out)); + assert!(text(&out).contains("missing clippy"), "{}", text(&out)); + scratch.tool("cargo-clippy", "clippy 0.1.99 (58cf8f9c35 2026-10-13)"); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); +} diff --git a/crates/cargo-tops/tests/support/mod.rs b/crates/cargo-tops/tests/support/mod.rs new file mode 100644 index 0000000..b4bb834 --- /dev/null +++ b/crates/cargo-tops/tests/support/mod.rs @@ -0,0 +1,17 @@ +//! Helpers the gate tests share. +use std::fs; +use std::path::Path; + +/// A `rustc` that prints the toolchain pin `init` writes into `rust-tops.lock`, +/// so a gate that asks the toolchain for its version never depends on which +/// compiler the machine running the test has. +pub(crate) fn stub_rustc(bin: &Path) { + let path = bin.join("rustc"); + fs::write(&path, "#!/bin/sh\necho 'rustc 1.99.1 (stub 2026-01-01)'\n") + .expect("write rustc stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod rustc stub"); + } +} diff --git a/schema/gate-report.schema.json b/schema/gate-report.schema.json index e65feb8..2b0217a 100644 --- a/schema/gate-report.schema.json +++ b/schema/gate-report.schema.json @@ -39,7 +39,7 @@ "enum": ["pass", "fail", "config-invalid", "environment"] }, "exit_code": { - "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable when a layer in range needs a pin (crap today); a command-line usage error is 2 too and writes no report; 3 the environment (a required tool or cargo metadata is missing, or report.json cannot be written).", + "description": "0 pass; 1 the gate ran and failed (a dry run included; an expired exemption is a failure, not a config error); 2 the config is invalid, or the request cannot be gated: rust-tops.yaml missing, unreadable or failing check, no enabled layer at or below --max-tier, or rust-tops.lock unreadable when a layer in range needs a pinned tool (every pinned tool a running layer needs reads the lock); a command-line usage error is 2 too and writes no report; 3 the environment (a required tool or cargo metadata is missing, a pinned tool prints another version than its pin in rust-tops.lock, or report.json cannot be written).", "type": "integer", "enum": [0, 1, 2, 3] }, @@ -78,6 +78,7 @@ "not-run", "vacuous", "tool-missing", + "tool-mismatch", "not-evaluated" ] }, From 0ba40e88d454792bbd3b24098ad7075bb841fefc Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 07:13:27 -0500 Subject: [PATCH 2/4] Read the lock strictly, write it safely, and keep pre-releases out of pins A pin that is not a string, a tools key that is not a table, a pin that is not dot-separated digits, and a lock with no tools table are errors naming the key (exit 2) in lock, check and the gate. A pre-release never matches a pin. An unknown key prints a warning, and only the three named libraries print skipped. A failed probe names its command and how it failed, in the same words on stdout, in the detail and in the problems, and each probe has ten seconds. lock --write rewrites only plain key = "value" lines under tools, refuses every other form by name without changing the file, reads the edited text back, prints wrote only for changed values, and leaves a partial pin that already matches. The protocol text follows what the gate asks. Co-Authored-By: Claude Sonnet 5.5 --- .config/nextest.toml | 10 + PROTOCOL.md | 18 +- crates/cargo-tops/src/gate/execute.rs | 112 +++--- crates/cargo-tops/src/gate/judge.rs | 27 +- crates/cargo-tops/src/lock.rs | 500 ++++++++++++++++--------- crates/cargo-tops/src/lock/probe.rs | 141 +++++++ crates/cargo-tops/src/lock/write.rs | 249 ++++++++++++ crates/cargo-tops/src/main.rs | 18 +- crates/cargo-tops/tests/gate_report.rs | 43 ++- crates/cargo-tops/tests/laws_gate.rs | 5 +- crates/cargo-tops/tests/lock_verify.rs | 234 +++++++++++- 11 files changed, 1094 insertions(+), 263 deletions(-) create mode 100644 crates/cargo-tops/src/lock/probe.rs create mode 100644 crates/cargo-tops/src/lock/write.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index ed829ad..c732786 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -56,3 +56,13 @@ slow-timeout = { period = "10s", terminate-after = 4 } [[profile.ci.overrides]] filter = "binary(gate_credit_nextest)" slow-timeout = { period = "10s", terminate-after = 6 } + +# These tests write fresh shell shims and run the `cargo-tops` binary against +# them, for the kit suite only; they match nothing in an adopter. A first run of +# each new script costs a fraction of a second on macOS, so alone the slowest +# ran in 1.9 s to 10 s. Under the full parallel suite at a load of 7 to 10 on +# the build machine, 3 of them timed out at 20 s in 4 runs. They get 60 s; the +# profile timeout is not raised. +[[profile.ci.overrides]] +filter = "binary(lock_verify)" +slow-timeout = { period = "10s", terminate-after = 6 } diff --git a/PROTOCOL.md b/PROTOCOL.md index 5e79ae4..dbd21c4 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -853,7 +853,9 @@ The template is [`config/drop-in/RUST-TOPS.md`](config/drop-in/RUST-TOPS.md). `c `rust-tops.lock` pins the tools the gate runs, under `[tools]`. A pin the gate does not check is only a comment, so the gate asks each tool for its version and compares the answer with the pin. -**What is asked.** One row per tool. The first line of the output is read, and the version is the first word after the tool's name, cut at the first character that is not a digit or a dot. +**The lock.** A pin is a string of dot-separated digits, such as `0.9.0` or `0.9`. A `[tools]` value that is not a string, a `tools` that is not a table, and a pin of a tool that is not dot-separated digits are errors that name the key: exit 2 in `cargo tops lock`, in `cargo tops check` and in the gate. `cargo tops lock` on a lock with no `[tools]` table is exit 2 too. A lock file that is not there pins nothing. + +**What is asked.** One row per tool. The first line of the output is read, and the version is the first word after the tool's name. Each ask has 10 seconds. | Lock key | Asked | Line starts with | |---|---|---| @@ -868,21 +870,21 @@ The template is [`config/drop-in/RUST-TOPS.md`](config/drop-in/RUST-TOPS.md). `c | `kani-verifier` | `cargo kani --version` | `Kani Rust Verifier ` | | `nushell` | `nu --version` | (the version alone) | -Keys that name a library or a runner (`proptest`, `criterion`, `callgrind-runner`) are not tools on `PATH`. They are skipped. +Keys that name a library or a runner (`proptest`, `criterion`, `callgrind-runner`) are not tools on `PATH`: they print `skipped`. Any other key is not a tool this protocol knows (a misspelt one, say): `lock --verify` prints `unknown` and does not check it, and the exit code is not changed. -**Matching.** A pin matches when its dot-separated parts are the first parts of the version found. `0.12` matches `0.12.0` and `0.12.3`, and does not match `0.120.0`. `0.12.3` does not match `0.12`. A line that does not have the shape above is a mismatch, and the line is named as found. +**Matching.** The digits and dots at the front of the first word after the tool's name are the version, and a `+build` part after them is cut. A pin matches when its dot-separated parts are the first parts of the version found: `0.12` matches `0.12.0` and `0.12.3`, and does not match `0.120.0`; `0.12.3` does not match `0.12`. A pre-release (`0.9.0-rc.1`, `1.99.1-nightly`) matches no pin, and the whole word is named as found. A line that does not have the shape above is a mismatch and is named as found; empty output reads `no version line`. A command that cannot be run, does not exit 0, or runs past the limit is a missing tool, and the line names the command and how it failed (`exited 101`, `timed out`). **In the gate.** -- Before the commands of a layer that runs, the gate asks each pinned tool that layer needs, once per gate. The layers and tools are: `static-lint` (`rustc`, `clippy`, `rustfmt`, `cargo-deny`), `unit`, `integration`, `property` and `acceptance` (`cargo-nextest`), `doc` (`rustc`), `coverage` (`cargo-llvm-cov`, `cargo-nextest`), `crap` (`cargo-crap`), `security-advisory` (`cargo-deny`), `mutation` (`cargo-mutants`), `fuzz` (`cargo-fuzz`) and `formal-kani` (`kani-verifier`). +- Before the commands of a layer that runs a command, the gate asks each pinned tool that layer needs, once per gate. The layers and tools are: `static-lint` (`rustc`, `clippy`, `rustfmt`, `cargo-deny`), `unit`, `integration`, `property` and `acceptance` (`cargo-nextest`), `doc` (`rustc`), `coverage` (`cargo-llvm-cov`, `cargo-nextest`), `crap` (`cargo-crap`), `security-advisory` (`cargo-deny`), `mutation` (`cargo-mutants`), `fuzz` (`cargo-fuzz`) and `formal-kani` (`kani-verifier`). `fuzz` and `formal-kani` have no gate command in this release, so the gate does not ask for `cargo-fuzz` or `kani-verifier`. - A tool that prints another version fails its layer with the verdict `tool-mismatch`. The layer's `detail`, its problem and the line on stdout carry the same words: `mismatch : pin , found `. The kind is `environment` and the exit code is 3. None of the layer's commands run. -- A tool that cannot be run is the `tool-missing` outcome, as before. +- A tool that cannot be run is the `tool-missing` verdict, in the same words in the same three places: `missing : pin , `` `. - A layer that takes another layer's run takes its verdict, `tool-mismatch` included. - A layer above `--max-tier`, or one that is off, asks nothing. A tool with no pin in the lock is not asked. With no lock file the gate asks nothing. -- A `rust-tops.lock` that cannot be read or parsed, when a layer in range needs a pinned tool, is a config error: exit 2. +- A `rust-tops.lock` that cannot be read or parsed, or that holds a pin that cannot be read, when a layer in range needs a pinned tool, is a config error: exit 2. -**`cargo tops lock --verify [--max-tier N] [--path P]`.** Asks every tool in the lock, or, with `--max-tier`, only the tools that the enabled layers at or below `N` need, and prints one line per tool: `ok`, `mismatch`, `missing` or `skipped`. Exit 0 when none is a mismatch or missing, 2 when the lock cannot be read (or the config cannot be loaded for `--max-tier`), 3 otherwise. It never writes a file. +**`cargo tops lock --verify [--max-tier N] [--path P]`.** Asks every tool in the lock, or, with `--max-tier`, only the tools that the enabled layers at or below `N` need (this includes `cargo-fuzz` from tier 3 and `kani-verifier` from tier 4, which the gate does not ask), and prints one line per tool: `ok`, `mismatch`, `missing`, `skipped` or `unknown`. A lock with an empty `[tools]` prints `nothing pinned in [tools]`. Exit 0 when none is a mismatch or missing, 2 when the lock cannot be read, has no `[tools]` table or holds a pin that cannot be read (or the config cannot be loaded for `--max-tier`), 3 otherwise. It never writes a file. -**`cargo tops lock --write [--path P]`.** Replaces each `[tools]` value with the version the tool prints now. Comments, order, spacing and other tables are kept. A tool that cannot be run keeps its value. +**`cargo tops lock --write [--path P]`.** Replaces the value of each pinned tool whose pin does not match what the tool prints now, with that release version; a partial pin that already matches is left alone, and a tool that cannot be run or prints a pre-release keeps its value. Only a plain `key = "value"` line under `[tools]` is rewritten, and comments, order, spacing and other tables are kept. Any other form of a key to be written (a quoted or dotted key, an inline table, a literal or multi-line string, a header written `[ tools ]`) is exit 2 naming the key, and the lock is not changed. The edited lock is read back before it is written. It prints `wrote` only for a value it changed. The reference workflow runs `cargo tops lock --verify` after the tools are installed and before the gate (section 17). diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index 07d4cb4..cd2d97e 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -13,7 +13,7 @@ use super::{Fault, Mode}; use crate::config::Layer; use crate::evidence; use crate::facts::{Outcome, Run}; -use crate::lock::{self, Line, Verdict}; +use crate::lock::{self, Line}; use crate::mutation::{self, Outcomes}; use crate::steps::{self, Base, Kind, MutationPlan, Step}; use crate::{laws, TOOL_LOCK}; @@ -83,43 +83,36 @@ const CARGO_BUILTINS: [&str; 6] = ["test", "build", "check", "doc", "run", "benc /// The first line ` --version` prints, or `None` when the tool does /// not answer. A `cargo` subcommand asks itself; a built-in asks `cargo`. -fn version_of( - dir: &Path, - argv: &[String], - cache: &mut HashMap>, -) -> Option { +fn version_of(dir: &Path, argv: &[String], cache: &mut Answers) -> Option { let (program, rest) = argv.split_first()?; ask_cached( dir, &version_ask(program, rest.first().map(String::as_str)), cache, ) + .ok() + .filter(|line| !line.is_empty()) } -/// The first line `asks` prints, or `None` when it cannot run or fails. Each -/// distinct argv is run once per gate, so a pinned tool the gate checked and -/// the version the report names come from the same answer. -fn ask_cached( - dir: &Path, - asks: &[&str], - cache: &mut HashMap>, -) -> Option { +/// What each version command answered: its first non-empty line, or why it did +/// not answer. +type Answers = HashMap>; + +/// The first line `asks` prints (empty when it prints none), or why it did not +/// answer within the limit. Each distinct argv is run once per gate, so a +/// pinned tool the gate checked and the version the report names come from +/// the same answer. +fn ask_cached(dir: &Path, asks: &[&str], cache: &mut Answers) -> Result { cache .entry(asks.join(" ")) .or_insert_with(|| { - let (head, tail) = asks.split_first()?; - let out = Command::new(head) - .args(tail) - .current_dir(dir) - .output() - .ok()?; - out.status.success().then(|| { - String::from_utf8_lossy(&out.stdout) - .lines() + lock::run_probe(dir, asks, lock::PROBE_LIMIT).map(|out| { + out.lines() .map(str::trim) .find(|line| !line.is_empty()) - .map(str::to_string) - })? + .unwrap_or_default() + .to_string() + }) }) .clone() } @@ -182,7 +175,7 @@ struct Runner<'p, 'a> { /// `false` for a dry run: commands are listed, never run. live: bool, done: Executed<'a>, - versions: HashMap>, + versions: Answers, /// What each pinned tool answered, asked once per gate. asked: HashMap<&'static str, Line>, } @@ -232,32 +225,34 @@ impl<'a> Runner<'_, 'a> { } /// The run that stands for `layer` when a pinned tool it needs is at - /// another version ([`Outcome::ToolMismatch`], the note naming tool, pin - /// and found) or will not run ([`Outcome::ToolMissing`]); `None` when - /// every tool answers with its pin. Each tool is asked once per gate. + /// another version ([`Outcome::ToolMismatch`]) or will not answer + /// ([`Outcome::ToolMissing`]), the note carrying every such tool in the + /// words of [`lock::Line::text`]; `None` when every tool answers with its + /// pin. Each tool is asked once per gate. fn ask_pins(&mut self, layer: &Layer) -> Option { let dir = self.dir; - let mut mismatched = Vec::new(); - let mut missing: Option = None; + let mut stops = Vec::new(); for (row, pin) in self.plan.pins_of(&layer.id) { let line = self.asked.entry(row.key).or_insert_with(|| { lock::check_row(row, pin, &mut |argv| { ask_cached(dir, argv, &mut self.versions) }) }); - match &line.verdict { - Verdict::Mismatch { .. } => mismatched.push((row.argv.join(" "), line.text())), - Verdict::Missing { asked } => { - missing.get_or_insert_with(|| asked.clone()); - } - Verdict::Ok { .. } | Verdict::Skipped { .. } => {} - } + stops.extend(line.stop()); } - if let Some((argv, _)) = mismatched.first() { - let notes: Vec<&str> = mismatched.iter().map(|(_, text)| text.as_str()).collect(); - return Some(mismatch_run(&layer.id, argv, notes.join("; "))); - } - missing.map(|asked| missing_run(&layer.id, asked)) + let first = stops.first()?; + let note: Vec<&str> = stops.iter().map(|stop| stop.text.as_str()).collect(); + let outcome = if stops.iter().any(|stop| stop.mismatch) { + Outcome::ToolMismatch + } else { + Outcome::ToolMissing + }; + Some(pin_stop_run( + &layer.id, + outcome, + &first.asked, + note.join("; "), + )) } fn shared(&mut self, layer: &'a Layer, from: &'static str) { @@ -457,33 +452,24 @@ pub(crate) fn run_chain(dir: &Path, layer: &str, chain: &[Step]) -> Result } /// A layer that stopped because a pinned tool prints another version than its -/// pin. The same `note` is the stdout line, the layer's detail and its problem. -fn mismatch_run(layer: &str, argv: &str, note: String) -> Run { - println!("tool-mismatch [{layer}]: {note}"); +/// pin (`outcome` is a mismatch) or would not answer (a missing tool). The same `note` is the stdout line, +/// the layer's detail and its problem. +fn pin_stop_run(layer: &str, outcome: Outcome, asked: &str, note: String) -> Run { + let label = if outcome == Outcome::ToolMismatch { + "tool-mismatch" + } else { + "tool-missing" + }; + println!("{label} [{layer}]: {note}"); Run { layer: layer.to_string(), cmd: String::new(), - step: argv.to_string(), - outcome: Outcome::ToolMismatch, + step: asked.to_string(), + outcome, note: Some(note), } } -/// A layer that stopped because a pinned tool will not run: the same outcome -/// and words as a command whose tool is not on `PATH`. -fn missing_run(layer: &str, asked: String) -> Run { - let words: Vec = asked.split_whitespace().map(str::to_string).collect(); - let binary = steps::tool_binary(&words).unwrap_or_else(|| asked.clone()); - println!("{}", steps::MissingTool { binary }); - Run { - layer: layer.to_string(), - cmd: String::new(), - step: asked, - outcome: Outcome::ToolMissing, - note: None, - } -} - /// A mutation layer's run, and what its `outcomes.json` held. pub(crate) struct Mutated { pub(crate) run: Run, diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index c1ef03c..cd7fa4c 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -485,21 +485,26 @@ pub(crate) fn laws_evaluated(cases: &Path) -> Option<(usize, usize)> { pub(crate) fn run_failure(run: &Run) -> Option { match run.outcome { Outcome::Passed | Outcome::Vacuous => None, - // A tool at another version always carries its note: tool, pin, found. - Outcome::Failed | Outcome::ToolMismatch => Some(match &run.note { - Some(note) => format!("{}: {note}", run.layer), - None => format!("{}: `{}` failed", run.layer, run.step), - }), - Outcome::ToolMissing => { - let missing = steps::MissingTool { - binary: steps::tool_binary(std::slice::from_ref(&run.step)) - .unwrap_or_else(|| run.step.clone()), - }; - Some(format!("{}: {missing}", run.layer)) + Outcome::Failed | Outcome::ToolMismatch | Outcome::ToolMissing => { + Some(format!("{}: {}", run.layer, failure_words(run))) } } } +/// What failed, in the words the run kept (a note), or for a command whose +/// tool is not on `PATH` the way that is said, or the step that failed. +fn failure_words(run: &Run) -> String { + match (&run.note, run.outcome) { + (Some(note), _) => note.clone(), + (None, Outcome::ToolMissing) => steps::MissingTool { + binary: steps::tool_binary(std::slice::from_ref(&run.step)) + .unwrap_or_else(|| run.step.clone()), + } + .to_string(), + (None, _) => format!("`{}` failed", run.step), + } +} + #[cfg(test)] mod tests { use super::{clear_cases, crate_prefix, laws_verdict, Judgement}; diff --git a/crates/cargo-tops/src/lock.rs b/crates/cargo-tops/src/lock.rs index 881c0de..275514b 100644 --- a/crates/cargo-tops/src/lock.rs +++ b/crates/cargo-tops/src/lock.rs @@ -11,9 +11,14 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use std::process::{Command, ExitCode}; +use std::process::ExitCode; -use anyhow::{Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; + +mod probe; +mod write; + +pub(crate) use probe::{probe_in, run_probe, PROBE_LIMIT}; /// One pinned tool: its lock key, the argv that prints its version, and the /// text the first line starts with before the version. @@ -127,18 +132,28 @@ pub(crate) fn tools_for<'a>(layers: impl IntoIterator) -> Vec<&' } /// The version on the first line of a tool's output, read by the row's rule: -/// the first word after the prefix, cut at the first character that is not a -/// digit or a dot. `None` when the line does not have that shape. +/// the first word after the prefix. The digits and dots at its front are the +/// version; a `+build` part after them is cut. A word that goes on with +/// anything else (`-rc.1`, `-nightly`) is a pre-release, and the whole word is +/// the version, which no pin matches. `None` when the line does not have that +/// shape. pub(crate) fn parse_version(row: &Row, output: &str) -> Option { let line = output.lines().next()?.trim(); let word = line.strip_prefix(row.prefix)?.split_whitespace().next()?; - let version: String = word - .chars() - .take_while(|c| c.is_ascii_digit() || *c == '.') - .collect(); - let version = version.trim_end_matches('.'); - // Only digits and dots are left, so a malformed version has an empty part: - // `1..2`, `.5`, or nothing at all. + let (number, rest) = word.split_at( + word.find(|c: char| !(c.is_ascii_digit() || c == '.')) + .unwrap_or(word.len()), + ); + let number = number.trim_end_matches('.'); + if number.is_empty() { + return None; + } + let version = if rest.is_empty() || rest.starts_with('+') { + number + } else { + word + }; + // A malformed version has an empty part: `1..2`, `.5`, or nothing at all. version .split('.') .all(|part| !part.is_empty()) @@ -146,11 +161,19 @@ pub(crate) fn parse_version(row: &Row, output: &str) -> Option { } /// Whether `found` satisfies `pin`. Whole dot-separated parts only: `0.12` -/// matches `0.12.3` and not `0.120.0`; `0.12.3` does not match `0.12`. +/// matches `0.12.3` and not `0.120.0`; `0.12.3` does not match `0.12`. A +/// pre-release (`0.12.0-rc.1`) matches no pin. pub(crate) fn pin_matches(pin: &str, found: &str) -> bool { let pin_parts: Vec<&str> = pin.split('.').collect(); let found_parts: Vec<&str> = found.split('.').collect(); - found_parts.starts_with(&pin_parts) + !found.contains('-') && found_parts.starts_with(&pin_parts) +} + +/// Whether `text` is dot-separated digits, the only shape a pin or a version +/// written to the lock has. +pub(crate) fn is_dotted_digits(text: &str) -> bool { + text.split('.') + .all(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit())) } /// What asking one tool came to. @@ -163,74 +186,122 @@ pub(crate) enum Verdict { Mismatch { found: String, }, - /// The tool could not be run, or exited non-zero; `asked` is the command. + /// The tool could not be run, or did not exit 0 in time: `asked` is the + /// command and `why` how it failed. Missing { asked: String, + why: String, }, /// Nothing to ask: a library or name row, or a tool with no pin. Skipped { why: &'static str, }, + /// A lock key that names no tool rust-tops knows: not checked. + Unknown, } #[derive(Debug, PartialEq, Eq)] pub(crate) struct Line { pub(crate) key: String, pub(crate) pin: String, + /// The version command of the tool. + pub(crate) asked: String, pub(crate) verdict: Verdict, } +/// Why a layer cannot run on a pinned tool: the same words on stdout, in the +/// layer's `detail` and in its problems. +pub(crate) struct Stop { + /// `true` for another version, `false` for a tool that would not answer. + pub(crate) mismatch: bool, + /// The command that was asked. + pub(crate) asked: String, + pub(crate) text: String, +} + impl Line { pub(crate) fn passes(&self) -> bool { - matches!(self.verdict, Verdict::Ok { .. } | Verdict::Skipped { .. }) + matches!( + self.verdict, + Verdict::Ok { .. } | Verdict::Skipped { .. } | Verdict::Unknown + ) } - /// The line `--verify` prints, and the detail of a `tool-mismatch`. + /// The line `--verify` prints, and the detail of a stopped layer. pub(crate) fn text(&self) -> String { - let Line { key, pin, verdict } = self; + let Line { + key, pin, verdict, .. + } = self; match verdict { Verdict::Ok { found } => format!("ok {key} {found} (pin {pin})"), Verdict::Mismatch { found } => { format!("mismatch {key}: pin {pin}, found {found}") } - Verdict::Missing { .. } => format!("missing {key}: pin {pin}, not found"), + Verdict::Missing { asked, why } => { + format!("missing {key}: pin {pin}, `{asked}` {why}") + } Verdict::Skipped { why } => format!("skipped {key}: {why}"), + Verdict::Unknown => format!("unknown {key}: not a tool rust-tops knows, not checked"), } } + + /// What stops a layer: another version or a tool that would not answer. + pub(crate) fn stop(&self) -> Option { + let (mismatch, asked) = match &self.verdict { + Verdict::Mismatch { .. } => (true, self.asked.clone()), + Verdict::Missing { asked, .. } => (false, asked.clone()), + _ => return None, + }; + Some(Stop { + mismatch, + asked, + text: self.text(), + }) + } } -/// Reads `[tools]` from a lock text. Values that are not strings are dropped. -pub(crate) fn read_pins(text: &str) -> Result> { +/// Reads `[tools]` from a lock text: `None` when there is no such table. A +/// `tools` that is not a table, a value that is not a string, and a pin of a +/// tool that is not dotted digits are errors naming the key. +pub(crate) fn read_lock(text: &str) -> Result>> { let lock: toml::Table = toml::from_str(text).context("parse rust-tops.lock")?; - Ok(lock - .get("tools") - .and_then(toml::Value::as_table) - .map(|tools| { - tools - .iter() - .filter_map(|(key, value)| Some((key.clone(), value.as_str()?.to_string()))) - .collect() - }) - .unwrap_or_default()) + let Some(tools) = lock.get("tools") else { + return Ok(None); + }; + let tools = tools + .as_table() + .ok_or_else(|| anyhow!("[tools] must be a table, found {}", tools.type_str()))?; + tools + .iter() + .map(|(key, value)| Ok((key.clone(), pin_of(key, value)?))) + .collect::>() + .map(Some) } -/// A probe that runs each argv in `dir` and returns its stdout, or `None` -/// when it could not run or failed. -pub(crate) fn probe_in(dir: &Path) -> impl FnMut(&[&str]) -> Option + '_ { - move |argv| { - let (program, args) = argv.split_first()?; - let output = Command::new(program) - .args(args) - .current_dir(dir) - .output() - .ok()?; - output - .status - .success() - .then(|| String::from_utf8_lossy(&output.stdout).into_owned()) +/// The pin `value` is for `key`. +fn pin_of(key: &str, value: &toml::Value) -> Result { + let pin = value.as_str().ok_or_else(|| { + anyhow!( + "[tools] {key}: a pin is a string such as \"0.9.0\", found {}", + value.type_str() + ) + })?; + if row(key).is_some() && !is_dotted_digits(pin) { + bail!( + "[tools] {key} = {pin:?}: a pin is dot-separated digits such as \"0.9.0\" or \"0.9\"" + ); } + Ok(pin.to_string()) } +/// The pins of a lock text; no `[tools]` table is no pins. +pub(crate) fn read_pins(text: &str) -> Result> { + Ok(read_lock(text)?.unwrap_or_default()) +} + +/// What a probe returns: the stdout of the command, or why it did not answer. +type Probe<'a> = dyn FnMut(&[&str]) -> Result + 'a; + /// The row for a lock key. pub(crate) fn row(key: &str) -> Option<&'static Row> { ROWS.iter().find(|row| row.key == key) @@ -249,25 +320,25 @@ pub(crate) fn pins_for<'a>( } /// Asks one pinned tool. `probe` runs an argv and returns its stdout. -pub(crate) fn check_row( - row: &Row, - pin: &str, - probe: &mut dyn FnMut(&[&str]) -> Option, -) -> Line { +pub(crate) fn check_row(row: &Row, pin: &str, probe: &mut Probe<'_>) -> Line { let verdict = match probe(row.argv) { - None => Verdict::Missing { + Err(why) => Verdict::Missing { asked: row.argv.join(" "), + why, }, - Some(out) => match row.present.iter().find(|argv| probe(argv).is_none()) { - Some(argv) => Verdict::Missing { - asked: argv.join(" "), - }, + Ok(out) => match row + .present + .iter() + .find_map(|argv| probe(argv).err().map(|why| (argv.join(" "), why))) + { + Some((asked, why)) => Verdict::Missing { asked, why }, None => version_verdict(row, pin, &out), }, }; Line { key: row.key.to_string(), pin: pin.to_string(), + asked: row.argv.join(" "), verdict, } } @@ -276,8 +347,13 @@ pub(crate) fn check_row( /// the shape of the row is a mismatch, and the line is named as found. fn version_verdict(row: &Row, pin: &str, out: &str) -> Verdict { let Some(found) = parse_version(row, out) else { + let line = out.lines().next().unwrap_or_default().trim(); return Verdict::Mismatch { - found: out.lines().next().unwrap_or_default().trim().to_string(), + found: if line.is_empty() { + "no version line".to_string() + } else { + line.to_string() + }, }; }; if pin_matches(pin, &found) { @@ -289,11 +365,11 @@ fn version_verdict(row: &Row, pin: &str, out: &str) -> Verdict { /// Checks the pins. With `only`, the tools in it (a tool with no pin is /// skipped); without, every pin in the lock, and a lock key with no row is -/// skipped. +/// skipped (a library or a runner) or unknown (anything else). pub(crate) fn verify( pins: &BTreeMap, only: Option<&[&str]>, - probe: &mut dyn FnMut(&[&str]) -> Option, + probe: &mut Probe<'_>, ) -> Vec { let mut lines = tool_lines(pins, only, probe); if only.is_none() { @@ -306,7 +382,7 @@ pub(crate) fn verify( fn tool_lines( pins: &BTreeMap, only: Option<&[&str]>, - probe: &mut dyn FnMut(&[&str]) -> Option, + probe: &mut Probe<'_>, ) -> Vec { ROWS.iter() .filter(|row| only.is_none_or(|only| only.contains(&row.key))) @@ -315,6 +391,7 @@ fn tool_lines( None => only.map(|_| Line { key: row.key.to_string(), pin: String::new(), + asked: row.argv.join(" "), verdict: Verdict::Skipped { why: "no pin in rust-tops.lock", }, @@ -323,69 +400,50 @@ fn tool_lines( .collect() } -/// The lock keys that name no tool: a library or a runner. Skipped. +/// The keys that are pinned in the lock but are not in the table of tools: +/// the three libraries and runners that are not tools on `PATH` are skipped, +/// and any other key (a misspelt tool, say) is named as unknown. +const NOT_TOOLS: [&str; 3] = ["proptest", "criterion", "callgrind-runner"]; + fn unmatched_keys(pins: &BTreeMap) -> Vec { pins.iter() .filter(|(key, _)| row(key).is_none()) .map(|(key, pin)| Line { key: key.clone(), pin: pin.clone(), - verdict: Verdict::Skipped { - why: "a library or name, not a tool on PATH", + asked: String::new(), + verdict: if NOT_TOOLS.contains(&key.as_str()) { + Verdict::Skipped { + why: "a library or name, not a tool on PATH", + } + } else { + Verdict::Unknown }, }) .collect() } -/// The version each pinned tool prints now, for `--write`. A tool that is -/// missing or prints no version is left out. +/// The version each pinned tool prints now, for `--write`: the tools that +/// answered with a release version, as dot-separated digits. A tool that did +/// not answer, or printed a pre-release, is left out. pub(crate) fn found_versions( pins: &BTreeMap, - probe: &mut dyn FnMut(&[&str]) -> Option, + probe: &mut Probe<'_>, ) -> BTreeMap { ROWS.iter() .filter(|row| pins.contains_key(row.key)) - .filter_map(|row| { - let found = parse_version(row, &probe(row.argv)?)?; - Some((row.key.to_string(), found)) - }) + .filter_map(|row| Some((row.key.to_string(), release_version(row, probe)?))) .collect() } -/// `text` with the `[tools]` values in `found` replaced. Only the quoted value -/// changes; comments, order, spacing and other tables are kept byte for byte. -pub(crate) fn write_values(text: &str, found: &BTreeMap) -> String { - let mut in_tools = false; - let mut out = String::with_capacity(text.len()); - for line in text.split_inclusive('\n') { - let trimmed = line.trim_start(); - if trimmed.starts_with('[') { - in_tools = trimmed.trim_end().starts_with("[tools]"); - } - out.push_str(&if in_tools { - replace_value(line, found) - } else { - line.to_string() - }); +/// The release version `row`'s tool prints, when the tool and the commands that +/// show it is present all answer. +fn release_version(row: &Row, probe: &mut Probe<'_>) -> Option { + let out = probe(row.argv).ok()?; + if row.present.iter().any(|argv| probe(argv).is_err()) { + return None; } - out -} - -fn replace_value(line: &str, found: &BTreeMap) -> String { - let Some((key, rest)) = line.split_once('=') else { - return line.to_string(); - }; - let Some(new) = found.get(key.trim()) else { - return line.to_string(); - }; - let Some(open) = rest.find('"') else { - return line.to_string(); - }; - let Some(len) = rest[open + 1..].find('"') else { - return line.to_string(); - }; - let close = open + 1 + len; - format!("{key}={}{new}{}", &rest[..=open], &rest[close..]) + parse_version(row, &out).filter(|found| is_dotted_digits(found)) } /// What `cargo tops lock` was asked to do. @@ -397,16 +455,19 @@ pub(crate) enum Action { Write, } -/// `cargo tops lock`. An unreadable lock, or a config that will not load for +/// `cargo tops lock`. A lock that is unreadable, has no `[tools]` table or +/// holds a pin that cannot be read, or a config that will not load for /// `--max-tier`, is an error (exit 2); a missing or mismatched tool is exit 3. pub(crate) fn command(dir: &Path, action: &Action) -> anyhow::Result { let path = dir.join(crate::TOOL_LOCK); let text = std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; - let pins = read_pins(&text).with_context(|| format!("read {}", path.display()))?; + let pins = read_lock(&text) + .with_context(|| format!("read {}", path.display()))? + .ok_or_else(|| anyhow!("{} has no [tools] table", path.display()))?; match action { Action::Verify { max_tier } => verify_command(dir, &pins, *max_tier), - Action::Write => write_command(dir, &path, &text, &pins), + Action::Write => write::write_command(dir, &path, &text, &pins), } } @@ -428,6 +489,9 @@ fn verify_command( ) -> anyhow::Result { let needed = max_tier.map(|tier| tools_in_range(dir, tier)).transpose()?; let lines = verify(pins, needed.as_deref(), &mut probe_in(dir)); + if lines.is_empty() { + println!("nothing pinned in [tools]"); + } for line in &lines { println!("{}", line.text()); } @@ -438,36 +502,6 @@ fn verify_command( }) } -/// One line for each pinned tool `--write` is about: the old and new value -/// when they differ, and that a tool that could not be run keeps its value. -fn write_lines(pins: &BTreeMap, found: &BTreeMap) -> Vec { - pins.iter() - .filter(|(key, _)| row(key).is_some()) - .map(|(key, pin)| match found.get(key) { - Some(now) if now != pin => format!("wrote {key}: {pin} -> {now}"), - Some(_) => format!("kept {key} {pin}"), - None => format!("kept {key} {pin}: not found"), - }) - .collect() -} - -fn write_command( - dir: &Path, - path: &Path, - text: &str, - pins: &BTreeMap, -) -> anyhow::Result { - let found = found_versions(pins, &mut probe_in(dir)); - for line in write_lines(pins, &found) { - println!("{line}"); - } - let written = write_values(text, &found); - if written != text { - std::fs::write(path, written).with_context(|| format!("write {}", path.display()))?; - } - Ok(ExitCode::SUCCESS) -} - #[cfg(test)] mod tests { use super::*; @@ -555,7 +589,7 @@ mod tests { assert_eq!( texts, [ - "missing clippy: pin 1.99.1, not found", + "missing clippy: pin 1.99.1, `cargo clippy --version` exited 101", "ok rustfmt 1.99.1 (pin 1.99.1)", ] ); @@ -570,10 +604,14 @@ mod tests { ] { assert_eq!(parse_version(row("cargo-nextest"), out), None, "{out:?}"); } - // A suffix after the digits and dots is cut, and a trailing dot goes. + // A build part is cut, and a trailing dot goes; a pre-release keeps its + // whole word, which no pin matches. for (out, want) in [ - ("cargo-nextest 1.2-rc\n", "1.2"), + ("cargo-nextest 1.2+build.5\n", "1.2"), ("cargo-nextest 1.\n", "1"), + ("cargo-nextest 1.2-rc.1\n", "1.2-rc.1"), + ("cargo-nextest 1.2.3-nightly (x)\n", "1.2.3-nightly"), + ("cargo-nextest 1.2abc\n", "1.2abc"), ] { assert_eq!( parse_version(row("cargo-nextest"), out).as_deref(), @@ -583,28 +621,6 @@ mod tests { } } - #[test] - fn write_lines_say_what_changed_and_leave_library_rows_out() { - let pins = BTreeMap::from([ - ("cargo-crap".to_string(), "0.9.0".to_string()), - ("cargo-deny".to_string(), "0.18.9".to_string()), - ("cargo-nextest".to_string(), "0.9.145".to_string()), - ("proptest".to_string(), "1".to_string()), - ]); - let found = BTreeMap::from([ - ("cargo-deny".to_string(), "0.18.9".to_string()), - ("cargo-nextest".to_string(), "0.9.150".to_string()), - ]); - assert_eq!( - write_lines(&pins, &found), - [ - "kept cargo-crap 0.9.0: not found", - "kept cargo-deny 0.18.9", - "wrote cargo-nextest: 0.9.145 -> 0.9.150", - ] - ); - } - #[test] fn a_line_without_the_prefix_or_a_version_reads_as_none() { for out in [ @@ -638,16 +654,42 @@ mod tests { assert!(!pin_matches("0.12.3", "0.12")); assert!(!pin_matches("1", "10.0.0")); assert!(pin_matches("1", "1.4.2")); + // A pre-release matches no pin, however few parts the pin has. + for pin in ["0", "0.9", "0.9.0"] { + assert!(!pin_matches(pin, "0.9.0-rc.1"), "{pin}"); + assert!(!pin_matches(pin, "0.9.0-nightly"), "{pin}"); + } + } + + #[test] + fn dotted_digits_are_the_only_pins_and_versions_written() { + for good in ["1", "0.12", "0.9.145"] { + assert!(is_dotted_digits(good), "{good}"); + } + for bad in [ + "", + "1.", + ".1", + "1..2", + "v1", + "1.x", + "=1", + "1.0-rc", + "gungraun-runner", + ] { + assert!(!is_dotted_digits(bad), "{bad:?}"); + } } fn probe_for<'a>( table: &'a [(&'a str, Option<&'a str>)], - ) -> impl FnMut(&[&str]) -> Option + 'a { + ) -> impl FnMut(&[&str]) -> Result + 'a { move |argv| { table .iter() .find(|(tool, _)| argv.contains(tool)) .and_then(|(_, out)| out.map(str::to_string)) + .ok_or_else(|| "exited 101".to_string()) } } @@ -678,7 +720,7 @@ mod tests { assert_eq!( texts, [ - "missing cargo-crap: pin 0.9.0, not found", + "missing cargo-crap: pin 0.9.0, `cargo crap --version` exited 101", "mismatch cargo-deny: pin 0.18.9, found what", ] ); @@ -693,7 +735,7 @@ mod tests { "gungraun-runner".to_string(), ), ]); - let mut probe = |_: &[&str]| -> Option { panic!("asked a library row") }; + let mut probe = |_: &[&str]| -> Result { panic!("asked a library row") }; let lines = verify(&pins, None, &mut probe); assert_eq!(lines.len(), 2); assert!(lines @@ -719,26 +761,134 @@ mod tests { } #[test] - fn write_changes_only_the_values_it_found() { - let text = "# keep me\n[tools]\nrustc = \"1.99.1\" # trailing\n# cargo-nextest note\ncargo-nextest = \"0.9.145\"\nproptest = \"1\"\n\n[notes]\nrustc = \"untouched\"\n"; - let found = BTreeMap::from([ - ("cargo-nextest".to_string(), "0.9.150".to_string()), - ("rustc".to_string(), "1.99.2".to_string()), + fn read_lock_reads_strings_and_names_every_other_kind_of_value() { + let pins = read_pins("[tools]\nrustc = \"1.99.1\"\nproptest = \"1\"\ncallgrind-runner = \"gungraun-runner\"\n[notes]\nx = \"y\"\n").expect("pins"); + assert_eq!(pins.len(), 3); + assert_eq!( + pins.get("callgrind-runner").map(String::as_str), + Some("gungraun-runner") + ); + assert_eq!(read_lock("[notes]\nx = \"y\"\n").expect("no tools"), None); + assert!(read_pins("[notes]\n").expect("no pins").is_empty()); + for (lock, words) in [ + ("[tools\n", "parse"), + ( + "[tools]\nrustc = 1\n", + "[tools] rustc: a pin is a string such as \"0.9.0\", found integer", + ), + ("[tools]\nrustc = 1.5\n", "found float"), + ("[tools]\nrustc = { v = 1 }\n", "found table"), + ("[tools]\nrustc = [\"1\"]\n", "found array"), + ("[tools]\nrustc = true\n", "found boolean"), + ("tools = \"x\"\n", "[tools] must be a table, found string"), + ("[[tools]]\n", "[tools] must be a table, found array"), + ( + "[tools]\nrustc = \"=1\"\n", + "[tools] rustc = \"=1\": a pin is dot-separated digits", + ), + ] { + let err = read_lock(lock).expect_err(lock); + assert!(format!("{err:#}").contains(words), "{lock}: {err:#}"); + } + } + + #[test] + fn an_empty_or_garbled_line_is_a_mismatch_that_says_what_was_printed() { + let pins = BTreeMap::from([("cargo-crap".to_string(), "0.9.0".to_string())]); + for (out, found) in [ + ("", "no version line"), + ("\n", "no version line"), + ("what\n", "what"), + ] { + let mut probe = |_: &[&str]| Ok(out.to_string()); + let lines = verify(&pins, None, &mut probe); + assert_eq!( + lines.first().map(Line::text), + Some(format!("mismatch cargo-crap: pin 0.9.0, found {found}")), + "{out:?}" + ); + } + } + + #[test] + fn a_failed_probe_carries_its_command_and_reason_into_the_stop() { + let pins = BTreeMap::from([("cargo-crap".to_string(), "0.9.0".to_string())]); + let mut probe = |_: &[&str]| Err("timed out after 10s".to_string()); + let lines = verify(&pins, None, &mut probe); + let stop = lines.first().and_then(Line::stop).expect("a stop"); + assert!(!stop.mismatch); + assert_eq!(stop.asked, "cargo crap --version"); + assert_eq!( + stop.text, + "missing cargo-crap: pin 0.9.0, `cargo crap --version` timed out after 10s" + ); + let mut ok = |_: &[&str]| Ok("cargo-crap 0.9.1\n".to_string()); + let lines = verify(&pins, None, &mut ok); + let stop = lines.first().and_then(Line::stop).expect("a stop"); + assert!(stop.mismatch); + assert_eq!(stop.asked, "cargo crap --version"); + let mut fine = |_: &[&str]| Ok("cargo-crap 0.9.0\n".to_string()); + assert!(verify(&pins, None, &mut fine) + .first() + .and_then(Line::stop) + .is_none()); + } + + #[test] + fn a_failing_presence_command_names_that_command() { + let pins = BTreeMap::from([("rustfmt".to_string(), "1.99.1".to_string())]); + let mut probe = probe_for(&[("rustc", Some("rustc 1.99.1 (x)\n")), ("fmt", None)]); + let lines = verify(&pins, None, &mut probe); + assert_eq!( + lines.first().map(Line::text).as_deref(), + Some("missing rustfmt: pin 1.99.1, `cargo fmt --version` exited 101") + ); + } + + #[test] + fn unknown_keys_pass_with_a_warning_and_only_the_named_libraries_are_skipped() { + let pins = BTreeMap::from([ + ("criterion".to_string(), "0.8".to_string()), + ("cargo-nexttest".to_string(), "0.9.145".to_string()), ]); + let lines = verify( + &pins, + None, + &mut |_: &[&str]| Err("never asked".to_string()), + ); + let texts: Vec = lines.iter().map(Line::text).collect(); assert_eq!( - write_values(text, &found), - "# keep me\n[tools]\nrustc = \"1.99.2\" # trailing\n# cargo-nextest note\ncargo-nextest = \"0.9.150\"\nproptest = \"1\"\n\n[notes]\nrustc = \"untouched\"\n" + texts, + [ + "unknown cargo-nexttest: not a tool rust-tops knows, not checked", + "skipped criterion: a library or name, not a tool on PATH", + ] ); + assert!(lines.iter().all(Line::passes)); } #[test] - fn read_pins_reads_tools_only_and_rejects_bad_toml() { - let pins = - read_pins("[tools]\nrustc = \"1.99.1\"\nn = 3\n[notes]\nx = \"y\"\n").expect("pins"); + fn found_versions_leave_out_a_tool_that_will_not_answer_and_a_pre_release() { + let pins = BTreeMap::from([ + ("cargo-crap".to_string(), "0.9.0".to_string()), + ("cargo-deny".to_string(), "0.18.9".to_string()), + ("cargo-fuzz".to_string(), "0.12".to_string()), + ("clippy".to_string(), "1.99.1".to_string()), + ("rustc".to_string(), "1.99.1".to_string()), + ]); + let mut probe = probe_for(&[ + ("crap", None), + ("deny", Some("cargo-deny 0.18.9-rc.1\n")), + ("fuzz", Some("cargo-fuzz 0.12.3\n")), + ("clippy", None), + ("rustc", Some("rustc 1.99.2 (x)\n")), + ]); assert_eq!( - pins, - BTreeMap::from([("rustc".to_string(), "1.99.1".to_string())]) + found_versions(&pins, &mut probe), + BTreeMap::from([ + ("cargo-fuzz".to_string(), "0.12.3".to_string()), + ("rustc".to_string(), "1.99.2".to_string()), + ]) ); - assert!(read_pins("[tools\n").is_err()); } } diff --git a/crates/cargo-tops/src/lock/probe.rs b/crates/cargo-tops/src/lock/probe.rs new file mode 100644 index 0000000..bdf8f4c --- /dev/null +++ b/crates/cargo-tops/src/lock/probe.rs @@ -0,0 +1,141 @@ +//! Asking a tool for its version: one command, a time limit, and a reason in +//! words when it does not answer. +use std::io::Read; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +/// How long one `--version` ask may take. +pub(crate) const PROBE_LIMIT: Duration = Duration::from_secs(10); + +/// A probe that runs each argv in `dir` with the [`PROBE_LIMIT`]. +pub(crate) fn probe_in(dir: &Path) -> impl FnMut(&[&str]) -> Result + '_ { + move |argv| run_probe(dir, argv, PROBE_LIMIT) +} + +/// What `argv` printed on stdout when it exited 0 within `limit`; otherwise +/// the reason, as `could not run`, `exited N`, `was stopped by a signal` or +/// `timed out after `. +pub(crate) fn run_probe(dir: &Path, argv: &[&str], limit: Duration) -> Result { + let (program, args) = argv.split_first().ok_or("no command")?; + let mut child = Command::new(program) + .args(args) + .current_dir(dir) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|_| "could not run".to_string())?; + let reader = child.stdout.take().map(|mut out| { + std::thread::spawn(move || { + let mut bytes = Vec::new(); + out.read_to_end(&mut bytes).ok(); + String::from_utf8_lossy(&bytes).into_owned() + }) + }); + wait_within(&mut child, limit)?; + Ok(reader + .and_then(|reader| reader.join().ok()) + .unwrap_or_default()) +} + +/// Waits for `child` to exit 0, or kills it when `limit` has passed. +fn wait_within(child: &mut Child, limit: Duration) -> Result<(), String> { + let deadline = Instant::now() + limit; + loop { + let status = child + .try_wait() + .map_err(|_| "could not wait for it".to_string())?; + match status { + Some(status) => return finished(status), + None if Instant::now() >= deadline => return give_up(child, limit), + None => std::thread::sleep(Duration::from_millis(10)), + } + } +} + +fn finished(status: ExitStatus) -> Result<(), String> { + if status.success() { + Ok(()) + } else { + Err(exit_words(status)) + } +} + +/// Kills `child`, which has run past `limit`, and says so. +fn give_up(child: &mut Child, limit: Duration) -> Result<(), String> { + child.kill().ok(); + child.wait().ok(); + Err(format!("timed out after {limit:?}")) +} + +fn exit_words(status: ExitStatus) -> String { + status.code().map_or_else( + || "was stopped by a signal".to_string(), + |code| format!("exited {code}"), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn here() -> std::path::PathBuf { + std::env::current_dir().expect("cwd") + } + + #[test] + fn a_command_that_exits_0_gives_its_stdout() { + let out = run_probe(&here(), &["sh", "-c", "echo hi; echo err >&2"], PROBE_LIMIT); + assert_eq!(out.as_deref(), Ok("hi\n")); + } + + #[test] + fn a_command_that_exits_non_zero_says_which_code() { + let out = run_probe(&here(), &["sh", "-c", "echo hi; exit 101"], PROBE_LIMIT); + assert_eq!(out, Err("exited 101".to_string())); + } + + #[test] + fn a_command_that_does_not_exist_could_not_run() { + let out = run_probe(&here(), &["a-tool-nobody-has-installed"], PROBE_LIMIT); + assert_eq!(out, Err("could not run".to_string())); + assert_eq!( + run_probe(&here(), &[], PROBE_LIMIT), + Err("no command".to_string()) + ); + } + + #[test] + fn a_command_that_runs_past_the_limit_is_killed_and_says_so() { + let began = Instant::now(); + let out = run_probe( + &here(), + &["sh", "-c", "sleep 30"], + Duration::from_millis(200), + ); + assert_eq!(out, Err("timed out after 200ms".to_string())); + assert!(began.elapsed() < Duration::from_secs(10)); + } + + #[test] + fn a_command_killed_by_a_signal_says_so() { + let out = run_probe(&here(), &["sh", "-c", "kill -9 $$"], PROBE_LIMIT); + assert_eq!(out, Err("was stopped by a signal".to_string())); + } + + #[test] + fn the_limit_is_ten_seconds() { + assert_eq!(PROBE_LIMIT, Duration::from_secs(10)); + } + + #[test] + fn the_probe_runs_in_the_directory_it_is_given() { + let dir = std::env::temp_dir(); + let out = probe_in(&dir)(&["pwd"]).expect("pwd"); + assert_eq!( + std::path::Path::new(out.trim()).canonicalize().ok(), + dir.canonicalize().ok() + ); + } +} diff --git a/crates/cargo-tops/src/lock/write.rs b/crates/cargo-tops/src/lock/write.rs new file mode 100644 index 0000000..153b3dd --- /dev/null +++ b/crates/cargo-tops/src/lock/write.rs @@ -0,0 +1,249 @@ +//! `cargo tops lock --write`: rewrite the `[tools]` values the tools print +//! now, and nothing else. +//! +//! Only a plain `key = "value"` line under a `[tools]` header is rewritten, so +//! comments, order and spacing are kept byte for byte. Any other form of the +//! same key (quoted or dotted key, inline table, literal or multi-line +//! string) is refused, and the lock is left as it was. The edited text is read +//! back before it is written. +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::process::ExitCode; + +use anyhow::{bail, Context, Result}; + +use super::probe::probe_in; +use super::{found_versions, pin_matches, read_pins, row}; + +/// `cargo tops lock --write` on the lock at `path`, read as `text`. +pub(crate) fn write_command( + dir: &Path, + path: &Path, + text: &str, + pins: &BTreeMap, +) -> Result { + let found = found_versions(pins, &mut probe_in(dir)); + let changes = changes(pins, &found); + let written = write_values(text, &changes)?; + check_read_back(&written, &changes)?; + if written != text { + std::fs::write(path, written).with_context(|| format!("write {}", path.display()))?; + } + for line in write_lines(pins, &found) { + println!("{line}"); + } + Ok(ExitCode::SUCCESS) +} + +/// The values to write: each found version that its pin does not already +/// match. A partial pin such as `0.12` that matches `0.12.3` is left alone. +fn changes( + pins: &BTreeMap, + found: &BTreeMap, +) -> BTreeMap { + found + .iter() + .filter(|(key, now)| pins.get(*key).is_some_and(|pin| !pin_matches(pin, now))) + .map(|(key, now)| (key.clone(), now.clone())) + .collect() +} + +/// One line for each pinned tool `--write` is about, true of the edit: the old +/// and new value for a changed one, and that a tool with no version to write +/// keeps its value. +pub(crate) fn write_lines( + pins: &BTreeMap, + found: &BTreeMap, +) -> Vec { + pins.iter() + .filter(|(key, _)| row(key).is_some()) + .map(|(key, pin)| match found.get(key) { + Some(now) if !pin_matches(pin, now) => format!("wrote {key}: {pin} -> {now}"), + Some(_) => format!("kept {key} {pin}"), + None => format!("kept {key} {pin}: no version to write"), + }) + .collect() +} + +/// `text` with the `[tools]` value of each key in `changes` replaced, or an +/// error naming the keys that are not on a plain `key = "value"` line. +pub(crate) fn write_values(text: &str, changes: &BTreeMap) -> Result { + let mut pending: BTreeSet<&str> = changes.keys().map(String::as_str).collect(); + let mut in_tools = false; + let mut out = String::with_capacity(text.len()); + for line in text.split_inclusive('\n') { + let trimmed = line.trim_start(); + if trimmed.starts_with('[') { + in_tools = is_tools_header(trimmed); + } + out.push_str(&if in_tools { + rewritten(line, changes, &mut pending) + } else { + line.to_string() + }); + } + if pending.is_empty() { + return Ok(out); + } + let keys: Vec<&str> = pending.into_iter().collect(); + bail!( + "not written, the lock is unchanged: {} not on a plain `key = \"value\"` line under [tools]", + keys.join(", ") + ) +} + +/// `line` of the `[tools]` table, with its value replaced when it is the plain +/// line of a key in `changes`; the key is then no longer pending. +fn rewritten( + line: &str, + changes: &BTreeMap, + pending: &mut BTreeSet<&str>, +) -> String { + for (key, new) in changes { + if let Some(replaced) = replace_plain(line, key, new) { + pending.remove(key.as_str()); + return replaced; + } + } + line.to_string() +} + +/// Whether a header line (already trimmed at the front) is `[tools]`, with an +/// optional comment after it. +fn is_tools_header(trimmed: &str) -> bool { + trimmed.strip_prefix("[tools]").is_some_and(|rest| { + let rest = rest.trim(); + rest.is_empty() || rest.starts_with('#') + }) +} + +/// The line with `key`'s value replaced by `new`, when the line is exactly +/// `key = "basic string"` with an optional comment, in any spacing. +fn replace_plain(line: &str, key: &str, new: &str) -> Option { + let (body, eol) = line.split_at(line.trim_end_matches(['\r', '\n']).len()); + let (indent, rest) = body.split_at(body.len() - body.trim_start().len()); + let after_key = rest.strip_prefix(key)?; + let quote = after_key.find('"')?; + let (between, value_and_tail) = after_key.split_at(quote); + if between.trim() != "=" { + return None; + } + let value_and_tail = value_and_tail.strip_prefix('"')?; + let (value, tail) = value_and_tail.split_at(value_and_tail.find('"')?); + let tail = tail.strip_prefix('"')?; + let plain_tail = tail.trim().is_empty() || tail.trim_start().starts_with('#'); + (plain_tail && !value.contains('\\')) + .then(|| format!("{indent}{key}{between}\"{new}\"{tail}{eol}")) +} + +/// The edited lock must read back with every changed pin at its new value. +fn check_read_back(written: &str, changes: &BTreeMap) -> Result<()> { + let pins = read_pins(written).context("the edited lock does not read back")?; + for (key, new) in changes { + if pins.get(key) != Some(new) { + bail!("the edited lock does not read back with {key} = \"{new}\"; nothing was written"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(pairs: &[(&str, &str)]) -> BTreeMap { + pairs + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect() + } + + #[test] + fn a_plain_line_changes_only_its_value() { + let text = "# keep me\n[tools]\nrustc = \"1.99.1\" # the toolchain\n# nextest note\ncargo-nextest=\"0.9.145\"#c\r\ncargo-crap = \"0.9.0\"\nproptest = \"1\"\n\n[notes]\nrustc = \"untouched\"\n"; + let changes = map(&[("cargo-nextest", "0.9.150"), ("rustc", "1.99.2")]); + assert_eq!( + write_values(text, &changes).expect("written"), + "# keep me\n[tools]\nrustc = \"1.99.2\" # the toolchain\n# nextest note\ncargo-nextest=\"0.9.150\"#c\r\ncargo-crap = \"0.9.0\"\nproptest = \"1\"\n\n[notes]\nrustc = \"untouched\"\n" + ); + } + + #[test] + fn a_header_with_a_comment_is_the_tools_table_and_a_sub_table_is_not() { + let text = "[tools] # pins\nrustc = \"1.99.1\"\n[tools.extra]\ncargo-crap = \"0.9.0\"\n"; + let out = write_values(text, &map(&[("rustc", "1.99.2")])).expect("written"); + assert_eq!( + out, + "[tools] # pins\nrustc = \"1.99.2\"\n[tools.extra]\ncargo-crap = \"0.9.0\"\n" + ); + assert!(write_values(text, &map(&[("cargo-crap", "0.9.1")])).is_err()); + } + + #[test] + fn every_other_form_of_the_key_is_refused_by_name() { + for line in [ + "\"cargo-crap\" = \"0.9.0\"", + "cargo-crap = '0.9.0'", + "cargo-crap = \"\"\"0.9.0\"\"\"", + "cargo-crap = \"0.9.0\" extra", + "cargo-crap = { version = \"0.9.0\" }", + "cargo-crap = \"0\\u002e9.0\"", + "cargo-crap-x = \"0.9.0\"", + ] { + let text = format!("[tools]\n{line}\n"); + let err = write_values(&text, &map(&[("cargo-crap", "0.9.1")])).expect_err(line); + assert!(err.to_string().contains("cargo-crap"), "{line}: {err}"); + assert!(err.to_string().contains("unchanged"), "{line}: {err}"); + } + } + + #[test] + fn a_key_outside_the_tools_table_is_not_rewritten() { + let text = "[notes]\ncargo-crap = \"0.9.0\"\n"; + assert!(write_values(text, &map(&[("cargo-crap", "0.9.1")])).is_err()); + assert_eq!( + write_values(text, &BTreeMap::new()).expect("nothing to do"), + text + ); + } + + #[test] + fn only_a_changed_value_is_a_wrote_line() { + let pins = map(&[ + ("cargo-crap", "0.9.0"), + ("cargo-deny", "0.18.9"), + ("cargo-fuzz", "0.12"), + ("cargo-nextest", "0.9.145"), + ("proptest", "1"), + ]); + let found = map(&[ + ("cargo-deny", "0.18.9"), + ("cargo-fuzz", "0.12.3"), + ("cargo-nextest", "0.9.150"), + ]); + assert_eq!( + write_lines(&pins, &found), + [ + "kept cargo-crap 0.9.0: no version to write", + "kept cargo-deny 0.18.9", + "kept cargo-fuzz 0.12", + "wrote cargo-nextest: 0.9.145 -> 0.9.150", + ] + ); + } + + #[test] + fn changes_leave_out_a_pin_that_already_matches() { + let pins = map(&[("cargo-fuzz", "0.12"), ("cargo-crap", "0.9.0")]); + let found = map(&[("cargo-fuzz", "0.12.3"), ("cargo-crap", "0.9.5")]); + assert_eq!(changes(&pins, &found), map(&[("cargo-crap", "0.9.5")])); + } + + #[test] + fn the_edit_must_read_back_with_every_new_value() { + let changes = map(&[("cargo-crap", "0.9.5")]); + assert!(check_read_back("[tools]\ncargo-crap = \"0.9.5\"\n", &changes).is_ok()); + assert!(check_read_back("[tools]\ncargo-crap = \"0.9.0\"\n", &changes).is_err()); + assert!(check_read_back("[tools\n", &changes).is_err()); + } +} diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index d25da4c..6f47b2f 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -110,8 +110,9 @@ impl LockArgs { } } -/// The commands that act on a project: the directory `--path` names, with its -/// `rust-tops.yaml` and `rust-tops.lock`. +/// The commands that take a `--path` to a project that is already set up, and +/// read its `rust-tops.yaml` or `rust-tops.lock`. `init` writes them, and +/// `version` reads the lock of the working directory. #[derive(Subcommand)] enum ProjectCommand { /// Validate rust-tops.yaml against the protocol schema (typed check) @@ -265,11 +266,24 @@ fn run_project(command: ProjectCommand) -> Result { } } +/// A `rust-tops.lock` that is there must read as pins: a pin that is not a +/// string, a `tools` that is not a table, or a pin that is not dotted digits +/// is an error naming the key. No lock is no pins. +fn check_lock(path: &Path) -> Result<()> { + let file = path.join(TOOL_LOCK); + let Ok(text) = fs::read_to_string(&file) else { + return Ok(()); + }; + lock::read_pins(&text).with_context(|| format!("{}", file.display()))?; + Ok(()) +} + /// `cargo tops check`: an expired exemption is a warning here and a failure /// of the gate; the config is still valid. fn check_command(path: &Path) -> Result { let cfg = load_config(path)?; check(&cfg)?; + check_lock(path)?; for problem in gate::judge::expired_problems(&cfg) { println!("warning: {problem}"); } diff --git a/crates/cargo-tops/tests/gate_report.rs b/crates/cargo-tops/tests/gate_report.rs index 476f993..aed01ae 100644 --- a/crates/cargo-tops/tests/gate_report.rs +++ b/crates/cargo-tops/tests/gate_report.rs @@ -720,8 +720,21 @@ fn clippy_that_will_not_run_is_a_missing_tool_not_a_version_miss() { answer_version(&fixture, "clippy", None); let (out, text) = gate(&fixture, &[], None); assert_eq!(out.status.code(), Some(3), "{text}"); - assert!(text.contains("static-lint: `cargo` not on PATH"), "{text}"); + // The correction: a probe that failed names the tool and the command it ran + // (it used to say `cargo` was not on PATH), once, on stdout, in the + // layer's detail and in the problems. + let said = "missing clippy: pin 1.99.1, `cargo clippy --version` exited 101"; + assert!( + text.contains(&format!("tool-missing [static-lint]: {said}")), + "{text}" + ); + assert!(text.contains(&format!("static-lint: {said}")), "{text}"); let report = report(&fixture); + assert_eq!(at(layer(&report, "static-lint"), "/detail"), said); + assert!( + problems(&report).contains(&format!("static-lint: {said}")), + "{report}" + ); assert_eq!( at(layer(&report, "static-lint"), "/verdict"), "tool-missing" @@ -782,6 +795,34 @@ fn a_layer_that_takes_another_layers_run_keeps_that_runs_mismatch() { assert_eq!(at(layer(&report, "property"), "/detail"), said); } +#[test] +fn a_pre_release_of_a_pinned_tool_is_a_tool_mismatch_not_a_pass() { + let fixture = fixture("pre-release"); + answer_version(&fixture, "deny", Some("cargo-deny 0.18.9-rc.1")); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(3), "{text}"); + assert!( + text.contains("mismatch cargo-deny: pin 0.18.9, found 0.18.9-rc.1"), + "{text}" + ); +} + +#[test] +fn a_pin_that_is_not_a_string_stops_the_gate_with_exit_2_naming_the_key() { + let fixture = fixture("lock-not-a-string"); + fs::write( + fixture.repo.join("rust-tops.lock"), + "[tools]\ncargo-deny = 0.18\n", + ) + .expect("write lock"); + let (out, text) = gate(&fixture, &[], None); + assert_eq!(out.status.code(), Some(2), "{text}"); + assert!(text.contains("cargo-deny"), "{text}"); + assert!(text.contains("found float"), "{text}"); + let log = cargo_log(&fixture); + assert!(!log.lines().any(|line| line.starts_with("deny ")), "{log}"); +} + fn cargo_log(fixture: &Fixture) -> String { fs::read_to_string(fixture.root.join("cargo.log")).unwrap_or_default() } diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 7186487..164b3b9 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -2178,7 +2178,10 @@ fn a_pinned_cargo_crap_that_is_not_installed_is_missing_not_a_version_miss() { stub_cargo_failing(&fixture, "\"crap --version\"", 101); let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); assert_eq!(out.status.code(), Some(3), "{text}"); - assert!(text.contains("crap: `cargo-crap` not on PATH"), "{text}"); + assert!( + text.contains("crap: missing cargo-crap: pin 0.9.0, `cargo crap --version` exited 101"), + "{text}" + ); assert!(!text.contains("tool-mismatch"), "{text}"); let report = report_json(&fixture); assert_eq!(at(&layer_of(&report, "crap"), "/verdict"), "tool-missing"); diff --git a/crates/cargo-tops/tests/lock_verify.rs b/crates/cargo-tops/tests/lock_verify.rs index c8a6ae5..2710d97 100644 --- a/crates/cargo-tops/tests/lock_verify.rs +++ b/crates/cargo-tops/tests/lock_verify.rs @@ -35,7 +35,7 @@ impl Scratch { let scratch = Scratch { root }; scratch.script( "cargo", - "#!/bin/sh\nsub=\"$1\"; shift\nexec \"${0%/*}/cargo-$sub\" \"$@\"\n", + "#!/bin/sh\nsub=\"$1\"; shift\n# Cargo says exit 101 for a subcommand it does not have.\n[ -x \"${0%/*}/cargo-$sub\" ] || exit 101\nexec \"${0%/*}/cargo-$sub\" \"$@\"\n", ); scratch } @@ -194,7 +194,7 @@ fn a_pinned_tool_that_is_not_on_path_is_missing_and_exits_3() { let out = scratch.run(&["--verify"]); assert_eq!(out.status.code(), Some(3), "{}", text(&out)); assert!( - text(&out).contains("missing cargo-crap: pin 0.9.0, not found"), + text(&out).contains("missing cargo-crap: pin 0.9.0, `cargo crap --version` exited 101"), "{}", text(&out) ); @@ -320,3 +320,233 @@ fn clippy_and_rustfmt_must_run_even_when_rustc_matches_the_pin() { let out = scratch.run(&["--verify"]); assert_eq!(out.status.code(), Some(0), "{}", text(&out)); } + +/// Runs `lock ` on the lock text and returns the output. +fn verify_lock(name: &str, lock: &str, shims: &[(&str, &str)]) -> (Scratch, Output) { + let scratch = Scratch::new(name); + scratch.lock(lock); + for (shim, line) in shims { + scratch.tool(shim, line); + } + let out = scratch.run(&["--verify"]); + (scratch, out) +} + +#[test] +fn a_pin_that_is_not_a_string_is_a_lock_error_naming_the_key() { + for (lock, key) in [ + ("[tools]\ncargo-crap = 0.9\n", "cargo-crap"), + ("[tools]\ncargo-mutants = 27\n", "cargo-mutants"), + ( + "[tools]\ncargo-crap = { version = \"0.9.0\" }\n", + "cargo-crap", + ), + ("[tools]\nrustc = true\n", "rustc"), + ] { + let (_scratch, out) = verify_lock("not-a-string", lock, &[]); + assert_eq!(out.status.code(), Some(2), "{lock}: {}", text(&out)); + assert!(text(&out).contains(key), "{lock}: {}", text(&out)); + } +} + +#[test] +fn a_tools_key_that_is_not_a_table_is_a_lock_error() { + for lock in [ + "[[tools]]\nrustc = \"1.99.1\"\n", + "tools = \"pinned below\"\n", + ] { + let (_scratch, out) = verify_lock("tools-not-a-table", lock, &[]); + assert_eq!(out.status.code(), Some(2), "{lock}: {}", text(&out)); + assert!(text(&out).contains("tools"), "{lock}: {}", text(&out)); + } +} + +#[test] +fn a_lock_with_no_tools_table_is_a_lock_error_and_an_empty_one_says_so() { + let (_scratch, out) = verify_lock("no-tools", "[notes]\nschema = \"1.3.0\"\n", &[]); + assert_eq!(out.status.code(), Some(2), "{}", text(&out)); + assert!(text(&out).contains("[tools]"), "{}", text(&out)); + let (_scratch, out) = verify_lock("empty-tools", "[tools]\n", &[]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + assert!( + text(&out).contains("nothing pinned in [tools]"), + "{}", + text(&out) + ); +} + +#[test] +fn a_pin_that_is_not_dotted_digits_is_a_lock_error() { + for pin in ["=0.9.0", "v0.9.0", "^0.9", "0.9.*", "", "0..9", "0.9."] { + let lock = format!("[tools]\ncargo-crap = \"{pin}\"\n"); + let (_scratch, out) = verify_lock("bad-pin", &lock, &[]); + assert_eq!(out.status.code(), Some(2), "{pin:?}: {}", text(&out)); + assert!(text(&out).contains("cargo-crap"), "{pin:?}: {}", text(&out)); + } +} + +#[test] +fn a_pre_release_never_matches_a_release_pin_and_a_build_suffix_does() { + for (shim, line, lock) in [ + ( + "cargo-crap", + "cargo-crap 0.9.0-rc.1", + "[tools]\ncargo-crap = \"0.9.0\"\n", + ), + ( + "cargo-crap", + "cargo-crap 0.9.0-beta.1", + "[tools]\ncargo-crap = \"0.9\"\n", + ), + ( + "rustc", + "rustc 1.99.1-nightly (58cf8f9c3 2026-10-13)", + "[tools]\nrustc = \"1.99.1\"\n", + ), + ( + "rustc", + "rustc 1.99.1-beta.3 (58cf8f9c3 2026-10-13)", + "[tools]\nrustc = \"1.99\"\n", + ), + ( + "nu", + "0.108.0-nightly.3", + "[tools]\nnushell = \"0.108.0\"\n", + ), + ] { + let (_scratch, out) = verify_lock("pre-release", lock, &[(shim, line)]); + assert_eq!(out.status.code(), Some(3), "{line}: {}", text(&out)); + assert!(text(&out).contains("mismatch"), "{line}: {}", text(&out)); + } + let (_scratch, out) = verify_lock( + "build-suffix", + "[tools]\ncargo-crap = \"0.9.0\"\n", + &[("cargo-crap", "cargo-crap 0.9.0+abc")], + ); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); +} + +#[test] +fn empty_output_reads_as_no_version_line() { + let (_scratch, out) = verify_lock( + "empty-output", + "[tools]\ncargo-crap = \"0.9.0\"\n", + &[("cargo-crap", "")], + ); + assert_eq!(out.status.code(), Some(3), "{}", text(&out)); + assert!( + text(&out).contains("mismatch cargo-crap: pin 0.9.0, found no version line"), + "{}", + text(&out) + ); +} + +#[test] +fn an_unknown_key_warns_and_only_the_three_named_rows_are_skipped() { + let lock = "[tools]\nproptest = \"1\"\ncriterion = \"0.8\"\ncallgrind-runner = \"gungraun-runner\"\ncargo-nexttest = \"0.9.145\"\n"; + let (_scratch, out) = verify_lock("unknown-key", lock, &[]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + let said = text(&out); + assert!( + said.contains("unknown cargo-nexttest: not a tool rust-tops knows, not checked"), + "{said}" + ); + for name in ["proptest", "criterion", "callgrind-runner"] { + assert!(said.contains(&format!("skipped {name}:")), "{said}"); + } + assert!(!said.contains("skipped cargo-nexttest"), "{said}"); +} + +#[test] +fn a_probe_that_fails_is_named_with_its_argv_and_how_it_failed() { + let scratch = Scratch::new("probe-failed"); + scratch.lock("[tools]\ncargo-crap = \"0.9.0\"\n"); + scratch.script( + "cargo-crap", + "#!/bin/sh\necho 'cargo-crap 0.9.0'\nexit 101\n", + ); + let out = scratch.run(&["--verify"]); + assert_eq!(out.status.code(), Some(3), "{}", text(&out)); + assert!( + text(&out).contains("missing cargo-crap: pin 0.9.0, `cargo crap --version` exited 101"), + "{}", + text(&out) + ); +} + +#[test] +fn check_refuses_a_lock_that_cannot_be_read_as_pins() { + let scratch = Scratch::new("check-lock"); + init(&scratch, "library-core"); + let run = |scratch: &Scratch| { + Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["check", "--path"]) + .arg(scratch.repo()) + .env("PATH", scratch.bin()) + .output() + .expect("spawn cargo-tops check") + }; + let ok = run(&scratch); + assert_eq!(ok.status.code(), Some(0), "{}", text(&ok)); + scratch.lock("[tools]\ncargo-crap = 0.9\n"); + let out = run(&scratch); + assert_eq!(out.status.code(), Some(2), "{}", text(&out)); + assert!(text(&out).contains("cargo-crap"), "{}", text(&out)); +} + +/// A lock written in a form `--write` will not rewrite, and the key it names. +const UNWRITABLE: [(&str, &str); 6] = [ + ("[tools]\n\"cargo-crap\" = \"0.9.0\"\n", "cargo-crap"), + ("tools.cargo-crap = \"0.9.0\"\n", "cargo-crap"), + ("tools = { cargo-crap = \"0.9.0\" }\n", "cargo-crap"), + ("[tools]\ncargo-crap = '0.9.0'\n", "cargo-crap"), + ("[tools]\ncargo-crap = \"\"\"0.9.0\"\"\"\n", "cargo-crap"), + ("[ tools ]\ncargo-crap = \"0.9.0\"\n", "cargo-crap"), +]; + +#[test] +fn write_refuses_any_form_it_cannot_rewrite_and_changes_nothing() { + for (lock, key) in UNWRITABLE { + let scratch = Scratch::new("write-refuses"); + scratch.lock(lock); + scratch.tool("cargo-crap", "cargo-crap 0.9.5"); + let out = scratch.run(&["--write"]); + assert_eq!(out.status.code(), Some(2), "{lock}: {}", text(&out)); + assert!(text(&out).contains(key), "{lock}: {}", text(&out)); + assert!(!text(&out).contains("wrote"), "{lock}: {}", text(&out)); + assert_eq!(scratch.read_lock(), lock, "{lock}"); + } +} + +#[test] +fn write_leaves_a_partial_pin_that_already_matches_and_prints_wrote_only_for_changes() { + let scratch = Scratch::new("write-partial"); + let lock = "[tools]\ncargo-fuzz = \"0.12\"\ncargo-crap = \"0.9.0\"\n"; + scratch.lock(lock); + scratch.tool("cargo-fuzz", "cargo-fuzz 0.12.3"); + scratch.tool("cargo-crap", "cargo-crap 0.9.5"); + let out = scratch.run(&["--write"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + assert_eq!( + scratch.read_lock(), + "[tools]\ncargo-fuzz = \"0.12\"\ncargo-crap = \"0.9.5\"\n" + ); + let said = text(&out); + assert!(said.contains("kept cargo-fuzz 0.12"), "{said}"); + assert!( + said.contains("wrote cargo-crap: 0.9.0 -> 0.9.5"), + "{said}" + ); + assert!(!said.contains("wrote cargo-fuzz"), "{said}"); +} + +#[test] +fn write_never_puts_a_pre_release_into_the_lock() { + let scratch = Scratch::new("write-pre-release"); + let lock = "[tools]\ncargo-crap = \"0.9.0\"\n"; + scratch.lock(lock); + scratch.tool("cargo-crap", "cargo-crap 0.9.5-rc.1"); + let out = scratch.run(&["--write"]); + assert_eq!(out.status.code(), Some(0), "{}", text(&out)); + assert_eq!(scratch.read_lock(), lock); +} From 61afbd51ce0146b1e9bb67e24bfbdaefe064130c Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 09:11:52 -0500 Subject: [PATCH 3/4] Give the gate-running kit tests the time the pin asks need The gate tests run the whole gate several times against stubs, and each gate now asks the pinned tools for their versions. Three test binaries get a longer limit in the kit config, with the measured times in the comment. The version ask looks for its answer sooner and backs off. Co-Authored-By: Claude Sonnet 5.5 --- .config/nextest.toml | 11 +++++++++++ crates/cargo-tops/src/lock/probe.rs | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index c732786..23e38d8 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -66,3 +66,14 @@ slow-timeout = { period = "10s", terminate-after = 6 } [[profile.ci.overrides]] filter = "binary(lock_verify)" slow-timeout = { period = "10s", terminate-after = 6 } + +# These tests run the whole gate against stub tools several times each, for +# the kit suite only; they match nothing in an adopter. Since the gate asks each +# pinned tool for its version, a gate runs about nine more commands. The slowest +# test of them ran in 2.3 s on the base and 3.4 s to 3.9 s with the pins, alone +# on the build machine. In the parallel run under coverage it timed out at 20 s +# in 3 of 4 gates at a load of 6 to 13. They get 60 s; the profile timeout is not +# raised. +[[profile.ci.overrides]] +filter = "binary(gate_survivors) | binary(gate_exemptions) | binary(gate_base)" +slow-timeout = { period = "10s", terminate-after = 6 } diff --git a/crates/cargo-tops/src/lock/probe.rs b/crates/cargo-tops/src/lock/probe.rs index bdf8f4c..c0a6f6a 100644 --- a/crates/cargo-tops/src/lock/probe.rs +++ b/crates/cargo-tops/src/lock/probe.rs @@ -42,6 +42,8 @@ pub(crate) fn run_probe(dir: &Path, argv: &[&str], limit: Duration) -> Result Result<(), String> { let deadline = Instant::now() + limit; + // Most asks answer within milliseconds, so look soon and back off. + let mut pause = Duration::from_millis(1); loop { let status = child .try_wait() @@ -49,7 +51,10 @@ fn wait_within(child: &mut Child, limit: Duration) -> Result<(), String> { match status { Some(status) => return finished(status), None if Instant::now() >= deadline => return give_up(child, limit), - None => std::thread::sleep(Duration::from_millis(10)), + None => { + std::thread::sleep(pause); + pause = (pause * 2).min(Duration::from_millis(20)); + } } } } From b9bd6a20059331208e79ad42f5898c7f366867d4 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 09:35:04 -0500 Subject: [PATCH 4/4] Look at a running version ask at a fixed interval The backoff was no faster and left a mutation that no test could tell apart. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/lock/probe.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/cargo-tops/src/lock/probe.rs b/crates/cargo-tops/src/lock/probe.rs index c0a6f6a..8df80de 100644 --- a/crates/cargo-tops/src/lock/probe.rs +++ b/crates/cargo-tops/src/lock/probe.rs @@ -8,6 +8,9 @@ use std::time::{Duration, Instant}; /// How long one `--version` ask may take. pub(crate) const PROBE_LIMIT: Duration = Duration::from_secs(10); +/// How often a running ask is looked at. +const POLL: Duration = Duration::from_millis(5); + /// A probe that runs each argv in `dir` with the [`PROBE_LIMIT`]. pub(crate) fn probe_in(dir: &Path) -> impl FnMut(&[&str]) -> Result + '_ { move |argv| run_probe(dir, argv, PROBE_LIMIT) @@ -42,8 +45,6 @@ pub(crate) fn run_probe(dir: &Path, argv: &[&str], limit: Duration) -> Result Result<(), String> { let deadline = Instant::now() + limit; - // Most asks answer within milliseconds, so look soon and back off. - let mut pause = Duration::from_millis(1); loop { let status = child .try_wait() @@ -51,10 +52,7 @@ fn wait_within(child: &mut Child, limit: Duration) -> Result<(), String> { match status { Some(status) => return finished(status), None if Instant::now() >= deadline => return give_up(child, limit), - None => { - std::thread::sleep(pause); - pause = (pause * 2).min(Duration::from_millis(20)); - } + None => std::thread::sleep(POLL), } } }