diff --git a/.config/nextest.toml b/.config/nextest.toml index ea98ebe..ed829ad 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -48,3 +48,11 @@ slow-timeout = { period = "10s", terminate-after = 4 } [[profile.ci.overrides]] filter = "test(=fresh_init_then_gate_passes_for_every_class)" slow-timeout = { period = "10s", terminate-after = 4 } + +# These tests run real `cargo nextest` on a small crate, one build each, for +# the kit suite only; they match nothing in an adopter. Alone each ran in +# 1.4 s to 6.2 s. Under the full suite on a loaded runner they take longer, +# so they get 60 s; the profile timeout is not raised. +[[profile.ci.overrides]] +filter = "binary(gate_credit_nextest)" +slow-timeout = { period = "10s", terminate-after = 6 } diff --git a/Cargo.lock b/Cargo.lock index 4a494b4..3573a5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,7 @@ version = "0.4.0" dependencies = [ "anyhow", "clap", + "proptest", "regex", "rustix", "serde", diff --git a/PROTOCOL.md b/PROTOCOL.md index a9d57f7..6fdcde7 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -391,6 +391,10 @@ fn encode_empty_emits_single_zero_block() { } ``` +#### 6.5.1 What the gate reads + +The gate runs the `acceptance` layer the way it runs `property` (§9.5): the same nextest command, narrowed to the tests it selects by name, with its own filter `gates.acceptance.test_filter` (default `acceptance`). A test counts for the layer when one `::`-separated segment of its nextest name starts with the filter, so `acceptance_encode` and `journeys::acceptance_encode` count and `an_acceptance_helper` does not. When the layer takes the coverage run instead of running itself, it passes only if that run's JUnit report holds a passed test that matches. + ### 6.6 Compile-fail / UI (proc-macros, tricky APIs) `trybuild` or `ui_test`. These are the mutation tests of the type system: if a change makes an intended failure compile, the suite must go red. @@ -596,6 +600,16 @@ Rules: A good property kills operator mutants that example tests miss (`<` vs `<=`, off-by-one on max). If a survivor is an inequality on a bound, add a property at the bound, do not add a single magic example unless the bound is a spec constant. +### 9.5 What the gate reads + +`gates.property.test_filter` (default `property`) names the tests that count as property tests: a test counts when one `::`-separated segment of its nextest name starts with the filter. The `property` layer passes only on a passed test that matches. + +- **It runs itself** when no coverage run is credited. Its command is the shared nextest run with `-E 'test(/(^|::)/)'` and `--no-tests=fail`, so a filter that selects nothing fails the layer. A filter is letters, digits and underscore only (`^[A-Za-z0-9_]+$`), so it is a plain regex literal and needs no escaping. `check` refuses any other value, and the gate exits 2. +- **It takes the coverage run** when its command carries `--workspace`, `--all-features` and `--profile ci`. The coverage run must also have written a nextest JUnit report, holding at least one passed test that matches. The gate reads the file where nextest wrote it: the workspace root from `cargo metadata`, then nextest's `[store] dir` from `.config/nextest.toml` (default `target/nextest`), then `ci/` with `path` from `[profile.ci.junit]`. `CARGO_TARGET_DIR` and `build.target-dir` do not move it. A report dated before the credited run began, or in the future, was not written by that run, so it is stale and not evidence; the gate never deletes it. No match, only failed or skipped matches, no report, or a stale report is `fail`. The report's `detail`, the failure line and `problems` all name the filter, the path read (relative to the workspace root), and the fix. +- **The same rule** applies to the whole workspace, not to one crate, so credit equals running. + +The same keys exist for `acceptance` under `gates.acceptance` (§6.5.1). + --- ## 10. Fuzzing (Tier 3) diff --git a/crates/cargo-tops/Cargo.toml b/crates/cargo-tops/Cargo.toml index ebe1490..ef0e7c9 100644 --- a/crates/cargo-tops/Cargo.toml +++ b/crates/cargo-tops/Cargo.toml @@ -39,5 +39,8 @@ toml = "1" # `openat`/`mkdirat` without `unsafe`. 1.1.6 clears RUSTSEC-2026-0486. rustix = { version = "1.1.6", default-features = false, features = ["fs", "std"] } +[dev-dependencies] +proptest = "1" + [lints] workspace = true diff --git a/crates/cargo-tops/src/config.rs b/crates/cargo-tops/src/config.rs index ad2bafc..f21a9e4 100644 --- a/crates/cargo-tops/src/config.rs +++ b/crates/cargo-tops/src/config.rs @@ -44,6 +44,32 @@ pub(crate) struct Gates { /// Absent means the remote's default branch (`origin/HEAD`). #[serde(skip_serializing_if = "Option::is_none")] pub base_ref: Option, + /// The `property` gate. Only `test_filter` is read here; the rest of the + /// object stays as written. + #[serde(default, skip_serializing)] + pub property: Option, + /// The `acceptance` gate, as [`Gates::property`]. + #[serde(default, skip_serializing)] + pub acceptance: Option, +} + +impl Gates { + /// The `test_filter` of `layer` (`property` or `acceptance`): the name + /// prefix a test must start a `::` segment with to count (PROTOCOL.md + /// §6, §9). It defaults to the layer's id. + pub(crate) fn test_filter(&self, layer: &str) -> String { + let gate = match layer { + "property" => &self.property, + "acceptance" => &self.acceptance, + _ => &None, + }; + gate.as_ref() + .and_then(|gate| gate.get("test_filter")) + .and_then(serde_json::Value::as_str) + .filter(|filter| !filter.is_empty()) + .unwrap_or(layer) + .to_string() + } } #[derive(Debug, Deserialize, Serialize, PartialEq)] @@ -69,3 +95,44 @@ pub(crate) struct CrapGate { pub(crate) struct MutationGate { pub unexplained_survivors_max: i64, } + +#[cfg(test)] +mod tests { + use super::Gates; + use serde_json::json; + + fn gates(extra: &serde_json::Value) -> Gates { + let mut doc = + json!({"coverage": {}, "crap": {}, "mutation": {"unexplained_survivors_max": 0}}); + if let (Some(base), Some(more)) = (doc.as_object_mut(), extra.as_object()) { + base.extend(more.clone()); + } + serde_json::from_value(doc).expect("gates") + } + + #[test] + fn a_test_filter_defaults_to_the_layer_id() { + let none = gates(&json!({})); + assert_eq!(none.test_filter("property"), "property"); + assert_eq!(none.test_filter("acceptance"), "acceptance"); + let other = gates(&json!({"property": {"framework": "proptest"}})); + assert_eq!(other.test_filter("property"), "property"); + } + + #[test] + fn each_layer_reads_its_own_test_filter() { + let set = gates(&json!({ + "property": {"test_filter": "prop_"}, + "acceptance": {"test_filter": "journey_"}, + })); + assert_eq!(set.test_filter("property"), "prop_"); + assert_eq!(set.test_filter("acceptance"), "journey_"); + assert_eq!(set.test_filter("unit"), "unit"); + } + + #[test] + fn an_empty_test_filter_is_the_default() { + let empty = gates(&json!({"property": {"test_filter": ""}})); + assert_eq!(empty.test_filter("property"), "property"); + } +} diff --git a/crates/cargo-tops/src/credit.rs b/crates/cargo-tops/src/credit.rs index 873c7f0..83f77b9 100644 --- a/crates/cargo-tops/src/credit.rs +++ b/crates/cargo-tops/src/credit.rs @@ -4,7 +4,9 @@ //! a layer and [`recorded_outcome`] after the source layer has finished. //! A missing or failed coverage run is not a pass. A coverage command that //! lacks `--workspace`, `--all-features`, or the nextest profile grants no -//! credit, and `unit`, `integration`, and `property` run themselves. +//! credit, and `unit`, `integration`, `property`, and `acceptance` run +//! themselves. `property` and `acceptance` also need evidence from the run +//! ([`crate::evidence`]). #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] #![allow(dead_code, reason = "main.rs calls the credit decision")] @@ -66,7 +68,10 @@ pub(crate) fn recorded_outcome(query: &Query<'_>) -> Option { fn takes_coverage(query: &Query<'_>) -> bool { query.coverage_in_range && coverage_grants_credit(query.coverage_argv) - && matches!(query.layer, "unit" | "integration" | "property") + && matches!( + query.layer, + "unit" | "integration" | "property" | "acceptance" + ) } fn takes_unit(query: &Query<'_>) -> bool { diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs new file mode 100644 index 0000000..fe627fa --- /dev/null +++ b/crates/cargo-tops/src/evidence.rs @@ -0,0 +1,450 @@ +//! Evidence that a credited layer's tests ran: the credited run's nextest +//! JUnit report. +//! +//! `property` and `acceptance` may take another layer's run (see +//! [`crate::credit`]). That run passes for them only if its JUnit report +//! holds a test whose name matches the layer's `test_filter` (see [`matches`]) +//! and that test passed. This module reads that one file, from where nextest +//! writes it: the workspace root `cargo metadata` names, nextest's +//! `[store] dir` (default `target/nextest`), the profile, then the +//! `[profile.ci.junit]` path. A report older than the credited run is not +//! evidence. Nothing is built and no test runs, so reading it costs no +//! second test run. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +/// The nextest profile whose report is read. +pub(crate) const PROFILE: &str = "ci"; + +/// Where nextest keeps its reports, under the workspace root, when +/// `.config/nextest.toml` has no `[store] dir`. +const DEFAULT_STORE: &str = "target/nextest"; + +/// How many matching tests passed and how many did not. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Tally { + pub passed: usize, + pub failed: usize, +} + +/// Whether a nextest test name matches `filter` (HN-L2): one `::`-separated +/// segment of the name starts with it. `property_x` and `tests::property_x` +/// match `property`; `a_property_x` does not. A layer that runs itself +/// selects tests with the same rule. +#[must_use] +pub(crate) fn matches(name: &str, filter: &str) -> bool { + name.split("::").any(|segment| segment.starts_with(filter)) +} + +/// Counts the tests in a JUnit report whose name matches `filter`. +/// +/// A test with a `failure`, `error` or `skipped` child did not pass. An empty +/// `filter` matches every test. +#[must_use] +pub(crate) fn tally(xml: &str, filter: &str) -> Tally { + let mut tally = Tally::default(); + for (name, passed) in testcases(xml) { + if !matches(&name, filter) { + continue; + } + if passed { + tally.passed += 1; + } else { + tally.failed += 1; + } + } + tally +} + +/// Each `testcase` as its name and whether it passed. +fn testcases(xml: &str) -> Vec<(String, bool)> { + let mut found = Vec::new(); + let mut rest = xml; + while let Some(at) = rest.find("') else { + break; + }; + let (body, tail) = if tag.ends_with('/') { + ("", after) + } else { + after.split_once("").unwrap_or((after, "")) + }; + if let Some(name) = attribute(tag, "name") { + found.push((name, !has_verdict_child(body))); + } + rest = tail; + } + found +} + +fn has_verdict_child(body: &str) -> bool { + [" Option { + let marker = format!(" {key}=\""); + let value = tag.split_once(&marker)?.1; + Some(unescape(value.split_once('"')?.0)) +} + +fn unescape(text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +/// The `path` of `[profile.ci.junit]` in the text of `.config/nextest.toml`. +#[must_use] +pub(crate) fn junit_name(nextest_toml: &str) -> Option { + let doc: toml::Table = toml::from_str(nextest_toml).ok()?; + doc.get("profile")? + .get(PROFILE)? + .get("junit")? + .get("path")? + .as_str() + .map(str::to_string) +} + +/// The `dir` of `[store]` in the text of `.config/nextest.toml`, relative to +/// the workspace root. nextest keeps its reports there. +#[must_use] +pub(crate) fn store_dir(nextest_toml: &str) -> String { + toml::from_str::(nextest_toml) + .ok() + .and_then(|doc| doc.get("store")?.get("dir")?.as_str().map(str::to_string)) + .unwrap_or_else(|| DEFAULT_STORE.to_string()) +} + +/// The report the credited run wrote: `//ci/`, with +/// the store dir and the file name from `.config/nextest.toml` under `root`. +/// +/// # Errors +/// The setting is missing, so the run writes no report to read. +pub(crate) fn junit_path(root: &Path) -> Result { + let text = fs::read_to_string(root.join(".config").join("nextest.toml")).unwrap_or_default(); + junit_name(&text) + .map(|name| root.join(store_dir(&text)).join(PROFILE).join(name)) + .ok_or_else(|| { + "no JUnit report is configured: set `path` under `[profile.ci.junit]` in \ + .config/nextest.toml so the credited run writes one" + .to_string() + }) +} + +/// Whether the credited run, which began at `since`, held a passed test +/// matching `filter` in a report it wrote. +/// +/// Returns how many matching tests passed. The file is only read, never +/// removed. +/// +/// # Errors +/// The detail for a `fail` (the judge adds the layer): it names the filter +/// and the report path read (relative to `root`), or the +/// `[profile.ci.junit]` setting. +pub(crate) fn credit( + layer: &str, + root: &Path, + filter: &str, + since: SystemTime, +) -> Result { + let who = format!("`{layer}` (filter `{filter}`)"); + let path = + junit_path(root).map_err(|why| format!("{why}; {who} has no evidence without it"))?; + let shown = path + .strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(); + let xml = fs::read_to_string(&path).map_err(|_| { + format!( + "no JUnit report at {shown}, so {who} has no evidence; the credited run must write \ + one (`[profile.ci.junit]` and `[store] dir` in .config/nextest.toml)" + ) + })?; + // Written during the credited run: not before it began, not after now. + let fresh = fs::metadata(&path) + .and_then(|meta| meta.modified()) + .is_ok_and(|written| written >= since && written <= SystemTime::now()); + if !fresh { + return Err(format!( + "the JUnit report {shown} was not written during the credited run (it is older \ + than the run, or dated in the future), so it is no evidence for {who}; make that \ + run write it (`[profile.ci.junit]` and `[store] dir` in .config/nextest.toml)" + )); + } + let found = tally(&xml, filter); + if found.passed > 0 { + return Ok(found.passed); + } + let why = if found.failed > 0 { + "every matching test failed or was skipped" + } else { + "no test matches" + }; + Err(format!( + "the credited run's report {shown} holds no passed test whose name has a `::` segment \ + starting with `{filter}` ({why}); add one, or set gates.{layer}.test_filter" + )) +} + +#[cfg(test)] +mod tests { + use super::{credit, junit_name, junit_path, matches, store_dir, tally, Tally}; + use std::fs; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + const REPORT: &str = r#" + + + + + boom + a <testcase name="property_fake"/> + + + + fine + + + + +"#; + + #[test] + fn only_a_passed_test_whose_name_matches_the_filter_counts() { + assert_eq!( + tally(REPORT, "property"), + Tally { + passed: 2, + failed: 3 + } + ); + assert_eq!( + tally(REPORT, "plain"), + Tally { + passed: 1, + failed: 0 + } + ); + assert_eq!(tally(REPORT, "acceptance"), Tally::default()); + assert_eq!(tally(REPORT, "Property"), Tally::default()); + assert_eq!(tally("", "property"), Tally::default()); + } + + #[test] + fn a_segment_must_start_with_the_filter() { + assert!(matches("property_x", "property")); + assert!(matches("tests::property_x", "property")); + assert!(matches("a::property::b", "property")); + assert!(!matches("a_property_x", "property")); + assert!(!matches("tests::a_property", "property")); + assert!(!matches("Property_x", "property")); + assert!(matches("anything", "")); + } + + #[test] + fn a_meta_test_that_only_mentions_the_filter_is_not_evidence() { + let xml = r#""#; + assert_eq!(tally(xml, "property"), Tally::default()); + } + + #[test] + fn escaped_text_inside_a_test_is_not_a_test() { + assert_eq!(tally(REPORT, "property_fake"), Tally::default()); + } + + #[test] + fn an_unclosed_tag_ends_the_scan_without_a_count() { + assert_eq!( + tally(""#; + assert_eq!(tally(open, "property").failed, 1); + } + + #[test] + fn a_name_is_unescaped() { + let xml = r#""#; + assert_eq!(tally(xml, "property_a&b\"c'd").passed, 1); + } + + #[test] + fn the_report_name_is_the_junit_path_of_the_ci_profile() { + let toml = "[profile.ci]\nretries = 0\n[profile.ci.junit]\npath = \"j.xml\"\n"; + assert_eq!(junit_name(toml).as_deref(), Some("j.xml")); + assert_eq!(junit_name("[profile.ci]\nretries = 0\n"), None); + assert_eq!( + junit_name("[profile.default.junit]\npath = \"j.xml\"\n"), + None + ); + assert_eq!(junit_name("not toml ["), None); + assert_eq!(junit_name("[profile.ci.junit]\npath = 3\n"), None); + } + + fn fixture(config: Option<&str>, report: Option<&str>) -> tempdir::Dir { + let dir = tempdir::Dir::new(); + if let Some(config) = config { + fs::create_dir_all(dir.path().join(".config")).unwrap(); + fs::write(dir.path().join(".config/nextest.toml"), config).unwrap(); + } + if let Some(report) = report { + let at = dir.path().join("target/nextest/ci"); + fs::create_dir_all(&at).unwrap(); + fs::write(at.join("junit.xml"), report).unwrap(); + } + dir + } + + const CONFIG: &str = "[profile.ci.junit]\npath = \"junit.xml\"\n"; + + fn run(dir: &tempdir::Dir, filter: &str) -> Result { + credit("property", dir.path(), filter, UNIX_EPOCH) + } + + #[test] + fn a_passed_matching_test_is_credit() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + assert_eq!(run(&dir, "property"), Ok(2)); + assert_eq!(run(&dir, "plain"), Ok(1)); + } + + #[test] + fn no_match_names_the_filter_and_the_path_read() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + let why = run(&dir, "acceptance").unwrap_err(); + assert!(why.contains("`acceptance`"), "{why}"); + assert!(why.contains("target/nextest/ci/junit.xml"), "{why}"); + assert!(why.contains("no test matches"), "{why}"); + assert!(why.contains("gates.property.test_filter"), "{why}"); + assert!(!why.contains(dir.path().to_str().unwrap_or("?")), "{why}"); + } + + #[test] + fn a_failed_match_is_not_evidence() { + let xml = r#""#; + let dir = fixture(Some(CONFIG), Some(xml)); + let why = run(&dir, "property").unwrap_err(); + assert!(why.contains("failed or was skipped"), "{why}"); + } + + #[test] + fn a_missing_report_names_the_junit_setting() { + let dir = fixture(Some(CONFIG), None); + let why = run(&dir, "property").unwrap_err(); + assert!( + why.contains("no JUnit report at target/nextest/ci/junit.xml"), + "{why}" + ); + assert!(why.contains("`property` (filter `property`)"), "{why}"); + assert!(why.contains("[profile.ci.junit]"), "{why}"); + } + + #[test] + fn no_junit_setting_names_the_junit_setting() { + let dir = fixture(None, Some(REPORT)); + let why = run(&dir, "property").unwrap_err(); + assert!(why.starts_with("no JUnit report is configured"), "{why}"); + assert!(why.contains("`property` (filter `property`)"), "{why}"); + assert!(why.contains("[profile.ci.junit]"), "{why}"); + let bare = fixture(Some("[profile.ci]\nretries = 0\n"), Some(REPORT)); + assert!(run(&bare, "property").is_err()); + assert!(junit_path(bare.path()).is_err()); + } + + #[test] + fn the_file_name_comes_from_the_config() { + let dir = fixture(Some("[profile.ci.junit]\npath = \"out/j.xml\"\n"), None); + let path = junit_path(dir.path()).unwrap(); + assert_eq!(path, dir.path().join("target/nextest/ci/out/j.xml")); + } + + #[test] + fn a_moved_store_dir_moves_where_the_report_is_read() { + let config = "[store]\ndir = \"target/nt\"\n[profile.ci.junit]\npath = \"junit.xml\"\n"; + assert_eq!(store_dir(config), "target/nt"); + assert_eq!(store_dir(CONFIG), "target/nextest"); + assert_eq!(store_dir("not toml ["), "target/nextest"); + assert_eq!(store_dir("[store]\ndir = 3\n"), "target/nextest"); + let dir = fixture(Some(config), None); + assert_eq!( + junit_path(dir.path()).unwrap(), + dir.path().join("target/nt/ci/junit.xml") + ); + // The old location holds a matching report; the moved store holds none. + let old = fixture(Some(config), Some(REPORT)); + let why = run(&old, "property").unwrap_err(); + assert!( + why.contains("no JUnit report at target/nt/ci/junit.xml"), + "{why}" + ); + } + + #[test] + fn a_report_dated_in_the_future_is_not_evidence() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + let file = fs::File::options() + .write(true) + .open(dir.path().join("target/nextest/ci/junit.xml")) + .unwrap(); + file.set_modified(SystemTime::now() + Duration::from_secs(3600)) + .unwrap(); + let why = run(&dir, "property").unwrap_err(); + assert!(why.contains("not written during the credited run"), "{why}"); + } + + #[test] + fn a_report_older_than_the_run_is_not_evidence_and_is_kept() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + let later = SystemTime::now() + Duration::from_secs(3600); + let why = credit("property", dir.path(), "property", later).unwrap_err(); + assert!(why.contains("target/nextest/ci/junit.xml"), "{why}"); + assert!(why.contains("not written during the credited run"), "{why}"); + assert!(why.contains("`property` (filter `property`)"), "{why}"); + assert!(dir.path().join("target/nextest/ci/junit.xml").is_file()); + assert_eq!( + credit("property", dir.path(), "property", UNIX_EPOCH), + Ok(2) + ); + } + + /// A scratch directory removed on drop. It removes only what it made. + mod tempdir { + use std::path::{Path, PathBuf}; + + pub(super) struct Dir(PathBuf); + + impl Dir { + pub(super) fn new() -> Self { + use std::sync::atomic::{AtomicUsize, Ordering}; + static N: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "rt-evidence-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } + + pub(super) fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for Dir { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).ok(); + } + } + } +} diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index b57cf56..285f4bb 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Result}; use super::plan::{GatePlan, Unit, What, CRAP_VERSION}; use super::{Fault, Mode}; use crate::config::Layer; +use crate::evidence; use crate::facts::{Outcome, Run}; use crate::steps::{self, Base, Kind, MutationPlan, Step}; use crate::{laws, TOOL_LOCK}; @@ -20,9 +21,26 @@ pub(crate) struct Prepared { pub(crate) judge: Option, } +/// When a layer's commands began, as wall time and as a stopwatch. +struct Clock { + began: SystemTime, + watch: Instant, +} + +impl Clock { + fn start() -> Self { + Self { + began: SystemTime::now(), + watch: Instant::now(), + } + } +} + /// Wall time and tool version of one layer's commands. pub(crate) struct Timing { pub(crate) layer: String, + /// When its commands began: a report older than this is not theirs. + pub(crate) began: SystemTime, pub(crate) seconds: f64, pub(crate) tool_version: Option, } @@ -187,7 +205,7 @@ impl<'a> Runner<'_, 'a> { ); self.done.listed += 1; if self.live { - let clock = Instant::now(); + let clock = Clock::start(); let run = run_mutation(self.dir, &layer.id, mutation).map_err(Fault::env)?; // The tool is asked for its version only when it ran: a vacuous // or failed diff runs no `cargo mutants`. @@ -195,7 +213,7 @@ impl<'a> Runner<'_, 'a> { && run.cmd.contains(&mutation.mutants.display())) .then(|| version_of(self.dir, &mutation.mutants.argv, &mut self.versions)) .flatten(); - self.done.record(run, clock, version); + self.done.record(run, &clock, version); } Ok(()) } @@ -215,7 +233,7 @@ impl<'a> Runner<'_, 'a> { if !self.live { return Ok(()); } - let clock = Instant::now(); + let clock = Clock::start(); let run = match pin { Some(pin) => run_pinned_crap(self.dir, &layer.id, chain, pin), None => run_chain(self.dir, &layer.id, chain), @@ -233,7 +251,7 @@ impl<'a> Runner<'_, 'a> { .filter(|_| run.outcome != Outcome::ToolMissing) .and_then(|argv| version_of(self.dir, argv, &mut self.versions)), }; - self.done.record(run, clock, version); + self.done.record(run, &clock, version); Ok(()) } @@ -252,14 +270,46 @@ impl<'a> Runner<'_, 'a> { } } + /// The run `layer` takes from `from`. `property` and `acceptance` pass + /// only when that run also wrote a JUnit report holding a passed test + /// they select by name; the report is read, never produced (HN-P3). + fn credited(&self, layer: &str, from: &str) -> Run { + let mut run = self.plan.tests.take(layer, from, &self.done.runs); + let Some(filter) = self.plan.filters.for_layer(layer) else { + return run; + }; + if run.outcome != Outcome::Passed { + return run; + } + let root = laws::workspace_root(self.dir).unwrap_or_else(|| self.dir.to_path_buf()); + // A report older than the credited run's own start is not its report. + let since = self + .done + .timing + .iter() + .find(|timing| timing.layer == from) + .map_or(self.done.started, |timing| timing.began); + if let Err(why) = evidence::credit(layer, &root, filter, since) { + run.outcome = Outcome::Failed; + run.note = Some(why); + } + run + } + /// A layer that adds no command records its source's run. fn take_shared(&mut self) { for (layer, from) in self.done.shared.clone() { - let run = self.plan.tests.take(layer, from, &self.done.runs); - println!("[{layer}] took the {from} run's outcome: {}", run.outcome); + let run = self.credited(layer, from); + match &run.note { + Some(note) => { + println!("[{layer}] took the {from} run, but it is no credit: {note}"); + } + None => println!("[{layer}] took the {from} run's outcome: {}", run.outcome), + } self.done.runs.push(run); self.done.timing.push(Timing { layer: layer.to_string(), + began: SystemTime::now(), seconds: 0.0, tool_version: None, }); @@ -269,10 +319,11 @@ impl<'a> Runner<'_, 'a> { impl Executed<'_> { /// Keep `run` with the time since `clock` and the tool's version. - fn record(&mut self, run: Run, clock: Instant, tool_version: Option) { + fn record(&mut self, run: Run, clock: &Clock, tool_version: Option) { self.timing.push(Timing { layer: run.layer.clone(), - seconds: clock.elapsed().as_secs_f64(), + began: clock.began, + seconds: clock.watch.elapsed().as_secs_f64(), tool_version, }); self.runs.push(run); @@ -475,6 +526,7 @@ mod tests { fn timing(layer: &str, seconds: f64) -> Timing { Timing { layer: layer.to_string(), + began: SystemTime::UNIX_EPOCH, seconds, tool_version: None, } diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 78e8161..53313c2 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -111,13 +111,13 @@ fn run_problems(done: &Executed<'_>) -> Vec { let mut failed: Vec = done.not_run.clone(); failed.extend(done.runs.iter().filter_map(|run| { match done.shared.iter().find(|(layer, _)| *layer == run.layer) { - Some((_, from)) => run.outcome.fails_the_gate().then(|| { + Some((_, from)) if run.note.is_none() => run.outcome.fails_the_gate().then(|| { format!( "{}: no command of its own; the {from} run it took is {}", run.layer, run.outcome ) }), - None => run_failure(run), + _ => run_failure(run), } })); failed diff --git a/crates/cargo-tops/src/gate/mod.rs b/crates/cargo-tops/src/gate/mod.rs index cb52d3a..69bc0fa 100644 --- a/crates/cargo-tops/src/gate/mod.rs +++ b/crates/cargo-tops/src/gate/mod.rs @@ -127,7 +127,7 @@ pub(crate) fn config_error(dir: &Path, error: &anyhow::Error, opts: Options) -> #[cfg(test)] mod tests { use super::judge::{command_count, ran_commands}; - use super::plan::{plan_for, Plan, TestCommands}; + use super::plan::{plan_for, Filters, Plan, TestCommands}; use crate::facts::{Outcome, Run}; use crate::steps::{self, Base, Kind, Step}; use crate::{config, credit}; @@ -142,7 +142,12 @@ mod tests { } fn chains(id: &str) -> Vec> { - match plan_for(&layer(id), Path::new("/r"), &Base::unused()) { + match plan_for( + &layer(id), + Path::new("/r"), + &Base::unused(), + &Filters::default(), + ) { Some(Plan::Chains(chains)) => chains, _ => panic!("{id} has no chains"), } @@ -280,7 +285,7 @@ mod tests { .map(layer) .collect(); let enabled: Vec<&config::Layer> = layers.iter().collect(); - let tests = TestCommands::new(&enabled, Path::new("/r")); + let tests = TestCommands::new(&enabled, Path::new("/r"), &Filters::default()); assert!(tests.coverage_in_range); assert!( credit::coverage_grants_credit(&tests.coverage), @@ -300,7 +305,7 @@ mod tests { .copied() .filter(|l| l.id != "coverage") .collect(); - let tests = TestCommands::new(&unit_only, Path::new("/r")); + let tests = TestCommands::new(&unit_only, Path::new("/r"), &Filters::default()); assert!(!tests.coverage_in_range); assert!(tests.coverage.is_empty()); assert_eq!(tests.shared_from("integration"), Some("unit")); @@ -402,7 +407,9 @@ mod tests { source: steps::BaseSource::Flag, name: "origin/trunk".to_string(), }; - let Some(Plan::Mutation(plan)) = plan_for(&layer("mutation"), report, &base) else { + let Some(Plan::Mutation(plan)) = + plan_for(&layer("mutation"), report, &base, &Filters::default()) + else { panic!("mutation has no plan"); }; assert_eq!(plan.diff_path, report.join("rust-tops.diff")); @@ -448,7 +455,6 @@ mod tests { #[test] fn unmapped_layers_have_no_plan() { for id in [ - "acceptance", "fuzz", "miri", "sanitizer", @@ -458,7 +464,13 @@ mod tests { "laws", ] { assert!( - plan_for(&layer(id), Path::new("/r"), &Base::unused()).is_none(), + plan_for( + &layer(id), + Path::new("/r"), + &Base::unused(), + &Filters::default() + ) + .is_none(), "{id}" ); } diff --git a/crates/cargo-tops/src/gate/plan.rs b/crates/cargo-tops/src/gate/plan.rs index df0fba8..a5c9a57 100644 --- a/crates/cargo-tops/src/gate/plan.rs +++ b/crates/cargo-tops/src/gate/plan.rs @@ -35,12 +35,12 @@ pub(crate) struct TestCommands { } impl TestCommands { - pub(crate) fn new(enabled: &[&config::Layer], report: &Path) -> Self { + pub(crate) fn new(enabled: &[&config::Layer], report: &Path, filters: &Filters) -> Self { let argv = |id: &str| { enabled .iter() .find(|layer| layer.id == id) - .and_then(|layer| tool_plan(layer, report)) + .and_then(|layer| tool_plan(layer, report, filters)) .and_then(|plan| match plan { Plan::Chains(chains) => chains .into_iter() @@ -163,20 +163,6 @@ const TOOL_LAYERS: &[(&str, &[&[&str]])] = &[ "doc", &[&["cargo", "test", "--doc", "--workspace", "--all-features"]], ), - ( - "property", - &[&[ - "cargo", - "nextest", - "run", - "--workspace", - "--all-features", - "--profile", - "ci", - "--no-tests=fail", - "property", - ]], - ), ( "security-advisory", &[&["cargo", "deny", "check", "advisories"]], @@ -184,6 +170,54 @@ const TOOL_LAYERS: &[(&str, &[&[&str]])] = &[ ("formal-kani", &[&["cargo", "kani", "--workspace"]]), ]; +/// The name prefixes `property` and `acceptance` select tests by. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Filters { + pub(crate) property: String, + pub(crate) acceptance: String, +} + +impl Default for Filters { + fn default() -> Self { + Self { + property: "property".to_string(), + acceptance: "acceptance".to_string(), + } + } +} + +impl Filters { + pub(crate) fn of(gates: &config::Gates) -> Self { + Self { + property: gates.test_filter("property"), + acceptance: gates.test_filter("acceptance"), + } + } + + /// The prefix of `layer`, for the two layers that select by name. + pub(crate) fn for_layer(&self, layer: &str) -> Option<&str> { + match layer { + "property" => Some(&self.property), + "acceptance" => Some(&self.acceptance), + _ => None, + } + } +} + +/// The shared nextest run, narrowed to the tests with a `::`-separated name +/// segment that starts with `filter` (HN-L2, a word of `[A-Za-z0-9_]`). The same rule reads the JUnit +/// report of a credited run, so credit equals running. +fn selected_tests(filter: &str) -> Vec { + // `check` holds a filter to `[A-Za-z0-9_]+`, which is a regex literal. + let expr = format!("test(/(^|::){filter}/)"); + let argv = NEXTEST.iter().map(|arg| (*arg).to_string()).chain([ + "--no-tests=fail".to_string(), + "-E".to_string(), + expr, + ]); + vec![Step::tool(argv)] +} + /// The test run `unit` and `integration` share. const NEXTEST: [&str; 7] = [ "cargo", @@ -217,20 +251,28 @@ fn crap_steps(report: &Path) -> Vec { /// `report` is [`laws::artifact_dir`]: the coverage LCOV, the CRAP report /// and the mutation diff are written there. `base` is where the mutation /// diff starts. -pub(crate) fn plan_for(layer: &config::Layer, report: &Path, base: &Base) -> Option { +pub(crate) fn plan_for( + layer: &config::Layer, + report: &Path, + base: &Base, + filters: &Filters, +) -> Option { if layer.id == "mutation" { Some(Plan::Mutation(MutationPlan::new(report, base.clone()))) } else { - tool_plan(layer, report) + tool_plan(layer, report, filters) } } /// The plan of every layer but `mutation`, which also needs a base. -fn tool_plan(layer: &config::Layer, report: &Path) -> Option { +fn tool_plan(layer: &config::Layer, report: &Path, filters: &Filters) -> Option { let id = layer.id.as_str(); match id { "coverage" => Some(Plan::Chains(vec![steps::coverage_steps(report)])), "crap" => Some(Plan::Chains(vec![crap_steps(report)])), + "property" | "acceptance" => filters + .for_layer(id) + .map(|filter| Plan::Chains(vec![selected_tests(filter)])), _ => TOOL_LAYERS .iter() .find(|(layer_id, _)| *layer_id == id) @@ -270,6 +312,8 @@ pub(crate) struct GatePlan<'a> { /// `report.json` live here, as an absolute path. pub(crate) report: PathBuf, pub(crate) tests: TestCommands, + /// The name prefixes of `property` and `acceptance`. + pub(crate) filters: Filters, pub(crate) crap_pin: Option, pub(crate) units: Vec>, } @@ -346,13 +390,17 @@ fn units_of<'a>( tests: &TestCommands, report: &Path, base: &Base, + filters: &Filters, ) -> Vec> { enabled .iter() .copied() .filter(|layer| layer.id != "laws") .map(|layer| { - let what = match (tests.shared_from(&layer.id), plan_for(layer, report, base)) { + let what = match ( + tests.shared_from(&layer.id), + plan_for(layer, report, base, filters), + ) { (Some(from), _) => What::Shared(from), (None, Some(Plan::Chains(chains))) => What::Chains(chains), (None, Some(Plan::Mutation(plan))) => What::Mutation(plan), @@ -381,7 +429,8 @@ pub(crate) fn build<'a>( ))); } let report = report_dir(dir)?; - let tests = TestCommands::new(&enabled, &report); + let filters = Filters::of(&cfg.gates); + let tests = TestCommands::new(&enabled, &report, &filters); let base = base_for(&enabled, dir, base_flag, cfg.gates.base_ref.as_deref()); let crap_pin = if enabled.iter().any(|layer| layer.id == "crap") { tool_pin(dir, "cargo-crap").map_err(Fault::config)? @@ -396,17 +445,18 @@ pub(crate) fn build<'a>( above: ids(cfg, |l| l.enabled && crate::tiers::tier_for(l) > max_tier), // A layer turned off still shows in the run: it is not run, not omitted. disabled: ids(cfg, |l| !l.enabled), - units: units_of(&enabled, &tests, &report, &base), + units: units_of(&enabled, &tests, &report, &base, &filters), enabled, report, tests, + filters, crap_pin, }) } #[cfg(test)] mod tests { - use super::{base_for, GatePlan, TestCommands}; + use super::{base_for, plan_for, Filters, GatePlan, Plan, TestCommands}; use crate::config::Layer; use crate::steps::{Base, BaseSource}; use std::path::PathBuf; @@ -451,6 +501,43 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } + #[test] + fn the_selecting_layers_run_only_tests_with_a_name_prefix() { + let filters = Filters { + property: "prop_".to_string(), + acceptance: "journey_9".to_string(), + }; + let shared = ["cargo", "nextest", "run", "--workspace", "--all-features"] + .into_iter() + .chain(["--profile", "ci", "--no-tests=fail", "-E"]); + for (id, expr) in [ + ("property", "test(/(^|::)prop_/)"), + ("acceptance", "test(/(^|::)journey_9/)"), + ] { + let Some(Plan::Chains(chains)) = + plan_for(&layer(id), &PathBuf::new(), &Base::unused(), &filters) + else { + panic!("{id} has no plan"); + }; + let argv: Vec = chains + .into_iter() + .flatten() + .flat_map(|step| step.argv) + .collect(); + let want: Vec = shared.clone().chain([expr]).map(str::to_string).collect(); + assert_eq!(argv, want, "{id}"); + } + } + + #[test] + fn only_the_two_selecting_layers_have_a_filter() { + let filters = Filters::default(); + assert_eq!(filters.for_layer("property"), Some("property")); + assert_eq!(filters.for_layer("acceptance"), Some("acceptance")); + assert_eq!(filters.for_layer("unit"), None); + assert_eq!(filters.for_layer("integration"), None); + } + #[test] fn a_layer_with_no_unit_in_the_plan_has_no_commands() { let plan = GatePlan { @@ -466,6 +553,7 @@ mod tests { unit: Vec::new(), integration: Vec::new(), }, + filters: super::Filters::default(), crap_pin: None, units: Vec::new(), }; diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index 5397d9c..279cde0 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -410,16 +410,12 @@ fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { done.seconds(id), version.as_deref(), ); - let unmapped = plan - .units - .iter() - .any(|unit| unit.layer.id == id && matches!(unit.what, What::NotRun)); - if unmapped { - entry.insert("detail".into(), json!("enabled and no command is mapped")); - } if let Some(numbers) = numbers { entry.insert("judged".into(), numbers); } + if let Some(detail) = layer_detail(plan, done, id) { + entry.insert("detail".into(), json!(detail)); + } if id == "mutation" { name_the_base(plan, done, &mut entry); } @@ -448,11 +444,7 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map json!({ "ref": null, "source": null, "tried": tried }), }; put_judged(entry, "base", base); - let ran = done - .runs - .iter() - .find(|run| run.layer == "mutation") - .and_then(|run| run.note.clone()); + let ran = run_note(done, "mutation"); let planned = (!matches!(mutation.base, Base::Found { .. })) .then(|| steps::stop_note(&mutation.base, Outcome::Vacuous).map(|(_, note)| note)) .flatten(); @@ -461,6 +453,28 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map, done: &Executed<'_>, id: &str) -> Option { + let unmapped = plan + .units + .iter() + .any(|unit| unit.layer.id == id && matches!(unit.what, What::NotRun)); + if unmapped { + return Some("enabled and no command is mapped".to_string()); + } + run_note(done, id) +} + +/// The note `layer`'s run kept: why it stopped, in the words the report and +/// the failure line both use. +fn run_note(done: &Executed<'_>, layer: &str) -> Option { + done.runs + .iter() + .find(|run| run.layer == layer) + .and_then(|run| run.note.clone()) +} + /// Set `key` in the row's `judged` object, keeping what else is there. fn put_judged(entry: &mut Map, key: &str, value: Value) { let judged = entry diff --git a/crates/cargo-tops/src/laws.rs b/crates/cargo-tops/src/laws.rs index 658b894..7a6b194 100644 --- a/crates/cargo-tops/src/laws.rs +++ b/crates/cargo-tops/src/laws.rs @@ -58,9 +58,13 @@ pub(crate) fn artifact_dir(project: &Path) -> PathBuf { resolve_report_dir(project, from_env.as_deref(), meta.as_deref()) } +/// What one `cargo metadata` said: the target directory, whether it could +/// not run, and the workspace root. +type Metadata = (Option, bool, Option); + thread_local! { /// The target directory `cargo metadata` named, and whether it could not run. - static META_CACHE: RefCell, bool)>> = RefCell::new(HashMap::new()); + static META_CACHE: RefCell> = RefCell::new(HashMap::new()); #[cfg(test)] static META_SPAWNS: Cell = const { Cell::new(0) }; } @@ -69,7 +73,7 @@ fn metadata_target_once(project: &Path) -> Option { metadata_cached(project).0 } -fn metadata_cached(project: &Path) -> (Option, bool) { +fn metadata_cached(project: &Path) -> Metadata { let key = project .canonicalize() .unwrap_or_else(|_| project.to_path_buf()); @@ -126,6 +130,13 @@ pub(crate) fn metadata_unavailable(project: &Path) -> bool { !project.join("Cargo.toml").is_file() || metadata_cached(project).1 } +/// The workspace root `cargo metadata` named for `project`, from the same +/// single `cargo metadata` the report directory uses. nextest keeps its +/// store under this root. +pub(crate) fn workspace_root(project: &Path) -> Option { + metadata_cached(project).2.map(PathBuf::from) +} + #[cfg(test)] fn metadata_target(project: &Path) -> Option { metadata_probe(project).0 @@ -133,10 +144,10 @@ fn metadata_target(project: &Path) -> Option { /// The target directory, and whether `cargo metadata` could not run or /// failed. Output that names no target directory is not a failure. -fn metadata_probe(project: &Path) -> (Option, bool) { +fn metadata_probe(project: &Path) -> Metadata { let manifest = project.join("Cargo.toml"); if !manifest.is_file() { - return (None, false); + return (None, false, None); } #[cfg(test)] META_SPAWNS.with(|count| count.set(count.get() + 1)); @@ -151,16 +162,19 @@ fn metadata_probe(project: &Path) -> (Option, bool) { .arg(&manifest) .output(); let Ok(output) = output else { - return (None, true); + return (None, true, None); }; if !output.status.success() { - return (None, true); + return (None, true, None); } - let target = String::from_utf8(output.stdout) + let doc = String::from_utf8(output.stdout) .ok() - .and_then(|text| serde_json::from_str::(&text).ok()) - .and_then(|value| value.get("target_directory")?.as_str().map(str::to_string)); - (target, false) + .and_then(|text| serde_json::from_str::(&text).ok()); + let field = |key: &str| { + doc.as_ref() + .and_then(|value| value.get(key)?.as_str().map(str::to_string)) + }; + (field("target_directory"), false, field("workspace_root")) } /// The `bend` binary that reports the pinned version. @@ -806,6 +820,31 @@ mod tests { ); } + #[test] + fn the_workspace_root_is_read_from_the_same_metadata() { + let dir = std::env::temp_dir().join(format!("cargo-tops-root-{}", std::process::id())); + #[expect( + clippy::let_underscore_must_use, + reason = "already in this crate when the strict table was turned on" + )] + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("src")).unwrap(); + fs::write( + dir.join("Cargo.toml"), + "[package]\nname = \"root-fixture\"\nversion = \"0.0.0\"\nedition = \"2021\"\n", + ) + .unwrap(); + fs::write(dir.join("src/lib.rs"), "// fixture\n").unwrap(); + let root = workspace_root(&dir).expect("cargo metadata"); + assert_eq!(root, dir); + assert_eq!(workspace_root(&dir.join("absent")), None); + #[expect( + clippy::let_underscore_must_use, + reason = "already in this crate when the strict table was turned on" + )] + let _ = fs::remove_dir_all(&dir); + } + #[test] fn metadata_target_reads_target_directory() { let dir = std::env::temp_dir().join(format!("cargo-tops-meta-{}", std::process::id())); diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index cda1415..4ad0711 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -24,6 +24,7 @@ mod config; mod coverage_floor; mod crap_ratchet; mod credit; +mod evidence; mod exemptions; mod facts; mod floors; diff --git a/crates/cargo-tops/templates/rust-tops.schema.json b/crates/cargo-tops/templates/rust-tops.schema.json index d70f0d0..cbfe42b 100644 --- a/crates/cargo-tops/templates/rust-tops.schema.json +++ b/crates/cargo-tops/templates/rust-tops.schema.json @@ -221,6 +221,21 @@ "required_on_classes": { "type": "array", "items": { "type": "string" } + }, + "test_filter": { + "type": "string", + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." + } + } + }, + "acceptance": { + "type": "object", + "properties": { + "test_filter": { + "type": "string", + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." } } }, diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs new file mode 100644 index 0000000..5ee6b15 --- /dev/null +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -0,0 +1,483 @@ +//! A credited `property` or `acceptance` layer passes only when the credited +//! run's nextest JUnit report holds a passed test with a `::` segment of its +//! name starting with the layer's `test_filter` (PROTOCOL.md §6, §9). +//! +//! Cargo and its plugins are stubs. The cargo stub plays the coverage run: +//! it writes the JUnit report the test put in the fixture's `reports/` to +//! `target/nextest/ci/junit.xml` in the repo, the way nextest does, and +//! writes no report when the test left none. Only the layers the protocol +//! requires of a library-core instance, plus the one under test, are on. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::{json, Value}; + +/// Answers `crap --version` and `metadata`, copies the stand-in reports the +/// way the floors tests do, and, for `llvm-cov nextest`, plays nextest: the +/// JUnit report in `$REPORTS/junit.xml` goes to the path nextest uses. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json"; exit 0; fi +if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/coverage-fails" ]; then exit 1; fi +if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/junit.xml" ]; then + mkdir -p "$REPORTS/../repo/target/nextest/ci" + cp "$REPORTS/junit.xml" "$REPORTS/../repo/target/nextest/ci/junit.xml" +fi +prev= +for arg in "$@"; do + case $prev in + --output-path | --output) + mkdir -p "$(dirname "$arg")" + name=$(basename "$arg") + if [ -f "$REPORTS/$name" ]; then cp "$REPORTS/$name" "$arg"; else printf 'lcov\n' > "$arg"; fi + ;; + esac + prev=$arg +done +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + reports: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +/// A library-core instance named `fixture` with coverage, the layers the +/// protocol requires of the class, and `extra` on, and coverage's JUnit +/// report `junit` (or none). +fn fixture(name: &str, extra: &[&str], junit: Option<&str>) -> Fixture { + let root = + std::env::temp_dir().join(format!("cargo-tops-credit-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + reports: root.join("reports"), + root, + }; + for dir in [&fixture.bin, &fixture.reports] { + fs::create_dir_all(dir).expect("mkdir"); + } + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + stage_tools_and_reports(&fixture, junit); + enable_only(&fixture, extra); + fixture +} + +/// The tool stubs, the coverage summary, and coverage's JUnit report. +fn stage_tools_and_reports(fixture: &Fixture, junit: Option<&str>) { + stub(&fixture.bin, "cargo", CARGO_STUB); + for plugin in [ + "cargo-llvm-cov", + "cargo-crap", + "cargo-nextest", + "cargo-deny", + ] { + stub(&fixture.bin, plugin, "#!/bin/sh\nexit 0\n"); + } + fs::write( + fixture.reports.join("coverage.json"), + json!({"type": "llvm.coverage.json.export", "data": [{"files": [ + {"filename": "src/lib.rs", "summary": { + "lines": {"covered": 100, "count": 100}, + "regions": {"covered": 100, "count": 100}}}]}]}) + .to_string(), + ) + .expect("write summary"); + if let Some(xml) = junit { + fs::write(fixture.reports.join("junit.xml"), xml).expect("write junit"); + } +} + +/// Coverage, the required layers and `extra` on; every other layer off. +fn enable_only(fixture: &Fixture, extra: &[&str]) { + edit(fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = + matches!(id, "coverage" | "security-advisory" | "property") || extra.contains(&id); + layer + .as_object_mut() + .expect("a layer is an object") + .insert("enabled".to_string(), Value::Bool(on)); + } + }); +} + +/// Sets `gates..test_filter`. +fn set_filter(fixture: &Fixture, layer: &str, filter: &str) { + edit(fixture, |doc| { + let gates = doc + .get_mut("gates") + .and_then(Value::as_object_mut) + .expect("gates"); + gates + .entry(layer.to_string()) + .or_insert_with(|| json!({})) + .as_object_mut() + .expect("a gate is an object") + .insert("test_filter".to_string(), json!(filter)); + }); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A JUnit report holding these tests, each `(name, passed)`. +fn junit(tests: &[(&str, bool)]) -> String { + let cases: String = tests + .iter() + .map(|&(name, passed)| { + if passed { + format!("") + } else { + format!( + "\ + " + ) + } + }) + .collect(); + format!( + "\n\ + {cases}" + ) +} + +/// The `problems` of the `report.json` the last gate run wrote. +fn problems(fixture: &Fixture) -> Vec { + let report = + fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")).expect("report.json"); + let report: Value = serde_json::from_str(&report).expect("report json"); + report + .get("problems") + .and_then(Value::as_array) + .map(|all| { + all.iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default() +} + +/// The gate, and the verdict and detail `report.json` gives `layer`. +fn gate(fixture: &Fixture, layer: &str) -> (Option, String, String, String) { + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let dirs = std::iter::once(fixture.bin.clone()).chain(std::env::split_paths(&inherited)); + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["gate", "--path"]) + .arg(&fixture.repo) + .env("PATH", std::env::join_paths(dirs).expect("PATH")) + .env("REPORTS", &fixture.reports) + .env_remove("CARGO_TARGET_DIR") + .output() + .expect("spawn cargo-tops"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let report = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("the gate wrote report.json"); + let report: Value = serde_json::from_str(&report).expect("report json"); + let row = report + .get("layers") + .and_then(Value::as_array) + .and_then(|rows| { + rows.iter() + .find(|row| row.get("id").and_then(Value::as_str) == Some(layer)) + }) + .expect("the layer has a row"); + let field = |key: &str| { + row.get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + (out.status.code(), text, field("verdict"), field("detail")) +} + +#[test] +fn a_coverage_run_with_no_property_test_fails_property() { + let fixture = fixture("none", &[], Some(&junit(&[("plain", true)]))); + let (code, text, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail", "{text}"); + assert!(detail.contains("`property`"), "{detail}"); + assert!(detail.contains("target/nextest/ci/junit.xml"), "{detail}"); + // The failure line and `problems` say the same, and never blame coverage. + let problems = problems(&fixture); + assert!( + problems.iter().any(|p| p == &format!("property: {detail}")), + "{problems:?}" + ); + assert!( + !problems.iter().any(|p| p.contains("run it took is")), + "{problems:?}" + ); + assert!( + text.contains(&format!( + "[property] took the coverage run, but it is no credit: {detail}" + )), + "{text}" + ); + assert!(!text.contains("run it took is"), "{text}"); +} + +#[test] +fn a_property_test_in_the_coverage_run_passes_property() { + let fixture = fixture("some", &[], Some(&junit(&[("property_roundtrip", true)]))); + let (code, text, verdict, _) = gate(&fixture, "property"); + assert_eq!(code, Some(0), "{text}"); + assert_eq!(verdict, "pass", "{text}"); +} + +#[test] +fn a_failed_property_test_is_not_evidence() { + let fixture = fixture( + "failed", + &[], + Some(&junit(&[("property_roundtrip", false)])), + ); + let (code, _, verdict, detail) = gate(&fixture, "property"); + assert_ne!(code, Some(0)); + assert_eq!(verdict, "fail"); + assert!(detail.contains("failed or was skipped"), "{detail}"); +} + +#[test] +fn a_missing_junit_report_fails_and_names_the_setting() { + let fixture = fixture("nojunit", &[], None); + let (code, text, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail", "{text}"); + assert!(detail.contains("[profile.ci.junit]"), "{detail}"); + let problems = problems(&fixture); + assert!( + problems.iter().any(|p| p == &format!("property: {detail}")), + "{problems:?}" + ); + assert!( + text.contains("but it is no credit: no JUnit report at"), + "{text}" + ); +} + +#[test] +fn a_failed_coverage_run_fails_property_without_reading_evidence() { + let fixture = fixture("covfail", &[], Some(&junit(&[("property_ok", true)]))); + fs::write(fixture.reports.join("coverage-fails"), "").expect("mark"); + let (code, text, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail"); + // Coverage really failed, so the problem blames it, and no evidence + // text is given for a run that never produced any. + assert!(!detail.contains("JUnit"), "{detail}"); + assert!(problems(&fixture) + .iter() + .any(|p| p.contains("run it took is Failed"))); +} + +#[test] +fn unit_and_integration_need_no_matching_test() { + let fixture = fixture( + "unit", + &["unit", "integration"], + Some(&junit(&[("property_ok", true)])), + ); + for layer in ["unit", "integration"] { + let (_, text, verdict, _) = gate(&fixture, layer); + assert_eq!(verdict, "pass", "{layer}: {text}"); + } +} + +#[test] +fn a_custom_test_filter_is_honoured() { + let tests = junit(&[("prop_roundtrip", true)]); + let fixture = fixture("filter", &[], Some(&tests)); + set_filter(&fixture, "property", "prop_"); + let (code, text, verdict, _) = gate(&fixture, "property"); + assert_eq!(code, Some(0), "{text}"); + assert_eq!(verdict, "pass", "{text}"); + // The default filter finds nothing in a report that names only `prop_`. + set_filter(&fixture, "property", "property"); + let (_, _, verdict, detail) = gate(&fixture, "property"); + assert_eq!(verdict, "fail"); + assert!(detail.contains("`property`"), "{detail}"); +} + +#[test] +fn acceptance_follows_the_same_rule_with_its_own_filter() { + let with = fixture( + "accept-pass", + &["acceptance"], + Some(&junit(&[ + ("property_a", true), + ("acceptance_journey", true), + ])), + ); + let (code, text, verdict, _) = gate(&with, "acceptance"); + assert_eq!(code, Some(0), "{text}"); + assert_eq!(verdict, "pass", "{text}"); + + let without = fixture( + "accept-fail", + &["acceptance"], + Some(&junit(&[("property_a", true)])), + ); + let (code, text, verdict, detail) = gate(&without, "acceptance"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail", "{text}"); + assert!(detail.contains("`acceptance`"), "{detail}"); + + set_filter(&without, "acceptance", "property"); + let (_, _, verdict, _) = gate(&without, "acceptance"); + assert_eq!(verdict, "pass", "its own filter, not property's"); +} + +/// `check` or `gate` on the fixture: the exit code and what it printed. +fn run_tops(fixture: &Fixture, command: &str) -> (Option, String) { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([command, "--path"]) + .arg(&fixture.repo) + .output() + .expect("spawn cargo-tops"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stderr), + String::from_utf8_lossy(&out.stdout) + ); + (out.status.code(), text) +} + +#[test] +fn a_test_filter_outside_the_name_characters_is_refused_with_exit_2() { + let fixture = fixture("badfilter", &[], None); + for (layer, filter) in [ + ("property", "p::prop"), + ("property", "a.b"), + ("property", "a b"), + ("property", "a/b"), + ("property", "\\"), + ("property", ""), + ("acceptance", "x::"), + ] { + set_filter(&fixture, layer, filter); + for command in ["check", "gate"] { + let (code, text) = run_tops(&fixture, command); + assert_eq!(code, Some(2), "{command} {layer} {filter:?}: {text}"); + assert!( + text.contains("test_filter"), + "{command} {layer} {filter:?}: {text}" + ); + } + set_filter(&fixture, layer, "prop_09"); + } + let (code, text) = run_tops(&fixture, "check"); + assert_eq!(code, Some(0), "{text}"); +} + +/// The kit holds the property tests its own `property` layer credits. A +/// test cannot run the plant (remove `tests/property.rs` and the example +/// crate's `property_*` tests, then gate: `property` fails; restore only +/// `tests/property.rs`: it passes). It guards what the plant relies on: the +/// file has at least three proptest properties, each starting with +/// `property`, and no other test in this crate starts a name segment with +/// `property`, so the file is what the layer's evidence rests on here. +#[test] +fn the_kit_has_at_least_three_property_tests() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let source = + fs::read_to_string(root.join("tests/property.rs")).expect("the kit's tests/property.rs"); + let named = source.matches("fn property_").count(); + assert!(named >= 3, "{named} property tests"); + assert!(source.contains("proptest!"), "they use proptest"); + assert!(manifest_has_proptest(root), "proptest is a dev-dependency"); + let strays = tests_named_property_elsewhere(root); + assert!( + strays.is_empty(), + "other kit tests start with property: {strays:?}" + ); +} + +fn manifest_has_proptest(root: &Path) -> bool { + fs::read_to_string(root.join("Cargo.toml")).is_ok_and(|text| text.contains("proptest")) +} + +/// `#[test]` functions in this crate, outside `tests/property.rs`, whose +/// name starts with `property`. +fn tests_named_property_elsewhere(root: &Path) -> Vec { + let mut found = Vec::new(); + for dir in ["src", "tests"] { + let mut stack = vec![root.join(dir)]; + while let Some(path) = stack.pop() { + if path.is_dir() { + stack.extend( + fs::read_dir(&path) + .expect("read dir") + .flatten() + .map(|e| e.path()), + ); + } else if path.extension().is_some_and(|ext| ext == "rs") + && path.file_name().is_some_and(|name| name != "property.rs") + { + let text = fs::read_to_string(&path).unwrap_or_default(); + let mut is_test = false; + for line in text.lines().map(str::trim_start) { + if line.starts_with("fn property") && is_test { + found.push(format!("{}: {}", path.display(), line)); + } + if line.starts_with("#[test]") { + is_test = true; + } else if !line.starts_with("#[") { + is_test = false; + } + } + } + } + } + found +} diff --git a/crates/cargo-tops/tests/gate_credit_nextest.rs b/crates/cargo-tops/tests/gate_credit_nextest.rs new file mode 100644 index 0000000..231548b --- /dev/null +++ b/crates/cargo-tops/tests/gate_credit_nextest.rs @@ -0,0 +1,327 @@ +//! The credited run is real nextest. The cargo stub plays `cargo llvm-cov +//! nextest` by running `cargo nextest run --profile ci` on a small crate, so +//! the JUnit report is written where nextest writes it, not where the gate +//! looks. These tests cover a `CARGO_TARGET_DIR` that moves the build, a +//! `[store] dir` that moves nextest's reports, and a report left behind by an +//! earlier run (PROTOCOL.md §9.5). +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::Value; + +/// `llvm-cov nextest` writes the stand-in LCOV, then runs real nextest when +/// `$REAL_NEXTEST` is set. `llvm-cov report` writes a fully covered summary. +/// `cargo metadata` is the real cargo; any other subcommand passes. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ "$1 $2" = "llvm-cov nextest" ]; then + prev= + for arg in "$@"; do + case $prev in --output-path) mkdir -p "$(dirname "$arg")"; printf 'lcov\n' > "$arg" ;; esac + prev=$arg + done + if [ -n "${REAL_NEXTEST:-}" ]; then + PATH="$REAL_PATH" "$REAL_CARGO" nextest run --workspace --all-features --profile ci >&2 || exit 1 + fi + exit 0 +fi +if [ "$1 $2" = "llvm-cov report" ]; then + prev= + for arg in "$@"; do + case $prev in --output-path) mkdir -p "$(dirname "$arg")"; printf '%s\n' '{"data":[{"files":[{"filename":"src/lib.rs","summary":{"lines":{"count":10,"covered":10},"regions":{"count":10,"covered":10}}}]}]}' > "$arg" ;; esac + prev=$arg + done + exit 0 +fi +# Only `cargo metadata` is real. Any other subcommand (`cargo deny`, which +# the runners do not have) passes, so no other layer can fail this gate. +if [ "$1" = metadata ]; then + PATH="$REAL_PATH" exec "$REAL_CARGO" "$@" +fi +exit 0 +"#; + +const WITH_TEST: &str = + "#[cfg(test)]\nmod tests {\n #[test]\n fn property_roundtrip() {}\n}\n"; +const WITHOUT_TEST: &str = "#[cfg(test)]\nmod tests {\n #[test]\n fn it_works() {}\n}\n"; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +/// A real one-crate repo: coverage, `security-advisory` and `property` on. +fn fixture(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-nx-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + set_tests(&fixture, WITH_TEST); + stub(&fixture.bin, "cargo", CARGO_STUB); + for plugin in ["cargo-llvm-cov", "cargo-deny"] { + stub(&fixture.bin, plugin, "#!/bin/sh\nexit 0\n"); + } + enable_required_layers(&fixture); + fixture +} + +/// Coverage, `security-advisory` and `property` on; every other layer off. +fn enable_required_layers(fixture: &Fixture) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + for layer in doc + .pointer_mut("/layers") + .and_then(Value::as_array_mut) + .expect("layers") + { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!(id, "coverage" | "security-advisory" | "property"); + layer + .as_object_mut() + .expect("a layer is an object") + .insert("enabled".to_string(), Value::Bool(on)); + } + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +fn set_tests(fixture: &Fixture, source: &str) { + fs::write(fixture.repo.join("src/lib.rs"), source).expect("write lib.rs"); +} + +fn append_nextest_config(fixture: &Fixture, text: &str) { + let path = fixture.repo.join(".config/nextest.toml"); + let old = fs::read_to_string(&path).expect("init writes .config/nextest.toml"); + fs::write(path, format!("{old}\n{text}\n")).expect("write nextest.toml"); +} + +/// What one gate run left: the exit code, stdout and stderr, and the +/// `property` row and `problems` of `report.json`. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Credited { + RealNextest, + /// The credited run writes no report. + Idle, +} + +struct Gate { + code: Option, + text: String, + verdict: String, + detail: String, + problems: Vec, +} + +/// The first `cargo` on the inherited `PATH`, before any stub. +fn real_cargo(path: &std::ffi::OsStr) -> PathBuf { + std::env::split_paths(path) + .map(|dir| dir.join("cargo")) + .find(|candidate| candidate.is_file()) + .expect("a real cargo on PATH") +} + +/// Runs the gate. `credited` says whether the credited run is real nextest; +/// `target_dir` is a `CARGO_TARGET_DIR`. +fn gate(fixture: &Fixture, credited: Credited, target_dir: Option<&Path>) -> Gate { + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let dirs = std::iter::once(fixture.bin.clone()).chain(std::env::split_paths(&inherited)); + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.args(["gate", "--path"]) + .arg(&fixture.repo) + .env("PATH", std::env::join_paths(dirs).expect("PATH")) + .env("REAL_PATH", &inherited) + .env("REAL_CARGO", real_cargo(&inherited)) + .env_remove("CARGO_TARGET_DIR") + .env_remove("REAL_NEXTEST"); + if credited == Credited::RealNextest { + cmd.env("REAL_NEXTEST", "1"); + } + if let Some(dir) = target_dir { + cmd.env("CARGO_TARGET_DIR", dir); + } + let out = cmd.output().expect("spawn cargo-tops"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let report_dir = target_dir.map_or_else(|| fixture.repo.join("target"), Path::to_path_buf); + let (verdict, detail, problems) = read_report(&report_dir); + Gate { + code: out.status.code(), + verdict, + detail, + problems, + text, + } +} + +/// The `property` row's verdict and detail, and the `problems`, of the +/// `report.json` under `report_dir`. +fn read_report(report_dir: &Path) -> (String, String, Vec) { + let report = fs::read_to_string(report_dir.join("rust-tops/report.json")) + .expect("the gate wrote report.json"); + let report: Value = serde_json::from_str(&report).expect("report json"); + let row = report + .get("layers") + .and_then(Value::as_array) + .and_then(|rows| { + rows.iter() + .find(|row| row.get("id").and_then(Value::as_str) == Some("property")) + }) + .expect("a property row"); + let text_of = |key: &str| { + row.get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + let problems = report + .get("problems") + .and_then(Value::as_array) + .map(|all| { + all.iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + (text_of("verdict"), text_of("detail"), problems) +} + +#[test] +fn a_cargo_target_dir_does_not_move_the_evidence() { + let fixture = fixture("target-dir"); + let target = fixture.root.join("elsewhere"); + let run = gate(&fixture, Credited::RealNextest, Some(&target)); + assert_eq!(run.code, Some(0), "{}", run.text); + assert_eq!(run.verdict, "pass", "{}", run.text); + // nextest wrote under the workspace root, not under the target dir. + assert!(fixture.repo.join("target/nextest/ci/junit.xml").is_file()); + assert!(!target.join("nextest").exists()); +} + +#[test] +fn a_moved_store_dir_is_read_where_nextest_wrote_it() { + let fixture = fixture("store"); + append_nextest_config(&fixture, "[store]\ndir = \"target/nt\""); + let run = gate(&fixture, Credited::RealNextest, None); + assert_eq!(run.code, Some(0), "{}", run.text); + assert_eq!(run.verdict, "pass", "{}", run.text); + assert!(fixture.repo.join("target/nt/ci/junit.xml").is_file()); +} + +#[test] +fn a_report_from_an_earlier_run_is_stale_and_kept() { + let fixture = fixture("stale"); + let first = gate(&fixture, Credited::RealNextest, None); + assert_eq!(first.verdict, "pass", "{}", first.text); + let old = fixture.repo.join("target/nextest/ci/junit.xml"); + assert!(old.is_file()); + + // The test goes, the store moves, and the next run writes elsewhere: + // the old report still holds the test, and is not read. + set_tests(&fixture, WITHOUT_TEST); + append_nextest_config(&fixture, "[store]\ndir = \"target/nt\""); + let moved = gate(&fixture, Credited::RealNextest, None); + assert_eq!(moved.code, Some(1), "{}", moved.text); + assert_eq!(moved.verdict, "fail", "{}", moved.text); + assert!( + moved.detail.contains("target/nt/ci/junit.xml"), + "{}", + moved.detail + ); + assert!(moved.detail.contains("no test matches"), "{}", moved.detail); + + // A credited run that writes no report leaves only the old one behind. + let config = fixture.repo.join(".config/nextest.toml"); + let text = fs::read_to_string(&config) + .expect("read") + .replace("target/nt", "target/nextest"); + fs::write(&config, text).expect("write"); + let idle = gate(&fixture, Credited::Idle, None); + assert_eq!(idle.code, Some(1), "{}", idle.text); + assert_eq!(idle.verdict, "fail", "{}", idle.text); + assert!( + idle.detail.contains("target/nextest/ci/junit.xml"), + "{}", + idle.detail + ); + assert!( + idle.detail.contains("not written during the credited run"), + "{}", + idle.detail + ); + assert!(old.is_file(), "the gate never deletes the report"); +} + +#[test] +fn a_failed_evidence_check_is_worded_once_in_detail_problems_and_stdout() { + let fixture = fixture("wording"); + set_tests(&fixture, WITHOUT_TEST); + let run = gate(&fixture, Credited::RealNextest, None); + assert_eq!(run.code, Some(1), "{}", run.text); + let line = "no passed test whose name has a `::` segment starting with `property`"; + assert!(run.detail.contains(line), "{}", run.detail); + let problem = run + .problems + .iter() + .find(|problem| problem.starts_with("property:")) + .expect("a property problem"); + assert!(problem.contains(line), "{problem}"); + assert!(problem.contains("target/nextest/ci/junit.xml"), "{problem}"); + assert!( + run.text.contains(&format!( + "[property] took the coverage run, but it is no credit: {}", + run.detail + )), + "{}", + run.text + ); + assert!(!run.text.contains("run it took is Failed"), "{}", run.text); + assert!( + !run.problems.iter().any(|p| p.contains("run it took is")), + "{:?}", + run.problems + ); +} diff --git a/crates/cargo-tops/tests/gate_floors.rs b/crates/cargo-tops/tests/gate_floors.rs index 8802c2b..2dacac0 100644 --- a/crates/cargo-tops/tests/gate_floors.rs +++ b/crates/cargo-tops/tests/gate_floors.rs @@ -37,6 +37,11 @@ for arg in "$@"; do mkdir -p "$(dirname "$arg")" name=$(basename "$arg") if [ -f "$REPORTS/$name" ]; then cp "$REPORTS/$name" "$arg"; else printf 'lcov\n' > "$arg"; fi + if [ "$1 $2" = "llvm-cov nextest" ]; then + j="$PWD/target/nextest/ci" + mkdir -p "$j" + printf '%s\n' '' > "$j/junit.xml" + fi ;; esac prev=$arg diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 921b4a9..41c0144 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -37,7 +37,10 @@ const INIT_CLASSES: [&str; 8] = [ /// `cargo llvm-cov` and `cargo crap` do. Logs its argv to `$CARGO_STUB_LOG`. /// The file is one JSON document that reads as a coverage summary (one /// fully covered `src/lib.rs`) and as a CRAP report with no function, so -/// the coverage floors and the CRAP ratchet pass. +/// the coverage floors and the CRAP ratchet pass. A `llvm-cov nextest` run +/// also writes the JUnit report nextest writes, holding one passed `property` +/// and one passed `acceptance` test, so a layer that takes the coverage run +/// has the evidence it needs (PROTOCOL.md §6, §9). const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then printf '%s\n' "$*" >> "$CARGO_STUB_LOG" @@ -53,6 +56,11 @@ for arg in "$@"; do --output-path | --output) mkdir -p "$(dirname "$arg")" printf '%s\n' '{"data":[{"files":[{"filename":"src/lib.rs","summary":{"lines":{"count":1,"covered":1},"regions":{"count":1,"covered":1}}}]}],"entries":[]}' > "$arg" + if [ "$1 $2" = "llvm-cov nextest" ]; then + j="$PWD/target/nextest/ci" + mkdir -p "$j" + printf '%s\n' '' > "$j/junit.xml" + fi ;; esac prev=$arg @@ -486,7 +494,7 @@ fn coverage_run(fixture: &Fixture) -> String { ) } const PROPERTY_RUN: &str = - "nextest run --workspace --all-features --profile ci --no-tests=fail property"; + "nextest run --workspace --all-features --profile ci --no-tests=fail -E test(/(^|::)property/)"; /// `laws evaluated: N/53` from gate output. fn laws_evaluated(text: &str) -> (usize, usize) { @@ -1413,7 +1421,7 @@ fn below_coverage_unit_runs_once_integration_takes_it_and_property_runs_itself() "{text}" ); assert!( - text.contains("\n==> [property] cargo nextest run --workspace --all-features --profile ci --no-tests=fail property\n"), + text.contains("\n==> [property] cargo nextest run --workspace --all-features --profile ci --no-tests=fail -E test(/(^|::)property/)\n"), "{text}" ); assert!(!text.contains("==> [coverage]"), "{text}"); @@ -1451,12 +1459,12 @@ fn below_coverage_unit_runs_once_integration_takes_it_and_property_runs_itself() fn a_property_filter_that_matches_nothing_fails() { // nextest exits 4 when `--no-tests=fail` finds no test; the stub does too. let fixture = fixture_with("one-run-empty-filter", &["property"]); - stub_cargo_failing(&fixture, "*' --no-tests=fail property'", 4); + stub_cargo_failing(&fixture, "*' --no-tests=fail -E test(/(^|::)property/)'", 4); let (out, text) = gate(&fixture, path_with_only_stubs(&fixture.bin)); assert_eq!(out.status.code(), Some(1), "{text}"); assert_eq!(nextest_runs(&fixture), [PROPERTY_RUN], "{text}"); assert!( - text.contains("\n - property: `cargo nextest run --workspace --all-features --profile ci --no-tests=fail property` failed\n"), + text.contains("\n - property: `cargo nextest run --workspace --all-features --profile ci --no-tests=fail -E test(/(^|::)property/)` failed\n"), "{text}" ); assert!(!text.contains("\nok: "), "{text}"); @@ -1634,10 +1642,14 @@ fn a_custom_cargo_target_dir_holds_lcov_crap_and_the_laws_report() { "{name} is not under CARGO_TARGET_DIR: {text}" ); } + // The gate's artifacts stay out of the project `target/`. nextest does + // not follow `CARGO_TARGET_DIR` for its own store, so the stub's JUnit + // report is under the project `target/nextest`, as real nextest's is. assert!( - !fixture.repo.join("target").exists(), + !fixture.repo.join("target/rust-tops").exists(), "the project target/ got an artifact while CARGO_TARGET_DIR was set: {text}" ); + assert!(fixture.repo.join("target/nextest/ci/junit.xml").is_file()); let lcov = report.join("lcov.info"); let log = cargo_log(&fixture); for run in [ @@ -1858,23 +1870,29 @@ fn a_real_diff_runs_mutants_on_the_report_dir_diff() { #[test] fn an_enabled_unmapped_layer_in_range_is_not_run_and_fails() { - // `init` writes `acceptance` off. Turned on, it is in range at tier 1 - // and has no command: NotRun, and the gate exits 1. No exemption. - let fixture = fixture_with("not-run", &["static-lint", "acceptance", "laws"]); + // `miri` has no command in the gate. Turned on and in range at tier 3 it + // is NotRun, and the gate exits 1. No exemption. (This was `acceptance` + // at tier 1 until protocol 1.4.0 gave that layer `property`'s command.) + let fixture = fixture_with("not-run", &["static-lint", "miri", "laws"]); adopt(&fixture, Origin::None); stub_bend(&fixture); - let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); + let (out, text) = gate_with( + &fixture, + path_without_bend(&fixture.bin), + None, + &["--max-tier", "3"], + ); assert_eq!(out.status.code(), Some(1), "{text}"); assert!( - text.contains("NotRun [acceptance]: enabled at tier 1 (<= --max-tier 1)"), + text.contains("NotRun [miri]: enabled at tier 3 (<= --max-tier 3)"), "{text}" ); assert!( - text.contains(" - acceptance: NotRun (enabled at tier 1, no command mapped"), + text.contains(" - miri: NotRun (enabled at tier 3, no command mapped"), "{text}" ); - assert!(!text.contains("skip acceptance"), "{text}"); - assert!(text.contains("none ran for: acceptance"), "{text}"); + assert!(!text.contains("skip miri"), "{text}"); + assert!(text.contains("none ran for: miri"), "{text}"); let yaml = fs::read_to_string(fixture.repo.join("rust-tops.yaml")).expect("yaml"); let doc: serde_json::Value = serde_saphyr::from_str(&yaml).expect("yaml parses"); #[expect( @@ -1889,7 +1907,7 @@ fn an_enabled_unmapped_layer_in_range_is_not_run_and_fails() { &fixture, path_without_bend(&fixture.bin), None, - &["--dry-run"], + &["--max-tier", "3", "--dry-run"], ); assert_eq!(out.status.code(), Some(1), "{text}"); assert!( diff --git a/crates/cargo-tops/tests/property.rs b/crates/cargo-tops/tests/property.rs new file mode 100644 index 0000000..455f3ec --- /dev/null +++ b/crates/cargo-tops/tests/property.rs @@ -0,0 +1,152 @@ +//! Properties of two rules in `PROTOCOL.md`, written from the spec and not +//! from the code: an exemption `expires` is a real `YYYY-MM-DD` day and a +//! record counts only until that day (§19), and an exemption `path` glob +//! matches a path the way the exclude globs do (§19, `scope`). +//! +//! The crate is a binary, so this file compiles the three leaf modules it +//! tests from `src/`. Every test name holds `property`: that is what the +//! kit's `property` layer asks nextest for, and it is how the gate knows the +//! layer ran a test and not just the coverage run. + +#[path = "../src/exemptions.rs"] +mod exemptions; +#[path = "../src/scope.rs"] +mod scope; +#[path = "../src/steps.rs"] +mod steps; + +use exemptions::{civil_date, Exemption}; +use proptest::prelude::*; +use scope::glob_matches; + +/// The days in `month` of `year`, from the Gregorian rule, not from the code. +fn days_in(year: u32, month: u32) -> u32 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + _ if year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400)) => { + 29 + } + _ => 28, + } +} + +/// A path component with no glob character, from a small alphabet so a +/// component can repeat. +fn component() -> impl Strategy { + "[a-c0-9_.]{1,4}" +} + +proptest! { + // A date is valid exactly when its month and day name a real day. + #[test] + fn property_an_expiry_is_valid_exactly_when_it_is_a_real_day( + year in 1u32..=9999, + month in 0u32..=13, + day in 0u32..=32, + ) { + let text = format!("{year:04}-{month:02}-{day:02}"); + let real = (1..=12).contains(&month) && (1..=days_in(year, month)).contains(&day); + prop_assert_eq!(steps::valid_expiry(&text), real, "{}", text); + } + + // Text that is not `YYYY-MM-DD` is never a day. + #[test] + fn property_a_text_that_is_not_the_date_shape_is_never_a_day( + text in "[0-9a-z/ .-]{0,12}", + ) { + let shaped = text.len() == 10 + && text.char_indices().all(|(at, ch)| { + if at == 4 || at == 7 { ch == '-' } else { ch.is_ascii_digit() } + }); + if !shaped { + prop_assert!(!steps::valid_expiry(&text), "{}", text); + } + } + + // Every day the calendar counts is a real day, and a later day sorts + // after an earlier one, which is how an expiry is compared with today. + #[test] + fn property_the_calendar_gives_real_days_in_order(days in 0u64..2_000_000) { + let today = civil_date(days); + let tomorrow = civil_date(days + 1); + prop_assert!(steps::valid_expiry(&today), "{}", today); + prop_assert!(tomorrow > today, "{} then {}", today, tomorrow); + } + + // The record in §19 counts until its expiry day and not on or after it. + #[test] + fn property_an_exemption_is_live_exactly_before_its_expiry_day( + expiry in 0u64..2_000_000, + today in 0u64..2_000_000, + ) { + let record = Exemption { + path: "src/**".to_string(), + symbol: None, + expires: civil_date(expiry), + }; + prop_assert_eq!(record.live(&civil_date(today)), expiry > today); + } + + // A glob made of a path's own components matches that path. + #[test] + fn property_a_glob_equal_to_a_path_matches_it( + parts in prop::collection::vec(component(), 1..5), + ) { + let path = parts.join("/"); + prop_assert!(glob_matches(&path, &path), "{}", path); + } + + // A glob also matches the tail of a path that starts at a directory. + #[test] + fn property_a_glob_matches_a_path_that_ends_in_it( + dirs in prop::collection::vec(component(), 0..4), + glob_parts in prop::collection::vec(component(), 1..4), + ) { + let glob = glob_parts.join("/"); + let path = dirs.iter().chain(glob_parts.iter()).cloned().collect::>().join("/"); + prop_assert!(glob_matches(&glob, &path), "{} in {}", glob, path); + } + + // `**` stands for any number of directories. + #[test] + fn property_a_double_star_matches_any_depth( + dirs in prop::collection::vec(component(), 0..5), + leaf in "[a-c]{1,4}", + ) { + let path = dirs.iter().cloned().chain(std::iter::once(leaf.clone())) + .collect::>().join("/"); + let glob = format!("**/{leaf}"); + prop_assert!(glob_matches(&glob, &path), "{}", path); + prop_assert!(glob_matches("**", &path), "{}", path); + } + + // `*` is any run inside one component and never crosses a `/`. The + // alphabets differ so `dir` cannot also start a later part of the path. + #[test] + fn property_a_star_stays_inside_one_path_component( + dir in "d[a-c]{1,3}", + first in "x[a-c]{0,3}", + second in "x[a-c]{0,3}", + ) { + let glob = format!("{dir}/*"); + let one = format!("{dir}/{first}"); + let two = format!("{dir}/{first}/{second}"); + prop_assert!(glob_matches(&glob, &one)); + prop_assert!(!glob_matches(&glob, &two)); + } + + // `?` is exactly one byte. + #[test] + fn property_a_question_mark_is_exactly_one_byte( + head in "[a-c]{0,3}", + mid in "[a-z]", + tail in "[a-c]{0,3}", + ) { + let glob = format!("{head}?{tail}"); + let one = format!("{head}{mid}{tail}"); + let none = format!("{head}{tail}"); + prop_assert!(glob_matches(&glob, &one)); + prop_assert!(!glob_matches(&glob, &none)); + } +} diff --git a/schema/rust-tops.schema.json b/schema/rust-tops.schema.json index d70f0d0..cbfe42b 100644 --- a/schema/rust-tops.schema.json +++ b/schema/rust-tops.schema.json @@ -221,6 +221,21 @@ "required_on_classes": { "type": "array", "items": { "type": "string" } + }, + "test_filter": { + "type": "string", + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." + } + } + }, + "acceptance": { + "type": "object", + "properties": { + "test_filter": { + "type": "string", + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." } } },