From af51f6dbdc8ddf96fbec4f301576a160b4a447b5 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 20:50:04 -0500 Subject: [PATCH 1/9] Draft the credit evidence reader, the kit's property tests and the credit oracle evidence.rs reads a credited run's nextest JUnit report and passes a layer only on a passed test whose name holds the layer's test_filter. It is held under credit.rs until the gate wiring lands. tests/property.rs has nine proptest properties on the expiry date and glob rules. gate_credit.rs is the stub-driven oracle for the wiring and is red until then. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 1 + crates/cargo-tops/Cargo.toml | 3 + crates/cargo-tops/src/credit.rs | 4 + crates/cargo-tops/src/evidence.rs | 346 +++++++++++++++++++++++++ crates/cargo-tops/tests/gate_credit.rs | 320 +++++++++++++++++++++++ crates/cargo-tops/tests/property.rs | 152 +++++++++++ 6 files changed, 826 insertions(+) create mode 100644 crates/cargo-tops/src/evidence.rs create mode 100644 crates/cargo-tops/tests/gate_credit.rs create mode 100644 crates/cargo-tops/tests/property.rs diff --git a/Cargo.lock b/Cargo.lock index 4a494b4..3573a5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,7 @@ version = "0.4.0" dependencies = [ "anyhow", "clap", + "proptest", "regex", "rustix", "serde", diff --git a/crates/cargo-tops/Cargo.toml b/crates/cargo-tops/Cargo.toml index ebe1490..ef0e7c9 100644 --- a/crates/cargo-tops/Cargo.toml +++ b/crates/cargo-tops/Cargo.toml @@ -39,5 +39,8 @@ toml = "1" # `openat`/`mkdirat` without `unsafe`. 1.1.6 clears RUSTSEC-2026-0486. rustix = { version = "1.1.6", default-features = false, features = ["fs", "std"] } +[dev-dependencies] +proptest = "1" + [lints] workspace = true diff --git a/crates/cargo-tops/src/credit.rs b/crates/cargo-tops/src/credit.rs index 873c7f0..a3ccc5f 100644 --- a/crates/cargo-tops/src/credit.rs +++ b/crates/cargo-tops/src/credit.rs @@ -8,6 +8,10 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] #![allow(dead_code, reason = "main.rs calls the credit decision")] +// Held here until the gate wiring moves it to main.rs. +#[path = "evidence.rs"] +pub(crate) mod evidence; + /// Nextest profile a coverage command must pass for credit. pub(crate) const NEXTEST_PROFILE: &str = "ci"; diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs new file mode 100644 index 0000000..5fe2f7b --- /dev/null +++ b/crates/cargo-tops/src/evidence.rs @@ -0,0 +1,346 @@ +//! Evidence that a credited layer's tests ran: the credited run's nextest +//! JUnit report. +//! +//! `property` and `acceptance` may take another layer's run (see +//! [`crate::credit`]). That run passes for them only if its JUnit report +//! holds a test whose name contains the layer's `test_filter` and that test +//! passed. This module reads that one file. It spawns nothing and builds +//! nothing, so reading it costs no second test run. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + +use std::fs; +use std::path::{Path, PathBuf}; + +/// The nextest profile whose report is read. +pub(crate) const PROFILE: &str = "ci"; + +/// Where nextest keeps a profile's reports under the target directory. +const STORE: &str = "nextest"; + +/// How many matching tests passed and how many did not. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Tally { + pub passed: usize, + pub failed: usize, +} + +/// Counts the tests in a JUnit report whose name contains `filter`. +/// +/// A test with a `failure`, `error` or `skipped` child did not pass. An empty +/// `filter` matches every test. +#[must_use] +pub(crate) fn tally(xml: &str, filter: &str) -> Tally { + let mut tally = Tally::default(); + for (name, passed) in testcases(xml) { + if !name.contains(filter) { + continue; + } + if passed { + tally.passed += 1; + } else { + tally.failed += 1; + } + } + tally +} + +/// Each `testcase` as its name and whether it passed. +fn testcases(xml: &str) -> Vec<(String, bool)> { + let mut found = Vec::new(); + let mut rest = xml; + while let Some(at) = rest.find("') else { break }; + let tag = from.get(..end).unwrap_or_default(); + let after = from.get(end + 1..).unwrap_or_default(); + let (body, tail) = if tag.ends_with('/') { + ("", after) + } else { + after.split_once("").unwrap_or((after, "")) + }; + if let Some(name) = attribute(tag, "name") { + found.push((name, !has_verdict_child(body))); + } + rest = tail; + } + found +} + +fn has_verdict_child(body: &str) -> bool { + [" Option { + let marker = format!(" {key}=\""); + let value = tag.split_once(&marker)?.1; + Some(unescape(value.split_once('"')?.0)) +} + +fn unescape(text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +/// The `path` of `[profile.ci.junit]` in the text of `.config/nextest.toml`. +#[must_use] +pub(crate) fn junit_name(nextest_toml: &str) -> Option { + let doc: toml::Table = toml::from_str(nextest_toml).ok()?; + doc.get("profile")? + .get(PROFILE)? + .get("junit")? + .get("path")? + .as_str() + .map(str::to_string) +} + +/// The report the credited run wrote: `/nextest/ci/`, with the +/// file name from `.config/nextest.toml` under `root`. +/// +/// # Errors +/// The setting is missing, so the run writes no report to read. +pub(crate) fn junit_path(root: &Path, target: &Path) -> Result { + let config = root.join(".config").join("nextest.toml"); + let name = fs::read_to_string(&config) + .ok() + .and_then(|text| junit_name(&text)); + name.map(|name| target.join(STORE).join(PROFILE).join(name)) + .ok_or_else(|| { + "no JUnit report is configured: set `path` under `[profile.ci.junit]` in \ + .config/nextest.toml so the credited run writes one" + .to_string() + }) +} + +/// Whether the credited run held a passed test matching `filter`. +/// +/// Returns how many matching tests passed. +/// +/// # Errors +/// The detail for a `fail`: it names the filter, and the report path read +/// (relative to `root`) or the `[profile.ci.junit]` setting. +pub(crate) fn credit( + layer: &str, + root: &Path, + target: &Path, + filter: &str, +) -> Result { + let path = junit_path(root, target).map_err(|why| format!("{layer}: {why}"))?; + let shown = path + .strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(); + let xml = fs::read_to_string(&path).map_err(|_| { + format!( + "{layer}: no JUnit report at {shown}; the credited run must write one \ + (`[profile.ci.junit]` in .config/nextest.toml)" + ) + })?; + let found = tally(&xml, filter); + if found.passed > 0 { + return Ok(found.passed); + } + let why = if found.failed > 0 { + "every matching test failed or was skipped" + } else { + "no test matches" + }; + Err(format!( + "{layer}: the credited run's report {shown} holds no passed test whose name contains \ + `{filter}` ({why}); add one, or set gates.{layer}.test_filter" + )) +} + +#[cfg(test)] +mod tests { + use super::{credit, junit_name, junit_path, tally, Tally}; + use std::fs; + use std::path::Path; + + const REPORT: &str = r#" + + + + + boom + a <testcase name="property_fake"/> + + + + fine + + + + +"#; + + #[test] + fn only_a_passed_test_whose_name_contains_the_filter_counts() { + assert_eq!( + tally(REPORT, "property"), + Tally { + passed: 2, + failed: 3 + } + ); + assert_eq!( + tally(REPORT, "plain"), + Tally { + passed: 1, + failed: 0 + } + ); + assert_eq!(tally(REPORT, "acceptance"), Tally::default()); + assert_eq!(tally(REPORT, "Property"), Tally::default()); + assert_eq!(tally("", "property"), Tally::default()); + } + + #[test] + fn escaped_text_inside_a_test_is_not_a_test() { + assert_eq!(tally(REPORT, "property_fake"), Tally::default()); + } + + #[test] + fn an_unclosed_tag_ends_the_scan_without_a_count() { + assert_eq!( + tally(""#; + assert_eq!(tally(open, "property").failed, 1); + } + + #[test] + fn a_name_is_unescaped() { + let xml = r#""#; + assert_eq!(tally(xml, "a&b\"c'd").passed, 1); + } + + #[test] + fn the_report_name_is_the_junit_path_of_the_ci_profile() { + let toml = "[profile.ci]\nretries = 0\n[profile.ci.junit]\npath = \"j.xml\"\n"; + assert_eq!(junit_name(toml).as_deref(), Some("j.xml")); + assert_eq!(junit_name("[profile.ci]\nretries = 0\n"), None); + assert_eq!( + junit_name("[profile.default.junit]\npath = \"j.xml\"\n"), + None + ); + assert_eq!(junit_name("not toml ["), None); + assert_eq!(junit_name("[profile.ci.junit]\npath = 3\n"), None); + } + + fn fixture(config: Option<&str>, report: Option<&str>) -> tempdir::Dir { + let dir = tempdir::Dir::new(); + if let Some(config) = config { + fs::create_dir_all(dir.path().join(".config")).unwrap(); + fs::write(dir.path().join(".config/nextest.toml"), config).unwrap(); + } + if let Some(report) = report { + let at = dir.path().join("target/nextest/ci"); + fs::create_dir_all(&at).unwrap(); + fs::write(at.join("junit.xml"), report).unwrap(); + } + dir + } + + const CONFIG: &str = "[profile.ci.junit]\npath = \"junit.xml\"\n"; + + fn run(dir: &tempdir::Dir, filter: &str) -> Result { + credit("property", dir.path(), &dir.path().join("target"), filter) + } + + #[test] + fn a_passed_matching_test_is_credit() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + assert_eq!(run(&dir, "property"), Ok(2)); + assert_eq!(run(&dir, "plain"), Ok(1)); + } + + #[test] + fn no_match_names_the_filter_and_the_path_read() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + let why = run(&dir, "acceptance").unwrap_err(); + assert!(why.contains("`acceptance`"), "{why}"); + assert!(why.contains("target/nextest/ci/junit.xml"), "{why}"); + assert!(why.contains("no test matches"), "{why}"); + assert!(why.contains("gates.property.test_filter"), "{why}"); + assert!(!why.contains(dir.path().to_str().unwrap_or("?")), "{why}"); + } + + #[test] + fn a_failed_match_is_not_evidence() { + let xml = r#""#; + let dir = fixture(Some(CONFIG), Some(xml)); + let why = run(&dir, "property").unwrap_err(); + assert!(why.contains("failed or was skipped"), "{why}"); + } + + #[test] + fn a_missing_report_names_the_junit_setting() { + let dir = fixture(Some(CONFIG), None); + let why = run(&dir, "property").unwrap_err(); + assert!( + why.contains("no JUnit report at target/nextest/ci/junit.xml"), + "{why}" + ); + assert!(why.contains("[profile.ci.junit]"), "{why}"); + } + + #[test] + fn no_junit_setting_names_the_junit_setting() { + let dir = fixture(None, Some(REPORT)); + let why = run(&dir, "property").unwrap_err(); + assert!( + why.starts_with("property: no JUnit report is configured"), + "{why}" + ); + assert!(why.contains("[profile.ci.junit]"), "{why}"); + let bare = fixture(Some("[profile.ci]\nretries = 0\n"), Some(REPORT)); + assert!(run(&bare, "property").is_err()); + assert!(junit_path(bare.path(), Path::new("t")).is_err()); + } + + #[test] + fn the_file_name_comes_from_the_config() { + let dir = fixture(Some("[profile.ci.junit]\npath = \"out/j.xml\"\n"), None); + let path = junit_path(dir.path(), Path::new("t")).unwrap(); + assert_eq!(path, Path::new("t/nextest/ci/out/j.xml")); + } + + /// A scratch directory removed on drop. It removes only what it made. + mod tempdir { + use std::path::{Path, PathBuf}; + + pub(super) struct Dir(PathBuf); + + impl Dir { + pub(super) fn new() -> Self { + use std::sync::atomic::{AtomicUsize, Ordering}; + static N: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "rt-evidence-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } + + pub(super) fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for Dir { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).ok(); + } + } + } +} diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs new file mode 100644 index 0000000..d16940c --- /dev/null +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -0,0 +1,320 @@ +//! A credited `property` or `acceptance` layer passes only when the credited +//! run's nextest JUnit report holds a passed test whose name contains the +//! layer's `test_filter` (PROTOCOL.md §6, §9). +//! +//! Cargo and its plugins are stubs. The cargo stub plays the coverage run: +//! it writes the JUnit report the test put in the fixture's `reports/` to +//! `target/nextest/ci/junit.xml` in the repo, the way nextest does, and +//! writes no report when the test left none. Only the layers the protocol +//! requires of a library-core instance, plus the one under test, are on. +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::{json, Value}; + +/// Answers `crap --version` and `metadata`, copies the stand-in reports the +/// way the floors tests do, and, for `llvm-cov nextest`, plays nextest: the +/// JUnit report in `$REPORTS/junit.xml` goes to the path nextest uses. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi +if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json"; exit 0; fi +if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/junit.xml" ]; then + mkdir -p "$REPORTS/../repo/target/nextest/ci" + cp "$REPORTS/junit.xml" "$REPORTS/../repo/target/nextest/ci/junit.xml" +fi +prev= +for arg in "$@"; do + case $prev in + --output-path | --output) + mkdir -p "$(dirname "$arg")" + name=$(basename "$arg") + if [ -f "$REPORTS/$name" ]; then cp "$REPORTS/$name" "$arg"; else printf 'lcov\n' > "$arg"; fi + ;; + esac + prev=$arg +done +exit 0 +"#; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, + reports: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +/// A library-core instance named `fixture` with coverage, the layers the +/// protocol requires of the class, and `extra` on, and coverage's JUnit +/// report `junit` (or none). +fn fixture(name: &str, extra: &[&str], junit: Option<&str>) -> Fixture { + let root = + std::env::temp_dir().join(format!("cargo-tops-credit-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + reports: root.join("reports"), + root, + }; + for dir in [&fixture.bin, &fixture.reports] { + fs::create_dir_all(dir).expect("mkdir"); + } + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", + ) + .expect("write Cargo.toml"); + stage_tools_and_reports(&fixture, junit); + enable_only(&fixture, extra); + fixture +} + +/// The tool stubs, the coverage summary, and coverage's JUnit report. +fn stage_tools_and_reports(fixture: &Fixture, junit: Option<&str>) { + stub(&fixture.bin, "cargo", CARGO_STUB); + for plugin in [ + "cargo-llvm-cov", + "cargo-crap", + "cargo-nextest", + "cargo-deny", + ] { + stub(&fixture.bin, plugin, "#!/bin/sh\nexit 0\n"); + } + fs::write( + fixture.reports.join("coverage.json"), + json!({"type": "llvm.coverage.json.export", "data": [{"files": [ + {"filename": "src/lib.rs", "summary": { + "lines": {"covered": 100, "count": 100}, + "regions": {"covered": 100, "count": 100}}}]}]}) + .to_string(), + ) + .expect("write summary"); + if let Some(xml) = junit { + fs::write(fixture.reports.join("junit.xml"), xml).expect("write junit"); + } +} + +/// Coverage, the required layers and `extra` on; every other layer off. +fn enable_only(fixture: &Fixture, extra: &[&str]) { + edit(fixture, |doc| { + let layers = doc.pointer_mut("/layers").and_then(Value::as_array_mut); + for layer in layers.expect("layers") { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = + matches!(id, "coverage" | "security-advisory" | "property") || extra.contains(&id); + layer + .as_object_mut() + .expect("a layer is an object") + .insert("enabled".to_string(), Value::Bool(on)); + } + }); +} + +/// Sets `gates..test_filter`. +fn set_filter(fixture: &Fixture, layer: &str, filter: &str) { + edit(fixture, |doc| { + let gates = doc + .get_mut("gates") + .and_then(Value::as_object_mut) + .expect("gates"); + gates + .entry(layer.to_string()) + .or_insert_with(|| json!({})) + .as_object_mut() + .expect("a gate is an object") + .insert("test_filter".to_string(), json!(filter)); + }); +} + +fn edit(fixture: &Fixture, change: impl FnOnce(&mut Value)) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + change(&mut doc); + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +/// A JUnit report holding these tests, each `(name, passed)`. +fn junit(tests: &[(&str, bool)]) -> String { + let cases: String = tests + .iter() + .map(|&(name, passed)| { + if passed { + format!("") + } else { + format!( + "\ + " + ) + } + }) + .collect(); + format!( + "\n\ + {cases}" + ) +} + +/// The gate, and the verdict and detail `report.json` gives `layer`. +fn gate(fixture: &Fixture, layer: &str) -> (Option, String, String, String) { + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let dirs = std::iter::once(fixture.bin.clone()).chain(std::env::split_paths(&inherited)); + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args(["gate", "--path"]) + .arg(&fixture.repo) + .env("PATH", std::env::join_paths(dirs).expect("PATH")) + .env("REPORTS", &fixture.reports) + .env_remove("CARGO_TARGET_DIR") + .output() + .expect("spawn cargo-tops"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let report = fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")) + .expect("the gate wrote report.json"); + let report: Value = serde_json::from_str(&report).expect("report json"); + let row = report + .get("layers") + .and_then(Value::as_array) + .and_then(|rows| { + rows.iter() + .find(|row| row.get("id").and_then(Value::as_str) == Some(layer)) + }) + .expect("the layer has a row"); + let field = |key: &str| { + row.get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + (out.status.code(), text, field("verdict"), field("detail")) +} + +#[test] +fn a_coverage_run_with_no_property_test_fails_property() { + let fixture = fixture("none", &[], Some(&junit(&[("plain", true)]))); + let (code, text, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail", "{text}"); + assert!(detail.contains("`property`"), "{detail}"); + assert!(detail.contains("target/nextest/ci/junit.xml"), "{detail}"); +} + +#[test] +fn a_property_test_in_the_coverage_run_passes_property() { + let fixture = fixture("some", &[], Some(&junit(&[("property_roundtrip", true)]))); + let (code, text, verdict, _) = gate(&fixture, "property"); + assert_eq!(code, Some(0), "{text}"); + assert_eq!(verdict, "pass", "{text}"); +} + +#[test] +fn a_failed_property_test_is_not_evidence() { + let fixture = fixture( + "failed", + &[], + Some(&junit(&[("property_roundtrip", false)])), + ); + let (code, _, verdict, detail) = gate(&fixture, "property"); + assert_ne!(code, Some(0)); + assert_eq!(verdict, "fail"); + assert!(detail.contains("failed or was skipped"), "{detail}"); +} + +#[test] +fn a_missing_junit_report_fails_and_names_the_setting() { + let fixture = fixture("nojunit", &[], None); + let (code, text, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail", "{text}"); + assert!(detail.contains("[profile.ci.junit]"), "{detail}"); +} + +#[test] +fn a_custom_test_filter_is_honoured() { + let tests = junit(&[("prop_roundtrip", true)]); + let fixture = fixture("filter", &[], Some(&tests)); + set_filter(&fixture, "property", "prop_"); + let (code, text, verdict, _) = gate(&fixture, "property"); + assert_eq!(code, Some(0), "{text}"); + assert_eq!(verdict, "pass", "{text}"); + // The default filter finds nothing in a report that names only `prop_`. + set_filter(&fixture, "property", "property"); + let (_, _, verdict, detail) = gate(&fixture, "property"); + assert_eq!(verdict, "fail"); + assert!(detail.contains("`property`"), "{detail}"); +} + +#[test] +fn acceptance_follows_the_same_rule_with_its_own_filter() { + let with = fixture( + "accept-pass", + &["acceptance"], + Some(&junit(&[ + ("property_a", true), + ("acceptance_journey", true), + ])), + ); + let (code, text, verdict, _) = gate(&with, "acceptance"); + assert_eq!(code, Some(0), "{text}"); + assert_eq!(verdict, "pass", "{text}"); + + let without = fixture( + "accept-fail", + &["acceptance"], + Some(&junit(&[("property_a", true)])), + ); + let (code, text, verdict, detail) = gate(&without, "acceptance"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail", "{text}"); + assert!(detail.contains("`acceptance`"), "{detail}"); + + set_filter(&without, "acceptance", "property"); + let (_, _, verdict, _) = gate(&without, "acceptance"); + assert_eq!(verdict, "pass", "its own filter, not property's"); +} + +/// The kit holds the property tests its own `property` layer credits. This is +/// the cheap half of the plant: removing `tests/property.rs` leaves the kit's +/// coverage run with no test named `property`, and the gate fails the layer. +#[test] +fn the_kit_has_at_least_three_property_tests() { + let source = + fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/property.rs")) + .expect("the kit's tests/property.rs"); + let named = source.matches("fn property_").count(); + assert!(named >= 3, "{named} property tests"); + assert!(source.contains("proptest!"), "they use proptest"); +} diff --git a/crates/cargo-tops/tests/property.rs b/crates/cargo-tops/tests/property.rs new file mode 100644 index 0000000..455f3ec --- /dev/null +++ b/crates/cargo-tops/tests/property.rs @@ -0,0 +1,152 @@ +//! Properties of two rules in `PROTOCOL.md`, written from the spec and not +//! from the code: an exemption `expires` is a real `YYYY-MM-DD` day and a +//! record counts only until that day (§19), and an exemption `path` glob +//! matches a path the way the exclude globs do (§19, `scope`). +//! +//! The crate is a binary, so this file compiles the three leaf modules it +//! tests from `src/`. Every test name holds `property`: that is what the +//! kit's `property` layer asks nextest for, and it is how the gate knows the +//! layer ran a test and not just the coverage run. + +#[path = "../src/exemptions.rs"] +mod exemptions; +#[path = "../src/scope.rs"] +mod scope; +#[path = "../src/steps.rs"] +mod steps; + +use exemptions::{civil_date, Exemption}; +use proptest::prelude::*; +use scope::glob_matches; + +/// The days in `month` of `year`, from the Gregorian rule, not from the code. +fn days_in(year: u32, month: u32) -> u32 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + _ if year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400)) => { + 29 + } + _ => 28, + } +} + +/// A path component with no glob character, from a small alphabet so a +/// component can repeat. +fn component() -> impl Strategy { + "[a-c0-9_.]{1,4}" +} + +proptest! { + // A date is valid exactly when its month and day name a real day. + #[test] + fn property_an_expiry_is_valid_exactly_when_it_is_a_real_day( + year in 1u32..=9999, + month in 0u32..=13, + day in 0u32..=32, + ) { + let text = format!("{year:04}-{month:02}-{day:02}"); + let real = (1..=12).contains(&month) && (1..=days_in(year, month)).contains(&day); + prop_assert_eq!(steps::valid_expiry(&text), real, "{}", text); + } + + // Text that is not `YYYY-MM-DD` is never a day. + #[test] + fn property_a_text_that_is_not_the_date_shape_is_never_a_day( + text in "[0-9a-z/ .-]{0,12}", + ) { + let shaped = text.len() == 10 + && text.char_indices().all(|(at, ch)| { + if at == 4 || at == 7 { ch == '-' } else { ch.is_ascii_digit() } + }); + if !shaped { + prop_assert!(!steps::valid_expiry(&text), "{}", text); + } + } + + // Every day the calendar counts is a real day, and a later day sorts + // after an earlier one, which is how an expiry is compared with today. + #[test] + fn property_the_calendar_gives_real_days_in_order(days in 0u64..2_000_000) { + let today = civil_date(days); + let tomorrow = civil_date(days + 1); + prop_assert!(steps::valid_expiry(&today), "{}", today); + prop_assert!(tomorrow > today, "{} then {}", today, tomorrow); + } + + // The record in §19 counts until its expiry day and not on or after it. + #[test] + fn property_an_exemption_is_live_exactly_before_its_expiry_day( + expiry in 0u64..2_000_000, + today in 0u64..2_000_000, + ) { + let record = Exemption { + path: "src/**".to_string(), + symbol: None, + expires: civil_date(expiry), + }; + prop_assert_eq!(record.live(&civil_date(today)), expiry > today); + } + + // A glob made of a path's own components matches that path. + #[test] + fn property_a_glob_equal_to_a_path_matches_it( + parts in prop::collection::vec(component(), 1..5), + ) { + let path = parts.join("/"); + prop_assert!(glob_matches(&path, &path), "{}", path); + } + + // A glob also matches the tail of a path that starts at a directory. + #[test] + fn property_a_glob_matches_a_path_that_ends_in_it( + dirs in prop::collection::vec(component(), 0..4), + glob_parts in prop::collection::vec(component(), 1..4), + ) { + let glob = glob_parts.join("/"); + let path = dirs.iter().chain(glob_parts.iter()).cloned().collect::>().join("/"); + prop_assert!(glob_matches(&glob, &path), "{} in {}", glob, path); + } + + // `**` stands for any number of directories. + #[test] + fn property_a_double_star_matches_any_depth( + dirs in prop::collection::vec(component(), 0..5), + leaf in "[a-c]{1,4}", + ) { + let path = dirs.iter().cloned().chain(std::iter::once(leaf.clone())) + .collect::>().join("/"); + let glob = format!("**/{leaf}"); + prop_assert!(glob_matches(&glob, &path), "{}", path); + prop_assert!(glob_matches("**", &path), "{}", path); + } + + // `*` is any run inside one component and never crosses a `/`. The + // alphabets differ so `dir` cannot also start a later part of the path. + #[test] + fn property_a_star_stays_inside_one_path_component( + dir in "d[a-c]{1,3}", + first in "x[a-c]{0,3}", + second in "x[a-c]{0,3}", + ) { + let glob = format!("{dir}/*"); + let one = format!("{dir}/{first}"); + let two = format!("{dir}/{first}/{second}"); + prop_assert!(glob_matches(&glob, &one)); + prop_assert!(!glob_matches(&glob, &two)); + } + + // `?` is exactly one byte. + #[test] + fn property_a_question_mark_is_exactly_one_byte( + head in "[a-c]{0,3}", + mid in "[a-z]", + tail in "[a-c]{0,3}", + ) { + let glob = format!("{head}?{tail}"); + let one = format!("{head}{mid}{tail}"); + let none = format!("{head}{tail}"); + prop_assert!(glob_matches(&glob, &one)); + prop_assert!(!glob_matches(&glob, &none)); + } +} From 94242eb5ad6b70f54719787c6fc62339152dd461 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 20:54:46 -0500 Subject: [PATCH 2/9] Match a test when a name segment starts with the filter A credited layer passes on a passed test with a ::-separated name segment that starts with test_filter, so a test that only mentions the filter mid-name is no evidence. The kit guard checks no other kit test starts a name with property. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/evidence.rs | 38 +++++++++++++++--- crates/cargo-tops/tests/gate_credit.rs | 53 +++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs index 5fe2f7b..23f5fc8 100644 --- a/crates/cargo-tops/src/evidence.rs +++ b/crates/cargo-tops/src/evidence.rs @@ -3,7 +3,7 @@ //! //! `property` and `acceptance` may take another layer's run (see //! [`crate::credit`]). That run passes for them only if its JUnit report -//! holds a test whose name contains the layer's `test_filter` and that test +//! holds a test whose name matches the layer's `test_filter` (see [`matches`]) and that test //! passed. This module reads that one file. It spawns nothing and builds //! nothing, so reading it costs no second test run. #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] @@ -24,7 +24,16 @@ pub(crate) struct Tally { pub failed: usize, } -/// Counts the tests in a JUnit report whose name contains `filter`. +/// Whether a nextest test name matches `filter` (HN-L2): one `::`-separated +/// segment of the name starts with it. `property_x` and `tests::property_x` +/// match `property`; `a_property_x` does not. A layer that runs itself +/// selects tests with the same rule. +#[must_use] +pub(crate) fn matches(name: &str, filter: &str) -> bool { + name.split("::").any(|segment| segment.starts_with(filter)) +} + +/// Counts the tests in a JUnit report whose name matches `filter`. /// /// A test with a `failure`, `error` or `skipped` child did not pass. An empty /// `filter` matches every test. @@ -32,7 +41,7 @@ pub(crate) struct Tally { pub(crate) fn tally(xml: &str, filter: &str) -> Tally { let mut tally = Tally::default(); for (name, passed) in testcases(xml) { - if !name.contains(filter) { + if !matches(&name, filter) { continue; } if passed { @@ -152,14 +161,14 @@ pub(crate) fn credit( "no test matches" }; Err(format!( - "{layer}: the credited run's report {shown} holds no passed test whose name contains \ + "{layer}: the credited run's report {shown} holds no passed test whose name has a `::` segment starting with \ `{filter}` ({why}); add one, or set gates.{layer}.test_filter" )) } #[cfg(test)] mod tests { - use super::{credit, junit_name, junit_path, tally, Tally}; + use super::{credit, junit_name, junit_path, matches, tally, Tally}; use std::fs; use std::path::Path; @@ -181,7 +190,7 @@ mod tests { "#; #[test] - fn only_a_passed_test_whose_name_contains_the_filter_counts() { + fn only_a_passed_test_whose_name_matches_the_filter_counts() { assert_eq!( tally(REPORT, "property"), Tally { @@ -201,6 +210,23 @@ mod tests { assert_eq!(tally("", "property"), Tally::default()); } + #[test] + fn a_segment_must_start_with_the_filter() { + assert!(matches("property_x", "property")); + assert!(matches("tests::property_x", "property")); + assert!(matches("a::property::b", "property")); + assert!(!matches("a_property_x", "property")); + assert!(!matches("tests::a_property", "property")); + assert!(!matches("Property_x", "property")); + assert!(matches("anything", "")); + } + + #[test] + fn a_meta_test_that_only_mentions_the_filter_is_not_evidence() { + let xml = r#""#; + assert_eq!(tally(xml, "property"), Tally::default()); + } + #[test] fn escaped_text_inside_a_test_is_not_a_test() { assert_eq!(tally(REPORT, "property_fake"), Tally::default()); diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs index d16940c..a36af91 100644 --- a/crates/cargo-tops/tests/gate_credit.rs +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -306,15 +306,58 @@ fn acceptance_follows_the_same_rule_with_its_own_filter() { assert_eq!(verdict, "pass", "its own filter, not property's"); } -/// The kit holds the property tests its own `property` layer credits. This is -/// the cheap half of the plant: removing `tests/property.rs` leaves the kit's -/// coverage run with no test named `property`, and the gate fails the layer. +/// The kit holds the property tests its own `property` layer credits. A +/// test cannot run the plant (remove `tests/property.rs` and the example +/// crate's `property_*` tests, then gate: `property` fails; restore only +/// `tests/property.rs`: it passes). It guards what the plant relies on: the +/// file has at least three proptest properties, each starting with +/// `property`, and no other test in this crate starts a name segment with +/// `property`, so the file is what the layer's evidence rests on here. #[test] fn the_kit_has_at_least_three_property_tests() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); let source = - fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/property.rs")) - .expect("the kit's tests/property.rs"); + fs::read_to_string(root.join("tests/property.rs")).expect("the kit's tests/property.rs"); let named = source.matches("fn property_").count(); assert!(named >= 3, "{named} property tests"); assert!(source.contains("proptest!"), "they use proptest"); + assert!(manifest_has_proptest(root), "proptest is a dev-dependency"); + let strays = tests_named_property_elsewhere(root); + assert!( + strays.is_empty(), + "other kit tests start with property: {strays:?}" + ); +} + +fn manifest_has_proptest(root: &Path) -> bool { + fs::read_to_string(root.join("Cargo.toml")).is_ok_and(|text| text.contains("proptest")) +} + +/// Test functions in this crate, outside `tests/property.rs`, whose name +/// starts with `property` (or has a `property` module segment). +fn tests_named_property_elsewhere(root: &Path) -> Vec { + let mut found = Vec::new(); + for dir in ["src", "tests"] { + let mut stack = vec![root.join(dir)]; + while let Some(path) = stack.pop() { + if path.is_dir() { + stack.extend( + fs::read_dir(&path) + .expect("read dir") + .flatten() + .map(|e| e.path()), + ); + } else if path.extension().is_some_and(|ext| ext == "rs") + && path.file_name().is_some_and(|name| name != "property.rs") + { + let text = fs::read_to_string(&path).unwrap_or_default(); + found.extend( + text.lines() + .filter(|line| line.trim_start().starts_with("fn property")) + .map(|line| format!("{}: {}", path.display(), line.trim())), + ); + } + } + } + found } From 9977269564e3355758c299ebcf02ba1158c27837 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 20:54:56 -0500 Subject: [PATCH 3/9] Fix the unescape test for the segment rule Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/evidence.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs index 23f5fc8..4f32b2e 100644 --- a/crates/cargo-tops/src/evidence.rs +++ b/crates/cargo-tops/src/evidence.rs @@ -245,7 +245,7 @@ mod tests { #[test] fn a_name_is_unescaped() { let xml = r#""#; - assert_eq!(tally(xml, "a&b\"c'd").passed, 1); + assert_eq!(tally(xml, "property_a&b\"c'd").passed, 1); } #[test] From 8a6200ca0a70f155371af511478d5dfdfc71302c Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 20:55:09 -0500 Subject: [PATCH 4/9] Count only test functions in the kit guard Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/tests/gate_credit.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs index a36af91..859bf27 100644 --- a/crates/cargo-tops/tests/gate_credit.rs +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -333,8 +333,8 @@ fn manifest_has_proptest(root: &Path) -> bool { fs::read_to_string(root.join("Cargo.toml")).is_ok_and(|text| text.contains("proptest")) } -/// Test functions in this crate, outside `tests/property.rs`, whose name -/// starts with `property` (or has a `property` module segment). +/// `#[test]` functions in this crate, outside `tests/property.rs`, whose +/// name starts with `property`. fn tests_named_property_elsewhere(root: &Path) -> Vec { let mut found = Vec::new(); for dir in ["src", "tests"] { @@ -351,11 +351,17 @@ fn tests_named_property_elsewhere(root: &Path) -> Vec { && path.file_name().is_some_and(|name| name != "property.rs") { let text = fs::read_to_string(&path).unwrap_or_default(); - found.extend( - text.lines() - .filter(|line| line.trim_start().starts_with("fn property")) - .map(|line| format!("{}: {}", path.display(), line.trim())), - ); + let mut is_test = false; + for line in text.lines().map(str::trim_start) { + if line.starts_with("fn property") && is_test { + found.push(format!("{}: {}", path.display(), line)); + } + if line.starts_with("#[test]") { + is_test = true; + } else if !line.starts_with("#[") { + is_test = false; + } + } } } } From 15db36b4d3fd85e94695bd41bf54e697b8c6b66b Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:11:59 -0500 Subject: [PATCH 5/9] Credit property and acceptance only on a matching passed test A credited property or acceptance layer passes only when the credited run's nextest JUnit report holds a passed test with a :: segment starting with the layer's test_filter (default property or acceptance). A layer that runs itself selects tests with the same rule through -E and --no-tests=fail. acceptance gets property's command and credit rule; it was NotRun. gates.property.test_filter and gates.acceptance.test_filter join the schema, section 6.5.1 and section 9.5. mod evidence moves into main.rs. Co-Authored-By: Claude Sonnet 5.5 --- PROTOCOL.md | 14 ++ crates/cargo-tops/src/config.rs | 67 +++++++++ crates/cargo-tops/src/credit.rs | 13 +- crates/cargo-tops/src/evidence.rs | 13 +- crates/cargo-tops/src/gate/execute.rs | 22 ++- crates/cargo-tops/src/gate/mod.rs | 26 +++- crates/cargo-tops/src/gate/plan.rs | 134 +++++++++++++++--- crates/cargo-tops/src/gate/report.rs | 9 ++ crates/cargo-tops/src/main.rs | 1 + .../templates/rust-tops.schema.json | 15 ++ crates/cargo-tops/tests/gate_credit.rs | 24 ++++ crates/cargo-tops/tests/gate_floors.rs | 5 + crates/cargo-tops/tests/laws_gate.rs | 42 ++++-- schema/rust-tops.schema.json | 15 ++ 14 files changed, 341 insertions(+), 59 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index a9d57f7..56dce23 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -391,6 +391,10 @@ fn encode_empty_emits_single_zero_block() { } ``` +#### 6.5.1 What the gate reads + +The gate runs the `acceptance` layer the way it runs `property` (§9.5): the same nextest command, narrowed to the tests it selects by name, with its own filter `gates.acceptance.test_filter` (default `acceptance`). A test counts for the layer when one `::`-separated segment of its nextest name starts with the filter, so `acceptance_encode` and `journeys::acceptance_encode` count and `an_acceptance_helper` does not. When the layer takes the coverage run instead of running itself, it passes only if that run's JUnit report holds a passed test that matches. + ### 6.6 Compile-fail / UI (proc-macros, tricky APIs) `trybuild` or `ui_test`. These are the mutation tests of the type system: if a change makes an intended failure compile, the suite must go red. @@ -596,6 +600,16 @@ Rules: A good property kills operator mutants that example tests miss (`<` vs `<=`, off-by-one on max). If a survivor is an inequality on a bound, add a property at the bound, do not add a single magic example unless the bound is a spec constant. +### 9.5 What the gate reads + +`gates.property.test_filter` (default `property`) names the tests that count as property tests: a test counts when one `::`-separated segment of its nextest name starts with the filter. The `property` layer passes only on a passed test that matches. + +- **It runs itself** when no coverage run is credited. Its command is the shared nextest run with `-E 'test(/(^|::)/)'` and `--no-tests=fail`, so a filter that selects nothing fails the layer. +- **It takes the coverage run** when its command carries `--workspace`, `--all-features` and `--profile ci`. The coverage run must also have written a nextest JUnit report at `/nextest/ci/`, with `path` from `[profile.ci.junit]` in `.config/nextest.toml`, holding at least one passed test that matches. The gate reads that file and runs nothing more. No match, only failed or skipped matches, or no report is `fail`, and the report row's `detail` names the filter and the path read, or the `[profile.ci.junit]` setting. +- **The same rule** applies to the whole workspace, not to one crate, so credit equals running. + +The same keys exist for `acceptance` under `gates.acceptance` (§6.5.1). + --- ## 10. Fuzzing (Tier 3) diff --git a/crates/cargo-tops/src/config.rs b/crates/cargo-tops/src/config.rs index ad2bafc..f21a9e4 100644 --- a/crates/cargo-tops/src/config.rs +++ b/crates/cargo-tops/src/config.rs @@ -44,6 +44,32 @@ pub(crate) struct Gates { /// Absent means the remote's default branch (`origin/HEAD`). #[serde(skip_serializing_if = "Option::is_none")] pub base_ref: Option, + /// The `property` gate. Only `test_filter` is read here; the rest of the + /// object stays as written. + #[serde(default, skip_serializing)] + pub property: Option, + /// The `acceptance` gate, as [`Gates::property`]. + #[serde(default, skip_serializing)] + pub acceptance: Option, +} + +impl Gates { + /// The `test_filter` of `layer` (`property` or `acceptance`): the name + /// prefix a test must start a `::` segment with to count (PROTOCOL.md + /// §6, §9). It defaults to the layer's id. + pub(crate) fn test_filter(&self, layer: &str) -> String { + let gate = match layer { + "property" => &self.property, + "acceptance" => &self.acceptance, + _ => &None, + }; + gate.as_ref() + .and_then(|gate| gate.get("test_filter")) + .and_then(serde_json::Value::as_str) + .filter(|filter| !filter.is_empty()) + .unwrap_or(layer) + .to_string() + } } #[derive(Debug, Deserialize, Serialize, PartialEq)] @@ -69,3 +95,44 @@ pub(crate) struct CrapGate { pub(crate) struct MutationGate { pub unexplained_survivors_max: i64, } + +#[cfg(test)] +mod tests { + use super::Gates; + use serde_json::json; + + fn gates(extra: &serde_json::Value) -> Gates { + let mut doc = + json!({"coverage": {}, "crap": {}, "mutation": {"unexplained_survivors_max": 0}}); + if let (Some(base), Some(more)) = (doc.as_object_mut(), extra.as_object()) { + base.extend(more.clone()); + } + serde_json::from_value(doc).expect("gates") + } + + #[test] + fn a_test_filter_defaults_to_the_layer_id() { + let none = gates(&json!({})); + assert_eq!(none.test_filter("property"), "property"); + assert_eq!(none.test_filter("acceptance"), "acceptance"); + let other = gates(&json!({"property": {"framework": "proptest"}})); + assert_eq!(other.test_filter("property"), "property"); + } + + #[test] + fn each_layer_reads_its_own_test_filter() { + let set = gates(&json!({ + "property": {"test_filter": "prop_"}, + "acceptance": {"test_filter": "journey_"}, + })); + assert_eq!(set.test_filter("property"), "prop_"); + assert_eq!(set.test_filter("acceptance"), "journey_"); + assert_eq!(set.test_filter("unit"), "unit"); + } + + #[test] + fn an_empty_test_filter_is_the_default() { + let empty = gates(&json!({"property": {"test_filter": ""}})); + assert_eq!(empty.test_filter("property"), "property"); + } +} diff --git a/crates/cargo-tops/src/credit.rs b/crates/cargo-tops/src/credit.rs index a3ccc5f..83f77b9 100644 --- a/crates/cargo-tops/src/credit.rs +++ b/crates/cargo-tops/src/credit.rs @@ -4,14 +4,12 @@ //! a layer and [`recorded_outcome`] after the source layer has finished. //! A missing or failed coverage run is not a pass. A coverage command that //! lacks `--workspace`, `--all-features`, or the nextest profile grants no -//! credit, and `unit`, `integration`, and `property` run themselves. +//! credit, and `unit`, `integration`, `property`, and `acceptance` run +//! themselves. `property` and `acceptance` also need evidence from the run +//! ([`crate::evidence`]). #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] #![allow(dead_code, reason = "main.rs calls the credit decision")] -// Held here until the gate wiring moves it to main.rs. -#[path = "evidence.rs"] -pub(crate) mod evidence; - /// Nextest profile a coverage command must pass for credit. pub(crate) const NEXTEST_PROFILE: &str = "ci"; @@ -70,7 +68,10 @@ pub(crate) fn recorded_outcome(query: &Query<'_>) -> Option { fn takes_coverage(query: &Query<'_>) -> bool { query.coverage_in_range && coverage_grants_credit(query.coverage_argv) - && matches!(query.layer, "unit" | "integration" | "property") + && matches!( + query.layer, + "unit" | "integration" | "property" | "acceptance" + ) } fn takes_unit(query: &Query<'_>) -> bool { diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs index 4f32b2e..5842413 100644 --- a/crates/cargo-tops/src/evidence.rs +++ b/crates/cargo-tops/src/evidence.rs @@ -131,7 +131,7 @@ pub(crate) fn junit_path(root: &Path, target: &Path) -> Result /// Returns how many matching tests passed. /// /// # Errors -/// The detail for a `fail`: it names the filter, and the report path read +/// The detail for a `fail` (the judge adds the layer): it names the filter, and the report path read /// (relative to `root`) or the `[profile.ci.junit]` setting. pub(crate) fn credit( layer: &str, @@ -139,7 +139,7 @@ pub(crate) fn credit( target: &Path, filter: &str, ) -> Result { - let path = junit_path(root, target).map_err(|why| format!("{layer}: {why}"))?; + let path = junit_path(root, target)?; let shown = path .strip_prefix(root) .unwrap_or(&path) @@ -147,7 +147,7 @@ pub(crate) fn credit( .to_string(); let xml = fs::read_to_string(&path).map_err(|_| { format!( - "{layer}: no JUnit report at {shown}; the credited run must write one \ + "no JUnit report at {shown}; the credited run must write one \ (`[profile.ci.junit]` in .config/nextest.toml)" ) })?; @@ -161,7 +161,7 @@ pub(crate) fn credit( "no test matches" }; Err(format!( - "{layer}: the credited run's report {shown} holds no passed test whose name has a `::` segment starting with \ + "the credited run's report {shown} holds no passed test whose name has a `::` segment starting with \ `{filter}` ({why}); add one, or set gates.{layer}.test_filter" )) } @@ -322,10 +322,7 @@ mod tests { fn no_junit_setting_names_the_junit_setting() { let dir = fixture(None, Some(REPORT)); let why = run(&dir, "property").unwrap_err(); - assert!( - why.starts_with("property: no JUnit report is configured"), - "{why}" - ); + assert!(why.starts_with("no JUnit report is configured"), "{why}"); assert!(why.contains("[profile.ci.junit]"), "{why}"); let bare = fixture(Some("[profile.ci]\nretries = 0\n"), Some(REPORT)); assert!(run(&bare, "property").is_err()); diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index b57cf56..299d848 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Result}; use super::plan::{GatePlan, Unit, What, CRAP_VERSION}; use super::{Fault, Mode}; use crate::config::Layer; +use crate::evidence; use crate::facts::{Outcome, Run}; use crate::steps::{self, Base, Kind, MutationPlan, Step}; use crate::{laws, TOOL_LOCK}; @@ -252,10 +253,29 @@ impl<'a> Runner<'_, 'a> { } } + /// The run `layer` takes from `from`. `property` and `acceptance` pass + /// only when that run also wrote a JUnit report holding a passed test + /// they select by name; the report is read, never produced (HN-P3). + fn credited(&self, layer: &str, from: &str) -> Run { + let mut run = self.plan.tests.take(layer, from, &self.done.runs); + let Some(filter) = self.plan.filters.for_layer(layer) else { + return run; + }; + if run.outcome != Outcome::Passed { + return run; + } + let target = self.plan.report.parent().unwrap_or(&self.plan.report); + if let Err(why) = evidence::credit(layer, self.dir, target, filter) { + run.outcome = Outcome::Failed; + run.note = Some(why); + } + run + } + /// A layer that adds no command records its source's run. fn take_shared(&mut self) { for (layer, from) in self.done.shared.clone() { - let run = self.plan.tests.take(layer, from, &self.done.runs); + let run = self.credited(layer, from); println!("[{layer}] took the {from} run's outcome: {}", run.outcome); self.done.runs.push(run); self.done.timing.push(Timing { diff --git a/crates/cargo-tops/src/gate/mod.rs b/crates/cargo-tops/src/gate/mod.rs index cb52d3a..69bc0fa 100644 --- a/crates/cargo-tops/src/gate/mod.rs +++ b/crates/cargo-tops/src/gate/mod.rs @@ -127,7 +127,7 @@ pub(crate) fn config_error(dir: &Path, error: &anyhow::Error, opts: Options) -> #[cfg(test)] mod tests { use super::judge::{command_count, ran_commands}; - use super::plan::{plan_for, Plan, TestCommands}; + use super::plan::{plan_for, Filters, Plan, TestCommands}; use crate::facts::{Outcome, Run}; use crate::steps::{self, Base, Kind, Step}; use crate::{config, credit}; @@ -142,7 +142,12 @@ mod tests { } fn chains(id: &str) -> Vec> { - match plan_for(&layer(id), Path::new("/r"), &Base::unused()) { + match plan_for( + &layer(id), + Path::new("/r"), + &Base::unused(), + &Filters::default(), + ) { Some(Plan::Chains(chains)) => chains, _ => panic!("{id} has no chains"), } @@ -280,7 +285,7 @@ mod tests { .map(layer) .collect(); let enabled: Vec<&config::Layer> = layers.iter().collect(); - let tests = TestCommands::new(&enabled, Path::new("/r")); + let tests = TestCommands::new(&enabled, Path::new("/r"), &Filters::default()); assert!(tests.coverage_in_range); assert!( credit::coverage_grants_credit(&tests.coverage), @@ -300,7 +305,7 @@ mod tests { .copied() .filter(|l| l.id != "coverage") .collect(); - let tests = TestCommands::new(&unit_only, Path::new("/r")); + let tests = TestCommands::new(&unit_only, Path::new("/r"), &Filters::default()); assert!(!tests.coverage_in_range); assert!(tests.coverage.is_empty()); assert_eq!(tests.shared_from("integration"), Some("unit")); @@ -402,7 +407,9 @@ mod tests { source: steps::BaseSource::Flag, name: "origin/trunk".to_string(), }; - let Some(Plan::Mutation(plan)) = plan_for(&layer("mutation"), report, &base) else { + let Some(Plan::Mutation(plan)) = + plan_for(&layer("mutation"), report, &base, &Filters::default()) + else { panic!("mutation has no plan"); }; assert_eq!(plan.diff_path, report.join("rust-tops.diff")); @@ -448,7 +455,6 @@ mod tests { #[test] fn unmapped_layers_have_no_plan() { for id in [ - "acceptance", "fuzz", "miri", "sanitizer", @@ -458,7 +464,13 @@ mod tests { "laws", ] { assert!( - plan_for(&layer(id), Path::new("/r"), &Base::unused()).is_none(), + plan_for( + &layer(id), + Path::new("/r"), + &Base::unused(), + &Filters::default() + ) + .is_none(), "{id}" ); } diff --git a/crates/cargo-tops/src/gate/plan.rs b/crates/cargo-tops/src/gate/plan.rs index df0fba8..bcf2e83 100644 --- a/crates/cargo-tops/src/gate/plan.rs +++ b/crates/cargo-tops/src/gate/plan.rs @@ -35,12 +35,12 @@ pub(crate) struct TestCommands { } impl TestCommands { - pub(crate) fn new(enabled: &[&config::Layer], report: &Path) -> Self { + pub(crate) fn new(enabled: &[&config::Layer], report: &Path, filters: &Filters) -> Self { let argv = |id: &str| { enabled .iter() .find(|layer| layer.id == id) - .and_then(|layer| tool_plan(layer, report)) + .and_then(|layer| tool_plan(layer, report, filters)) .and_then(|plan| match plan { Plan::Chains(chains) => chains .into_iter() @@ -163,20 +163,6 @@ const TOOL_LAYERS: &[(&str, &[&[&str]])] = &[ "doc", &[&["cargo", "test", "--doc", "--workspace", "--all-features"]], ), - ( - "property", - &[&[ - "cargo", - "nextest", - "run", - "--workspace", - "--all-features", - "--profile", - "ci", - "--no-tests=fail", - "property", - ]], - ), ( "security-advisory", &[&["cargo", "deny", "check", "advisories"]], @@ -184,6 +170,54 @@ const TOOL_LAYERS: &[(&str, &[&[&str]])] = &[ ("formal-kani", &[&["cargo", "kani", "--workspace"]]), ]; +/// The name prefixes `property` and `acceptance` select tests by. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Filters { + pub(crate) property: String, + pub(crate) acceptance: String, +} + +impl Default for Filters { + fn default() -> Self { + Self { + property: "property".to_string(), + acceptance: "acceptance".to_string(), + } + } +} + +impl Filters { + pub(crate) fn of(gates: &config::Gates) -> Self { + Self { + property: gates.test_filter("property"), + acceptance: gates.test_filter("acceptance"), + } + } + + /// The prefix of `layer`, for the two layers that select by name. + pub(crate) fn for_layer(&self, layer: &str) -> Option<&str> { + match layer { + "property" => Some(&self.property), + "acceptance" => Some(&self.acceptance), + _ => None, + } + } +} + +/// The shared nextest run, narrowed to the tests with a `::`-separated name +/// segment that starts with `filter` (HN-L2). The same rule reads the JUnit +/// report of a credited run, so credit equals running. +fn selected_tests(filter: &str) -> Vec { + let literal = regex::escape(filter).replace('/', "\\/"); + let expr = format!("test(/(^|::){literal}/)"); + let argv = NEXTEST.iter().map(|arg| (*arg).to_string()).chain([ + "--no-tests=fail".to_string(), + "-E".to_string(), + expr, + ]); + vec![Step::tool(argv)] +} + /// The test run `unit` and `integration` share. const NEXTEST: [&str; 7] = [ "cargo", @@ -217,20 +251,28 @@ fn crap_steps(report: &Path) -> Vec { /// `report` is [`laws::artifact_dir`]: the coverage LCOV, the CRAP report /// and the mutation diff are written there. `base` is where the mutation /// diff starts. -pub(crate) fn plan_for(layer: &config::Layer, report: &Path, base: &Base) -> Option { +pub(crate) fn plan_for( + layer: &config::Layer, + report: &Path, + base: &Base, + filters: &Filters, +) -> Option { if layer.id == "mutation" { Some(Plan::Mutation(MutationPlan::new(report, base.clone()))) } else { - tool_plan(layer, report) + tool_plan(layer, report, filters) } } /// The plan of every layer but `mutation`, which also needs a base. -fn tool_plan(layer: &config::Layer, report: &Path) -> Option { +fn tool_plan(layer: &config::Layer, report: &Path, filters: &Filters) -> Option { let id = layer.id.as_str(); match id { "coverage" => Some(Plan::Chains(vec![steps::coverage_steps(report)])), "crap" => Some(Plan::Chains(vec![crap_steps(report)])), + "property" | "acceptance" => filters + .for_layer(id) + .map(|filter| Plan::Chains(vec![selected_tests(filter)])), _ => TOOL_LAYERS .iter() .find(|(layer_id, _)| *layer_id == id) @@ -270,6 +312,8 @@ pub(crate) struct GatePlan<'a> { /// `report.json` live here, as an absolute path. pub(crate) report: PathBuf, pub(crate) tests: TestCommands, + /// The name prefixes of `property` and `acceptance`. + pub(crate) filters: Filters, pub(crate) crap_pin: Option, pub(crate) units: Vec>, } @@ -346,13 +390,17 @@ fn units_of<'a>( tests: &TestCommands, report: &Path, base: &Base, + filters: &Filters, ) -> Vec> { enabled .iter() .copied() .filter(|layer| layer.id != "laws") .map(|layer| { - let what = match (tests.shared_from(&layer.id), plan_for(layer, report, base)) { + let what = match ( + tests.shared_from(&layer.id), + plan_for(layer, report, base, filters), + ) { (Some(from), _) => What::Shared(from), (None, Some(Plan::Chains(chains))) => What::Chains(chains), (None, Some(Plan::Mutation(plan))) => What::Mutation(plan), @@ -381,7 +429,8 @@ pub(crate) fn build<'a>( ))); } let report = report_dir(dir)?; - let tests = TestCommands::new(&enabled, &report); + let filters = Filters::of(&cfg.gates); + let tests = TestCommands::new(&enabled, &report, &filters); let base = base_for(&enabled, dir, base_flag, cfg.gates.base_ref.as_deref()); let crap_pin = if enabled.iter().any(|layer| layer.id == "crap") { tool_pin(dir, "cargo-crap").map_err(Fault::config)? @@ -396,17 +445,18 @@ pub(crate) fn build<'a>( above: ids(cfg, |l| l.enabled && crate::tiers::tier_for(l) > max_tier), // A layer turned off still shows in the run: it is not run, not omitted. disabled: ids(cfg, |l| !l.enabled), - units: units_of(&enabled, &tests, &report, &base), + units: units_of(&enabled, &tests, &report, &base, &filters), enabled, report, tests, + filters, crap_pin, }) } #[cfg(test)] mod tests { - use super::{base_for, GatePlan, TestCommands}; + use super::{base_for, plan_for, Filters, GatePlan, Plan, TestCommands}; use crate::config::Layer; use crate::steps::{Base, BaseSource}; use std::path::PathBuf; @@ -451,6 +501,43 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } + #[test] + fn the_selecting_layers_run_only_tests_with_a_name_prefix() { + let filters = Filters { + property: "prop_".to_string(), + acceptance: "a/b.c".to_string(), + }; + let shared = ["cargo", "nextest", "run", "--workspace", "--all-features"] + .into_iter() + .chain(["--profile", "ci", "--no-tests=fail", "-E"]); + for (id, expr) in [ + ("property", "test(/(^|::)prop_/)"), + ("acceptance", r"test(/(^|::)a\/b\.c/)"), + ] { + let Some(Plan::Chains(chains)) = + plan_for(&layer(id), &PathBuf::new(), &Base::unused(), &filters) + else { + panic!("{id} has no plan"); + }; + let argv: Vec = chains + .into_iter() + .flatten() + .flat_map(|step| step.argv) + .collect(); + let want: Vec = shared.clone().chain([expr]).map(str::to_string).collect(); + assert_eq!(argv, want, "{id}"); + } + } + + #[test] + fn only_the_two_selecting_layers_have_a_filter() { + let filters = Filters::default(); + assert_eq!(filters.for_layer("property"), Some("property")); + assert_eq!(filters.for_layer("acceptance"), Some("acceptance")); + assert_eq!(filters.for_layer("unit"), None); + assert_eq!(filters.for_layer("integration"), None); + } + #[test] fn a_layer_with_no_unit_in_the_plan_has_no_commands() { let plan = GatePlan { @@ -466,6 +553,7 @@ mod tests { unit: Vec::new(), integration: Vec::new(), }, + filters: super::Filters::default(), crap_pin: None, units: Vec::new(), }; diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index 5397d9c..5ee566e 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -420,6 +420,15 @@ fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { if let Some(numbers) = numbers { entry.insert("judged".into(), numbers); } + // A layer stopped on a note (the credit evidence of `property`) says why. + let noted = done + .runs + .iter() + .find(|run| run.layer == id) + .and_then(|run| run.note.clone()); + if let Some(note) = noted { + entry.insert("detail".into(), json!(note)); + } if id == "mutation" { name_the_base(plan, done, &mut entry); } diff --git a/crates/cargo-tops/src/main.rs b/crates/cargo-tops/src/main.rs index cda1415..4ad0711 100644 --- a/crates/cargo-tops/src/main.rs +++ b/crates/cargo-tops/src/main.rs @@ -24,6 +24,7 @@ mod config; mod coverage_floor; mod crap_ratchet; mod credit; +mod evidence; mod exemptions; mod facts; mod floors; diff --git a/crates/cargo-tops/templates/rust-tops.schema.json b/crates/cargo-tops/templates/rust-tops.schema.json index d70f0d0..1ba0a9a 100644 --- a/crates/cargo-tops/templates/rust-tops.schema.json +++ b/crates/cargo-tops/templates/rust-tops.schema.json @@ -221,6 +221,21 @@ "required_on_classes": { "type": "array", "items": { "type": "string" } + }, + "test_filter": { + "type": "string", + "minLength": 1, + "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." + } + } + }, + "acceptance": { + "type": "object", + "properties": { + "test_filter": { + "type": "string", + "minLength": 1, + "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." } } }, diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs index 859bf27..4fc2d18 100644 --- a/crates/cargo-tops/tests/gate_credit.rs +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -19,6 +19,7 @@ use serde_json::{json, Value}; const CARGO_STUB: &str = r#"#!/bin/sh if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json"; exit 0; fi +if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/coverage-fails" ]; then exit 1; fi if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/junit.xml" ]; then mkdir -p "$REPORTS/../repo/target/nextest/ci" cp "$REPORTS/junit.xml" "$REPORTS/../repo/target/nextest/ci/junit.xml" @@ -306,6 +307,29 @@ fn acceptance_follows_the_same_rule_with_its_own_filter() { assert_eq!(verdict, "pass", "its own filter, not property's"); } +#[test] +fn a_failed_coverage_run_fails_property_without_reading_evidence() { + let fixture = fixture("covfail", &[], Some(&junit(&[("property_ok", true)]))); + fs::write(fixture.reports.join("coverage-fails"), "").expect("mark"); + let (code, _, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1)); + assert_eq!(verdict, "fail"); + assert!(!detail.contains("JUnit"), "{detail}"); +} + +#[test] +fn unit_and_integration_need_no_matching_test() { + let fixture = fixture( + "unit", + &["unit", "integration"], + Some(&junit(&[("property_ok", true)])), + ); + for layer in ["unit", "integration"] { + let (_, text, verdict, _) = gate(&fixture, layer); + assert_eq!(verdict, "pass", "{layer}: {text}"); + } +} + /// The kit holds the property tests its own `property` layer credits. A /// test cannot run the plant (remove `tests/property.rs` and the example /// crate's `property_*` tests, then gate: `property` fails; restore only diff --git a/crates/cargo-tops/tests/gate_floors.rs b/crates/cargo-tops/tests/gate_floors.rs index 8802c2b..155394b 100644 --- a/crates/cargo-tops/tests/gate_floors.rs +++ b/crates/cargo-tops/tests/gate_floors.rs @@ -37,6 +37,11 @@ for arg in "$@"; do mkdir -p "$(dirname "$arg")" name=$(basename "$arg") if [ -f "$REPORTS/$name" ]; then cp "$REPORTS/$name" "$arg"; else printf 'lcov\n' > "$arg"; fi + if [ "$1 $2" = "llvm-cov nextest" ]; then + j="$(dirname "$arg")/../nextest/ci" + mkdir -p "$j" + printf '%s\n' '' > "$j/junit.xml" + fi ;; esac prev=$arg diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index 921b4a9..faf16cb 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -37,7 +37,10 @@ const INIT_CLASSES: [&str; 8] = [ /// `cargo llvm-cov` and `cargo crap` do. Logs its argv to `$CARGO_STUB_LOG`. /// The file is one JSON document that reads as a coverage summary (one /// fully covered `src/lib.rs`) and as a CRAP report with no function, so -/// the coverage floors and the CRAP ratchet pass. +/// the coverage floors and the CRAP ratchet pass. A `llvm-cov nextest` run +/// also writes the JUnit report nextest writes, holding one passed `property` +/// and one passed `acceptance` test, so a layer that takes the coverage run +/// has the evidence it needs (PROTOCOL.md §6, §9). const CARGO_STUB: &str = r#"#!/bin/sh if [ -n "${CARGO_STUB_LOG:-}" ]; then printf '%s\n' "$*" >> "$CARGO_STUB_LOG" @@ -53,6 +56,11 @@ for arg in "$@"; do --output-path | --output) mkdir -p "$(dirname "$arg")" printf '%s\n' '{"data":[{"files":[{"filename":"src/lib.rs","summary":{"lines":{"count":1,"covered":1},"regions":{"count":1,"covered":1}}}]}],"entries":[]}' > "$arg" + if [ "$1 $2" = "llvm-cov nextest" ]; then + j="$(dirname "$arg")/../nextest/ci" + mkdir -p "$j" + printf '%s\n' '' > "$j/junit.xml" + fi ;; esac prev=$arg @@ -486,7 +494,7 @@ fn coverage_run(fixture: &Fixture) -> String { ) } const PROPERTY_RUN: &str = - "nextest run --workspace --all-features --profile ci --no-tests=fail property"; + "nextest run --workspace --all-features --profile ci --no-tests=fail -E test(/(^|::)property/)"; /// `laws evaluated: N/53` from gate output. fn laws_evaluated(text: &str) -> (usize, usize) { @@ -1413,7 +1421,7 @@ fn below_coverage_unit_runs_once_integration_takes_it_and_property_runs_itself() "{text}" ); assert!( - text.contains("\n==> [property] cargo nextest run --workspace --all-features --profile ci --no-tests=fail property\n"), + text.contains("\n==> [property] cargo nextest run --workspace --all-features --profile ci --no-tests=fail -E test(/(^|::)property/)\n"), "{text}" ); assert!(!text.contains("==> [coverage]"), "{text}"); @@ -1451,12 +1459,12 @@ fn below_coverage_unit_runs_once_integration_takes_it_and_property_runs_itself() fn a_property_filter_that_matches_nothing_fails() { // nextest exits 4 when `--no-tests=fail` finds no test; the stub does too. let fixture = fixture_with("one-run-empty-filter", &["property"]); - stub_cargo_failing(&fixture, "*' --no-tests=fail property'", 4); + stub_cargo_failing(&fixture, "*' --no-tests=fail -E test(/(^|::)property/)'", 4); let (out, text) = gate(&fixture, path_with_only_stubs(&fixture.bin)); assert_eq!(out.status.code(), Some(1), "{text}"); assert_eq!(nextest_runs(&fixture), [PROPERTY_RUN], "{text}"); assert!( - text.contains("\n - property: `cargo nextest run --workspace --all-features --profile ci --no-tests=fail property` failed\n"), + text.contains("\n - property: `cargo nextest run --workspace --all-features --profile ci --no-tests=fail -E test(/(^|::)property/)` failed\n"), "{text}" ); assert!(!text.contains("\nok: "), "{text}"); @@ -1858,23 +1866,29 @@ fn a_real_diff_runs_mutants_on_the_report_dir_diff() { #[test] fn an_enabled_unmapped_layer_in_range_is_not_run_and_fails() { - // `init` writes `acceptance` off. Turned on, it is in range at tier 1 - // and has no command: NotRun, and the gate exits 1. No exemption. - let fixture = fixture_with("not-run", &["static-lint", "acceptance", "laws"]); + // `miri` has no command in the gate. Turned on and in range at tier 3 it + // is NotRun, and the gate exits 1. No exemption. (This was `acceptance` + // at tier 1 until protocol 1.4.0 gave that layer `property`'s command.) + let fixture = fixture_with("not-run", &["static-lint", "miri", "laws"]); adopt(&fixture, Origin::None); stub_bend(&fixture); - let (out, text) = gate(&fixture, path_without_bend(&fixture.bin)); + let (out, text) = gate_with( + &fixture, + path_without_bend(&fixture.bin), + None, + &["--max-tier", "3"], + ); assert_eq!(out.status.code(), Some(1), "{text}"); assert!( - text.contains("NotRun [acceptance]: enabled at tier 1 (<= --max-tier 1)"), + text.contains("NotRun [miri]: enabled at tier 3 (<= --max-tier 3)"), "{text}" ); assert!( - text.contains(" - acceptance: NotRun (enabled at tier 1, no command mapped"), + text.contains(" - miri: NotRun (enabled at tier 3, no command mapped"), "{text}" ); - assert!(!text.contains("skip acceptance"), "{text}"); - assert!(text.contains("none ran for: acceptance"), "{text}"); + assert!(!text.contains("skip miri"), "{text}"); + assert!(text.contains("none ran for: miri"), "{text}"); let yaml = fs::read_to_string(fixture.repo.join("rust-tops.yaml")).expect("yaml"); let doc: serde_json::Value = serde_saphyr::from_str(&yaml).expect("yaml parses"); #[expect( @@ -1889,7 +1903,7 @@ fn an_enabled_unmapped_layer_in_range_is_not_run_and_fails() { &fixture, path_without_bend(&fixture.bin), None, - &["--dry-run"], + &["--max-tier", "3", "--dry-run"], ); assert_eq!(out.status.code(), Some(1), "{text}"); assert!( diff --git a/schema/rust-tops.schema.json b/schema/rust-tops.schema.json index d70f0d0..1ba0a9a 100644 --- a/schema/rust-tops.schema.json +++ b/schema/rust-tops.schema.json @@ -221,6 +221,21 @@ "required_on_classes": { "type": "array", "items": { "type": "string" } + }, + "test_filter": { + "type": "string", + "minLength": 1, + "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." + } + } + }, + "acceptance": { + "type": "object", + "properties": { + "test_filter": { + "type": "string", + "minLength": 1, + "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." } } }, From 1e0bc7303966660bd3381dd1d1639bf0054e75af Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:16:53 -0500 Subject: [PATCH 6/9] Read one run note for the mutation base and the credit detail Split the tag at its closing bracket instead of counting past it. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/evidence.rs | 6 +++--- crates/cargo-tops/src/gate/report.rs | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs index 5842413..bb65f53 100644 --- a/crates/cargo-tops/src/evidence.rs +++ b/crates/cargo-tops/src/evidence.rs @@ -59,9 +59,9 @@ fn testcases(xml: &str) -> Vec<(String, bool)> { let mut rest = xml; while let Some(at) = rest.find("') else { break }; - let tag = from.get(..end).unwrap_or_default(); - let after = from.get(end + 1..).unwrap_or_default(); + let Some((tag, after)) = from.split_once('>') else { + break; + }; let (body, tail) = if tag.ends_with('/') { ("", after) } else { diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index 5ee566e..0924372 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -421,12 +421,7 @@ fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { entry.insert("judged".into(), numbers); } // A layer stopped on a note (the credit evidence of `property`) says why. - let noted = done - .runs - .iter() - .find(|run| run.layer == id) - .and_then(|run| run.note.clone()); - if let Some(note) = noted { + if let Some(note) = run_note(done, id) { entry.insert("detail".into(), json!(note)); } if id == "mutation" { @@ -457,11 +452,7 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map json!({ "ref": null, "source": null, "tried": tried }), }; put_judged(entry, "base", base); - let ran = done - .runs - .iter() - .find(|run| run.layer == "mutation") - .and_then(|run| run.note.clone()); + let ran = run_note(done, "mutation"); let planned = (!matches!(mutation.base, Base::Found { .. })) .then(|| steps::stop_note(&mutation.base, Outcome::Vacuous).map(|(_, note)| note)) .flatten(); @@ -470,6 +461,15 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map, layer: &str) -> Option { + done.runs + .iter() + .find(|run| run.layer == layer) + .and_then(|run| run.note.clone()) +} + /// Set `key` in the row's `judged` object, keeping what else is there. fn put_judged(entry: &mut Map, key: &str, value: Value) { let judged = entry From ca06ba88dd75cd91b14d583fb6b753716887ff61 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:21:38 -0500 Subject: [PATCH 7/9] Say in one place why a layer row has a detail A layer with no mapped command and a layer whose run kept a note both explain themselves in the report. One function now says which, which takes two branches out of unit_row. Co-Authored-By: Claude Sonnet 5.5 --- crates/cargo-tops/src/gate/report.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/cargo-tops/src/gate/report.rs b/crates/cargo-tops/src/gate/report.rs index 0924372..279cde0 100644 --- a/crates/cargo-tops/src/gate/report.rs +++ b/crates/cargo-tops/src/gate/report.rs @@ -410,19 +410,11 @@ fn unit_row(parts: &Parts<'_>, id: &str, tier: u8) -> Map { done.seconds(id), version.as_deref(), ); - let unmapped = plan - .units - .iter() - .any(|unit| unit.layer.id == id && matches!(unit.what, What::NotRun)); - if unmapped { - entry.insert("detail".into(), json!("enabled and no command is mapped")); - } if let Some(numbers) = numbers { entry.insert("judged".into(), numbers); } - // A layer stopped on a note (the credit evidence of `property`) says why. - if let Some(note) = run_note(done, id) { - entry.insert("detail".into(), json!(note)); + if let Some(detail) = layer_detail(plan, done, id) { + entry.insert("detail".into(), json!(detail)); } if id == "mutation" { name_the_base(plan, done, &mut entry); @@ -461,6 +453,19 @@ fn name_the_base(plan: &GatePlan<'_>, done: &Executed<'_>, entry: &mut Map, done: &Executed<'_>, id: &str) -> Option { + let unmapped = plan + .units + .iter() + .any(|unit| unit.layer.id == id && matches!(unit.what, What::NotRun)); + if unmapped { + return Some("enabled and no command is mapped".to_string()); + } + run_note(done, id) +} + /// The note `layer`'s run kept: why it stopped, in the words the report and /// the failure line both use. fn run_note(done: &Executed<'_>, layer: &str) -> Option { From c18c15263096312385a9ccad51f008e6ab2500a5 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Thu, 19 Nov 2026 23:50:50 -0500 Subject: [PATCH 8/9] Read credit evidence where nextest writes it and word its failure once The JUnit report is read from the workspace root cargo metadata names plus nextest's store dir, never from CARGO_TARGET_DIR, and a report older than the credited run is not evidence. A failed evidence check is carried as the run note into stdout, detail and problems, and no longer blames a passing coverage run. test_filter is held to letters, digits and underscore in both schemas, so check refuses anything else and no escaping is needed. Real nextest tests cover a moved target dir, a moved store and a stale report. Co-Authored-By: Claude Sonnet 5.5 --- PROTOCOL.md | 4 +- crates/cargo-tops/src/evidence.rs | 118 +++++-- crates/cargo-tops/src/gate/execute.rs | 11 +- crates/cargo-tops/src/gate/judge.rs | 4 +- crates/cargo-tops/src/gate/plan.rs | 10 +- crates/cargo-tops/src/laws.rs | 59 +++- .../templates/rust-tops.schema.json | 8 +- crates/cargo-tops/tests/gate_credit.rs | 97 +++++- .../cargo-tops/tests/gate_credit_nextest.rs | 322 ++++++++++++++++++ crates/cargo-tops/tests/gate_floors.rs | 2 +- crates/cargo-tops/tests/laws_gate.rs | 8 +- schema/rust-tops.schema.json | 8 +- 12 files changed, 572 insertions(+), 79 deletions(-) create mode 100644 crates/cargo-tops/tests/gate_credit_nextest.rs diff --git a/PROTOCOL.md b/PROTOCOL.md index 56dce23..5661e3a 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -604,8 +604,8 @@ A good property kills operator mutants that example tests miss (`<` vs `<=`, off `gates.property.test_filter` (default `property`) names the tests that count as property tests: a test counts when one `::`-separated segment of its nextest name starts with the filter. The `property` layer passes only on a passed test that matches. -- **It runs itself** when no coverage run is credited. Its command is the shared nextest run with `-E 'test(/(^|::)/)'` and `--no-tests=fail`, so a filter that selects nothing fails the layer. -- **It takes the coverage run** when its command carries `--workspace`, `--all-features` and `--profile ci`. The coverage run must also have written a nextest JUnit report at `/nextest/ci/`, with `path` from `[profile.ci.junit]` in `.config/nextest.toml`, holding at least one passed test that matches. The gate reads that file and runs nothing more. No match, only failed or skipped matches, or no report is `fail`, and the report row's `detail` names the filter and the path read, or the `[profile.ci.junit]` setting. +- **It runs itself** when no coverage run is credited. Its command is the shared nextest run with `-E 'test(/(^|::)/)'` and `--no-tests=fail`, so a filter that selects nothing fails the layer. A filter is letters, digits and underscore only (`^[A-Za-z0-9_]+$`), so it is a plain regex literal and needs no escaping. `check` refuses any other value, and the gate exits 2. +- **It takes the coverage run** when its command carries `--workspace`, `--all-features` and `--profile ci`. The coverage run must also have written a nextest JUnit report, holding at least one passed test that matches. The gate reads the file where nextest wrote it: the workspace root from `cargo metadata`, then nextest's `[store] dir` from `.config/nextest.toml` (default `target/nextest`), then `ci/` with `path` from `[profile.ci.junit]`. `CARGO_TARGET_DIR` and `build.target-dir` do not move it. A report older than the start of the gate run is stale and is not evidence; the gate never deletes it. No match, only failed or skipped matches, no report, or a stale report is `fail`. The report's `detail`, the failure line and `problems` all name the filter, the path read (relative to the workspace root), and the fix. - **The same rule** applies to the whole workspace, not to one crate, so credit equals running. The same keys exist for `acceptance` under `gates.acceptance` (§6.5.1). diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs index bb65f53..264375d 100644 --- a/crates/cargo-tops/src/evidence.rs +++ b/crates/cargo-tops/src/evidence.rs @@ -3,19 +3,25 @@ //! //! `property` and `acceptance` may take another layer's run (see //! [`crate::credit`]). That run passes for them only if its JUnit report -//! holds a test whose name matches the layer's `test_filter` (see [`matches`]) and that test -//! passed. This module reads that one file. It spawns nothing and builds -//! nothing, so reading it costs no second test run. +//! holds a test whose name matches the layer's `test_filter` (see [`matches`]) +//! and that test passed. This module reads that one file, from where nextest +//! writes it: the workspace root `cargo metadata` names, nextest's +//! `[store] dir` (default `target/nextest`), the profile, then the +//! `[profile.ci.junit]` path. A report older than the credited run is not +//! evidence. Nothing is built and no test runs, so reading it costs no +//! second test run. #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] use std::fs; use std::path::{Path, PathBuf}; +use std::time::SystemTime; /// The nextest profile whose report is read. pub(crate) const PROFILE: &str = "ci"; -/// Where nextest keeps a profile's reports under the target directory. -const STORE: &str = "nextest"; +/// Where nextest keeps its reports, under the workspace root, when +/// `.config/nextest.toml` has no `[store] dir`. +const DEFAULT_STORE: &str = "target/nextest"; /// How many matching tests passed and how many did not. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -108,17 +114,25 @@ pub(crate) fn junit_name(nextest_toml: &str) -> Option { .map(str::to_string) } -/// The report the credited run wrote: `/nextest/ci/`, with the -/// file name from `.config/nextest.toml` under `root`. +/// The `dir` of `[store]` in the text of `.config/nextest.toml`, relative to +/// the workspace root. nextest keeps its reports there. +#[must_use] +pub(crate) fn store_dir(nextest_toml: &str) -> String { + toml::from_str::(nextest_toml) + .ok() + .and_then(|doc| doc.get("store")?.get("dir")?.as_str().map(str::to_string)) + .unwrap_or_else(|| DEFAULT_STORE.to_string()) +} + +/// The report the credited run wrote: `//ci/`, with +/// the store dir and the file name from `.config/nextest.toml` under `root`. /// /// # Errors /// The setting is missing, so the run writes no report to read. -pub(crate) fn junit_path(root: &Path, target: &Path) -> Result { - let config = root.join(".config").join("nextest.toml"); - let name = fs::read_to_string(&config) - .ok() - .and_then(|text| junit_name(&text)); - name.map(|name| target.join(STORE).join(PROFILE).join(name)) +pub(crate) fn junit_path(root: &Path) -> Result { + let text = fs::read_to_string(root.join(".config").join("nextest.toml")).unwrap_or_default(); + junit_name(&text) + .map(|name| root.join(store_dir(&text)).join(PROFILE).join(name)) .ok_or_else(|| { "no JUnit report is configured: set `path` under `[profile.ci.junit]` in \ .config/nextest.toml so the credited run writes one" @@ -126,20 +140,23 @@ pub(crate) fn junit_path(root: &Path, target: &Path) -> Result }) } -/// Whether the credited run held a passed test matching `filter`. +/// Whether the credited run, which began at `since`, held a passed test +/// matching `filter` in a report it wrote. /// -/// Returns how many matching tests passed. +/// Returns how many matching tests passed. The file is only read, never +/// removed. /// /// # Errors -/// The detail for a `fail` (the judge adds the layer): it names the filter, and the report path read -/// (relative to `root`) or the `[profile.ci.junit]` setting. +/// The detail for a `fail` (the judge adds the layer): it names the filter +/// and the report path read (relative to `root`), or the +/// `[profile.ci.junit]` setting. pub(crate) fn credit( layer: &str, root: &Path, - target: &Path, filter: &str, + since: SystemTime, ) -> Result { - let path = junit_path(root, target)?; + let path = junit_path(root)?; let shown = path .strip_prefix(root) .unwrap_or(&path) @@ -148,9 +165,19 @@ pub(crate) fn credit( let xml = fs::read_to_string(&path).map_err(|_| { format!( "no JUnit report at {shown}; the credited run must write one \ - (`[profile.ci.junit]` in .config/nextest.toml)" + (`[profile.ci.junit]` and `[store] dir` in .config/nextest.toml)" ) })?; + let fresh = fs::metadata(&path) + .and_then(|meta| meta.modified()) + .is_ok_and(|written| written >= since); + if !fresh { + return Err(format!( + "the JUnit report {shown} is older than the credited run, so it is not evidence \ + of this run; make that run write it (`[profile.ci.junit]` and `[store] dir` in \ + .config/nextest.toml)" + )); + } let found = tally(&xml, filter); if found.passed > 0 { return Ok(found.passed); @@ -161,16 +188,16 @@ pub(crate) fn credit( "no test matches" }; Err(format!( - "the credited run's report {shown} holds no passed test whose name has a `::` segment starting with \ - `{filter}` ({why}); add one, or set gates.{layer}.test_filter" + "the credited run's report {shown} holds no passed test whose name has a `::` segment \ + starting with `{filter}` ({why}); add one, or set gates.{layer}.test_filter" )) } #[cfg(test)] mod tests { - use super::{credit, junit_name, junit_path, matches, tally, Tally}; + use super::{credit, junit_name, junit_path, matches, store_dir, tally, Tally}; use std::fs; - use std::path::Path; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; const REPORT: &str = r#" @@ -278,7 +305,7 @@ mod tests { const CONFIG: &str = "[profile.ci.junit]\npath = \"junit.xml\"\n"; fn run(dir: &tempdir::Dir, filter: &str) -> Result { - credit("property", dir.path(), &dir.path().join("target"), filter) + credit("property", dir.path(), filter, UNIX_EPOCH) } #[test] @@ -326,14 +353,49 @@ mod tests { assert!(why.contains("[profile.ci.junit]"), "{why}"); let bare = fixture(Some("[profile.ci]\nretries = 0\n"), Some(REPORT)); assert!(run(&bare, "property").is_err()); - assert!(junit_path(bare.path(), Path::new("t")).is_err()); + assert!(junit_path(bare.path()).is_err()); } #[test] fn the_file_name_comes_from_the_config() { let dir = fixture(Some("[profile.ci.junit]\npath = \"out/j.xml\"\n"), None); - let path = junit_path(dir.path(), Path::new("t")).unwrap(); - assert_eq!(path, Path::new("t/nextest/ci/out/j.xml")); + let path = junit_path(dir.path()).unwrap(); + assert_eq!(path, dir.path().join("target/nextest/ci/out/j.xml")); + } + + #[test] + fn a_moved_store_dir_moves_where_the_report_is_read() { + let config = "[store]\ndir = \"target/nt\"\n[profile.ci.junit]\npath = \"junit.xml\"\n"; + assert_eq!(store_dir(config), "target/nt"); + assert_eq!(store_dir(CONFIG), "target/nextest"); + assert_eq!(store_dir("not toml ["), "target/nextest"); + assert_eq!(store_dir("[store]\ndir = 3\n"), "target/nextest"); + let dir = fixture(Some(config), None); + assert_eq!( + junit_path(dir.path()).unwrap(), + dir.path().join("target/nt/ci/junit.xml") + ); + // The old location holds a matching report; the moved store holds none. + let old = fixture(Some(config), Some(REPORT)); + let why = run(&old, "property").unwrap_err(); + assert!( + why.contains("no JUnit report at target/nt/ci/junit.xml"), + "{why}" + ); + } + + #[test] + fn a_report_older_than_the_run_is_not_evidence_and_is_kept() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + let later = SystemTime::now() + Duration::from_secs(3600); + let why = credit("property", dir.path(), "property", later).unwrap_err(); + assert!(why.contains("target/nextest/ci/junit.xml"), "{why}"); + assert!(why.contains("older than the credited run"), "{why}"); + assert!(dir.path().join("target/nextest/ci/junit.xml").is_file()); + assert_eq!( + credit("property", dir.path(), "property", UNIX_EPOCH), + Ok(2) + ); } /// A scratch directory removed on drop. It removes only what it made. diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index 299d848..9d8dbed 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -264,8 +264,8 @@ impl<'a> Runner<'_, 'a> { if run.outcome != Outcome::Passed { return run; } - let target = self.plan.report.parent().unwrap_or(&self.plan.report); - if let Err(why) = evidence::credit(layer, self.dir, target, filter) { + let root = laws::workspace_root(self.dir).unwrap_or_else(|| self.dir.to_path_buf()); + if let Err(why) = evidence::credit(layer, &root, filter, self.done.started) { run.outcome = Outcome::Failed; run.note = Some(why); } @@ -276,7 +276,12 @@ impl<'a> Runner<'_, 'a> { fn take_shared(&mut self) { for (layer, from) in self.done.shared.clone() { let run = self.credited(layer, from); - println!("[{layer}] took the {from} run's outcome: {}", run.outcome); + match &run.note { + Some(note) => { + println!("[{layer}] took the {from} run, but it is no credit: {note}"); + } + None => println!("[{layer}] took the {from} run's outcome: {}", run.outcome), + } self.done.runs.push(run); self.done.timing.push(Timing { layer: layer.to_string(), diff --git a/crates/cargo-tops/src/gate/judge.rs b/crates/cargo-tops/src/gate/judge.rs index 78e8161..53313c2 100644 --- a/crates/cargo-tops/src/gate/judge.rs +++ b/crates/cargo-tops/src/gate/judge.rs @@ -111,13 +111,13 @@ fn run_problems(done: &Executed<'_>) -> Vec { let mut failed: Vec = done.not_run.clone(); failed.extend(done.runs.iter().filter_map(|run| { match done.shared.iter().find(|(layer, _)| *layer == run.layer) { - Some((_, from)) => run.outcome.fails_the_gate().then(|| { + Some((_, from)) if run.note.is_none() => run.outcome.fails_the_gate().then(|| { format!( "{}: no command of its own; the {from} run it took is {}", run.layer, run.outcome ) }), - None => run_failure(run), + _ => run_failure(run), } })); failed diff --git a/crates/cargo-tops/src/gate/plan.rs b/crates/cargo-tops/src/gate/plan.rs index bcf2e83..a5c9a57 100644 --- a/crates/cargo-tops/src/gate/plan.rs +++ b/crates/cargo-tops/src/gate/plan.rs @@ -205,11 +205,11 @@ impl Filters { } /// The shared nextest run, narrowed to the tests with a `::`-separated name -/// segment that starts with `filter` (HN-L2). The same rule reads the JUnit +/// segment that starts with `filter` (HN-L2, a word of `[A-Za-z0-9_]`). The same rule reads the JUnit /// report of a credited run, so credit equals running. fn selected_tests(filter: &str) -> Vec { - let literal = regex::escape(filter).replace('/', "\\/"); - let expr = format!("test(/(^|::){literal}/)"); + // `check` holds a filter to `[A-Za-z0-9_]+`, which is a regex literal. + let expr = format!("test(/(^|::){filter}/)"); let argv = NEXTEST.iter().map(|arg| (*arg).to_string()).chain([ "--no-tests=fail".to_string(), "-E".to_string(), @@ -505,14 +505,14 @@ mod tests { fn the_selecting_layers_run_only_tests_with_a_name_prefix() { let filters = Filters { property: "prop_".to_string(), - acceptance: "a/b.c".to_string(), + acceptance: "journey_9".to_string(), }; let shared = ["cargo", "nextest", "run", "--workspace", "--all-features"] .into_iter() .chain(["--profile", "ci", "--no-tests=fail", "-E"]); for (id, expr) in [ ("property", "test(/(^|::)prop_/)"), - ("acceptance", r"test(/(^|::)a\/b\.c/)"), + ("acceptance", "test(/(^|::)journey_9/)"), ] { let Some(Plan::Chains(chains)) = plan_for(&layer(id), &PathBuf::new(), &Base::unused(), &filters) diff --git a/crates/cargo-tops/src/laws.rs b/crates/cargo-tops/src/laws.rs index 658b894..7a6b194 100644 --- a/crates/cargo-tops/src/laws.rs +++ b/crates/cargo-tops/src/laws.rs @@ -58,9 +58,13 @@ pub(crate) fn artifact_dir(project: &Path) -> PathBuf { resolve_report_dir(project, from_env.as_deref(), meta.as_deref()) } +/// What one `cargo metadata` said: the target directory, whether it could +/// not run, and the workspace root. +type Metadata = (Option, bool, Option); + thread_local! { /// The target directory `cargo metadata` named, and whether it could not run. - static META_CACHE: RefCell, bool)>> = RefCell::new(HashMap::new()); + static META_CACHE: RefCell> = RefCell::new(HashMap::new()); #[cfg(test)] static META_SPAWNS: Cell = const { Cell::new(0) }; } @@ -69,7 +73,7 @@ fn metadata_target_once(project: &Path) -> Option { metadata_cached(project).0 } -fn metadata_cached(project: &Path) -> (Option, bool) { +fn metadata_cached(project: &Path) -> Metadata { let key = project .canonicalize() .unwrap_or_else(|_| project.to_path_buf()); @@ -126,6 +130,13 @@ pub(crate) fn metadata_unavailable(project: &Path) -> bool { !project.join("Cargo.toml").is_file() || metadata_cached(project).1 } +/// The workspace root `cargo metadata` named for `project`, from the same +/// single `cargo metadata` the report directory uses. nextest keeps its +/// store under this root. +pub(crate) fn workspace_root(project: &Path) -> Option { + metadata_cached(project).2.map(PathBuf::from) +} + #[cfg(test)] fn metadata_target(project: &Path) -> Option { metadata_probe(project).0 @@ -133,10 +144,10 @@ fn metadata_target(project: &Path) -> Option { /// The target directory, and whether `cargo metadata` could not run or /// failed. Output that names no target directory is not a failure. -fn metadata_probe(project: &Path) -> (Option, bool) { +fn metadata_probe(project: &Path) -> Metadata { let manifest = project.join("Cargo.toml"); if !manifest.is_file() { - return (None, false); + return (None, false, None); } #[cfg(test)] META_SPAWNS.with(|count| count.set(count.get() + 1)); @@ -151,16 +162,19 @@ fn metadata_probe(project: &Path) -> (Option, bool) { .arg(&manifest) .output(); let Ok(output) = output else { - return (None, true); + return (None, true, None); }; if !output.status.success() { - return (None, true); + return (None, true, None); } - let target = String::from_utf8(output.stdout) + let doc = String::from_utf8(output.stdout) .ok() - .and_then(|text| serde_json::from_str::(&text).ok()) - .and_then(|value| value.get("target_directory")?.as_str().map(str::to_string)); - (target, false) + .and_then(|text| serde_json::from_str::(&text).ok()); + let field = |key: &str| { + doc.as_ref() + .and_then(|value| value.get(key)?.as_str().map(str::to_string)) + }; + (field("target_directory"), false, field("workspace_root")) } /// The `bend` binary that reports the pinned version. @@ -806,6 +820,31 @@ mod tests { ); } + #[test] + fn the_workspace_root_is_read_from_the_same_metadata() { + let dir = std::env::temp_dir().join(format!("cargo-tops-root-{}", std::process::id())); + #[expect( + clippy::let_underscore_must_use, + reason = "already in this crate when the strict table was turned on" + )] + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("src")).unwrap(); + fs::write( + dir.join("Cargo.toml"), + "[package]\nname = \"root-fixture\"\nversion = \"0.0.0\"\nedition = \"2021\"\n", + ) + .unwrap(); + fs::write(dir.join("src/lib.rs"), "// fixture\n").unwrap(); + let root = workspace_root(&dir).expect("cargo metadata"); + assert_eq!(root, dir); + assert_eq!(workspace_root(&dir.join("absent")), None); + #[expect( + clippy::let_underscore_must_use, + reason = "already in this crate when the strict table was turned on" + )] + let _ = fs::remove_dir_all(&dir); + } + #[test] fn metadata_target_reads_target_directory() { let dir = std::env::temp_dir().join(format!("cargo-tops-meta-{}", std::process::id())); diff --git a/crates/cargo-tops/templates/rust-tops.schema.json b/crates/cargo-tops/templates/rust-tops.schema.json index 1ba0a9a..cbfe42b 100644 --- a/crates/cargo-tops/templates/rust-tops.schema.json +++ b/crates/cargo-tops/templates/rust-tops.schema.json @@ -224,8 +224,8 @@ }, "test_filter": { "type": "string", - "minLength": 1, - "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." } } }, @@ -234,8 +234,8 @@ "properties": { "test_filter": { "type": "string", - "minLength": 1, - "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." } } }, diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs index 4fc2d18..21db140 100644 --- a/crates/cargo-tops/tests/gate_credit.rs +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -19,7 +19,6 @@ use serde_json::{json, Value}; const CARGO_STUB: &str = r#"#!/bin/sh if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json"; exit 0; fi -if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/coverage-fails" ]; then exit 1; fi if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/junit.xml" ]; then mkdir -p "$REPORTS/../repo/target/nextest/ci" cp "$REPORTS/junit.xml" "$REPORTS/../repo/target/nextest/ci/junit.xml" @@ -186,6 +185,23 @@ fn junit(tests: &[(&str, bool)]) -> String { ) } +/// The `problems` of the `report.json` the last gate run wrote. +fn problems(fixture: &Fixture) -> Vec { + let report = + fs::read_to_string(fixture.repo.join("target/rust-tops/report.json")).expect("report.json"); + let report: Value = serde_json::from_str(&report).expect("report json"); + report + .get("problems") + .and_then(Value::as_array) + .map(|all| { + all.iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default() +} + /// The gate, and the verdict and detail `report.json` gives `layer`. fn gate(fixture: &Fixture, layer: &str) -> (Option, String, String, String) { let inherited = std::env::var_os("PATH").unwrap_or_default(); @@ -231,6 +247,23 @@ fn a_coverage_run_with_no_property_test_fails_property() { assert_eq!(verdict, "fail", "{text}"); assert!(detail.contains("`property`"), "{detail}"); assert!(detail.contains("target/nextest/ci/junit.xml"), "{detail}"); + // The failure line and `problems` say the same, and never blame coverage. + let problems = problems(&fixture); + assert!( + problems.iter().any(|p| p == &format!("property: {detail}")), + "{problems:?}" + ); + assert!( + !problems.iter().any(|p| p.contains("run it took is")), + "{problems:?}" + ); + assert!( + text.contains(&format!( + "[property] took the coverage run, but it is no credit: {detail}" + )), + "{text}" + ); + assert!(!text.contains("run it took is"), "{text}"); } #[test] @@ -261,6 +294,15 @@ fn a_missing_junit_report_fails_and_names_the_setting() { assert_eq!(code, Some(1), "{text}"); assert_eq!(verdict, "fail", "{text}"); assert!(detail.contains("[profile.ci.junit]"), "{detail}"); + let problems = problems(&fixture); + assert!( + problems.iter().any(|p| p == &format!("property: {detail}")), + "{problems:?}" + ); + assert!( + text.contains("but it is no credit: no JUnit report at"), + "{text}" + ); } #[test] @@ -307,27 +349,46 @@ fn acceptance_follows_the_same_rule_with_its_own_filter() { assert_eq!(verdict, "pass", "its own filter, not property's"); } -#[test] -fn a_failed_coverage_run_fails_property_without_reading_evidence() { - let fixture = fixture("covfail", &[], Some(&junit(&[("property_ok", true)]))); - fs::write(fixture.reports.join("coverage-fails"), "").expect("mark"); - let (code, _, verdict, detail) = gate(&fixture, "property"); - assert_eq!(code, Some(1)); - assert_eq!(verdict, "fail"); - assert!(!detail.contains("JUnit"), "{detail}"); +/// `check` or `gate` on the fixture: the exit code and what it printed. +fn run_tops(fixture: &Fixture, command: &str) -> (Option, String) { + let out = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([command, "--path"]) + .arg(&fixture.repo) + .output() + .expect("spawn cargo-tops"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stderr), + String::from_utf8_lossy(&out.stdout) + ); + (out.status.code(), text) } #[test] -fn unit_and_integration_need_no_matching_test() { - let fixture = fixture( - "unit", - &["unit", "integration"], - Some(&junit(&[("property_ok", true)])), - ); - for layer in ["unit", "integration"] { - let (_, text, verdict, _) = gate(&fixture, layer); - assert_eq!(verdict, "pass", "{layer}: {text}"); +fn a_test_filter_outside_the_name_characters_is_refused_with_exit_2() { + let fixture = fixture("badfilter", &[], None); + for (layer, filter) in [ + ("property", "p::prop"), + ("property", "a.b"), + ("property", "a b"), + ("property", "a/b"), + ("property", "\\"), + ("property", ""), + ("acceptance", "x::"), + ] { + set_filter(&fixture, layer, filter); + for command in ["check", "gate"] { + let (code, text) = run_tops(&fixture, command); + assert_eq!(code, Some(2), "{command} {layer} {filter:?}: {text}"); + assert!( + text.contains("test_filter"), + "{command} {layer} {filter:?}: {text}" + ); + } + set_filter(&fixture, layer, "prop_09"); } + let (code, text) = run_tops(&fixture, "check"); + assert_eq!(code, Some(0), "{text}"); } /// The kit holds the property tests its own `property` layer credits. A diff --git a/crates/cargo-tops/tests/gate_credit_nextest.rs b/crates/cargo-tops/tests/gate_credit_nextest.rs new file mode 100644 index 0000000..7e417f4 --- /dev/null +++ b/crates/cargo-tops/tests/gate_credit_nextest.rs @@ -0,0 +1,322 @@ +//! The credited run is real nextest. The cargo stub plays `cargo llvm-cov +//! nextest` by running `cargo nextest run --profile ci` on a small crate, so +//! the JUnit report is written where nextest writes it, not where the gate +//! looks. These tests cover a `CARGO_TARGET_DIR` that moves the build, a +//! `[store] dir` that moves nextest's reports, and a report left behind by an +//! earlier run (PROTOCOL.md §9.5). +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::Value; + +/// `llvm-cov nextest` writes the stand-in LCOV, then runs real nextest when +/// `$REAL_NEXTEST` is set. `llvm-cov report` writes a fully covered summary. +/// Everything else (`cargo metadata`) is the real cargo. +const CARGO_STUB: &str = r#"#!/bin/sh +if [ "$1 $2" = "llvm-cov nextest" ]; then + prev= + for arg in "$@"; do + case $prev in --output-path) mkdir -p "$(dirname "$arg")"; printf 'lcov\n' > "$arg" ;; esac + prev=$arg + done + if [ -n "${REAL_NEXTEST:-}" ]; then + PATH="$REAL_PATH" "$REAL_CARGO" nextest run --workspace --all-features --profile ci >&2 || exit 1 + fi + exit 0 +fi +if [ "$1 $2" = "llvm-cov report" ]; then + prev= + for arg in "$@"; do + case $prev in --output-path) mkdir -p "$(dirname "$arg")"; printf '%s\n' '{"data":[{"files":[{"filename":"src/lib.rs","summary":{"lines":{"count":10,"covered":10},"regions":{"count":10,"covered":10}}}]}]}' > "$arg" ;; esac + prev=$arg + done + exit 0 +fi +PATH="$REAL_PATH" exec "$REAL_CARGO" "$@" +"#; + +const WITH_TEST: &str = + "#[cfg(test)]\nmod tests {\n #[test]\n fn property_roundtrip() {}\n}\n"; +const WITHOUT_TEST: &str = "#[cfg(test)]\nmod tests {\n #[test]\n fn it_works() {}\n}\n"; + +struct Fixture { + root: PathBuf, + repo: PathBuf, + bin: PathBuf, +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).ok(); + } +} + +fn stub(bin: &Path, name: &str, body: &str) { + let path = bin.join(name); + fs::write(&path, body).expect("write stub"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod stub"); + } +} + +/// A real one-crate repo: coverage, `security-advisory` and `property` on. +fn fixture(name: &str) -> Fixture { + let root = std::env::temp_dir().join(format!("cargo-tops-nx-{name}-{}", std::process::id())); + fs::remove_dir_all(&root).ok(); + let fixture = Fixture { + repo: root.join("repo"), + bin: root.join("bin"), + root, + }; + fs::create_dir_all(&fixture.bin).expect("mkdir"); + let init = Command::new(env!("CARGO_BIN_EXE_cargo-tops")) + .args([ + "init", + "--class", + "library-core", + "--name", + "fixture", + "--path", + ]) + .arg(&fixture.repo) + .output() + .expect("spawn init"); + assert!(init.status.success(), "init failed"); + fs::write( + fixture.repo.join("Cargo.toml"), + "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .expect("write Cargo.toml"); + fs::create_dir_all(fixture.repo.join("src")).expect("mkdir src"); + set_tests(&fixture, WITH_TEST); + stub(&fixture.bin, "cargo", CARGO_STUB); + for plugin in ["cargo-llvm-cov", "cargo-deny"] { + stub(&fixture.bin, plugin, "#!/bin/sh\nexit 0\n"); + } + enable_required_layers(&fixture); + fixture +} + +/// Coverage, `security-advisory` and `property` on; every other layer off. +fn enable_required_layers(fixture: &Fixture) { + let path = fixture.repo.join("rust-tops.yaml"); + let mut doc: Value = + serde_saphyr::from_str(&fs::read_to_string(&path).expect("read")).expect("yaml"); + for layer in doc + .pointer_mut("/layers") + .and_then(Value::as_array_mut) + .expect("layers") + { + let id = layer.get("id").and_then(Value::as_str).unwrap_or_default(); + let on = matches!(id, "coverage" | "security-advisory" | "property"); + layer + .as_object_mut() + .expect("a layer is an object") + .insert("enabled".to_string(), Value::Bool(on)); + } + fs::write(&path, serde_json::to_string_pretty(&doc).expect("json")).expect("write yaml"); +} + +fn set_tests(fixture: &Fixture, source: &str) { + fs::write(fixture.repo.join("src/lib.rs"), source).expect("write lib.rs"); +} + +fn append_nextest_config(fixture: &Fixture, text: &str) { + let path = fixture.repo.join(".config/nextest.toml"); + let old = fs::read_to_string(&path).expect("init writes .config/nextest.toml"); + fs::write(path, format!("{old}\n{text}\n")).expect("write nextest.toml"); +} + +/// What one gate run left: the exit code, stdout and stderr, and the +/// `property` row and `problems` of `report.json`. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Credited { + RealNextest, + /// The credited run writes no report. + Idle, +} + +struct Gate { + code: Option, + text: String, + verdict: String, + detail: String, + problems: Vec, +} + +/// The first `cargo` on the inherited `PATH`, before any stub. +fn real_cargo(path: &std::ffi::OsStr) -> PathBuf { + std::env::split_paths(path) + .map(|dir| dir.join("cargo")) + .find(|candidate| candidate.is_file()) + .expect("a real cargo on PATH") +} + +/// Runs the gate. `credited` says whether the credited run is real nextest; +/// `target_dir` is a `CARGO_TARGET_DIR`. +fn gate(fixture: &Fixture, credited: Credited, target_dir: Option<&Path>) -> Gate { + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let dirs = std::iter::once(fixture.bin.clone()).chain(std::env::split_paths(&inherited)); + let mut cmd = Command::new(env!("CARGO_BIN_EXE_cargo-tops")); + cmd.args(["gate", "--path"]) + .arg(&fixture.repo) + .env("PATH", std::env::join_paths(dirs).expect("PATH")) + .env("REAL_PATH", &inherited) + .env("REAL_CARGO", real_cargo(&inherited)) + .env_remove("CARGO_TARGET_DIR") + .env_remove("REAL_NEXTEST"); + if credited == Credited::RealNextest { + cmd.env("REAL_NEXTEST", "1"); + } + if let Some(dir) = target_dir { + cmd.env("CARGO_TARGET_DIR", dir); + } + let out = cmd.output().expect("spawn cargo-tops"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let report_dir = target_dir.map_or_else(|| fixture.repo.join("target"), Path::to_path_buf); + let (verdict, detail, problems) = read_report(&report_dir); + Gate { + code: out.status.code(), + verdict, + detail, + problems, + text, + } +} + +/// The `property` row's verdict and detail, and the `problems`, of the +/// `report.json` under `report_dir`. +fn read_report(report_dir: &Path) -> (String, String, Vec) { + let report = fs::read_to_string(report_dir.join("rust-tops/report.json")) + .expect("the gate wrote report.json"); + let report: Value = serde_json::from_str(&report).expect("report json"); + let row = report + .get("layers") + .and_then(Value::as_array) + .and_then(|rows| { + rows.iter() + .find(|row| row.get("id").and_then(Value::as_str) == Some("property")) + }) + .expect("a property row"); + let text_of = |key: &str| { + row.get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + let problems = report + .get("problems") + .and_then(Value::as_array) + .map(|all| { + all.iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + (text_of("verdict"), text_of("detail"), problems) +} + +#[test] +fn a_cargo_target_dir_does_not_move_the_evidence() { + let fixture = fixture("target-dir"); + let target = fixture.root.join("elsewhere"); + let run = gate(&fixture, Credited::RealNextest, Some(&target)); + assert_eq!(run.code, Some(0), "{}", run.text); + assert_eq!(run.verdict, "pass", "{}", run.text); + // nextest wrote under the workspace root, not under the target dir. + assert!(fixture.repo.join("target/nextest/ci/junit.xml").is_file()); + assert!(!target.join("nextest").exists()); +} + +#[test] +fn a_moved_store_dir_is_read_where_nextest_wrote_it() { + let fixture = fixture("store"); + append_nextest_config(&fixture, "[store]\ndir = \"target/nt\""); + let run = gate(&fixture, Credited::RealNextest, None); + assert_eq!(run.code, Some(0), "{}", run.text); + assert_eq!(run.verdict, "pass", "{}", run.text); + assert!(fixture.repo.join("target/nt/ci/junit.xml").is_file()); +} + +#[test] +fn a_report_from_an_earlier_run_is_stale_and_kept() { + let fixture = fixture("stale"); + let first = gate(&fixture, Credited::RealNextest, None); + assert_eq!(first.verdict, "pass", "{}", first.text); + let old = fixture.repo.join("target/nextest/ci/junit.xml"); + assert!(old.is_file()); + + // The test goes, the store moves, and the next run writes elsewhere: + // the old report still holds the test, and is not read. + set_tests(&fixture, WITHOUT_TEST); + append_nextest_config(&fixture, "[store]\ndir = \"target/nt\""); + let moved = gate(&fixture, Credited::RealNextest, None); + assert_eq!(moved.code, Some(1), "{}", moved.text); + assert_eq!(moved.verdict, "fail", "{}", moved.text); + assert!( + moved.detail.contains("target/nt/ci/junit.xml"), + "{}", + moved.detail + ); + assert!(moved.detail.contains("no test matches"), "{}", moved.detail); + + // A credited run that writes no report leaves only the old one behind. + let config = fixture.repo.join(".config/nextest.toml"); + let text = fs::read_to_string(&config) + .expect("read") + .replace("target/nt", "target/nextest"); + fs::write(&config, text).expect("write"); + let idle = gate(&fixture, Credited::Idle, None); + assert_eq!(idle.code, Some(1), "{}", idle.text); + assert_eq!(idle.verdict, "fail", "{}", idle.text); + assert!( + idle.detail.contains("target/nextest/ci/junit.xml"), + "{}", + idle.detail + ); + assert!( + idle.detail.contains("older than the credited run"), + "{}", + idle.detail + ); + assert!(old.is_file(), "the gate never deletes the report"); +} + +#[test] +fn a_failed_evidence_check_is_worded_once_in_detail_problems_and_stdout() { + let fixture = fixture("wording"); + set_tests(&fixture, WITHOUT_TEST); + let run = gate(&fixture, Credited::RealNextest, None); + assert_eq!(run.code, Some(1), "{}", run.text); + let line = "no passed test whose name has a `::` segment starting with `property`"; + assert!(run.detail.contains(line), "{}", run.detail); + let problem = run + .problems + .iter() + .find(|problem| problem.starts_with("property:")) + .expect("a property problem"); + assert!(problem.contains(line), "{problem}"); + assert!(problem.contains("target/nextest/ci/junit.xml"), "{problem}"); + assert!( + run.text.contains(&format!( + "[property] took the coverage run, but it is no credit: {}", + run.detail + )), + "{}", + run.text + ); + assert!(!run.text.contains("run it took is Failed"), "{}", run.text); + assert!( + !run.problems.iter().any(|p| p.contains("run it took is")), + "{:?}", + run.problems + ); +} diff --git a/crates/cargo-tops/tests/gate_floors.rs b/crates/cargo-tops/tests/gate_floors.rs index 155394b..2dacac0 100644 --- a/crates/cargo-tops/tests/gate_floors.rs +++ b/crates/cargo-tops/tests/gate_floors.rs @@ -38,7 +38,7 @@ for arg in "$@"; do name=$(basename "$arg") if [ -f "$REPORTS/$name" ]; then cp "$REPORTS/$name" "$arg"; else printf 'lcov\n' > "$arg"; fi if [ "$1 $2" = "llvm-cov nextest" ]; then - j="$(dirname "$arg")/../nextest/ci" + j="$PWD/target/nextest/ci" mkdir -p "$j" printf '%s\n' '' > "$j/junit.xml" fi diff --git a/crates/cargo-tops/tests/laws_gate.rs b/crates/cargo-tops/tests/laws_gate.rs index faf16cb..41c0144 100644 --- a/crates/cargo-tops/tests/laws_gate.rs +++ b/crates/cargo-tops/tests/laws_gate.rs @@ -57,7 +57,7 @@ for arg in "$@"; do mkdir -p "$(dirname "$arg")" printf '%s\n' '{"data":[{"files":[{"filename":"src/lib.rs","summary":{"lines":{"count":1,"covered":1},"regions":{"count":1,"covered":1}}}]}],"entries":[]}' > "$arg" if [ "$1 $2" = "llvm-cov nextest" ]; then - j="$(dirname "$arg")/../nextest/ci" + j="$PWD/target/nextest/ci" mkdir -p "$j" printf '%s\n' '' > "$j/junit.xml" fi @@ -1642,10 +1642,14 @@ fn a_custom_cargo_target_dir_holds_lcov_crap_and_the_laws_report() { "{name} is not under CARGO_TARGET_DIR: {text}" ); } + // The gate's artifacts stay out of the project `target/`. nextest does + // not follow `CARGO_TARGET_DIR` for its own store, so the stub's JUnit + // report is under the project `target/nextest`, as real nextest's is. assert!( - !fixture.repo.join("target").exists(), + !fixture.repo.join("target/rust-tops").exists(), "the project target/ got an artifact while CARGO_TARGET_DIR was set: {text}" ); + assert!(fixture.repo.join("target/nextest/ci/junit.xml").is_file()); let lcov = report.join("lcov.info"); let log = cargo_log(&fixture); for run in [ diff --git a/schema/rust-tops.schema.json b/schema/rust-tops.schema.json index 1ba0a9a..cbfe42b 100644 --- a/schema/rust-tops.schema.json +++ b/schema/rust-tops.schema.json @@ -224,8 +224,8 @@ }, "test_filter": { "type": "string", - "minLength": 1, - "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the property layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: property. A credited run must hold a passed test that matches (PROTOCOL.md section 9)." } } }, @@ -234,8 +234,8 @@ "properties": { "test_filter": { "type": "string", - "minLength": 1, - "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." + "pattern": "^[A-Za-z0-9_]+$", + "description": "A test counts for the acceptance layer when a :: separated segment of its nextest name starts with this text: letters, digits and underscore only. Default: acceptance. A credited run must hold a passed test that matches (PROTOCOL.md section 6)." } } }, From 31b534fbc2eb31b00709de0e9877776d783d7be1 Mon Sep 17 00:00:00 2001 From: VirtualMachinist Date: Fri, 20 Nov 2026 00:15:28 -0500 Subject: [PATCH 9/9] Stub only cargo metadata in the real-nextest gate tests, and tighten the stale rule The real-nextest fixture passed every cargo subcommand but metadata to a real cargo that has no deny on a runner, so security-advisory failed the gate; only metadata is real now. Every evidence failure names the layer and its filter. The stale rule compares the report's mtime with the credited run's own start and refuses a future date. The failed-coverage and unit/integration tests come back, and the real-nextest tests get a longer slow timeout. Co-Authored-By: Claude Sonnet 5.5 --- .config/nextest.toml | 8 ++++ PROTOCOL.md | 2 +- crates/cargo-tops/src/evidence.rs | 35 ++++++++++++---- crates/cargo-tops/src/gate/execute.rs | 41 +++++++++++++++---- crates/cargo-tops/tests/gate_credit.rs | 33 ++++++++++++++- .../cargo-tops/tests/gate_credit_nextest.rs | 11 +++-- 6 files changed, 109 insertions(+), 21 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index ea98ebe..ed829ad 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -48,3 +48,11 @@ slow-timeout = { period = "10s", terminate-after = 4 } [[profile.ci.overrides]] filter = "test(=fresh_init_then_gate_passes_for_every_class)" slow-timeout = { period = "10s", terminate-after = 4 } + +# These tests run real `cargo nextest` on a small crate, one build each, for +# the kit suite only; they match nothing in an adopter. Alone each ran in +# 1.4 s to 6.2 s. Under the full suite on a loaded runner they take longer, +# so they get 60 s; the profile timeout is not raised. +[[profile.ci.overrides]] +filter = "binary(gate_credit_nextest)" +slow-timeout = { period = "10s", terminate-after = 6 } diff --git a/PROTOCOL.md b/PROTOCOL.md index 5661e3a..6fdcde7 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -605,7 +605,7 @@ A good property kills operator mutants that example tests miss (`<` vs `<=`, off `gates.property.test_filter` (default `property`) names the tests that count as property tests: a test counts when one `::`-separated segment of its nextest name starts with the filter. The `property` layer passes only on a passed test that matches. - **It runs itself** when no coverage run is credited. Its command is the shared nextest run with `-E 'test(/(^|::)/)'` and `--no-tests=fail`, so a filter that selects nothing fails the layer. A filter is letters, digits and underscore only (`^[A-Za-z0-9_]+$`), so it is a plain regex literal and needs no escaping. `check` refuses any other value, and the gate exits 2. -- **It takes the coverage run** when its command carries `--workspace`, `--all-features` and `--profile ci`. The coverage run must also have written a nextest JUnit report, holding at least one passed test that matches. The gate reads the file where nextest wrote it: the workspace root from `cargo metadata`, then nextest's `[store] dir` from `.config/nextest.toml` (default `target/nextest`), then `ci/` with `path` from `[profile.ci.junit]`. `CARGO_TARGET_DIR` and `build.target-dir` do not move it. A report older than the start of the gate run is stale and is not evidence; the gate never deletes it. No match, only failed or skipped matches, no report, or a stale report is `fail`. The report's `detail`, the failure line and `problems` all name the filter, the path read (relative to the workspace root), and the fix. +- **It takes the coverage run** when its command carries `--workspace`, `--all-features` and `--profile ci`. The coverage run must also have written a nextest JUnit report, holding at least one passed test that matches. The gate reads the file where nextest wrote it: the workspace root from `cargo metadata`, then nextest's `[store] dir` from `.config/nextest.toml` (default `target/nextest`), then `ci/` with `path` from `[profile.ci.junit]`. `CARGO_TARGET_DIR` and `build.target-dir` do not move it. A report dated before the credited run began, or in the future, was not written by that run, so it is stale and not evidence; the gate never deletes it. No match, only failed or skipped matches, no report, or a stale report is `fail`. The report's `detail`, the failure line and `problems` all name the filter, the path read (relative to the workspace root), and the fix. - **The same rule** applies to the whole workspace, not to one crate, so credit equals running. The same keys exist for `acceptance` under `gates.acceptance` (§6.5.1). diff --git a/crates/cargo-tops/src/evidence.rs b/crates/cargo-tops/src/evidence.rs index 264375d..fe627fa 100644 --- a/crates/cargo-tops/src/evidence.rs +++ b/crates/cargo-tops/src/evidence.rs @@ -156,7 +156,9 @@ pub(crate) fn credit( filter: &str, since: SystemTime, ) -> Result { - let path = junit_path(root)?; + let who = format!("`{layer}` (filter `{filter}`)"); + let path = + junit_path(root).map_err(|why| format!("{why}; {who} has no evidence without it"))?; let shown = path .strip_prefix(root) .unwrap_or(&path) @@ -164,18 +166,19 @@ pub(crate) fn credit( .to_string(); let xml = fs::read_to_string(&path).map_err(|_| { format!( - "no JUnit report at {shown}; the credited run must write one \ - (`[profile.ci.junit]` and `[store] dir` in .config/nextest.toml)" + "no JUnit report at {shown}, so {who} has no evidence; the credited run must write \ + one (`[profile.ci.junit]` and `[store] dir` in .config/nextest.toml)" ) })?; + // Written during the credited run: not before it began, not after now. let fresh = fs::metadata(&path) .and_then(|meta| meta.modified()) - .is_ok_and(|written| written >= since); + .is_ok_and(|written| written >= since && written <= SystemTime::now()); if !fresh { return Err(format!( - "the JUnit report {shown} is older than the credited run, so it is not evidence \ - of this run; make that run write it (`[profile.ci.junit]` and `[store] dir` in \ - .config/nextest.toml)" + "the JUnit report {shown} was not written during the credited run (it is older \ + than the run, or dated in the future), so it is no evidence for {who}; make that \ + run write it (`[profile.ci.junit]` and `[store] dir` in .config/nextest.toml)" )); } let found = tally(&xml, filter); @@ -342,6 +345,7 @@ mod tests { why.contains("no JUnit report at target/nextest/ci/junit.xml"), "{why}" ); + assert!(why.contains("`property` (filter `property`)"), "{why}"); assert!(why.contains("[profile.ci.junit]"), "{why}"); } @@ -350,6 +354,7 @@ mod tests { let dir = fixture(None, Some(REPORT)); let why = run(&dir, "property").unwrap_err(); assert!(why.starts_with("no JUnit report is configured"), "{why}"); + assert!(why.contains("`property` (filter `property`)"), "{why}"); assert!(why.contains("[profile.ci.junit]"), "{why}"); let bare = fixture(Some("[profile.ci]\nretries = 0\n"), Some(REPORT)); assert!(run(&bare, "property").is_err()); @@ -384,13 +389,27 @@ mod tests { ); } + #[test] + fn a_report_dated_in_the_future_is_not_evidence() { + let dir = fixture(Some(CONFIG), Some(REPORT)); + let file = fs::File::options() + .write(true) + .open(dir.path().join("target/nextest/ci/junit.xml")) + .unwrap(); + file.set_modified(SystemTime::now() + Duration::from_secs(3600)) + .unwrap(); + let why = run(&dir, "property").unwrap_err(); + assert!(why.contains("not written during the credited run"), "{why}"); + } + #[test] fn a_report_older_than_the_run_is_not_evidence_and_is_kept() { let dir = fixture(Some(CONFIG), Some(REPORT)); let later = SystemTime::now() + Duration::from_secs(3600); let why = credit("property", dir.path(), "property", later).unwrap_err(); assert!(why.contains("target/nextest/ci/junit.xml"), "{why}"); - assert!(why.contains("older than the credited run"), "{why}"); + assert!(why.contains("not written during the credited run"), "{why}"); + assert!(why.contains("`property` (filter `property`)"), "{why}"); assert!(dir.path().join("target/nextest/ci/junit.xml").is_file()); assert_eq!( credit("property", dir.path(), "property", UNIX_EPOCH), diff --git a/crates/cargo-tops/src/gate/execute.rs b/crates/cargo-tops/src/gate/execute.rs index 9d8dbed..285f4bb 100644 --- a/crates/cargo-tops/src/gate/execute.rs +++ b/crates/cargo-tops/src/gate/execute.rs @@ -21,9 +21,26 @@ pub(crate) struct Prepared { pub(crate) judge: Option, } +/// When a layer's commands began, as wall time and as a stopwatch. +struct Clock { + began: SystemTime, + watch: Instant, +} + +impl Clock { + fn start() -> Self { + Self { + began: SystemTime::now(), + watch: Instant::now(), + } + } +} + /// Wall time and tool version of one layer's commands. pub(crate) struct Timing { pub(crate) layer: String, + /// When its commands began: a report older than this is not theirs. + pub(crate) began: SystemTime, pub(crate) seconds: f64, pub(crate) tool_version: Option, } @@ -188,7 +205,7 @@ impl<'a> Runner<'_, 'a> { ); self.done.listed += 1; if self.live { - let clock = Instant::now(); + let clock = Clock::start(); let run = run_mutation(self.dir, &layer.id, mutation).map_err(Fault::env)?; // The tool is asked for its version only when it ran: a vacuous // or failed diff runs no `cargo mutants`. @@ -196,7 +213,7 @@ impl<'a> Runner<'_, 'a> { && run.cmd.contains(&mutation.mutants.display())) .then(|| version_of(self.dir, &mutation.mutants.argv, &mut self.versions)) .flatten(); - self.done.record(run, clock, version); + self.done.record(run, &clock, version); } Ok(()) } @@ -216,7 +233,7 @@ impl<'a> Runner<'_, 'a> { if !self.live { return Ok(()); } - let clock = Instant::now(); + let clock = Clock::start(); let run = match pin { Some(pin) => run_pinned_crap(self.dir, &layer.id, chain, pin), None => run_chain(self.dir, &layer.id, chain), @@ -234,7 +251,7 @@ impl<'a> Runner<'_, 'a> { .filter(|_| run.outcome != Outcome::ToolMissing) .and_then(|argv| version_of(self.dir, argv, &mut self.versions)), }; - self.done.record(run, clock, version); + self.done.record(run, &clock, version); Ok(()) } @@ -265,7 +282,14 @@ impl<'a> Runner<'_, 'a> { return run; } let root = laws::workspace_root(self.dir).unwrap_or_else(|| self.dir.to_path_buf()); - if let Err(why) = evidence::credit(layer, &root, filter, self.done.started) { + // A report older than the credited run's own start is not its report. + let since = self + .done + .timing + .iter() + .find(|timing| timing.layer == from) + .map_or(self.done.started, |timing| timing.began); + if let Err(why) = evidence::credit(layer, &root, filter, since) { run.outcome = Outcome::Failed; run.note = Some(why); } @@ -285,6 +309,7 @@ impl<'a> Runner<'_, 'a> { self.done.runs.push(run); self.done.timing.push(Timing { layer: layer.to_string(), + began: SystemTime::now(), seconds: 0.0, tool_version: None, }); @@ -294,10 +319,11 @@ impl<'a> Runner<'_, 'a> { impl Executed<'_> { /// Keep `run` with the time since `clock` and the tool's version. - fn record(&mut self, run: Run, clock: Instant, tool_version: Option) { + fn record(&mut self, run: Run, clock: &Clock, tool_version: Option) { self.timing.push(Timing { layer: run.layer.clone(), - seconds: clock.elapsed().as_secs_f64(), + began: clock.began, + seconds: clock.watch.elapsed().as_secs_f64(), tool_version, }); self.runs.push(run); @@ -500,6 +526,7 @@ mod tests { fn timing(layer: &str, seconds: f64) -> Timing { Timing { layer: layer.to_string(), + began: SystemTime::UNIX_EPOCH, seconds, tool_version: None, } diff --git a/crates/cargo-tops/tests/gate_credit.rs b/crates/cargo-tops/tests/gate_credit.rs index 21db140..5ee6b15 100644 --- a/crates/cargo-tops/tests/gate_credit.rs +++ b/crates/cargo-tops/tests/gate_credit.rs @@ -1,6 +1,6 @@ //! A credited `property` or `acceptance` layer passes only when the credited -//! run's nextest JUnit report holds a passed test whose name contains the -//! layer's `test_filter` (PROTOCOL.md §6, §9). +//! run's nextest JUnit report holds a passed test with a `::` segment of its +//! name starting with the layer's `test_filter` (PROTOCOL.md §6, §9). //! //! Cargo and its plugins are stubs. The cargo stub plays the coverage run: //! it writes the JUnit report the test put in the fixture's `reports/` to @@ -19,6 +19,7 @@ use serde_json::{json, Value}; const CARGO_STUB: &str = r#"#!/bin/sh if [ "$1 $2" = "crap --version" ]; then echo "cargo-crap 0.9.0"; exit 0; fi if [ "$1" = metadata ]; then cat "$REPORTS/metadata.json"; exit 0; fi +if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/coverage-fails" ]; then exit 1; fi if [ "$1 $2" = "llvm-cov nextest" ] && [ -f "$REPORTS/junit.xml" ]; then mkdir -p "$REPORTS/../repo/target/nextest/ci" cp "$REPORTS/junit.xml" "$REPORTS/../repo/target/nextest/ci/junit.xml" @@ -305,6 +306,34 @@ fn a_missing_junit_report_fails_and_names_the_setting() { ); } +#[test] +fn a_failed_coverage_run_fails_property_without_reading_evidence() { + let fixture = fixture("covfail", &[], Some(&junit(&[("property_ok", true)]))); + fs::write(fixture.reports.join("coverage-fails"), "").expect("mark"); + let (code, text, verdict, detail) = gate(&fixture, "property"); + assert_eq!(code, Some(1), "{text}"); + assert_eq!(verdict, "fail"); + // Coverage really failed, so the problem blames it, and no evidence + // text is given for a run that never produced any. + assert!(!detail.contains("JUnit"), "{detail}"); + assert!(problems(&fixture) + .iter() + .any(|p| p.contains("run it took is Failed"))); +} + +#[test] +fn unit_and_integration_need_no_matching_test() { + let fixture = fixture( + "unit", + &["unit", "integration"], + Some(&junit(&[("property_ok", true)])), + ); + for layer in ["unit", "integration"] { + let (_, text, verdict, _) = gate(&fixture, layer); + assert_eq!(verdict, "pass", "{layer}: {text}"); + } +} + #[test] fn a_custom_test_filter_is_honoured() { let tests = junit(&[("prop_roundtrip", true)]); diff --git a/crates/cargo-tops/tests/gate_credit_nextest.rs b/crates/cargo-tops/tests/gate_credit_nextest.rs index 7e417f4..231548b 100644 --- a/crates/cargo-tops/tests/gate_credit_nextest.rs +++ b/crates/cargo-tops/tests/gate_credit_nextest.rs @@ -12,7 +12,7 @@ use serde_json::Value; /// `llvm-cov nextest` writes the stand-in LCOV, then runs real nextest when /// `$REAL_NEXTEST` is set. `llvm-cov report` writes a fully covered summary. -/// Everything else (`cargo metadata`) is the real cargo. +/// `cargo metadata` is the real cargo; any other subcommand passes. const CARGO_STUB: &str = r#"#!/bin/sh if [ "$1 $2" = "llvm-cov nextest" ]; then prev= @@ -33,7 +33,12 @@ if [ "$1 $2" = "llvm-cov report" ]; then done exit 0 fi -PATH="$REAL_PATH" exec "$REAL_CARGO" "$@" +# Only `cargo metadata` is real. Any other subcommand (`cargo deny`, which +# the runners do not have) passes, so no other layer can fail this gate. +if [ "$1" = metadata ]; then + PATH="$REAL_PATH" exec "$REAL_CARGO" "$@" +fi +exit 0 "#; const WITH_TEST: &str = @@ -283,7 +288,7 @@ fn a_report_from_an_earlier_run_is_stale_and_kept() { idle.detail ); assert!( - idle.detail.contains("older than the credited run"), + idle.detail.contains("not written during the credited run"), "{}", idle.detail );