diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a425bd30..0592bd9025 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,16 @@ workspace carried at those versions without a tag. ## Unreleased +- A one-pass writer for the counts of the bootstrap count cases, #92. + Under `LEXLEAN_COUNTS=fast` each of `BS-05` to `BS-08` and `BS-11` to + `BS-15` records every count it checks in `target/count-report/` instead of + failing at the first that moved, and stops before its evaluator and rustc + runs (`LEXLEAN_COUNTS=report` runs it whole); `cargo xtask counts --write` + carries the moved counts into the case's constants, its §31 row, register + statement and scenario, and the prose of §17.23 that repeats them, and + lists every number it cannot place unambiguously. A landing that moved + the counts took a run and a hand mirror per stage of the cases; it takes + one pass. Unset, the cases assert as before. The register is unchanged. - `rustix` 1.1.6 (RUSTSEC-2026-0486: the extended-attribute functions marked unwritten bytes as initialized for a zero-length buffer), so `cargo deny` passes; `errno` and `tempfile` resolve `windows-sys` 0.59, the version `fs4` 0.13 and `rustix` 1.1.6 both accept. diff --git a/crates/conformance/src/cases/bootstrap/rust_assemble.rs b/crates/conformance/src/cases/bootstrap/rust_assemble.rs index d0c32590ed..59f0854217 100644 --- a/crates/conformance/src/cases/bootstrap/rust_assemble.rs +++ b/crates/conformance/src/cases/bootstrap/rust_assemble.rs @@ -351,11 +351,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-14", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-14") { + return; + } let checked = support::checked_project(&project); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); let roots = Roots::of(&project); diff --git a/crates/conformance/src/cases/bootstrap/rust_bodies.rs b/crates/conformance/src/cases/bootstrap/rust_bodies.rs index a4b5bd3232..6de7dc4aca 100644 --- a/crates/conformance/src/cases/bootstrap/rust_bodies.rs +++ b/crates/conformance/src/cases/bootstrap/rust_bodies.rs @@ -618,11 +618,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-11", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-11") { + return; + } let checked = support::checked_project(&project); let uses = super::lower(&checked, &project, TYPING, "useTrees"); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); diff --git a/crates/conformance/src/cases/bootstrap/rust_loops.rs b/crates/conformance/src/cases/bootstrap/rust_loops.rs index e275dc1a1e..a8a1dae406 100644 --- a/crates/conformance/src/cases/bootstrap/rust_loops.rs +++ b/crates/conformance/src/cases/bootstrap/rust_loops.rs @@ -494,11 +494,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-13", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-13") { + return; + } let checked = support::checked_project(&project); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); let roots = Roots::of(&project); diff --git a/crates/conformance/src/cases/bootstrap/rust_lower.rs b/crates/conformance/src/cases/bootstrap/rust_lower.rs index 96eab30759..3783ee4508 100644 --- a/crates/conformance/src/cases/bootstrap/rust_lower.rs +++ b/crates/conformance/src/cases/bootstrap/rust_lower.rs @@ -912,11 +912,18 @@ pub(super) fn case() { .iter() .filter(|vector| vector.nodes <= EVALUATOR_NODES) .collect(); - assert_eq!( - (rules.len(), all.len(), evaluator.len()), - (RULE_PROGRAMS, VECTORS, VECTORS_WITHIN), - "the programs of the rules, the vectors, and those the evaluator is given" + crate::counts!( + "BS-07", + "the programs of the rules, the vectors, and those the evaluator is given", + { + RULE_PROGRAMS: rules.len(), + VECTORS: all.len(), + VECTORS_WITHIN: evaluator.len(), + } ); + if crate::counts::stop_before_runs("BS-07") { + return; + } eprintln!( "BS-07: {} vectors ({} within {EVALUATOR_NODES} nodes) ({:?})", all.len(), diff --git a/crates/conformance/src/cases/bootstrap/rust_moves.rs b/crates/conformance/src/cases/bootstrap/rust_moves.rs index 92a98f56da..ab77a8de42 100644 --- a/crates/conformance/src/cases/bootstrap/rust_moves.rs +++ b/crates/conformance/src/cases/bootstrap/rust_moves.rs @@ -519,15 +519,21 @@ pub(super) fn case() { .map(run) .collect(); assert_eq!( - ( - base.len(), - small_runs.len(), - evaluator_mutants.len(), - rustc_mutants.len() - ), - (CRATES, WITHIN, EVALUATOR_MUTANTS, RUSTC_MUTANTS), - "the crates the moves are run on, those within the evaluator's bound, and the mutants" + (evaluator_mutants.len(), rustc_mutants.len()), + (EVALUATOR_MUTANTS, RUSTC_MUTANTS), + "the mutants" ); + crate::counts!( + "BS-08", + "the crates the moves are run on, and those within the evaluator's bound", + { + CRATES: base.len(), + WITHIN: small_runs.len(), + } + ); + if crate::counts::stop_before_runs("BS-08") { + return; + } eprintln!( "BS-08: {} crates ({} within {EVALUATOR_NODES} expressions), {} + {} mutants ({:?})", base.len(), diff --git a/crates/conformance/src/cases/bootstrap/rust_package.rs b/crates/conformance/src/cases/bootstrap/rust_package.rs index 1541366c2d..851b2d3217 100644 --- a/crates/conformance/src/cases/bootstrap/rust_package.rs +++ b/crates/conformance/src/cases/bootstrap/rust_package.rs @@ -916,27 +916,22 @@ pub(super) fn case() { within.len(), started.elapsed() ); - assert_eq!( - ( - manifests.len(), - renders.len(), - rules.len(), - shaped.len(), - mutants.len(), - render_mutants.len(), - within.len() - ), - ( - MANIFESTS, - ACCEPTED, - HEADER_RULES, - RENDER_RULES, - HEADER_MUTANTS, - RENDER_MUTANTS, - EVALUATOR_RENDERS - ), - "the manifests, the accepted ones, the rule inputs, the mutants, and the renders the evaluator is given" + crate::counts!( + "BS-15", + "the manifests, the accepted ones, the rule inputs, the mutants, and the renders the evaluator is given", + { + MANIFESTS: manifests.len(), + ACCEPTED: renders.len(), + HEADER_RULES: rules.len(), + RENDER_RULES: shaped.len(), + HEADER_MUTANTS: mutants.len(), + RENDER_MUTANTS: render_mutants.len(), + EVALUATOR_RENDERS: within.len(), + } ); + if crate::counts::stop_before_runs("BS-15") { + return; + } let checked = support::checked_project(&project); let check_root = super::lower(&checked, &project, MODULE, "checkManifest"); diff --git a/crates/conformance/src/cases/bootstrap/rust_print.rs b/crates/conformance/src/cases/bootstrap/rust_print.rs index 8caf850b8a..2ddd381560 100644 --- a/crates/conformance/src/cases/bootstrap/rust_print.rs +++ b/crates/conformance/src/cases/bootstrap/rust_print.rs @@ -1318,16 +1318,16 @@ pub(super) fn case() { "the committed library ends with the printed items" ); } - assert_eq!( - (manifests, fixtures, distinct, all.len(), packages.len()), - ( - COMMITTED_MANIFESTS, - COMMITTED_FIXTURES, - COMMITTED_PROGRAMS, - COMMITTED_CRATES, - COMMITTED_PACKAGES - ), - "the committed manifests, fixtures, distinct programs, distinct crates, and packages" + crate::counts!( + "BS-06", + "the committed manifests, fixtures, distinct programs, distinct crates, and packages", + { + COMMITTED_MANIFESTS: manifests, + COMMITTED_FIXTURES: fixtures, + COMMITTED_PROGRAMS: distinct, + COMMITTED_CRATES: all.len(), + COMMITTED_PACKAGES: packages.len(), + } ); eprintln!( "BS-06: {manifests} manifests, {fixtures} fixtures, {distinct} programs, {} crates, {} packages ({:?})", @@ -1386,25 +1386,18 @@ pub(super) fn case() { started.elapsed() ); - assert_eq!( - ( - crate_runs.len(), - committed_count, - refused(&crate_runs), - expr_runs.len(), - hand_exprs, - refused(&expr_runs) - ), - ( - VECTORS_CRATE, - COMMITTED_DISTINCT, - REFUSED_CRATE, - VECTORS_EXPR, - HAND_EXPR, - 0 - ), - "the vectors of printCrate and printExpr, the committed crates among them, and the refused" + crate::counts!( + "BS-06", + "the vectors of printCrate and printExpr, the committed crates among them, and the refused", + { + VECTORS_CRATE: crate_runs.len(), + COMMITTED_DISTINCT: committed_count, + REFUSED_CRATE: refused(&crate_runs), + VECTORS_EXPR: expr_runs.len(), + HAND_EXPR: hand_exprs, + } ); + assert_eq!(refused(&expr_runs), 0, "no printExpr vector is refused"); // The corpora of the normative host are made here, so that every count // the row states is asserted before the first leg runs. let evaluator_crates: Vec = crate_runs @@ -1422,23 +1415,20 @@ pub(super) fn case() { let (seeded_crates, seeded_exprs) = mutants(&bases, &mut seeded, EVALUATOR_MUTANTS); let mut large = super::Seeded(0x0B55_0066); let (large_crates, large_exprs) = mutants(&bases, &mut large, RUSTC_MUTANTS); - assert_eq!( - ( - evaluator_crates.len(), - evaluator_exprs.len(), - seeded_crates.len(), - seeded_exprs.len(), - large_crates.len() + large_exprs.len() - ), - ( - EVALUATOR_CRATE, - VECTORS_EXPR, - SEEDED_CRATE, - SEEDED_EXPR, - RUSTC_SEEDED_RUNS - ), - "the vectors and the seeded mutants of the evaluator, and the further mutants of rustc" + crate::counts!( + "BS-06", + "the vectors and the seeded mutants of the evaluator, and the further mutants of rustc", + { + EVALUATOR_CRATE: evaluator_crates.len(), + VECTORS_EXPR: evaluator_exprs.len(), + SEEDED_CRATE: seeded_crates.len(), + SEEDED_EXPR: seeded_exprs.len(), + RUSTC_SEEDED_RUNS: large_crates.len() + large_exprs.len(), + } ); + if crate::counts::stop_before_runs("BS-06") { + return; + } let checked = support::checked_project(&project); let crate_root = super::lower(&checked, &project, MODULE, "printCrate"); let expr_root = super::lower(&checked, &project, MODULE, "printExpr"); @@ -1496,7 +1486,13 @@ pub(super) fn case() { ], "seeded", ); - assert_eq!(large_runs, RUSTC_SEEDED_RUNS); + crate::counts!( + "BS-06", + "the further mutants rustc ran", + { + RUSTC_SEEDED_RUNS: large_runs, + } + ); // The gate is armed: each defect planted in a copy of the source is a // disagreement with the oracle. let mut detection: BTreeMap<&str, Vec> = BTreeMap::new(); diff --git a/crates/conformance/src/cases/bootstrap/rust_succeed.rs b/crates/conformance/src/cases/bootstrap/rust_succeed.rs index 3be225d1a7..68b9e7eb4c 100644 --- a/crates/conformance/src/cases/bootstrap/rust_succeed.rs +++ b/crates/conformance/src/cases/bootstrap/rust_succeed.rs @@ -222,11 +222,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-12", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-12") { + return; + } let checked = support::checked_project(&project); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); let roots = Roots::of(&project); diff --git a/crates/conformance/src/cases/bootstrap/rust_validate.rs b/crates/conformance/src/cases/bootstrap/rust_validate.rs index 395d6ea61d..4bf4e3bbb7 100644 --- a/crates/conformance/src/cases/bootstrap/rust_validate.rs +++ b/crates/conformance/src/cases/bootstrap/rust_validate.rs @@ -3041,15 +3041,15 @@ pub(super) fn case() { // The counts SPEC.md §17.23, the register statement, and the scenario // state are asserted here, so a repository that grows fails the case // instead of leaving a stale statement behind. - assert_eq!( - (manifests, fixtures, distinct, all.len()), - ( - COMMITTED_MANIFESTS, - COMMITTED_FIXTURES, - COMMITTED_PROGRAMS, - COMMITTED_CRATES - ), - "the committed manifests, fixtures, distinct programs, and distinct crates" + crate::counts!( + "BS-05", + "the committed manifests, fixtures, distinct programs, and distinct crates", + { + COMMITTED_MANIFESTS: manifests, + COMMITTED_FIXTURES: fixtures, + COMMITTED_PROGRAMS: distinct, + COMMITTED_CRATES: all.len(), + } ); let bases = small(&all, BASE_CAP); let (mut rule_validate, rule_correspond) = rule_vectors(&bases, RULES_EACH); @@ -3059,25 +3059,23 @@ pub(super) fn case() { .chain(machine_crates()) .chain(list_match_crates()) .collect(); - assert_eq!( - (rule_validate.len(), rule_correspond.len()), - (RULE_VALIDATE_RUNS, RULE_CORRESPOND_RUNS), - "the rule runs: each reachable refusal {RULES_EACH} times" + crate::counts!( + "BS-05", + &format!("the rule runs: each reachable refusal {RULES_EACH} times"), + { + RULE_VALIDATE_RUNS: rule_validate.len(), + RULE_CORRESPOND_RUNS: rule_correspond.len(), + } ); - assert_eq!( - ( - branch_crates().len(), - loop_crates().len(), - machine_crates().len(), - list_match_crates().len() - ), - ( - BRANCH_CRATES, - LOOP_CRATES, - MACHINE_CRATES, - LIST_MATCH_CRATES - ), - "the hand-built crates" + crate::counts!( + "BS-05", + "the hand-built crates", + { + BRANCH_CRATES: branch_crates().len(), + LOOP_CRATES: loop_crates().len(), + MACHINE_CRATES: machine_crates().len(), + LIST_MATCH_CRATES: list_match_crates().len(), + } ); rule_validate.extend(by_hand.iter().map(validate_run)); let mut rule_correspond = rule_correspond; @@ -3110,20 +3108,15 @@ pub(super) fn case() { ); let vectors_validate = merged(&[&accepted_validate, &rule_validate]); let vectors_correspond = merged(&[&accepted_correspond, &rule_correspond]); - assert_eq!( - ( - vectors_validate.len(), - refused(&vectors_validate), - vectors_correspond.len(), - refused(&vectors_correspond) - ), - ( - VECTORS_VALIDATE, - REFUSED_VALIDATE, - VECTORS_CORRESPOND, - REFUSED_CORRESPOND - ), - "the vectors of validateCrate and correspondCrate, and the refused among them" + crate::counts!( + "BS-05", + "the vectors of validateCrate and correspondCrate, and the refused among them", + { + VECTORS_VALIDATE: vectors_validate.len(), + REFUSED_VALIDATE: refused(&vectors_validate), + VECTORS_CORRESPOND: vectors_correspond.len(), + REFUSED_CORRESPOND: refused(&vectors_correspond), + } ); // The corpora of the normative host are made here, so that every count // the row states is asserted before the first leg runs. @@ -3143,36 +3136,37 @@ pub(super) fn case() { .map(|index| accepted_correspond[*index].clone()) .chain(rule_correspond.iter().cloned()) .collect(); - assert_eq!( - ( - within.len(), - evaluator_validate.len(), - evaluator_correspond.len() - ), - (EVALUATOR_CRATES, EVALUATOR_VALIDATE, EVALUATOR_CORRESPOND), - "the crates of at most {EVALUATOR_CAP} expressions and the vectors the evaluator is given" + crate::counts!( + "BS-05", + &format!("the crates of at most {EVALUATOR_CAP} expressions and the vectors the evaluator is given"), + { + EVALUATOR_CRATES: within.len(), + EVALUATOR_VALIDATE: evaluator_validate.len(), + EVALUATOR_CORRESPOND: evaluator_correspond.len(), + } ); let mut seeded = Seeded(0x0B55_0005); let (seeded_validate, seeded_correspond) = mutants(&bases, &mut seeded, EVALUATOR_MUTANTS); assert_eq!( - ( - seeded_validate.len(), - seeded_correspond.len(), - refused(&seeded_validate), - refused(&seeded_correspond) - ), - ( - EVALUATOR_MUTANTS, - EVALUATOR_MUTANTS, - SEEDED_REFUSED_VALIDATE, - SEEDED_REFUSED_CORRESPOND - ), - "the seeded mutants the evaluator is given, and the refused among them" + (seeded_validate.len(), seeded_correspond.len()), + (EVALUATOR_MUTANTS, EVALUATOR_MUTANTS), + "the seeded mutants the evaluator is given" + ); + crate::counts!( + "BS-05", + "the refused among the seeded mutants", + { + SEEDED_REFUSED_VALIDATE: refused(&seeded_validate), + SEEDED_REFUSED_CORRESPOND: refused(&seeded_correspond), + } ); eprintln!( "BS-05: the vectors are the counted ones ({:?})", started.elapsed() ); + if crate::counts::stop_before_runs("BS-05") { + return; + } let checked = support::checked_project(&project); let validate_root = super::lower(&checked, &project, MODULE, "validateCrate"); let correspond_root = super::lower(&checked, &project, MODULE, "correspondCrate"); diff --git a/crates/conformance/src/counts.rs b/crates/conformance/src/counts.rs new file mode 100755 index 0000000000..1e05c0c445 --- /dev/null +++ b/crates/conformance/src/counts.rs @@ -0,0 +1,209 @@ +//! The counts the bootstrap cases assert (SPEC.md §17.23): what the committed +//! corpora hold and what the case derives from them, each a named constant of +//! the case, so a repository that grows fails the case instead of leaving a +//! stale statement behind. +//! +//! A landing moves several of them at once, and the cases check them in +//! stages, the cheapest first. Under `LEXLEAN_COUNTS` a case records every +//! count it checks instead of stopping at the first that moved, so one run +//! reports all of them; `cargo xtask counts` reads the report and rewrites the +//! constants and the statements that state them. +//! +//! - unset: a count that moved fails the case, as an `assert_eq!` would; +//! - `report`: every count is recorded and the case runs on, whole; +//! - `fast`: as `report`, and the case stops at [`stop_before_runs`], before +//! the evaluator and rustc runs that take most of its time. +//! +//! The report is one JSON line per checked count, in +//! `target/count-report/.jsonl`, rewritten by each run of the case. + +use std::collections::BTreeSet; +use std::io::Write as _; +use std::sync::Mutex; + +use camino::Utf8PathBuf; + +/// One count a case checks: the constant that states it, the value the +/// constant holds, and the value the case observed. +#[derive(Debug, Clone, Copy)] +pub struct Count { + /// The constant's name in the case's source. + pub name: &'static str, + /// The constant's value. + pub expected: usize, + /// What the case observed. + pub observed: usize, +} + +/// One recorded count of a report line. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Recorded { + /// The case's conformance ID, `BS-05` say. + pub id: String, + /// The case's source file, relative to the repository root. + pub file: String, + /// What the checked group of counts is. + pub what: String, + /// The constant's name. + pub name: String, + /// The constant's value. + pub expected: usize, + /// What the case observed. + pub observed: usize, +} + +/// How the counts are checked, read from `LEXLEAN_COUNTS`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// A count that moved fails the case. + Assert, + /// Every count is recorded and the case runs whole. + Report, + /// Every count is recorded and the case stops before its runs. + Fast, +} + +/// The mode `LEXLEAN_COUNTS` names. +/// +/// # Panics +/// +/// When the variable holds anything but `report` or `fast`, so a misspelt +/// mode is not taken for the asserting one. +#[must_use] +pub fn mode() -> Mode { + match std::env::var("LEXLEAN_COUNTS").as_deref() { + Err(_) | Ok("") => Mode::Assert, + Ok("report") => Mode::Report, + Ok("fast") => Mode::Fast, + Ok(other) => panic!("LEXLEAN_COUNTS is `report` or `fast`, not `{other}`"), + } +} + +/// The directory of the reports. +#[must_use] +pub fn report_dir() -> Utf8PathBuf { + crate::support::repo_root().join("target/count-report") +} + +/// The cases whose report this process has begun, each rewritten once. +static BEGUN: Mutex> = Mutex::new(BTreeSet::new()); + +/// Check a group of counts of the case `id` in `file`. Under the asserting +/// mode a moved count fails the case with both tuples; otherwise each count +/// is recorded and every moved one is said on stderr. +/// +/// # Panics +/// +/// Under the asserting mode, when a count moved; under the others, when the +/// report cannot be written. +pub fn check(id: &str, file: &str, what: &str, counts: &[Count]) { + let mode = mode(); + if mode == Mode::Assert { + let observed: Vec = counts.iter().map(|count| count.observed).collect(); + let expected: Vec = counts.iter().map(|count| count.expected).collect(); + assert!( + observed == expected, + "{id}: {what}\n observed: {observed:?}\n expected: {expected:?} ({})", + counts + .iter() + .map(|count| count.name) + .collect::>() + .join(", ") + ); + return; + } + let dir = report_dir(); + std::fs::create_dir_all(&dir).expect("the count report's directory"); + let path = dir.join(format!("{id}.jsonl")); + let fresh = BEGUN + .lock() + .expect("the begun reports") + .insert(id.to_owned()); + let mut out = std::fs::OpenOptions::new() + .create(true) + .write(true) + .append(!fresh) + .truncate(fresh) + .open(&path) + .expect("the count report"); + for count in counts { + let line = serde_json::to_string(&Recorded { + id: id.to_owned(), + file: file.to_owned(), + what: what.to_owned(), + name: count.name.to_owned(), + expected: count.expected, + observed: count.observed, + }) + .expect("a recorded count is JSON"); + writeln!(out, "{line}").expect("the count report"); + if count.expected != count.observed { + eprintln!( + "{id}: count moved: {} {} -> {} ({what})", + count.name, count.expected, count.observed + ); + } + } +} + +/// Whether the case is to stop here, before the runs that take most of its +/// time: under `LEXLEAN_COUNTS=fast`, once every count before them is +/// recorded. Says so on stderr when it is. +#[must_use] +pub fn stop_before_runs(id: &str) -> bool { + let stop = mode() == Mode::Fast; + if stop { + eprintln!( + "{id}: LEXLEAN_COUNTS=fast: the counts before the runs are in {}; stopping before the runs", + report_dir().join(format!("{id}.jsonl")) + ); + } + stop +} + +/// Every report line under `dir`, in file order. +/// +/// # Errors +/// +/// When a report cannot be read or a line is not a recorded count. +pub fn read_reports(dir: &std::path::Path) -> Result, String> { + let mut paths: Vec = match std::fs::read_dir(dir) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "jsonl")) + .collect(), + Err(error) => return Err(format!("{}: {error}", dir.display())), + }; + paths.sort(); + let mut out = Vec::new(); + for path in paths { + let text = std::fs::read_to_string(&path) + .map_err(|error| format!("{}: {error}", path.display()))?; + for (index, line) in text.lines().enumerate() { + out.push( + serde_json::from_str(line) + .map_err(|error| format!("{}:{}: {error}", path.display(), index + 1))?, + ); + } + } + Ok(out) +} + +/// Check the named counts of a bootstrap case: `counts!("BS-05", "what", +/// { CONSTANT: observed, ... })`. See [`check`]. +#[macro_export] +macro_rules! counts { + ($id:literal, $what:expr, { $( $name:ident : $observed:expr ),+ $(,)? }) => { + $crate::counts::check( + $id, + file!(), + &$what, + &[ $( $crate::counts::Count { + name: stringify!($name), + expected: $name, + observed: $observed, + } ),+ ], + ) + }; +} diff --git a/crates/conformance/src/lib.rs b/crates/conformance/src/lib.rs index 3a974e2dcb..eed1e3dee7 100644 --- a/crates/conformance/src/lib.rs +++ b/crates/conformance/src/lib.rs @@ -13,6 +13,7 @@ pub mod calculus; pub mod cases; pub mod certification; pub mod certification_library; +pub mod counts; pub mod differential; pub mod evidence; pub mod fixtures; diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index f6090f3784..84ec1de64d 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xtask" -description = "Repository gates: validate-model, validate-spec-links, verify-examples, check-golden, check-reproducibility, check-fixtures, check-calculus, check-production-export, check-package, check-test-inventory, bdd, ci-summary, ci-hints, ci-shards, release-artifacts, release-check, validate." +description = "Repository gates: validate-model, validate-spec-links, verify-examples, check-golden, check-reproducibility, check-fixtures, check-calculus, check-production-export, check-package, counts, check-test-inventory, bdd, ci-summary, ci-hints, ci-shards, release-artifacts, release-check, validate." version.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/xtask/src/counts.rs b/xtask/src/counts.rs new file mode 100755 index 0000000000..a194f60817 --- /dev/null +++ b/xtask/src/counts.rs @@ -0,0 +1,829 @@ +//! `cargo xtask counts [--write]`: carry the counts a bootstrap case reported +//! under `LEXLEAN_COUNTS` into the constants and statements that state them +//! (SPEC.md §17.23, the §31 row, the register, the scenario). +//! +//! A count is rewritten only where the rewrite is unambiguous: the constant's +//! literal must hold the reported value; a number in a statement is rewritten +//! when no other constant of the case holds the same value, or when it is the +//! sum of exactly one pair of the case's counts; prose of §17.23 is rewritten +//! where the number is followed by the same words as in the case's row. +//! Everything else is listed for a reader, never guessed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use repo_conformance::counts::{read_reports, Recorded}; + +use crate::Fail; + +/// A number in a line of prose. +struct Token { + start: usize, + end: usize, + value: usize, + commas: bool, +} + +/// Whether `bytes[at..]` is a comma group: `,` and three digits, no fourth. +fn comma_group(bytes: &[u8], at: usize) -> bool { + bytes.get(at) == Some(&b',') + && at + 4 <= bytes.len() + && bytes[at + 1..at + 4].iter().all(u8::is_ascii_digit) + && !bytes.get(at + 4).is_some_and(u8::is_ascii_digit) +} + +/// The numbers of `line`: digit runs, with comma groups of three, that are not +/// part of a name, an ID, a section number, or a decimal. +fn tokens(line: &str) -> Vec { + let bytes = line.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if !bytes[i].is_ascii_digit() { + i += 1; + continue; + } + let start = i; + let mut end = i; + while end < bytes.len() && bytes[end].is_ascii_digit() { + end += 1; + } + let mut commas = false; + if end - start <= 3 { + while comma_group(bytes, end) { + commas = true; + end += 4; + } + } + i = end; + let before = line[..start].chars().next_back(); + let named = before.is_some_and(|c| c.is_alphanumeric() || "_-.§".contains(c)); + let decimal = + bytes.get(end) == Some(&b'.') && bytes.get(end + 1).is_some_and(u8::is_ascii_digit); + let joined = bytes + .get(end) + .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_'); + if named || decimal || joined { + continue; + } + let digits: String = line[start..end] + .chars() + .filter(char::is_ascii_digit) + .collect(); + if let Ok(value) = digits.parse() { + out.push(Token { + start, + end, + value, + commas, + }); + } + } + out +} + +fn grouped(digits: &str, separator: char) -> String { + let mut out = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + out.push(separator); + } + out.push(digit); + } + out +} + +/// `value` with comma groups of three when `commas`. +fn render(value: usize, commas: bool) -> String { + let digits = value.to_string(); + if commas && digits.len() > 3 { + grouped(&digits, ',') + } else { + digits + } +} + +/// A Rust literal for `value`: grouped by `_` when the old literal was, or +/// when it has five digits or more. +fn rust_literal(value: usize, underscores: bool) -> String { + let digits = value.to_string(); + if underscores || digits.len() >= 5 { + grouped(&digits, '_') + } else { + digits + } +} + +/// `line` with every number in `map` replaced at once, in the number's own +/// style (a number under 1,000 that grows past it takes commas when the line +/// writes its other thousands with them), and how often each was replaced. +fn rewrite(line: &str, map: &BTreeMap) -> (String, BTreeMap) { + let found = tokens(line); + let line_commas = found.iter().any(|token| token.commas); + let mut out = String::new(); + let mut applied = BTreeMap::new(); + let mut last = 0; + for token in found { + if let Some(new) = map.get(&token.value) { + out.push_str(&line[last..token.start]); + let commas = token.commas || (token.value < 1_000 && line_commas); + out.push_str(&render(*new, commas)); + last = token.end; + *applied.entry(token.value).or_insert(0) += 1; + } + } + out.push_str(&line[last..]); + (out, applied) +} + +/// The text after byte `end` of `line`: at most three words, and never past a +/// table cell's end. +fn following(line: &str, end: usize) -> String { + let rest = &line[end..]; + let mut words = 0; + let mut cut = rest.len(); + let mut in_word = false; + for (index, c) in rest.char_indices() { + if c == '|' { + cut = index; + break; + } + if c.is_whitespace() { + if in_word { + words += 1; + in_word = false; + if words == 3 { + cut = index; + break; + } + } + } else { + in_word = true; + } + } + rest[..cut].to_owned() +} + +/// One constant whose count moved. +#[derive(Debug, Clone)] +struct Moved { + id: String, + file: String, + name: String, + old: usize, + new: usize, +} + +/// A constant of a case's source: its value, its line, and its literal. +type Constant = (usize, usize, String); + +/// The `usize` constants of a case's source, by name. +fn constants(source: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + for (index, line) in source.lines().enumerate() { + let Some(rest) = line.trim_start().strip_prefix("const ") else { + continue; + }; + let Some((name, rest)) = rest.split_once(": usize = ") else { + continue; + }; + let Some(literal) = rest.strip_suffix(';') else { + continue; + }; + if let Ok(value) = literal.replace('_', "").parse() { + out.insert(name.to_owned(), (value, index, literal.to_owned())); + } + } + out +} + +/// The lines of `text` joined back, keeping a final newline. +fn joined(lines: &[String], text: &str) -> String { + let mut out = lines.join("\n"); + if text.ends_with('\n') { + out.push('\n'); + } + out +} + +/// Every count of the report, by its case, file, and constant: the values the +/// constant held and the values observed. +type Seen = BTreeMap<(String, String, String), (BTreeSet, BTreeSet)>; + +fn seen_of(records: &[Recorded]) -> Seen { + let mut seen = Seen::new(); + for record in records { + let entry = seen + .entry((record.id.clone(), record.file.clone(), record.name.clone())) + .or_default(); + entry.0.insert(record.expected); + entry.1.insert(record.observed); + } + seen +} + +/// The single value of a set the caller knows has one. +fn only(set: &BTreeSet) -> usize { + *set.first().expect("a set of one value") +} + +/// A case's numbers to rewrite, and each with the words that follow it in +/// the case's §31 row. +type Anchored = (String, BTreeMap, Vec<(usize, String)>); + +/// The numbers of a case to rewrite, and the words that follow each in its row. +type Numbers = (BTreeMap, Vec<(usize, String)>); + +/// The files a write touches, read once. +struct Texts { + sources: BTreeMap, + spec: String, + ids: String, + features: BTreeMap, +} + +/// Run the task. +/// +/// # Errors +/// +/// When there is no report, or a file cannot be read or written. +pub fn counts(root: &Path, write: bool) -> Result<(), Fail> { + let dir = root.join("target/count-report"); + let records = read_reports(&dir).map_err(|error| { + format!( + "counts: no report ({error}); run the bootstrap parts under LEXLEAN_COUNTS=fast first" + ) + })?; + if records.is_empty() { + return Err("counts: the report is empty".into()); + } + let seen = seen_of(&records); + let mut report = String::new(); + let mut flagged: Vec = Vec::new(); + let mut texts = Texts { + sources: BTreeMap::new(), + spec: std::fs::read_to_string(root.join("SPEC.md"))?, + ids: std::fs::read_to_string(root.join("model/ids.toml"))?, + features: BTreeMap::new(), + }; + for entry in std::fs::read_dir(root.join("features/suites"))? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "feature") { + let text = std::fs::read_to_string(&path)?; + texts.features.insert(path, text); + } + } + let moved = moved_counts(root, &seen, &mut texts, &mut report, &mut flagged)?; + if moved.is_empty() { + let _ = writeln!(report, "counts: no count moved"); + } + let originals = texts.sources.clone(); + for item in &moved { + write_constant(item, &mut texts, &mut report); + } + let mut verification = String::new(); + let mut anchored: Vec = Vec::new(); + let cases: BTreeSet<&str> = moved.iter().map(|item| item.id.as_str()).collect(); + for id in cases { + let mine: Vec<&Moved> = moved.iter().filter(|item| item.id == id).collect(); + if let Some((map, contexts)) = statements( + id, + &mine, + &constants(&originals[&mine[0].file]), + &seen, + &mut texts, + &mut report, + &mut flagged, + ) { + anchored.push((id.to_owned(), map, contexts)); + } + let names: Vec<&str> = mine.iter().map(|item| item.name.as_str()).collect(); + let olds: Vec = mine.iter().map(|item| render(item.old, true)).collect(); + let news: Vec = mine.iter().map(|item| render(item.new, true)).collect(); + let _ = writeln!( + verification, + "| `{id}` {} | {} | {} |", + names.join(", "), + olds.join(", "), + news.join(", ") + ); + } + if !moved.is_empty() { + let derived: Vec<(usize, usize)> = anchored + .iter() + .flat_map(|(_, map, _)| map.iter().map(|(old, new)| (*old, *new))) + .filter(|(old, _)| !moved.iter().any(|item| item.old == *old)) + .collect(); + let shared = shared_map(root, &moved, &derived)?; + prose(&anchored, &shared, &mut texts, &mut report, &mut flagged); + } + print!("{report}"); + if !verification.is_empty() { + println!( + "\nVERIFICATION.md rows for the landing (counts | before | after):\n{verification}" + ); + } + if !flagged.is_empty() { + println!("For a reader:"); + for line in &flagged { + println!(" {line}"); + } + } + if moved.is_empty() { + return Ok(()); + } + if write { + for (file, text) in &texts.sources { + if originals.get(file) != Some(text) { + std::fs::write(root.join(file), text)?; + } + } + std::fs::write(root.join("SPEC.md"), &texts.spec)?; + std::fs::write(root.join("model/ids.toml"), &texts.ids)?; + for (path, text) in &texts.features { + std::fs::write(path, text)?; + } + println!( + "\nwritten; next: cargo xtask validate-model --write, then the parts without LEXLEAN_COUNTS" + ); + } else { + println!("\n(dry run: --write applies it)"); + } + Ok(()) +} + +/// The counts that moved and whose constant still holds the old value. +fn moved_counts( + root: &Path, + seen: &Seen, + texts: &mut Texts, + report: &mut String, + flagged: &mut Vec, +) -> Result, Fail> { + let mut moved = Vec::new(); + for ((id, file, name), (expected, observed)) in seen { + if !texts.sources.contains_key(file) { + let source = std::fs::read_to_string(root.join(file)) + .map_err(|error| format!("counts: {file}: {error}"))?; + texts.sources.insert(file.clone(), source); + } + if expected.len() != 1 || observed.len() != 1 { + flagged.push(format!( + "{id} {name}: the report holds {expected:?} and observes {observed:?}; the checks of one constant disagree, which is no count to write" + )); + continue; + } + let (old, new) = (only(expected), only(observed)); + if old == new { + continue; + } + match constants(&texts.sources[file]).get(name).map(|constant| constant.0) { + Some(value) if value == new => { + let _ = writeln!(report, "{id} {name}: already {new}"); + } + Some(value) if value == old => moved.push(Moved { + id: id.clone(), + file: file.clone(), + name: name.clone(), + old, + new, + }), + Some(value) => flagged.push(format!( + "{id} {name}: the source holds {value} and the report {old} -> {new}; rerun the case" + )), + None => flagged.push(format!( + "{id} {name}: {file} has no `const {name}: usize = ;`; write {old} -> {new} by hand" + )), + } + } + Ok(moved) +} + +/// Rewrite the constant's literal in its source. +fn write_constant(item: &Moved, texts: &mut Texts, report: &mut String) { + let source = &texts.sources[&item.file]; + let (_, index, literal) = constants(source)[&item.name].clone(); + let new_literal = rust_literal(item.new, literal.contains('_')); + let lines: Vec = source + .lines() + .enumerate() + .map(|(at, line)| { + if at == index { + line.replacen(&format!("= {literal};"), &format!("= {new_literal};"), 1) + } else { + line.to_owned() + } + }) + .collect(); + let text = joined(&lines, source); + texts.sources.insert(item.file.clone(), text); + let _ = writeln!( + report, + "{} {}: {} -> {} ({})", + item.id, item.name, item.old, item.new, item.file + ); +} + +/// The numbers of a case's statements to rewrite: each moved count whose old +/// value no other constant of the case holds, and each sum of one pair of its +/// counts that the row states. +fn number_map( + id: &str, + mine: &[&Moved], + table: &BTreeMap, + seen: &Seen, + row: &str, + report: &mut String, + flagged: &mut Vec, +) -> BTreeMap { + let mut map = BTreeMap::new(); + for item in mine { + let others: Vec<&String> = table + .iter() + .filter(|(name, (value, _, _))| { + *name != &item.name + && *value == item.old + && !mine + .iter() + .any(|other| &other.name == *name && other.new == item.new) + }) + .map(|(name, _)| name) + .collect(); + if others.is_empty() { + map.insert(item.old, item.new); + } else { + flagged.push(format!( + "{id} {}: {} is also the value of {others:?}; rewrite its statements by hand ({} -> {})", + item.name, item.old, item.old, item.new + )); + } + } + let counted: Vec<(&String, usize, usize)> = seen + .iter() + .filter(|((case, _, _), (expected, observed))| { + case == id && expected.len() == 1 && observed.len() == 1 + }) + .map(|((_, _, name), (expected, observed))| (name, only(expected), only(observed))) + .collect(); + let known: BTreeSet = table.values().map(|constant| constant.0).collect(); + for token in tokens(row) { + if known.contains(&token.value) || map.contains_key(&token.value) { + continue; + } + let mut sums = BTreeSet::new(); + let mut pairs = Vec::new(); + for (i, (a, a_old, a_new)) in counted.iter().enumerate() { + for (b, b_old, b_new) in &counted[i + 1..] { + if a_old + b_old == token.value && (a_old != a_new || b_old != b_new) { + sums.insert(a_new + b_new); + pairs.push(format!("{a} + {b}")); + } + } + } + match sums.len() { + 0 => {} + 1 => { + let new = only(&sums); + let _ = writeln!( + report, + "{id}: {} is {}: -> {new}", + token.value, + pairs.join(" or ") + ); + map.insert(token.value, new); + } + _ => flagged.push(format!( + "{id}: {} is the sum of {}, which move differently; rewrite it by hand", + token.value, + pairs.join(", ") + )), + } + } + map +} + +/// Rewrite the numbers of the case's §31 row, register statement, and +/// scenario; return the numbers and the words that follow each in the row. +fn statements( + id: &str, + mine: &[&Moved], + table: &BTreeMap, + seen: &Seen, + texts: &mut Texts, + report: &mut String, + flagged: &mut Vec, +) -> Option { + let row_prefix = format!("| `{id}` |"); + let Some(row) = texts + .spec + .lines() + .find(|line| line.starts_with(&row_prefix)) + .map(str::to_owned) + else { + flagged.push(format!("{id}: SPEC.md has no §31 row")); + return None; + }; + let map = number_map(id, mine, table, seen, &row, report, flagged); + let contexts: Vec<(usize, String)> = tokens(&row) + .iter() + .filter(|token| map.contains_key(&token.value)) + .map(|token| (token.value, collapsed(&following(&row, token.end)))) + .filter(|(_, words)| words.trim().len() >= 4) + .collect(); + let (new_row, applied) = rewrite(&row, &map); + let _ = writeln!(report, "{id}: §31 row: {}", applied_text(&applied, &map)); + for (old, new) in &map { + if !applied.contains_key(old) { + let _ = writeln!(report, "{id}: the §31 row does not state {old} (-> {new})"); + } + } + let mut lines: Vec = texts.spec.lines().map(str::to_owned).collect(); + if let Some(at) = lines.iter().position(|line| line == &row) { + lines[at] = new_row; + } + texts.spec = joined(&lines, &texts.spec); + + let marker = format!("id = \"{id}\""); + let mut lines: Vec = texts.ids.lines().map(str::to_owned).collect(); + let statement = lines + .iter() + .position(|line| line == &marker) + .and_then(|at| { + lines[at..] + .iter() + .position(|line| line.starts_with("statement = ")) + .map(|offset| at + offset) + }); + if let Some(at) = statement { + let (line, applied) = rewrite(&lines[at], &map); + lines[at] = line; + let _ = writeln!(report, "{id}: register: {}", applied_text(&applied, &map)); + texts.ids = joined(&lines, &texts.ids); + } else { + flagged.push(format!("{id}: model/ids.toml has no statement")); + } + + let tag = format!("@{id} "); + for text in texts.features.values_mut() { + let mut lines: Vec = text.lines().map(str::to_owned).collect(); + let Some(at) = lines + .iter() + .position(|line| line.trim_start().starts_with(&tag)) + else { + continue; + }; + if let Some(next) = lines.get(at + 1) { + let (line, applied) = rewrite(next, &map); + lines[at + 1] = line; + let _ = writeln!(report, "{id}: scenario: {}", applied_text(&applied, &map)); + } + *text = joined(&lines, text); + } + Some((map, contexts)) +} + +/// `text` with every run of whitespace one space. +fn collapsed(text: &str) -> String { + let mut out = String::new(); + let mut space = false; + for c in text.chars() { + if c.is_whitespace() { + space = true; + } else { + if space { + out.push(' '); + space = false; + } + out.push(c); + } + } + if space { + out.push(' '); + } + out +} + +/// The numbers every constant holding them moves alike: each moved old +/// value that no constant of any bootstrap case holds unless it moves to the +/// same new value, and each sum of counts (`derived`) that no constant holds. +/// Prose of §17.23 states such a number without naming a case (the committed +/// manifests every case reads, say). +fn shared_map( + root: &Path, + moved: &[Moved], + derived: &[(usize, usize)], +) -> Result, Fail> { + let mut holders: Vec<(String, String, usize)> = Vec::new(); + let dir = root.join("crates/conformance/src/cases/bootstrap"); + let mut paths: Vec = std::fs::read_dir(&dir)? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .collect(); + paths.push(root.join("crates/conformance/src/cases/bootstrap.rs")); + for path in paths { + if path.extension().is_none_or(|ext| ext != "rs") { + continue; + } + let file = path + .strip_prefix(root) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + let source = std::fs::read_to_string(&path)?; + for (name, (value, _, _)) in constants(&source) { + holders.push((file.clone(), name, value)); + } + } + let mut out = BTreeMap::new(); + let olds: BTreeSet = moved + .iter() + .map(|item| item.old) + .chain(derived.iter().map(|(old, _)| *old)) + .collect(); + for old in olds { + let news: BTreeSet = moved + .iter() + .filter(|item| item.old == old) + .map(|item| item.new) + .chain( + derived + .iter() + .filter(|(from, _)| *from == old) + .map(|(_, new)| *new), + ) + .collect(); + let still = holders.iter().any(|(file, name, value)| { + *value == old + && !moved + .iter() + .any(|item| &item.file == file && &item.name == name) + }); + if news.len() == 1 && !still { + out.insert(old, only(&news)); + } + } + Ok(out) +} + +/// `old -> new (times)` for each replaced number. +fn applied_text(applied: &BTreeMap, map: &BTreeMap) -> String { + if applied.is_empty() { + return "nothing".to_owned(); + } + applied + .iter() + .map(|(old, times)| format!("{old} -> {} (x{times})", map[old])) + .collect::>() + .join(", ") +} + +/// The text after byte `end` of line `at`, running on over the next lines of +/// its paragraph, with whitespace collapsed: enough to compare with a row's +/// words across the prose's line breaks. +fn after(lines: &[String], at: usize, end: usize) -> String { + let mut text = lines[at][end..].to_owned(); + for line in lines.iter().skip(at + 1).take(3) { + if line.trim().is_empty() { + break; + } + text.push(' '); + text.push_str(line); + } + collapsed(&text) +} + +/// Rewrite the numbers of the prose of §17.23: each a case's row states that +/// the row's words follow there too, and each every holder moves alike +/// (`shared`); then list the moved numbers it still states. Matches are taken +/// on the prose as it was, then applied at once. +fn prose( + anchored: &[Anchored], + shared: &BTreeMap, + texts: &mut Texts, + report: &mut String, + flagged: &mut Vec, +) { + let mut lines: Vec = texts.spec.lines().map(str::to_owned).collect(); + let Some(start) = lines.iter().position(|line| line.starts_with("### 17.23 ")) else { + flagged.push("SPEC.md has no §17.23".to_owned()); + return; + }; + let end = lines[start + 1..] + .iter() + .position(|line| line.starts_with("### ") || line.starts_with("## ")) + .map_or(lines.len(), |offset| start + 1 + offset); + let mut edits: Vec<(usize, usize, usize, String, String)> = Vec::new(); + for at in start..end { + if lines[at].starts_with('|') { + continue; + } + for token in tokens(&lines[at]) { + let words = after(&lines, at, token.end); + let mut by: Option<(usize, String)> = None; + for (id, map, contexts) in anchored { + if contexts.iter().any(|(old, context)| { + *old == token.value && words.starts_with(context.as_str()) + }) { + by = Some((map[&token.value], id.clone())); + break; + } + } + if by.is_none() { + if let Some(new) = shared.get(&token.value) { + by = Some((*new, "every holder".to_owned())); + } + } + if let Some((new, why)) = by { + edits.push((at, token.start, token.end, render(new, token.commas), why)); + } + } + } + for (at, start, end, new, why) in edits.iter().rev() { + let old = lines[*at][*start..*end].to_owned(); + lines[*at].replace_range(*start..*end, new); + let _ = writeln!(report, "SPEC.md:{}: {old} -> {new} ({why})", at + 1); + } + let moved: BTreeMap = anchored + .iter() + .flat_map(|(id, map, _)| { + map.iter() + .map(move |(old, new)| (*old, (*new, id.as_str()))) + }) + .collect(); + for at in start..end { + if lines[at].starts_with('|') { + continue; + } + let done: BTreeSet = edits + .iter() + .filter(|edit| edit.0 == at) + .map(|edit| edit.1) + .collect(); + for token in tokens(&lines[at]) { + if done.contains(&token.start) { + continue; + } + if let Some((new, id)) = moved.get(&token.value) { + flagged.push(format!( + "SPEC.md:{} still states {} (-> {new} for {id}?) before \"{}\"", + at + 1, + token.value, + after(&lines, at, token.end) + .chars() + .take(40) + .collect::() + .trim() + )); + } + } + } + texts.spec = joined(&lines, &texts.spec); +} + +#[cfg(test)] +mod tests { + use super::{after, collapsed, following, render, rewrite, rust_literal, tokens}; + use std::collections::BTreeMap; + + #[test] + fn prose_words_are_read_across_its_line_breaks_within_a_paragraph() { + let lines: Vec = ["run 2,927 vectors: 714", "of `printCrate` (546", "", "next"] + .iter() + .map(|line| (*line).to_owned()) + .collect(); + let words = after(&lines, 0, "run 2,927".len()); + assert_eq!(words, " vectors: 714 of `printCrate` (546"); + assert!(words.starts_with(&collapsed(" vectors: 714\nof"))); + } + + #[test] + fn numbers_are_read_with_their_groups_and_not_from_names() { + let line = "the 2,838 vectors of `BS-05` (§17.16, 0x0B55) are 693 and 2,145, 3522."; + let values: Vec = tokens(line).iter().map(|token| token.value).collect(); + assert_eq!(values, vec![2_838, 693, 2_145, 3_522]); + } + + #[test] + fn a_rewrite_is_simultaneous_and_keeps_the_style() { + let map = BTreeMap::from([(693, 714), (714, 800), (2_145, 2_213)]); + let (line, applied) = rewrite("693 of 714, then 2,145 and 2145", &map); + assert_eq!(line, "714 of 800, then 2,213 and 2213"); + assert_eq!(applied[&2_145], 2); + let (grown, _) = rewrite("999 and 1,000", &BTreeMap::from([(999, 1_001)])); + assert_eq!(grown, "1,001 and 1,000"); + } + + #[test] + fn literals_and_contexts() { + assert_eq!(rust_literal(2_989, true), "2_989"); + assert_eq!(rust_literal(3_522, false), "3522"); + assert_eq!(rust_literal(12_000, false), "12_000"); + assert_eq!(render(1_532, true), "1,532"); + assert_eq!( + following("509 runs of `validateCrate` and more", 3), + " runs of `validateCrate`" + ); + assert_eq!(following("66 | rest", 2), " "); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9887022ba5..93bee814d7 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -12,6 +12,7 @@ use camino::Utf8PathBuf; mod audit; mod ci; mod codegen; +mod counts; mod release_manifest; mod spec_links; @@ -36,6 +37,7 @@ fn main() -> ExitCode { "check-calculus" => check_calculus(&root, write), "check-production-export" => check_production_export(write), "check-package" => check_package(&root), + "counts" => counts::counts(&root, write), "check-test-inventory" => ci::check_test_inventory(&root), "bdd" => ci::bdd(&root), "ci-summary" => ci::ci_summary(&root, &std::env::args().skip(2).collect::>()), @@ -58,6 +60,8 @@ fn main() -> ExitCode { check-calculus §17.14-§17.16: the compiler project's modules, the target and\n\ GNAF fixtures, the Rust packages, and configuration equal\n\ their generator\n\ + counts §17.23: write the counts a bootstrap case reported under\n\ + LEXLEAN_COUNTS=fast|report into its constants and statements\n\ check-production-export §24.7: the consumer's exported packages equal the committed\n\ packages under tests/production-export\n\ check-package §30.3, RP-12: package lexlean crate, verify offline build & identity\n\