From 9f533bfeeed979cdda387826be3f795abb29fa33 Mon Sep 17 00:00:00 2001 From: humuhumu33 Date: Sun, 8 Nov 2026 23:57:34 +0000 Subject: [PATCH 1/2] Bootstrap count cases: one pass to report every moved count, and a writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The count cases BS-05 to BS-08 and BS-11 to BS-15 assert their counts in stages, the cheapest first, so a landing that moves several of them failed at the first stage, was mirrored by hand, run again, and failed at the next: four rounds for SL-14, each a run of the nine parts and a hand edit of seven files. Each count check is now `crate::counts!(id, what, { CONSTANT: observed })` (crates/conformance/src/counts.rs). Unset, `LEXLEAN_COUNTS` keeps the assertion: a moved count fails the case with both tuples. Under `report` every count is recorded in target/count-report/.jsonl and the case runs on; under `fast` it also stops before its evaluator and rustc runs. Constants that drive the computation (the mutant draws of BS-05, BS-07 and BS-08, the bounds) stay plain assertions, so they are never reported as counts. `cargo xtask counts [--write]` reads the report and rewrites each moved constant whose literal holds the reported value; in the case's §31 row, register statement and scenario, every number whose value no other constant of the case holds, and each sum of one pair of its counts; and in the prose of §17.23, a number the row's words follow there too, or one every constant of every bootstrap case that holds it moves alike. It prints the VERIFICATION.md rows of the landing and lists every number it cannot place unambiguously, never guessing. Checked: on release/v0.4 the nine parts under LEXLEAN_COUNTS=fast take about four minutes and record 58 counts, none moved. On #415 (SL-13) one fast pass and one write produced its whole count mirror: 23 constants, the nine rows, statements and scenarios, 18 prose numbers including the derived 2,927 of BS-06, with four numbers left to a reader (273 is also a seeded-mutant count of BS-05; 117 is a timing). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 + .../src/cases/bootstrap/rust_assemble.rs | 15 +- .../src/cases/bootstrap/rust_bodies.rs | 15 +- .../src/cases/bootstrap/rust_loops.rs | 15 +- .../src/cases/bootstrap/rust_lower.rs | 30 +- .../src/cases/bootstrap/rust_moves.rs | 22 +- .../src/cases/bootstrap/rust_package.rs | 35 +- .../src/cases/bootstrap/rust_print.rs | 86 +- .../src/cases/bootstrap/rust_succeed.rs | 15 +- .../src/cases/bootstrap/rust_validate.rs | 118 ++- crates/conformance/src/counts.rs | 209 +++++ crates/conformance/src/lib.rs | 1 + xtask/src/counts.rs | 829 ++++++++++++++++++ xtask/src/main.rs | 4 + 14 files changed, 1238 insertions(+), 166 deletions(-) create mode 100755 crates/conformance/src/counts.rs create mode 100755 xtask/src/counts.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 32eb1be762..050bee23d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,16 @@ workspace carried at those versions without a tag. ## Unreleased +- A one-pass writer for the counts of the bootstrap count cases, #92. + Under `LEXLEAN_COUNTS=fast` each of `BS-05` to `BS-08` and `BS-11` to + `BS-15` records every count it checks in `target/count-report/` instead of + failing at the first that moved, and stops before its evaluator and rustc + runs (`LEXLEAN_COUNTS=report` runs it whole); `cargo xtask counts --write` + carries the moved counts into the case's constants, its §31 row, register + statement and scenario, and the prose of §17.23 that repeats them, and + lists every number it cannot place unambiguously. A landing that moved + the counts took a run and a hand mirror per stage of the cases; it takes + one pass. Unset, the cases assert as before. The register is unchanged. - Maps and sets in the standard library, the fifth member the self-hosted compiler needs, #276 (SPEC.md §17.22, `SL-14`). The module `LexLeanStd.Map` holds ten production roots for `rust-std` over strictly ascending lists of diff --git a/crates/conformance/src/cases/bootstrap/rust_assemble.rs b/crates/conformance/src/cases/bootstrap/rust_assemble.rs index b9bb8da8a2..fae28f261b 100644 --- a/crates/conformance/src/cases/bootstrap/rust_assemble.rs +++ b/crates/conformance/src/cases/bootstrap/rust_assemble.rs @@ -351,11 +351,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-14", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-14") { + return; + } let checked = support::checked_project(&project); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); let roots = Roots::of(&project); diff --git a/crates/conformance/src/cases/bootstrap/rust_bodies.rs b/crates/conformance/src/cases/bootstrap/rust_bodies.rs index a4b5bd3232..6de7dc4aca 100644 --- a/crates/conformance/src/cases/bootstrap/rust_bodies.rs +++ b/crates/conformance/src/cases/bootstrap/rust_bodies.rs @@ -618,11 +618,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-11", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-11") { + return; + } let checked = support::checked_project(&project); let uses = super::lower(&checked, &project, TYPING, "useTrees"); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); diff --git a/crates/conformance/src/cases/bootstrap/rust_loops.rs b/crates/conformance/src/cases/bootstrap/rust_loops.rs index fe173338a0..e2d49963b5 100644 --- a/crates/conformance/src/cases/bootstrap/rust_loops.rs +++ b/crates/conformance/src/cases/bootstrap/rust_loops.rs @@ -494,11 +494,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-13", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-13") { + return; + } let checked = support::checked_project(&project); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); let roots = Roots::of(&project); diff --git a/crates/conformance/src/cases/bootstrap/rust_lower.rs b/crates/conformance/src/cases/bootstrap/rust_lower.rs index 679def7ebc..c3d05f99ec 100644 --- a/crates/conformance/src/cases/bootstrap/rust_lower.rs +++ b/crates/conformance/src/cases/bootstrap/rust_lower.rs @@ -917,22 +917,22 @@ pub(super) fn case() { let mut large = Seeded(0x00B5_0207); let rustc_mutants = seeded_mutants(&sources, &mut large, RUSTC_MUTANTS, BASE_NODES); assert_eq!( - ( - rules.len(), - all.len(), - evaluator.len(), - evaluator_mutants.len(), - rustc_mutants.len() - ), - ( - RULE_PROGRAMS, - VECTORS, - VECTORS_WITHIN, - EVALUATOR_MUTANTS, - RUSTC_MUTANTS - ), - "the programs of the rules, the vectors, those the evaluator is given, and the mutants" + (evaluator_mutants.len(), rustc_mutants.len()), + (EVALUATOR_MUTANTS, RUSTC_MUTANTS), + "the mutants" + ); + crate::counts!( + "BS-07", + "the programs of the rules, the vectors, and those the evaluator is given", + { + RULE_PROGRAMS: rules.len(), + VECTORS: all.len(), + VECTORS_WITHIN: evaluator.len(), + } ); + if crate::counts::stop_before_runs("BS-07") { + return; + } eprintln!( "BS-07: {} vectors ({} within {EVALUATOR_NODES} nodes), {} + {} mutants ({:?})", all.len(), diff --git a/crates/conformance/src/cases/bootstrap/rust_moves.rs b/crates/conformance/src/cases/bootstrap/rust_moves.rs index 4b7c75a7fe..c1b3d2bd3e 100644 --- a/crates/conformance/src/cases/bootstrap/rust_moves.rs +++ b/crates/conformance/src/cases/bootstrap/rust_moves.rs @@ -519,15 +519,21 @@ pub(super) fn case() { .map(run) .collect(); assert_eq!( - ( - base.len(), - small_runs.len(), - evaluator_mutants.len(), - rustc_mutants.len() - ), - (CRATES, WITHIN, EVALUATOR_MUTANTS, RUSTC_MUTANTS), - "the crates the moves are run on, those within the evaluator's bound, and the mutants" + (evaluator_mutants.len(), rustc_mutants.len()), + (EVALUATOR_MUTANTS, RUSTC_MUTANTS), + "the mutants" ); + crate::counts!( + "BS-08", + "the crates the moves are run on, and those within the evaluator's bound", + { + CRATES: base.len(), + WITHIN: small_runs.len(), + } + ); + if crate::counts::stop_before_runs("BS-08") { + return; + } eprintln!( "BS-08: {} crates ({} within {EVALUATOR_NODES} expressions), {} + {} mutants ({:?})", base.len(), diff --git a/crates/conformance/src/cases/bootstrap/rust_package.rs b/crates/conformance/src/cases/bootstrap/rust_package.rs index 0f5a9b10d0..4eda76973d 100644 --- a/crates/conformance/src/cases/bootstrap/rust_package.rs +++ b/crates/conformance/src/cases/bootstrap/rust_package.rs @@ -916,27 +916,22 @@ pub(super) fn case() { within.len(), started.elapsed() ); - assert_eq!( - ( - manifests.len(), - renders.len(), - rules.len(), - shaped.len(), - mutants.len(), - render_mutants.len(), - within.len() - ), - ( - MANIFESTS, - ACCEPTED, - HEADER_RULES, - RENDER_RULES, - HEADER_MUTANTS, - RENDER_MUTANTS, - EVALUATOR_RENDERS - ), - "the manifests, the accepted ones, the rule inputs, the mutants, and the renders the evaluator is given" + crate::counts!( + "BS-15", + "the manifests, the accepted ones, the rule inputs, the mutants, and the renders the evaluator is given", + { + MANIFESTS: manifests.len(), + ACCEPTED: renders.len(), + HEADER_RULES: rules.len(), + RENDER_RULES: shaped.len(), + HEADER_MUTANTS: mutants.len(), + RENDER_MUTANTS: render_mutants.len(), + EVALUATOR_RENDERS: within.len(), + } ); + if crate::counts::stop_before_runs("BS-15") { + return; + } let checked = support::checked_project(&project); let check_root = super::lower(&checked, &project, MODULE, "checkManifest"); diff --git a/crates/conformance/src/cases/bootstrap/rust_print.rs b/crates/conformance/src/cases/bootstrap/rust_print.rs index a51713f5ac..0795c3a184 100644 --- a/crates/conformance/src/cases/bootstrap/rust_print.rs +++ b/crates/conformance/src/cases/bootstrap/rust_print.rs @@ -1318,16 +1318,16 @@ pub(super) fn case() { "the committed library ends with the printed items" ); } - assert_eq!( - (manifests, fixtures, distinct, all.len(), packages.len()), - ( - COMMITTED_MANIFESTS, - COMMITTED_FIXTURES, - COMMITTED_PROGRAMS, - COMMITTED_CRATES, - COMMITTED_PACKAGES - ), - "the committed manifests, fixtures, distinct programs, distinct crates, and packages" + crate::counts!( + "BS-06", + "the committed manifests, fixtures, distinct programs, distinct crates, and packages", + { + COMMITTED_MANIFESTS: manifests, + COMMITTED_FIXTURES: fixtures, + COMMITTED_PROGRAMS: distinct, + COMMITTED_CRATES: all.len(), + COMMITTED_PACKAGES: packages.len(), + } ); eprintln!( "BS-06: {manifests} manifests, {fixtures} fixtures, {distinct} programs, {} crates, {} packages ({:?})", @@ -1386,25 +1386,18 @@ pub(super) fn case() { started.elapsed() ); - assert_eq!( - ( - crate_runs.len(), - committed_count, - refused(&crate_runs), - expr_runs.len(), - hand_exprs, - refused(&expr_runs) - ), - ( - VECTORS_CRATE, - COMMITTED_DISTINCT, - REFUSED_CRATE, - VECTORS_EXPR, - HAND_EXPR, - 0 - ), - "the vectors of printCrate and printExpr, the committed crates among them, and the refused" + crate::counts!( + "BS-06", + "the vectors of printCrate and printExpr, the committed crates among them, and the refused", + { + VECTORS_CRATE: crate_runs.len(), + COMMITTED_DISTINCT: committed_count, + REFUSED_CRATE: refused(&crate_runs), + VECTORS_EXPR: expr_runs.len(), + HAND_EXPR: hand_exprs, + } ); + assert_eq!(refused(&expr_runs), 0, "no printExpr vector is refused"); // The corpora of the normative host are made here, so that every count // the row states is asserted before the first leg runs. let evaluator_crates: Vec = crate_runs @@ -1422,23 +1415,20 @@ pub(super) fn case() { let (seeded_crates, seeded_exprs) = mutants(&bases, &mut seeded, EVALUATOR_MUTANTS); let mut large = super::Seeded(0x0B55_0066); let (large_crates, large_exprs) = mutants(&bases, &mut large, RUSTC_MUTANTS); - assert_eq!( - ( - evaluator_crates.len(), - evaluator_exprs.len(), - seeded_crates.len(), - seeded_exprs.len(), - large_crates.len() + large_exprs.len() - ), - ( - EVALUATOR_CRATE, - VECTORS_EXPR, - SEEDED_CRATE, - SEEDED_EXPR, - RUSTC_SEEDED_RUNS - ), - "the vectors and the seeded mutants of the evaluator, and the further mutants of rustc" + crate::counts!( + "BS-06", + "the vectors and the seeded mutants of the evaluator, and the further mutants of rustc", + { + EVALUATOR_CRATE: evaluator_crates.len(), + VECTORS_EXPR: evaluator_exprs.len(), + SEEDED_CRATE: seeded_crates.len(), + SEEDED_EXPR: seeded_exprs.len(), + RUSTC_SEEDED_RUNS: large_crates.len() + large_exprs.len(), + } ); + if crate::counts::stop_before_runs("BS-06") { + return; + } let checked = support::checked_project(&project); let crate_root = super::lower(&checked, &project, MODULE, "printCrate"); let expr_root = super::lower(&checked, &project, MODULE, "printExpr"); @@ -1496,7 +1486,13 @@ pub(super) fn case() { ], "seeded", ); - assert_eq!(large_runs, RUSTC_SEEDED_RUNS); + crate::counts!( + "BS-06", + "the further mutants rustc ran", + { + RUSTC_SEEDED_RUNS: large_runs, + } + ); // The gate is armed: each defect planted in a copy of the source is a // disagreement with the oracle. let mut detection: BTreeMap<&str, Vec> = BTreeMap::new(); diff --git a/crates/conformance/src/cases/bootstrap/rust_succeed.rs b/crates/conformance/src/cases/bootstrap/rust_succeed.rs index e1e6322f42..0d663e5eca 100644 --- a/crates/conformance/src/cases/bootstrap/rust_succeed.rs +++ b/crates/conformance/src/cases/bootstrap/rust_succeed.rs @@ -222,11 +222,18 @@ pub(super) fn case() { mutated.len(), started.elapsed() ); - assert_eq!( - (all.len(), within.len(), mutated.len()), - (VECTORS, VECTORS_WITHIN, EVALUATOR_MUTANTS), - "the vectors, those the evaluator is given, and the mutants" + crate::counts!( + "BS-12", + "the vectors, those the evaluator is given, and the mutants", + { + VECTORS: all.len(), + VECTORS_WITHIN: within.len(), + EVALUATOR_MUTANTS: mutated.len(), + } ); + if crate::counts::stop_before_runs("BS-12") { + return; + } let checked = support::checked_project(&project); let moves = super::lower(&checked, &project, MOVES, "applyMoves"); let roots = Roots::of(&project); diff --git a/crates/conformance/src/cases/bootstrap/rust_validate.rs b/crates/conformance/src/cases/bootstrap/rust_validate.rs index 08658212d9..f9ad74a9da 100644 --- a/crates/conformance/src/cases/bootstrap/rust_validate.rs +++ b/crates/conformance/src/cases/bootstrap/rust_validate.rs @@ -3041,15 +3041,15 @@ pub(super) fn case() { // The counts SPEC.md §17.23, the register statement, and the scenario // state are asserted here, so a repository that grows fails the case // instead of leaving a stale statement behind. - assert_eq!( - (manifests, fixtures, distinct, all.len()), - ( - COMMITTED_MANIFESTS, - COMMITTED_FIXTURES, - COMMITTED_PROGRAMS, - COMMITTED_CRATES - ), - "the committed manifests, fixtures, distinct programs, and distinct crates" + crate::counts!( + "BS-05", + "the committed manifests, fixtures, distinct programs, and distinct crates", + { + COMMITTED_MANIFESTS: manifests, + COMMITTED_FIXTURES: fixtures, + COMMITTED_PROGRAMS: distinct, + COMMITTED_CRATES: all.len(), + } ); let bases = small(&all, BASE_CAP); let (mut rule_validate, rule_correspond) = rule_vectors(&bases, RULES_EACH); @@ -3059,25 +3059,23 @@ pub(super) fn case() { .chain(machine_crates()) .chain(list_match_crates()) .collect(); - assert_eq!( - (rule_validate.len(), rule_correspond.len()), - (RULE_VALIDATE_RUNS, RULE_CORRESPOND_RUNS), - "the rule runs: each reachable refusal {RULES_EACH} times" + crate::counts!( + "BS-05", + &format!("the rule runs: each reachable refusal {RULES_EACH} times"), + { + RULE_VALIDATE_RUNS: rule_validate.len(), + RULE_CORRESPOND_RUNS: rule_correspond.len(), + } ); - assert_eq!( - ( - branch_crates().len(), - loop_crates().len(), - machine_crates().len(), - list_match_crates().len() - ), - ( - BRANCH_CRATES, - LOOP_CRATES, - MACHINE_CRATES, - LIST_MATCH_CRATES - ), - "the hand-built crates" + crate::counts!( + "BS-05", + "the hand-built crates", + { + BRANCH_CRATES: branch_crates().len(), + LOOP_CRATES: loop_crates().len(), + MACHINE_CRATES: machine_crates().len(), + LIST_MATCH_CRATES: list_match_crates().len(), + } ); rule_validate.extend(by_hand.iter().map(validate_run)); let mut rule_correspond = rule_correspond; @@ -3110,20 +3108,15 @@ pub(super) fn case() { ); let vectors_validate = merged(&[&accepted_validate, &rule_validate]); let vectors_correspond = merged(&[&accepted_correspond, &rule_correspond]); - assert_eq!( - ( - vectors_validate.len(), - refused(&vectors_validate), - vectors_correspond.len(), - refused(&vectors_correspond) - ), - ( - VECTORS_VALIDATE, - REFUSED_VALIDATE, - VECTORS_CORRESPOND, - REFUSED_CORRESPOND - ), - "the vectors of validateCrate and correspondCrate, and the refused among them" + crate::counts!( + "BS-05", + "the vectors of validateCrate and correspondCrate, and the refused among them", + { + VECTORS_VALIDATE: vectors_validate.len(), + REFUSED_VALIDATE: refused(&vectors_validate), + VECTORS_CORRESPOND: vectors_correspond.len(), + REFUSED_CORRESPOND: refused(&vectors_correspond), + } ); // The corpora of the normative host are made here, so that every count // the row states is asserted before the first leg runs. @@ -3143,36 +3136,37 @@ pub(super) fn case() { .map(|index| accepted_correspond[*index].clone()) .chain(rule_correspond.iter().cloned()) .collect(); - assert_eq!( - ( - within.len(), - evaluator_validate.len(), - evaluator_correspond.len() - ), - (EVALUATOR_CRATES, EVALUATOR_VALIDATE, EVALUATOR_CORRESPOND), - "the crates of at most {EVALUATOR_CAP} expressions and the vectors the evaluator is given" + crate::counts!( + "BS-05", + &format!("the crates of at most {EVALUATOR_CAP} expressions and the vectors the evaluator is given"), + { + EVALUATOR_CRATES: within.len(), + EVALUATOR_VALIDATE: evaluator_validate.len(), + EVALUATOR_CORRESPOND: evaluator_correspond.len(), + } ); let mut seeded = Seeded(0x0B55_0005); let (seeded_validate, seeded_correspond) = mutants(&bases, &mut seeded, EVALUATOR_MUTANTS); assert_eq!( - ( - seeded_validate.len(), - seeded_correspond.len(), - refused(&seeded_validate), - refused(&seeded_correspond) - ), - ( - EVALUATOR_MUTANTS, - EVALUATOR_MUTANTS, - SEEDED_REFUSED_VALIDATE, - SEEDED_REFUSED_CORRESPOND - ), - "the seeded mutants the evaluator is given, and the refused among them" + (seeded_validate.len(), seeded_correspond.len()), + (EVALUATOR_MUTANTS, EVALUATOR_MUTANTS), + "the seeded mutants the evaluator is given" + ); + crate::counts!( + "BS-05", + "the refused among the seeded mutants", + { + SEEDED_REFUSED_VALIDATE: refused(&seeded_validate), + SEEDED_REFUSED_CORRESPOND: refused(&seeded_correspond), + } ); eprintln!( "BS-05: the vectors are the counted ones ({:?})", started.elapsed() ); + if crate::counts::stop_before_runs("BS-05") { + return; + } let checked = support::checked_project(&project); let validate_root = super::lower(&checked, &project, MODULE, "validateCrate"); let correspond_root = super::lower(&checked, &project, MODULE, "correspondCrate"); diff --git a/crates/conformance/src/counts.rs b/crates/conformance/src/counts.rs new file mode 100755 index 0000000000..1e05c0c445 --- /dev/null +++ b/crates/conformance/src/counts.rs @@ -0,0 +1,209 @@ +//! The counts the bootstrap cases assert (SPEC.md §17.23): what the committed +//! corpora hold and what the case derives from them, each a named constant of +//! the case, so a repository that grows fails the case instead of leaving a +//! stale statement behind. +//! +//! A landing moves several of them at once, and the cases check them in +//! stages, the cheapest first. Under `LEXLEAN_COUNTS` a case records every +//! count it checks instead of stopping at the first that moved, so one run +//! reports all of them; `cargo xtask counts` reads the report and rewrites the +//! constants and the statements that state them. +//! +//! - unset: a count that moved fails the case, as an `assert_eq!` would; +//! - `report`: every count is recorded and the case runs on, whole; +//! - `fast`: as `report`, and the case stops at [`stop_before_runs`], before +//! the evaluator and rustc runs that take most of its time. +//! +//! The report is one JSON line per checked count, in +//! `target/count-report/.jsonl`, rewritten by each run of the case. + +use std::collections::BTreeSet; +use std::io::Write as _; +use std::sync::Mutex; + +use camino::Utf8PathBuf; + +/// One count a case checks: the constant that states it, the value the +/// constant holds, and the value the case observed. +#[derive(Debug, Clone, Copy)] +pub struct Count { + /// The constant's name in the case's source. + pub name: &'static str, + /// The constant's value. + pub expected: usize, + /// What the case observed. + pub observed: usize, +} + +/// One recorded count of a report line. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Recorded { + /// The case's conformance ID, `BS-05` say. + pub id: String, + /// The case's source file, relative to the repository root. + pub file: String, + /// What the checked group of counts is. + pub what: String, + /// The constant's name. + pub name: String, + /// The constant's value. + pub expected: usize, + /// What the case observed. + pub observed: usize, +} + +/// How the counts are checked, read from `LEXLEAN_COUNTS`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// A count that moved fails the case. + Assert, + /// Every count is recorded and the case runs whole. + Report, + /// Every count is recorded and the case stops before its runs. + Fast, +} + +/// The mode `LEXLEAN_COUNTS` names. +/// +/// # Panics +/// +/// When the variable holds anything but `report` or `fast`, so a misspelt +/// mode is not taken for the asserting one. +#[must_use] +pub fn mode() -> Mode { + match std::env::var("LEXLEAN_COUNTS").as_deref() { + Err(_) | Ok("") => Mode::Assert, + Ok("report") => Mode::Report, + Ok("fast") => Mode::Fast, + Ok(other) => panic!("LEXLEAN_COUNTS is `report` or `fast`, not `{other}`"), + } +} + +/// The directory of the reports. +#[must_use] +pub fn report_dir() -> Utf8PathBuf { + crate::support::repo_root().join("target/count-report") +} + +/// The cases whose report this process has begun, each rewritten once. +static BEGUN: Mutex> = Mutex::new(BTreeSet::new()); + +/// Check a group of counts of the case `id` in `file`. Under the asserting +/// mode a moved count fails the case with both tuples; otherwise each count +/// is recorded and every moved one is said on stderr. +/// +/// # Panics +/// +/// Under the asserting mode, when a count moved; under the others, when the +/// report cannot be written. +pub fn check(id: &str, file: &str, what: &str, counts: &[Count]) { + let mode = mode(); + if mode == Mode::Assert { + let observed: Vec = counts.iter().map(|count| count.observed).collect(); + let expected: Vec = counts.iter().map(|count| count.expected).collect(); + assert!( + observed == expected, + "{id}: {what}\n observed: {observed:?}\n expected: {expected:?} ({})", + counts + .iter() + .map(|count| count.name) + .collect::>() + .join(", ") + ); + return; + } + let dir = report_dir(); + std::fs::create_dir_all(&dir).expect("the count report's directory"); + let path = dir.join(format!("{id}.jsonl")); + let fresh = BEGUN + .lock() + .expect("the begun reports") + .insert(id.to_owned()); + let mut out = std::fs::OpenOptions::new() + .create(true) + .write(true) + .append(!fresh) + .truncate(fresh) + .open(&path) + .expect("the count report"); + for count in counts { + let line = serde_json::to_string(&Recorded { + id: id.to_owned(), + file: file.to_owned(), + what: what.to_owned(), + name: count.name.to_owned(), + expected: count.expected, + observed: count.observed, + }) + .expect("a recorded count is JSON"); + writeln!(out, "{line}").expect("the count report"); + if count.expected != count.observed { + eprintln!( + "{id}: count moved: {} {} -> {} ({what})", + count.name, count.expected, count.observed + ); + } + } +} + +/// Whether the case is to stop here, before the runs that take most of its +/// time: under `LEXLEAN_COUNTS=fast`, once every count before them is +/// recorded. Says so on stderr when it is. +#[must_use] +pub fn stop_before_runs(id: &str) -> bool { + let stop = mode() == Mode::Fast; + if stop { + eprintln!( + "{id}: LEXLEAN_COUNTS=fast: the counts before the runs are in {}; stopping before the runs", + report_dir().join(format!("{id}.jsonl")) + ); + } + stop +} + +/// Every report line under `dir`, in file order. +/// +/// # Errors +/// +/// When a report cannot be read or a line is not a recorded count. +pub fn read_reports(dir: &std::path::Path) -> Result, String> { + let mut paths: Vec = match std::fs::read_dir(dir) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "jsonl")) + .collect(), + Err(error) => return Err(format!("{}: {error}", dir.display())), + }; + paths.sort(); + let mut out = Vec::new(); + for path in paths { + let text = std::fs::read_to_string(&path) + .map_err(|error| format!("{}: {error}", path.display()))?; + for (index, line) in text.lines().enumerate() { + out.push( + serde_json::from_str(line) + .map_err(|error| format!("{}:{}: {error}", path.display(), index + 1))?, + ); + } + } + Ok(out) +} + +/// Check the named counts of a bootstrap case: `counts!("BS-05", "what", +/// { CONSTANT: observed, ... })`. See [`check`]. +#[macro_export] +macro_rules! counts { + ($id:literal, $what:expr, { $( $name:ident : $observed:expr ),+ $(,)? }) => { + $crate::counts::check( + $id, + file!(), + &$what, + &[ $( $crate::counts::Count { + name: stringify!($name), + expected: $name, + observed: $observed, + } ),+ ], + ) + }; +} diff --git a/crates/conformance/src/lib.rs b/crates/conformance/src/lib.rs index 3a974e2dcb..eed1e3dee7 100644 --- a/crates/conformance/src/lib.rs +++ b/crates/conformance/src/lib.rs @@ -13,6 +13,7 @@ pub mod calculus; pub mod cases; pub mod certification; pub mod certification_library; +pub mod counts; pub mod differential; pub mod evidence; pub mod fixtures; diff --git a/xtask/src/counts.rs b/xtask/src/counts.rs new file mode 100755 index 0000000000..a194f60817 --- /dev/null +++ b/xtask/src/counts.rs @@ -0,0 +1,829 @@ +//! `cargo xtask counts [--write]`: carry the counts a bootstrap case reported +//! under `LEXLEAN_COUNTS` into the constants and statements that state them +//! (SPEC.md §17.23, the §31 row, the register, the scenario). +//! +//! A count is rewritten only where the rewrite is unambiguous: the constant's +//! literal must hold the reported value; a number in a statement is rewritten +//! when no other constant of the case holds the same value, or when it is the +//! sum of exactly one pair of the case's counts; prose of §17.23 is rewritten +//! where the number is followed by the same words as in the case's row. +//! Everything else is listed for a reader, never guessed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use repo_conformance::counts::{read_reports, Recorded}; + +use crate::Fail; + +/// A number in a line of prose. +struct Token { + start: usize, + end: usize, + value: usize, + commas: bool, +} + +/// Whether `bytes[at..]` is a comma group: `,` and three digits, no fourth. +fn comma_group(bytes: &[u8], at: usize) -> bool { + bytes.get(at) == Some(&b',') + && at + 4 <= bytes.len() + && bytes[at + 1..at + 4].iter().all(u8::is_ascii_digit) + && !bytes.get(at + 4).is_some_and(u8::is_ascii_digit) +} + +/// The numbers of `line`: digit runs, with comma groups of three, that are not +/// part of a name, an ID, a section number, or a decimal. +fn tokens(line: &str) -> Vec { + let bytes = line.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if !bytes[i].is_ascii_digit() { + i += 1; + continue; + } + let start = i; + let mut end = i; + while end < bytes.len() && bytes[end].is_ascii_digit() { + end += 1; + } + let mut commas = false; + if end - start <= 3 { + while comma_group(bytes, end) { + commas = true; + end += 4; + } + } + i = end; + let before = line[..start].chars().next_back(); + let named = before.is_some_and(|c| c.is_alphanumeric() || "_-.§".contains(c)); + let decimal = + bytes.get(end) == Some(&b'.') && bytes.get(end + 1).is_some_and(u8::is_ascii_digit); + let joined = bytes + .get(end) + .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_'); + if named || decimal || joined { + continue; + } + let digits: String = line[start..end] + .chars() + .filter(char::is_ascii_digit) + .collect(); + if let Ok(value) = digits.parse() { + out.push(Token { + start, + end, + value, + commas, + }); + } + } + out +} + +fn grouped(digits: &str, separator: char) -> String { + let mut out = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + out.push(separator); + } + out.push(digit); + } + out +} + +/// `value` with comma groups of three when `commas`. +fn render(value: usize, commas: bool) -> String { + let digits = value.to_string(); + if commas && digits.len() > 3 { + grouped(&digits, ',') + } else { + digits + } +} + +/// A Rust literal for `value`: grouped by `_` when the old literal was, or +/// when it has five digits or more. +fn rust_literal(value: usize, underscores: bool) -> String { + let digits = value.to_string(); + if underscores || digits.len() >= 5 { + grouped(&digits, '_') + } else { + digits + } +} + +/// `line` with every number in `map` replaced at once, in the number's own +/// style (a number under 1,000 that grows past it takes commas when the line +/// writes its other thousands with them), and how often each was replaced. +fn rewrite(line: &str, map: &BTreeMap) -> (String, BTreeMap) { + let found = tokens(line); + let line_commas = found.iter().any(|token| token.commas); + let mut out = String::new(); + let mut applied = BTreeMap::new(); + let mut last = 0; + for token in found { + if let Some(new) = map.get(&token.value) { + out.push_str(&line[last..token.start]); + let commas = token.commas || (token.value < 1_000 && line_commas); + out.push_str(&render(*new, commas)); + last = token.end; + *applied.entry(token.value).or_insert(0) += 1; + } + } + out.push_str(&line[last..]); + (out, applied) +} + +/// The text after byte `end` of `line`: at most three words, and never past a +/// table cell's end. +fn following(line: &str, end: usize) -> String { + let rest = &line[end..]; + let mut words = 0; + let mut cut = rest.len(); + let mut in_word = false; + for (index, c) in rest.char_indices() { + if c == '|' { + cut = index; + break; + } + if c.is_whitespace() { + if in_word { + words += 1; + in_word = false; + if words == 3 { + cut = index; + break; + } + } + } else { + in_word = true; + } + } + rest[..cut].to_owned() +} + +/// One constant whose count moved. +#[derive(Debug, Clone)] +struct Moved { + id: String, + file: String, + name: String, + old: usize, + new: usize, +} + +/// A constant of a case's source: its value, its line, and its literal. +type Constant = (usize, usize, String); + +/// The `usize` constants of a case's source, by name. +fn constants(source: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + for (index, line) in source.lines().enumerate() { + let Some(rest) = line.trim_start().strip_prefix("const ") else { + continue; + }; + let Some((name, rest)) = rest.split_once(": usize = ") else { + continue; + }; + let Some(literal) = rest.strip_suffix(';') else { + continue; + }; + if let Ok(value) = literal.replace('_', "").parse() { + out.insert(name.to_owned(), (value, index, literal.to_owned())); + } + } + out +} + +/// The lines of `text` joined back, keeping a final newline. +fn joined(lines: &[String], text: &str) -> String { + let mut out = lines.join("\n"); + if text.ends_with('\n') { + out.push('\n'); + } + out +} + +/// Every count of the report, by its case, file, and constant: the values the +/// constant held and the values observed. +type Seen = BTreeMap<(String, String, String), (BTreeSet, BTreeSet)>; + +fn seen_of(records: &[Recorded]) -> Seen { + let mut seen = Seen::new(); + for record in records { + let entry = seen + .entry((record.id.clone(), record.file.clone(), record.name.clone())) + .or_default(); + entry.0.insert(record.expected); + entry.1.insert(record.observed); + } + seen +} + +/// The single value of a set the caller knows has one. +fn only(set: &BTreeSet) -> usize { + *set.first().expect("a set of one value") +} + +/// A case's numbers to rewrite, and each with the words that follow it in +/// the case's §31 row. +type Anchored = (String, BTreeMap, Vec<(usize, String)>); + +/// The numbers of a case to rewrite, and the words that follow each in its row. +type Numbers = (BTreeMap, Vec<(usize, String)>); + +/// The files a write touches, read once. +struct Texts { + sources: BTreeMap, + spec: String, + ids: String, + features: BTreeMap, +} + +/// Run the task. +/// +/// # Errors +/// +/// When there is no report, or a file cannot be read or written. +pub fn counts(root: &Path, write: bool) -> Result<(), Fail> { + let dir = root.join("target/count-report"); + let records = read_reports(&dir).map_err(|error| { + format!( + "counts: no report ({error}); run the bootstrap parts under LEXLEAN_COUNTS=fast first" + ) + })?; + if records.is_empty() { + return Err("counts: the report is empty".into()); + } + let seen = seen_of(&records); + let mut report = String::new(); + let mut flagged: Vec = Vec::new(); + let mut texts = Texts { + sources: BTreeMap::new(), + spec: std::fs::read_to_string(root.join("SPEC.md"))?, + ids: std::fs::read_to_string(root.join("model/ids.toml"))?, + features: BTreeMap::new(), + }; + for entry in std::fs::read_dir(root.join("features/suites"))? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "feature") { + let text = std::fs::read_to_string(&path)?; + texts.features.insert(path, text); + } + } + let moved = moved_counts(root, &seen, &mut texts, &mut report, &mut flagged)?; + if moved.is_empty() { + let _ = writeln!(report, "counts: no count moved"); + } + let originals = texts.sources.clone(); + for item in &moved { + write_constant(item, &mut texts, &mut report); + } + let mut verification = String::new(); + let mut anchored: Vec = Vec::new(); + let cases: BTreeSet<&str> = moved.iter().map(|item| item.id.as_str()).collect(); + for id in cases { + let mine: Vec<&Moved> = moved.iter().filter(|item| item.id == id).collect(); + if let Some((map, contexts)) = statements( + id, + &mine, + &constants(&originals[&mine[0].file]), + &seen, + &mut texts, + &mut report, + &mut flagged, + ) { + anchored.push((id.to_owned(), map, contexts)); + } + let names: Vec<&str> = mine.iter().map(|item| item.name.as_str()).collect(); + let olds: Vec = mine.iter().map(|item| render(item.old, true)).collect(); + let news: Vec = mine.iter().map(|item| render(item.new, true)).collect(); + let _ = writeln!( + verification, + "| `{id}` {} | {} | {} |", + names.join(", "), + olds.join(", "), + news.join(", ") + ); + } + if !moved.is_empty() { + let derived: Vec<(usize, usize)> = anchored + .iter() + .flat_map(|(_, map, _)| map.iter().map(|(old, new)| (*old, *new))) + .filter(|(old, _)| !moved.iter().any(|item| item.old == *old)) + .collect(); + let shared = shared_map(root, &moved, &derived)?; + prose(&anchored, &shared, &mut texts, &mut report, &mut flagged); + } + print!("{report}"); + if !verification.is_empty() { + println!( + "\nVERIFICATION.md rows for the landing (counts | before | after):\n{verification}" + ); + } + if !flagged.is_empty() { + println!("For a reader:"); + for line in &flagged { + println!(" {line}"); + } + } + if moved.is_empty() { + return Ok(()); + } + if write { + for (file, text) in &texts.sources { + if originals.get(file) != Some(text) { + std::fs::write(root.join(file), text)?; + } + } + std::fs::write(root.join("SPEC.md"), &texts.spec)?; + std::fs::write(root.join("model/ids.toml"), &texts.ids)?; + for (path, text) in &texts.features { + std::fs::write(path, text)?; + } + println!( + "\nwritten; next: cargo xtask validate-model --write, then the parts without LEXLEAN_COUNTS" + ); + } else { + println!("\n(dry run: --write applies it)"); + } + Ok(()) +} + +/// The counts that moved and whose constant still holds the old value. +fn moved_counts( + root: &Path, + seen: &Seen, + texts: &mut Texts, + report: &mut String, + flagged: &mut Vec, +) -> Result, Fail> { + let mut moved = Vec::new(); + for ((id, file, name), (expected, observed)) in seen { + if !texts.sources.contains_key(file) { + let source = std::fs::read_to_string(root.join(file)) + .map_err(|error| format!("counts: {file}: {error}"))?; + texts.sources.insert(file.clone(), source); + } + if expected.len() != 1 || observed.len() != 1 { + flagged.push(format!( + "{id} {name}: the report holds {expected:?} and observes {observed:?}; the checks of one constant disagree, which is no count to write" + )); + continue; + } + let (old, new) = (only(expected), only(observed)); + if old == new { + continue; + } + match constants(&texts.sources[file]).get(name).map(|constant| constant.0) { + Some(value) if value == new => { + let _ = writeln!(report, "{id} {name}: already {new}"); + } + Some(value) if value == old => moved.push(Moved { + id: id.clone(), + file: file.clone(), + name: name.clone(), + old, + new, + }), + Some(value) => flagged.push(format!( + "{id} {name}: the source holds {value} and the report {old} -> {new}; rerun the case" + )), + None => flagged.push(format!( + "{id} {name}: {file} has no `const {name}: usize = ;`; write {old} -> {new} by hand" + )), + } + } + Ok(moved) +} + +/// Rewrite the constant's literal in its source. +fn write_constant(item: &Moved, texts: &mut Texts, report: &mut String) { + let source = &texts.sources[&item.file]; + let (_, index, literal) = constants(source)[&item.name].clone(); + let new_literal = rust_literal(item.new, literal.contains('_')); + let lines: Vec = source + .lines() + .enumerate() + .map(|(at, line)| { + if at == index { + line.replacen(&format!("= {literal};"), &format!("= {new_literal};"), 1) + } else { + line.to_owned() + } + }) + .collect(); + let text = joined(&lines, source); + texts.sources.insert(item.file.clone(), text); + let _ = writeln!( + report, + "{} {}: {} -> {} ({})", + item.id, item.name, item.old, item.new, item.file + ); +} + +/// The numbers of a case's statements to rewrite: each moved count whose old +/// value no other constant of the case holds, and each sum of one pair of its +/// counts that the row states. +fn number_map( + id: &str, + mine: &[&Moved], + table: &BTreeMap, + seen: &Seen, + row: &str, + report: &mut String, + flagged: &mut Vec, +) -> BTreeMap { + let mut map = BTreeMap::new(); + for item in mine { + let others: Vec<&String> = table + .iter() + .filter(|(name, (value, _, _))| { + *name != &item.name + && *value == item.old + && !mine + .iter() + .any(|other| &other.name == *name && other.new == item.new) + }) + .map(|(name, _)| name) + .collect(); + if others.is_empty() { + map.insert(item.old, item.new); + } else { + flagged.push(format!( + "{id} {}: {} is also the value of {others:?}; rewrite its statements by hand ({} -> {})", + item.name, item.old, item.old, item.new + )); + } + } + let counted: Vec<(&String, usize, usize)> = seen + .iter() + .filter(|((case, _, _), (expected, observed))| { + case == id && expected.len() == 1 && observed.len() == 1 + }) + .map(|((_, _, name), (expected, observed))| (name, only(expected), only(observed))) + .collect(); + let known: BTreeSet = table.values().map(|constant| constant.0).collect(); + for token in tokens(row) { + if known.contains(&token.value) || map.contains_key(&token.value) { + continue; + } + let mut sums = BTreeSet::new(); + let mut pairs = Vec::new(); + for (i, (a, a_old, a_new)) in counted.iter().enumerate() { + for (b, b_old, b_new) in &counted[i + 1..] { + if a_old + b_old == token.value && (a_old != a_new || b_old != b_new) { + sums.insert(a_new + b_new); + pairs.push(format!("{a} + {b}")); + } + } + } + match sums.len() { + 0 => {} + 1 => { + let new = only(&sums); + let _ = writeln!( + report, + "{id}: {} is {}: -> {new}", + token.value, + pairs.join(" or ") + ); + map.insert(token.value, new); + } + _ => flagged.push(format!( + "{id}: {} is the sum of {}, which move differently; rewrite it by hand", + token.value, + pairs.join(", ") + )), + } + } + map +} + +/// Rewrite the numbers of the case's §31 row, register statement, and +/// scenario; return the numbers and the words that follow each in the row. +fn statements( + id: &str, + mine: &[&Moved], + table: &BTreeMap, + seen: &Seen, + texts: &mut Texts, + report: &mut String, + flagged: &mut Vec, +) -> Option { + let row_prefix = format!("| `{id}` |"); + let Some(row) = texts + .spec + .lines() + .find(|line| line.starts_with(&row_prefix)) + .map(str::to_owned) + else { + flagged.push(format!("{id}: SPEC.md has no §31 row")); + return None; + }; + let map = number_map(id, mine, table, seen, &row, report, flagged); + let contexts: Vec<(usize, String)> = tokens(&row) + .iter() + .filter(|token| map.contains_key(&token.value)) + .map(|token| (token.value, collapsed(&following(&row, token.end)))) + .filter(|(_, words)| words.trim().len() >= 4) + .collect(); + let (new_row, applied) = rewrite(&row, &map); + let _ = writeln!(report, "{id}: §31 row: {}", applied_text(&applied, &map)); + for (old, new) in &map { + if !applied.contains_key(old) { + let _ = writeln!(report, "{id}: the §31 row does not state {old} (-> {new})"); + } + } + let mut lines: Vec = texts.spec.lines().map(str::to_owned).collect(); + if let Some(at) = lines.iter().position(|line| line == &row) { + lines[at] = new_row; + } + texts.spec = joined(&lines, &texts.spec); + + let marker = format!("id = \"{id}\""); + let mut lines: Vec = texts.ids.lines().map(str::to_owned).collect(); + let statement = lines + .iter() + .position(|line| line == &marker) + .and_then(|at| { + lines[at..] + .iter() + .position(|line| line.starts_with("statement = ")) + .map(|offset| at + offset) + }); + if let Some(at) = statement { + let (line, applied) = rewrite(&lines[at], &map); + lines[at] = line; + let _ = writeln!(report, "{id}: register: {}", applied_text(&applied, &map)); + texts.ids = joined(&lines, &texts.ids); + } else { + flagged.push(format!("{id}: model/ids.toml has no statement")); + } + + let tag = format!("@{id} "); + for text in texts.features.values_mut() { + let mut lines: Vec = text.lines().map(str::to_owned).collect(); + let Some(at) = lines + .iter() + .position(|line| line.trim_start().starts_with(&tag)) + else { + continue; + }; + if let Some(next) = lines.get(at + 1) { + let (line, applied) = rewrite(next, &map); + lines[at + 1] = line; + let _ = writeln!(report, "{id}: scenario: {}", applied_text(&applied, &map)); + } + *text = joined(&lines, text); + } + Some((map, contexts)) +} + +/// `text` with every run of whitespace one space. +fn collapsed(text: &str) -> String { + let mut out = String::new(); + let mut space = false; + for c in text.chars() { + if c.is_whitespace() { + space = true; + } else { + if space { + out.push(' '); + space = false; + } + out.push(c); + } + } + if space { + out.push(' '); + } + out +} + +/// The numbers every constant holding them moves alike: each moved old +/// value that no constant of any bootstrap case holds unless it moves to the +/// same new value, and each sum of counts (`derived`) that no constant holds. +/// Prose of §17.23 states such a number without naming a case (the committed +/// manifests every case reads, say). +fn shared_map( + root: &Path, + moved: &[Moved], + derived: &[(usize, usize)], +) -> Result, Fail> { + let mut holders: Vec<(String, String, usize)> = Vec::new(); + let dir = root.join("crates/conformance/src/cases/bootstrap"); + let mut paths: Vec = std::fs::read_dir(&dir)? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .collect(); + paths.push(root.join("crates/conformance/src/cases/bootstrap.rs")); + for path in paths { + if path.extension().is_none_or(|ext| ext != "rs") { + continue; + } + let file = path + .strip_prefix(root) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + let source = std::fs::read_to_string(&path)?; + for (name, (value, _, _)) in constants(&source) { + holders.push((file.clone(), name, value)); + } + } + let mut out = BTreeMap::new(); + let olds: BTreeSet = moved + .iter() + .map(|item| item.old) + .chain(derived.iter().map(|(old, _)| *old)) + .collect(); + for old in olds { + let news: BTreeSet = moved + .iter() + .filter(|item| item.old == old) + .map(|item| item.new) + .chain( + derived + .iter() + .filter(|(from, _)| *from == old) + .map(|(_, new)| *new), + ) + .collect(); + let still = holders.iter().any(|(file, name, value)| { + *value == old + && !moved + .iter() + .any(|item| &item.file == file && &item.name == name) + }); + if news.len() == 1 && !still { + out.insert(old, only(&news)); + } + } + Ok(out) +} + +/// `old -> new (times)` for each replaced number. +fn applied_text(applied: &BTreeMap, map: &BTreeMap) -> String { + if applied.is_empty() { + return "nothing".to_owned(); + } + applied + .iter() + .map(|(old, times)| format!("{old} -> {} (x{times})", map[old])) + .collect::>() + .join(", ") +} + +/// The text after byte `end` of line `at`, running on over the next lines of +/// its paragraph, with whitespace collapsed: enough to compare with a row's +/// words across the prose's line breaks. +fn after(lines: &[String], at: usize, end: usize) -> String { + let mut text = lines[at][end..].to_owned(); + for line in lines.iter().skip(at + 1).take(3) { + if line.trim().is_empty() { + break; + } + text.push(' '); + text.push_str(line); + } + collapsed(&text) +} + +/// Rewrite the numbers of the prose of §17.23: each a case's row states that +/// the row's words follow there too, and each every holder moves alike +/// (`shared`); then list the moved numbers it still states. Matches are taken +/// on the prose as it was, then applied at once. +fn prose( + anchored: &[Anchored], + shared: &BTreeMap, + texts: &mut Texts, + report: &mut String, + flagged: &mut Vec, +) { + let mut lines: Vec = texts.spec.lines().map(str::to_owned).collect(); + let Some(start) = lines.iter().position(|line| line.starts_with("### 17.23 ")) else { + flagged.push("SPEC.md has no §17.23".to_owned()); + return; + }; + let end = lines[start + 1..] + .iter() + .position(|line| line.starts_with("### ") || line.starts_with("## ")) + .map_or(lines.len(), |offset| start + 1 + offset); + let mut edits: Vec<(usize, usize, usize, String, String)> = Vec::new(); + for at in start..end { + if lines[at].starts_with('|') { + continue; + } + for token in tokens(&lines[at]) { + let words = after(&lines, at, token.end); + let mut by: Option<(usize, String)> = None; + for (id, map, contexts) in anchored { + if contexts.iter().any(|(old, context)| { + *old == token.value && words.starts_with(context.as_str()) + }) { + by = Some((map[&token.value], id.clone())); + break; + } + } + if by.is_none() { + if let Some(new) = shared.get(&token.value) { + by = Some((*new, "every holder".to_owned())); + } + } + if let Some((new, why)) = by { + edits.push((at, token.start, token.end, render(new, token.commas), why)); + } + } + } + for (at, start, end, new, why) in edits.iter().rev() { + let old = lines[*at][*start..*end].to_owned(); + lines[*at].replace_range(*start..*end, new); + let _ = writeln!(report, "SPEC.md:{}: {old} -> {new} ({why})", at + 1); + } + let moved: BTreeMap = anchored + .iter() + .flat_map(|(id, map, _)| { + map.iter() + .map(move |(old, new)| (*old, (*new, id.as_str()))) + }) + .collect(); + for at in start..end { + if lines[at].starts_with('|') { + continue; + } + let done: BTreeSet = edits + .iter() + .filter(|edit| edit.0 == at) + .map(|edit| edit.1) + .collect(); + for token in tokens(&lines[at]) { + if done.contains(&token.start) { + continue; + } + if let Some((new, id)) = moved.get(&token.value) { + flagged.push(format!( + "SPEC.md:{} still states {} (-> {new} for {id}?) before \"{}\"", + at + 1, + token.value, + after(&lines, at, token.end) + .chars() + .take(40) + .collect::() + .trim() + )); + } + } + } + texts.spec = joined(&lines, &texts.spec); +} + +#[cfg(test)] +mod tests { + use super::{after, collapsed, following, render, rewrite, rust_literal, tokens}; + use std::collections::BTreeMap; + + #[test] + fn prose_words_are_read_across_its_line_breaks_within_a_paragraph() { + let lines: Vec = ["run 2,927 vectors: 714", "of `printCrate` (546", "", "next"] + .iter() + .map(|line| (*line).to_owned()) + .collect(); + let words = after(&lines, 0, "run 2,927".len()); + assert_eq!(words, " vectors: 714 of `printCrate` (546"); + assert!(words.starts_with(&collapsed(" vectors: 714\nof"))); + } + + #[test] + fn numbers_are_read_with_their_groups_and_not_from_names() { + let line = "the 2,838 vectors of `BS-05` (§17.16, 0x0B55) are 693 and 2,145, 3522."; + let values: Vec = tokens(line).iter().map(|token| token.value).collect(); + assert_eq!(values, vec![2_838, 693, 2_145, 3_522]); + } + + #[test] + fn a_rewrite_is_simultaneous_and_keeps_the_style() { + let map = BTreeMap::from([(693, 714), (714, 800), (2_145, 2_213)]); + let (line, applied) = rewrite("693 of 714, then 2,145 and 2145", &map); + assert_eq!(line, "714 of 800, then 2,213 and 2213"); + assert_eq!(applied[&2_145], 2); + let (grown, _) = rewrite("999 and 1,000", &BTreeMap::from([(999, 1_001)])); + assert_eq!(grown, "1,001 and 1,000"); + } + + #[test] + fn literals_and_contexts() { + assert_eq!(rust_literal(2_989, true), "2_989"); + assert_eq!(rust_literal(3_522, false), "3522"); + assert_eq!(rust_literal(12_000, false), "12_000"); + assert_eq!(render(1_532, true), "1,532"); + assert_eq!( + following("509 runs of `validateCrate` and more", 3), + " runs of `validateCrate`" + ); + assert_eq!(following("66 | rest", 2), " "); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9887022ba5..93bee814d7 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -12,6 +12,7 @@ use camino::Utf8PathBuf; mod audit; mod ci; mod codegen; +mod counts; mod release_manifest; mod spec_links; @@ -36,6 +37,7 @@ fn main() -> ExitCode { "check-calculus" => check_calculus(&root, write), "check-production-export" => check_production_export(write), "check-package" => check_package(&root), + "counts" => counts::counts(&root, write), "check-test-inventory" => ci::check_test_inventory(&root), "bdd" => ci::bdd(&root), "ci-summary" => ci::ci_summary(&root, &std::env::args().skip(2).collect::>()), @@ -58,6 +60,8 @@ fn main() -> ExitCode { check-calculus §17.14-§17.16: the compiler project's modules, the target and\n\ GNAF fixtures, the Rust packages, and configuration equal\n\ their generator\n\ + counts §17.23: write the counts a bootstrap case reported under\n\ + LEXLEAN_COUNTS=fast|report into its constants and statements\n\ check-production-export §24.7: the consumer's exported packages equal the committed\n\ packages under tests/production-export\n\ check-package §30.3, RP-12: package lexlean crate, verify offline build & identity\n\ From 9204a44e20ef2b5797ac36f10f05dd65aaaec50a Mon Sep 17 00:00:00 2001 From: humuhumu33 Date: Mon, 9 Nov 2026 09:24:55 +0000 Subject: [PATCH 2/2] xtask: the crate description names the counts task (RP-05) RP-05 holds the description of xtask/Cargo.toml to every task main.rs dispatches; `counts` is one now. Co-Authored-By: Claude Opus 5.5 --- xtask/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index f6090f3784..84ec1de64d 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xtask" -description = "Repository gates: validate-model, validate-spec-links, verify-examples, check-golden, check-reproducibility, check-fixtures, check-calculus, check-production-export, check-package, check-test-inventory, bdd, ci-summary, ci-hints, ci-shards, release-artifacts, release-check, validate." +description = "Repository gates: validate-model, validate-spec-links, verify-examples, check-golden, check-reproducibility, check-fixtures, check-calculus, check-production-export, check-package, counts, check-test-inventory, bdd, ci-summary, ci-hints, ci-shards, release-artifacts, release-check, validate." version.workspace = true edition.workspace = true rust-version.workspace = true