From 1c11efb6e9be957295a1e1d668752085ddd6f6d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 18:15:05 +0000 Subject: [PATCH] AU-01: read the corpus one module at a time The case collected every semantic module of the repository as a value before checking any. The corpus includes modules of tens of megabytes, so AU-01 alone peaked at 11.2 GiB resident in the debug test process. The bdd-verification leg runs it on four test threads beside AU-08, AU-09 and the Lean processes of NE-01 and the VR cases on a 16 GB runner, and the leg intermittently exhausted memory and swap and lost the runner, failing whichever case was running then (NE-01 on #405, AU-08, OR-01, VR-11 on v0.4 pull requests). corpus() is now a lazy iterator over the walk: the same modules, in the same order, under the same assertions, each dropped once checked. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3M8ULnzEQwLLngyBBX --- CHANGELOG.md | 10 +++++++ crates/conformance/src/cases/authoring.rs | 34 +++++++++++------------ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b88dae65c..4f76b80ae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,16 @@ workspace carried at those versions without a tag. ## Unreleased +- `AU-01` reads the committed corpus one module at a time (SPEC.md §9.4). It + collected every semantic module of the repository as a value before checking + any, and the corpus holds modules of tens of megabytes, so the case alone + peaked at 11.2 GiB of resident memory in a debug test process. The + `bdd-verification` leg runs it on four test threads beside `AU-08`, `AU-09` + and the Lean processes of `NE-01` and the `VR` cases on a 16 GB runner, and + the leg intermittently exhausted memory and swap and lost the runner, failing + whichever case was running then. The case checks the same modules with the + same assertions; each is dropped once checked. + - Canonical JSON refuses duplicate keys that differ only in their escapes (SPEC.md §21.7, `AR-11`, #475). The duplicate check compared each key as written, so `{"a":1,"\u0061":2}` passed it and `Json::parse` returned `{"a":2}`, dropping the first member; keys are now diff --git a/crates/conformance/src/cases/authoring.rs b/crates/conformance/src/cases/authoring.rs index 8989617e1b..1395f6e7dd 100644 --- a/crates/conformance/src/cases/authoring.rs +++ b/crates/conformance/src/cases/authoring.rs @@ -75,10 +75,15 @@ const CORE_SPEC: &str = "lexlean/core-module/2"; /// Every semantic module of a committed source, as canonical JSON, with the /// file it is in. A module that is not well formed is a negative fixture's /// business and is left out. -fn corpus() -> Vec<(String, Json)> { +/// +/// The modules are read one at a time, as the walk reaches them, and each is +/// dropped when the caller is done with it: the corpus includes modules of +/// tens of megabytes, and held whole as values it peaked at over 11 GiB of a +/// debug test process, most of the 16 GB runner its CI leg shares with the +/// part's other cases (§9.4). +fn corpus() -> impl Iterator { let root = repo_root(); - let mut modules = Vec::new(); - for entry in walkdir::WalkDir::new(root.as_std_path()) + walkdir::WalkDir::new(root.as_std_path()) .into_iter() .filter_entry(|entry| { let name = entry.file_name().to_string_lossy(); @@ -88,21 +93,16 @@ fn corpus() -> Vec<(String, Json)> { && !support::holds_cache_tag(entry) }) .flatten() - { - let path = entry.path(); - if !path.to_string_lossy().ends_with(".lex.tex") { - continue; - } - let Ok(text) = std::fs::read_to_string(path) else { - continue; - }; - if let Ok(value) = support::try_module_json(&text) { - if let Ok(json) = Json::parse(value.to_string().as_bytes()) { - modules.push((path.display().to_string(), json)); + .filter_map(|entry| { + let path = entry.path(); + if !path.to_string_lossy().ends_with(".lex.tex") { + return None; } - } - } - modules + let text = std::fs::read_to_string(path).ok()?; + let value = support::try_module_json(&text).ok()?; + let json = Json::parse(value.to_string().as_bytes()).ok()?; + Some((path.display().to_string(), json)) + }) } // ---------------------------------------------------------------------------