From 9727a8c61e43d5199d82ef64dc7846dd55a30394 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 17:26:24 +0000 Subject: [PATCH 1/2] conformance: run one of BS-05..BS-08 per 8 GiB at a time Each checks its own copy of the bootstrap project; the four together OOM-killed the 16 GiB aarch64 runner mid cargo test (runner lost with "shutdown signal"). Measured: BS-07 alone 4.4 GiB, BS-05 with BS-07 past 10 GiB. A process-wide gate in cases::run admits one per 8 GiB of MemTotal (one on that runner); other cases are unaffected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3QVAsEzzaQPMzJnNz1 --- CHANGELOG.md | 8 ++++ crates/conformance/src/cases/mod.rs | 69 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d25c7f8e7..5714c5e194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,14 @@ workspace carried at those versions without a tag. ## Unreleased +- At most one of BS-05..BS-08 per 8 GiB of the host's memory runs at a time in + one test process (no SPEC, language, or identity change). Each checks its own + copy of the bootstrap project: BS-07 alone peaks at 4.4 GiB, BS-05 with BS-07 + passes 10 GiB, and the four together were OOM-killed on the 16 GiB aarch64 + runner, which lost the `build and non-Lean tests (aarch64-unknown-linux-gnu)` + job mid-run. On that runner they now run one at a time. A waiting case is + not timed. + - A semantic module nests as deeply as the project's `max_scope_depth` allows (SPEC.md §17.17, §17.21.3, `SM-12`; no new diagnostic code; no language identity moves). The front end read a module's JSON with a fixed 128-level diff --git a/crates/conformance/src/cases/mod.rs b/crates/conformance/src/cases/mod.rs index baba52ebaf..873f043d32 100644 --- a/crates/conformance/src/cases/mod.rs +++ b/crates/conformance/src/cases/mod.rs @@ -79,6 +79,74 @@ mod wasi; /// The wall time of one case, written to the CI evidence directory when the /// environment names one (SPEC.md §22.12) and nowhere else. It is written when /// the case ends, by panic as well, and a failure to write it is dropped. +/// How many of the heaviest bootstrap cases run at once in one process. +/// Each checks its own copy of the whole bootstrap project and then works +/// over the committed corpus: BS-07 alone passes 4 GiB, BS-05 with BS-07 +/// passes 10 GiB, and BS-05..BS-08 together were OOM-killed on the 16 GiB +/// aarch64 runner. The harness runs one thread per processor, so without a +/// bound the processors, not the memory, decide how many overlap. A case +/// waits here before it starts, so its recorded wall time excludes the wait. +mod heavy { + use std::sync::{Condvar, Mutex}; + + /// The cases bounded: the four whose lanes split the macOS, Windows, and + /// Intel hosts. + const HEAVY: [&str; 4] = ["BS-05", "BS-06", "BS-07", "BS-08"]; + + /// The memory one of them is given. + const BUDGET_KIB: u64 = 8 * 1024 * 1024; + + static RUNNING: Mutex = Mutex::new(0); + static FREED: Condvar = Condvar::new(); + + /// One case per [`BUDGET_KIB`] of the host's memory, at least one; + /// unbounded where the total is not readable. + fn capacity() -> usize { + std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|text| { + let line = text.lines().find(|line| line.starts_with("MemTotal:"))?; + line.split_whitespace().nth(1)?.parse::().ok() + }) + .map_or(usize::MAX, |total| { + usize::try_from(total / BUDGET_KIB) + .unwrap_or(usize::MAX) + .max(1) + }) + } + + /// Admission of one heavy case, released when dropped, on a panic too. + pub(super) struct Admitted; + + impl Drop for Admitted { + fn drop(&mut self) { + let mut running = RUNNING + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *running -= 1; + FREED.notify_one(); + } + } + + /// Waits until a heavy case may start; other cases start at once. + pub(super) fn admit(id: &str) -> Option { + if !HEAVY.contains(&id) { + return None; + } + let limit = capacity(); + let mut running = RUNNING + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while *running >= limit { + running = FREED + .wait(running) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + *running += 1; + Some(Admitted) + } +} + struct Timed { id: String, started: std::time::Instant, @@ -119,6 +187,7 @@ impl Drop for Timed { /// Panics when the case's assertion fails, and for an ID with no wired /// case — an unimplemented registered capability must not pass silently. pub fn run(id: &str) { + let _admitted = heavy::admit(id); let _timed = Timed::start(id); let prefix = id.split('-').next().unwrap_or_default(); match prefix { From 3ccd3fa2374bf80887769d0af2615e1575dbbfac Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 18:12:11 +0000 Subject: [PATCH 2/2] conformance: plan 10 GiB per heavy bootstrap case; document the bound's scope A 16 GiB host then runs BS-05..BS-08 one at a time whatever its exact MemTotal (review: 8 GiB gave two slots at exactly 16 GiB). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3QVAsEzzaQPMzJnNz1 --- CHANGELOG.md | 14 +++++++------- crates/conformance/src/cases/mod.rs | 24 +++++++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f480444e..fcbc71df1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,13 +17,13 @@ workspace carried at those versions without a tag. ## Unreleased -- At most one of BS-05..BS-08 per 8 GiB of the host's memory runs at a time in - one test process (no SPEC, language, or identity change). Each checks its own - copy of the bootstrap project: BS-07 alone peaks at 4.4 GiB, BS-05 with BS-07 - passes 10 GiB, and the four together were OOM-killed on the 16 GiB aarch64 - runner, which lost the `build and non-Lean tests (aarch64-unknown-linux-gnu)` - job mid-run. On that runner they now run one at a time. A waiting case is - not timed. +- At most one of BS-05..BS-08 per 10 GiB of the host's memory runs at a time + in one test process (no SPEC, language, or identity change), so a 16 GiB host + runs them one at a time. Each checks its own copy of the bootstrap project: + on a 16 GiB host BS-07 alone peaks at 4.4 GiB, and BS-05 with BS-07, like + the four together, was OOM-killed. The 16 GiB aarch64 runner lost the + `build and non-Lean tests (aarch64-unknown-linux-gnu)` job while the four + ran together. A waiting case is not timed. ## 0.4.3 diff --git a/crates/conformance/src/cases/mod.rs b/crates/conformance/src/cases/mod.rs index 873f043d32..357b62d1e4 100644 --- a/crates/conformance/src/cases/mod.rs +++ b/crates/conformance/src/cases/mod.rs @@ -81,11 +81,19 @@ mod wasi; /// the case ends, by panic as well, and a failure to write it is dropped. /// How many of the heaviest bootstrap cases run at once in one process. /// Each checks its own copy of the whole bootstrap project and then works -/// over the committed corpus: BS-07 alone passes 4 GiB, BS-05 with BS-07 -/// passes 10 GiB, and BS-05..BS-08 together were OOM-killed on the 16 GiB -/// aarch64 runner. The harness runs one thread per processor, so without a -/// bound the processors, not the memory, decide how many overlap. A case -/// waits here before it starts, so its recorded wall time excludes the wait. +/// over the committed corpus. Measured on a 4-processor host with 16 GiB +/// (`MemTotal` 16473776 KiB) under a 13.4 GiB memory limit: BS-07 alone +/// peaks at 4.4 GiB, BS-05 with BS-07 was OOM-killed past 10 GiB, and +/// BS-05..BS-08 together within a minute. The harness runs one thread per +/// processor, so without a bound the processors, not the memory, decide how +/// many overlap. A case waits here before it starts, so its recorded wall +/// time excludes the wait. +/// +/// The bound is a planning figure for this process alone, not a guarantee: +/// it reads the host's total memory once per admission and ignores cgroup +/// limits, current pressure, other processes, and the other cases, and where +/// `/proc/meminfo` is unreadable (macOS, Windows, whose lanes already run +/// these cases apart) it admits every case. mod heavy { use std::sync::{Condvar, Mutex}; @@ -93,8 +101,10 @@ mod heavy { /// Intel hosts. const HEAVY: [&str; 4] = ["BS-05", "BS-06", "BS-07", "BS-08"]; - /// The memory one of them is given. - const BUDGET_KIB: u64 = 8 * 1024 * 1024; + /// The memory one of them is planned for: more than a measured pair, so + /// a host with less than 20 GiB, the 16 GiB runners whatever their exact + /// `MemTotal`, runs one at a time. + const BUDGET_KIB: u64 = 10 * 1024 * 1024; static RUNNING: Mutex = Mutex::new(0); static FREED: Condvar = Condvar::new();