diff --git a/CHANGELOG.md b/CHANGELOG.md index ed3f74c5da..fcbc71df1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,14 @@ workspace carried at those versions without a tag. ## Unreleased +- At most one of BS-05..BS-08 per 10 GiB of the host's memory runs at a time + in one test process (no SPEC, language, or identity change), so a 16 GiB host + runs them one at a time. Each checks its own copy of the bootstrap project: + on a 16 GiB host BS-07 alone peaks at 4.4 GiB, and BS-05 with BS-07, like + the four together, was OOM-killed. The 16 GiB aarch64 runner lost the + `build and non-Lean tests (aarch64-unknown-linux-gnu)` job while the four + ran together. A waiting case is not timed. + ## 0.4.3 - A semantic module nests as deeply as the project's `max_scope_depth` allows diff --git a/crates/conformance/src/cases/mod.rs b/crates/conformance/src/cases/mod.rs index baba52ebaf..357b62d1e4 100644 --- a/crates/conformance/src/cases/mod.rs +++ b/crates/conformance/src/cases/mod.rs @@ -79,6 +79,84 @@ mod wasi; /// The wall time of one case, written to the CI evidence directory when the /// environment names one (SPEC.md §22.12) and nowhere else. It is written when /// the case ends, by panic as well, and a failure to write it is dropped. +/// How many of the heaviest bootstrap cases run at once in one process. +/// Each checks its own copy of the whole bootstrap project and then works +/// over the committed corpus. Measured on a 4-processor host with 16 GiB +/// (`MemTotal` 16473776 KiB) under a 13.4 GiB memory limit: BS-07 alone +/// peaks at 4.4 GiB, BS-05 with BS-07 was OOM-killed past 10 GiB, and +/// BS-05..BS-08 together within a minute. The harness runs one thread per +/// processor, so without a bound the processors, not the memory, decide how +/// many overlap. A case waits here before it starts, so its recorded wall +/// time excludes the wait. +/// +/// The bound is a planning figure for this process alone, not a guarantee: +/// it reads the host's total memory once per admission and ignores cgroup +/// limits, current pressure, other processes, and the other cases, and where +/// `/proc/meminfo` is unreadable (macOS, Windows, whose lanes already run +/// these cases apart) it admits every case. +mod heavy { + use std::sync::{Condvar, Mutex}; + + /// The cases bounded: the four whose lanes split the macOS, Windows, and + /// Intel hosts. + const HEAVY: [&str; 4] = ["BS-05", "BS-06", "BS-07", "BS-08"]; + + /// The memory one of them is planned for: more than a measured pair, so + /// a host with less than 20 GiB, the 16 GiB runners whatever their exact + /// `MemTotal`, runs one at a time. + const BUDGET_KIB: u64 = 10 * 1024 * 1024; + + static RUNNING: Mutex = Mutex::new(0); + static FREED: Condvar = Condvar::new(); + + /// One case per [`BUDGET_KIB`] of the host's memory, at least one; + /// unbounded where the total is not readable. + fn capacity() -> usize { + std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|text| { + let line = text.lines().find(|line| line.starts_with("MemTotal:"))?; + line.split_whitespace().nth(1)?.parse::().ok() + }) + .map_or(usize::MAX, |total| { + usize::try_from(total / BUDGET_KIB) + .unwrap_or(usize::MAX) + .max(1) + }) + } + + /// Admission of one heavy case, released when dropped, on a panic too. + pub(super) struct Admitted; + + impl Drop for Admitted { + fn drop(&mut self) { + let mut running = RUNNING + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *running -= 1; + FREED.notify_one(); + } + } + + /// Waits until a heavy case may start; other cases start at once. + pub(super) fn admit(id: &str) -> Option { + if !HEAVY.contains(&id) { + return None; + } + let limit = capacity(); + let mut running = RUNNING + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while *running >= limit { + running = FREED + .wait(running) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + *running += 1; + Some(Admitted) + } +} + struct Timed { id: String, started: std::time::Instant, @@ -119,6 +197,7 @@ impl Drop for Timed { /// Panics when the case's assertion fails, and for an ID with no wired /// case — an unimplemented registered capability must not pass silently. pub fn run(id: &str) { + let _admitted = heavy::admit(id); let _timed = Timed::start(id); let prefix = id.split('-').next().unwrap_or_default(); match prefix {