From 12728aec986bf84e3ebc42bdbd425a6b2fbfe97e Mon Sep 17 00:00:00 2001 From: McSon2 Date: Mon, 13 Jul 2026 00:33:20 +0200 Subject: [PATCH 1/2] fix(lgs): bound books cache memory with mmap-backed storage and LRU eviction The LGS kept every played mode's books fully decompressed in an anonymous heap buffer, cached forever. With real publish files (0.5-1 GiB compressed, 5-19 GiB decompressed per mode) a session touching a few modes grew past physical RAM and pushed the whole machine into swap: measured 27 GB footprint after 14 minutes on a 24 GB machine. - Stream-decompress each books file into an unlinked temp file and mmap it read-only. Pages are file-backed and clean, so the OS reclaims them under memory pressure instead of swapping. Measured footprint for four cached tachyon modes (~15 GiB decompressed): 22 MB dirty, down from 27 GB. - Evict least-recently-used modes beyond 8 entries or 40 GiB of total decompressed bytes. In-flight spins keep their Arc alive, so eviction is safe mid-request. - Widen id_to_range offsets from u32 to u64. Decompressed books routinely exceed 4 GiB (tachyon BONUS: 5.4 GiB), where u32 offsets silently wrapped and corrupted every read past the 4 GiB boundary. Verified by forcing the last book of a 5.4 GiB file: payout now matches the lookup table exactly. - Build the id index while the decompressed stream is being written (IndexingWriter harvests ids at line starts), instead of re-reading and JSON-parsing the multi-GiB buffer afterwards. Falls back to the previous exhaustive serde scan whenever the line-based index does not cover every id present in the weights table (e.g. records not separated by newlines). Cold mode load on a 19 GiB books file: 68 s -> 21 s; warm hits unchanged. - Size the id map modestly and shrink_to_fit once counts are known; the previous len/512 pre-allocation reserved ~500 MB per large mode. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 12 ++ Cargo.toml | 2 + crates/lgs/Cargo.toml | 3 + crates/lgs/src/math_engine.rs | 202 ++++++++++++++++++++++++++++++---- 4 files changed, 198 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5320ef7..314e3c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2377,6 +2377,8 @@ dependencies = [ "dirs 5.0.1", "getrandom 0.2.17", "include_dir", + "memchr", + "memmap2", "mime_guess", "once_cell", "parking_lot", @@ -2389,6 +2391,7 @@ dependencies = [ "serde", "serde_json", "sonic-rs", + "tempfile", "thiserror 2.0.18", "time", "tokio", @@ -2593,6 +2596,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" diff --git a/Cargo.toml b/Cargo.toml index 2ee5d6c..7e39831 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,8 @@ thiserror = "2.0" once_cell = "1.20" parking_lot = "0.12" memmap2 = "0.9" +tempfile = "3" +memchr = "2" arc-swap = "1.7" uuid = { version = "1.10", features = ["v4", "serde"] } url = "2.5" diff --git a/crates/lgs/Cargo.toml b/crates/lgs/Cargo.toml index 5bc75d9..59b0017 100644 --- a/crates/lgs/Cargo.toml +++ b/crates/lgs/Cargo.toml @@ -45,3 +45,6 @@ thiserror.workspace = true once_cell.workspace = true parking_lot.workspace = true arc-swap.workspace = true +memmap2.workspace = true +tempfile.workspace = true +memchr.workspace = true diff --git a/crates/lgs/src/math_engine.rs b/crates/lgs/src/math_engine.rs index 7b7a4d6..01f954e 100644 --- a/crates/lgs/src/math_engine.rs +++ b/crates/lgs/src/math_engine.rs @@ -2,6 +2,7 @@ use crate::config::ServerConfig; use crate::error::{AppError, AppResult}; use crate::types::{GameConfig, GameMode, WeightEntry}; use dashmap::DashMap; +use memmap2::Mmap; use rand::RngCore; use serde::Serialize; use serde_json::value::RawValue; @@ -12,12 +13,16 @@ use tokio::fs; use tokio::sync::OnceCell; pub struct BooksIndex { - pub buffer: Vec, + /// Decompressed books, backed by an unlinked temp file via mmap. File-backed + /// pages stay clean, so the OS can reclaim them under memory pressure + /// instead of pushing multi-GB buffers to swap. + buffer: Mmap, /// Maps each book's `id` field to its (start, end) byte range in `buffer`. /// Built by scanning every line at load time — indexing by `id` rather than /// by line position because math-sdk writes `library[sim+1] = Book(sim)`, /// so line N contains id N-1 (not id N as the name might suggest). - pub id_to_range: HashMap, + /// Offsets are u64: decompressed books routinely exceed 4 GiB. + pub id_to_range: HashMap, } pub struct WeightSampler { @@ -31,10 +36,20 @@ pub struct ModeAssets { pub books: Arc, } +/// Decompressed books are huge (frequently several GiB per mode), so only the +/// most recently used modes stay cached; older entries are dropped, releasing +/// their temp-file-backed mmaps. In-flight spins keep their `Arc` +/// alive, so eviction is safe mid-request. The cache is bounded both by entry +/// count and by total decompressed bytes, since each cached mode holds its +/// decompressed size in temp-file disk space. +const MAX_CACHED_MODES: usize = 8; +const MAX_CACHED_BYTES: u64 = 40 * 1024 * 1024 * 1024; + pub struct MathEngine { cfg: ServerConfig, configs: DashMap>>>, modes: DashMap>>>, + mode_lru: parking_lot::Mutex>, } impl MathEngine { @@ -43,6 +58,28 @@ impl MathEngine { cfg, configs: DashMap::new(), modes: DashMap::new(), + mode_lru: parking_lot::Mutex::new(Vec::new()), + } + } + + fn touch_mode_cache(&self, key: &str, bytes: u64) { + let evicted: Vec<(String, u64)> = { + let mut lru = self.mode_lru.lock(); + lru.retain(|(k, _)| k != key); + lru.push((key.to_string(), bytes)); + let mut evicted = Vec::new(); + // Always keep at least the entry just touched, whatever its size. + while lru.len() > 1 + && (lru.len() > MAX_CACHED_MODES + || lru.iter().map(|(_, b)| *b).sum::() > MAX_CACHED_BYTES) + { + evicted.push(lru.remove(0)); + } + evicted + }; + for (old, old_bytes) in evicted { + self.modes.remove(&old); + tracing::info!(mode = %old, gib = old_bytes / (1024 * 1024 * 1024), "evicted books cache (LRU)"); } } @@ -108,20 +145,29 @@ impl MathEngine { let key = format!("{game}:{}", mode.name); let cell = self .modes - .entry(key) + .entry(key.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone(); let game = game.to_string(); let mode = mode.clone(); - cell.get_or_try_init(|| async move { + let assets = cell.get_or_try_init(|| async move { let weights_bytes = self.read_file(&game, &mode.weights).await?; - let books_bytes = self.read_file(&game, &mode.events).await?; let weights_text = String::from_utf8(weights_bytes) .map_err(|e| AppError::Parse(format!("weights utf8: {e}")))?; let sampler = parse_weights(&weights_text)?; - let books = decompress_and_index(&books_bytes)?; + // Stream the compressed books straight from disk: at up to ~1 GiB + // compressed, buffering the whole file first would leave a same- + // sized hole in the allocator's large-block cache on every load. + let books_path = self.file_path(&game, &mode.events); + let books_file = std::fs::File::open(&books_path) + .map_err(|e| AppError::Parse(format!("read {}: {e}", books_path.display())))?; + let required_ids: Vec = sampler.entries.iter().map(|e| e.event_id).collect(); + let books = decompress_and_index( + std::io::BufReader::with_capacity(4 << 20, books_file), + &required_ids, + )?; Ok::, AppError>(Arc::new(ModeAssets { sampler: Arc::new(sampler), @@ -129,7 +175,9 @@ impl MathEngine { })) }) .await - .cloned() + .cloned()?; + self.touch_mode_cache(&key, assets.books.buffer.len() as u64); + Ok(assets) } pub async fn preload(&self, game: &str) -> AppResult<()> { @@ -398,38 +446,148 @@ fn parse_weights(text: &str) -> AppResult { }) } -fn decompress_and_index(compressed: &[u8]) -> AppResult { - let buffer = zstd::decode_all(compressed).map_err(|e| AppError::Zstd(e.to_string()))?; +fn decompress_and_index( + compressed: impl std::io::Read, + required_ids: &[u32], +) -> AppResult { + // Stream-decompress into an unlinked temp file, then mmap it read-only. + // The temp file has no path (already deleted); the OS frees the disk space + // as soon as the mmap is dropped. + let file = tempfile::tempfile().map_err(|e| AppError::Zstd(format!("temp books file: {e}")))?; + + // Fast path: math-sdk publish files are strict JSONL, so ids can be + // harvested from line starts while the decompressed stream is being + // written out — the multi-GiB buffer is never re-read. Trust the result + // only if it accounts for every id the weights table can ask for; + // otherwise (adjacent records, multi-line values…) fall back to an + // exhaustive JSON stream scan of the mmap. + let mut writer = IndexingWriter::new(std::io::BufWriter::with_capacity(4 << 20, file)); + zstd::stream::copy_decode(compressed, &mut writer) + .map_err(|e| AppError::Zstd(e.to_string()))?; + let (writer, mut id_to_range) = writer.finish(); + let file = writer + .into_inner() + .map_err(|e| AppError::Zstd(format!("flush books: {e}")))?; + let buffer = + unsafe { Mmap::map(&file) }.map_err(|e| AppError::Zstd(format!("mmap books: {e}")))?; + + if !required_ids.iter().all(|id| id_to_range.contains_key(id)) { + tracing::warn!("line-based books index incomplete; falling back to full JSON scan"); + id_to_range = index_by_json_stream(&buffer)?; + } + + Ok(BooksIndex { + buffer, + id_to_range, + }) +} + +/// The first bytes of a line always suffice to read its `id` field +/// (`{"id":N` with optional whitespace), so no full line is ever buffered. +const LINE_HEAD_CAP: usize = 64; + +/// Write adapter that forwards the decompressed stream to `inner` while +/// building the id → byte-range index from line boundaries on the fly. +struct IndexingWriter { + inner: W, + offset: u64, + line_start: u64, + head: Vec, + id_to_range: HashMap, +} + +impl IndexingWriter { + fn new(inner: W) -> Self { + Self { + inner, + offset: 0, + line_start: 0, + head: Vec::with_capacity(LINE_HEAD_CAP), + id_to_range: HashMap::with_capacity(64 * 1024), + } + } - let mut id_to_range = HashMap::with_capacity(buffer.len() / 512 + 1); + fn feed(&mut self, buf: &[u8]) { + let mut pos = 0usize; + while pos < buf.len() { + match memchr::memchr(b'\n', &buf[pos..]) { + Some(i) => { + let take = i.min(LINE_HEAD_CAP.saturating_sub(self.head.len())); + self.head.extend_from_slice(&buf[pos..pos + take]); + if let Some(id) = read_id_field(&self.head) { + let nl_abs = self.offset + (pos + i) as u64; + self.id_to_range.insert(id, (self.line_start, nl_abs)); + self.line_start = nl_abs + 1; + } else { + self.line_start = self.offset + (pos + i) as u64 + 1; + } + self.head.clear(); + pos += i + 1; + } + None => { + let take = (buf.len() - pos).min(LINE_HEAD_CAP.saturating_sub(self.head.len())); + self.head.extend_from_slice(&buf[pos..pos + take]); + break; + } + } + } + self.offset += buf.len() as u64; + } + + fn finish(mut self) -> (W, HashMap) { + // Trailing line without a final newline. + if self.offset > self.line_start { + if let Some(id) = read_id_field(&self.head) { + self.id_to_range.insert(id, (self.line_start, self.offset)); + } + } + self.id_to_range.shrink_to_fit(); + (self.inner, self.id_to_range) + } +} + +impl std::io::Write for IndexingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.inner.write_all(buf)?; + self.feed(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +fn index_by_json_stream(buffer: &[u8]) -> AppResult> { + // Books average well above 512 bytes each (often tens of KB), so sizing the + // map from len/512 over-allocates by orders of magnitude on multi-GiB + // files; start modest and shrink once the real count is known. + let mut id_to_range = HashMap::with_capacity(64 * 1024); let mut stream = - serde_json::Deserializer::from_slice(&buffer).into_iter::(); + serde_json::Deserializer::from_slice(buffer).into_iter::(); while let Some(item) = { let start = stream.byte_offset(); stream.next().map(|item| (start, item)) } { let (start, item) = item; item.map_err(|e| AppError::Parse(format!("books json stream at byte {start}: {e}")))?; - index_record(&buffer, start, stream.byte_offset(), &mut id_to_range); + index_record(buffer, start, stream.byte_offset(), &mut id_to_range); } - - Ok(BooksIndex { - buffer, - id_to_range, - }) + id_to_range.shrink_to_fit(); + Ok(id_to_range) } fn index_record( buffer: &[u8], record_start: usize, record_end: usize, - id_to_range: &mut HashMap, + id_to_range: &mut HashMap, ) { if record_end <= record_start { return; } if let Some(id) = read_id_field(&buffer[record_start..record_end]) { - id_to_range.insert(id, (record_start as u32, record_end as u32)); + id_to_range.insert(id, (record_start as u64, record_end as u64)); } } @@ -530,7 +688,7 @@ mod tests { "#, ); - let books = decompress_and_index(&compressed).expect("index books"); + let books = decompress_and_index(&compressed[..], &[1, 2]).expect("index books"); let raw = read_event(&books, 2).expect("read event"); assert_eq!(raw.get(), r#"[{"symbol":"B"}]"#); @@ -542,7 +700,9 @@ mod tests { br#"{"id":10,"events":[{"bonus":false}]}{"id":11,"events":[{"bonus":true}]}"#, ); - let books = decompress_and_index(&compressed).expect("index books"); + // Adjacent records defeat the line-based fast path; requiring id 11 + // must force the fallback JSON stream scan, which still finds it. + let books = decompress_and_index(&compressed[..], &[10, 11]).expect("index books"); let raw = read_event(&books, 11).expect("read event"); assert_eq!(raw.get(), r#"[{"bonus":true}]"#); From 7f080039fbcb849731cd6507661bc5fe9abc4133 Mon Sep 17 00:00:00 2001 From: McSon2 Date: Mon, 13 Jul 2026 00:33:20 +0200 Subject: [PATCH 2/2] fix(lgs): accept integer or float mode costs in index.json Recent math-sdk publishes emit "cost": 300.0 rather than an integer, which failed the u64 deserializer and broke every endpoint of the game with a 500. Accept both forms and round to the RGS integer cost model. Co-Authored-By: Claude Fable 5 --- crates/lgs/src/types.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/lgs/src/types.rs b/crates/lgs/src/types.rs index ac3b1d5..d57ab1f 100644 --- a/crates/lgs/src/types.rs +++ b/crates/lgs/src/types.rs @@ -55,11 +55,23 @@ pub struct Round { #[derive(Debug, Clone, Deserialize)] pub struct GameMode { pub name: String, + /// Cost multiplier of the mode. math-sdk emits this either as an integer + /// or as a float (`"cost": 300.0`) depending on the generator version, so + /// accept both and round to the RGS's integer cost model. + #[serde(deserialize_with = "de_cost")] pub cost: u64, pub events: String, pub weights: String, } +fn de_cost<'de, D: serde::Deserializer<'de>>(d: D) -> Result { + let raw = f64::deserialize(d)?; + if !raw.is_finite() || raw < 0.0 { + return Err(serde::de::Error::custom(format!("invalid mode cost {raw}"))); + } + Ok((raw.round() as u64).max(1)) +} + #[derive(Debug, Clone, Deserialize)] pub struct GameConfig { pub modes: Vec,