diff --git a/crates/ltk_hashdb/src/layered.rs b/crates/ltk_hashdb/src/layered.rs new file mode 100644 index 0000000..816953c --- /dev/null +++ b/crates/ltk_hashdb/src/layered.rs @@ -0,0 +1,330 @@ +//! An in-memory overlay layered over an ordered list of read-only base tables. + +use std::borrow::Cow; +use std::collections::HashMap; + +use crate::{Casing, HashDb, HashKind, KeyWidth}; + +/// The key configuration a base hashes under. Every base in a [`LayeredHashDb`] +/// must agree on this triple; a base that diverges can never be hit by a caller's +/// precomputed probe (see the type-level invariant). +fn base_config(db: &HashDb) -> (KeyWidth, HashKind, Casing) { + (db.key_width(), db.hash_kind(), db.casing()) +} + +/// A writable in-memory overlay on top of ordered read-only [`HashDb`] bases. +/// +/// Lookups consult the overlay first, then each base in push order; the first hit +/// wins. Base files are never mutated. This generalises [`ExtendedHashDb`] (one +/// base) to the N-base case consumers need when several tables (e.g. League's +/// `game` and `lcu`) sit under one overlay. +/// +/// # Base configuration invariant +/// +/// Lookups take a `u64` the caller already computed, and each base binary-searches +/// its own key set with that raw value - there is no per-base re-hashing. So every +/// base (and any path registered via [`insert_path`](Self::insert_path)) must share +/// the same key configuration: [`key_width`](HashDb::key_width), +/// [`hash_kind`](HashDb::hash_kind), and [`casing`](HashDb::casing). A base that +/// diverges is silently unreachable - the caller's probes were hashed for a +/// different scheme, so they can never match it. [`push_base`](Self::push_base) and +/// [`from_bases`](Self::from_bases) `debug_assert!` this; release builds skip the +/// check. League's `game`/`lcu` tables are uniform (XXH64 / U64 / case-insensitive), +/// so the common path always satisfies it. +/// +/// [`ExtendedHashDb`]: crate::ExtendedHashDb +#[derive(Default)] +pub struct LayeredHashDb { + overlay: HashMap>, + + /// Bases in priority order: earlier ones shadow later ones. + bases: Vec, +} + +impl LayeredHashDb { + /// An empty layered db: no overlay, no bases. Everything resolves to `None`. + pub fn new() -> Self { + Self::default() + } + + /// Layer an empty overlay over `bases`, in the given priority order (`bases[0]` + /// shadows `bases[1]`, and so on). In debug builds, asserts every base shares + /// the first one's key configuration (see the type-level invariant). + pub fn from_bases(bases: Vec) -> Self { + if let Some((first, rest)) = bases.split_first() { + let cfg = base_config(first); + for db in rest { + debug_assert_eq!( + base_config(db), + cfg, + "LayeredHashDb bases must share key config (key_width/hash_kind/casing); \ + a divergent base is unreachable by a caller's precomputed probe" + ); + } + } + + Self { + overlay: HashMap::new(), + bases, + } + } + + /// Append a lower-priority base below all existing ones. + pub fn push_base(&mut self, db: HashDb) { + if let Some(first) = self.bases.first() { + debug_assert_eq!( + base_config(&db), + base_config(first), + "LayeredHashDb bases must share key config (key_width/hash_kind/casing); \ + a divergent base is unreachable by a caller's precomputed probe" + ); + } + + self.bases.push(db); + } + + /// Insert an overlay entry (e.g. a runtime mod hash). Shadows every base. + pub fn insert(&mut self, hash: u64, path: impl Into>) { + self.overlay.insert(hash, path.into()); + } + + /// Bulk-insert overlay entries. + pub fn extend<'a>(&mut self, it: impl IntoIterator) { + self.overlay + .extend(it.into_iter().map(|(k, p)| (k, Box::from(p)))); + } + + /// Hash `path` with the **first** base's algorithm/casing/width, insert it into + /// the overlay, and return the hash - "register this path" without knowing the + /// algorithm. Returns `None` when there are no bases (no algorithm to hash with). + /// + /// Bases with differing key widths make this first-base-wins; callers mixing + /// widths should precompute the hash and use [`insert`](Self::insert) instead. + pub fn insert_path(&mut self, path: &str) -> Option { + let hash = self.bases.first()?.hash_path(path); + self.insert(hash, path); + Some(hash) + } + + /// Overlay first, then each base in push order; the first hit wins. + pub fn get(&self, hash: u64) -> Option> { + if let Some(path) = self.overlay.get(&hash) { + return Some(Cow::Borrowed(&**path)); + } + self.bases.iter().find_map(|base| base.get(hash)) + } + + /// Membership test across the overlay and every base; never touches an arena. + pub fn contains(&self, hash: u64) -> bool { + self.overlay.contains_key(&hash) || self.bases.iter().any(|base| base.contains(hash)) + } + + /// Staged bulk resolve. The overlay is consulted first, then each base's + /// [`get_batch`](HashDb::get_batch) handles only the residual misses, so every + /// base's frames decompress at most once per call. Results are yielded in input + /// order. This is the payoff over calling [`get`](Self::get) N times. + pub fn get_batch<'a>( + &'a self, + hashes: &'a [u64], + ) -> impl Iterator>)> + 'a { + let mut results: Vec>> = Vec::new(); + results.resize_with(hashes.len(), || None); + + // Layer 0: overlay, O(1) per hash. Positions still missing stay in + // `residual`, in input order. + let mut residual: Vec = Vec::new(); + for (i, &h) in hashes.iter().enumerate() { + match self.overlay.get(&h) { + Some(path) => results[i] = Some(Cow::Borrowed(&**path)), + None => residual.push(i), + } + } + + // Layers 1..: each base in push order, on the shrinking residual set. `sub` + // is built in residual order, and `base.get_batch` yields in input order, so + // the zip lines up positionally while the base re-sorts by arena offset + // internally (preserving frame coalescing). + for base in &self.bases { + if residual.is_empty() { + break; + } + let sub: Vec = residual.iter().map(|&i| hashes[i]).collect(); + let mut next: Vec = Vec::new(); + for ((_, opt), &pos) in base.get_batch(&sub).zip(&residual) { + match opt { + Some(path) => results[pos] = Some(path), + None => next.push(pos), + } + } + residual = next; + } + + hashes.iter().copied().zip(results) + } + + /// The base tables, in priority order. + pub fn bases(&self) -> &[HashDb] { + &self.bases + } + + /// Number of overlay entries. + pub fn overlay_len(&self) -> usize { + self.overlay.len() + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::LayeredHashDb; + use crate::{Compression, HashDb, HashDbWriter, KeyWidth}; + + /// Build a raw (uncompressed) U64 table from `(hash, path)` pairs. + fn raw_db(entries: &[(u64, &str)]) -> HashDb { + let mut w = HashDbWriter::new(KeyWidth::U64, Compression::None); + for &(h, p) in entries { + w.insert(h, p); + } + let mut out = Cursor::new(Vec::new()); + w.build(&mut out).expect("build"); + HashDb::open_bytes(out.into_inner()).expect("open") + } + + /// Build a raw table with an explicit key width, for config-mismatch tests. + fn raw_db_width(width: KeyWidth, entries: &[(u64, &str)]) -> HashDb { + let mut w = HashDbWriter::new(width, Compression::None); + for &(h, p) in entries { + w.insert(h, p); + } + let mut out = Cursor::new(Vec::new()); + w.build(&mut out).expect("build"); + HashDb::open_bytes(out.into_inner()).expect("open") + } + + /// Build a compressed U64 table whose 100 clustered paths span several frames. + fn compressed_db(frame_size: u32) -> HashDb { + let mut w = HashDbWriter::new( + KeyWidth::U64, + Compression::Zeekstd { + frame_size, + level: 3, + }, + ); + for i in 0..100u64 { + w.insert( + i * 3, + &format!("assets/characters/champ{i}/skins/skin{i}.bin"), + ); + } + let mut out = Cursor::new(Vec::new()); + w.build(&mut out).expect("build"); + HashDb::open_bytes(out.into_inner()).expect("open") + } + + #[test] + fn layering_order_shadows_lower_layers() { + let base0 = raw_db(&[(1, "base0/one"), (2, "base0/two")]); + let base1 = raw_db(&[(2, "base1/two"), (3, "base1/three")]); + let mut db = LayeredHashDb::from_bases(vec![base0, base1]); + db.insert(1, "overlay/one"); + + // Overlay shadows base 0. + assert_eq!(db.get(1).as_deref(), Some("overlay/one")); + // Base 0 shadows base 1 on a shared key. + assert_eq!(db.get(2).as_deref(), Some("base0/two")); + // Falls through to base 1. + assert_eq!(db.get(3).as_deref(), Some("base1/three")); + // Total miss. + assert_eq!(db.get(999), None); + assert!(!db.contains(999)); + assert!(db.contains(3)); + } + + #[test] + fn get_and_get_batch_agree_in_input_order() { + let base0 = raw_db(&[(10, "base0/ten"), (20, "shadowed")]); + let base1 = raw_db(&[(20, "base1/twenty"), (30, "base1/thirty")]); + let mut db = LayeredHashDb::from_bases(vec![base0, base1]); + db.insert(5, "overlay/five"); + + // Mixed set: overlay hit, base-0 hit, base-1 hit, miss, duplicate. + let probes = [5u64, 10, 20, 30, 999, 10]; + let batch: Vec<_> = db + .get_batch(&probes) + .map(|(h, o)| (h, o.map(|c| c.into_owned()))) + .collect(); + let expected: Vec<_> = probes + .iter() + .map(|&h| (h, db.get(h).map(|c| c.into_owned()))) + .collect(); + assert_eq!(batch, expected); + } + + #[test] + fn get_batch_preserves_frame_coalescing() { + // Small frames so the 100 clustered paths span several frames. + let base = compressed_db(256); + let frames = base.decompressions(); // 0 before any read + assert_eq!(frames, 0); + let db = LayeredHashDb::from_bases(vec![base]); + + // Batch every real key (i*3 for i in 0..100) plus some misses. + let mut probes: Vec = (0..100u64).map(|i| i * 3).collect(); + probes.extend([1, 2, 4, u64::MAX]); + let hits = db.get_batch(&probes).filter(|(_, o)| o.is_some()).count(); + assert_eq!(hits, 100); + + // Coalesced: decompressions track #frames, well below the 100 hits (the + // fixture spans ~20 frames at this size). Per-hit resolution would be ~100. + // No public frame-count accessor here, so assert against the hit count. + let decomps = db.bases()[0].decompressions(); + assert!(decomps > 0, "hits must decompress"); + assert!( + decomps < hits as u64, + "batch decompressed {decomps} times for {hits} clustered hits - coalescing defeated" + ); + } + + #[test] + fn empty_and_no_base() { + let mut db = LayeredHashDb::new(); + assert_eq!(db.get(1), None); + assert_eq!(db.insert_path("some/path"), None); + + // Overlay still works with no bases. + db.insert(42, "manual"); + assert_eq!(db.get(42).as_deref(), Some("manual")); + assert_eq!(db.overlay_len(), 1); + assert!(db.bases().is_empty()); + } + + #[test] + #[should_panic(expected = "must share key config")] + fn push_base_rejects_divergent_key_config() { + let u64_base = raw_db_width(KeyWidth::U64, &[(1, "u64/one")]); + let u32_base = raw_db_width(KeyWidth::U32, &[(2, "u32/two")]); + let mut db = LayeredHashDb::from_bases(vec![u64_base]); + + // Debug-only guard: layering a U32 base under a U64 base is unreachable. + db.push_base(u32_base); + } + + #[test] + #[should_panic(expected = "must share key config")] + fn from_bases_rejects_divergent_key_config() { + let u64_base = raw_db_width(KeyWidth::U64, &[(1, "u64/one")]); + let u32_base = raw_db_width(KeyWidth::U32, &[(2, "u32/two")]); + + let _ = LayeredHashDb::from_bases(vec![u64_base, u32_base]); + } + + #[test] + fn insert_path_uses_first_base() { + let base = raw_db(&[(1, "seed")]); + let mut db = LayeredHashDb::from_bases(vec![base]); + let path = "assets/characters/aatrox/aatrox.bin"; + let hash = db.insert_path(path).expect("has a base"); + assert_eq!(db.get(hash).as_deref(), Some(path)); + } +} diff --git a/crates/ltk_hashdb/src/lib.rs b/crates/ltk_hashdb/src/lib.rs index c019baa..edb17b8 100644 --- a/crates/ltk_hashdb/src/lib.rs +++ b/crates/ltk_hashdb/src/lib.rs @@ -12,6 +12,7 @@ mod error; mod extended; mod hash; mod header; +mod layered; mod reader; mod writer; @@ -19,6 +20,7 @@ pub use error::{BuildError, OpenError, VerifyError}; pub use extended::ExtendedHashDb; pub use hash::{Casing, HashKind}; pub use header::{FORMAT_VERSION, HEADER_SIZE, MAGIC}; +pub use layered::LayeredHashDb; pub use reader::HashDb; pub use writer::{BuildStats, HashDbWriter}; diff --git a/crates/ltk_hashdb/src/reader.rs b/crates/ltk_hashdb/src/reader.rs index 876a995..4f412e3 100644 --- a/crates/ltk_hashdb/src/reader.rs +++ b/crates/ltk_hashdb/src/reader.rs @@ -153,7 +153,7 @@ impl HashDb { /// once (same-directory hashes cluster into the same frames). Yielded in input order. pub fn get_batch<'a>( &'a self, - hashes: &'a [u64], + hashes: &[u64], ) -> impl Iterator>)> + 'a { let indices: Vec> = hashes.iter().map(|&h| self.index_of(h)).collect(); let mut order: Vec = (0..hashes.len()).collect(); @@ -168,7 +168,9 @@ impl HashDb { results[p] = self.str_at_cached(i, &mut cache); } } - hashes.iter().copied().zip(results) + + let out: Vec<(u64, Option>)> = hashes.iter().copied().zip(results).collect(); + out.into_iter() } pub fn len(&self) -> usize { @@ -208,6 +210,12 @@ impl HashDb { self.header.arena_compressed_size } + /// Number of zstd frames this table has decompressed over its lifetime. + #[cfg(test)] + pub(crate) fn decompressions(&self) -> u64 { + self.decompressions.load(Ordering::Relaxed) + } + /// Hash a path string with **this table's** algorithm and casing rule (from /// the `hash_kind` header field - falling back on key width when /// unspecified - and the `case_insensitive` flag). diff --git a/crates/ltk_mimir_cache/src/store.rs b/crates/ltk_mimir_cache/src/store.rs index 1441986..cf8d385 100644 --- a/crates/ltk_mimir_cache/src/store.rs +++ b/crates/ltk_mimir_cache/src/store.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; -use ltk_hashdb::HashDb; +use ltk_hashdb::{HashDb, LayeredHashDb}; use crate::manifest::{Manifest, Source, TableEntry}; use crate::{ @@ -107,6 +107,33 @@ impl HashStore { Ok(HashDb::open(self.path_for(table)?)?) } + /// Open several tables, pairing each with its result so callers can warn-and-skip + /// missing ones instead of aborting on the first error. Results are returned in + /// `tables` order. + pub fn open_many(&self, tables: &[Table]) -> Vec<(Table, Result)> { + tables.iter().map(|&t| (t, self.open(t))).collect() + } + + /// Open `tables`, layer the ones that opened into a [`LayeredHashDb`] (in the + /// given priority order - earlier tables shadow later ones), and return the + /// per-table open errors for the caller to log. + /// + /// A tool stays usable when a table is missing: its hashes just miss. This is + /// the shape most WAD consumers want - e.g. + /// `open_layered(&[Table::Game, Table::Lcu])`. + pub fn open_layered(&self, tables: &[Table]) -> (LayeredHashDb, Vec<(Table, OpenError)>) { + let mut layered = LayeredHashDb::new(); + let mut errors = Vec::new(); + for (table, res) in self.open_many(tables) { + match res { + Ok(db) => layered.push_base(db), + Err(e) => errors.push((table, e)), + } + } + + (layered, errors) + } + /// Try to become the single updater without blocking. `Ok(None)` means another /// process is already updating. Hold the returned guard across /// download/build/[`commit`](HashStore::commit)/[`gc`](HashStore::gc). diff --git a/crates/ltk_mimir_cache/tests/store.rs b/crates/ltk_mimir_cache/tests/store.rs index a73c056..10cb3c3 100644 --- a/crates/ltk_mimir_cache/tests/store.rs +++ b/crates/ltk_mimir_cache/tests/store.rs @@ -120,6 +120,101 @@ fn open_missing_manifest_and_table_error() { )); } +#[test] +fn open_layered_skips_missing_and_respects_order() { + let tmp = tempdir().unwrap(); + let store = HashStore::at(tmp.path()); + + // Two tables sharing a key with different values, plus a unique key each. + let game = &[(0xAAAA_u64, "game/shared"), (0x1111, "game/only")][..]; + let lcu = &[(0xAAAA_u64, "lcu/shared"), (0x2222, "lcu/only")][..]; + store + .commit( + &[ + CommitItem::new( + Table::Game, + "v1", + build_table(&tmp.path().join("g.lhdb"), game), + ), + CommitItem::new( + Table::Lcu, + "v1", + build_table(&tmp.path().join("l.lhdb"), lcu), + ), + ], + None, + ) + .unwrap(); + + // Both present: no errors, each table's unique key resolves, and the earlier + // table shadows the later one on the shared key. + let (db, errors) = store.open_layered(&[Table::Game, Table::Lcu]); + assert!(errors.is_empty()); + assert_eq!(db.get(0x1111).as_deref(), Some("game/only")); + assert_eq!(db.get(0x2222).as_deref(), Some("lcu/only")); + assert_eq!(db.get(0xAAAA).as_deref(), Some("game/shared")); + assert_eq!(db.bases().len(), 2); + + // Reversing the request order flips which table wins the shared key. + let (db, _) = store.open_layered(&[Table::Lcu, Table::Game]); + assert_eq!(db.get(0xAAAA).as_deref(), Some("lcu/shared")); +} + +#[test] +fn open_layered_reports_missing_table_but_stays_usable() { + let tmp = tempdir().unwrap(); + let store = HashStore::at(tmp.path()); + + // Only Game is committed; Lcu is absent. + store + .commit( + &[CommitItem::new( + Table::Game, + "v1", + build_table(&tmp.path().join("g.lhdb"), ENTRIES), + )], + None, + ) + .unwrap(); + + let (db, errors) = store.open_layered(&[Table::Game, Table::Lcu]); + + // The present table still resolves; the missing one is reported, not fatal. + assert!(db.contains(0x1111)); + assert_eq!(db.bases().len(), 1); + assert_eq!(errors.len(), 1); + assert!(matches!( + errors[0], + (Table::Lcu, OpenError::TableNotFound(Table::Lcu)) + )); +} + +#[test] +fn open_many_pairs_each_table_with_its_result() { + let tmp = tempdir().unwrap(); + let store = HashStore::at(tmp.path()); + store + .commit( + &[CommitItem::new( + Table::Game, + "v1", + build_table(&tmp.path().join("g.lhdb"), ENTRIES), + )], + None, + ) + .unwrap(); + + let results = store.open_many(&[Table::Game, Table::Lcu]); + assert_eq!(results.len(), 2); + assert_eq!(results[0].0, Table::Game); + assert!(results[0].1.is_ok()); + assert_eq!(results[1].0, Table::Lcu); + assert!(matches!( + results[1].1, + Err(OpenError::TableNotFound(Table::Lcu)) + )); +} + #[test] fn update_lock_is_exclusive_then_reacquirable() { let tmp = tempdir().unwrap(); diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index 02995b1..69e4ba3 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -163,6 +163,45 @@ The base file is never mutated. `ext.base()` exposes the underlying `HashDb`; not persisted - if you want them shared or durable, contribute them upstream to the CommunityDragon txt lists (the canonical source). +### Layering several base tables under one overlay + +When a workload spans **more than one** table - WAD chunk resolution consults both +`Game` and `Lcu` - reach for `LayeredHashDb` instead of hand-rolling a `Vec` +plus fallback. It is `ExtendedHashDb` generalised to N ordered bases: lookups consult +the overlay first, then each base in push order, first hit wins. + +```rust +use ltk_mimir_cache::{HashStore, Table}; + +let store = HashStore::discover()?; + +// Open the WAD path tables into one layered reader; missing tables are reported, +// not fatal - the tool stays usable and their hashes just miss. +let (mut db, errors) = store.open_layered(&[Table::Game, Table::Lcu]); +for (table, e) in &errors { + eprintln!("skipping {table:?}: {e}"); +} + +db.insert(precomputed_hash, "assets/mymod/custom.bin"); // overlay writes as before + +// Batch-resolve a WAD's chunk hashes: the overlay is checked first, then each base +// handles only the residual misses, so every base's frames decompress at most once. +for (hash, path) in db.get_batch(&chunk_hashes) { + match path { + Some(p) => println!("{p}"), + None => println!("{hash:016x} (unknown)"), + } +} +``` + +`open_layered` is the convenience most WAD consumers want; `open_many` is the +lower-level primitive it's built from - it pairs each requested table with its +`Result` so you can warn-and-skip instead of aborting on the first +missing one. `db.bases()` exposes the opened tables in priority order, `overlay_len()` +counts overlay entries, and `insert_path` hashes with the **first** base's algorithm +(returning `None` when there are no bases). The League-domain hex fallback for a total +miss stays in your tool - mimir returns `Option`, it never invents a hex string. + ## Getting and updating tables Tables are published as GitHub release assets: each release carries every table as an