diff --git a/Cargo.lock b/Cargo.lock index 9067191dff..d8bc30ebb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9276,6 +9276,7 @@ dependencies = [ "rkyv", "rustc_version", "ruvector-core 2.5.1", + "ruvector-e-means", "ruvector-turboquant", "serde", "serde_json", @@ -9538,6 +9539,17 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "ruvector-e-means" +version = "0.1.0" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "ruvector-economy-wasm" version = "0.1.0" @@ -10409,6 +10421,7 @@ name = "ruvector-pq-search" version = "2.3.0" dependencies = [ "rand 0.8.6", + "ruvector-e-means", "serde", "serde_json", "thiserror 2.0.18", @@ -10513,6 +10526,7 @@ version = "0.1.0" dependencies = [ "criterion 0.5.1", "rand 0.8.6", + "ruvector-e-means", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index e57eb682cc..adea40f9c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,8 @@ exclude = ["crates/prime-radiant", "patches/candle-core-0.8.4", "patches/candle- # unify with this workspace. The pure cores it wraps are members below. "edge", "edge/*"] members = [ + # Shared EMA clustering core and portable npm WASM engine (ADR-359). + "crates/ruvector-e-means", # ruvector Python SDK (ADR-352): PyO3 + maturin bindings. Salvaged from # the stale feature/python-sdk-m1 branch (M1 RaBitQ scope) and extended. "crates/ruvector-py", diff --git a/crates/ruvector-core/Cargo.toml b/crates/ruvector-core/Cargo.toml index 92d49a9c73..e1ec5f1267 100644 --- a/crates/ruvector-core/Cargo.toml +++ b/crates/ruvector-core/Cargo.toml @@ -17,6 +17,8 @@ hnsw_rs = { workspace = true, optional = true } # Turbo4 codec + unified encoding plane (ADR-296/297). Dependency-free and # WASM-safe, so it is unconditional: the codec plane exists on every build. ruvector-turboquant = { version = "2.3.0", path = "../ruvector-turboquant" } +# Shared CPU/WASM e-means optimizer, used by the explicit PQ training API. +ruvector-e-means = { version = "=0.1.0", path = "../ruvector-e-means" } simsimd = { workspace = true, optional = true } rayon = { workspace = true, optional = true } crossbeam = { workspace = true, optional = true } diff --git a/crates/ruvector-core/src/quantization.rs b/crates/ruvector-core/src/quantization.rs index 8d3ffed16b..e03c454965 100644 --- a/crates/ruvector-core/src/quantization.rs +++ b/crates/ruvector-core/src/quantization.rs @@ -109,6 +109,83 @@ pub struct ProductQuantized { } impl ProductQuantized { + /// Train product-quantization codebooks with the shared e-means optimizer. + /// + /// `config.clusters` selects the number of entries in each codebook (1–256). + /// Every vector must have the same nonzero dimension, evenly divisible by + /// `num_subspaces`, and contain only finite values. The returned codebooks + /// use the same representation as [`Self::train`] and work with + /// [`Self::encode`]. No input vector is encoded until the caller requests it. + /// + /// This API is explicit because e-means' automatic hyperparameters were + /// fitted on large embedding datasets; callers training small PQ subspaces + /// can set `batch_size` and `peak_step` in the configuration. + pub fn train_e_means( + vectors: &[Vec], + num_subspaces: usize, + config: &ruvector_e_means::Config, + ) -> Result { + use crate::error::RuvectorError; + + let dimensions = vectors.first().map_or(0, Vec::len); + if vectors.is_empty() || dimensions == 0 { + return Err(RuvectorError::InvalidInput( + "e-means PQ training requires nonempty vectors".into(), + )); + } + if num_subspaces == 0 || dimensions % num_subspaces != 0 { + return Err(RuvectorError::InvalidParameter( + "Subspace count must be nonzero and divide vector dimensions".into(), + )); + } + if config.clusters == 0 || config.clusters > 256 || config.clusters > vectors.len() { + return Err(RuvectorError::InvalidParameter( + "Codebook size must be in 1..=256 and no greater than the vector count".into(), + )); + } + for vector in vectors { + if vector.len() != dimensions { + return Err(RuvectorError::DimensionMismatch { + expected: dimensions, + actual: vector.len(), + }); + } + if vector.iter().any(|value| !value.is_finite()) { + return Err(RuvectorError::InvalidInput( + "e-means PQ training vectors must contain only finite values".into(), + )); + } + } + + let subspace_dim = dimensions / num_subspaces; + let capacity = vectors.len().checked_mul(subspace_dim).ok_or_else(|| { + RuvectorError::InvalidInput("e-means PQ training shape overflows usize".into()) + })?; + let mut codebooks = Vec::with_capacity(num_subspaces); + let mut subspace_vectors = Vec::with_capacity(capacity); + for subspace_idx in 0..num_subspaces { + subspace_vectors.clear(); + let start = subspace_idx * subspace_dim; + for vector in vectors { + subspace_vectors.extend_from_slice(&vector[start..start + subspace_dim]); + } + let model = ruvector_e_means::fit(&subspace_vectors, subspace_dim, config) + .map_err(|error| RuvectorError::InvalidParameter(error.to_string()))?; + codebooks.push( + model + .centroids + .chunks_exact(subspace_dim) + .map(<[f32]>::to_vec) + .collect(), + ); + } + + Ok(Self { + codes: Vec::new(), + codebooks, + }) + } + /// Train product quantization on a set of vectors pub fn train( vectors: &[Vec], @@ -168,16 +245,28 @@ impl ProductQuantized { let start = subspace_idx * subspace_dim; let end = start + subspace_dim; let subvector = &vector[start..end]; + // Keep the existing f32 distance, so codes for finite distances are + // unchanged. Only a candidate whose f32 squared distance overflows + // is re-measured in f64, keeping its nearest ordering. + let squared_distance = |centroid: &[f32]| -> f64 { + let fast = euclidean_squared(subvector, centroid); + if fast.is_finite() { + f64::from(fast) + } else { + subvector + .iter() + .zip(centroid) + .map(|(&x, &y)| (f64::from(x) - f64::from(y)).powi(2)) + .sum() + } + }; // Find nearest centroid let code = codebook .iter() .enumerate() - .min_by(|(_, a), (_, b)| { - let dist_a = euclidean_squared(subvector, a); - let dist_b = euclidean_squared(subvector, b); - dist_a.partial_cmp(&dist_b).unwrap() - }) + .map(|(idx, centroid)| (idx, squared_distance(centroid))) + .min_by(|(_, a), (_, b)| a.total_cmp(b)) .map(|(idx, _)| idx as u8) .unwrap_or(0); diff --git a/crates/ruvector-core/tests/e_means_quantization.rs b/crates/ruvector-core/tests/e_means_quantization.rs new file mode 100644 index 0000000000..83a0ce75e0 --- /dev/null +++ b/crates/ruvector-core/tests/e_means_quantization.rs @@ -0,0 +1,144 @@ +//! Integration evidence for the shared e-means PQ training path. + +use ruvector_core::error::RuvectorError; +use ruvector_core::ProductQuantized; +use ruvector_e_means::Config; + +fn config(clusters: usize) -> Config { + Config { + clusters, + epochs: 3, + seed: 17, + ..Config::default() + } +} + +#[test] +fn subspace_codebooks_recover_column_means_and_keep_the_existing_format() { + let vectors = vec![ + vec![1.0, 2.0, 10.0, 20.0], + vec![3.0, 4.0, 30.0, 40.0], + vec![5.0, 6.0, 50.0, 60.0], + ]; + let trained = ProductQuantized::train_e_means(&vectors, 2, &config(1)).unwrap(); + assert!(trained.codes.is_empty()); + assert_eq!( + trained.codebooks, + vec![vec![vec![3.0, 4.0]], vec![vec![30.0, 40.0]]] + ); + assert_eq!(trained.encode(&vectors[0]), vec![0, 0]); + + // Existing persisted PQ consumers use the same serde representation and + // reconstruct a vector by looking up each subspace's encoded centroid. + let json = serde_json::to_vec(&trained).unwrap(); + let restored: ProductQuantized = serde_json::from_slice(&json).unwrap(); + let codes = restored.encode(&vectors[2]); + let reconstructed: Vec = codes + .iter() + .enumerate() + .flat_map(|(subspace, &code)| restored.codebooks[subspace][code as usize].clone()) + .collect(); + assert_eq!(reconstructed, vec![3.0, 4.0, 30.0, 40.0]); +} + +#[test] +fn identical_seed_and_options_produce_identical_codebooks() { + let vectors: Vec> = (0..80) + .map(|i| vec![i as f32 / 10.0, (i % 7) as f32, -(i as f32), 2.0]) + .collect(); + let mut options = config(4); + options.batch_size = Some(7); // Exercise the final short batch. + let first = ProductQuantized::train_e_means(&vectors, 2, &options).unwrap(); + let second = ProductQuantized::train_e_means(&vectors, 2, &options).unwrap(); + assert_eq!(first.codebooks, second.codebooks); + assert_eq!(first.codebooks.len(), 2); + assert!(first.codebooks.iter().all(|book| book.len() == 4)); + assert!(first + .codebooks + .iter() + .flatten() + .all(|centroid| centroid.len() == 2 && centroid.iter().all(|v| v.is_finite()))); +} + +#[test] +fn maximum_u8_codebook_size_is_usable() { + let vectors: Vec> = (0..256).map(|i| vec![i as f32]).collect(); + let mut options = config(256); + options.epochs = 1; + options.batch_size = Some(256); + let trained = ProductQuantized::train_e_means(&vectors, 1, &options).unwrap(); + assert_eq!(trained.codebooks[0].len(), 256); + let codes: std::collections::BTreeSet = + vectors.iter().map(|v| trained.encode(v)[0]).collect(); + assert_eq!(codes.len(), 256); +} + +#[test] +fn large_finite_inputs_keep_the_nearest_centroid_when_squared_f32_distances_overflow() { + let magnitude = f32::MAX / 4.0; + let vectors = vec![vec![-magnitude], vec![magnitude]]; + let mut options = config(2); + options.epochs = 1; + options.batch_size = Some(2); + let trained = ProductQuantized::train_e_means(&vectors, 1, &options).unwrap(); + // Pick a query past the second centroid. Both candidate squared distances + // overflow f32, but the second remains strictly nearer in f64. + let query = trained.codebooks[0][1][0] * 2.0; + assert!(query.is_finite()); + for centroid in &trained.codebooks[0] { + assert!((query - centroid[0]).powi(2).is_infinite()); + } + assert_eq!(trained.encode(&[query]), vec![1]); +} + +#[test] +fn malformed_training_inputs_return_errors() { + let valid = vec![vec![1.0, 2.0, 3.0, 4.0], vec![2.0, 3.0, 4.0, 5.0]]; + for (vectors, subspaces) in [ + (Vec::new(), 1), + (vec![Vec::new()], 1), + (valid.clone(), 0), + (valid.clone(), 3), + (valid.clone(), 8), + (vec![valid[0].clone(), vec![1.0]], 2), + (vec![vec![f32::NAN, 0.0]], 1), + (vec![vec![f32::INFINITY, 0.0]], 1), + ] { + assert!(ProductQuantized::train_e_means(&vectors, subspaces, &config(1)).is_err()); + } + let error = + ProductQuantized::train_e_means(&[vec![1.0, 2.0], vec![1.0]], 1, &config(1)).unwrap_err(); + assert!(matches!(error, RuvectorError::DimensionMismatch { .. })); +} + +#[test] +fn invalid_optimizer_settings_return_errors() { + let vectors = vec![vec![1.0, 2.0], vec![3.0, 4.0]]; + for clusters in [0, 3, 257] { + assert!(ProductQuantized::train_e_means(&vectors, 1, &config(clusters)).is_err()); + } + let mut options = config(1); + options.epochs = 0; + assert!(ProductQuantized::train_e_means(&vectors, 1, &options).is_err()); + options = config(1); + options.batch_size = Some(0); + assert!(ProductQuantized::train_e_means(&vectors, 1, &options).is_err()); + options = config(1); + options.peak_step = Some(f64::NAN); + assert!(ProductQuantized::train_e_means(&vectors, 1, &options).is_err()); +} + +#[test] +fn finite_distance_codes_keep_the_existing_f32_ordering() { + // These rows tie in f32 but not in f64. Existing encode picks the first + // f32 minimum, and codes for already-trained indexes must not change. + let bits = |v: [u32; 4]| v.map(f32::from_bits).to_vec(); + let query = bits([1059241166, 3208695764, 3205419074, 1055842996]); + let first = bits([1063676912, 3210863684, 1060297430, 3212197722]); + let second = bits([1052646232, 3210863675, 1060297430, 3212197722]); + let quantized = ProductQuantized { + codes: Vec::new(), + codebooks: vec![vec![first, second]], + }; + assert_eq!(quantized.encode(&query), vec![0]); +} diff --git a/crates/ruvector-e-means/Cargo.toml b/crates/ruvector-e-means/Cargo.toml new file mode 100644 index 0000000000..9b457c4e2a --- /dev/null +++ b/crates/ruvector-e-means/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "ruvector-e-means" +version = "0.1.0" +edition = "2021" +rust-version = "1.81" +license = "MIT" +description = "Deterministic CPU and WebAssembly e-means clustering with EMA centroid updates" +repository = "https://github.com/ruvnet/ruvector" +homepage = "https://ruv.io" +documentation = "https://docs.rs/ruvector-e-means" +keywords = ["clustering", "k-means", "ema", "quantization", "machine-learning"] +categories = ["algorithms", "science"] +include = ["src/**", "tests/**", "examples/**", "Cargo.toml", "LICENSE", "README.md"] + +[lib] +crate-type = ["rlib", "cdylib"] + +[features] +default = [] +serde = ["dep:serde"] +wasm = ["serde", "dep:serde_json", "dep:wasm-bindgen"] + +[dependencies] +rand_chacha = "=0.3.1" +rand_core = "=0.6.4" +serde = { version = "1.0", features = ["derive"], optional = true } +serde_json = { version = "1.0", optional = true } +wasm-bindgen = { version = "=0.2.125", optional = true } + +[dev-dependencies] +serde_json = "1.0" diff --git a/crates/ruvector-e-means/LICENSE b/crates/ruvector-e-means/LICENSE new file mode 100644 index 0000000000..2dd524ac38 --- /dev/null +++ b/crates/ruvector-e-means/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 rUv + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/ruvector-e-means/README.md b/crates/ruvector-e-means/README.md new file mode 100644 index 0000000000..092c9b8f3f --- /dev/null +++ b/crates/ruvector-e-means/README.md @@ -0,0 +1,34 @@ +# ruvector-e-means + +![e-means training loop: rows are assigned in mini-batches, EMA counts and sums move the centroids with a cosine-decaying step, a dead centroid is revived by splitting a busy donor, and the last epoch mean gives the codebook](https://raw.githubusercontent.com/ruvnet/ruvector/main/npm/packages/e-means/assets/e-means-flow.svg) + +CPU reference implementation of [e-means (Bessa et al., 2026)](https://arxiv.org/abs/2611.09501), used by RuVector's PQ and IVF crates. MIT-licensed original Rust implementation; no upstream implementation code is copied. + +```rust +use ruvector_e_means::{fit, Config}; +let data = [1.0f32, 2.0, 3.0, 4.0]; // two 2-dimensional rows +let config = Config { clusters: 1, ..Config::default() }; +let model = fit(&data, 2, &config)?; +assert_eq!(model.centroids, [2.0, 3.0]); +# Ok::<(), ruvector_e_means::Error>(()) +``` + +The implementation includes seeded random initialization, EMA counts/sums initialized at 1e-10, cosine scheduling, automatic batch/step scaling, weighted donor splitting every five updates, and final-epoch mean or main EMA finalization. Prediction and returned MSE use exact squared-L2 distances in original input order. All rows, including a partial final batch, are visited each epoch. Scratch memory is O(n + kd). Accumulation and distance arithmetic use f64; codebooks use f32. Extreme revival perturbations saturate to finite f32. Dead slots outnumbering available live donors wait for the next revival pass. + +Assignment prunes candidates once their partial squared distance cannot beat the current best, checking every eight coordinates above 16 dimensions. Small vectors retain the scalar path. A zero-distance winner stops the search. Ordered f64 addition and lowest-index ties are preserved. Training reuses revival scratch and omits last-epoch buffers in `Ema` mode. The npm bridge returns owned typed arrays and frees its temporary Rust model after materialization. + +Scaling laws were fitted on large, high-dimensional embeddings; override `batch_size` and `peak_step` for other distributions. Last-epoch means average assignments observed during the last epoch, when centroids were moving; this is not a separate Lloyd pass. Empty final clusters retain their EMA centroid. A final exact pass recomputes assignments and MSE against the returned codebook. `mse` is the mean squared L2 distance per row, not per coordinate. + +This version does not implement GPU/low-precision kernels, distributed or approximate IVF assignment, or slow-EMA finalization. It makes no claim to reproduce the paper's GPU speedups or universally outperform Lloyd. Existing consumer training defaults remain available. + +Native Rust has no default features. `serde` adds config/model serialization; `wasm` adds the wasm-bindgen bridge used by `@ruvector/e-means`. Native core MSRV is Rust 1.81; WASM builds use the toolchain required by the pinned wasm-bindgen dependency. See [ADR-359](https://github.com/ruvnet/ruvector/blob/main/docs/adr/ADR-359-e-means-shared-clustering.md) for integration, validation, and staged release instructions. + +Run the deterministic native benchmark with `cargo run -p ruvector-e-means --release --example benchmark -- --case all --repeats 5 --seed 42 --finalization both`. Dataset generation and warmup are excluded from median fit/predict times. The [local measurements](https://github.com/ruvnet/ruvector/blob/main/docs/research/e-means/performance-2026-11-11.json) include baseline comparisons, allocation counts and unchanged-output checks; gains depend on shape and distribution. + +Reusable `quantization::ProductQuantizer` and `ivf::IvfPartition` modules add +checked byte-code encode/decode, subspace-major codebooks, and stable inverted +postings. Their private shape fields and serde restoration reject malformed +persisted state. `ruvector-pq-search` and the IVF/RAIRS/SEIL family expose explicit +e-means training using this dependency. See [ADR-360](https://github.com/ruvnet/ruvector/blob/main/docs/adr/ADR-360-e-means-ecosystem-integrations.md) +for the native/npm integration matrix, MetaHarness evaluation, and Cloudflare +static-WASM entrypoints. These additions preserve existing training defaults. diff --git a/crates/ruvector-e-means/examples/benchmark.rs b/crates/ruvector-e-means/examples/benchmark.rs new file mode 100644 index 0000000000..5112f8032d --- /dev/null +++ b/crates/ruvector-e-means/examples/benchmark.rs @@ -0,0 +1,177 @@ +//! Portable native fit/predict benchmark. No extra dependencies or CPU flags. +//! +//! cargo run -p ruvector-e-means --release --example benchmark -- +//! --case all --repeats 5 --seed 42 --finalization both +//! +//! Dataset creation and warmup are excluded from reported median timings. + +use ruvector_e_means::{fit, Config, Finalization}; +use std::{hint::black_box, time::Instant}; + +#[derive(Clone, Copy)] +pub struct Case { + pub name: &'static str, + pub samples: usize, + pub clusters: usize, + pub dimensions: usize, +} + +pub const CASES: [Case; 4] = [ + Case { + name: "embedding128", + samples: 4096, + clusters: 64, + dimensions: 128, + }, + Case { + name: "embedding768", + samples: 2048, + clusters: 32, + dimensions: 768, + }, + Case { + name: "pq8", + samples: 8192, + clusters: 256, + dimensions: 8, + }, + Case { + name: "revival", + samples: 2048, + clusters: 64, + dimensions: 32, + }, +]; + +/// Deterministic synthetic inputs, not a reproduction of the paper's datasets. +/// Cases cover clustered and diffuse normalized embeddings, diffuse low-D +/// codebook training, and duplicate/zero rows that exercise cluster revival. +pub fn generate(case: Case, seed: u64) -> Vec { + let mut state = seed; + let mut random = || { + state = state.wrapping_add(0x9e3779b97f4a7c15); + let mut value = state; + value = (value ^ (value >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d049bb133111eb); + value ^= value >> 31; + ((value >> 40) as f32 / 16_777_216.0) * 2.0 - 1.0 + }; + let centers: Vec = (0..case.clusters * case.dimensions) + .map(|_| random()) + .collect(); + let mut data = Vec::with_capacity(case.samples * case.dimensions); + for i in 0..case.samples { + for d in 0..case.dimensions { + let value = match case.name { + "embedding128" => { + centers[(i % case.clusters) * case.dimensions + d] + random() * 0.1 + } + "revival" if i % 5 == 0 => 0.0, + "revival" => ((i % 4) as f32 - 1.5) * if d % 2 == 0 { 1.0 } else { -1.0 }, + _ => random(), + }; + data.push(value); + } + if case.name.starts_with("embedding") { + let row = &mut data[i * case.dimensions..(i + 1) * case.dimensions]; + let norm = row + .iter() + .map(|&v| f64::from(v).powi(2)) + .sum::() + .sqrt(); + for value in row { + *value = (f64::from(*value) / norm) as f32; + } + } + } + data +} + +pub fn digest(centroids: &[f32], assignments: &[usize]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for bits in centroids + .iter() + .map(|v| u64::from(v.to_bits())) + .chain(assignments.iter().map(|&v| v as u64)) + { + for byte in bits.to_le_bytes() { + hash = (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3); + } + } + hash +} + +pub fn median_ms(samples: &mut [f64]) -> f64 { + samples.sort_by(f64::total_cmp); + let middle = samples.len() / 2; + if samples.len() % 2 == 0 { + (samples[middle - 1] + samples[middle]) / 2.0 + } else { + samples[middle] + } +} + +fn main() -> Result<(), Box> { + let mut case_name = String::from("all"); + let mut repeats = 5usize; + let mut seed = 42u64; + let mut finalization = String::from("both"); + let mut args = std::env::args().skip(1); + while let Some(flag) = args.next() { + if flag == "--help" { + println!("benchmark [--case all|embedding128|embedding768|pq8|revival] [--repeats 1..99] [--seed u64] [--finalization both|lastEpochMean|ema]"); + return Ok(()); + } + let value = args.next().ok_or("every option requires a value")?; + match flag.as_str() { + "--case" => case_name = value, + "--repeats" => repeats = value.parse()?, + "--seed" => seed = value.parse()?, + "--finalization" => finalization = value, + _ => return Err(format!("unknown option: {flag}").into()), + } + } + if !(1..=99).contains(&repeats) + || !["both", "lastEpochMean", "ema"].contains(&finalization.as_str()) + { + return Err("invalid repeats or finalization".into()); + } + let cases: Vec = CASES + .into_iter() + .filter(|case| case_name == "all" || case.name == case_name) + .collect(); + if cases.is_empty() { + return Err("unknown benchmark case".into()); + } + for case in cases { + let data = generate(case, seed); + for (label, mode) in [ + ("lastEpochMean", Finalization::LastEpochMean), + ("ema", Finalization::Ema), + ] { + if finalization != "both" && finalization != label { + continue; + } + let config = Config { + clusters: case.clusters, + seed, + finalization: mode, + ..Config::default() + }; + let model = fit(black_box(&data), case.dimensions, &config)?; + black_box(model.predict(black_box(&data))?); + let mut fit_ms = Vec::with_capacity(repeats); + let mut predict_ms = Vec::with_capacity(repeats); + for _ in 0..repeats { + let start = Instant::now(); + black_box(fit(black_box(&data), case.dimensions, black_box(&config))?); + fit_ms.push(start.elapsed().as_secs_f64() * 1000.0); + let start = Instant::now(); + black_box(model.predict(black_box(&data))?); + predict_ms.push(start.elapsed().as_secs_f64() * 1000.0); + } + println!("{{\"case\":\"{}\",\"samples\":{},\"clusters\":{},\"dimensions\":{},\"seed\":{},\"epochs\":{},\"repeats\":{},\"finalization\":\"{}\",\"fitMedianMs\":{:.6},\"predictMedianMs\":{:.6},\"mse\":{:.12},\"revivedClusters\":{},\"digest\":\"{:016x}\"}}", case.name, case.samples, case.clusters, case.dimensions, seed, config.epochs, repeats, label, median_ms(&mut fit_ms), median_ms(&mut predict_ms), model.mse, model.revived_clusters, digest(&model.centroids, &model.assignments)); + } + } + Ok(()) +} diff --git a/crates/ruvector-e-means/src/config.rs b/crates/ruvector-e-means/src/config.rs new file mode 100644 index 0000000000..9e63d6a575 --- /dev/null +++ b/crates/ruvector-e-means/src/config.rs @@ -0,0 +1,115 @@ +use crate::Error; + +/// Final centroid estimator. Slow-EMA finalization is reserved for a future API. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub enum Finalization { + #[default] + LastEpochMean, + Ema, +} + +/// Fit options. Overrides are useful for tiny or low-dimensional datasets where +/// the paper's empirically fitted scaling laws are outside their training range. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(rename_all = "camelCase", deny_unknown_fields) +)] +pub struct Config { + pub clusters: usize, + #[cfg_attr(feature = "serde", serde(default = "default_epochs"))] + pub epochs: usize, + #[cfg_attr(feature = "serde", serde(default))] + pub batch_size: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub peak_step: Option, + #[cfg_attr(feature = "serde", serde(default = "default_seed"))] + pub seed: u64, + #[cfg_attr(feature = "serde", serde(default))] + pub finalization: Finalization, +} + +fn default_epochs() -> usize { + 3 +} + +fn default_seed() -> u64 { + 42 +} + +impl Default for Config { + fn default() -> Self { + Self { + clusters: 8, + epochs: default_epochs(), + batch_size: None, + peak_step: None, + seed: default_seed(), + finalization: Finalization::LastEpochMean, + } + } +} + +/// Resolved scaling-law parameters, inspectable before allocating a model. +#[derive(Debug, Clone, Copy)] +pub struct Parameters { + pub batch_size: usize, + pub peak_step: f64, + pub updates: usize, +} + +impl Config { + /// Table 2 and equations (6)-(7) of arXiv:2611.09501v1. Batch law uses + /// coefficient 0.035, divided by 8 above two million samples, rounded to + /// the nearest power of two and capped at n. Tail batches are included. + pub fn parameters(&self, samples: usize) -> Result { + if samples == 0 || self.clusters == 0 || self.clusters > samples { + return Err(Error("clusters must be in 1..=number of samples")); + } + if self.epochs == 0 { + return Err(Error("epochs must be positive")); + } + let n = samples as f64; + let k = self.clusters as f64; + let t = self.epochs as f64; + let batch_size = match self.batch_size { + Some(0) => return Err(Error("batch size must be positive")), + Some(b) => b.min(samples), + None => { + let coefficient = if samples > 2_000_000 { + 0.035 / 8.0 + } else { + 0.035 + }; + let law = coefficient * k.powf(0.13) * n.powf(0.88) * t.powf(0.81); + 2.0_f64.powf(law.max(1.0).log2().round()).min(n) as usize + } + }; + let peak_step = self.peak_step.unwrap_or_else(|| { + (0.824 * k.powf(-0.46) * (batch_size as f64).powf(0.90) * n.powf(-0.45) * t.powf(-0.11)) + .min(1.0) + }); + if !peak_step.is_finite() || peak_step <= 0.0 || peak_step > 1.0 { + return Err(Error("peak step must be finite and in (0, 1]")); + } + let batches = samples / batch_size + usize::from(samples % batch_size != 0); + let updates = batches + .checked_mul(self.epochs) + .ok_or(Error("update count overflow"))?; + Ok(Parameters { + batch_size, + peak_step, + updates, + }) + } +} + +/// Equation (5), zero-based batch index t and total M updates. Its denominator +/// is M as printed in the paper: the last update approaches the 0.1 floor. +pub fn cosine_step(peak_step: f64, update: usize, total_updates: usize) -> f64 { + let fraction = (update as f64) / (total_updates.max(1) as f64); + peak_step * (0.1 + 0.9 * 0.5 * (1.0 + (std::f64::consts::PI * fraction).cos())) +} diff --git a/crates/ruvector-e-means/src/distance.rs b/crates/ruvector-e-means/src/distance.rs new file mode 100644 index 0000000000..03b9753679 --- /dev/null +++ b/crates/ruvector-e-means/src/distance.rs @@ -0,0 +1,189 @@ +//! Exact squared-L2 assignment with bounded work on losing candidates. + +/// Inputs have already passed the public API's shape and finiteness checks. +/// All squared terms are nonnegative: once a partial distance reaches the +/// best distance, that later centroid cannot win, including in a tie. +pub(crate) fn nearest(row: &[f32], centroids: &[f32], dimensions: usize) -> (usize, f64) { + // End-to-end diffuse PQ benchmarks favor the original scalar path at small + // dimensions: pruning checks cost more than the arithmetic they can skip. + if dimensions <= 16 { + exhaustive(row, centroids, dimensions) + } else { + pruned::<8>(row, centroids, dimensions) + } +} + +fn exhaustive(row: &[f32], centroids: &[f32], dimensions: usize) -> (usize, f64) { + let mut best = (0, f64::INFINITY); + for (j, centroid) in centroids.chunks_exact(dimensions).enumerate() { + let distance: f64 = row + .iter() + .zip(centroid) + .map(|(&x, &y)| (f64::from(x) - f64::from(y)).powi(2)) + .sum(); + if distance < best.1 { + best = (j, distance); + if distance == 0.0 { + break; + } + } + } + best +} + +fn pruned(row: &[f32], centroids: &[f32], dimensions: usize) -> (usize, f64) { + let mut best = (0, f64::INFINITY); + for (j, centroid) in centroids.chunks_exact(dimensions).enumerate() { + let mut distance = 0.0; + for (xs, ys) in row.chunks(BLOCK).zip(centroid.chunks(BLOCK)) { + // Retain left-to-right f64 addition. Reassociating these terms, or + // expanding norms and dot products, can change near-tie winners. + for (&x, &y) in xs.iter().zip(ys) { + let difference = f64::from(x) - f64::from(y); + distance += difference * difference; + } + if distance >= best.1 { + break; + } + } + if distance < best.1 { + best = (j, distance); + if distance == 0.0 { + break; + } + } + } + best +} + +#[cfg(test)] +mod tests { + use super::nearest; + + /// The pre-optimization exhaustive implementation is an independent oracle. + fn oracle(row: &[f32], centroids: &[f32], dimensions: usize) -> (usize, f64) { + let mut best = (0, f64::INFINITY); + for (j, centroid) in centroids.chunks_exact(dimensions).enumerate() { + let distance: f64 = row + .iter() + .zip(centroid) + .map(|(&x, &y)| (f64::from(x) - f64::from(y)).powi(2)) + .sum(); + if distance < best.1 { + best = (j, distance); + } + } + best + } + + fn compare(row: &[f32], centroids: &[f32]) { + let expected = oracle(row, centroids, row.len()); + let actual = nearest(row, centroids, row.len()); + assert_eq!(actual.0, expected.0, "{} dimensions", row.len()); + assert_eq!(actual.1.to_bits(), expected.1.to_bits()); + } + + fn random(state: &mut u64) -> f32 { + *state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + ((*state >> 40) as f32 / 16777216.0) * 2.0 - 1.0 + } + + #[test] + fn assignment_matches_exhaustive_on_diffuse_and_separated_data() { + let mut state = 42; + for dimensions in [ + 1, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 127, 128, 129, 384, 385, 768, 769, + ] { + for clusters in [1, 2, 7, 32] { + for separated in [false, true] { + let centroids = (0..clusters * dimensions) + .map(|i| { + random(&mut state) + + if separated { + (i / dimensions) as f32 * 4.0 + } else { + 0.0 + } + }) + .collect::>(); + for i in 0..13 { + let row = (0..dimensions) + .map(|d| { + if separated { + centroids[(i % clusters) * dimensions + d] + + random(&mut state) * 0.01 + } else { + random(&mut state) + } + }) + .collect::>(); + compare(&row, ¢roids); + } + } + } + } + } + + #[test] + fn ties_and_zero_distances_keep_the_lowest_index() { + for dimensions in [1, 3, 8, 16, 32, 128, 384, 768] { + let row = vec![0.0; dimensions]; + let mut centroids = vec![1.0; dimensions]; + centroids.extend(vec![-1.0; dimensions]); + compare(&row, ¢roids); + assert_eq!(nearest(&row, ¢roids, dimensions).0, 0); + centroids.extend(vec![-0.0; dimensions]); + centroids.extend(vec![0.0; dimensions]); + compare(&row, ¢roids); + assert_eq!(nearest(&row, ¢roids, dimensions).0, 2); + } + } + + #[test] + fn extreme_finite_coordinates_and_subnormals_preserve_exact_distance() { + let values = [ + f32::MAX, + -f32::MAX, + f32::MIN_POSITIVE, + f32::from_bits(1), + -f32::from_bits(1), + 0.0, + -0.0, + 1.0, + ]; + for dimensions in [1, 3, 8, 16, 32, 128, 384, 768] { + for offset in 0..values.len() { + let row = (0..dimensions) + .map(|d| values[(d + offset) % values.len()]) + .collect::>(); + let centroids = (0..7 * dimensions) + .map(|d| values[(d * 3 + offset + 1) % values.len()]) + .collect::>(); + compare(&row, ¢roids); + } + } + } + + #[test] + fn near_ties_and_awkward_tail_dimensions_preserve_ordered_addition() { + for dimensions in [3, 7, 8, 9, 15, 16, 17, 31, 32, 33, 127, 129, 385, 769] { + let row = vec![0.0; dimensions]; + let mut first = vec![1.0; dimensions]; + let mut second = first.clone(); + first[dimensions - 1] = f32::from_bits(1.0_f32.to_bits() + 1); + let mut centroids = first.clone(); + centroids.extend_from_slice(&second); + compare(&row, ¢roids); + assert_eq!(nearest(&row, ¢roids, dimensions).0, 1); + + // Tiny trailing contributions disappear when added after a huge + // leading square; summation reassociation must not break this tie. + first[0] = 1e20; + second[0] = 1e20; + let mut centroids = first; + centroids.extend(second); + compare(&row, ¢roids); + assert_eq!(nearest(&row, ¢roids, dimensions).0, 0); + } + } +} diff --git a/crates/ruvector-e-means/src/ivf.rs b/crates/ruvector-e-means/src/ivf.rs new file mode 100644 index 0000000000..136f78d200 --- /dev/null +++ b/crates/ruvector-e-means/src/ivf.rs @@ -0,0 +1,155 @@ +//! Stable inverted-file postings over a trained e-means centroid model. + +use crate::{fit, validate_rows, Config, Error, Model}; + +/// A centroid model plus compact, deterministic inverted lists of input row IDs. +/// Posting `j` occupies `indices[offsets[j]..offsets[j + 1]]`. Row IDs within +/// every list are ascending and every training row occurs exactly once. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct IvfPartition { + model: Model, + offsets: Vec, + indices: Vec, +} + +impl IvfPartition { + /// Train centroids and build postings from the final nearest assignments. + pub fn fit(data: &[f32], dimensions: usize, config: &Config) -> Result { + Self::from_model(fit(data, dimensions, config)?) + } + + /// Build postings for an existing, validated fitted model. + pub fn from_model(model: Model) -> Result { + let clusters = validate_model(&model)?; + let mut offsets = vec![0usize; clusters + 1]; + for &assignment in &model.assignments { + offsets[assignment + 1] += 1; + } + for i in 1..offsets.len() { + offsets[i] += offsets[i - 1]; + } + let mut cursors = offsets[..clusters].to_vec(); + let mut indices = vec![0usize; model.assignments.len()]; + for (row, &assignment) in model.assignments.iter().enumerate() { + indices[cursors[assignment]] = row; + cursors[assignment] += 1; + } + Ok(Self { + model, + offsets, + indices, + }) + } + + /// Restore a persisted partition, checking offsets, labels, and row IDs. + /// Repeated, omitted, reordered, or incorrectly assigned row IDs reject. + pub fn from_parts( + model: Model, + offsets: Vec, + indices: Vec, + ) -> Result { + let clusters = validate_model(&model)?; + let samples = model.assignments.len(); + if offsets.len() != clusters + 1 + || offsets.first() != Some(&0) + || offsets.last() != Some(&samples) + || indices.len() != samples + { + return Err(Error( + "IVF offsets and indices must cover every training row", + )); + } + if offsets + .windows(2) + .any(|pair| pair[0] > pair[1] || pair[1] > samples) + { + return Err(Error( + "IVF offsets must be monotonic and within the row count", + )); + } + for cluster in 0..clusters { + let posting = &indices[offsets[cluster]..offsets[cluster + 1]]; + if posting.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(Error("IVF posting row IDs must be strictly ascending")); + } + for &row in posting { + if row >= samples || model.assignments[row] != cluster { + return Err(Error("IVF row ID is out of range or in the wrong posting")); + } + } + } + Ok(Self { + model, + offsets, + indices, + }) + } + + /// Trained centroids, final labels, and training metrics. + pub fn model(&self) -> &Model { + &self.model + } + + /// Exclusive posting boundaries, of length `clusters + 1`. + pub fn offsets(&self) -> &[usize] { + &self.offsets + } + + /// Input row IDs grouped by cluster, stable within each cluster. + pub fn indices(&self) -> &[usize] { + &self.indices + } + + /// Borrow one inverted list, returning an error for an invalid cluster. + pub fn posting(&self, cluster: usize) -> Result<&[usize], Error> { + if cluster >= self.offsets.len() - 1 { + return Err(Error("IVF cluster index is out of range")); + } + Ok(&self.indices[self.offsets[cluster]..self.offsets[cluster + 1]]) + } + + /// Consume this partition without copying its trained centroid model. + pub fn into_model(self) -> Model { + self.model + } +} + +fn validate_model(model: &Model) -> Result { + validate_rows(&model.centroids, model.dimensions)?; + let clusters = model.centroids.len() / model.dimensions; + let samples = model.assignments.len(); + if clusters == 0 || samples < clusters || model.assignments.iter().any(|&j| j >= clusters) { + return Err(Error( + "IVF model must have valid nonempty centroids and assignments", + )); + } + if !model.mse.is_finite() + || model.mse < 0.0 + || !model.peak_step.is_finite() + || model.peak_step <= 0.0 + || model.peak_step > 1.0 + || model.batch_size == 0 + || model.batch_size > samples + || model.updates == 0 + { + return Err(Error("IVF model has invalid fitted training metrics")); + } + Ok(clusters) +} + +#[cfg(feature = "serde")] +impl<'de> serde::Deserialize<'de> for IvfPartition { + fn deserialize>(deserializer: D) -> Result { + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase", deny_unknown_fields)] + struct Wire { + model: Model, + offsets: Vec, + indices: Vec, + } + let wire = ::deserialize(deserializer)?; + Self::from_parts(wire.model, wire.offsets, wire.indices).map_err(serde::de::Error::custom) + } +} diff --git a/crates/ruvector-e-means/src/lib.rs b/crates/ruvector-e-means/src/lib.rs new file mode 100644 index 0000000000..959539093d --- /dev/null +++ b/crates/ruvector-e-means/src/lib.rs @@ -0,0 +1,81 @@ +//! e-means CPU reference implementation of Bessa et al., arXiv:2611.09501v1. +//! +//! Uses exact squared-L2 assignment, f64 sufficient statistics and seeded +//! ChaCha8 shuffles. This does not implement the paper's GPU or approximate IVF +//! kernels. See [`fit`] and [`Config`]. + +mod config; +mod distance; +pub mod ivf; +pub mod quantization; +mod trainer; +#[cfg(feature = "wasm")] +mod wasm; + +pub use config::{cosine_step, Config, Finalization, Parameters}; +pub(crate) use distance::nearest; +pub use ivf::IvfPartition; +pub use quantization::ProductQuantizer; +pub use trainer::fit; + +/// Validation failure; invalid input is rejected before training starts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(pub(crate) &'static str); + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} + +impl std::error::Error for Error {} + +/// A trained flat, row-major codebook and freshly recomputed training metrics. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct Model { + pub centroids: Vec, + pub dimensions: usize, + /// Nearest-centroid assignments in original input order, after finalization. + pub assignments: Vec, + /// Mean squared L2 distance per vector, not divided by dimensionality. + pub mse: f64, + pub batch_size: usize, + pub peak_step: f64, + pub updates: usize, + pub revived_clusters: usize, +} + +impl Model { + /// Assign new vectors to this codebook. Empty queries return no assignments. + /// Public/deserialized model fields are validated on every call. + pub fn predict(&self, data: &[f32]) -> Result, Error> { + predict(data, self.dimensions, &self.centroids) + } +} + +/// Exact squared-L2 prediction with deterministic lowest-index tie breaking. +pub fn predict(data: &[f32], dimensions: usize, centroids: &[f32]) -> Result, Error> { + validate_rows(data, dimensions)?; + validate_rows(centroids, dimensions)?; + if centroids.is_empty() { + return Err(Error("centroids must not be empty")); + } + Ok(data + .chunks_exact(dimensions) + .map(|row| nearest(row, centroids, dimensions).0) + .collect()) +} + +pub(crate) fn validate_rows(data: &[f32], dimensions: usize) -> Result<(), Error> { + if dimensions == 0 || data.len() % dimensions != 0 { + return Err(Error( + "dimensions must be positive and divide the data length", + )); + } + if data.iter().any(|v| !v.is_finite()) { + return Err(Error("vectors must contain only finite values")); + } + Ok(()) +} diff --git a/crates/ruvector-e-means/src/quantization.rs b/crates/ruvector-e-means/src/quantization.rs new file mode 100644 index 0000000000..1fb7f7989a --- /dev/null +++ b/crates/ruvector-e-means/src/quantization.rs @@ -0,0 +1,181 @@ +//! Checked product-quantization codebooks trained by the shared e-means kernel. + +use crate::{fit, nearest, validate_rows, Config, Error}; + +/// A reusable PQ codebook with private, validated layout metadata. +/// +/// Centroids are subspace-major: `[subspace][cluster][subspace_dimension]`. +/// Codes are row-major: one u8 per subspace per input vector. Each subspace +/// uses the supplied configuration unchanged, including its deterministic seed. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct ProductQuantizer { + dimensions: usize, + subspaces: usize, + clusters: usize, + centroids: Vec, +} + +impl ProductQuantizer { + /// Train one e-means codebook per equally sized subspace. + pub fn fit( + data: &[f32], + dimensions: usize, + subspaces: usize, + config: &Config, + ) -> Result { + validate_rows(data, dimensions)?; + config.parameters(data.len() / dimensions)?; + Self::validate_shape(dimensions, subspaces, config.clusters)?; + let sub_dim = dimensions / subspaces; + let mut centroids = Vec::with_capacity(dimensions * config.clusters); + let mut sub_data = Vec::with_capacity(data.len() / subspaces); + for subspace in 0..subspaces { + sub_data.clear(); + let start = subspace * sub_dim; + for row in data.chunks_exact(dimensions) { + sub_data.extend_from_slice(&row[start..start + sub_dim]); + } + centroids.extend(fit(&sub_data, sub_dim, config)?.centroids); + } + Self::from_centroids(dimensions, subspaces, config.clusters, centroids) + } + + /// Restore a codebook, rejecting inconsistent shapes or nonfinite values. + pub fn from_centroids( + dimensions: usize, + subspaces: usize, + clusters: usize, + centroids: Vec, + ) -> Result { + Self::validate_parts(dimensions, subspaces, clusters, ¢roids)?; + Ok(Self { + dimensions, + subspaces, + clusters, + centroids, + }) + } + + /// Validate a borrowed codebook layout without allocating a copy. + pub fn validate_parts( + dimensions: usize, + subspaces: usize, + clusters: usize, + centroids: &[f32], + ) -> Result<(), Error> { + Self::validate_shape(dimensions, subspaces, clusters)?; + if centroids.len() != dimensions * clusters { + return Err(Error( + "PQ centroid length must equal dimensions times clusters", + )); + } + validate_rows(centroids, dimensions / subspaces) + } + + fn validate_shape(dimensions: usize, subspaces: usize, clusters: usize) -> Result<(), Error> { + if dimensions == 0 || subspaces == 0 || dimensions % subspaces != 0 { + return Err(Error( + "PQ subspaces must be positive and divide positive dimensions", + )); + } + if !(1..=256).contains(&clusters) { + return Err(Error("PQ clusters must be in 1..=256 for u8 codes")); + } + dimensions + .checked_mul(clusters) + .ok_or(Error("PQ centroid shape overflows usize"))?; + Ok(()) + } + + /// Original vector dimensionality. + pub fn dimensions(&self) -> usize { + self.dimensions + } + + /// Number of equally sized subspaces, and bytes in each encoded vector. + pub fn subspaces(&self) -> usize { + self.subspaces + } + + /// Number of centroids in each subspace. + pub fn clusters(&self) -> usize { + self.clusters + } + + /// Dimensionality of each subvector. + pub fn subspace_dimensions(&self) -> usize { + self.dimensions / self.subspaces + } + + /// Borrow the flat subspace-major codebook. + pub fn centroids(&self) -> &[f32] { + &self.centroids + } + + /// Consume the quantizer and return its existing flat codebook allocation. + pub fn into_centroids(self) -> Vec { + self.centroids + } + + /// Encode finite row-major vectors with exact f64 nearest-centroid assignment. + /// Empty input produces an empty code array; ties choose the lowest code. + pub fn encode(&self, data: &[f32]) -> Result, Error> { + validate_rows(data, self.dimensions)?; + let sub_dim = self.subspace_dimensions(); + let mut codes = Vec::with_capacity(data.len() / sub_dim); + for row in data.chunks_exact(self.dimensions) { + for (subspace, subvector) in row.chunks_exact(sub_dim).enumerate() { + let start = subspace * self.clusters * sub_dim; + let book = &self.centroids[start..start + self.clusters * sub_dim]; + codes.push(nearest(subvector, book, sub_dim).0 as u8); + } + } + Ok(codes) + } + + /// Decode row-major PQ codes, rejecting incomplete rows and invalid codes. + pub fn decode(&self, codes: &[u8]) -> Result, Error> { + if codes.len() % self.subspaces != 0 { + return Err(Error("PQ code length must be divisible by subspaces")); + } + if codes.iter().any(|&code| usize::from(code) >= self.clusters) { + return Err(Error("PQ code is outside the trained codebook")); + } + let capacity = (codes.len() / self.subspaces) + .checked_mul(self.dimensions) + .ok_or(Error("decoded PQ shape overflows usize"))?; + let sub_dim = self.subspace_dimensions(); + let mut data = Vec::with_capacity(capacity); + for row in codes.chunks_exact(self.subspaces) { + for (subspace, &code) in row.iter().enumerate() { + let start = (subspace * self.clusters + usize::from(code)) * sub_dim; + data.extend_from_slice(&self.centroids[start..start + sub_dim]); + } + } + Ok(data) + } +} + +#[cfg(feature = "serde")] +impl<'de> serde::Deserialize<'de> for ProductQuantizer { + fn deserialize>(deserializer: D) -> Result { + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase", deny_unknown_fields)] + struct Wire { + dimensions: usize, + subspaces: usize, + clusters: usize, + centroids: Vec, + } + let wire = ::deserialize(deserializer)?; + Self::from_centroids( + wire.dimensions, + wire.subspaces, + wire.clusters, + wire.centroids, + ) + .map_err(serde::de::Error::custom) + } +} diff --git a/crates/ruvector-e-means/src/trainer.rs b/crates/ruvector-e-means/src/trainer.rs new file mode 100644 index 0000000000..5ded7db45b --- /dev/null +++ b/crates/ruvector-e-means/src/trainer.rs @@ -0,0 +1,242 @@ +use crate::{cosine_step, nearest, validate_rows, Config, Error, Finalization, Model}; +use rand_chacha::ChaCha8Rng; +use rand_core::{RngCore, SeedableRng}; + +const INITIAL_COUNT: f64 = 1e-10; +const MIN_COUNT: f64 = 1e-300; +const REVIVAL_THRESHOLD: f64 = 0.1; +// BF16 epsilon, as used by Appendix H, even though this implementation is f32. +const SPLIT_DELTA: f64 = 1.0 / 128.0; + +/// Train on finite flat row-major f32 vectors. The algorithm follows equations +/// (2)-(8) of arXiv:2611.09501v1, using exact CPU assignment. Shuffled partial +/// batches are included, so every row participates in every epoch. Scratch +/// memory is O(n + kd); no n-by-k distance matrix is allocated. +pub fn fit(data: &[f32], dimensions: usize, config: &Config) -> Result { + validate_rows(data, dimensions)?; + let samples = data.len() / dimensions; + let params = config.parameters(samples)?; + let k = config.clusters; + let mut rng = ChaCha8Rng::seed_from_u64(config.seed); + let mut order: Vec = (0..samples).collect(); + shuffle(&mut order, &mut rng); + let mut centroids = Vec::with_capacity(k * dimensions); + for &i in &order[..k] { + centroids.extend_from_slice(&data[i * dimensions..(i + 1) * dimensions]); + } + let mut counts = vec![INITIAL_COUNT; k]; + let mut sums: Vec = centroids + .iter() + .map(|&v| f64::from(v) * INITIAL_COUNT) + .collect(); + let mut batch_counts = vec![0.0; k]; + let mut batch_sums = vec![0.0; sums.len()]; + let use_last_mean = config.finalization == Finalization::LastEpochMean; + let mut last_counts = if use_last_mean { + vec![0usize; k] + } else { + Vec::new() + }; + let mut last_sums = if use_last_mean { + vec![0.0; sums.len()] + } else { + Vec::new() + }; + let mut revival_scratch = RevivalScratch::default(); + let mut updates = 0; + let mut revived_clusters = 0; + for epoch in 0..config.epochs { + let record_last = use_last_mean && epoch + 1 == config.epochs; + shuffle(&mut order, &mut rng); + for batch in order.chunks(params.batch_size) { + batch_counts.fill(0.0); + batch_sums.fill(0.0); + for &i in batch { + let row = &data[i * dimensions..(i + 1) * dimensions]; + let (j, _) = nearest(row, ¢roids, dimensions); + batch_counts[j] += 1.0; + let batch_sum = &mut batch_sums[j * dimensions..(j + 1) * dimensions]; + for (sum, &x) in batch_sum.iter_mut().zip(row) { + *sum += f64::from(x); + } + if record_last { + last_counts[j] += 1; + let last_sum = &mut last_sums[j * dimensions..(j + 1) * dimensions]; + for (sum, &x) in last_sum.iter_mut().zip(row) { + *sum += f64::from(x); + } + } + } + let step = cosine_step(params.peak_step, updates, params.updates); + for j in 0..k { + let count = (1.0 - step) * counts[j] + step * batch_counts[j]; + let denominator = count.max(MIN_COUNT); + counts[j] = denominator; + let range = j * dimensions..(j + 1) * dimensions; + let sum_row = &mut sums[range.clone()]; + let centroid_row = &mut centroids[range.clone()]; + if count < MIN_COUNT { + // A peak step of 1 can erase empty-cluster statistics. + // Preserve that centroid with a negligible pseudo-count. + for (sum, centroid) in sum_row.iter_mut().zip(centroid_row) { + *sum = denominator * f64::from(*centroid); + *centroid = bounded_f32(*sum / denominator); + } + } else { + for ((sum, &batch_sum), centroid) in + sum_row.iter_mut().zip(&batch_sums[range]).zip(centroid_row) + { + *sum = (1.0 - step) * *sum + step * batch_sum; + *centroid = bounded_f32(*sum / denominator); + } + } + } + updates += 1; + if updates % 5 == 0 { + revived_clusters += revive( + &mut counts, + &mut sums, + &mut centroids, + dimensions, + &mut rng, + &mut revival_scratch, + ); + } + } + } + if use_last_mean { + for (j, &count) in last_counts.iter().enumerate() { + if count > 0 { + for d in 0..dimensions { + centroids[j * dimensions + d] = + bounded_f32(last_sums[j * dimensions + d] / count as f64); + } + } + } + } + let mut assignments = Vec::with_capacity(samples); + let mut loss = 0.0; + for row in data.chunks_exact(dimensions) { + let (j, distance) = nearest(row, ¢roids, dimensions); + assignments.push(j); + loss += distance; + } + Ok(Model { + centroids, + dimensions, + assignments, + mse: loss / samples as f64, + batch_size: params.batch_size, + peak_step: params.peak_step, + updates, + revived_clusters, + }) +} + +fn bounded_f32(value: f64) -> f32 { + value.clamp(-f64::from(f32::MAX), f64::from(f32::MAX)) as f32 +} + +fn shuffle(order: &mut [usize], rng: &mut ChaCha8Rng) { + for i in (1..order.len()).rev() { + let limit = (i + 1) as u64; + // Rejection sampling avoids modulo bias, including on wasm32. + let threshold = limit.wrapping_neg() % limit; + let value = loop { + let value = rng.next_u64(); + if value >= threshold { + break value; + } + }; + order.swap(i, (value % limit) as usize); + } +} + +#[derive(Default)] +struct RevivalScratch { + dead: Vec, + available: Vec, +} + +fn revive( + counts: &mut [f64], + sums: &mut [f64], + centroids: &mut [f32], + dimensions: usize, + rng: &mut ChaCha8Rng, + scratch: &mut RevivalScratch, +) -> usize { + let threshold = REVIVAL_THRESHOLD * counts.iter().sum::() / counts.len() as f64; + scratch.dead.clear(); + scratch + .dead + .extend((0..counts.len()).filter(|&j| counts[j] < threshold)); + if scratch.dead.is_empty() { + return 0; + } + // Select live donors proportional to their EMA count, without replacement. + // Newly revived slots cannot donate again in the same revival pass. + let available = &mut scratch.available; + available.clear(); + available.extend((0..counts.len()).filter(|&j| counts[j] >= threshold)); + let mut revived = 0; + for &j in &scratch.dead { + let mass: f64 = available.iter().map(|&i| counts[i]).sum(); + if mass <= 0.0 { + break; + } + let unit = (rng.next_u64() >> 11) as f64 / ((1u64 << 53) as f64); + let mut pick = unit * mass; + let mut donor_pos = available.len() - 1; + for (pos, &i) in available.iter().enumerate() { + pick -= counts[i]; + if pick <= 0.0 { + donor_pos = pos; + break; + } + } + let donor = available.remove(donor_pos); + counts[donor] *= 0.5; + counts[j] = counts[donor]; + for d in 0..dimensions { + let shift = if d % 2 == 0 { + SPLIT_DELTA + } else { + -SPLIT_DELTA + }; + let value = f64::from(centroids[donor * dimensions + d]); + centroids[j * dimensions + d] = bounded_f32(value * (1.0 + shift)); + centroids[donor * dimensions + d] = bounded_f32(value * (1.0 - shift)); + sums[j * dimensions + d] = f64::from(centroids[j * dimensions + d]) * counts[j]; + sums[donor * dimensions + d] = + f64::from(centroids[donor * dimensions + d]) * counts[donor]; + } + revived += 1; + } + revived +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn donor_split_preserves_mass_and_weighted_sum() { + let mut counts = vec![10.0, 0.0]; + let mut sums = vec![40.0, 80.0, 0.0, 0.0]; + let mut centroids = vec![4.0, 8.0, 0.0, 0.0]; + let n = revive( + &mut counts, + &mut sums, + &mut centroids, + 2, + &mut ChaCha8Rng::seed_from_u64(2), + &mut RevivalScratch::default(), + ); + assert_eq!(n, 1); + assert_eq!(counts, [5.0, 5.0]); + assert_eq!(sums[0] + sums[2], 40.0); + assert_eq!(sums[1] + sums[3], 80.0); + assert_ne!(¢roids[..2], ¢roids[2..]); + } +} diff --git a/crates/ruvector-e-means/src/wasm.rs b/crates/ruvector-e-means/src/wasm.rs new file mode 100644 index 0000000000..6888112de3 --- /dev/null +++ b/crates/ruvector-e-means/src/wasm.rs @@ -0,0 +1,92 @@ +use crate::{fit, predict, Config, Model}; +use wasm_bindgen::prelude::*; + +/// Compatibility transport and an independent parity oracle for typed output. +#[wasm_bindgen(js_name = fitJson)] +pub fn fit_json(data: &[f32], dimensions: usize, config_json: &str) -> Result { + let config: Config = + serde_json::from_str(config_json).map_err(|e| JsError::new(&e.to_string()))?; + let model = fit(data, dimensions, &config).map_err(|e| JsError::new(&e.to_string()))?; + serde_json::to_string(&model).map_err(|e| JsError::new(&e.to_string())) +} + +/// A temporary owner for one fit result. JavaScript takes independent output +/// arrays and then frees this handle, releasing any vectors not taken. +#[wasm_bindgen] +pub struct FittedModel { + model: Model, +} + +#[wasm_bindgen] +impl FittedModel { + /// wasm-bindgen copies the returned vector into a caller-owned Float32Array + /// before freeing its Rust allocation. No view into WASM memory escapes. + #[wasm_bindgen(js_name = takeCentroids)] + pub fn take_centroids(&mut self) -> Vec { + std::mem::take(&mut self.model.centroids) + } + + /// Cluster indices fit u32 because wasm32 arrays and sample counts do. + #[wasm_bindgen(js_name = takeAssignments)] + pub fn take_assignments(&mut self) -> Vec { + std::mem::take(&mut self.model.assignments) + .into_iter() + .map(|index| index as u32) + .collect() + } + + #[wasm_bindgen(getter)] + pub fn dimensions(&self) -> usize { + self.model.dimensions + } + + #[wasm_bindgen(getter)] + pub fn mse(&self) -> f64 { + self.model.mse + } + + #[wasm_bindgen(getter, js_name = batchSize)] + pub fn batch_size(&self) -> usize { + self.model.batch_size + } + + #[wasm_bindgen(getter, js_name = peakStep)] + pub fn peak_step(&self) -> f64 { + self.model.peak_step + } + + #[wasm_bindgen(getter)] + pub fn updates(&self) -> usize { + self.model.updates + } + + #[wasm_bindgen(getter, js_name = revivedClusters)] + pub fn revived_clusters(&self) -> usize { + self.model.revived_clusters + } +} + +/// Only the small configuration crosses JSON; result vectors stay numeric. +#[wasm_bindgen(js_name = fitTyped)] +pub fn fit_typed( + data: &[f32], + dimensions: usize, + config_json: &str, +) -> Result { + let config: Config = + serde_json::from_str(config_json).map_err(|e| JsError::new(&e.to_string()))?; + fit(data, dimensions, &config) + .map(|model| FittedModel { model }) + .map_err(|e| JsError::new(&e.to_string())) +} + +#[wasm_bindgen(js_name = predictFlat)] +pub fn predict_flat( + data: &[f32], + dimensions: usize, + centroids: &[f32], +) -> Result, JsError> { + predict(data, dimensions, centroids) + .map(|v| v.into_iter().map(|j| j as u32).collect()) + .map_err(|e| JsError::new(&e.to_string())) +} diff --git a/crates/ruvector-e-means/tests/clustering.rs b/crates/ruvector-e-means/tests/clustering.rs new file mode 100644 index 0000000000..3f340f2b76 --- /dev/null +++ b/crates/ruvector-e-means/tests/clustering.rs @@ -0,0 +1,234 @@ +use ruvector_e_means::{cosine_step, fit, predict, Config, Finalization}; + +#[test] +fn ema_uses_decayed_sums_and_counts_including_short_batches() { + let model = fit( + &[0.0, 1.0, 9.0], + 1, + &Config { + clusters: 1, + epochs: 1, + batch_size: Some(2), + peak_step: Some(1.0), + finalization: Finalization::Ema, + ..Config::default() + }, + ) + .unwrap(); + // First step=1, second step=.55. For any shuffled tail x, the + // correct ratio is (.45*(10-x)+.55*x)/(.45*2+.55*1). + // Averaging batch centroids directly gives a different answer. + let possible: Vec = [0.0, 1.0, 9.0] + .into_iter() + .map(|tail| (0.45 * (10.0 - tail) + 0.55 * tail) / 1.45) + .collect(); + assert!(possible + .iter() + .any(|x| (*x - f64::from(model.centroids[0])).abs() < 1e-6)); + assert_eq!(model.updates, 2); + let center = f64::from(model.centroids[0]); + let expected_mse = (center.powi(2) + (1.0 - center).powi(2) + (9.0 - center).powi(2)) / 3.0; + assert!((model.mse - expected_mse).abs() < 1e-12); +} + +#[test] +fn last_epoch_mean_matches_independent_mean_and_mse_with_tail_batch() { + let data = [1., 2., 3., 4., 5., 6., 7., 8., 9., 10.]; + let config = Config { + clusters: 1, + epochs: 2, + batch_size: Some(2), + ..Config::default() + }; + let model = fit(&data, 2, &config).unwrap(); + assert_eq!(model.centroids, [5., 6.]); + assert_eq!(model.assignments, [0; 5]); + assert_eq!(model.mse, 16.0); + assert_eq!(model.updates, 6); // ceil(5/2) * 2, no dropped rows. + assert_eq!(model.predict(&data).unwrap(), model.assignments); +} + +#[test] +fn scaling_laws_match_table_two_and_large_dataset_rule() { + let params = Config { + clusters: 1024, + epochs: 4, + ..Config::default() + } + .parameters(1_000_000) + .unwrap(); + assert_eq!(params.batch_size, 65536); + assert_eq!(params.peak_step, 1.0); + let params = Config { + clusters: 1024, + epochs: 4, + batch_size: Some(256), + ..Config::default() + } + .parameters(1_000_000) + .unwrap(); + assert!((params.peak_step - 0.008558124524629177).abs() < 1e-14); + let params = Config { + clusters: 1000, + ..Config::default() + } + .parameters(3_000_000) + .unwrap(); + assert_eq!(params.batch_size, 16384); + assert!((params.peak_step - 0.2299943961117698).abs() < 1e-14); +} + +#[test] +fn schedule_matches_equation_five() { + assert_eq!(cosine_step(0.5, 0, 20), 0.5); + assert!((cosine_step(0.5, 10, 20) - 0.275).abs() < 1e-15); + assert_eq!(cosine_step(0.5, 20, 20), 0.05); + let steps: Vec = (0..20).map(|i| cosine_step(0.5, i, 20)).collect(); + assert!(steps.windows(2).all(|s| s[1] < s[0])); + assert!(steps[19] > 0.05); +} + +#[test] +fn separates_three_clouds_and_reproduces_seed() { + let mut data = Vec::new(); + for j in 0..3 { + for i in 0..48 { + for d in 0..8 { + data.push(j as f32 * 40.0 + ((i * 13 + d * 7) % 19) as f32 * 0.01); + } + } + } + let config = Config { + clusters: 3, + epochs: 5, + batch_size: Some(12), + ..Config::default() + }; + let a = fit(&data, 8, &config).unwrap(); + let b = fit(&data, 8, &config).unwrap(); + assert_eq!(a.centroids, b.centroids); + assert_eq!(a.assignments, b.assignments); + assert!(a.mse < 0.1, "mse {}", a.mse); + let leaders: Vec<_> = [0, 48, 96].into_iter().map(|i| a.assignments[i]).collect(); + assert_ne!(leaders[0], leaders[1]); + assert_ne!(leaders[0], leaders[2]); + assert_ne!(leaders[1], leaders[2]); + for cloud in a.assignments.chunks_exact(48) { + assert!(cloud.iter().all(|j| *j == cloud[0])); + } +} + +#[test] +fn duplicate_and_zero_vectors_remain_finite_with_revival_and_step_one() { + for value in [0., 12., f32::MAX, -f32::MAX] { + for finalization in [Finalization::LastEpochMean, Finalization::Ema] { + let model = fit( + &[value; 96], + 3, + &Config { + clusters: 8, + epochs: 4, + batch_size: Some(2), + peak_step: Some(1.0), + finalization, + ..Config::default() + }, + ) + .unwrap(); + assert!(model.centroids.iter().all(|x| x.is_finite())); + assert!(model.mse.is_finite()); + assert!(model.revived_clusters > 0); + assert!(model.assignments.iter().all(|j| *j < 8)); + } + } +} + +#[test] +fn handles_full_batch_k_equals_n_and_large_coordinates() { + let data = [f32::MAX, -f32::MAX, -f32::MAX, f32::MAX]; + let model = fit( + &data, + 2, + &Config { + clusters: 2, + batch_size: Some(100), + ..Config::default() + }, + ) + .unwrap(); + assert_eq!(model.batch_size, 2); + assert_eq!(model.mse, 0.0); + assert_ne!(model.assignments[0], model.assignments[1]); +} + +#[test] +fn validates_before_training_and_checks_mutated_models_on_predict() { + let good = Config { + clusters: 1, + ..Config::default() + }; + for (data, dim) in [ + (vec![], 2), + (vec![1.], 0), + (vec![1.], 2), + (vec![f32::NAN], 1), + (vec![f32::INFINITY], 1), + ] { + assert!(fit(&data, dim, &good).is_err()); + } + for config in [ + Config { + clusters: 0, + ..good.clone() + }, + Config { + clusters: 3, + ..good.clone() + }, + Config { + epochs: 0, + ..good.clone() + }, + Config { + batch_size: Some(0), + ..good.clone() + }, + Config { + peak_step: Some(0.), + ..good.clone() + }, + Config { + peak_step: Some(-0.1), + ..good.clone() + }, + Config { + peak_step: Some(1.1), + ..good.clone() + }, + Config { + peak_step: Some(f64::NAN), + ..good.clone() + }, + Config { + epochs: usize::MAX, + batch_size: Some(1), + ..good.clone() + }, + ] { + assert!(fit(&[1., 2.], 1, &config).is_err()); + } + let mut model = fit(&[1., 2.], 1, &good).unwrap(); + assert!(model.predict(&[]).unwrap().is_empty()); + model.dimensions = 0; + assert!(model.predict(&[1.]).is_err()); + assert!(predict(&[1., 2.], 2, &[1.]).is_err()); + assert!(predict(&[1.], 1, &[]).is_err()); +} + +#[cfg(feature = "serde")] +#[test] +fn serializable_public_api_is_available() { + fn serializable serde::Deserialize<'de>>() {} + serializable::(); + serializable::(); +} diff --git a/crates/ruvector-e-means/tests/ivf.rs b/crates/ruvector-e-means/tests/ivf.rs new file mode 100644 index 0000000000..067c016487 --- /dev/null +++ b/crates/ruvector-e-means/tests/ivf.rs @@ -0,0 +1,103 @@ +use ruvector_e_means::{Config, IvfPartition, Model}; + +fn model() -> Model { + Model { + centroids: vec![0.0, 10.0], + dimensions: 1, + assignments: vec![1, 0, 1, 0], + mse: 0.0, + batch_size: 4, + peak_step: 0.2, + updates: 1, + revived_clusters: 0, + } +} + +#[test] +fn stable_postings_group_every_input_row_exactly_once() { + let partition = IvfPartition::from_model(model()).unwrap(); + assert_eq!(partition.offsets(), [0, 2, 4]); + assert_eq!(partition.indices(), [1, 3, 0, 2]); + assert_eq!(partition.posting(0).unwrap(), [1, 3]); + assert_eq!(partition.posting(1).unwrap(), [0, 2]); + assert!(partition.posting(2).is_err()); + let restored = IvfPartition::from_parts( + model(), + partition.offsets().to_vec(), + partition.indices().to_vec(), + ) + .unwrap(); + assert_eq!(restored.indices(), partition.indices()); +} + +#[test] +fn fit_partitions_final_predictions_in_original_row_order() { + let data = [10.0, -10.0, 11.0, -11.0, 12.0, -12.0]; + let partition = IvfPartition::fit( + &data, + 1, + &Config { + clusters: 2, + epochs: 5, + ..Config::default() + }, + ) + .unwrap(); + let model = partition.model(); + assert_eq!(model.predict(&data).unwrap(), model.assignments); + let mut ids = partition.indices().to_vec(); + ids.sort_unstable(); + assert_eq!(ids, (0..data.len()).collect::>()); + for cluster in 0..2 { + let posting = partition.posting(cluster).unwrap(); + assert!(posting.windows(2).all(|p| p[0] < p[1])); + assert!(posting.iter().all(|&id| model.assignments[id] == cluster)); + } +} + +#[test] +fn malformed_offsets_labels_and_row_permutations_reject() { + for (offsets, indices) in [ + (vec![1, 2, 4], vec![1, 3, 0, 2]), + (vec![0, 3, 4], vec![1, 3, 0, 2]), + (vec![0, 5, 4], vec![1, 3, 0, 2]), + (vec![0, 2], vec![1, 3, 0, 2]), + (vec![0, 2, 4], vec![1, 1, 0, 2]), + (vec![0, 2, 4], vec![3, 1, 0, 2]), + (vec![0, 2, 4], vec![1, 3, 0, 9]), + (vec![0, 2, 4], vec![1, 3, 0]), + ] { + assert!(IvfPartition::from_parts(model(), offsets, indices).is_err()); + } + let mut invalid = model(); + invalid.assignments[0] = 2; + assert!(IvfPartition::from_model(invalid).is_err()); + let mut invalid = model(); + invalid.centroids[0] = f32::INFINITY; + assert!(IvfPartition::from_model(invalid).is_err()); + let mut invalid = model(); + invalid.dimensions = 0; + assert!(IvfPartition::from_model(invalid).is_err()); + let mut invalid = model(); + invalid.batch_size = 0; + assert!(IvfPartition::from_model(invalid).is_err()); +} + +#[cfg(feature = "serde")] +#[test] +fn serde_roundtrip_rejects_corrupted_postings_and_models() { + let partition = IvfPartition::from_model(model()).unwrap(); + let value = serde_json::to_value(&partition).unwrap(); + let restored: IvfPartition = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(restored.offsets(), partition.offsets()); + assert_eq!(restored.indices(), partition.indices()); + let mut invalid = value.clone(); + invalid["indices"] = serde_json::json!([1, 1, 0, 2]); + assert!(serde_json::from_value::(invalid).is_err()); + let mut invalid = value.clone(); + invalid["model"]["assignments"] = serde_json::json!([2, 0, 1, 0]); + assert!(serde_json::from_value::(invalid).is_err()); + let mut invalid = value; + invalid["offsets"] = serde_json::json!([0, 4, 2]); + assert!(serde_json::from_value::(invalid).is_err()); +} diff --git a/crates/ruvector-e-means/tests/quantization.rs b/crates/ruvector-e-means/tests/quantization.rs new file mode 100644 index 0000000000..7f88a70061 --- /dev/null +++ b/crates/ruvector-e-means/tests/quantization.rs @@ -0,0 +1,113 @@ +use ruvector_e_means::{Config, ProductQuantizer}; + +#[test] +fn subspace_layout_codes_and_reconstruction_have_an_independent_oracle() { + let pq = ProductQuantizer::from_centroids( + 4, + 2, + 2, + vec![0.0, 0.0, 10.0, 10.0, 100.0, 100.0, 200.0, 200.0], + ) + .unwrap(); + let data = [0.0, 0.0, 200.0, 200.0, 10.0, 10.0, 100.0, 100.0]; + assert_eq!(pq.encode(&data).unwrap(), [0, 1, 1, 0]); + assert_eq!(pq.decode(&[0, 1, 1, 0]).unwrap(), data); + assert_eq!(pq.encode(&[5.0, 5.0, 150.0, 150.0]).unwrap(), [0, 0]); + assert!(pq.encode(&[]).unwrap().is_empty()); + assert!(pq.decode(&[]).unwrap().is_empty()); +} + +#[test] +fn fitted_single_codebooks_recover_column_means() { + let pq = ProductQuantizer::fit( + &[1.0, 10.0, 3.0, 30.0, 5.0, 50.0], + 2, + 2, + &Config { + clusters: 1, + batch_size: Some(2), + ..Config::default() + }, + ) + .unwrap(); + assert_eq!( + ( + pq.dimensions(), + pq.subspaces(), + pq.clusters(), + pq.subspace_dimensions() + ), + (2, 2, 1, 1) + ); + assert_eq!(pq.centroids(), [3.0, 30.0]); + assert_eq!( + pq.decode(&pq.encode(&[2.0, 20.0, 9.0, 90.0]).unwrap()) + .unwrap(), + [3.0, 30.0, 3.0, 30.0] + ); +} + +#[test] +fn checked_pq_boundaries_reject_malformed_and_nonfinite_data() { + for (dimensions, subspaces, clusters, centroids) in [ + (0, 1, 1, vec![]), + (2, 0, 1, vec![0.0; 2]), + (3, 2, 1, vec![0.0; 3]), + (2, 1, 0, vec![]), + (1, 1, 257, vec![0.0; 257]), + (2, 1, 1, vec![0.0]), + (1, 1, 1, vec![f32::NAN]), + (usize::MAX, 1, 256, vec![]), + ] { + assert!( + ProductQuantizer::from_centroids(dimensions, subspaces, clusters, centroids).is_err() + ); + } + let pq = ProductQuantizer::from_centroids(2, 2, 1, vec![0.0; 2]).unwrap(); + assert!(pq.encode(&[1.0]).is_err()); + assert!(pq.encode(&[f32::INFINITY, 0.0]).is_err()); + assert!(pq.decode(&[0]).is_err()); + assert!(pq.decode(&[0, 1]).is_err()); + assert!(ProductQuantizer::fit(&[], 2, 2, &Config::default()).is_err()); + assert!(ProductQuantizer::fit( + &[0.0; 8], + 2, + 2, + &Config { + clusters: 5, + ..Config::default() + } + ) + .is_err()); +} + +#[test] +fn all_u8_codes_and_large_finite_distances_remain_valid() { + let pq = + ProductQuantizer::from_centroids(1, 1, 256, (0..256).map(|v| v as f32).collect()).unwrap(); + assert_eq!(pq.encode(&[255.0]).unwrap(), [255]); + assert_eq!(pq.decode(&[255]).unwrap(), [255.0]); + let magnitude = f32::MAX / 4.0; + let pq = ProductQuantizer::from_centroids(1, 1, 2, vec![-magnitude, magnitude]).unwrap(); + assert_eq!(pq.encode(&[magnitude * 2.0]).unwrap(), [1]); +} + +#[cfg(feature = "serde")] +#[test] +fn serde_roundtrip_validates_the_codebook_before_it_becomes_usable() { + let pq = ProductQuantizer::from_centroids(2, 2, 2, vec![0.0, 1.0, 10.0, 20.0]).unwrap(); + let value = serde_json::to_value(&pq).unwrap(); + let restored: ProductQuantizer = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(restored.encode(&[1.0, 20.0]).unwrap(), [1, 1]); + for field in ["dimensions", "subspaces", "clusters"] { + let mut invalid = value.clone(); + invalid[field] = 0.into(); + assert!(serde_json::from_value::(invalid).is_err()); + } + let mut invalid = value.clone(); + invalid["centroids"] = serde_json::json!([0.0]); + assert!(serde_json::from_value::(invalid).is_err()); + let mut invalid = value; + invalid["extra"] = true.into(); + assert!(serde_json::from_value::(invalid).is_err()); +} diff --git a/crates/ruvector-pq-search/Cargo.toml b/crates/ruvector-pq-search/Cargo.toml index 7dab2653a3..362e2f5341 100644 --- a/crates/ruvector-pq-search/Cargo.toml +++ b/crates/ruvector-pq-search/Cargo.toml @@ -16,6 +16,7 @@ name = "pq-benchmark" path = "src/main.rs" [dependencies] +ruvector-e-means = { version = "=0.1.0", path = "../ruvector-e-means" } rand = { workspace = true } thiserror = { workspace = true } serde = { workspace = true } diff --git a/crates/ruvector-pq-search/src/codebook.rs b/crates/ruvector-pq-search/src/codebook.rs index 638c9c8d70..1193099169 100644 --- a/crates/ruvector-pq-search/src/codebook.rs +++ b/crates/ruvector-pq-search/src/codebook.rs @@ -1,9 +1,20 @@ -//! PQ codebook: M sub-spaces, K centroids each, trained with Lloyd's k-means. +//! PQ codebook: M sub-spaces, K centroids, with Lloyd and explicit e-means training. use crate::l2_sq; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; +/// Validation or shared-optimizer failure in the checked e-means PQ APIs. +#[derive(Debug, thiserror::Error)] +pub enum EmeansPqError { + /// Malformed codebook, vector, or probe configuration. + #[error("invalid e-means PQ input: {0}")] + InvalidInput(&'static str), + /// Validation failure from the shared clustering implementation. + #[error(transparent)] + Clustering(#[from] ruvector_e_means::Error), +} + /// Configuration for product quantization. #[derive(Debug, Clone)] pub struct PqConfig { @@ -11,7 +22,7 @@ pub struct PqConfig { pub m: usize, /// Number of centroids per sub-space (must be ≤ 256 for u8 codes). pub k: usize, - /// Number of Lloyd's iterations. + /// Training budget: Lloyd iterations, or epochs for explicit e-means training. pub iterations: usize, /// RNG seed (deterministic builds). pub seed: u64, @@ -54,6 +65,44 @@ pub struct PqCodebook { } impl PqCodebook { + /// Train checked e-means subspace codebooks in the existing flat PQ layout. + /// `options.clusters` is the number of u8 centroids per subspace. The same + /// configuration is used for every subspace; Lloyd defaults are unchanged. + pub fn train_e_means( + subspaces: usize, + vectors: &[f32], + dim: usize, + options: &ruvector_e_means::Config, + ) -> Result { + let quantizer = ruvector_e_means::ProductQuantizer::fit(vectors, dim, subspaces, options)?; + Ok(Self { + config: PqConfig { + m: subspaces, + k: options.clusters, + iterations: options.epochs, + seed: options.seed, + }, + sub_dim: quantizer.subspace_dimensions(), + dim, + centroids: quantizer.into_centroids(), + }) + } + + pub(crate) fn validate_e_means_layout(&self) -> Result<(), EmeansPqError> { + ruvector_e_means::ProductQuantizer::validate_parts( + self.dim, + self.config.m, + self.config.k, + &self.centroids, + )?; + if self.sub_dim != self.dim / self.config.m { + return Err(EmeansPqError::InvalidInput( + "codebook sub_dim disagrees with its shape", + )); + } + Ok(()) + } + /// Train a PQ codebook from a slice of flat row-major vectors. /// `vectors`: n × dim flat layout. pub fn train(config: PqConfig, vectors: &[f32], dim: usize) -> Self { diff --git a/crates/ruvector-pq-search/src/ivf_pq.rs b/crates/ruvector-pq-search/src/ivf_pq.rs index 123ed2f902..c0f67eca5c 100644 --- a/crates/ruvector-pq-search/src/ivf_pq.rs +++ b/crates/ruvector-pq-search/src/ivf_pq.rs @@ -4,7 +4,11 @@ //! k-means on raw vectors. At query time, probe the `n_probe` nearest cells //! and run ADC only within those lists. Reduces scan from O(n) to O(n/n_lists × n_probe). -use crate::{codebook::PqCodebook, encoder::encode_vector, l2_sq, PqSearch, SearchResult}; +use crate::{ + codebook::{EmeansPqError, PqCodebook}, + encoder::encode_vector, + l2_sq, PqSearch, SearchResult, +}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; @@ -21,6 +25,60 @@ pub struct IvfPqIndex { } impl IvfPqIndex { + /// Train coarse e-means centroids while retaining the existing IVF+PQ + /// posting, code, and ADC search representation. `options.clusters` sets + /// the number of lists; the supplied codebook may also be e-means-trained. + pub fn new_e_means( + codebook: PqCodebook, + n_probe: usize, + training_vecs: &[f32], + options: &ruvector_e_means::Config, + ) -> Result { + codebook.validate_e_means_layout()?; + if n_probe == 0 || n_probe > options.clusters { + return Err(EmeansPqError::InvalidInput( + "n_probe must be in 1..=coarse clusters", + )); + } + let dim = codebook.dim; + let coarse = ruvector_e_means::fit(training_vecs, dim, options)?; + Ok(Self { + codebook, + coarse_centroids: coarse.centroids, + lists: vec![Vec::new(); options.clusters], + n_lists: options.clusters, + n_probe, + dim, + next_id: 0, + }) + } + + /// Insert one finite, correctly shaped vector without a partial mutation + /// on validation failure. Existing `PqSearch::insert` remains available. + pub fn try_insert(&mut self, vector: &[f32]) -> Result<(), EmeansPqError> { + self.validate_vector(vector)?; + if self.next_id == usize::MAX { + return Err(EmeansPqError::InvalidInput("vector ID overflow")); + } + ::insert(self, vector); + Ok(()) + } + + /// Search with explicit query validation before accessing legacy ADC code. + pub fn try_search(&self, query: &[f32], k: usize) -> Result, EmeansPqError> { + self.validate_vector(query)?; + Ok(::search(self, query, k)) + } + + fn validate_vector(&self, vector: &[f32]) -> Result<(), EmeansPqError> { + if vector.len() != self.dim || vector.iter().any(|value| !value.is_finite()) { + return Err(EmeansPqError::InvalidInput( + "vector must match dimensions and contain only finite values", + )); + } + Ok(()) + } + /// Build an IVF+PQ index. /// `n_lists`: number of Voronoi partitions. /// `n_probe`: cells to probe at query time. diff --git a/crates/ruvector-pq-search/tests/e_means.rs b/crates/ruvector-pq-search/tests/e_means.rs new file mode 100644 index 0000000000..4e1ff254fe --- /dev/null +++ b/crates/ruvector-pq-search/tests/e_means.rs @@ -0,0 +1,73 @@ +use ruvector_e_means::Config; +use ruvector_pq_search::{decode_vector, encode_vector, IvfPqIndex, PqCodebook}; + +fn data() -> Vec { + (0..8).flat_map(|i| [i as f32; 4]).collect() +} + +fn options(clusters: usize) -> Config { + Config { + clusters, + epochs: 1, + batch_size: Some(8), + seed: 7, + ..Config::default() + } +} + +#[test] +fn e_means_codebooks_roundtrip_existing_codes_and_adc_search() { + let data = data(); + let codebook = PqCodebook::train_e_means(2, &data, 4, &options(8)).unwrap(); + assert_eq!(codebook.centroids.len(), 8 * 4); + for row in data.as_chunks::<4>().0 { + let codes = encode_vector(&codebook, row); + assert_eq!(codes.len(), 2); + assert_eq!(decode_vector(&codebook, &codes), row.as_slice()); + let table = codebook.build_adc_table(row); + assert_eq!(codebook.adc_distance(&table, &codes), 0.0); + } + let mut index = IvfPqIndex::new_e_means(codebook, 2, &data, &options(2)).unwrap(); + assert!(index.try_insert(&[1.0]).is_err()); + assert!(index.try_insert(&[f32::NAN; 4]).is_err()); + for row in data.as_chunks::<4>().0 { + index.try_insert(row).unwrap(); + } + let results = index.try_search(&[3.0; 4], 3).unwrap(); + assert_eq!(results[0].id, 3); + assert_eq!( + results.iter().map(|r| r.distance).collect::>(), + [0.0, 4.0, 4.0] + ); + let mut ids: Vec = results.iter().map(|r| r.id).collect(); + ids.sort_unstable(); + assert_eq!(ids, [2, 3, 4]); + assert!(index.try_search(&[0.0], 1).is_err()); + assert!(index.try_search(&[f32::INFINITY; 4], 1).is_err()); + assert!(index.try_search(&[0.0; 4], 0).unwrap().is_empty()); +} + +#[test] +fn checked_constructors_reject_invalid_training_and_public_codebook_fields() { + let data = data(); + assert!(PqCodebook::train_e_means(0, &data, 4, &options(2)).is_err()); + assert!(PqCodebook::train_e_means(3, &data, 4, &options(2)).is_err()); + assert!(PqCodebook::train_e_means(2, &[0.0; 3], 4, &options(2)).is_err()); + assert!(PqCodebook::train_e_means(2, &[f32::NAN; 32], 4, &options(2)).is_err()); + assert!(PqCodebook::train_e_means(2, &data, 4, &options(9)).is_err()); + let codebook = PqCodebook::train_e_means(2, &data, 4, &options(8)).unwrap(); + for probes in [0, 3] { + assert!(IvfPqIndex::new_e_means(codebook.clone(), probes, &data, &options(2)).is_err()); + } + let mut corrupt = codebook.clone(); + corrupt.sub_dim = 0; + assert!(IvfPqIndex::new_e_means(corrupt, 1, &data, &options(2)).is_err()); + let mut corrupt = codebook.clone(); + corrupt.config.m = 0; + assert!(IvfPqIndex::new_e_means(corrupt, 1, &data, &options(2)).is_err()); + let mut corrupt = codebook.clone(); + corrupt.centroids[0] = f32::NAN; + assert!(IvfPqIndex::new_e_means(corrupt, 1, &data, &options(2)).is_err()); + assert!(IvfPqIndex::new_e_means(codebook.clone(), 1, &[], &options(2)).is_err()); + assert!(IvfPqIndex::new_e_means(codebook, 1, &[0.0; 3], &options(2)).is_err()); +} diff --git a/crates/ruvector-rairs/Cargo.toml b/crates/ruvector-rairs/Cargo.toml index bd069d489a..c36b5ffd60 100644 --- a/crates/ruvector-rairs/Cargo.toml +++ b/crates/ruvector-rairs/Cargo.toml @@ -14,6 +14,7 @@ name = "rairs-demo" path = "src/main.rs" [dependencies] +ruvector-e-means = { version = "=0.1.0", path = "../ruvector-e-means" } rand = "0.8" serde = { version = "1", features = ["derive"] } diff --git a/crates/ruvector-rairs/src/ivf.rs b/crates/ruvector-rairs/src/ivf.rs index 160c1bcb08..14fb0cf5b7 100644 --- a/crates/ruvector-rairs/src/ivf.rs +++ b/crates/ruvector-rairs/src/ivf.rs @@ -22,6 +22,22 @@ pub struct IvfFlat { } impl IvfFlat { + /// Train with the shared e-means optimizer instead of Lloyd's algorithm. + /// `options.clusters` must match this index's configured cluster count; + /// epochs and seed come from `options`. Successful retraining clears all + /// inserted vectors and restarts IDs; validation failure preserves state. + pub fn train_e_means( + &mut self, + corpus: &[Vec], + options: &ruvector_e_means::Config, + ) -> Result<(), RairsError> { + let centroids = train_e_means_centroids(corpus, self.dim, self.nclusters, options)?; + self.lists = vec![Vec::new(); centroids.len()]; + self.centroids = centroids; + self.total = 0; + Ok(()) + } + /// Create a new untrained IvfFlat index. /// /// * `dim` — vector dimensionality @@ -59,6 +75,54 @@ impl IvfFlat { } } +/// Shared validated centroid training for the IVF, RAIRS, and SEIL layouts. +pub(crate) fn train_e_means_centroids( + corpus: &[Vec], + dimensions: usize, + clusters: usize, + options: &ruvector_e_means::Config, +) -> Result>, RairsError> { + if corpus.is_empty() { + return Err(RairsError::EmptyCorpus); + } + if dimensions == 0 || options.clusters != clusters { + return Err(RairsError::InvalidParam( + "e-means requires positive dimensions and a matching cluster count".into(), + )); + } + options + .parameters(corpus.len()) + .map_err(|error| RairsError::InvalidParam(error.to_string()))?; + let capacity = corpus + .len() + .checked_mul(dimensions) + .ok_or_else(|| RairsError::InvalidParam("training shape overflows usize".into()))?; + for row in corpus { + if row.len() != dimensions { + return Err(RairsError::DimMismatch { + expected: dimensions, + got: row.len(), + }); + } + if row.iter().any(|value| !value.is_finite()) { + return Err(RairsError::InvalidParam( + "e-means training vectors must contain only finite values".into(), + )); + } + } + let mut data = Vec::with_capacity(capacity); + for row in corpus { + data.extend_from_slice(row); + } + let model = ruvector_e_means::fit(&data, dimensions, options) + .map_err(|error| RairsError::InvalidParam(error.to_string()))?; + Ok(model + .centroids + .chunks_exact(dimensions) + .map(<[f32]>::to_vec) + .collect()) +} + impl AnnIndex for IvfFlat { fn add(&mut self, vectors: &[Vec]) -> Result<(), RairsError> { if self.centroids.is_empty() { diff --git a/crates/ruvector-rairs/src/rairs.rs b/crates/ruvector-rairs/src/rairs.rs index da9c342c2f..b87f99f7cd 100644 --- a/crates/ruvector-rairs/src/rairs.rs +++ b/crates/ruvector-rairs/src/rairs.rs @@ -34,6 +34,26 @@ pub struct RairsStrict { } impl RairsStrict { + /// Train explicit e-means centroids, preserving this variant's redundant + /// assignment and search layout. Cluster count must match the constructor; + /// the supplied options control epochs and seed. A failed fit preserves + /// current state; successful retraining clears vectors and resets IDs. + pub fn train_e_means( + &mut self, + corpus: &[Vec], + options: &ruvector_e_means::Config, + ) -> Result<(), RairsError> { + if !self.lambda.is_finite() { + return Err(RairsError::InvalidParam("lambda must be finite".into())); + } + let centroids = + crate::ivf::train_e_means_centroids(corpus, self.dim, self.nclusters, options)?; + self.lists = vec![Vec::new(); centroids.len()]; + self.centroids = centroids; + self.total = 0; + Ok(()) + } + /// Create a new untrained RairsStrict index. /// /// `lambda` is the RAIR amplification factor (paper default = 1.0). diff --git a/crates/ruvector-rairs/src/seil.rs b/crates/ruvector-rairs/src/seil.rs index 19d7d6a1c5..124c5aebf0 100644 --- a/crates/ruvector-rairs/src/seil.rs +++ b/crates/ruvector-rairs/src/seil.rs @@ -48,6 +48,26 @@ pub struct RairsSeil { } impl RairsSeil { + /// Train explicit e-means centroids while retaining SEIL block storage and + /// redundant assignment. Cluster count must match the constructor; options + /// provide epochs and seed. Failure preserves state; success clears stored + /// vectors and restarts their IDs at zero. + pub fn train_e_means( + &mut self, + corpus: &[Vec], + options: &ruvector_e_means::Config, + ) -> Result<(), RairsError> { + if !self.lambda.is_finite() { + return Err(RairsError::InvalidParam("lambda must be finite".into())); + } + let centroids = + crate::ivf::train_e_means_centroids(corpus, self.dim, self.nclusters, options)?; + self.lists = vec![Vec::new(); centroids.len()]; + self.centroids = centroids; + self.total = 0; + Ok(()) + } + /// Create a new untrained RairsSeil index. pub fn new(dim: usize, nclusters: usize, max_iter: usize, seed: u64, lambda: f32) -> Self { Self { diff --git a/crates/ruvector-rairs/tests/e_means.rs b/crates/ruvector-rairs/tests/e_means.rs new file mode 100644 index 0000000000..388ac60f5f --- /dev/null +++ b/crates/ruvector-rairs/tests/e_means.rs @@ -0,0 +1,114 @@ +use ruvector_e_means::Config; +use ruvector_rairs::{AnnIndex, IvfFlat, RairsSeil, RairsStrict}; + +fn corpus() -> Vec> { + (0..40) + .map(|i| vec![i as f32, i as f32 * 0.1, (i % 3) as f32, -(i as f32)]) + .collect() +} + +fn options() -> Config { + Config { + clusters: 4, + epochs: 3, + batch_size: Some(7), + seed: 17, + ..Config::default() + } +} + +fn assert_full_probe_ground_truth(index: &dyn AnnIndex, vectors: &[Vec]) { + let query = [17.37, 1.65, 2.13, -17.91]; + let mut truth: Vec<(usize, f64)> = vectors + .iter() + .enumerate() + .map(|(id, row)| { + ( + id, + row.iter() + .zip(query) + .map(|(&x, y)| (f64::from(x) - f64::from(y)).powi(2)) + .sum::() + .sqrt(), + ) + }) + .collect(); + truth.sort_by(|a, b| a.1.total_cmp(&b.1)); + let results = index.search(&query, 5, index.num_lists()).unwrap(); + assert_eq!(results.len(), 5); + for (actual, expected) in results.iter().zip(truth) { + assert_eq!(actual.id, expected.0); + assert!((f64::from(actual.distance) - expected.1).abs() < 0.001); + } +} + +#[test] +fn all_rairs_layouts_use_e_means_and_preserve_search_and_training_atomicity() { + let vectors = corpus(); + let config = options(); + macro_rules! check { + ($index:expr) => {{ + let mut index = $index; + index.train_e_means(&vectors, &config).unwrap(); + index.add(&vectors).unwrap(); + assert_eq!(index.len(), vectors.len()); + assert_full_probe_ground_truth(&index, &vectors); + let before = index.search(&vectors[17], 5, 4).unwrap(); + let mut invalid = vectors.clone(); + invalid[3].pop(); + assert!(index.train_e_means(&invalid, &config).is_err()); + invalid = vectors.clone(); + invalid[3][0] = f32::NAN; + assert!(index.train_e_means(&invalid, &config).is_err()); + assert!(index.train_e_means(&[], &config).is_err()); + let mut bad_options = config.clone(); + bad_options.clusters = 5; + assert!(index.train_e_means(&vectors, &bad_options).is_err()); + bad_options = config.clone(); + bad_options.epochs = 0; + assert!(index.train_e_means(&vectors, &bad_options).is_err()); + assert_eq!(index.len(), vectors.len()); + assert_eq!(index.search(&vectors[17], 5, 4).unwrap(), before); + index.train_e_means(&vectors, &config).unwrap(); + assert!(index.is_empty()); + index.add(&vectors[10..12]).unwrap(); + assert_eq!(index.len(), 2); + assert_eq!(index.search(&vectors[10], 1, 4).unwrap()[0].id, 0); + }}; + } + check!(IvfFlat::new(4, 4, 25, 42)); + check!(RairsStrict::new(4, 4, 25, 42, 1.0)); + check!(RairsSeil::new(4, 4, 25, 42, 1.0)); +} + +#[test] +fn invalid_constructor_settings_reject_without_a_training_panic() { + let vectors = corpus(); + assert!(IvfFlat::new(0, 4, 25, 42) + .train_e_means(&vectors, &options()) + .is_err()); + assert!(IvfFlat::new(4, 0, 25, 42) + .train_e_means( + &vectors, + &Config { + clusters: 0, + ..options() + } + ) + .is_err()); + assert!(IvfFlat::new(4, 41, 25, 42) + .train_e_means( + &vectors, + &Config { + clusters: 41, + ..options() + } + ) + .is_err()); + assert!(RairsStrict::new(4, 4, 25, 42, f32::NAN) + .train_e_means(&vectors, &options()) + .is_err()); + assert!(RairsSeil::new(4, 4, 25, 42, f32::INFINITY) + .train_e_means(&vectors, &options()) + .is_err()); +} diff --git a/crates/ruvector-wasm/clustering.d.mts b/crates/ruvector-wasm/clustering.d.mts new file mode 100644 index 0000000000..2ca0a76f10 --- /dev/null +++ b/crates/ruvector-wasm/clustering.d.mts @@ -0,0 +1,2 @@ +export { init, fit, predict } from '@ruvector/e-means'; +export type { EMeansOptions, EMeansModel } from '@ruvector/e-means'; diff --git a/crates/ruvector-wasm/clustering.mjs b/crates/ruvector-wasm/clustering.mjs new file mode 100644 index 0000000000..f1f8835355 --- /dev/null +++ b/crates/ruvector-wasm/clustering.mjs @@ -0,0 +1,2 @@ +/** Shared Rust/WASM clustering; initialize explicitly before numerical calls. */ +export { init, fit, predict } from '@ruvector/e-means'; diff --git a/crates/ruvector-wasm/clustering/cloudflare.d.mts b/crates/ruvector-wasm/clustering/cloudflare.d.mts new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/crates/ruvector-wasm/clustering/cloudflare.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/crates/ruvector-wasm/clustering/cloudflare.mjs b/crates/ruvector-wasm/clustering/cloudflare.mjs new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/crates/ruvector-wasm/clustering/cloudflare.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/crates/ruvector-wasm/clustering/ivf.d.mts b/crates/ruvector-wasm/clustering/ivf.d.mts new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/crates/ruvector-wasm/clustering/ivf.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/crates/ruvector-wasm/clustering/ivf.mjs b/crates/ruvector-wasm/clustering/ivf.mjs new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/crates/ruvector-wasm/clustering/ivf.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/crates/ruvector-wasm/clustering/metaharness.d.mts b/crates/ruvector-wasm/clustering/metaharness.d.mts new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/crates/ruvector-wasm/clustering/metaharness.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/crates/ruvector-wasm/clustering/metaharness.mjs b/crates/ruvector-wasm/clustering/metaharness.mjs new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/crates/ruvector-wasm/clustering/metaharness.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/crates/ruvector-wasm/clustering/quantization.d.mts b/crates/ruvector-wasm/clustering/quantization.d.mts new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/crates/ruvector-wasm/clustering/quantization.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/crates/ruvector-wasm/clustering/quantization.mjs b/crates/ruvector-wasm/clustering/quantization.mjs new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/crates/ruvector-wasm/clustering/quantization.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/crates/ruvector-wasm/package.json b/crates/ruvector-wasm/package.json index bd16bae8d1..0fb4030afd 100644 --- a/crates/ruvector-wasm/package.json +++ b/crates/ruvector-wasm/package.json @@ -13,6 +13,26 @@ "./adapter": { "types": "./src/adapter.d.ts", "default": "./src/adapter.js" + }, + "./clustering": { + "types": "./clustering.d.mts", + "import": "./clustering.mjs" + }, + "./clustering/quantization": { + "types": "./clustering/quantization.d.mts", + "import": "./clustering/quantization.mjs" + }, + "./clustering/ivf": { + "types": "./clustering/ivf.d.mts", + "import": "./clustering/ivf.mjs" + }, + "./clustering/metaharness": { + "types": "./clustering/metaharness.d.mts", + "import": "./clustering/metaharness.mjs" + }, + "./clustering/cloudflare": { + "types": "./clustering/cloudflare.d.mts", + "import": "./clustering/cloudflare.mjs" } }, "files": [ @@ -21,7 +41,10 @@ "src/adapter.d.ts", "src/worker.js", "src/worker-pool.js", - "src/indexeddb.js" + "src/indexeddb.js", + "clustering.mjs", + "clustering.d.mts", + "clustering/" ], "scripts": { "build": "npm run build:web && npm run build:simd && npm run build:bundler", @@ -59,6 +82,9 @@ "bugs": { "url": "https://github.com/ruvnet/ruvector/issues" }, + "dependencies": { + "@ruvector/e-means": "0.1.0" + }, "devDependencies": { "wasm-pack": "^0.12.1" } diff --git a/docs/adr/ADR-359-e-means-shared-clustering.md b/docs/adr/ADR-359-e-means-shared-clustering.md new file mode 100644 index 0000000000..cff693fa71 --- /dev/null +++ b/docs/adr/ADR-359-e-means-shared-clustering.md @@ -0,0 +1,251 @@ +# ADR-359: Shared e-means clustering for RuVector and ruv-FANN + +- **Status**: Accepted; local implementation, registry publication pending +- **Date**: 2026-11-11 +- **Source**: Bessa et al., [e-means: outperforming k-means (in one-tenth the epochs)](https://arxiv.org/abs/2611.09501), [version 1](https://arxiv.org/html/2611.09501v1), 2026-11-06 +- **Related**: [ruv-FANN ADR-013](https://github.com/ruvnet/ruv-FANN/blob/main/docs/adr/ADR-013-e-means-clustering.md) + +## Context and scope + +RuVector trains product-quantization codebooks with Lloyd k-means. ruv-FANN +consumers also need unsupervised clustering of input embeddings without changing +their supervised labels or neural optimizers. Maintaining independent clustering +implementations for Rust and JavaScript would make numerical fixes and paper +conformance diverge. + +The paper proposes mini-batch EMA sufficient statistics plus a particular +initialization, schedule, cluster revival, and finalization recipe. Its reported +GPU results motivate implementation, but do not establish performance of this CPU +implementation. Its fitted hyperparameters are less reliable for tiny and +low-dimensional data, including many PQ subspaces. This release therefore adds +explicit e-means APIs while preserving existing training defaults and persisted +codebook formats. + +## Decision and contracts + +Create `ruvector-e-means` 0.1.0 as a separately publishable, MIT-licensed crate in +`crates/ruvector-e-means`, registered in the workspace. The native core requires +Rust 1.81, has no default features, and has no dependency on either consumer. +`serde` adds config/model serialization. `wasm` adds a thin wasm-bindgen bridge. +Pin wasm-bindgen and its build CLI to the existing workspace version 0.2.125 so +this feature does not upgrade other packages' WASM dependency stack. + +The public core API is `fit(flat_f32_rows, dimensions, &Config) -> Result` +and exact `predict`. `Config` exposes clusters, epochs (default 3), optional batch +and peak-step overrides, seed (default 42), and finalization. `Model` returns flat +centroids, assignments, dimensions, squared-L2 MSE per vector, resolved batch and +step, update count, and number of revived clusters. Assignments are recomputed +against the returned centroids in original row order. Public/deserialized +codebooks are validated when passed to prediction. + +Implement the paper's primary recipe: + +1. Seeded ChaCha8 sampling without replacement for initial centroids; initialize + counts to `1e-10` and sums to count times centroid (equation 4). +2. Resolve `B = 0.035 k^0.13 n^0.88 T^0.81`, divided by eight when + `n > 2,000,000`, rounded to the nearest power of two and capped at `n`. + Resolve `alpha0 = min(0.824 k^-0.46 B^0.90 n^-0.45 T^-0.11, 1)` using + the effective batch size (equations 6–7, table 2). +3. Shuffle each epoch. Assign rows by exact squared L2 and accumulate per-cluster + batch counts/sums. Update every cluster's statistics with equation 2 and + the cosine step from equation 5, including its 0.1 floor and `t/M` denominator. +4. Every five batches revive counts below 0.1 times the mean count. Draw live + donors proportional to count, without replacement. Split count in half and + perturb child coordinates by alternating `1 +/- 1/128` (Appendix H). +5. Finalize using the mean of assignments recorded during the last epoch, or + return the main EMA centroids. Recompute nearest assignments and MSE in a + final exact pass. + +Use f64 sums, counts, and distances, and f32 codebooks. Retain a negligible +pseudo-count for a temporarily empty cluster when a step of one would otherwise +erase its state. Clamp extreme split coordinates to finite f32. Final clusters +without recorded rows retain their EMA centroid. If dead slots outnumber live +donors, remaining slots wait until a later revival pass. + +Two deliberate execution differences from the printed algorithm are explicit: +include short tail batches rather than dropping rows, with +`M = T * ceil(n/B)`; and recompute final nearest assignments/MSE rather than +reporting the loss of stale last-epoch assignments. Last-epoch mean is an average +of assignments made while centroids moved, not a separate Lloyd iteration, so we +do not promise monotonic improvement over the final EMA model. + +`@ruvector/e-means` 0.1.0 in `npm/packages/e-means` ships that same Rust core as +portable WebAssembly, typed ESM APIs, and the matching generated loader. +`await init()` loads bundled bytes in Node and fetches them in a browser; +`fit(Float32Array, dimensions, options)` and `predict` run synchronously after +initialization. No JavaScript numerical fallback or Rust installation is needed +by an installed consumer. Training calls are bounded by caller-selected sizes +and epochs and can block the caller; applications should use workers for long +runs. The Rust crate itself does not spawn threads or fetch datasets. + +## Consumer integration and release order + +- RuVector core directly depends on the new crate and exposes + `ProductQuantized::train_e_means(vectors, num_subspaces, &Config)`. It trains + each subspace with the shared implementation and retains u8 codes and the + current codebook serialization. f64 encode comparisons preserve nearest + ordering for large finite f32 coordinates. +- The main `ruvector` npm package depends on exact `@ruvector/e-means` 0.1.0 and + ships `ruvector/clustering.mjs` with matching ESM declarations. This explicit + subpath avoids eagerly initializing WASM or restricting legacy package + subpaths with a new exports map. +- ruv-FANN has an optional `clustering` Cargo feature with a version-only + dependency on `ruvector-e-means` 0.1.0, re-exported types/functions, and + `TrainingData::cluster_inputs`. It clusters inputs without mutating inputs + or outputs. Its npm facade depends on exact `@ruvector/e-means` 0.1.0 and + exposes `ruv-fann/clustering` without changing the root neural API. + +Publish the new crate and npm package before releasing dependent consumer +versions. Before registry publication, qualify ruv-FANN with an explicit local +Cargo patch and install the packed npm dependency in isolated consumer tests: + +```sh +cd ../ruv-FANN +cargo test --features clustering --test clustering_tests \ + --config 'patch.crates-io.ruvector-e-means.path="../ruvector/crates/ruvector-e-means"' +``` + +No sibling checkout path is committed in ruv-FANN's published dependency +contract. Its version-only dependency cannot resolve from a fresh registry +checkout until the new crate is published, even with the feature disabled. +Local lockfile qualification with a patch does not prove registry availability. +Do not claim publication based on a successful local build or archive check. + +## Alternatives and consequences + +- Copying the algorithm into both projects and npm would multiply maintenance + and correctness risk. A single Rust implementation compiled to WASM keeps + numerical behavior shared. +- Replacing all Lloyd defaults immediately risks quality regressions for small + PQ subspaces outside the scaling laws' fitted domain. Explicit APIs allow + evaluation with real embeddings and consumer-specific recall/latency gates. +- A GPU/PyTorch/Faiss runtime dependency would add substantial deployment cost. + This CPU reference and portable WASM establish the contract first. GPU, + low-precision, distributed/IVF assignment, slow-EMA finalization, arbitrary + streaming updates, and wholesale replacement of all clustering call sites are + outside this release. + +The algorithm stores O(n + kd) scratch state and performs O(Tnkd) exact-distance +work plus a final O(nkd) scoring pass. Training is synchronous. Seeded shuffles +are stable across native/WASM; platform transcendental routines can introduce +small floating-point differences, so cross-target numerical checks use an +appropriate tolerance rather than a universal bit-identity claim. + +## Acceptance and validation + +| Requirement | Executable evidence | +|---|---| +| Paper coefficients, cosine schedule, weighted mass-preserving revival | `cargo test -p ruvector-e-means` | +| Mean/MSE oracle, all tail rows visited, reproducibility, separated clouds | crate `tests/clustering.rs` | +| Empty/ragged/nonfinite shapes, invalid configs, overflow, duplicate/zero/extreme inputs | crate and consumer Rust tests; npm WASM tests | +| PQ representation, serialization, all 256 u8 codes, extreme nearest comparison | `cargo test -p ruvector-core --no-default-features --test e_means_quantization` | +| ruv-FANN adapter and unchanged supervised state | patched `clustering_tests` plus existing library tests | +| Same Rust core used through typed npm facade | `npm test` and `npm run typecheck` in the new package; consumer facade tests | +| Complete installable npm artifact without repository paths | `npm run test:consumer` in the new package | +| Browser default fetch/initialization | new package's headless browser consumer test | + +Build WASM with `npm run build` in the new package, using the pinned CLI. Its +prepack check executes the shipped artifact. Review and run the targeted checks +before release; this ADR does not certify the entire monorepo, GPU speedups, or +universal quality superiority. Rollback means use the retained Lloyd/neural APIs +or stop importing the clustering subpaths; existing persisted codebooks need no +migration. + +### Local qualification on 2026-11-11 + +- The shared crate passed 10 native tests with all features and strict Clippy. + `cargo package --allow-dirty` built and verified the archive. Its native core + also passed `cargo +1.81.0 check --locked` against the packaged manifest. +- Actual RuVector core passed 6 new integration tests and 36 existing + quantization tests. ruv-FANN passed 3 adapter tests and 207 existing library + tests with the local dependency patch. +- The final wasm-bindgen 0.2.125 artifact passed 9 Node numerical/API tests, + offline installed-archive qualification, strict TypeScript, and both consumer + facade checks. Packed ruv-FANN was also installed with the packed dependency + and exercised through its public subpath. +- Browser execution remains unverified here: the browser consumer timed out, + and independent Chromium `about:blank` startup also timed out, including a + single-process baseline. The checked-in browser test fails on that timeout; + no skip or simulated success is counted as a pass. Re-run it in a working + Chromium environment before claiming browser qualification. + +These are focused local checks. Neither registry publication nor full-monorepo +qualification was performed. + +## Optimization qualification + +Exact partial-distance pruning checks candidates every eight coordinates for +dimensions above 16, while small codebooks retain scalar distance evaluation. +Zero distance ends the search. These changes retain the scalar addition order +and lowest-index ties. Independent exhaustive-oracle tests cover diffuse and +separated inputs, extreme/subnormal coordinates, near ties, and dimensions with +incomplete final blocks. The optimization does not add approximate assignment +or change seeded shuffles, sufficient statistics, or finalizers. + +Training uses sliced centroid rows, reuses dead/donor scratch capacity, skips +donor-list construction when no cluster is dead, and omits last-epoch statistics +for EMA finalization. PQ encoding computes each candidate distance once. The +WASM wrapper keeps JSON for the small config and returns vectors as owned typed +arrays, releasing its temporary Rust handle in `finally`. Tests check repeated +fits, error-path cleanup, and output survival after WASM memory growth. + +The author's run on an AMD EPYC 9V74 environment, with five interleaved native +repetitions, recorded these median fit times for last-epoch-mean finalization. +The "Before" build is a pre-optimization version of this crate that is not in +the repository, so these ratios cannot be re-run from the tree: + +| Synthetic input (n × d, clusters) | Before | After | Speedup | +|---|---:|---:|---:| +| Clustered embeddings (4096 × 128, 64) | 130.05 ms | 70.57 ms | 1.84× | +| Diffuse embeddings (2048 × 768, 32) | 203.61 ms | 177.48 ms | 1.15× | +| Duplicate/zero rows (2048 × 32, 64) | 13.35 ms | 2.53 ms | 5.29× | + +All model fields and predictions matched the baseline bit for bit in eight +native scenarios. A 21-repeat 8D PQ follow-up measured 51.25→52.68 ms with +last-epoch mean and 50.28→50.83 ms with EMA; no PQ speedup is claimed. EMA mode +on the 768D fixture reduced peak additional requested heap from 721,664 to +525,312 bytes. Last-epoch-mean mode may retain a little more scratch capacity; +allocation-event reduction is distinct from peak-memory reduction. + +In the same author run, Node 24 WASM median-of-seven fit times were 36.19→15.00 ms (2.41×) for an +8192 × 64, 16-cluster fixture and 84.09→61.57 ms (1.37×) for one million +scalar rows with one cluster. Both match baseline centroids, assignments and +MSE. Comparing JSON and typed output using the same optimized kernel measured +75.97→58.41 ms on the million-row fixture; this includes the whole fit and +materialization, not just conversion time. + +Independent review reproduction (2026-11-11, ruvultra, Rust 1.99.1 with the +workspace release profile for both sides, one pinned core, five interleaved +rounds of five repetitions). The baseline is this tree with partial-distance +pruning disabled (`nearest` always takes the scalar path), so it isolates the +pruning change only. It does not include the other optimizations above: + +| Synthetic input (n × d, clusters) | Pruning off | Pruning on | Speedup | +|---|---:|---:|---:| +| Clustered embeddings (4096 × 128, 64) | 57.55 ms | 36.40 ms | 1.58× | +| Diffuse embeddings (2048 × 768, 32) | 97.95 ms | 98.30 ms | 1.00× | +| 8D PQ (8192 × 8, 256) | 28.83 ms | 28.80 ms | 1.00× | +| Duplicate/zero rows (2048 × 32, 64) | 3.65 ms | 1.44 ms | 2.54× | + +The npm `benchmark` 8192 × 64, 16-cluster fixture measured 17.0→7.5 ms +(2.26×) against a pruning-off WASM build. All eight native digests +(centroid bits and assignments) equal both the pruning-off build and the +digests recorded in the measurement artifact, and the WASM fixtures matched +with zero label or centroid differences. The shipped `.wasm` rebuilds +byte-for-byte from this source with Rust 1.99.1 and wasm-bindgen 0.2.125 +(`--remap-path-prefix` set to the author's home directory). Treat the author +ratios above as that environment's results against an earlier build, and the +reproduction as the checked effect of pruning. + +The [measurement artifact](../research/e-means/performance-2026-11-11.json) +records shapes, options, raw native samples, environment, allocation definitions, +and source/artifact checksums. The native benchmark example and npm `benchmark` +command reproduce current measurements. These synthetic CPU/WASM results do not +establish speedups on arbitrary datasets or the paper's GPU benchmarks. Browser +qualification and registry publication remain pending. + +Post-optimization validation passed 14 shared-crate tests, 6 PQ integration +tests, 20 existing core quantization tests, 207 ruv-FANN library tests and its +3 adapter tests. The npm runtime passed 13 numerical/ownership tests, installed +archive checks, both consumer facades and strict TypeScript. Strict Clippy, +crate archive verification, and the packaged native Rust 1.81 check also passed. diff --git a/docs/adr/ADR-360-e-means-ecosystem-integrations.md b/docs/adr/ADR-360-e-means-ecosystem-integrations.md new file mode 100644 index 0000000000..beca468bbf --- /dev/null +++ b/docs/adr/ADR-360-e-means-ecosystem-integrations.md @@ -0,0 +1,184 @@ +# ADR-360: e-means ecosystem submodules, MetaHarness, and Cloudflare Workers + +- **Status**: Accepted; local integration implemented, publication and deployment pending +- **Date**: 2026-11-11 +- **Related**: [ADR-359](./ADR-359-e-means-shared-clustering.md) + +## Outcome and acceptance contract + +Make the shared e-means implementation usable for existing PQ/IVF indexes, +neural-input workflows, measured MetaHarness tuning, and Cloudflare Workers. +Integrations must invoke the real Rust/WASM kernel, preserve current training +defaults, expose explicit typed entrypoints, and run from installed npm archives. +Here, submodules mean Rust modules and npm export subpaths; no Git submodule +checkout is required. + +Success requires numerical encode/decode and posting-list oracles; checked +restoration of persisted records; actual Darwin evolution with held-out scoring; +authenticated, resource-bounded Worker execution with statically imported WASM; +and real local workerd/R2 qualification. Hosted deployment, registry publication, +and live Vectorize mutation are outside this implementation request. + +## Module boundaries + +| Surface | Interface and responsibility | +|---|---| +| Rust `quantization` | `ProductQuantizer::fit`, checked `from_centroids`, byte `encode`/`decode`, private shape fields, validated serde restoration | +| Rust `ivf` | `IvfPartition::fit`, checked `from_model`/`from_parts`, stable offsets/indices, checked `posting` access | +| npm `/quantization` | `trainProductQuantizer`, `encodeProductQuantizer`, `decodeProductQuantizer`, validated owned JSON conversions | +| npm `/ivf` | `partition` and `postingList`, with owned typed arrays and stable input-row IDs | +| npm `/runtime` | Node-free `init(source)`, `fit`, `predict`; requires explicit bytes or a precompiled module | +| npm `/metaharness` | `createClusteringEvaluator` and bounded Node-only `tuneClustering`, using the actual Darwin numeric API | +| npm `/cloudflare` | Stateless Worker factory and explicit R2/Vectorize binding helpers | +| npm `/cloudflare/build` | Node-only local static-WASM builder, using esbuild; never deploys | +| npm `/wasm` | Raw artifact export for Wrangler's static WebAssembly import | + +PQ decomposition and posting construction wrap the shared fitter and nearest +assignment. There is no second JavaScript clustering optimizer. All PQ codes are +row-major, one u8 per subspace; centroids are subspace-major. Rust and JavaScript +use the same numeric layout, but JS JSON has an explicit schema envelope. Remove +that envelope before native serde deserialization, or construct the native +quantizer with its checked `from_centroids` API. Do not assume arbitrary JSON +records from the two APIs are interchangeable. + +## Actual consumers + +- `ruvector-core::ProductQuantized::train_e_means` remains available from ADR-359. +- `ruvector-pq-search` adds `PqCodebook::train_e_means`, + `IvfPqIndex::new_e_means`, and checked insertion/query wrappers. Existing PQ + layouts and ADC search behavior remain in place. +- `ruvector-rairs` adds explicit training to `IvfFlat`, `RairsStrict`, and + `RairsSeil`. Invalid training leaves state intact. Successful explicit + retraining resets inserted vectors and IDs, as documented by these methods. +- `@ruvector/core`, `ruvector`, and `@ruvector/all` ship `clustering.mjs` and + `clustering/{quantization,ivf,metaharness,cloudflare}.mjs`, with `.d.mts` + declarations. Their existing package subpaths remain accessible. +- The current `@ruvector/wasm` source at `crates/ruvector-wasm` exposes + `/clustering` and `/clustering/{quantization,ivf,metaharness,cloudflare}`. + Historical same-name manifest copies are not release sources for this change. +- `ruv-fann` and `@ruv-fann/integrations` expose corresponding clustering + subpaths. Existing WSP Cloudflare and neural MetaHarness entrypoints retain + their own meanings. + +Each npm consumer depends on exact `@ruvector/e-means` 0.1.0. It is an optional +dependency for `@ruvector/core`, whose legacy entrypoint supports Node 18; +clustering entrypoints require the shared package's Node 20 runtime. When optional +dependencies are omitted, install the shared package explicitly. The new modules +do not eagerly initialize WASM or import Darwin during ordinary SDK imports. + +Shared modules use f64 assignment arithmetic. Existing downstream ADC/scanning +APIs retain their existing f32 precision and legacy validation behavior; this +integration does not certify those algorithms for extreme f32 magnitudes or +silently replace their defaults. + +## MetaHarness control flow and evidence + +The evaluator synchronously copies training/holdout arrays and snapshots the +baseline, domain, and budgets before its first await. Holdout rows must be +nonempty and nonoverlapping with training rows. Each dispatch validates the +genome, reserves its work allowance, executes real WASM training, and separately +predicts and scores the holdout. Higher-is-better `primary = 1/(1+holdoutMse)` +matches Darwin's score contract. Records include actual training/holdout MSE, +fit/prediction timings, selected configuration, and resource estimates. + +Only one evaluation runs at a time, with at most 32 admitted dispatches, +64 MiB of input snapshots, and a caller-selected allowance capped at 1e9 +coordinate-scale work units. Admission includes centroid updates, tail batches, +and quadratic donor selection. Units are conservative estimates, not measured +instructions or a process-memory limit. Deadlines are cooperative; synchronous +fits and arbitrary host operations cannot be preempted. + +`tuneClustering` dynamically loads exact `@metaharness/darwin` 0.13.0, validates +the installed identity/API, and calls its real `evolveNumeric` at concurrency 1. +It checks worst-case search cost before execution and matches returned archive +records to observed evaluations. Regressed/unmeasured winners cannot replace +the baseline recommendation. A temporary archive is removed in `finally`. +Receipts and returned scores are independent copies; `sourceApplied` is always +false. The holdout guides selection and is not an independent generalization +test. SDK version checking is metadata validation, not binary attestation. + +The SDK is an optional peer, loaded only when tuning is invoked. Package tests +exercise both an installation without that peer and actual evolution with the +pinned peer installed. + +## Cloudflare execution and authority boundaries + +Wrangler imports `.wasm` as `WebAssembly.Module`; the Worker passes that static +module to the Node-free runtime. No Node filesystem loader, downloaded model, +runtime source compilation, or `instantiateStreaming` path is needed. + +The Worker exposes public `/health` and authenticated POST `/fit` and `/predict`. +Operations fail closed when no token is configured, except an explicit trusted +local `allowUnauthenticated` opt-in. The example reads `E_MEANS_BEARER_TOKEN` +from the caller's binding configuration; credentials are not packaged. + +Before entering WASM, the handler reads bounded UTF-8 JSON with a stream deadline +and validates exact fields, finite float32 values, shapes, options, and work. +Absolute caps include 1 MiB request bodies, 262,144 elements, 16,384 rows, +1,536 dimensions, 256 clusters, 32 epochs, and 1e8 estimated work units; defaults +are smaller. The work estimate includes updates and donor revival. These are +per-call bounds, not a guarantee against aggregate isolate exhaustion or a hard +CPU-time cap. Request vectors/models are not kept in global mutable state, and +HTTP responses expose no unexpected stack traces. + +Storage operations are explicit helper calls on caller-owned bindings. HTTP +fitting does not automatically store or export anything: + +- R2 stores only bounded centroid codebooks, not source vectors or assignments. + Writes require an acknowledgement; reads validate streamed bytes and schema + before constructing owned numeric arrays. PQ schema records are rejected + because their subspace-major layout is not a single centroid set. +- Vectorize V2 exports validate the entire snapshot and identifier/namespace + byte lengths before the first upsert. Batches are limited to 1,000 vectors + and dimensions to 1,536. Every acknowledgement must carry a mutation ID. + An acknowledgement does not prove immediate query visibility. +- Export failures report prior acknowledged rows/mutation IDs. The failed batch + may also have applied; no transaction, rollback, retry, or deletion is implied. + Use a versioned prefix/namespace for revised codebooks, since upserts do not + remove stale IDs from an older, larger codebook. + +Official Cloudflare references: [WASM bundling](https://developers.cloudflare.com/workers/wrangler/bundling/), +[Vectorize limits](https://developers.cloudflare.com/vectorize/platform/limits/), +[asynchronous mutations](https://developers.cloudflare.com/vectorize/get-started/intro/). +The connected Cloudflare documentation tool was used to verify these contracts. + +## Packaging, validation, and rollback + +Use the isolated package lock in `npm/packages/e-means` for integration tooling. +Miniflare 4.20260730.0 and esbuild 0.28.2 are development-only. Scoped overrides +select patched sharp 0.35.5 and undici 7.30.1; re-run workerd tests and the +lockfile-aware audit when updating them. The workspace lock records this tool +graph beneath the e-means workspace without upgrading unrelated dependency +versions. Whole-workspace installation remains separately constrained by the +repository's existing platform-specific workspace packages. + +| Requirement | Check | +|---|---| +| Native layout, persisted-state validation, real index consumers | `cargo test -p ruvector-e-means --features serde -p ruvector-pq-search -p ruvector-rairs` | +| Existing shared core and optional WASM bridge | `cargo test -p ruvector-e-means --all-features` | +| JS numerical, ownership, holdout, budget and HTTP boundaries | `npm test` in the shared package | +| Installed package without/with actual optional Darwin SDK | `npm run test:consumer` | +| New entrypoints from actual consumer archives | `npm run test:ecosystem` | +| Real static WASM and R2 in local workerd | `npm run test:cloudflare` | +| Public declarations | `npm run typecheck` and installed consumer checks | +| Newly selected tool dependency graph | `npm audit --package-lock-only --workspaces=false` | + +Consumer archive tests qualify the new clustering subpaths. Their unrelated +legacy root/prepack release builds are not qualified by this task. Vectorize +has binding-contract tests, not a claimed hosted index write. No Cloudflare +deployment or registry publication is inferred from passing local checks. + +Release the shared crate/package first, bump affected consumer release versions, +and verify registry installs before publishing consumers. Local ruv-FANN Rust +checks use the explicit Cargo patch from ADR-359. Rollback means stop using the +new feature methods or subpaths; existing codebooks, neural APIs, and search +defaults require no migration. Storage helpers never delete external resources. + +Local qualification passed 50 targeted Rust tests, 44 shared npm tests, 8 FANN +facade tests, and 3 Rust FANN adapter tests, plus strict declarations, Clippy, +archive verification and the packaged Rust 1.81 check. Installed consumers +exercised the optional-SDK absence path, actual Darwin evolution, all four +RuVector facade packages, and static-WASM/R2 execution in real local workerd. +The isolated integration-tool lockfile audit reported zero known advisories +after the scoped overrides. [Qualification evidence](../research/e-means/integrations-qualification.json) +records versions, scope limits, and current lock/artifact digests. diff --git a/docs/adr/INDEX.md b/docs/adr/INDEX.md index 798d610c43..bbd221b0e4 100644 --- a/docs/adr/INDEX.md +++ b/docs/adr/INDEX.md @@ -1,6 +1,6 @@ # ADR Index -**Next available ADR number: 350** +**Next available ADR number: 361** > Generated by `node scripts/adr-index.mjs` — do not edit by hand. > This file is the canonical allocation counter for new ADR numbers @@ -8,8 +8,8 @@ > historical artifacts and are cited as `ADR-NNN (slug)`. > CI gate: `node scripts/adr-index.mjs --check`. -- ADR files indexed: **376** (329 on the canonical counter, 47 in namespaced families) -- Highest allocated number: **ADR-349** +- ADR files indexed: **388** (341 on the canonical counter, 47 in namespaced families) +- Highest allocated number: **ADR-360** - Frozen duplicate numbers: **27** (spanning 61 files) | Number | Title | File | Last commit | Status | Duplicate | @@ -345,7 +345,16 @@ | ADR-345 | ADR-345: Mincut-Gated Forgetting — Structural Eviction Signal and Eviction Witnesses for Agent Memory | [`ADR-345-mincut-gated-forgetting.md`](./ADR-345-mincut-gated-forgetting.md) | 2026-09-05 | Rejected (for production use as designed). Experimental crate addition | | | ADR-346 | ADR-346: Deterministic, Non-Degenerate Witness Partitions in `ruvector-mincut` | [`ADR-346-deterministic-mincut-witness-partition.md`](./ADR-346-deterministic-mincut-witness-partition.md) | 2026-09-26 | Accepted. Non-breaking bug fix, merged into `ruvector-mincut` directly (no | | | ADR-347 | ADR-347: Ed25519 `WitnessSigner` for the TARL Ledger's Witness Chain | [`ADR-347-witness-signer-tarl-ledger.md`](./ADR-347-witness-signer-tarl-ledger.md) | 2026-09-16 | Accepted. New, opt-in, non-default module | | -| ADR-349 | ADR-349: pi.ruv.io — Three Independent 504 Mechanisms, a UTF-8 Crash, and the Unauthenticated Write Path | [`ADR-349-ruvbrain-504-mechanisms-and-write-path-auth.md`](./ADR-349-ruvbrain-504-mechanisms-and-write-path-auth.md) | 2026-09-26 | Accepted (code landed) | | +| ADR-349 | ADR-349: pi.ruv.io — Three Independent 504 Mechanisms, a UTF-8 Crash, and the Unauthenticated Write Path | [`ADR-349-ruvbrain-504-mechanisms-and-write-path-auth.md`](./ADR-349-ruvbrain-504-mechanisms-and-write-path-auth.md) | 2026-09-26 | Accepted (code landed; item 6 fixed by P5 in a follow-up). Three items are delib | | +| ADR-351 | ADR-351: ruvector edge — Multi-Tenant Rust/wasm Services on Cloudflare Workers + Durable Objects, Behind an Edge Authorization Server Federated to cognitum-one OAuth | [`ADR-351-ruvector-edge-multitenant-services.md`](./ADR-351-ruvector-edge-multitenant-services.md) | 2026-11-10 | **Proposed.** 2026-09-28; reconciled 2026-09-29 with the edge-authorization-serv | | +| ADR-352 | ADR-352: ruvector Python SDK, CLI, and MCP Server (PyO3 core, SOTA scope) | [`ADR-352-ruvector-python-sdk-cli-mcp.md`](./ADR-352-ruvector-python-sdk-cli-mcp.md) | 2026-11-10 | **Proposed, with M1 implemented and verified.** 2026-10-01. Supersedes the scope | | +| ADR-353 | ADR-353: RVF Stream Descriptor (RVSD) — verified streaming delivery (project Ofer) | [`ADR-353-rvf-stream-descriptor.md`](./ADR-353-rvf-stream-descriptor.md) | 2026-11-10 | Proposed (implemented on `feat/ofer-core`; section 10 records where the code ref | | +| ADR-354 | ADR-354: RVF dialect convergence and an effective Strict policy (project Ofer prerequisites) | [`ADR-354-rvf-dialect-convergence-and-strict-policy.md`](./ADR-354-rvf-dialect-convergence-and-strict-policy.md) | 2026-11-10 | Proposed | | +| ADR-355 | ADR-355: ruvector-weaver, a self-improving harness for the ruvnet package fabric | [`ADR-355-ruvector-weaver-ecosystem-fabric.md`](./ADR-355-ruvector-weaver-ecosystem-fabric.md) | 2026-11-10 | Accepted | | +| ADR-356 | ADR-356: Portfolio distribution and SEO standard for the ruvnet npm packages | [`ADR-356-portfolio-distribution-seo.md`](./ADR-356-portfolio-distribution-seo.md) | 2026-11-10 | Accepted | | +| ADR-358 | ADR-358: @ruvector/weight-space, a versioned ruv-fann weight registry indexed by RuVector | [`ADR-358-weight-space-ruv-fann-registry.md`](./ADR-358-weight-space-ruv-fann-registry.md) | 2026-11-10 | Accepted | | +| ADR-359 | ADR-359: Shared e-means clustering for RuVector and ruv-FANN | [`ADR-359-e-means-shared-clustering.md`](./ADR-359-e-means-shared-clustering.md) | | Accepted; local implementation, registry publication pending | | +| ADR-360 | ADR-360: e-means ecosystem submodules, MetaHarness, and Cloudflare Workers | [`ADR-360-e-means-ecosystem-integrations.md`](./ADR-360-e-means-ecosystem-integrations.md) | | Accepted; local integration implemented, publication and deployment pending | | | ADR-CE-001 | ADR-CE-001: Sheaf Laplacian Defines Coherence Witness | [`coherence-engine/ADR-CE-001-sheaf-laplacian-coherence.md`](./coherence-engine/ADR-CE-001-sheaf-laplacian-coherence.md) | 2026-08-20 | Accepted | | | ADR-CE-002 | ADR-CE-002: Incremental Coherence Computation | [`coherence-engine/ADR-CE-002-incremental-computation.md`](./coherence-engine/ADR-CE-002-incremental-computation.md) | 2026-08-20 | Accepted | | | ADR-CE-003 | ADR-CE-003: PostgreSQL + Ruvector Unified Substrate | [`coherence-engine/ADR-CE-003-hybrid-storage.md`](./coherence-engine/ADR-CE-003-hybrid-storage.md) | 2026-08-20 | Accepted | | diff --git a/docs/research/e-means/integrations-qualification.json b/docs/research/e-means/integrations-qualification.json new file mode 100644 index 0000000000..4cfee7cddd --- /dev/null +++ b/docs/research/e-means/integrations-qualification.json @@ -0,0 +1,70 @@ +{ + "observedAt": "2026-11-11T17:36:16.810277+00:00", + "scope": "New e-means modules and explicit ecosystem entrypoints; no registry publication, Cloudflare deployment, or hosted Vectorize mutation.", + "rust": { + "targetedTestsPassed": 50, + "packages": [ + "ruvector-e-means (all features)", + "ruvector-pq-search", + "ruvector-rairs" + ], + "strictClippyAllTargets": "passed", + "crateArchiveVerification": "passed", + "packagedNativeRust181Check": "passed" + }, + "npm": { + "sharedTestsPassed": 44, + "sharedFailures": 0, + "sharedSkips": 0, + "strictTypes": "passed", + "optionalSdkAbsentInstalledConsumer": "passed", + "realDarwinInstalledConsumer": "passed", + "newRuVectorArchiveEntryPoints": [ + "@ruvector/core/clustering.mjs", + "ruvector/clustering.mjs", + "@ruvector/all/clustering.mjs", + "@ruvector/wasm/clustering" + ], + "newRuVectorArchiveEntryPointsStatus": "passed; unrelated legacy root/prepack release builds excluded", + "fannSourceFacadeTestsPassed": 8, + "fannRustAdapterTestsPassed": 3 + }, + "cloudflare": { + "localWorkerd": "passed with static Rust WASM and no nodejs_compat", + "localR2BindingRoundtrip": "passed", + "installedArchiveWorker": "passed", + "vectorize": "binding-contract fixtures only; no hosted mutation", + "schemaIteratorAuthBodyDeadlineAndWorkBoundRegressions": "passed" + }, + "metaharness": { + "package": "@metaharness/darwin", + "version": "0.13.0", + "algorithm": "actual evolveNumeric", + "holdout": "Separate nonoverlapping caller-supplied snapshot; used for selection, not independent generalization evidence", + "sourceApplied": false, + "temporaryArchiveCleanup": "passed" + }, + "dependencyAudit": { + "scope": "Isolated canonical integration-tool package lock", + "counts": { + "info": 0, + "low": 0, + "moderate": 0, + "high": 0, + "critical": 0, + "total": 0 + }, + "miniflare": "4.20260730.0", + "scopedOverrides": { + "sharp": "0.35.5", + "undici": "7.30.1" + }, + "workspaceInstallLimitation": "Existing Darwin-only workspace package prevents a whole-workspace Linux npm install; isolated lock qualification is used." + }, + "sha256": { + "npm/packages/e-means/package-lock.json": "d4b066d2503c58c92927fac0872180e3f1cc5d9f3e2e9665ddd5d2fdf717d25b", + "npm/package-lock.json": "4d80b3ccec4d52dbbfc5191062cc57397b8f6aa0ff24e7211c45cdb06c3fa1b0", + "Cargo.lock": "45f6253a219ae8446ff24b4adc27061e18ea43b9e627f8c55756f8983f81fcf4", + "npm/packages/e-means/wasm/ruvector_e_means_bg.wasm": "2c8f8a21e89f7e809a02d967bb77935284f35115855ed8024fae1a9453f27b5b" + } +} diff --git a/docs/research/e-means/performance-2026-11-11.json b/docs/research/e-means/performance-2026-11-11.json new file mode 100644 index 0000000000..921bcd5632 --- /dev/null +++ b/docs/research/e-means/performance-2026-11-11.json @@ -0,0 +1,406 @@ +{ + "date": "2026-11-11", + "scope": "Local synthetic CPU/WASM measurements against the pre-optimization implementation from this task; not paper GPU results or a production-data guarantee.", + "nativeEnvironment": { + "rustc": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "system": "Linux-6.18.57-x86_64-with-glibc2.41", + "cpu": "AMD EPYC 9V74 80-Core Processor,virtualized", + "cpuQuota": "400000 100000", + "memoryLimitBytes": "17179869184", + "rustflags": "", + "profile": { + "optLevel": 3, + "lto": "fat", + "codegenUnits": 1 + }, + "repeats": 5, + "warmup": "one fit and predict per implementation,per scenario", + "ordering": "alternate baseline-first and current-first on each timing repetition", + "heapMeasurement": "allocation/reallocation events and peak additional live requested heap bytes during one fit; excludes preexisting dataset and models; notRSS", + "equivalence": "exact centroid f32 bits,assignments,dimension,mse f64 bits,peakstep f64 bits,batchsize,updates,revivedclusters,predictions", + "timingInstrumentation": "Same global tracking System allocator active in both implementations; timings include its atomic allocation/reallocation accounting.", + "pqFollowup": "pq21.jsonl uses same compiled binary,seed42,21interleavedrepetitions;pairedratios reported separately." + }, + "nativeResults": [ + { + "case": "embedding128", + "samples": 4096, + "clusters": 64, + "dimensions": 128, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "lastEpochMean", + "baselineFitMedianMs": 130.051781, + "currentFitMedianMs": 70.574425, + "fitSpeedup": 1.84276, + "baselinePredictMedianMs": 32.324955, + "currentPredictMedianMs": 15.459149, + "predictSpeedup": 2.09099, + "baselineFitSamplesMs": [131.467435, 127.331156, 130.242989, 130.051781, 122.74313], + "currentFitSamplesMs": [70.574425, 79.942167, 70.979476, 66.06521, 65.197374], + "baselinePredictSamplesMs": [37.515317, 33.257138, 32.324955, 32.27568, 31.45154], + "currentPredictSamplesMs": [27.529438, 15.707944, 15.284636, 14.883934, 15.459149], + "baselineAllocations": 65, + "currentAllocations": 18, + "baselinePeakBytes": 296448, + "currentPeakBytes": 297216, + "mse": 0.114103430514, + "revivedClusters": 35, + "digest": "d4c3e4ebdca756ec", + "bitwiseEqual": true + }, + { + "case": "embedding128", + "samples": 4096, + "clusters": 64, + "dimensions": 128, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "ema", + "baselineFitMedianMs": 127.289993, + "currentFitMedianMs": 68.394125, + "fitSpeedup": 1.86112, + "baselinePredictMedianMs": 32.655452, + "currentPredictMedianMs": 16.103738, + "predictSpeedup": 2.02782, + "baselineFitSamplesMs": [123.94487, 127.289993, 140.52791, 129.614864, 123.52662], + "currentFitSamplesMs": [68.394125, 68.124209, 68.149968, 98.981614, 69.446419], + "baselinePredictSamplesMs": [32.222961, 33.500103, 31.648978, 32.655452, 33.376668], + "currentPredictSamplesMs": [15.650256, 16.10431, 16.103738, 16.809221, 15.972451], + "baselineAllocations": 65, + "currentAllocations": 16, + "baselinePeakBytes": 296448, + "currentPeakBytes": 231168, + "mse": 0.117060715656, + "revivedClusters": 35, + "digest": "8fe5489fdb461bac", + "bitwiseEqual": true + }, + { + "case": "embedding768", + "samples": 2048, + "clusters": 32, + "dimensions": 768, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "lastEpochMean", + "baselineFitMedianMs": 203.612263, + "currentFitMedianMs": 177.482473, + "fitSpeedup": 1.14722, + "baselinePredictMedianMs": 52.71644, + "currentPredictMedianMs": 43.645892, + "predictSpeedup": 1.20782, + "baselineFitSamplesMs": [204.457404, 198.054146, 219.330426, 203.612263, 183.144237], + "currentFitSamplesMs": [177.8861, 177.482473, 182.109762, 173.68051, 169.356138], + "baselinePredictSamplesMs": [53.80124, 47.364421, 49.976874, 52.71644, 53.603404], + "currentPredictSamplesMs": [43.804434, 43.151491, 43.949389, 43.645892, 43.350748], + "baselineAllocations": 62, + "currentAllocations": 17, + "baselinePeakBytes": 721664, + "currentPeakBytes": 722176, + "mse": 0.973645490345, + "revivedClusters": 65, + "digest": "37a4d73d3bf84abb", + "bitwiseEqual": true + }, + { + "case": "embedding768", + "samples": 2048, + "clusters": 32, + "dimensions": 768, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "ema", + "baselineFitMedianMs": 192.918342, + "currentFitMedianMs": 184.687942, + "fitSpeedup": 1.04456, + "baselinePredictMedianMs": 51.914446, + "currentPredictMedianMs": 47.421477, + "predictSpeedup": 1.09475, + "baselineFitSamplesMs": [187.38364, 216.452527, 213.979302, 178.705455, 192.918342], + "currentFitSamplesMs": [177.536174, 184.687942, 184.713402, 191.928362, 180.864216], + "baselinePredictSamplesMs": [71.861448, 57.920893, 51.914446, 51.732527, 47.172022], + "currentPredictSamplesMs": [48.789807, 47.421477, 47.508818, 43.513473, 46.182311], + "baselineAllocations": 62, + "currentAllocations": 15, + "baselinePeakBytes": 721664, + "currentPeakBytes": 525312, + "mse": 0.97650976645, + "revivedClusters": 65, + "digest": "348b350dd42e4f2f", + "bitwiseEqual": true + }, + { + "case": "pq8", + "samples": 8192, + "clusters": 256, + "dimensions": 8, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "lastEpochMean", + "baselineFitMedianMs": 49.976593, + "currentFitMedianMs": 51.816974, + "fitSpeedup": 0.96448, + "baselinePredictMedianMs": 14.465709, + "currentPredictMedianMs": 12.975626, + "predictSpeedup": 1.11484, + "baselineFitSamplesMs": [47.05165, 51.369871, 48.388843, 55.925801, 49.976593], + "currentFitSamplesMs": [51.816974, 47.203168, 48.491608, 51.961502, 56.463701], + "baselinePredictSamplesMs": [15.078182, 11.836701, 12.312207, 14.465709, 16.043384], + "currentPredictSamplesMs": [12.975626, 13.535009, 19.086345, 12.357025, 12.182241], + "baselineAllocations": 72, + "currentAllocations": 9, + "baselinePeakBytes": 194560, + "currentPeakBytes": 194560, + "mse": 0.611907061092, + "revivedClusters": 0, + "digest": "d397aea176dfd1db", + "bitwiseEqual": true + }, + { + "case": "pq8", + "samples": 8192, + "clusters": 256, + "dimensions": 8, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "ema", + "baselineFitMedianMs": 49.940565, + "currentFitMedianMs": 53.106126, + "fitSpeedup": 0.94039, + "baselinePredictMedianMs": 12.497583, + "currentPredictMedianMs": 12.372008, + "predictSpeedup": 1.01015, + "baselineFitSamplesMs": [51.534639, 49.823182, 55.087438, 49.940565, 47.923417], + "currentFitSamplesMs": [53.106126, 63.783741, 48.189242, 54.477886, 52.573087], + "baselinePredictSamplesMs": [12.996287, 11.327925, 11.742278, 12.497583, 12.582941], + "currentPredictSamplesMs": [12.17396, 12.372008, 12.316084, 12.405534, 13.082613], + "baselineAllocations": 72, + "currentAllocations": 7, + "baselinePeakBytes": 194560, + "currentPeakBytes": 176128, + "mse": 0.614058884679, + "revivedClusters": 0, + "digest": "598a7d0a59609a23", + "bitwiseEqual": true + }, + { + "case": "revival", + "samples": 2048, + "clusters": 64, + "dimensions": 32, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "lastEpochMean", + "baselineFitMedianMs": 13.349986, + "currentFitMedianMs": 2.525902, + "fitSpeedup": 5.28524, + "baselinePredictMedianMs": 3.117462, + "currentPredictMedianMs": 0.248884, + "predictSpeedup": 12.52576, + "baselineFitSamplesMs": [12.900549, 15.271623, 12.593227, 17.385549, 13.349986], + "currentFitSamplesMs": [2.532672, 2.410038, 2.525902, 2.512823, 3.187197], + "baselinePredictSamplesMs": [3.40796, 3.115189, 2.963251, 4.092791, 3.117462], + "currentPredictSamplesMs": [0.321274, 0.246832, 0.337598, 0.248884, 0.248334], + "baselineAllocations": 80, + "currentAllocations": 19, + "baselinePeakBytes": 91648, + "currentPeakBytes": 92672, + "mse": 0.0, + "revivedClusters": 167, + "digest": "e510f1a7dd7e5fe7", + "bitwiseEqual": true + }, + { + "case": "revival", + "samples": 2048, + "clusters": 64, + "dimensions": 32, + "seed": 42, + "epochs": 3, + "repeats": 5, + "finalization": "ema", + "baselineFitMedianMs": 12.905517, + "currentFitMedianMs": 3.145295, + "fitSpeedup": 4.10312, + "baselinePredictMedianMs": 3.489793, + "currentPredictMedianMs": 0.781276, + "predictSpeedup": 4.46679, + "baselineFitSamplesMs": [15.14003, 12.905517, 13.234311, 12.580436, 12.535741], + "currentFitSamplesMs": [3.350463, 3.380248, 3.126326, 3.145295, 2.962339], + "baselinePredictSamplesMs": [4.709018, 3.083932, 3.489793, 3.906741, 2.897862], + "currentPredictSamplesMs": [0.772182, 0.765983, 0.833865, 0.7971, 0.781276], + "baselineAllocations": 80, + "currentAllocations": 17, + "baselinePeakBytes": 91648, + "currentPeakBytes": 75776, + "mse": 1.8328e-08, + "revivedClusters": 167, + "digest": "60ad9268e8f72725", + "bitwiseEqual": true + } + ], + "nativePqFollowup": [ + { + "case": "pq8", + "samples": 8192, + "clusters": 256, + "dimensions": 8, + "seed": 42, + "epochs": 3, + "repeats": 21, + "finalization": "lastEpochMean", + "baselineFitMedianMs": 51.24788, + "currentFitMedianMs": 52.682849, + "fitSpeedup": 0.97276, + "baselinePredictMedianMs": 13.382461, + "currentPredictMedianMs": 13.632527, + "predictSpeedup": 0.98166, + "baselineFitSamplesMs": [49.448291, 49.816586, 49.312958, 48.863471, 51.08877, 48.398356, 52.749219, 51.153607, 51.147177, 52.587776, 58.093673, 51.758298, 52.255166, 51.439568, 51.496994, 52.095585, 49.410925, 51.24788, 53.677175, 51.450254, 50.59035], + "currentFitSamplesMs": [50.906396, 53.363303, 49.171624, 48.485522, 51.836616, 54.628707, 52.223908, 53.907743, 51.938108, 53.92603, 59.165586, 53.207258, 54.037302, 52.044438, 52.682849, 54.567615, 52.528166, 51.875483, 54.145655, 50.834828, 54.102114], + "baselinePredictSamplesMs": [13.083401, 12.904091, 13.440829, 12.854857, 13.701972, 14.183566, 13.736564, 13.026355, 13.022549, 13.786629, 12.42468, 13.382461, 13.45503, 13.917757, 13.202941, 14.74445, 12.927286, 13.384143, 12.49818, 12.775959, 16.755818], + "currentPredictSamplesMs": [14.81667, 13.117412, 13.440328, 12.302315, 13.213417, 14.582297, 13.978999, 15.233878, 13.649152, 13.632527, 13.236692, 14.473603, 12.616579, 13.798918, 13.571535, 14.082564, 13.86665, 13.012905, 11.896034, 13.598897, 14.586924], + "baselineAllocations": 72, + "currentAllocations": 9, + "baselinePeakBytes": 194560, + "currentPeakBytes": 194560, + "mse": 0.611907061092, + "revivedClusters": 0, + "digest": "d397aea176dfd1db", + "bitwiseEqual": true + }, + { + "case": "pq8", + "samples": 8192, + "clusters": 256, + "dimensions": 8, + "seed": 42, + "epochs": 3, + "repeats": 21, + "finalization": "ema", + "baselineFitMedianMs": 50.276679, + "currentFitMedianMs": 50.832466, + "fitSpeedup": 0.98907, + "baselinePredictMedianMs": 12.737901, + "currentPredictMedianMs": 13.050141, + "predictSpeedup": 0.97607, + "baselineFitSamplesMs": [47.921643, 48.137958, 48.476228, 50.120504, 49.945341, 48.566244, 49.538017, 48.266904, 46.495408, 64.897544, 73.224287, 56.418893, 50.48981, 52.052792, 50.767058, 51.888285, 50.276679, 54.769221, 59.892563, 49.93207, 94.624515], + "currentFitSamplesMs": [49.980713, 48.920456, 50.832466, 48.000654, 48.827698, 49.990659, 46.3299, 52.639305, 50.077289, 61.600693, 57.869147, 65.716427, 51.168893, 50.068806, 50.157379, 50.630702, 51.047169, 53.231833, 55.708239, 51.757348, 109.263924], + "baselinePredictSamplesMs": [12.159581, 12.584019, 12.620094, 12.799063, 12.296727, 11.405556, 12.620986, 13.689182, 12.929981, 16.264949, 14.558451, 12.833325, 13.327659, 12.405751, 12.988378, 13.218344, 12.535867, 12.737901, 11.942053, 30.120994, 12.542518], + "currentPredictSamplesMs": [12.864371, 13.261529, 13.898338, 13.794552, 11.400368, 13.422412, 12.868548, 13.469041, 13.050141, 16.407775, 16.744091, 12.667987, 12.244189, 12.784041, 12.993115, 13.011372, 11.954952, 13.802284, 13.442983, 17.258065, 11.744716], + "baselineAllocations": 72, + "currentAllocations": 7, + "baselinePeakBytes": 194560, + "currentPeakBytes": 176128, + "mse": 0.614058884679, + "revivedClusters": 0, + "digest": "598a7d0a59609a23", + "bitwiseEqual": true + } + ], + "npm": { + "runtime": { + "node": "v24.22.1", + "platform": "linux", + "arch": "x64", + "cpu": "AMD EPYC 9V74 80-Core Processor", + "logicalCpus": 5 + }, + "repeats": 7, + "warmupsPerVariant": 2, + "measurement": "median wall-clock milliseconds, fully materialized results; explicit GC outside timing", + "sameKernelTransportMeasurement": "full fit plus output materialization using one optimized WASM kernel; difference includes JSON serialization/parsing and array conversion, not a conversion-only timer", + "checksum": 42347816.01348107, + "cases": [ + { + "name": "large-assignment-result", + "samples": 1000000, + "dimensions": 1, + "clusters": 1, + "epochs": 1, + "batchSize": 4096, + "fit": { + "optimizedMs": 61.56721399999992, + "baselineMs": 84.090687, + "speedup": 1.3658355078402624 + }, + "predict": { + "optimizedMs": 10.453125, + "baselineMs": 15.906072000000222, + "speedup": 1.5216571121076445 + }, + "sameKernelTransport": { + "jsonMs": 75.97168599999986, + "typedMs": 58.40519399999994, + "speedup": 1.3007693459591958 + }, + "baselineParity": { + "labelDifferences": 0, + "centroidMaxAbsDifference": 0, + "mseAbsoluteDifference": 0 + } + }, + { + "name": "embedding-codebook", + "samples": 8192, + "dimensions": 64, + "clusters": 16, + "epochs": 3, + "batchSize": 256, + "fit": { + "optimizedMs": 15.001260999999886, + "baselineMs": 36.18889299999955, + "speedup": 2.4123900650751846 + }, + "predict": { + "optimizedMs": 5.059164999999666, + "baselineMs": 13.240130000000136, + "speedup": 2.6170583485616716 + }, + "sameKernelTransport": { + "jsonMs": 12.692776000000777, + "typedMs": 12.346003999999994, + "speedup": 1.0280877926170107 + }, + "baselineParity": { + "labelDifferences": 0, + "centroidMaxAbsDifference": 0, + "mseAbsoluteDifference": 0 + } + } + ] + }, + "sourceAndArtifactSha256": { + "baselineRustSource": { + "config.rs": "b9ad83acdf082778bc9841b18833122b0f353586b6a2cae5e2047f87d4dff6d6", + "lib.rs": "627e108c9d0b2644f4cc75f5f09805c1f92bde683f56b50a6f474074a2ccb517", + "trainer.rs": "f67fc903d4995678716dbc450cba820f4f4e4d73c50686c7267c2de01d22b967", + "wasm.rs": "270d30ca46b32602a7549d4c37b5d0740275800babdc24f62b8df358008f6568" + }, + "optimizedRustSource": { + "config.rs": "b9ad83acdf082778bc9841b18833122b0f353586b6a2cae5e2047f87d4dff6d6", + "distance.rs": "b2bb4252b7c10a99d55d07a33c46f196743d8fbe071694340db4be76c5f22e05", + "lib.rs": "084ce5655335019fc6a11b0226f0ccfed6b48a55cde674686453dc8231d59562", + "trainer.rs": "1bfa1299152cc9dd188975e6e068752aac83d5cdd3bd199672472ad68d5b1af2", + "wasm.rs": "7d4fbe5c800e083446ecf5f767bbe9eb6ed603b0330ab9be7086d009d930ad00" + }, + "baselineWasm": "dd68d9bf52a7422f14f56819f6b49c55118fc40438a0282ab64cde6e066c348b", + "optimizedWasm": "2c8f8a21e89f7e809a02d967bb77935284f35115855ed8024fae1a9453f27b5b" + }, + "verification": { + "native": "All model fields and predictions match baseline bit for bit in all eight measured cases.", + "npm": "Baseline centroid/label/MSE differences are zero in both measured cases; typed and JSON transports of the optimized kernel compare equal.", + "memory": "Native peaks are additional live requested heap bytes, excluding the dataset and preexisting models; not process RSS. LastEpochMean peaks can rise slightly due to retained scratch capacity; EMA skips last-epoch buffers.", + "pq8": "21-repeat follow-up fit median ratios baseline/current are 0.97276 (lastEpochMean) and 0.98907 (ema). No PQ speedup is claimed.", + "browser": "Chromium startup failed even for about:blank in the earlier baseline; browser qualification remains pending." + }, + "reproduceCurrentNative": "cargo run -p ruvector-e-means --release --example benchmark -- --case all --repeats 5 --seed 42 --finalization both", + "reproduceNpm": "npm run benchmark -- --repeats 7 (in npm/packages/e-means); --baseline compares a supplied prior artifact." +} diff --git a/npm/package-lock.json b/npm/package-lock.json index 87b5cd8571..9ca843f88d 100644 --- a/npm/package-lock.json +++ b/npm/package-lock.json @@ -3581,6 +3581,10 @@ "node": ">= 18" } }, + "node_modules/@ruvector/e-means": { + "resolved": "packages/e-means", + "link": true + }, "node_modules/@ruvector/economy-wasm": { "version": "0.1.29", "resolved": "https://registry.npmjs.org/@ruvector/economy-wasm/-/economy-wasm-0.1.29.tgz", @@ -19007,7 +19011,8 @@ "@ruvector/sona": "^0.1.9", "@ruvector/tiny-dancer": "^0.1.23", "ruvector": "^0.3.7", - "rvlite": "^0.2.7" + "rvlite": "^0.2.7", + "@ruvector/e-means": "0.1.0" }, "engines": { "node": ">=20.0.0" @@ -19460,6 +19465,9 @@ "engines": { "node": ">=18.0.0" }, + "dependencies": { + "@ruvector/e-means": "0.1.0" + }, "optionalDependencies": { "ruvector-core-darwin-arm64": "0.1.34", "ruvector-core-darwin-x64": "0.1.34", @@ -19486,6 +19494,28 @@ "@ruvector/diskann-win32-x64-msvc": "0.1.3" } }, + "packages/e-means": { + "name": "@ruvector/e-means", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@metaharness/darwin": "0.13.0", + "esbuild": "0.28.2", + "miniflare": "4.20260730.0", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@metaharness/darwin": "0.13.0 || 0.13.1" + }, + "peerDependenciesMeta": { + "@metaharness/darwin": { + "optional": true + } + } + }, "packages/emergent-time": { "name": "@ruvector/emergent-time", "version": "0.1.3", @@ -21176,7 +21206,8 @@ "@ruvector/sona": "^0.1.11", "chalk": "^4.1.2", "commander": "^11.1.0", - "ora": "^5.4.1" + "ora": "^5.4.1", + "@ruvector/e-means": "0.1.0" }, "bin": { "ruvector": "bin/cli.js" @@ -22410,6 +22441,1510 @@ "engines": { "node": ">=18.0.0" } + }, + "packages/e-means/node_modules/@babel/helper-string-parser": { + "version": "7.29.13", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.13.tgz", + "integrity": "sha512-fWegr+WsEsiMVW4KUNdIIzzMTZSzNiqhy0//6+2MRi2d5NFNsL/AYwBQW8eBnRbTBVRnDVzjaJEUqKaCWjMMdQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "packages/e-means/node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "packages/e-means/node_modules/@babel/parser": { + "version": "7.29.13", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.13.tgz", + "integrity": "sha512-fJM50YYjlTiBhgdkGmC1tBpv+FMAyo4annf2X6Aks+0yRlaiVkYW/3f07xAfVVkYIoZ9fFZp+mEThUFnuybhkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.11" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "packages/e-means/node_modules/@babel/types": { + "version": "7.29.11", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.11.tgz", + "integrity": "sha512-WCsemW40Cpc6gouC1SrY6PBaZ+oLRlEVY9XBvZBI7il/HyUrb2nJ505wE1MHEFyDZNnrWDEuy4jL50WVHfTTSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.11", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "packages/e-means/node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260730.1.tgz", + "integrity": "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "packages/e-means/node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260730.1.tgz", + "integrity": "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "packages/e-means/node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260730.1.tgz", + "integrity": "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "packages/e-means/node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260730.1.tgz", + "integrity": "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "packages/e-means/node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260730.1.tgz", + "integrity": "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "packages/e-means/node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "packages/e-means/node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "packages/e-means/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "packages/e-means/node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz", + "integrity": "sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-darwin-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz", + "integrity": "sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz", + "integrity": "sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz", + "integrity": "sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz", + "integrity": "sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz", + "integrity": "sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz", + "integrity": "sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz", + "integrity": "sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz", + "integrity": "sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz", + "integrity": "sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz", + "integrity": "sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz", + "integrity": "sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz", + "integrity": "sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-linux-arm": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz", + "integrity": "sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linux-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz", + "integrity": "sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz", + "integrity": "sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz", + "integrity": "sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linux-s390x": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz", + "integrity": "sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linux-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz", + "integrity": "sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz", + "integrity": "sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz", + "integrity": "sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.4" + } + }, + "packages/e-means/node_modules/@img/sharp-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", + "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.5.tgz", + "integrity": "sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-win32-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz", + "integrity": "sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-win32-ia32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz", + "integrity": "sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@img/sharp-win32-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz", + "integrity": "sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "packages/e-means/node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "packages/e-means/node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "packages/e-means/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "packages/e-means/node_modules/@metaharness/darwin": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@metaharness/darwin/-/darwin-0.13.0.tgz", + "integrity": "sha512-L3SrQii70erHVDmo1qx0875IRpVujk86zAO7sxtSVChWCgAjabEmmYRYt72D1VekfUCbSWCgZKIktqUxVAKCFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.13" + }, + "bin": { + "metaharness-darwin": "dist/cli.js" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "packages/e-means/node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "packages/e-means/node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "packages/e-means/node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "packages/e-means/node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "packages/e-means/node_modules/@speed-highlight/core": { + "version": "1.2.24", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.24.tgz", + "integrity": "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==", + "dev": true, + "license": "CC0-1.0" + }, + "packages/e-means/node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "packages/e-means/node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "packages/e-means/node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "packages/e-means/node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "packages/e-means/node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "packages/e-means/node_modules/miniflare": { + "version": "4.20260730.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260730.0.tgz", + "integrity": "sha512-1Z9SB9r/o//80UA02Re3QhtcecSHAyAjf5EcKBfQVlQrCg7Miy79hl2PvtkwFLIaJ5rcrOPdDcRr577okwZPsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.28.0", + "workerd": "1.20260730.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "packages/e-means/node_modules/semver": { + "version": "7.8.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.6.tgz", + "integrity": "sha512-9lpSSU4Mly3mrtruwLs1eLK6+gzbzqeBFwRNjIjE+9LuDpWdYctrQlQ3ej1Ilgi7gknpsTRnPoLP8DpCHqHIJw==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "packages/e-means/node_modules/sharp": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", + "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.5", + "@img/sharp-darwin-x64": "0.35.5", + "@img/sharp-freebsd-wasm32": "0.35.5", + "@img/sharp-libvips-darwin-arm64": "1.3.4", + "@img/sharp-libvips-darwin-x64": "1.3.4", + "@img/sharp-libvips-linux-arm": "1.3.4", + "@img/sharp-libvips-linux-arm64": "1.3.4", + "@img/sharp-libvips-linux-ppc64": "1.3.4", + "@img/sharp-libvips-linux-riscv64": "1.3.4", + "@img/sharp-libvips-linux-s390x": "1.3.4", + "@img/sharp-libvips-linux-x64": "1.3.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4", + "@img/sharp-libvips-linuxmusl-x64": "1.3.4", + "@img/sharp-linux-arm": "0.35.5", + "@img/sharp-linux-arm64": "0.35.5", + "@img/sharp-linux-ppc64": "0.35.5", + "@img/sharp-linux-riscv64": "0.35.5", + "@img/sharp-linux-s390x": "0.35.5", + "@img/sharp-linux-x64": "0.35.5", + "@img/sharp-linuxmusl-arm64": "0.35.5", + "@img/sharp-linuxmusl-x64": "0.35.5", + "@img/sharp-webcontainers-wasm32": "0.35.5", + "@img/sharp-win32-arm64": "0.35.5", + "@img/sharp-win32-ia32": "0.35.5", + "@img/sharp-win32-x64": "0.35.5" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "packages/e-means/node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "packages/e-means/node_modules/tslib": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.9.1.tgz", + "integrity": "sha512-a4Ux9VKuIMHsOS7ENbEq6hK2ZGYIT27Rtl0E4eD9KqwxhKiQeMZE/9xb3IdYwWaSOyIBQLExhQgXDTYoiCxnWA==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "packages/e-means/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "packages/e-means/node_modules/undici": { + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "packages/e-means/node_modules/workerd": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260730.1.tgz", + "integrity": "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260730.1", + "@cloudflare/workerd-darwin-arm64": "1.20260730.1", + "@cloudflare/workerd-linux-64": "1.20260730.1", + "@cloudflare/workerd-linux-arm64": "1.20260730.1", + "@cloudflare/workerd-windows-64": "1.20260730.1" + } + }, + "packages/e-means/node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "packages/e-means/node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "packages/e-means/node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } } } } diff --git a/npm/package.json b/npm/package.json index 2e1505e593..274ecf56c5 100644 --- a/npm/package.json +++ b/npm/package.json @@ -30,6 +30,10 @@ "npm": ">=9.0.0" }, "overrides": { + "miniflare": { + "sharp": "0.35.5", + "undici": "7.30.1" + }, "node-forge": ">=1.4.0", "flatted": ">=3.3.3", "picomatch": ">=4.0.3", diff --git a/npm/packages/all/clustering.d.mts b/npm/packages/all/clustering.d.mts new file mode 100644 index 0000000000..2ca0a76f10 --- /dev/null +++ b/npm/packages/all/clustering.d.mts @@ -0,0 +1,2 @@ +export { init, fit, predict } from '@ruvector/e-means'; +export type { EMeansOptions, EMeansModel } from '@ruvector/e-means'; diff --git a/npm/packages/all/clustering.mjs b/npm/packages/all/clustering.mjs new file mode 100644 index 0000000000..f1f8835355 --- /dev/null +++ b/npm/packages/all/clustering.mjs @@ -0,0 +1,2 @@ +/** Shared Rust/WASM clustering; initialize explicitly before numerical calls. */ +export { init, fit, predict } from '@ruvector/e-means'; diff --git a/npm/packages/all/clustering/cloudflare.d.mts b/npm/packages/all/clustering/cloudflare.d.mts new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/npm/packages/all/clustering/cloudflare.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/npm/packages/all/clustering/cloudflare.mjs b/npm/packages/all/clustering/cloudflare.mjs new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/npm/packages/all/clustering/cloudflare.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/npm/packages/all/clustering/ivf.d.mts b/npm/packages/all/clustering/ivf.d.mts new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/npm/packages/all/clustering/ivf.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/npm/packages/all/clustering/ivf.mjs b/npm/packages/all/clustering/ivf.mjs new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/npm/packages/all/clustering/ivf.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/npm/packages/all/clustering/metaharness.d.mts b/npm/packages/all/clustering/metaharness.d.mts new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/npm/packages/all/clustering/metaharness.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/npm/packages/all/clustering/metaharness.mjs b/npm/packages/all/clustering/metaharness.mjs new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/npm/packages/all/clustering/metaharness.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/npm/packages/all/clustering/quantization.d.mts b/npm/packages/all/clustering/quantization.d.mts new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/npm/packages/all/clustering/quantization.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/npm/packages/all/clustering/quantization.mjs b/npm/packages/all/clustering/quantization.mjs new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/npm/packages/all/clustering/quantization.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/npm/packages/all/package.json b/npm/packages/all/package.json index 895268d823..95f7ea786c 100644 --- a/npm/packages/all/package.json +++ b/npm/packages/all/package.json @@ -21,12 +21,16 @@ "files": [ "index.js", "index.d.ts", - "README.md" + "README.md", + "clustering.mjs", + "clustering.d.mts", + "clustering/" ], "scripts": { "test": "node --test test/all.test.js" }, "dependencies": { + "@ruvector/e-means": "0.1.0", "ruvector": "^0.3.7", "@ruvector/rvf": "^0.4.1", "@ruvector/sona": "^0.1.9", diff --git a/npm/packages/core/clustering.d.mts b/npm/packages/core/clustering.d.mts new file mode 100644 index 0000000000..2ca0a76f10 --- /dev/null +++ b/npm/packages/core/clustering.d.mts @@ -0,0 +1,2 @@ +export { init, fit, predict } from '@ruvector/e-means'; +export type { EMeansOptions, EMeansModel } from '@ruvector/e-means'; diff --git a/npm/packages/core/clustering.mjs b/npm/packages/core/clustering.mjs new file mode 100644 index 0000000000..f1f8835355 --- /dev/null +++ b/npm/packages/core/clustering.mjs @@ -0,0 +1,2 @@ +/** Shared Rust/WASM clustering; initialize explicitly before numerical calls. */ +export { init, fit, predict } from '@ruvector/e-means'; diff --git a/npm/packages/core/clustering/cloudflare.d.mts b/npm/packages/core/clustering/cloudflare.d.mts new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/npm/packages/core/clustering/cloudflare.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/npm/packages/core/clustering/cloudflare.mjs b/npm/packages/core/clustering/cloudflare.mjs new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/npm/packages/core/clustering/cloudflare.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/npm/packages/core/clustering/ivf.d.mts b/npm/packages/core/clustering/ivf.d.mts new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/npm/packages/core/clustering/ivf.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/npm/packages/core/clustering/ivf.mjs b/npm/packages/core/clustering/ivf.mjs new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/npm/packages/core/clustering/ivf.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/npm/packages/core/clustering/metaharness.d.mts b/npm/packages/core/clustering/metaharness.d.mts new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/npm/packages/core/clustering/metaharness.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/npm/packages/core/clustering/metaharness.mjs b/npm/packages/core/clustering/metaharness.mjs new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/npm/packages/core/clustering/metaharness.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/npm/packages/core/clustering/quantization.d.mts b/npm/packages/core/clustering/quantization.d.mts new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/npm/packages/core/clustering/quantization.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/npm/packages/core/clustering/quantization.mjs b/npm/packages/core/clustering/quantization.mjs new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/npm/packages/core/clustering/quantization.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/npm/packages/core/package.json b/npm/packages/core/package.json index e42719839b..197ce9642b 100644 --- a/npm/packages/core/package.json +++ b/npm/packages/core/package.json @@ -21,7 +21,10 @@ "files": [ "index.js", "index.d.ts", - "README.md" + "README.md", + "clustering.mjs", + "clustering.d.mts", + "clustering/" ], "scripts": { "build:napi": "napi build --platform --release --cargo-cwd ../../../crates/ruvector-node", @@ -31,6 +34,9 @@ "devDependencies": { "@napi-rs/cli": "^2.18.0" }, + "dependencies": { + "@ruvector/e-means": "0.1.0" + }, "optionalDependencies": { "ruvector-core-linux-x64-gnu": "0.1.35", "ruvector-core-linux-arm64-gnu": "0.1.35", diff --git a/npm/packages/e-means/.gitignore b/npm/packages/e-means/.gitignore new file mode 100644 index 0000000000..7f84121734 --- /dev/null +++ b/npm/packages/e-means/.gitignore @@ -0,0 +1,2 @@ +# Keep the isolated integration-test dependency graph reproducible. +!package-lock.json diff --git a/npm/packages/e-means/LICENSE b/npm/packages/e-means/LICENSE new file mode 100644 index 0000000000..2dd524ac38 --- /dev/null +++ b/npm/packages/e-means/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 rUv + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/npm/packages/e-means/README.md b/npm/packages/e-means/README.md new file mode 100644 index 0000000000..121ff27951 --- /dev/null +++ b/npm/packages/e-means/README.md @@ -0,0 +1,101 @@ +# @ruvector/e-means + +![e-means training loop: rows are assigned in mini-batches, EMA counts and sums move the centroids with a cosine-decaying step, a dead centroid is revived by splitting a busy donor, and the last epoch mean gives the codebook](https://raw.githubusercontent.com/ruvnet/ruvector/main/npm/packages/e-means/assets/e-means-flow.svg) + +Portable WebAssembly bindings for the shared `ruvector-e-means` Rust crate, based on [e-means (arXiv:2611.09501v1)](https://arxiv.org/abs/2611.09501v1). Uses EMA counts and sums, cosine annealing, dead-cluster splitting and last-epoch mean finalization. + +```js +import { init, fit, predict } from '@ruvector/e-means'; + +await init(); +const data = new Float32Array([0, 0, 0, 2, 10, 10, 10, 12]); +const model = fit(data, 2, { clusters: 2, epochs: 3, seed: 42 }); +const labels = predict(data, 2, model.centroids); +``` + +Pass flat row-major `Float32Array` data and finite values. `fit` and `predict` are synchronous after `init`; input arrays are unchanged. Cluster labels break distance ties at the lowest centroid index. `mse` is mean squared Euclidean distance per vector. Output assignments are recomputed against the final codebook. + +The runtime transfers result vectors directly as typed arrays. Returned arrays belong to the caller and remain valid across later fits and WebAssembly memory growth. The temporary Rust result handle is released before `fit` returns; the small options object alone uses JSON transport. + +`batchSize` and `peakStep` can override the paper's automatic settings. `finalization: 'ema'` returns the training EMA instead of the default `'lastEpochMean'`. The empirical scaling laws target large embedding datasets; evaluate overrides on tiny or low-dimensional data. CPU/WASM throughput is not the paper's measured GPU throughput. GPU kernels, approximate assignment and the optional slow-EMA finalizer are outside this initial implementation. + +Node reads the bundled WASM from disk. In browsers, serve the `.wasm` artifact beside its generated JavaScript module, or call `init(bytes)` / `init(compiledModule)` to supply it explicitly. No native compilation, network request, or model download occurs during installation or Node initialization. Browser initialization fetches the bundled `.wasm` URL by default. + +From a source checkout: install the `wasm32-unknown-unknown` Rust target and `wasm-bindgen-cli` version `0.2.125`, then run `npm run build`, `npm test` and `npm run test:consumer` in this package. `npm pack` verifies the built runtime before creating an archive. The archive contains the runtime and needs neither Rust nor the source repository. + +`npm run typecheck` validates the declarations. `npm run test:browser` runs the same numerical oracle in headless Chromium against the default browser loader; set `E_MEANS_CHROMIUM` when Chromium is not on `PATH`. + +`npm run benchmark -- --baseline /path/to/previous/package --repeats 7` measures median fit and prediction time on one million scalar rows and an embedding codebook workload. It also compares JSON and typed output on the same current Rust kernel. The baseline argument is optional; retain the previous package's module and WASM together. Run benchmarks without other compute workloads; results describe the current machine and these fixtures. + +RuVector exposes the same API through `ruvector/clustering.mjs` once a `ruvector` release that depends on this package is published. A ruv-FANN facade is proposed separately and is not released. + +The package also ships explicit integration subpaths: + +| Import | Use | +|---|---| +| `@ruvector/e-means/quantization` | Train PQ subspace codebooks, encode byte codes, decode vectors, and validate JSON codebooks | +| `@ruvector/e-means/ivf` | Train coarse clusters and construct stable posting lists | +| `@ruvector/e-means/metaharness` | Measure held-out clustering quality and run bounded numeric evolution with Darwin 0.13.0 or 0.13.1 | +| `@ruvector/e-means/cloudflare` | Create an authenticated Worker and explicitly persist/export centroid codebooks through R2/Vectorize bindings | +| `@ruvector/e-means/runtime` | Initialize from explicit bytes or a precompiled WASM module without importing Node loaders | +| `@ruvector/e-means/cloudflare/build` | Build the shipped Worker example locally; requires development tooling, and never deploys | +| `@ruvector/e-means/wasm` | Static WASM module import for Wrangler | + +```js +import { init } from '@ruvector/e-means'; +import { trainProductQuantizer, encodeProductQuantizer, decodeProductQuantizer } + from '@ruvector/e-means/quantization'; +import { partition, postingList } from '@ruvector/e-means/ivf'; + +await init(); +const data = new Float32Array([0, 0, 0, 2, 10, 10, 10, 12]); +const options = { clusters: 2, epochs: 3, seed: 42 }; +const book = trainProductQuantizer(data, 2, 2, options); +const codes = encodeProductQuantizer(book, data); +const reconstructed = decodeProductQuantizer(book, codes); +const inverted = partition(data, 2, options); +const firstPosting = postingList(inverted, 0); +``` + +MetaHarness tuning requires the optional peer `@metaharness/darwin` at 0.13.0 or 0.13.1. +`createClusteringEvaluator` uses snapshotted, nonoverlapping training/holdout +arrays and explicit evaluation/work/deadline budgets. `tuneClustering` calls the +real numeric SDK, verifies observed records, and returns a recommendation with +`sourceApplied: false`. Selection uses the holdout; another test split is needed +for generalization claims. See the shipped `examples/metaharness.mjs`. + +For a Worker, statically import the module and supply the deployment's secret: + +```js +import wasmModule from '@ruvector/e-means/wasm'; +import { createClusteringWorker } from '@ruvector/e-means/cloudflare'; + +export default { + fetch(request, env) { + return createClusteringWorker({ + wasmModule, bearerToken: env.E_MEANS_BEARER_TOKEN, + }).fetch(request); + }, +}; +``` + +`/fit` and `/predict` require authentication by default and enforce body, shape, +stream-read and work bounds. R2 helpers persist only centroid codebooks. +Vectorize exports use explicit caller-owned bindings; acknowledgements are +asynchronous and failures can leave earlier batches applied. Use versioned +prefixes/namespaces because upserts do not delete stale IDs. PQ records have a +different layout and are rejected by these single-codebook helpers. + +The shipped `examples/cloudflare` contains a static-WASM builder and Wrangler +configuration. `npm run build:cloudflare` only writes local artifacts. `buildWorker` needs `esbuild` +installed by the caller and reports a clear error without it. Qualify +with `npm ci --ignore-scripts --workspaces=false`, `npm test`, +`npm run typecheck`, `npm run test:consumer`, and `npm run test:ecosystem`. +Local workerd/R2 tests use development dependencies; Vectorize is covered by +binding fixtures. No hosted deployment is implied. + +The `@ruvector/core`, `@ruvector/all`, `@ruvector/wasm` and `ruvector` sources +add clustering subpaths over these integrations; they reach users only with +those packages' next releases. See +[ADR-360](https://github.com/ruvnet/ruvector/blob/main/docs/adr/ADR-360-e-means-ecosystem-integrations.md) +for the exact paths, release order, and validation scope. diff --git a/npm/packages/e-means/assets/e-means-flow.svg b/npm/packages/e-means/assets/e-means-flow.svg new file mode 100644 index 0000000000..670e13813f --- /dev/null +++ b/npm/packages/e-means/assets/e-means-flow.svg @@ -0,0 +1,62 @@ + + e-means training loop + Three groups of points. Centroids start on random rows and move toward each group as mini-batches update exponential moving averages of counts and sums with a cosine-decaying step. A dead centroid is revived by splitting a busy donor, and the last epoch's mean gives the final codebook. + + + + + + + + + + + + + + + + + + + + + + + + e-means: mini-batch k-means with EMA statistics + + + + 1. assign batch rows, EMA-update counts and sums + + + + 2. step size follows a cosine schedule + + + + 3. every 5 updates, split a busy donor into a dead slot + + + + 4. finalize with the last epoch's mean (or the EMA) + Same Rust kernel natively and in WebAssembly; seeded and deterministic. + diff --git a/npm/packages/e-means/e-means.d.ts b/npm/packages/e-means/e-means.d.ts new file mode 100644 index 0000000000..6230ed769a --- /dev/null +++ b/npm/packages/e-means/e-means.d.ts @@ -0,0 +1,33 @@ +export interface EMeansOptions { + clusters: number; + /** Positive integer, default 3. */ + epochs?: number; + /** Positive integer; defaults to the paper's batch scaling law, capped at the row count. */ + batchSize?: number; + /** Peak EMA step in (0, 1]; defaults to the paper's step scaling law. */ + peakStep?: number; + /** Non-negative safe integer; default 42. */ + seed?: number; + finalization?: 'lastEpochMean' | 'ema'; +} + +export interface EMeansModel { + /** Flat row-major clusters × dimensions codebook. */ + centroids: Float32Array; + /** Nearest final centroid for every input row. */ + assignments: Uint32Array; + dimensions: number; + /** Mean squared Euclidean distance per vector, not per coordinate. */ + mse: number; + batchSize: number; + peakStep: number; + updates: number; + revivedClusters: number; +} + +/** Load the bundled WebAssembly runtime. In browsers the default .wasm URL must be served. */ +export function init(input?: BufferSource | WebAssembly.Module): Promise; +/** Requires init(). Inputs are not modified. */ +export function fit(data: Float32Array, dimensions: number, options: EMeansOptions): EMeansModel; +/** Requires init(). An empty input returns an empty assignment array. */ +export function predict(data: Float32Array, dimensions: number, centroids: Float32Array): Uint32Array; diff --git a/npm/packages/e-means/e-means.mjs b/npm/packages/e-means/e-means.mjs new file mode 100644 index 0000000000..9d74da55ff --- /dev/null +++ b/npm/packages/e-means/e-means.mjs @@ -0,0 +1,29 @@ +import { init as initRuntime } from './runtime.mjs'; +export { fit, predict } from './runtime.mjs'; + +let initialization; + +/** Load the bundled Rust runtime once; pass bytes or a compiled module to override it. */ +export async function init(input) { + if (!initialization) { + initialization = (async () => { + let source = input; + if (source === undefined) { + const url = new URL('./wasm/ruvector_e_means_bg.wasm', import.meta.url); + if (url.protocol === 'file:' && typeof process !== 'undefined' && process.versions?.node) { + const { readFile } = await import('node:fs/promises'); + source = await readFile(url); + } else { + source = url; + } + } + await initRuntime(source); + })().catch((error) => { + initialization = undefined; + const message = typeof error?.message === 'string' ? error.message : String(error); + if (message.startsWith('Could not initialize @ruvector/e-means:')) throw error; + throw new Error(`Could not initialize @ruvector/e-means: ${message}`, { cause: error }); + }); + } + await initialization; +} diff --git a/npm/packages/e-means/examples/cloudflare/build.d.ts b/npm/packages/e-means/examples/cloudflare/build.d.ts new file mode 100644 index 0000000000..eac7465c9d --- /dev/null +++ b/npm/packages/e-means/examples/cloudflare/build.d.ts @@ -0,0 +1,5 @@ +/** Node-only build tooling; requires caller-installed esbuild. Never deploys. */ +export function buildWorker(outDirectory: string, entryPoint?: string): Promise<{ + script: string; + wasm: string; +}>; diff --git a/npm/packages/e-means/examples/cloudflare/build.mjs b/npm/packages/e-means/examples/cloudflare/build.mjs new file mode 100644 index 0000000000..e253422332 --- /dev/null +++ b/npm/packages/e-means/examples/cloudflare/build.mjs @@ -0,0 +1,25 @@ +import { mkdir, copyFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +/** Build a static-WASM Worker locally; this function never deploys anything. */ +export async function buildWorker(outDirectory, entryPoint = fileURLToPath(new URL('./worker.mjs', import.meta.url))) { + if (typeof outDirectory !== 'string' || outDirectory.length === 0) throw new TypeError('outDirectory is required'); + let build; + try { ({ build } = await import('esbuild')); } + catch (cause) { throw new Error('buildWorker requires esbuild; install it as a development dependency', { cause }); } + const output = resolve(outDirectory), wasm = join(output, 'e-means.wasm'); + await mkdir(output, { recursive: true }); + await copyFile(fileURLToPath(import.meta.resolve('@ruvector/e-means/wasm')), wasm); + const script = join(output, 'worker.mjs'); + await build({ entryPoints: [entryPoint], outfile: script, bundle: true, format: 'esm', platform: 'browser', + target: 'es2022', legalComments: 'none', plugins: [{ name: 'e-means-static-wasm', setup(builder) { + builder.onResolve({ filter: /^@ruvector\/e-means\/wasm$/ }, () => ({ path: './e-means.wasm', external: true })); + } }] }); + return { script, wasm }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const artifact = await buildWorker(process.argv[2] ?? fileURLToPath(new URL('./dist/', import.meta.url))); + console.log(JSON.stringify(artifact)); +} diff --git a/npm/packages/e-means/examples/cloudflare/worker.mjs b/npm/packages/e-means/examples/cloudflare/worker.mjs new file mode 100644 index 0000000000..be36fb9ebd --- /dev/null +++ b/npm/packages/e-means/examples/cloudflare/worker.mjs @@ -0,0 +1,9 @@ +import wasmModule from '@ruvector/e-means/wasm'; +import { createClusteringWorker } from '@ruvector/e-means/cloudflare'; + +export default { + fetch(request, env) { + // The secret is supplied by the deployment's binding configuration. + return createClusteringWorker({ wasmModule, bearerToken: env.E_MEANS_BEARER_TOKEN }).fetch(request); + }, +}; diff --git a/npm/packages/e-means/examples/cloudflare/wrangler.jsonc b/npm/packages/e-means/examples/cloudflare/wrangler.jsonc new file mode 100644 index 0000000000..19ed83de99 --- /dev/null +++ b/npm/packages/e-means/examples/cloudflare/wrangler.jsonc @@ -0,0 +1,9 @@ +{ + "name": "ruvector-e-means", + "main": "dist/worker.mjs", + "compatibility_date": "2026-07-30", + "no_bundle": true, + "find_additional_modules": true, + "rules": [{ "type": "CompiledWasm", "globs": ["**/*.wasm"], "fallthrough": true }], + "observability": { "enabled": true } +} diff --git a/npm/packages/e-means/examples/metaharness.mjs b/npm/packages/e-means/examples/metaharness.mjs new file mode 100644 index 0000000000..a2114ea7e5 --- /dev/null +++ b/npm/packages/e-means/examples/metaharness.mjs @@ -0,0 +1,16 @@ +import { tuneClustering } from '@ruvector/e-means/metaharness'; + +const result = await tuneClustering({ + training: new Float32Array([0, 0, 0, 2, 10, 10, 10, 12]), + holdout: new Float32Array([0, 1, 10, 11]), + dimensions: 2, + baseline: { clusters: 2, epochs: 2, batchSize: 2, peakStep: 0.5, seed: 42 }, + domain: { peakStep: { min: 0.1, max: 1, type: 'float', scale: 'log' } }, + budget: { maxEvaluations: 3, maxDistanceCoordinates: 100_000, timeoutMs: 30_000 }, + generations: 1, + childrenPerGeneration: 2, + mutationSeed: 7, +}); +console.log(JSON.stringify({ baselineHoldoutMse: result.baseline.score.raw.holdoutMse, + winnerHoldoutMse: result.winner.score.raw.holdoutMse, recommendation: result.recommendation, + sourceApplied: result.sourceApplied }, null, 2)); diff --git a/npm/packages/e-means/integrations/cloudflare.d.ts b/npm/packages/e-means/integrations/cloudflare.d.ts new file mode 100644 index 0000000000..499c546dbb --- /dev/null +++ b/npm/packages/e-means/integrations/cloudflare.d.ts @@ -0,0 +1,58 @@ +export interface WorkerOptions { + /** Statically import @ruvector/e-means/wasm in a Worker. */ + wasmModule: WebAssembly.Module; + bearerToken?: string; + /** Explicit local/trusted opt-in; false by default. */ + allowUnauthenticated?: boolean; + maxRequestBytes?: number; + maxWork?: number; + bodyTimeoutMs?: number; +} +/** One row-major set of full-dimensional centroids, not subspace-major PQ books. */ +export interface CodebookInput { + dimensions: number; + centroids: Float32Array; + schema?: 'ruvector.e-means-codebook/1'; +} +export interface StoredCodebook extends CodebookInput { + schema: 'ruvector.e-means-codebook/1'; + clusters: number; +} +export interface R2Binding { + put(key: string, body: string, options?: { httpMetadata?: { contentType?: string } }): Promise; + get(key: string): Promise<{ size: number; body: ReadableStream } | null>; +} +export interface VectorizeVector { + id: string; + values: number[]; + metadata: { cluster: number }; + namespace?: string; +} +export interface VectorizeBinding { + upsert(vectors: VectorizeVector[]): Promise<{ mutationId: string }>; +} +export interface ExportOptions { + /** Use a versioned prefix for a new codebook; old centroid IDs are not deleted. */ + prefix: string; + namespace?: string; + batchSize?: number; +} +export interface ExportReceipt { submitted: number; mutationIds: string[] } +/** Does not persist request data. Operations fail closed unless authentication is configured. */ +export function createClusteringWorker(options: WorkerOptions): { fetch(request: Request): Promise }; +export function persistCodebook(bucket: Pick, key: string, model: CodebookInput): Promise<{ + key: string; dimensions: number; clusters: number; bytes: number; +}>; +export function loadCodebook(bucket: Pick, key: string): Promise; +/** Batches upserts; acknowledgement does not establish query visibility or atomicity. */ +export function exportCentroids(index: VectorizeBinding, model: CodebookInput, options: ExportOptions): Promise; +export class CentroidExportError extends Error { + constructor(submitted: number, mutationIds: readonly string[], cause: unknown); + /** Rows in previously acknowledged batches; the failing batch may also have applied. */ + submitted: number; + mutationIds: string[]; +} +export const WORKER_LIMITS: Readonly<{ + maxRequestBytes: number; maxWork: number; maxRows: number; maxDimensions: number; + maxClusters: number; maxEpochs: number; maxElements: number; maxCodebookBytes: number; +}>; diff --git a/npm/packages/e-means/integrations/cloudflare.mjs b/npm/packages/e-means/integrations/cloudflare.mjs new file mode 100644 index 0000000000..f8ed7db1e0 --- /dev/null +++ b/npm/packages/e-means/integrations/cloudflare.mjs @@ -0,0 +1,72 @@ +import { init, fit, predict } from '../runtime.mjs'; +import { workEstimate } from './clustering-validation.mjs'; +import { HTTPError, integer, LIMITS, plain, floats, fitOptions } from './cloudflare/validation.mjs'; +import { readJSON, response, tokenMatches } from './cloudflare/http.mjs'; + +export { persistCodebook, loadCodebook, exportCentroids, CentroidExportError } from './cloudflare/storage.mjs'; +export { LIMITS as WORKER_LIMITS } from './cloudflare/validation.mjs'; + +/** Private, stateless Worker handler. Pass a statically imported WASM module. */ +export function createClusteringWorker(options) { + const config = plain(options, ['wasmModule', 'bearerToken', 'allowUnauthenticated', + 'maxRequestBytes', 'maxWork', 'bodyTimeoutMs'], ['wasmModule'], 'Worker options'); + if (!(config.wasmModule instanceof WebAssembly.Module)) throw new TypeError('wasmModule must be a precompiled WebAssembly.Module'); + if (config.bearerToken !== undefined && (typeof config.bearerToken !== 'string' || + config.bearerToken.length < 24 || config.bearerToken.length > 1024 || !/^[\x21-\x7e]+$/.test(config.bearerToken))) { + throw new TypeError('bearerToken must be 24..1024 printable non-space ASCII characters'); + } + if (config.allowUnauthenticated !== undefined && typeof config.allowUnauthenticated !== 'boolean') { + throw new TypeError('allowUnauthenticated must be boolean'); + } + const maxBytes = integer(config.maxRequestBytes ?? 512 * 1024, 1, LIMITS.maxRequestBytes, 'maxRequestBytes'); + const maxWork = integer(config.maxWork ?? 10_000_000, 1, LIMITS.maxWork, 'maxWork'); + const timeoutMs = integer(config.bodyTimeoutMs ?? 5000, 1, 30_000, 'bodyTimeoutMs'); + const token = config.bearerToken; + return Object.freeze({ async fetch(request) { + try { + if (!(request instanceof Request)) throw new TypeError('request must be a Request'); + const url = new URL(request.url); + if (url.search) throw new HTTPError(400, 'invalid_query', 'Query parameters are unsupported'); + if (url.pathname === '/health') { + if (request.method !== 'GET') return response({ error: 'method_not_allowed' }, 405, { Allow: 'GET' }); + return response({ schema: 'ruvector.e-means-worker/1', stateless: true, + authenticatedOperations: Boolean(token), maxRequestBytes: maxBytes, maxWork }); + } + if (!['/fit', '/predict'].includes(url.pathname)) return response({ error: 'not_found' }, 404); + if (request.method !== 'POST') return response({ error: 'method_not_allowed' }, 405, { Allow: 'POST' }); + if (!token && config.allowUnauthenticated !== true) { + throw new HTTPError(503, 'authentication_unconfigured', 'Configure E_MEANS_BEARER_TOKEN before using operations'); + } + if (token && !tokenMatches(request.headers.get('authorization'), `Bearer ${token}`)) { + throw new HTTPError(401, 'unauthorized', 'A valid bearer token is required'); + } + const isFit = url.pathname === '/fit'; + const fields = isFit ? ['data', 'dimensions', 'options'] : ['data', 'dimensions', 'centroids']; + const body = plain(await readJSON(request, maxBytes, timeoutMs), fields, fields, 'request'); + const dimensions = integer(body.dimensions, 1, LIMITS.maxDimensions, 'dimensions'); + const data = floats(body.data, dimensions, 'data', !isFit); + if (isFit) { + const options = fitOptions(body.options, data.length / dimensions); + // Shared coordinate-scale admission estimate, including centroid + // statistics and O(k²) donor revival; not a hard CPU-clock limit. + const epochs = options.epochs ?? 3; + const work = workEstimate({ ...options, epochs }, data.length / dimensions, 0, dimensions); + if (work > maxWork) throw new HTTPError(413, 'work_budget_exceeded', 'Fit exceeds configured work budget'); + await init(config.wasmModule); + const model = fit(data, dimensions, options); + return response({ ...model, centroids: Array.from(model.centroids), assignments: Array.from(model.assignments) }); + } + const centroids = floats(body.centroids, dimensions, 'centroids'); + const clusters = integer(centroids.length / dimensions, 1, LIMITS.maxClusters, 'clusters'); + if (data.length + centroids.length > LIMITS.maxElements || data.length * clusters > maxWork) { + throw new HTTPError(413, 'work_budget_exceeded', 'Prediction exceeds configured work or element budget'); + } + await init(config.wasmModule); + return response({ assignments: Array.from(predict(data, dimensions, centroids)) }); + } catch (error) { + if (error instanceof HTTPError) return response({ error: error.code, message: error.message }, error.status); + if (error instanceof TypeError || error instanceof RangeError) return response({ error: 'invalid_input', message: error.message }, 422); + return response({ error: 'execution_failed', message: 'Clustering execution failed' }, 500); + } + } }); +} diff --git a/npm/packages/e-means/integrations/cloudflare/http.mjs b/npm/packages/e-means/integrations/cloudflare/http.mjs new file mode 100644 index 0000000000..3b7f58550e --- /dev/null +++ b/npm/packages/e-means/integrations/cloudflare/http.mjs @@ -0,0 +1,59 @@ +import { HTTPError } from './validation.mjs'; + +export function response(value, status = 200, headers = {}) { + return new Response(JSON.stringify(value), { status, headers: { + 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', ...headers, + } }); +} + +export function tokenMatches(value, token) { + if (typeof value !== 'string' || value.length !== token.length) return false; + let different = 0; + for (let index = 0; index < token.length; index++) different |= value.charCodeAt(index) ^ token.charCodeAt(index); + return different === 0; +} + +export async function readBytes(stream, limit, timeoutMs) { + if (!stream || typeof stream.getReader !== 'function') throw new HTTPError(400, 'invalid_body', 'A body is required'); + const reader = stream.getReader(); + let timer; + const cancel = () => { void reader.cancel('body rejected').catch(() => {}); }; + const read = async () => { + let bytes = new Uint8Array(Math.min(limit, 8192)), size = 0; + for (;;) { + const { value, done } = await reader.read(); + if (done) return bytes.subarray(0, size); + if (!(value instanceof Uint8Array)) throw new HTTPError(400, 'invalid_body', 'Body must contain bytes'); + if (value.byteLength > limit - size) { + cancel(); + throw new HTTPError(413, 'body_too_large', 'Body exceeds byte limit'); + } + const next = size + value.byteLength; + if (next > bytes.length) { + const grown = new Uint8Array(Math.min(limit, Math.max(next, bytes.length * 2))); + grown.set(bytes); bytes = grown; + } + bytes.set(value, size); size = next; + } + }; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => { cancel(); reject(new HTTPError(408, 'body_timeout', 'Body read timed out')); }, timeoutMs); + }); + try { return await Promise.race([read(), deadline]); } + finally { clearTimeout(timer); reader.releaseLock(); } +} + +export async function readJSON(request, limit, timeoutMs) { + const type = request.headers.get('content-type'); + if (!type || !/^application\/json(?:\s*;\s*charset\s*=\s*(?:utf-8|"utf-8"))?\s*$/i.test(type)) { + throw new HTTPError(415, 'unsupported_media_type', 'Content-Type must be UTF-8 application/json'); + } + const length = request.headers.get('content-length'); + if (length !== null && (!/^(0|[1-9][0-9]*)$/.test(length) || Number(length) > limit)) { + throw new HTTPError(413, 'body_too_large', 'Body exceeds byte limit'); + } + const bytes = await readBytes(request.body, limit, timeoutMs); + try { return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); } + catch { throw new HTTPError(400, 'invalid_json', 'Body must be valid UTF-8 JSON'); } +} diff --git a/npm/packages/e-means/integrations/cloudflare/storage.mjs b/npm/packages/e-means/integrations/cloudflare/storage.mjs new file mode 100644 index 0000000000..71ecee2799 --- /dev/null +++ b/npm/packages/e-means/integrations/cloudflare/storage.mjs @@ -0,0 +1,78 @@ +import { checkedCodebook, integer, LIMITS, plain, safeText } from './validation.mjs'; +import { readBytes } from './http.mjs'; + +/** Persist centroids only, never input vectors or assignments. Binding is caller-owned. */ +export async function persistCodebook(bucket, key, model) { + if (typeof bucket?.put !== 'function') throw new TypeError('an R2 put binding is required'); + safeText(key, 'key', 1024); + const checked = checkedCodebook(model); + const body = JSON.stringify({ ...checked, centroids: Array.from(checked.centroids) }); + const bytes = new TextEncoder().encode(body).byteLength; + if (bytes > LIMITS.maxCodebookBytes) throw new RangeError('codebook exceeds storage byte limit'); + const result = await bucket.put(key, body, { httpMetadata: { contentType: 'application/json' } }); + if (!result || typeof result !== 'object') throw new Error('R2 did not acknowledge the codebook write'); + return { key, dimensions: checked.dimensions, clusters: checked.clusters, bytes }; +} + +export async function loadCodebook(bucket, key) { + if (typeof bucket?.get !== 'function') throw new TypeError('an R2 get binding is required'); + safeText(key, 'key', 1024); + const stored = await bucket.get(key); + if (stored === null) return null; + if (!stored || !Number.isSafeInteger(stored.size) || stored.size < 0 || stored.size > LIMITS.maxCodebookBytes) { + throw new RangeError('stored codebook exceeds storage byte limit'); + } + const bytes = await readBytes(stored.body, LIMITS.maxCodebookBytes, 5000); + let parsed; + try { parsed = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); } + catch { throw new TypeError('stored codebook must be valid UTF-8 JSON'); } + const record = plain(parsed, ['schema', 'dimensions', 'clusters', 'centroids'], + ['schema', 'dimensions', 'clusters', 'centroids'], 'stored codebook'); + if (record.schema !== 'ruvector.e-means-codebook/1' || !Array.isArray(record.centroids) || + record.centroids.length > LIMITS.maxElements) throw new TypeError('unsupported stored codebook'); + for (const value of record.centroids) { + if (!Number.isFinite(value) || Math.abs(value) > 3.4028234663852886e38) throw new RangeError('stored centroids must be finite float32 values'); + } + const checked = checkedCodebook({ dimensions: record.dimensions, centroids: new Float32Array(record.centroids) }); + if (record.clusters !== checked.clusters) throw new TypeError('stored cluster count does not match centroid shape'); + return checked; +} + +export class CentroidExportError extends Error { + constructor(submitted, mutationIds, cause) { + super('Vectorize export failed after a partially submitted batch sequence', { cause }); + this.name = 'CentroidExportError'; this.submitted = submitted; this.mutationIds = [...mutationIds]; + } +} + +/** Submit centroids to Vectorize. Acknowledged mutations may not be query-visible yet. */ +export async function exportCentroids(index, model, options) { + if (typeof index?.upsert !== 'function') throw new TypeError('a Vectorize upsert binding is required'); + const config = plain(options, ['prefix', 'namespace', 'batchSize'], ['prefix'], 'Vectorize options'); + safeText(config.prefix, 'prefix', 64); + if (config.namespace !== undefined) safeText(config.namespace, 'namespace', 64); + const batchSize = integer(config.batchSize ?? 1000, 1, 1000, 'batchSize'); + const checked = checkedCodebook(model); + integer(checked.dimensions, 1, 1536, 'Vectorize dimensions'); + // Every ID uses the same prefix and a decimal ordinal; the last is longest. + safeText(`${config.prefix}:${checked.clusters - 1}`, 'vector id', 64); + let submitted = 0; + const mutationIds = []; + while (submitted < checked.clusters) { + const end = Math.min(checked.clusters, submitted + batchSize), vectors = []; + for (let ordinal = submitted; ordinal < end; ordinal++) { + const row = checked.centroids.subarray(ordinal * checked.dimensions, (ordinal + 1) * checked.dimensions); + vectors.push({ id: `${config.prefix}:${ordinal}`, values: Array.from(row), metadata: { cluster: ordinal }, + ...(config.namespace === undefined ? {} : { namespace: config.namespace }) }); + } + let result; + try { + result = await index.upsert(vectors); + safeText(result?.mutationId, 'Vectorize mutation id', 256); + } + catch (cause) { throw new CentroidExportError(submitted, mutationIds, cause); } + submitted = end; + mutationIds.push(result.mutationId); + } + return { submitted, mutationIds }; +} diff --git a/npm/packages/e-means/integrations/cloudflare/validation.mjs b/npm/packages/e-means/integrations/cloudflare/validation.mjs new file mode 100644 index 0000000000..bf054d7b7e --- /dev/null +++ b/npm/packages/e-means/integrations/cloudflare/validation.mjs @@ -0,0 +1,107 @@ +export const LIMITS = Object.freeze({ + maxRequestBytes: 1024 * 1024, + maxWork: 100_000_000, + maxRows: 16_384, + maxDimensions: 1536, + maxClusters: 256, + maxEpochs: 32, + maxElements: 262_144, + maxCodebookBytes: 8 * 1024 * 1024, +}); + +export class HTTPError extends Error { + constructor(status, code, message) { super(message); this.status = status; this.code = code; } +} + +export function plain(value, allowed, required, name) { + if (!value || typeof value !== 'object' || Array.isArray(value) || + ![Object.prototype, null].includes(Object.getPrototypeOf(value))) { + throw new TypeError(`${name} must be a plain object`); + } + const output = Object.create(null); + for (const key of Reflect.ownKeys(value)) { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (typeof key !== 'string' || !allowed.includes(key) || !Object.hasOwn(descriptor, 'value')) { + throw new TypeError(`unknown or accessor ${name} field`); + } + output[key] = descriptor.value; + } + if (required.some(key => !Object.hasOwn(output, key))) throw new TypeError(`missing ${name} field`); + return output; +} + +export function integer(value, min, max, name) { + if (!Number.isSafeInteger(value) || value < min || value > max) { + throw new RangeError(`${name} must be an integer in ${min}..${max}`); + } + return value; +} + +export function safeText(value, name, maxBytes) { + if (typeof value !== 'string' || value.length === 0 || /[\u0000-\u001f\u007f]/.test(value)) { + throw new TypeError(`${name} must be nonempty text without control characters`); + } + for (const character of value) { + const code = character.codePointAt(0); + if (code >= 0xd800 && code <= 0xdfff) throw new TypeError(`${name} must contain valid Unicode`); + } + if (new TextEncoder().encode(value).byteLength > maxBytes) throw new RangeError(`${name} exceeds ${maxBytes} UTF-8 bytes`); + return value; +} + +export function floats(values, dimensions, name, allowEmpty = false) { + if (!Array.isArray(values) || (!allowEmpty && values.length === 0) || + values.length > LIMITS.maxElements || values.length % dimensions !== 0) { + throw new RangeError(`${name} must be a bounded array of complete vectors`); + } + integer(values.length / dimensions, allowEmpty ? 0 : 1, LIMITS.maxRows, `${name} rows`); + for (const value of values) { + if (!Number.isFinite(value) || Math.abs(value) > 3.4028234663852886e38) { + throw new RangeError(`${name} must contain finite float32 values`); + } + } + return new Float32Array(values); +} + +export function fitOptions(value, rows) { + const options = plain(value, ['clusters', 'epochs', 'batchSize', 'peakStep', 'seed', 'finalization'], ['clusters'], 'fit options'); + integer(options.clusters, 1, Math.min(rows, LIMITS.maxClusters), 'clusters'); + if (options.epochs !== undefined) integer(options.epochs, 1, LIMITS.maxEpochs, 'epochs'); + if (options.batchSize !== undefined) integer(options.batchSize, 1, LIMITS.maxRows, 'batchSize'); + if (options.peakStep !== undefined && (!Number.isFinite(options.peakStep) || options.peakStep <= 0 || options.peakStep > 1)) { + throw new RangeError('peakStep must be in (0, 1]'); + } + if (options.seed !== undefined) integer(options.seed, 0, Number.MAX_SAFE_INTEGER, 'seed'); + if (options.finalization !== undefined && !['lastEpochMean', 'ema'].includes(options.finalization)) { + throw new TypeError('unsupported finalization'); + } + return options; +} + +export function checkedCodebook(value) { + if (!value || typeof value !== 'object') throw new TypeError('codebook is required'); + const schemaDescriptor = Object.getOwnPropertyDescriptor(value, 'schema'); + if (schemaDescriptor && (!Object.hasOwn(schemaDescriptor, 'value') || schemaDescriptor.value !== 'ruvector.e-means-codebook/1')) { + throw new TypeError('expected a single row-major codebook, not another model schema'); + } + const dimensionsDescriptor = Object.getOwnPropertyDescriptor(value, 'dimensions'); + const centroidsDescriptor = Object.getOwnPropertyDescriptor(value, 'centroids'); + if (!dimensionsDescriptor || !centroidsDescriptor || + !Object.hasOwn(dimensionsDescriptor, 'value') || !Object.hasOwn(centroidsDescriptor, 'value')) { + throw new TypeError('codebook dimensions and centroids must be own data properties'); + } + const dimensions = integer(dimensionsDescriptor.value, 1, LIMITS.maxElements, 'dimensions'); + const centroids = centroidsDescriptor.value; + if (!(centroids instanceof Float32Array) || centroids.length === 0 || + centroids.length > LIMITS.maxElements || centroids.length % dimensions !== 0) { + throw new RangeError('codebook must contain complete finite centroid rows'); + } + // Validate the owned snapshot, including when the caller supplies shared + // storage or overrides its typed-array iterator. + const owned = new Float32Array(centroids); + for (let index = 0; index < owned.length; index++) { + if (!Number.isFinite(owned[index])) throw new RangeError('centroids must be finite'); + } + return { schema: 'ruvector.e-means-codebook/1', dimensions, + clusters: owned.length / dimensions, centroids: owned }; +} diff --git a/npm/packages/e-means/integrations/clustering-validation.mjs b/npm/packages/e-means/integrations/clustering-validation.mjs new file mode 100644 index 0000000000..e7e058f72e --- /dev/null +++ b/npm/packages/e-means/integrations/clustering-validation.mjs @@ -0,0 +1,111 @@ +const PARAMS = ['clusters', 'epochs', 'batchSize', 'peakStep']; +const FIT_KEYS = [...PARAMS, 'seed', 'finalization']; +const MAX_U32 = 0xffff_ffff; +export const MAX_INPUT_BYTES = 64 * 1024 * 1024; +export const MAX_WORK = 1_000_000_000; + +export function object(value, allowed, name) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) throw new TypeError(`${name} must be an object`); + for (const key of Object.keys(value)) if (!allowed.includes(key)) throw new TypeError(`Unsupported ${name} field: ${key}`); + return value; +} + +export function integer(value, min, max, name) { + if (!Number.isSafeInteger(value) || value < min || value > max) throw new RangeError(`${name} must be an integer in ${min}..${max}`); + return value; +} + +function finite(value, min, max, name) { + if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max) throw new RangeError(`${name} must be finite in ${min}..${max}`); + return value; +} + +function rows(data, dimensions, name) { + if (!(data instanceof Float32Array)) throw new TypeError(`${name} must be a Float32Array`); + if (!data.length || data.length % dimensions) throw new RangeError(`${name} must contain nonempty complete vectors`); + for (let i = 0; i < data.length; i++) if (!Number.isFinite(data[i])) throw new RangeError(`${name} must contain only finite values`); + return data.length / dimensions; +} + +function config(value, samples) { + object(value, FIT_KEYS, 'baseline'); + const out = { clusters: value.clusters, epochs: value.epochs === undefined ? 3 : value.epochs, + seed: value.seed === undefined ? 42 : value.seed, finalization: value.finalization === undefined ? 'lastEpochMean' : value.finalization }; + integer(out.clusters, 1, samples, 'clusters'); + integer(out.epochs, 1, MAX_U32, 'epochs'); + integer(out.seed, 0, Number.MAX_SAFE_INTEGER, 'seed'); + if (!['lastEpochMean', 'ema'].includes(out.finalization)) throw new RangeError('Unsupported finalization'); + if (value.batchSize !== undefined) out.batchSize = integer(value.batchSize, 1, MAX_U32, 'batchSize'); + if (value.peakStep !== undefined) out.peakStep = finite(value.peakStep, Number.MIN_VALUE, 1, 'peakStep'); + return out; +} + +function domain(value, baseline, samples) { + object(value, PARAMS, 'domain'); + if (!Object.keys(value).length) throw new RangeError('domain must contain at least one tunable parameter'); + const result = {}; + for (const [name, supplied] of Object.entries(value)) { + object(supplied, ['min', 'max', 'type', 'scale', 'default'], `${name} domain`); + const min = finite(supplied.min, Number.MIN_VALUE, name === 'peakStep' ? 1 : MAX_U32, `${name}.min`); + const max = finite(supplied.max, min, name === 'peakStep' ? 1 : MAX_U32, `${name}.max`); + const type = name === 'peakStep' ? 'float' : 'int'; + if (supplied.type !== type || !['linear', 'log'].includes(supplied.scale)) throw new TypeError(`Invalid type or scale for ${name}`); + if (type === 'int') { integer(min, 1, MAX_U32, `${name}.min`); integer(max, min, MAX_U32, `${name}.max`); } + if (name === 'clusters' && max > samples) throw new RangeError('Cluster domain exceeds training rows'); + if (baseline[name] === undefined || baseline[name] < min || baseline[name] > max) throw new RangeError(`baseline ${name} must be explicitly inside its domain`); + if (supplied.default !== undefined && supplied.default !== baseline[name]) throw new RangeError(`${name}.default must match the baseline`); + result[name] = Object.freeze({ min, max, type, scale: supplied.scale, default: baseline[name] }); + } + return Object.freeze(result); +} + +// Coordinate-scale upper estimate, including short-batch statistics and O(k²) +// donor selection. It is an admission budget, not a measured instruction count. +export function workEstimate(config, n, holdout, d) { + const k = config.clusters, epochs = config.epochs; + // When automatic batch size is requested, B>=1 is a safe lower bound. + const batch = Math.min(config.batchSize ?? 1, n); + const updates = epochs * Math.ceil(n / batch); + const work = n * k * d * epochs + (n + holdout) * k * d + holdout * d + + n * d * (epochs + 1) + n * (epochs + 1) + + updates * k * (8 * d + 6) + Math.floor(updates / 5) * (3 * k * k + 6 * k * d + 3 * k); + if (!Number.isSafeInteger(work) || !Number.isSafeInteger(updates)) throw new RangeError('Clustering work estimate exceeds safe integer range'); + return work; +} + +export function checkOptions(value) { + object(value, ['training', 'holdout', 'dimensions', 'baseline', 'domain', 'budget', 'qualityTolerance'], 'evaluator options'); + const dimensions = integer(value.dimensions, 1, MAX_U32, 'dimensions'); + if (!(value.training instanceof Float32Array) || !(value.holdout instanceof Float32Array)) throw new TypeError('training and holdout must be Float32Array values'); + const bytes = value.training.byteLength + value.holdout.byteLength; + if (bytes > MAX_INPUT_BYTES) throw new RangeError('Input snapshots exceed 64 MiB'); + const n = rows(value.training, dimensions, 'training'), h = rows(value.holdout, dimensions, 'holdout'); + if (value.training.buffer === value.holdout.buffer && value.training.byteOffset < value.holdout.byteOffset + value.holdout.byteLength && + value.holdout.byteOffset < value.training.byteOffset + value.training.byteLength) throw new RangeError('training and holdout must use separate, nonoverlapping rows'); + const baseline = Object.freeze(config(value.baseline, n)); + const space = domain(value.domain, baseline, n); + object(value.budget, ['maxEvaluations', 'maxDistanceCoordinates', 'timeoutMs'], 'budget'); + const budget = Object.freeze({ maxEvaluations: integer(value.budget.maxEvaluations, 1, 32, 'maxEvaluations'), + maxDistanceCoordinates: integer(value.budget.maxDistanceCoordinates, 1, MAX_WORK, 'maxDistanceCoordinates'), + timeoutMs: integer(value.budget.timeoutMs === undefined ? 30_000 : value.budget.timeoutMs, 1, 120_000, 'timeoutMs') }); + const worst = { ...baseline }; + for (const [name, spec] of Object.entries(space)) worst[name] = name === 'batchSize' ? spec.min : spec.max; + const worstCaseWorkPerTrial = workEstimate(worst, n, h, dimensions); + if (worstCaseWorkPerTrial > budget.maxDistanceCoordinates) throw new RangeError(`Domain permits work ${worstCaseWorkPerTrial} above maxDistanceCoordinates`); + const qualityTolerance = finite(value.qualityTolerance === undefined ? 0 : value.qualityTolerance, 0, 1, 'qualityTolerance'); + return { training: new Float32Array(value.training), holdout: new Float32Array(value.holdout), dimensions, n, h, baseline, + domain: space, budget, qualityTolerance, numericSnapshotBytes: bytes, worstCaseWorkPerTrial }; +} + +export function checkGenome(value, space) { + const keys = Object.keys(space); + object(value, keys, 'genome'); + if (Object.keys(value).length !== keys.length) throw new RangeError('Genome must include every domain parameter'); + const out = {}; + for (const [name, spec] of Object.entries(space)) { + out[name] = spec.type === 'int' ? integer(value[name], spec.min, spec.max, name) : finite(value[name], spec.min, spec.max, name); + } + return out; +} + +export const clone = value => structuredClone(value); diff --git a/npm/packages/e-means/integrations/ivf.d.ts b/npm/packages/e-means/integrations/ivf.d.ts new file mode 100644 index 0000000000..c1345fb8ee --- /dev/null +++ b/npm/packages/e-means/integrations/ivf.d.ts @@ -0,0 +1,16 @@ +import type { EMeansModel, EMeansOptions } from '../e-means.js'; + +/** Caller-owned structured-clone-compatible records; postingList validates on every operation. */ +export interface IVFPartition { + schema: 'ruvector-e-means-ivf-v1'; + model: EMeansModel; + /** Length clusters+1. Cluster j occupies indices[offsets[j]..offsets[j+1]). */ + offsets: Uint32Array; + /** Original input indices, in ascending order within every cluster. */ + indices: Uint32Array; +} + +/** Requires init() from the root package. Trains centroids and returns owned posting arrays. */ +export function partition(data: Float32Array, dimensions: number, options: EMeansOptions): IVFPartition; +/** Validates shape, assignment consistency and stable unique indices; returns an owned list. */ +export function postingList(value: IVFPartition, cluster: number): Uint32Array; diff --git a/npm/packages/e-means/integrations/ivf.mjs b/npm/packages/e-means/integrations/ivf.mjs new file mode 100644 index 0000000000..f3fbec8217 --- /dev/null +++ b/npm/packages/e-means/integrations/ivf.mjs @@ -0,0 +1,63 @@ +import { fit } from '../e-means.mjs'; + +const schema = 'ruvector-e-means-ivf-v1'; + +/** Train coarse centroids and construct stable posting lists from exact Rust/WASM assignments. */ +export function partition(data, dimensions, options) { + const model = fit(data, dimensions, options); + const clusters = model.centroids.length / dimensions; + const offsets = new Uint32Array(clusters + 1); + for (const cluster of model.assignments) offsets[cluster + 1]++; + for (let cluster = 0; cluster < clusters; cluster++) offsets[cluster + 1] += offsets[cluster]; + const cursors = offsets.slice(0, clusters); + const indices = new Uint32Array(model.assignments.length); + for (let row = 0; row < model.assignments.length; row++) { + indices[cursors[model.assignments[row]]++] = row; + } + return { schema, model, offsets, indices }; +} + +function validated(value) { + if (value === null || typeof value !== 'object' || value.schema !== schema) { + throw new TypeError(`partition must use schema ${schema}`); + } + const { model, offsets, indices } = value; + if (model === null || typeof model !== 'object') throw new TypeError('partition must contain a model'); + const { dimensions, centroids, assignments, mse, batchSize, peakStep, updates, revivedClusters } = model; + if (!Number.isInteger(dimensions) || dimensions < 1 || dimensions > 0xffff_ffff) throw new RangeError('model dimensions must be positive'); + if (!(centroids instanceof Float32Array) || centroids.length === 0 || centroids.length % dimensions !== 0) { + throw new RangeError('model centroids must be complete Float32Array vectors'); + } + for (const coordinate of centroids) if (!Number.isFinite(coordinate)) throw new RangeError('model centroids must be finite'); + if (!Number.isFinite(mse) || mse < 0 || !Number.isFinite(peakStep) || peakStep <= 0 || peakStep > 1 || + !Number.isSafeInteger(batchSize) || batchSize < 1 || !Number.isSafeInteger(updates) || updates < 1 || + !Number.isSafeInteger(revivedClusters) || revivedClusters < 0) throw new RangeError('model metrics are invalid'); + if (!(assignments instanceof Uint32Array) || !(offsets instanceof Uint32Array) || !(indices instanceof Uint32Array)) { + throw new TypeError('assignments, offsets and indices must be Uint32Array values'); + } + const clusters = centroids.length / dimensions; + if (offsets.length !== clusters + 1 || offsets[0] !== 0 || offsets[clusters] !== indices.length || indices.length !== assignments.length) { + throw new RangeError('posting offsets must cover every model assignment exactly once'); + } + for (let cluster = 0; cluster < clusters; cluster++) { + const start = offsets[cluster]; + const end = offsets[cluster + 1]; + if (end < start || end > indices.length) throw new RangeError('posting offsets must be monotonic'); + let previous = -1; + for (let cursor = start; cursor < end; cursor++) { + const row = indices[cursor]; + if (row <= previous || row >= assignments.length || assignments[row] !== cluster) { + throw new RangeError('posting indices must be unique, stable and consistent with model assignments'); + } + previous = row; + } + } + return { clusters, offsets, indices }; +} + +/** Return an owned copy of one posting list after validating the caller-owned partition. */ +export function postingList(value, cluster) { + const { clusters, offsets, indices } = validated(value); + if (!Number.isInteger(cluster) || cluster < 0 || cluster >= clusters) throw new RangeError('cluster is outside the partition'); + return indices.slice(offsets[cluster], offsets[cluster + 1]); +} diff --git a/npm/packages/e-means/integrations/metaharness.d.ts b/npm/packages/e-means/integrations/metaharness.d.ts new file mode 100644 index 0000000000..58db50b900 --- /dev/null +++ b/npm/packages/e-means/integrations/metaharness.d.ts @@ -0,0 +1,115 @@ +import type { EMeansOptions } from '@ruvector/e-means'; + +export interface ClusteringParameter { + min: number; + max: number; + type: 'int' | 'float'; + scale: 'linear' | 'log'; + /** If supplied, must equal the baseline's value for this parameter. */ + default?: number; +} + +export interface ClusteringBudget { + /** Includes the baseline and every admitted dispatch. Integer 1..32. */ + maxEvaluations: number; + /** Conservative scalar-work allowance, including short-batch centroid updates and revival; at most 1e9. */ + maxDistanceCoordinates: number; + /** Cooperative deadline in milliseconds, 1..120000; default 30000. A synchronous fit cannot be preempted. */ + timeoutMs?: number; +} + +export interface ClusteringEvaluationOptions { + training: Float32Array; + /** Nonempty, nonoverlapping validation rows. Selection uses this split; supply another test split for generalization claims. */ + holdout: Float32Array; + dimensions: number; + baseline: EMeansOptions; + /** Searched parameters need explicit baseline values; peakStep uses float, all other fields use int. */ + domain: Partial>; + budget: ClusteringBudget; + /** Relative holdout-MSE degradation allowance in [0,1], default 0. */ + qualityTolerance?: number; +} + +export interface ClusteringScoreCard { + variantId: string; + /** Higher is better: 1/(1+holdoutMse). */ + primary: number; + regressed: boolean; + noopRate: 0; + costPerWin: 0; + raw: { + fitConfig: EMeansOptions; + trainMse: number; + holdoutMse: number; + baselineHoldoutMse: number; + fitMs: number; + holdoutMs: number; + estimatedWork: number; + updates: number; + revivedClusters: number; + trainingRows: number; + holdoutRows: number; + dimensions: number; + }; +} + +export interface ClusteringTrial { + variantId: string; + genome: Record; + score: ClusteringScoreCard; +} + +export interface ClusteringEvaluationReceipt { + schema: 'e-means.clustering-evaluation/1'; + baseline?: ClusteringTrial; + trials: ClusteringTrial[]; + baselineGenome: Record; + baselineConfig: EMeansOptions; + domain: Record; + dimensions: number; + trainingRows: number; + holdoutRows: number; + qualityTolerance: number; + budget: Required; + dispatchCount: number; + consumedWork: number; + worstCaseWorkPerTrial: number; + numericSnapshotBytes: number; + sourceApplied: false; + scope: string; + timingScope: string; + limits: { maxInputBytes: number; maxDistanceCoordinates: number; cooperativeDeadline: true; workUnit: string }; +} + +/** Structurally compatible with @metaharness/darwin's NumericEvaluator. */ +export interface ClusteringEvaluator { + readonly baselineGenome: Readonly>; + readonly domain: Readonly>>; + evaluate(genome: Readonly>, variantId: string): Promise; + /** Returns an independent snapshot. Changes never affect evaluator state. */ + receipt(): ClusteringEvaluationReceipt; +} + +export interface ClusteringTuningOptions extends ClusteringEvaluationOptions { + generations: number; + childrenPerGeneration: number; + mutationSeed?: number; +} + +export interface ClusteringTuningReceipt extends Omit { + schema: 'e-means.clustering-tuning/1'; + engine: { package: '@metaharness/darwin'; version: '0.13.0' | '0.13.1'; algorithm: 'evolveNumeric' }; + baseline: ClusteringTrial; + winner: ClusteringTrial; + recommendation: EMeansOptions; + winnerRejected: boolean; + improvedOnObservedHoldout: boolean; + search: { generations: number; childrenPerGeneration: number; mutationSeed: number; worstCaseDispatches: number; concurrency: 1 }; + temporaryArchiveRemoved: true; +} + +/** Copies inputs/configuration synchronously. First evaluate() must use baselineGenome. No SDK import occurs here. */ +export function createClusteringEvaluator(options: ClusteringEvaluationOptions): ClusteringEvaluator; +/** Node-only bounded search using the actual optional @metaharness/darwin 0.13.0 or 0.13.1 peer. Never applies a model. */ +export function tuneClustering(options: ClusteringTuningOptions): Promise; diff --git a/npm/packages/e-means/integrations/metaharness.mjs b/npm/packages/e-means/integrations/metaharness.mjs new file mode 100644 index 0000000000..5a14ba0ad5 --- /dev/null +++ b/npm/packages/e-means/integrations/metaharness.mjs @@ -0,0 +1,154 @@ +import { init, fit, predict } from '../e-means.mjs'; +import { checkOptions, checkGenome, clone, integer, object, workEstimate, MAX_INPUT_BYTES, MAX_WORK } from './clustering-validation.mjs'; + +function holdoutMse(data, dimensions, centroids) { + const assignments = predict(data, dimensions, centroids); + let loss = 0; + for (let row = 0; row < assignments.length; row++) { + const centroid = assignments[row] * dimensions; + for (let d = 0; d < dimensions; d++) { + const delta = data[row * dimensions + d] - centroids[centroid + d]; + loss += delta * delta; + } + } + return loss / assignments.length; +} + +/** NumericEvaluator compatible with published Darwin 0.13.0 and 0.13.1. Snapshots happen + * synchronously before any await. The first dispatch must be the baseline. + */ +export function createClusteringEvaluator(options) { + const config = checkOptions(options); + const baselineGenome = Object.freeze(Object.fromEntries(Object.keys(config.domain).map(key => [key, config.baseline[key]]))); + const trials = [], ids = new Set(); + let active = false, calls = 0, consumedWork = 0, baseline; + const started = performance.now(); + const check = () => { + if (performance.now() - started > config.budget.timeoutMs) throw new Error('Clustering cooperative deadline exceeded'); + }; + const receipt = () => clone({ schema: 'e-means.clustering-evaluation/1', baseline, trials, + baselineGenome, baselineConfig: config.baseline, domain: config.domain, + dimensions: config.dimensions, trainingRows: config.n, holdoutRows: config.h, + qualityTolerance: config.qualityTolerance, budget: config.budget, + dispatchCount: calls, consumedWork, worstCaseWorkPerTrial: config.worstCaseWorkPerTrial, + numericSnapshotBytes: config.numericSnapshotBytes, sourceApplied: false, + scope: 'Caller-supplied training and holdout snapshots. Holdout guides selection and is not an independent generalization test.', + timingScope: 'One synchronous Rust/WASM fit and one separate holdout prediction/MSE calculation per dispatch; initialization excluded.', + limits: { maxInputBytes: MAX_INPUT_BYTES, maxDistanceCoordinates: MAX_WORK, cooperativeDeadline: true, + workUnit: 'Conservative coordinate-scale work estimate including centroid updates and donor selection; not measured instructions.' } }); + + const evaluator = { + baselineGenome, + domain: config.domain, + async evaluate(genome, variantId) { + check(); + if (active) throw new Error('Concurrent clustering evaluations are not supported'); + if (calls >= config.budget.maxEvaluations) throw new RangeError('Clustering evaluation budget exhausted'); + calls++; + if (typeof variantId !== 'string' || !variantId || variantId.length > 128 || ids.has(variantId)) throw new TypeError('variantId must be a unique nonempty label of at most 128 characters'); + const selected = checkGenome(genome, config.domain); + if (!baseline && JSON.stringify(selected) !== JSON.stringify(baselineGenome)) throw new Error('First evaluation must use baselineGenome'); + const fitConfig = { ...config.baseline, ...selected }; + const work = workEstimate(fitConfig, config.n, config.h, config.dimensions); + if (work > config.budget.maxDistanceCoordinates - consumedWork) throw new RangeError('Clustering coordinate work budget exhausted'); + ids.add(variantId); consumedWork += work; active = true; + try { + await init(); + check(); + const fitStart = performance.now(); + const model = fit(config.training, config.dimensions, fitConfig); + const fitMs = performance.now() - fitStart; + check(); + const scoreStart = performance.now(); + const mse = holdoutMse(config.holdout, config.dimensions, model.centroids); + const holdoutMs = performance.now() - scoreStart; + check(); + if (!Number.isFinite(mse) || mse < 0) throw new Error('Nonfinite clustering evaluation'); + const baselineMse = baseline?.score.raw.holdoutMse ?? mse; + const score = { variantId, primary: 1 / (1 + mse), regressed: mse > baselineMse * (1 + config.qualityTolerance), + noopRate: 0, costPerWin: 0, raw: { fitConfig, trainMse: model.mse, holdoutMse: mse, baselineHoldoutMse: baselineMse, + fitMs, holdoutMs, estimatedWork: work, updates: model.updates, revivedClusters: model.revivedClusters, + trainingRows: config.n, holdoutRows: config.h, dimensions: config.dimensions } }; + const trial = clone({ variantId, genome: selected, score }); + trials.push(trial); + if (!baseline) baseline = trial; + return clone(score); + } finally { active = false; } + }, + receipt, + }; + return Object.freeze(evaluator); +} + +// Exact versions whose numeric API this integration was run against. +const DARWIN_VERSIONS = ['0.13.0', '0.13.1']; + +async function loadDarwin() { + const { readFile } = await import('node:fs/promises'); + let entry; + try { entry = import.meta.resolve('@metaharness/darwin'); } + catch (cause) { throw new Error(`tuneClustering requires optional peer @metaharness/darwin (${DARWIN_VERSIONS.join(' or ')})`, { cause }); } + const manifest = JSON.parse(await readFile(new URL('../package.json', entry), 'utf8')); + if (manifest.name !== '@metaharness/darwin' || !DARWIN_VERSIONS.includes(manifest.version)) throw new Error(`Expected @metaharness/darwin ${DARWIN_VERSIONS.join(' or ')}`); + const { evolveNumeric } = await import('@metaharness/darwin'); + if (typeof evolveNumeric !== 'function') throw new Error('Darwin numeric API is unavailable'); + return { evolveNumeric, version: manifest.version }; +} + +/** Run bounded, local numeric evolution. Never mutates source or applies a model. */ +export async function tuneClustering(options) { + const started = performance.now(); + object(options, ['training', 'holdout', 'dimensions', 'baseline', 'domain', 'budget', 'qualityTolerance', 'generations', 'childrenPerGeneration', 'mutationSeed'], 'tuning options'); + const { generations, childrenPerGeneration, mutationSeed = 0, ...evaluation } = options; + integer(generations, 1, 8, 'generations'); + integer(childrenPerGeneration, 1, 8, 'childrenPerGeneration'); + integer(mutationSeed, 0, 0xffff_ffff, 'mutationSeed'); + const evaluator = createClusteringEvaluator(evaluation); + const initial = evaluator.receipt(); + const check = () => { + if (performance.now() - started > initial.budget.timeoutMs) throw new Error('Clustering cooperative deadline exceeded'); + }; + const worstCaseDispatches = 1 + childrenPerGeneration + 2 * childrenPerGeneration * (generations - 1); + if (worstCaseDispatches > initial.budget.maxEvaluations) throw new RangeError('Darwin worst-case dispatches exceed maxEvaluations'); + if (worstCaseDispatches * initial.worstCaseWorkPerTrial > initial.budget.maxDistanceCoordinates) throw new RangeError('Darwin worst-case work exceeds maxDistanceCoordinates'); + const { evolveNumeric, version: darwinVersion } = await loadDarwin(); + check(); + const { mkdtemp, rm } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + let archive, failure, answer; + try { + archive = await mkdtemp(join(tmpdir(), 'e-means-darwin-')); + check(); + const result = await evolveNumeric({ genomeSpec: evaluator.domain, baselineGenome: evaluator.baselineGenome, + evaluator: { evaluate: async (...args) => { + try { return await evaluator.evaluate(...args); } + catch (error) { failure = error; throw error; } + } }, generations, childrenPerGeneration, seed: mutationSeed, concurrency: 1, + mutationSigma: 0.2, crossover: false, workRoot: archive }); + if (failure) throw failure; + check(); + const measured = evaluator.receipt(); + if (!Array.isArray(result?.records) || result.records.length !== measured.trials.length || measured.dispatchCount !== measured.trials.length) throw new Error('Darwin archive differs from observed evaluations'); + const observed = new Map(measured.trials.map(trial => [trial.variantId, trial])); + const verified = record => { + const trial = observed.get(record?.variant?.id); + if (!trial || JSON.stringify(trial.genome) !== JSON.stringify(record.variant.genome) || JSON.stringify(trial.score) !== JSON.stringify(record.score)) throw new Error('Darwin returned an unobserved result'); + return trial; + }; + for (const record of result.records) verified(record); + const baseline = verified(result.baseline); + if (JSON.stringify(baseline.genome) !== JSON.stringify(evaluator.baselineGenome)) throw new Error('Darwin changed the requested baseline'); + let winner = result.winner ? verified(result.winner) : baseline; + const winnerRejected = winner.score.regressed || winner.score.primary < baseline.score.primary; + if (winnerRejected) winner = baseline; + answer = { engine: { package: '@metaharness/darwin', version: darwinVersion, algorithm: 'evolveNumeric' }, + ...measured, schema: 'e-means.clustering-tuning/1', baseline, winner, + recommendation: clone(winner.score.raw.fitConfig), winnerRejected, + improvedOnObservedHoldout: winner.score.primary > baseline.score.primary, + search: { generations, childrenPerGeneration, mutationSeed, worstCaseDispatches, concurrency: 1 }, + sourceApplied: false, temporaryArchiveRemoved: true }; + } finally { if (archive) await rm(archive, { recursive: true, force: true }); } + check(); + return answer; +} diff --git a/npm/packages/e-means/integrations/quantization.d.ts b/npm/packages/e-means/integrations/quantization.d.ts new file mode 100644 index 0000000000..b1aa5ad55e --- /dev/null +++ b/npm/packages/e-means/integrations/quantization.d.ts @@ -0,0 +1,29 @@ +import type { EMeansOptions } from '../e-means.js'; + +/** Caller-owned mutable records. Every codec operation validates their shape and finite coordinates. */ +export interface ProductQuantizer { + schema: 'ruvector-e-means-pq-v1'; + dimensions: number; + subspaces: number; + /** 1..256, shared by every subspace. */ + clusters: number; + /** Flat subspace-major codebooks: subspaces × clusters × (dimensions/subspaces). */ + centroids: Float32Array; +} + +export interface ProductQuantizerJSON { + schema: 'ruvector-e-means-pq-v1'; + dimensions: number; + subspaces: number; + clusters: number; + centroids: number[]; +} + +/** Requires init() from the root package. Input data is borrowed; returned centroids are owned. */ +export function trainProductQuantizer(data: Float32Array, dimensions: number, subspaces: number, options: EMeansOptions): ProductQuantizer; +/** Requires init(). Codes are row-major, rows × subspaces bytes. Empty input is supported. */ +export function encodeProductQuantizer(model: ProductQuantizer, data: Float32Array): Uint8Array; +/** No runtime initialization required. Returns an owned reconstruction; empty codes are supported. */ +export function decodeProductQuantizer(model: ProductQuantizer, codes: Uint8Array): Float32Array; +export function productQuantizerToJSON(model: ProductQuantizer): ProductQuantizerJSON; +export function productQuantizerFromJSON(value: unknown): ProductQuantizer; diff --git a/npm/packages/e-means/integrations/quantization.mjs b/npm/packages/e-means/integrations/quantization.mjs new file mode 100644 index 0000000000..85537def17 --- /dev/null +++ b/npm/packages/e-means/integrations/quantization.mjs @@ -0,0 +1,144 @@ +import { fit, predict } from '../e-means.mjs'; + +const schema = 'ruvector-e-means-pq-v1'; +const maxU32 = 0xffff_ffff; + +function positiveInteger(value, name, maximum = maxU32) { + if (!Number.isInteger(value) || value < 1 || value > maximum) { + throw new RangeError(`${name} must be an integer in 1..${maximum}`); + } +} + +function shape(dimensions, subspaces, clusters) { + positiveInteger(dimensions, 'dimensions'); + positiveInteger(subspaces, 'subspaces', dimensions); + positiveInteger(clusters, 'clusters', 256); + if (dimensions % subspaces !== 0) throw new RangeError('dimensions must be divisible by subspaces'); + return dimensions / subspaces; +} + +function vectors(data, dimensions, allowEmpty = false) { + if (!(data instanceof Float32Array)) throw new TypeError('data must be a Float32Array'); + if ((!allowEmpty && data.length === 0) || data.length % dimensions !== 0) { + throw new RangeError('data must contain complete vectors'); + } + for (const value of data) if (!Number.isFinite(value)) throw new RangeError('data must be finite'); +} + +function validated(model) { + if (model === null || typeof model !== 'object' || model.schema !== schema) { + throw new TypeError(`model must use schema ${schema}`); + } + const { dimensions, subspaces, clusters, centroids } = model; + const subDimensions = shape(dimensions, subspaces, clusters); + if (!(centroids instanceof Float32Array)) throw new TypeError('centroids must be a Float32Array'); + if (centroids.length !== dimensions * clusters) throw new RangeError('centroid count does not match the codebook shape'); + for (const value of centroids) if (!Number.isFinite(value)) throw new RangeError('centroids must be finite'); + return { dimensions, subspaces, clusters, centroids, subDimensions }; +} + +function gather(data, dimensions, subDimensions, subspace, target) { + for (let row = 0; row < data.length / dimensions; row++) { + const start = row * dimensions + subspace * subDimensions; + target.set(data.subarray(start, start + subDimensions), row * subDimensions); + } +} + +function snapshotOptions(options) { + if (options === null || typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('options must contain clusters'); + } + // Read caller properties once so all subspaces use one stable configuration. + const config = { ...options }; + const supported = ['clusters', 'epochs', 'batchSize', 'peakStep', 'seed', 'finalization']; + for (const key of Object.keys(config)) if (!supported.includes(key)) throw new TypeError(`Unknown option: ${key}`); + if (config.epochs !== undefined) positiveInteger(config.epochs, 'epochs'); + if (config.batchSize !== undefined) positiveInteger(config.batchSize, 'batchSize'); + if (config.peakStep !== undefined && + (typeof config.peakStep !== 'number' || !Number.isFinite(config.peakStep) || config.peakStep <= 0 || config.peakStep > 1)) { + throw new RangeError('peakStep must be finite and in (0, 1]'); + } + if (config.seed !== undefined && (!Number.isSafeInteger(config.seed) || config.seed < 0)) throw new RangeError('seed must be a non-negative safe integer'); + if (config.finalization !== undefined && !['lastEpochMean', 'ema'].includes(config.finalization)) throw new RangeError('unsupported finalization'); + return config; +} + +/** Train independent subspace codebooks using the shared Rust/WASM fitter. Requires init(). */ +export function trainProductQuantizer(data, dimensions, subspaces, options) { + const config = snapshotOptions(options); + const clusters = config.clusters; + const subDimensions = shape(dimensions, subspaces, clusters); + vectors(data, dimensions); + const rows = data.length / dimensions; + if (clusters > rows) throw new RangeError('clusters cannot exceed the number of vectors'); + const centroids = new Float32Array(clusters * dimensions); + const subvector = new Float32Array(rows * subDimensions); + for (let subspace = 0; subspace < subspaces; subspace++) { + gather(data, dimensions, subDimensions, subspace, subvector); + const model = fit(subvector, subDimensions, config); + centroids.set(model.centroids, subspace * clusters * subDimensions); + } + return { schema, dimensions, subspaces, clusters, centroids }; +} + +/** Encode to row-major, one-byte-per-subspace codes using exact Rust/WASM assignment. */ +export function encodeProductQuantizer(model, data) { + const { dimensions, subspaces, clusters, centroids, subDimensions } = validated(model); + vectors(data, dimensions, true); + const rows = data.length / dimensions; + const codes = new Uint8Array(rows * subspaces); + const subvector = new Float32Array(rows * subDimensions); + for (let subspace = 0; subspace < subspaces; subspace++) { + gather(data, dimensions, subDimensions, subspace, subvector); + const start = subspace * clusters * subDimensions; + const assignments = predict(subvector, subDimensions, centroids.subarray(start, start + clusters * subDimensions)); + for (let row = 0; row < rows; row++) codes[row * subspaces + subspace] = assignments[row]; + } + return codes; +} + +/** Reconstruct row-major vectors from a validated codebook and byte codes. Does not require init(). */ +export function decodeProductQuantizer(model, codes) { + const { dimensions, subspaces, clusters, centroids, subDimensions } = validated(model); + if (!(codes instanceof Uint8Array)) throw new TypeError('codes must be a Uint8Array'); + if (codes.length % subspaces !== 0) throw new RangeError('codes must contain complete subspace groups'); + for (const code of codes) if (code >= clusters) throw new RangeError('code exceeds the codebook cluster count'); + const rows = codes.length / subspaces; + if (rows * dimensions > maxU32) throw new RangeError('decoded output exceeds the supported element count'); + const data = new Float32Array(rows * dimensions); + for (let row = 0; row < rows; row++) { + for (let subspace = 0; subspace < subspaces; subspace++) { + const code = codes[row * subspaces + subspace]; + const start = (subspace * clusters + code) * subDimensions; + data.set(centroids.subarray(start, start + subDimensions), row * dimensions + subspace * subDimensions); + } + } + return data; +} + +/** Return owned JSON-compatible codebook data. */ +export function productQuantizerToJSON(model) { + const { dimensions, subspaces, clusters, centroids } = validated(model); + return { schema, dimensions, subspaces, clusters, centroids: Array.from(centroids) }; +} + +/** Restore a validated, owned typed codebook from parsed JSON. */ +export function productQuantizerFromJSON(value) { + if (value === null || typeof value !== 'object' || value.schema !== schema) { + throw new TypeError(`model must use schema ${schema}`); + } + const { dimensions, subspaces, clusters, centroids } = value; + shape(dimensions, subspaces, clusters); + if (!Array.isArray(centroids) || centroids.length !== dimensions * clusters) { + throw new RangeError('JSON centroids must be an array matching the codebook shape'); + } + const owned = new Float32Array(centroids.length); + for (let index = 0; index < centroids.length; index++) { + const coordinate = centroids[index]; + if (typeof coordinate !== 'number' || !Number.isFinite(coordinate)) throw new RangeError('JSON centroids must be finite numbers'); + owned[index] = coordinate; + } + const model = { schema, dimensions, subspaces, clusters, centroids: owned }; + validated(model); + return model; +} diff --git a/npm/packages/e-means/package-lock.json b/npm/packages/e-means/package-lock.json new file mode 100644 index 0000000000..f5b191bec2 --- /dev/null +++ b/npm/packages/e-means/package-lock.json @@ -0,0 +1,1534 @@ +{ + "name": "@ruvector/e-means", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@ruvector/e-means", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@metaharness/darwin": "0.13.0", + "esbuild": "0.28.2", + "miniflare": "4.20260730.0", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@metaharness/darwin": "0.13.0 || 0.13.1" + }, + "peerDependenciesMeta": { + "@metaharness/darwin": { + "optional": true + } + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.13", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.13.tgz", + "integrity": "sha512-fWegr+WsEsiMVW4KUNdIIzzMTZSzNiqhy0//6+2MRi2d5NFNsL/AYwBQW8eBnRbTBVRnDVzjaJEUqKaCWjMMdQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.13", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.13.tgz", + "integrity": "sha512-fJM50YYjlTiBhgdkGmC1tBpv+FMAyo4annf2X6Aks+0yRlaiVkYW/3f07xAfVVkYIoZ9fFZp+mEThUFnuybhkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.11" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.11", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.11.tgz", + "integrity": "sha512-WCsemW40Cpc6gouC1SrY6PBaZ+oLRlEVY9XBvZBI7il/HyUrb2nJ505wE1MHEFyDZNnrWDEuy4jL50WVHfTTSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.11", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260730.1.tgz", + "integrity": "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260730.1.tgz", + "integrity": "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260730.1.tgz", + "integrity": "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260730.1.tgz", + "integrity": "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260730.1.tgz", + "integrity": "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz", + "integrity": "sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz", + "integrity": "sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.4" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz", + "integrity": "sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz", + "integrity": "sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz", + "integrity": "sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz", + "integrity": "sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz", + "integrity": "sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz", + "integrity": "sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz", + "integrity": "sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz", + "integrity": "sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz", + "integrity": "sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz", + "integrity": "sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz", + "integrity": "sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz", + "integrity": "sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz", + "integrity": "sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz", + "integrity": "sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz", + "integrity": "sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz", + "integrity": "sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz", + "integrity": "sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz", + "integrity": "sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz", + "integrity": "sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", + "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.5.tgz", + "integrity": "sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz", + "integrity": "sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz", + "integrity": "sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz", + "integrity": "sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@metaharness/darwin": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@metaharness/darwin/-/darwin-0.13.0.tgz", + "integrity": "sha512-L3SrQii70erHVDmo1qx0875IRpVujk86zAO7sxtSVChWCgAjabEmmYRYt72D1VekfUCbSWCgZKIktqUxVAKCFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.13" + }, + "bin": { + "metaharness-darwin": "dist/cli.js" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.24", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.24.tgz", + "integrity": "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/miniflare": { + "version": "4.20260730.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260730.0.tgz", + "integrity": "sha512-1Z9SB9r/o//80UA02Re3QhtcecSHAyAjf5EcKBfQVlQrCg7Miy79hl2PvtkwFLIaJ5rcrOPdDcRr577okwZPsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.28.0", + "workerd": "1.20260730.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/semver": { + "version": "7.8.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.6.tgz", + "integrity": "sha512-9lpSSU4Mly3mrtruwLs1eLK6+gzbzqeBFwRNjIjE+9LuDpWdYctrQlQ3ej1Ilgi7gknpsTRnPoLP8DpCHqHIJw==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", + "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.5", + "@img/sharp-darwin-x64": "0.35.5", + "@img/sharp-freebsd-wasm32": "0.35.5", + "@img/sharp-libvips-darwin-arm64": "1.3.4", + "@img/sharp-libvips-darwin-x64": "1.3.4", + "@img/sharp-libvips-linux-arm": "1.3.4", + "@img/sharp-libvips-linux-arm64": "1.3.4", + "@img/sharp-libvips-linux-ppc64": "1.3.4", + "@img/sharp-libvips-linux-riscv64": "1.3.4", + "@img/sharp-libvips-linux-s390x": "1.3.4", + "@img/sharp-libvips-linux-x64": "1.3.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4", + "@img/sharp-libvips-linuxmusl-x64": "1.3.4", + "@img/sharp-linux-arm": "0.35.5", + "@img/sharp-linux-arm64": "0.35.5", + "@img/sharp-linux-ppc64": "0.35.5", + "@img/sharp-linux-riscv64": "0.35.5", + "@img/sharp-linux-s390x": "0.35.5", + "@img/sharp-linux-x64": "0.35.5", + "@img/sharp-linuxmusl-arm64": "0.35.5", + "@img/sharp-linuxmusl-x64": "0.35.5", + "@img/sharp-webcontainers-wasm32": "0.35.5", + "@img/sharp-win32-arm64": "0.35.5", + "@img/sharp-win32-ia32": "0.35.5", + "@img/sharp-win32-x64": "0.35.5" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tslib": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.9.1.tgz", + "integrity": "sha512-a4Ux9VKuIMHsOS7ENbEq6hK2ZGYIT27Rtl0E4eD9KqwxhKiQeMZE/9xb3IdYwWaSOyIBQLExhQgXDTYoiCxnWA==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/workerd": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260730.1.tgz", + "integrity": "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260730.1", + "@cloudflare/workerd-darwin-arm64": "1.20260730.1", + "@cloudflare/workerd-linux-64": "1.20260730.1", + "@cloudflare/workerd-linux-arm64": "1.20260730.1", + "@cloudflare/workerd-windows-64": "1.20260730.1" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } + } + } +} diff --git a/npm/packages/e-means/package.json b/npm/packages/e-means/package.json new file mode 100644 index 0000000000..8dcf9ca38f --- /dev/null +++ b/npm/packages/e-means/package.json @@ -0,0 +1,120 @@ +{ + "name": "@ruvector/e-means", + "version": "0.1.0", + "description": "Deterministic e-means (EMA k-means) clustering for vectors: Rust core compiled to WebAssembly, with PQ codebook, IVF, MetaHarness tuning and Cloudflare Worker helpers", + "type": "module", + "main": "./e-means.mjs", + "types": "./e-means.d.ts", + "exports": { + ".": { + "types": "./e-means.d.ts", + "import": "./e-means.mjs" + }, + "./runtime": { + "types": "./runtime.d.ts", + "import": "./runtime.mjs" + }, + "./quantization": { + "types": "./integrations/quantization.d.ts", + "import": "./integrations/quantization.mjs" + }, + "./ivf": { + "types": "./integrations/ivf.d.ts", + "import": "./integrations/ivf.mjs" + }, + "./metaharness": { + "types": "./integrations/metaharness.d.ts", + "import": "./integrations/metaharness.mjs" + }, + "./cloudflare": { + "types": "./integrations/cloudflare.d.ts", + "import": "./integrations/cloudflare.mjs" + }, + "./cloudflare/build": { + "types": "./examples/cloudflare/build.d.ts", + "import": "./examples/cloudflare/build.mjs" + }, + "./wasm": { + "types": "./wasm-module.d.ts", + "default": "./wasm/ruvector_e_means_bg.wasm" + } + }, + "files": [ + "e-means.mjs", + "e-means.d.ts", + "runtime.mjs", + "runtime.d.ts", + "wasm-module.d.ts", + "integrations/", + "examples/cloudflare/", + "examples/metaharness.mjs", + "wasm/", + "scripts/check-artifact.mjs", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "node scripts/build.mjs", + "test": "node --test test/*.test.mjs", + "test:consumer": "node test/packed-consumer.mjs", + "test:ecosystem": "node test/ecosystem-consumers.mjs", + "test:cloudflare": "node --test test/cloudflare*.test.mjs", + "build:cloudflare": "node examples/cloudflare/build.mjs", + "prepack": "node scripts/check-artifact.mjs", + "typecheck": "tsc --noEmit -p test/tsconfig.json", + "test:browser": "node test/browser-consumer.mjs", + "benchmark": "node --expose-gc test/benchmark.mjs" + }, + "engines": { + "node": ">=20.0.0" + }, + "license": "MIT", + "homepage": "https://ruv.io", + "bugs": { + "url": "https://github.com/ruvnet/ruvector/issues" + }, + "repository": { + "type": "git", + "url": "https://github.com/ruvnet/ruvector.git", + "directory": "npm/packages/e-means" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "clustering", + "e-means", + "k-means", + "ema", + "webassembly", + "wasm", + "rust", + "product-quantization", + "ivf", + "vector-search", + "codebook", + "embeddings", + "cloudflare-workers", + "ruvector" + ], + "devDependencies": { + "typescript": "5.9.3", + "@metaharness/darwin": "0.13.0", + "esbuild": "0.28.2", + "miniflare": "4.20260730.0" + }, + "peerDependencies": { + "@metaharness/darwin": "0.13.0 || 0.13.1" + }, + "peerDependenciesMeta": { + "@metaharness/darwin": { + "optional": true + } + }, + "overrides": { + "miniflare": { + "sharp": "0.35.5", + "undici": "7.30.1" + } + } +} diff --git a/npm/packages/e-means/runtime.d.ts b/npm/packages/e-means/runtime.d.ts new file mode 100644 index 0000000000..ab711476c2 --- /dev/null +++ b/npm/packages/e-means/runtime.d.ts @@ -0,0 +1,4 @@ +export { fit, predict } from '@ruvector/e-means'; +export type { EMeansOptions, EMeansModel } from '@ruvector/e-means'; +/** No filesystem/default URL loader. Supply bytes or a precompiled module. */ +export function init(source: BufferSource | WebAssembly.Module): Promise; diff --git a/npm/packages/e-means/runtime.mjs b/npm/packages/e-means/runtime.mjs new file mode 100644 index 0000000000..d79984c862 --- /dev/null +++ b/npm/packages/e-means/runtime.mjs @@ -0,0 +1,96 @@ +import initWasm, { fitTyped, predictFlat } from './wasm/ruvector_e_means.js'; + +let initialized = false; +let initialization; +const maxU32 = 0xffff_ffff; + +/** Initialize from caller-supplied bytes or a precompiled module; no Node loader is reachable. */ +export async function init(source) { + if (initialized) return; + if (!initialization) { + initialization = (async () => { + if (source === undefined) throw new TypeError('runtime init requires WASM bytes or a precompiled module'); + await initWasm({ module_or_path: source }); + initialized = true; + })().catch((error) => { + initialization = undefined; + const message = typeof error?.message === 'string' ? error.message : String(error); + throw new Error(`Could not initialize @ruvector/e-means: ${message}`, { cause: error }); + }); + } + await initialization; +} + +function ready() { + if (!initialized) throw new Error('Call and await init() before using @ruvector/e-means'); +} + +function positiveInteger(value, name) { + if (!Number.isInteger(value) || value <= 0 || value > maxU32) { + throw new RangeError(`${name} must be an integer between 1 and ${maxU32}`); + } +} + +function vectors(data, dimensions, name, allowEmpty = false) { + if (!(data instanceof Float32Array)) throw new TypeError(`${name} must be a Float32Array`); + positiveInteger(dimensions, 'dimensions'); + if ((!allowEmpty && data.length === 0) || data.length % dimensions !== 0) { + throw new RangeError(`${name} must contain complete ${dimensions}-dimensional vectors`); + } + for (let index = 0; index < data.length; index++) { + if (!Number.isFinite(data[index])) throw new RangeError(`${name} must contain only finite values`); + } +} + +/** Cluster flat row-major f32 vectors using the e-means Rust implementation. */ +export function fit(data, dimensions, options) { + ready(); + vectors(data, dimensions, 'data'); + if (options === null || typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('options must be an object with clusters'); + } + const allowed = ['clusters', 'epochs', 'batchSize', 'peakStep', 'seed', 'finalization']; + for (const key of Object.keys(options)) { + if (!allowed.includes(key)) throw new TypeError(`Unknown option: ${key}`); + } + positiveInteger(options.clusters, 'clusters'); + if (options.clusters > data.length / dimensions) throw new RangeError('clusters cannot exceed the number of vectors'); + if (options.epochs !== undefined) positiveInteger(options.epochs, 'epochs'); + if (options.batchSize !== undefined) positiveInteger(options.batchSize, 'batchSize'); + if (options.peakStep !== undefined && + (typeof options.peakStep !== 'number' || !Number.isFinite(options.peakStep) || options.peakStep <= 0 || options.peakStep > 1)) { + throw new RangeError('peakStep must be a finite number in (0, 1]'); + } + if (options.seed !== undefined && (!Number.isSafeInteger(options.seed) || options.seed < 0)) { + throw new RangeError('seed must be a non-negative safe integer'); + } + if (options.finalization !== undefined && !['lastEpochMean', 'ema'].includes(options.finalization)) { + throw new RangeError('finalization must be lastEpochMean or ema'); + } + // Copy supported options explicitly: inherited properties and toJSON do not become Rust configuration. + const config = { clusters: options.clusters }; + for (const key of allowed.slice(1)) if (options[key] !== undefined) config[key] = options[key]; + const model = fitTyped(data, dimensions, JSON.stringify(config)); + try { + return { + centroids: model.takeCentroids(), + assignments: model.takeAssignments(), + dimensions: model.dimensions, + mse: model.mse, + batchSize: model.batchSize, + peakStep: model.peakStep, + updates: model.updates, + revivedClusters: model.revivedClusters, + }; + } finally { + model.free(); + } +} + +/** Assign vectors to their nearest centroid using squared Euclidean distance. */ +export function predict(data, dimensions, centroids) { + ready(); + vectors(data, dimensions, 'data', true); + vectors(centroids, dimensions, 'centroids'); + return predictFlat(data, dimensions, centroids); +} diff --git a/npm/packages/e-means/scripts/build.mjs b/npm/packages/e-means/scripts/build.mjs new file mode 100644 index 0000000000..4f907425eb --- /dev/null +++ b/npm/packages/e-means/scripts/build.mjs @@ -0,0 +1,23 @@ +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { mkdir } from 'node:fs/promises'; + +const root = fileURLToPath(new URL('../../../../', import.meta.url)); +const manifest = fileURLToPath(new URL('../../../../crates/ruvector-e-means/Cargo.toml', import.meta.url)); +const target = fileURLToPath(new URL('../../../../target/e-means-wasm/', import.meta.url)); +const output = fileURLToPath(new URL('../wasm/', import.meta.url)); + +function run(command, args) { + const result = spawnSync(command, args, { cwd: root, stdio: 'inherit' }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`${command} exited ${result.status}`); +} + +const version = spawnSync('wasm-bindgen', ['--version'], { encoding: 'utf8' }); +if (version.error || version.status !== 0 || version.stdout.trim() !== 'wasm-bindgen 0.2.125') { + throw new Error('Install wasm-bindgen-cli@0.2.125 and the wasm32-unknown-unknown Rust target before building'); +} +await mkdir(output, { recursive: true }); +run('cargo', ['build', '--locked', '--manifest-path', manifest, '--release', '--target', 'wasm32-unknown-unknown', '--features', 'wasm', '--target-dir', target]); +run('wasm-bindgen', [`${target}/wasm32-unknown-unknown/release/ruvector_e_means.wasm`, '--target', 'web', '--out-dir', output, '--out-name', 'ruvector_e_means']); +run(process.execPath, [fileURLToPath(new URL('./check-artifact.mjs', import.meta.url))]); diff --git a/npm/packages/e-means/scripts/check-artifact.mjs b/npm/packages/e-means/scripts/check-artifact.mjs new file mode 100644 index 0000000000..d87a846a50 --- /dev/null +++ b/npm/packages/e-means/scripts/check-artifact.mjs @@ -0,0 +1,11 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { init, fit } from '../e-means.mjs'; + +const bytes = await readFile(new URL('../wasm/ruvector_e_means_bg.wasm', import.meta.url)); +assert.deepEqual([...bytes.subarray(0, 4)], [0, 97, 115, 109], 'missing or invalid WebAssembly artifact'); +await init(bytes); +const model = fit(new Float32Array([0, 2]), 1, { clusters: 1, epochs: 1 }); +assert.equal(model.centroids[0], 1, 'bundled WASM must run the Rust mean finalizer'); +assert.equal(model.mse, 1); +console.error('Bundled e-means WASM verified'); diff --git a/npm/packages/e-means/test/benchmark.mjs b/npm/packages/e-means/test/benchmark.mjs new file mode 100644 index 0000000000..3a572d30ba --- /dev/null +++ b/npm/packages/e-means/test/benchmark.mjs @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { cpus } from 'node:os'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { init, fit, predict } from '../e-means.mjs'; +import { fitJson, fitTyped } from '../wasm/ruvector_e_means.js'; + +let baselinePath, repeats = 7; +for (let i = 2; i < process.argv.length; i++) { + const argument = process.argv[i]; + if (argument === '--baseline') baselinePath = process.argv[++i]; + else if (argument === '--repeats') repeats = Number(process.argv[++i]); + else throw new Error(`Unknown benchmark argument: ${argument}`); +} +assert.ok(Number.isInteger(repeats) && repeats >= 3 && repeats <= 31, '--repeats must be in 3..31'); +await init(); +let baseline; +if (baselinePath) { + baseline = await import(pathToFileURL(resolve(baselinePath, 'e-means.mjs')).href); + await baseline.init(); +} + +function materialize(handle) { + try { + return { + centroids: handle.takeCentroids(), assignments: handle.takeAssignments(), + dimensions: handle.dimensions, mse: handle.mse, batchSize: handle.batchSize, + peakStep: handle.peakStep, updates: handle.updates, revivedClusters: handle.revivedClusters, + }; + } finally { + handle.free(); + } +} + +function median(values) { + return [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; +} + +// Interleave variants and rotate their order to reduce ordering/thermal bias. +// GC is outside the timed region; each result is fully materialized and consumed. +let checksum = 0; +function measure(variants) { + const entries = Object.entries(variants), timings = new Map(entries.map(([name]) => [name, []])); + for (const [, operation] of entries) for (let i = 0; i < 2; i++) operation(); + for (let round = 0; round < repeats; round++) { + for (let i = 0; i < entries.length; i++) { + const [name, operation] = entries[(i + round) % entries.length]; + globalThis.gc?.(); + const start = performance.now(); + const result = operation(); + timings.get(name).push(performance.now() - start); + checksum += result instanceof Uint32Array ? result.length + (result[0] || 0) : result.mse + result.assignments.length; + } + } + return Object.fromEntries([...timings].map(([name, samples]) => [name, median(samples)])); +} + +function compareResults(a, b) { + assert.equal(a.assignments.length, b.assignments.length); + assert.equal(a.centroids.length, b.centroids.length); + let labelDifferences = 0, centroidMaxAbsDifference = 0; + for (let i = 0; i < a.assignments.length; i++) labelDifferences += Number(a.assignments[i] !== b.assignments[i]); + for (let i = 0; i < a.centroids.length; i++) centroidMaxAbsDifference = Math.max(centroidMaxAbsDifference, Math.abs(a.centroids[i] - b.centroids[i])); + return { labelDifferences, centroidMaxAbsDifference, mseAbsoluteDifference: Math.abs(a.mse - b.mse) }; +} + +const cases = [ + { name: 'large-assignment-result', samples: 1_000_000, dimensions: 1, clusters: 1, epochs: 1, batchSize: 4096 }, + { name: 'embedding-codebook', samples: 8192, dimensions: 64, clusters: 16, epochs: 3, batchSize: 256 }, +]; +const results = []; +for (const scenario of cases) { + const { samples, dimensions, clusters, epochs, batchSize } = scenario; + const data = new Float32Array(samples * dimensions); + for (let row = 0; row < samples; row++) { + for (let column = 0; column < dimensions; column++) { + data[row * dimensions + column] = (row % clusters) * 4 + (column % 7) * 0.1 + ((row * 17 + column * 31) % 97) / 9700; + } + } + const options = { clusters, epochs, batchSize, seed: 42 }, config = JSON.stringify(options); + const optimizedModel = fit(data, dimensions, options); + const jsonResult = JSON.parse(fitJson(data, dimensions, config)); + const jsonModel = { ...jsonResult, centroids: Float32Array.from(jsonResult.centroids), assignments: Uint32Array.from(jsonResult.assignments) }; + assert.deepEqual(optimizedModel, jsonModel, 'same optimized kernel must have identical JSON/typed results'); + const baselineModel = baseline?.fit(data, dimensions, options); + if (baselineModel) assert.deepEqual(optimizedModel, baselineModel, 'optimized fit must preserve baseline outputs on the measured fixture'); + const fitVariants = { optimizedMs: () => fit(data, dimensions, options) }; + if (baseline) fitVariants.baselineMs = () => baseline.fit(data, dimensions, options); + const fitTimes = measure(fitVariants); + if (baseline) fitTimes.speedup = fitTimes.baselineMs / fitTimes.optimizedMs; + + const transport = measure({ + jsonMs: () => { + const model = JSON.parse(fitJson(data, dimensions, config)); + return { ...model, centroids: Float32Array.from(model.centroids), assignments: Uint32Array.from(model.assignments) }; + }, + typedMs: () => materialize(fitTyped(data, dimensions, config)), + }); + transport.speedup = transport.jsonMs / transport.typedMs; + const predictVariants = { optimizedMs: () => predict(data, dimensions, optimizedModel.centroids) }; + if (baseline) predictVariants.baselineMs = () => baseline.predict(data, dimensions, baselineModel.centroids); + const predictTimes = measure(predictVariants); + if (baseline) predictTimes.speedup = predictTimes.baselineMs / predictTimes.optimizedMs; + results.push({ ...scenario, fit: fitTimes, predict: predictTimes, sameKernelTransport: transport, + baselineParity: baselineModel ? compareResults(optimizedModel, baselineModel) : undefined }); +} +console.log(JSON.stringify({ runtime: { node: process.version, platform: process.platform, arch: process.arch, + cpu: cpus()[0]?.model, logicalCpus: cpus().length }, repeats, warmupsPerVariant: 2, + measurement: 'median wall-clock milliseconds, fully materialized results; explicit GC outside timing', + sameKernelTransportMeasurement: 'full fit plus output materialization using one optimized WASM kernel; difference includes JSON serialization/parsing and array conversion, not a conversion-only timer', + baselinePath: baselinePath ? resolve(baselinePath) : undefined, checksum, cases: results }, null, 2)); diff --git a/npm/packages/e-means/test/browser-consumer.mjs b/npm/packages/e-means/test/browser-consumer.mjs new file mode 100644 index 0000000000..a38a304477 --- /dev/null +++ b/npm/packages/e-means/test/browser-consumer.mjs @@ -0,0 +1,90 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const page = ``; + +const routes = new Map([ + ['/e-means.mjs', ['../e-means.mjs', 'application/javascript']], + ['/wasm/ruvector_e_means.js', ['../wasm/ruvector_e_means.js', 'application/javascript']], + ['/wasm/ruvector_e_means_bg.wasm', ['../wasm/ruvector_e_means_bg.wasm', 'application/wasm']], +]); +let fetchedWasm = false; +const server = createServer(async (request, response) => { + try { + if (request.url === '/') { + response.writeHead(200, { 'content-type': 'text/html' }); + response.end(page); + return; + } + const route = routes.get(request.url); + if (!route) { + response.writeHead(404); + response.end(); + return; + } + if (request.url.endsWith('.wasm')) fetchedWasm = true; + const bytes = await readFile(new URL(route[0], import.meta.url)); + response.writeHead(200, { 'content-type': route[1] }); + response.end(bytes); + } catch (error) { + response.writeHead(500); + response.end(error.message); + } +}); + +const profile = await mkdtemp(join(tmpdir(), 'e-means-chromium-')); +let browser; +function stopBrowser() { + if (!browser?.pid) return; + try { process.kill(-browser.pid, 'SIGKILL'); } catch (error) { + if (error.code !== 'ESRCH') throw error; + } +} +try { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const chromium = process.env.E_MEANS_CHROMIUM || 'chromium'; + const url = `http://127.0.0.1:${server.address().port}/`; + const result = await new Promise((resolve, reject) => { + const child = browser = spawn(chromium, ['--headless', '--no-sandbox', '--disable-dev-shm-usage', + '--disable-gpu', '--disable-background-networking', '--no-first-run', + `--user-data-dir=${profile}`, '--dump-dom', '--virtual-time-budget=10000', url], + { stdio: ['ignore', 'pipe', 'pipe'], detached: true }); + let stdout = '', stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + const timer = setTimeout(() => { + stopBrowser(); + reject(new Error(`Chromium browser consumer exceeded 25 seconds\n${stdout}\n${stderr.slice(-2000)}`)); + }, 25_000); + child.once('error', (error) => { clearTimeout(timer); reject(error); }); + child.once('close', (status) => { clearTimeout(timer); resolve({ status, stdout, stderr }); }); + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /data-result="pass"/, result.stdout + '\n' + result.stderr.slice(-1500)); + assert.ok(fetchedWasm, 'browser must fetch the bundled .wasm through default init()'); + console.log('Chromium browser consumer passed: default WASM fetch, fit, predict and numerical oracle'); +} finally { + stopBrowser(); + await new Promise((resolve) => server.close(resolve)); + await rm(profile, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); +} diff --git a/npm/packages/e-means/test/cloudflare-storage.test.mjs b/npm/packages/e-means/test/cloudflare-storage.test.mjs new file mode 100644 index 0000000000..90b01c8acb --- /dev/null +++ b/npm/packages/e-means/test/cloudflare-storage.test.mjs @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { persistCodebook, loadCodebook, exportCentroids, CentroidExportError } from '../integrations/cloudflare.mjs'; + +const model = { dimensions: 2, centroids: new Float32Array([1, 2, 3, 4]), assignments: new Uint32Array([0, 1]), mse: 1 }; +function stored(body) { return { size: new TextEncoder().encode(body).length, body: new Response(body).body }; } + +test('R2 binding helpers persist only centroid codebooks and restore independent typed output', async () => { + let written; + const bucket = { async put(key, body, options) { written = { key, body, options }; return {}; }, async get() { return stored(written.body); } }; + const receipt = await persistCodebook(bucket, 'private/codebook.json', model); + assert.equal(receipt.clusters, 2); + const record = JSON.parse(written.body); + assert.deepEqual(Object.keys(record).sort(), ['centroids', 'clusters', 'dimensions', 'schema']); + assert.equal(written.options.httpMetadata.contentType, 'application/json'); + const loaded = await loadCodebook(bucket, 'private/codebook.json'); + assert.deepEqual(loaded.centroids, model.centroids); + loaded.centroids[0] = 99; + assert.equal(model.centroids[0], 1); + assert.equal(await loadCodebook({ get: async () => null }, 'missing'), null); +}); + +test('R2 writes snapshot caller-owned centroid data before the first asynchronous boundary', async () => { + const input = { dimensions: 1, centroids: new Float32Array([3, 7]) }; + let payload; + const promise = persistCodebook({ async put(key, body) { payload = body; await Promise.resolve(); return {}; } }, 'snapshot', input); + input.centroids.fill(100); + await promise; + assert.deepEqual(JSON.parse(payload).centroids, [3, 7]); +}); + +test('centroid validation inspects copied numeric storage even with an overridden iterator', async () => { + const centroids = new Float32Array([1, NaN]); + centroids[Symbol.iterator] = function* () { yield 0; }; + const input = { dimensions: 1, centroids }; + let writes = 0; + await assert.rejects(persistCodebook({ put: async () => { writes++; return {}; } }, 'bad', input), /finite/); + await assert.rejects(exportCentroids({ upsert: async () => { writes++; return { mutationId: 'bad' }; } }, input, { prefix: 'bad' }), /finite/); + assert.equal(writes, 0); +}); + +test('PQ envelopes and accessor schemas cannot be mistaken for full-dimensional codebooks', async () => { + let writes = 0, reads = 0; + const bucket = { put: async () => { writes++; return {}; } }; + const index = { upsert: async () => { writes++; return { mutationId: 'unexpected' }; } }; + const pq = { schema: 'ruvector-e-means-pq-v1', dimensions: 2, centroids: new Float32Array([1, 2, 3, 4]) }; + const accessor = { dimensions: 2, centroids: new Float32Array([1, 2]) }; + Object.defineProperty(accessor, 'schema', { get() { reads++; return 'ruvector.e-means-codebook/1'; } }); + for (const input of [pq, accessor]) { + await assert.rejects(persistCodebook(bucket, 'book', input), /single row-major codebook/); + await assert.rejects(exportCentroids(index, input, { prefix: 'book' }), /single row-major codebook/); + } + assert.equal(writes, 0); + assert.equal(reads, 0); +}); + +test('corrupt, oversized, nonfinite and malformed stored codebooks reject', async () => { + for (const record of [ + { schema: 'other', dimensions: 1, clusters: 1, centroids: [0] }, + { schema: 'ruvector.e-means-codebook/1', dimensions: 2, clusters: 1, centroids: [0] }, + { schema: 'ruvector.e-means-codebook/1', dimensions: 1, clusters: 2, centroids: [0] }, + { schema: 'ruvector.e-means-codebook/1', dimensions: 1, clusters: 1, centroids: ['1'] }, + { schema: 'ruvector.e-means-codebook/1', dimensions: 1, clusters: 1, centroids: [1e40] }, + ]) await assert.rejects(loadCodebook({ get: async () => stored(JSON.stringify(record)) }, 'bad')); + await assert.rejects(loadCodebook({ get: async () => ({ size: 10_000_000, body: new Response('x').body }) }, 'large')); + await assert.rejects(persistCodebook({ put: async () => null }, 'failed', model), /acknowledge/); + await assert.rejects(persistCodebook({ put: async () => undefined }, 'failed', model), /acknowledge/); + let writes = 0; + await assert.rejects(persistCodebook({ put: async () => { writes++; } }, 'invalid', { dimensions: 1, centroids: new Float32Array([Infinity]) })); + assert.equal(writes, 0); +}); + +test('Vectorize binding export batches static centroid records and propagates explicit namespace', async () => { + const batches = []; + const index = { async upsert(vectors) { batches.push(structuredClone(vectors)); return { mutationId: `m${batches.length}` }; } }; + const receipt = await exportCentroids(index, { dimensions: 1, centroids: Float32Array.from({ length: 2001 }, (_, i) => i) }, + { prefix: 'codebook', namespace: 'owned-namespace' }); + assert.deepEqual(batches.map(batch => batch.length), [1000, 1000, 1]); + assert.deepEqual(receipt, { submitted: 2001, mutationIds: ['m1', 'm2', 'm3'] }); + assert.deepEqual(batches[2][0], { id: 'codebook:2000', values: [2000], metadata: { cluster: 2000 }, namespace: 'owned-namespace' }); + assert.equal(new Set(batches.flat().map(v => v.id)).size, 2001); +}); + +test('Vectorize validates every record before writes and reports acknowledged batches on failure', async () => { + let calls = 0; + const index = { async upsert() { if (++calls === 2) throw new Error('binding failure'); return { mutationId: 'first' }; } }; + for (const [input, options] of [ + [model, { prefix: '😀'.repeat(16) }], [model, { prefix: 'x', namespace: '😀'.repeat(17) }], + [model, { prefix: 'x', batchSize: 1001 }], + [{ dimensions: 1537, centroids: new Float32Array(1537) }, { prefix: 'x' }], + [{ dimensions: 1, centroids: new Float32Array([NaN]) }, { prefix: 'x' }], + ]) await assert.rejects(exportCentroids(index, input, options)); + assert.equal(calls, 0); + await assert.rejects(exportCentroids(index, { dimensions: 1, centroids: new Float32Array(1001) }, { prefix: 'x' }), error => { + assert.ok(error instanceof CentroidExportError); + assert.equal(error.submitted, 1000); + assert.deepEqual(error.mutationIds, ['first']); + return true; + }); + for (const acknowledgement of [undefined, {}, { mutationId: '' }, { mutationId: 'x'.repeat(257) }]) { + await assert.rejects(exportCentroids({ upsert: async () => acknowledgement }, model, { prefix: 'x' }), error => { + assert.ok(error instanceof CentroidExportError); + assert.equal(error.submitted, 0); + assert.deepEqual(error.mutationIds, []); + return true; + }); + } +}); diff --git a/npm/packages/e-means/test/cloudflare-tsconfig.json b/npm/packages/e-means/test/cloudflare-tsconfig.json new file mode 100644 index 0000000000..6fa7091d6d --- /dev/null +++ b/npm/packages/e-means/test/cloudflare-tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "lib": ["ES2022", "DOM"], + "types": [] + }, + "include": ["cloudflare-types.ts"] +} diff --git a/npm/packages/e-means/test/cloudflare-types.ts b/npm/packages/e-means/test/cloudflare-types.ts new file mode 100644 index 0000000000..e9b6eabcc5 --- /dev/null +++ b/npm/packages/e-means/test/cloudflare-types.ts @@ -0,0 +1,25 @@ +import type { EMeansModel } from '@ruvector/e-means'; +import { + createClusteringWorker, persistCodebook, loadCodebook, exportCentroids, + type CodebookInput, type R2Binding, type VectorizeBinding, +} from '@ruvector/e-means/cloudflare'; + +declare const wasm: WebAssembly.Module; +declare const model: EMeansModel; +declare const request: Request; +const worker = createClusteringWorker({ wasmModule: wasm, maxWork: 1_000_000, bodyTimeoutMs: 1000 }); +const response: Promise = worker.fetch(request); +const bucket: R2Binding = { put: async () => ({}), get: async () => null }; +const index: VectorizeBinding = { upsert: async () => ({ mutationId: 'acknowledgement' }) }; +const input: CodebookInput = model; +void response; +void persistCodebook(bucket, 'model', input); +void loadCodebook(bucket, 'model'); +void exportCentroids(index, model, { prefix: 'model', namespace: 'owned' }); +// @ts-expect-error Worker configuration numbers cannot be strings. +createClusteringWorker({ wasmModule: wasm, maxWork: 'unbounded' }); +// @ts-expect-error Codebook storage requires owned float32 centroid arrays. +persistCodebook(bucket, 'model', { dimensions: 1, centroids: [0, 1] }); +// @ts-expect-error Vectorize returns an acknowledgement identity. +const invalidIndex: VectorizeBinding = { upsert: async () => ({}) }; +void invalidIndex; diff --git a/npm/packages/e-means/test/cloudflare-workerd.test.mjs b/npm/packages/e-means/test/cloudflare-workerd.test.mjs new file mode 100644 index 0000000000..eaf0df63f8 --- /dev/null +++ b/npm/packages/e-means/test/cloudflare-workerd.test.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { mkdtemp, readFile, rm, writeFile, mkdir, symlink } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { Miniflare } from 'miniflare'; +import { buildWorker } from '../examples/cloudflare/build.mjs'; + +const token = 'e-means-workerd-test-not-for-production'; +const config = { compatibilityDate: '2026-07-30', modules: true, + modulesRules: [{ type: 'CompiledWasm', include: ['**/*.wasm'], fallthrough: true }] }; + +test('packed npm static Rust WASM executes in real workerd without Node compatibility', { timeout: 60_000 }, async t => { + const temporary = await mkdtemp(join(tmpdir(), 'e-means-workerd-')); + const packageDir = fileURLToPath(new URL('../', import.meta.url)); + const run = (args, cwd) => { + const result = spawnSync('npm', args, { cwd, encoding: 'utf8', env: { ...process.env, npm_config_workspaces: 'false' } }); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + return result.stdout; + }; + const archive = JSON.parse(run(['pack', '--json', '--workspaces=false', '--pack-destination', temporary], packageDir))[0]; + const consumer = join(temporary, 'consumer'); + await mkdir(consumer); + await writeFile(join(consumer, 'package.json'), JSON.stringify({ private: true, type: 'module' })); + run(['install', '--ignore-scripts', '--no-audit', '--no-fund', '--offline', join(temporary, archive.filename)], consumer); + // esbuild is caller-supplied build tooling, not an installed runtime dependency. + await symlink(dirname(dirname(fileURLToPath(import.meta.resolve('esbuild')))), join(consumer, 'node_modules/esbuild'), 'dir'); + const installed = join(consumer, 'node_modules/@ruvector/e-means'); + const builder = await import(pathToFileURL(join(installed, 'examples/cloudflare/build.mjs')).href); + const output = join(temporary, 'output'); + const artifact = await builder.buildWorker(output); + const source = await readFile(artifact.script, 'utf8'); + assert.match(source, /\.\/e-means\.wasm/); + assert.doesNotMatch(source, /node:fs|node:crypto|node:os/); + const worker = new Miniflare({ ...config, modulesRoot: output, scriptPath: artifact.script, + bindings: { E_MEANS_BEARER_TOKEN: token } }); + t.after(async () => { await worker.dispose(); await rm(temporary, { recursive: true, force: true }); }); + const invoke = (path, value, authorization = `Bearer ${token}`) => worker.dispatchFetch(`https://worker.test${path}`, { + method: 'POST', headers: { 'content-type': 'application/json', authorization }, body: JSON.stringify(value), + }); + const health = await worker.dispatchFetch('https://worker.test/health'); + assert.equal(health.status, 200); + assert.equal((await health.json()).stateless, true); + const fitted = await invoke('/fit', { data: [1, 2, 3, 4, 5, 6], dimensions: 2, options: { clusters: 1 } }); + assert.equal(fitted.status, 200, await fitted.clone().text()); + const model = await fitted.json(); + assert.deepEqual(model.centroids, [3, 4]); + assert.deepEqual(model.assignments, [0, 0, 0]); + assert.equal(model.mse, 16 / 3); + const predicted = await invoke('/predict', { data: [1, 1, 9, 9], dimensions: 2, centroids: [0, 0, 10, 10] }); + assert.deepEqual(await predicted.json(), { assignments: [0, 1] }); + assert.equal((await invoke('/fit', {}, 'Bearer incorrect')).status, 401); + assert.equal((await invoke('/fit', { data: [1, 2], dimensions: 0, options: { clusters: 1 } })).status, 422); +}); + +test('R2 codebook helpers use the real local binding and Vectorize contract remains explicit', { timeout: 60_000 }, async t => { + const temporary = await mkdtemp(join(tmpdir(), 'e-means-r2-workerd-')); + const entry = join(temporary, 'entry.mjs'); + const adapter = fileURLToPath(new URL('../integrations/cloudflare.mjs', import.meta.url)); + await writeFile(entry, ` +import { persistCodebook, loadCodebook } from ${JSON.stringify(adapter)}; +export default { async fetch(request, env) { + const original = { dimensions: 2, centroids: new Float32Array([1, 2, 3, 4]) }; + const receipt = await persistCodebook(env.CODEBOOKS, 'test/codebook', original); + const restored = await loadCodebook(env.CODEBOOKS, 'test/codebook'); + return Response.json({ receipt, centroids: Array.from(restored.centroids), clusters: restored.clusters }); +} }; +`); + const artifact = await buildWorker(join(temporary, 'out'), entry); + const worker = new Miniflare({ ...config, modulesRoot: join(temporary, 'out'), scriptPath: artifact.script, r2Buckets: ['CODEBOOKS'] }); + t.after(async () => { await worker.dispose(); await rm(temporary, { recursive: true, force: true }); }); + const response = await worker.dispatchFetch('https://worker.test/storage'); + assert.equal(response.status, 200, await response.clone().text()); + const result = await response.json(); + assert.deepEqual(result.centroids, [1, 2, 3, 4]); + assert.equal(result.clusters, 2); + const bucket = await worker.getR2Bucket('CODEBOOKS'); + const stored = JSON.parse(await (await bucket.get('test/codebook')).text()); + assert.equal(stored.schema, 'ruvector.e-means-codebook/1'); + assert.equal(Object.hasOwn(stored, 'assignments'), false); +}); diff --git a/npm/packages/e-means/test/cloudflare.test.mjs b/npm/packages/e-means/test/cloudflare.test.mjs new file mode 100644 index 0000000000..7c081e83fb --- /dev/null +++ b/npm/packages/e-means/test/cloudflare.test.mjs @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; +import { createClusteringWorker, WORKER_LIMITS } from '../integrations/cloudflare.mjs'; + +const wasmModule = new WebAssembly.Module(await readFile(new URL('../wasm/ruvector_e_means_bg.wasm', import.meta.url))); +const token = 'e-means-test-token-not-for-production'; +const valid = { data: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dimensions: 2, + options: { clusters: 1, epochs: 2, batchSize: 2, seed: 42 } }; +const handler = createClusteringWorker({ wasmModule, bearerToken: token }); +function request(path, body, authorization = `Bearer ${token}`, headers = {}) { + return new Request(`https://worker.test${path}`, { method: 'POST', + headers: { 'content-type': 'application/json', authorization, ...headers }, body: JSON.stringify(body) }); +} + +test('Worker fits and predicts with the real precompiled Rust WASM module', async () => { + const response = await handler.fetch(request('/fit', valid)); + assert.equal(response.status, 200); + const fitted = await response.json(); + assert.deepEqual(fitted.centroids, [5, 6]); + assert.deepEqual(fitted.assignments, [0, 0, 0, 0, 0]); + assert.equal(fitted.mse, 16); + assert.equal(fitted.updates, 6); + const predicted = await handler.fetch(request('/predict', { data: [2, 2, 8, 8], dimensions: 2, centroids: [1, 1, 9, 9] })); + assert.deepEqual(await predicted.json(), { assignments: [0, 1] }); + const empty = await handler.fetch(request('/predict', { data: [], dimensions: 2, centroids: [1, 1] })); + assert.deepEqual(await empty.json(), { assignments: [] }); + assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.equal(response.headers.get('x-content-type-options'), 'nosniff'); +}); + +test('authentication fails closed and request fields cannot bypass it', async () => { + const unavailable = createClusteringWorker({ wasmModule }); + assert.equal((await unavailable.fetch(request('/fit', valid))).status, 503); + assert.equal((await handler.fetch(request('/fit', valid, 'Bearer incorrect'))).status, 401); + const health = await unavailable.fetch(new Request('https://worker.test/health')); + assert.equal(health.status, 200); + assert.equal((await health.json()).authenticatedOperations, false); + assert.equal((await handler.fetch(request('/fit', { ...valid, allowUnauthenticated: true }))).status, 422); + const local = createClusteringWorker({ wasmModule, allowUnauthenticated: true }); + assert.equal((await local.fetch(request('/fit', valid, ''))).status, 200); +}); + +test('shape, numeric, option, method and conservative work limits reject requests', async () => { + for (const body of [ + { ...valid, data: [] }, { ...valid, data: [1, 2, 3] }, + { ...valid, data: ['1', 2] }, { ...valid, data: [1e40, 2] }, + { ...valid, dimensions: 0 }, { ...valid, dimensions: WORKER_LIMITS.maxDimensions + 1 }, + { ...valid, options: { clusters: 0 } }, { ...valid, options: { clusters: 6 } }, + { ...valid, options: { clusters: 1, epochs: 0 } }, + { ...valid, options: { clusters: 1, batchSize: 0 } }, + { ...valid, options: { clusters: 1, peakStep: 0 } }, + { ...valid, options: { clusters: 1, seed: -1 } }, + { ...valid, options: { clusters: 1, finalization: 'slow' } }, + { ...valid, options: { clusters: 1, unknown: 1 } }, + ]) assert.equal((await handler.fetch(request('/fit', body))).status, 422, JSON.stringify(body)); + const limited = createClusteringWorker({ wasmModule, bearerToken: token, maxWork: 1 }); + assert.equal((await limited.fetch(request('/fit', valid))).status, 413); + assert.equal((await limited.fetch(request('/predict', { data: [1, 2], dimensions: 1, centroids: [0, 1] }))).status, 413); + // Assignment-only accounting would allow this ~4M-unit request; donor + // revival at every fifth batch makes the actual upper bound much larger. + assert.equal((await handler.fetch(request('/fit', { data: Array(256).fill(0), dimensions: 1, + options: { clusters: 256, epochs: 32, batchSize: 1 } }))).status, 413); + assert.equal((await handler.fetch(new Request('https://worker.test/fit'))).status, 405); + assert.equal((await handler.fetch(request('/fit?namespace=another-user', valid))).status, 400); + assert.equal((await handler.fetch(request('/persist', valid))).status, 404); + assert.equal((await handler.fetch(request('/fit', valid, `Bearer ${token}`, { 'content-type': 'text/plain' }))).status, 415); +}); + +test('streamed bodies have actual byte limits, strict UTF-8 parsing and read deadlines', async () => { + const small = createClusteringWorker({ wasmModule, bearerToken: token, maxRequestBytes: 32 }); + let cancelled = false; + const oversized = new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array(33)); }, cancel() { cancelled = true; } }); + const response = await small.fetch(new Request('https://worker.test/fit', { method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${token}` }, body: oversized, duplex: 'half' })); + assert.equal(response.status, 413); + assert.equal(cancelled, true); + const malformed = await handler.fetch(new Request('https://worker.test/fit', { method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${token}` }, body: new Uint8Array([0xff]) })); + assert.equal(malformed.status, 400); + const quick = createClusteringWorker({ wasmModule, bearerToken: token, bodyTimeoutMs: 20 }); + const stalled = new ReadableStream({ start() {}, cancel() {} }); + const timeout = await quick.fetch(new Request('https://worker.test/fit', { method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${token}` }, body: stalled, duplex: 'half' })); + assert.equal(timeout.status, 408); + const contentLength = await handler.fetch(request('/fit', valid, `Bearer ${token}`, { 'content-length': '999999999' })); + assert.equal(contentLength.status, 413); +}); + +test('Worker configuration is explicit and bounded', () => { + for (const options of [ + {}, { wasmModule: new Uint8Array() }, { wasmModule, bearerToken: 'short' }, + { wasmModule, maxWork: Infinity }, { wasmModule, maxRequestBytes: WORKER_LIMITS.maxRequestBytes + 1 }, + { wasmModule, allowUnauthenticated: 'true' }, { wasmModule, bodyTimeoutMs: 0 }, + ]) assert.throws(() => createClusteringWorker(options)); +}); diff --git a/npm/packages/e-means/test/e-means.test.mjs b/npm/packages/e-means/test/e-means.test.mjs new file mode 100644 index 0000000000..36b4a25f64 --- /dev/null +++ b/npm/packages/e-means/test/e-means.test.mjs @@ -0,0 +1,135 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { test } from 'node:test'; +import { init, fit, predict } from '../e-means.mjs'; + +test('numerical operations require completed initialization', () => { + const moduleUrl = new URL('../e-means.mjs', import.meta.url).href; + const child = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + import { fit, predict } from ${JSON.stringify(moduleUrl)}; + assert.throws(() => fit(new Float32Array([0]), 1, { clusters: 1 }), /await init/); + assert.throws(() => predict(new Float32Array([0]), 1, new Float32Array([0])), /await init/); + `], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); +}); + +test('initialization failure is retryable and concurrent calls share the runtime', async () => { + const moduleUrl = new URL('../e-means.mjs', import.meta.url).href; + const child = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + import { init, fit } from ${JSON.stringify(moduleUrl)}; + await assert.rejects(init(new Uint8Array([0, 1, 2])), /Could not initialize/); + await Promise.all([init(), init()]); + assert.equal(fit(new Float32Array([1, 3]), 1, { clusters: 1 }).centroids[0], 2); + `], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); +}); + +await init(); + +test('last-epoch mean uses every row, including a partial final batch', () => { + const data = new Float32Array([1, 2, 3, 4]); + const original = data.slice(); + const model = fit(data, 1, { clusters: 1, epochs: 1, batchSize: 3, peakStep: 1 }); + assert.deepEqual(data, original); + assert.deepEqual(model.centroids, new Float32Array([2.5])); + assert.deepEqual(model.assignments, new Uint32Array([0, 0, 0, 0])); + assert.equal(model.mse, 1.25); + assert.equal(model.updates, 2); + assert.equal(model.batchSize, 3); + assert.equal(model.dimensions, 1); +}); + +test('shared native/WASM golden case agrees on the exact codebook and diagnostics', () => { + const model = fit(new Float32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 2, + { clusters: 1, epochs: 2, batchSize: 2, seed: 42 }); + assert.deepEqual([...model.centroids], [5, 6]); + assert.deepEqual([...model.assignments], [0, 0, 0, 0, 0]); + assert.equal(model.mse, 16); + assert.equal(model.updates, 6); + assert.equal(model.batchSize, 2); +}); + +test('EMA maintains decayed sums and counts across unequal batch sizes', () => { + const model = fit(new Float32Array([0, 1, 9]), 1, + { clusters: 1, epochs: 1, batchSize: 2, peakStep: 1, finalization: 'ema', seed: 42 }); + // At t=1 of M=2, alpha=.55. Denominator is .45*2+.55*1=1.45. + // Enumerate the three possible final rows so the oracle is independent of shuffle order. + const possible = [3.103448275862069, 3.1724137931034484, 3.724137931034483]; + const center = model.centroids[0]; + assert.ok(possible.some((expected) => Math.abs(expected - center) < 1e-6), `center ${center}`); + assert.ok(Math.abs(model.mse - ([0, 1, 9].reduce((sum, x) => sum + (x - center) ** 2, 0) / 3)) < 1e-10); +}); + +test('separated clusters reconstruct accurately and output labels match the final codebook', () => { + const data = new Float32Array(256 * 2); + for (let i = 0; i < 256; ++i) { + const center = i < 128 ? -10 : 10; + data[i * 2] = center + (i % 5) * 0.01; + data[i * 2 + 1] = center - (i % 7) * 0.01; + } + const options = { clusters: 2, epochs: 4, batchSize: 32, seed: 7 }; + const model = fit(data, 2, options); + assert.ok(model.mse < 0.01, `MSE ${model.mse}`); + assert.deepEqual(model.assignments, predict(data, 2, model.centroids)); + assert.notEqual(model.assignments[0], model.assignments[128]); + assert.deepEqual(fit(data, 2, options), model, 'same seed reproduces the same fit'); + assert.equal(model.centroids.length, 4); + for (const v of model.centroids) assert.ok(Number.isFinite(v)); +}); + +test('nearest centroid handles ties, empty queries and f32 extreme values', () => { + const centroids = new Float32Array([-1, 1]); + assert.deepEqual(predict(new Float32Array([-2, 0, 2]), 1, centroids), new Uint32Array([0, 0, 1])); + assert.deepEqual(predict(new Float32Array(), 1, centroids), new Uint32Array()); + const large = new Float32Array([-3e38, 3e38]); + const model = fit(large, 1, { clusters: 1, epochs: 1 }); + assert.equal(model.centroids[0], 0); + assert.ok(Number.isFinite(model.mse)); +}); + +test('duplicate and zero vectors never create nonfinite centroids', () => { + for (const finalization of ['lastEpochMean', 'ema']) { + const data = new Float32Array(96); + const model = fit(data, 3, { clusters: 8, epochs: 3, batchSize: 2, peakStep: 1, finalization }); + assert.equal(model.centroids.length, 24); + assert.equal(model.mse, 0); + for (const v of model.centroids) assert.ok(Number.isFinite(v)); + } +}); + +test('invalid shapes, nonfinite input and malformed config reject before WASM coercion', () => { + const data = new Float32Array([0, 1]); + const config = { clusters: 1 }; + for (const bad of [0, -1, 1.5, NaN, Infinity, 2 ** 32, '1']) { + assert.throws(() => fit(data, bad, config), /dimensions/); + } + for (const bad of [[], [0, 1], new Float64Array([0, 1]), null]) { + assert.throws(() => fit(bad, 1, config), /Float32Array/); + } + assert.throws(() => fit(new Float32Array(), 1, config), /complete/); + assert.throws(() => fit(new Float32Array([1, 2, 3]), 2, config), /complete/); + for (const bad of [NaN, Infinity, -Infinity]) { + assert.throws(() => fit(new Float32Array([0, bad]), 1, config), /finite/); + assert.throws(() => predict(data, 1, new Float32Array([bad])), /finite/); + } + for (const key of ['clusters', 'epochs', 'batchSize']) { + for (const bad of [0, -1, 0.5, NaN, Infinity, 2 ** 32, '1', null]) { + assert.throws(() => fit(data, 1, { ...config, [key]: bad }), new RegExp(key)); + } + } + assert.throws(() => fit(data, 1, { clusters: 3 }), /number of vectors/); + for (const bad of [0, -1, 2, NaN, Infinity, '1', null]) { + assert.throws(() => fit(data, 1, { ...config, peakStep: bad }), /peakStep/); + } + for (const bad of [-1, 1.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity, '1', null]) { + assert.throws(() => fit(data, 1, { ...config, seed: bad }), /seed/); + } + for (const bad of [undefined, null, [], 1, 'config']) { + assert.throws(() => fit(data, 1, bad), /options/); + } + assert.throws(() => fit(data, 1, { ...config, finalization: 'slowEma' }), /finalization/); + assert.throws(() => fit(data, 1, { ...config, epohs: 3 }), /Unknown option/); + assert.throws(() => predict(data, 1, new Float32Array()), /complete/); +}); diff --git a/npm/packages/e-means/test/ecosystem-consumers.mjs b/npm/packages/e-means/test/ecosystem-consumers.mjs new file mode 100644 index 0000000000..3136e26e6d --- /dev/null +++ b/npm/packages/e-means/test/ecosystem-consumers.mjs @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repository = fileURLToPath(new URL('../../../../', import.meta.url)); +const packageRoot = fileURLToPath(new URL('../', import.meta.url)); +const temporary = await mkdtemp(join(tmpdir(), 'e-means-ecosystem-')); +function run(command, args, cwd = temporary) { + const result = spawnSync(command, args, { + cwd, encoding: 'utf8', timeout: 180_000, + env: { ...process.env, npm_config_workspaces: 'false' }, + }); + assert.equal(result.status, 0, `${command} ${args.join(' ')}\n${result.stdout}\n${result.stderr}`); + return result.stdout; +} +try { + await writeFile(join(temporary, 'package.json'), JSON.stringify({ private: true, type: 'module' })); + const packages = [ + { path: packageRoot, check: true }, + { path: join(repository, 'npm/packages/core') }, + { path: join(repository, 'npm/packages/ruvector') }, + { path: join(repository, 'npm/packages/all') }, + { path: join(repository, 'crates/ruvector-wasm') }, + ]; + const archives = []; + for (const item of packages) { + const args = ['pack', '--json', '--workspaces=false', '--pack-destination', temporary]; + // The shared artifact's normal prepack executes its actual WASM. Consumer + // prepack lanes build unrelated database SDKs; this test qualifies only the + // newly shipped clustering subpaths, not those packages' legacy root builds. + if (!item.check) args.push('--ignore-scripts'); + const [packed] = JSON.parse(run('npm', args, item.path)); + const paths = packed.files.map(file => file.path); + if (item.check) { + for (const name of ['runtime.mjs', 'wasm/ruvector_e_means_bg.wasm', 'integrations/metaharness.mjs', 'integrations/cloudflare.mjs']) { + assert.ok(paths.includes(name), `missing shared artifact ${name}`); + } + } else { + for (const name of ['clustering.mjs', 'clustering.d.mts', 'clustering/quantization.mjs', 'clustering/ivf.mjs', 'clustering/metaharness.mjs', 'clustering/cloudflare.mjs']) { + assert.ok(paths.includes(name), `missing consumer artifact ${name}`); + } + } + archives.push(join(temporary, packed.filename)); + } + run('npm', ['install', '--ignore-scripts', '--omit=optional', '--no-audit', '--no-fund', ...archives]); + await writeFile(join(temporary, 'consumer.mjs'), ` + import assert from 'node:assert/strict'; + import { readFile } from 'node:fs/promises'; + const cases = [ + ['@ruvector/core/clustering.mjs', '@ruvector/core/clustering/'], + ['ruvector/clustering.mjs', 'ruvector/clustering/'], + ['@ruvector/all/clustering.mjs', '@ruvector/all/clustering/'], + ['@ruvector/wasm/clustering', '@ruvector/wasm/clustering/'], + ]; + for (const [root, prefix] of cases) { + const extension = root.endsWith('.mjs') ? '.mjs' : ''; + const core = await import(root); + await core.init(); + const data = new Float32Array([0, 0, 0, 2, 10, 10, 10, 12]); + const quantization = await import(prefix + 'quantization' + extension); + const book = quantization.trainProductQuantizer(data, 2, 2, { clusters: 1, epochs: 1 }); + const codes = quantization.encodeProductQuantizer(book, data); + assert.ok(codes instanceof Uint8Array); + assert.deepEqual([...quantization.decodeProductQuantizer(book, codes)], [5,6,5,6,5,6,5,6]); + const ivf = await import(prefix + 'ivf' + extension); + const partitions = ivf.partition(data, 2, { clusters: 1 }); + assert.deepEqual([...ivf.postingList(partitions, 0)], [0,1,2,3]); + const meta = await import(prefix + 'metaharness' + extension); + const evaluator = meta.createClusteringEvaluator({ training: data, + holdout: new Float32Array([5,6]), dimensions: 2, + baseline: { clusters: 1, epochs: 1, batchSize: 4 }, + domain: { batchSize: { min: 1, max: 4, type: 'int', scale: 'linear' } }, + budget: { maxEvaluations: 1, maxDistanceCoordinates: 100000, timeoutMs: 30000 } }); + const score = await evaluator.evaluate(evaluator.baselineGenome, 'installed-baseline'); + assert.equal(score.raw.holdoutMse, 0); + const edge = await import(prefix + 'cloudflare' + extension); + const bytes = await readFile(new URL(import.meta.resolve('@ruvector/e-means/wasm'))); + const wasmModule = await WebAssembly.compile(bytes); + const token = 'local-installed-consumer-test-token'; + const worker = edge.createClusteringWorker({ wasmModule, bearerToken: token }); + const response = await worker.fetch(new Request('https://local.test/fit', { + method: 'POST', headers: { authorization: 'Bearer ' + token, 'content-type': 'application/json' }, + body: JSON.stringify({ data: [1,2,3,4], dimensions: 2, options: { clusters: 1 } }), + })); + assert.equal(response.status, 200); + assert.deepEqual((await response.json()).centroids, [2,3]); + console.log(root + ': PQ, IVF, heldout evaluator and Worker facades passed'); + } + `); + process.stdout.write(run(process.execPath, ['consumer.mjs'])); + const installed = JSON.parse(await readFile(join(temporary, 'node_modules/@ruvector/e-means/package.json'))); + assert.equal(installed.version, '0.1.0'); + console.log('Installed ecosystem subpaths passed; consumer root/prepack release lanes were not qualified'); +} finally { + await rm(temporary, { recursive: true, force: true }); +} diff --git a/npm/packages/e-means/test/ivf.test.mjs b/npm/packages/e-means/test/ivf.test.mjs new file mode 100644 index 0000000000..9e8d09acdd --- /dev/null +++ b/npm/packages/e-means/test/ivf.test.mjs @@ -0,0 +1,60 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { init, fit, predict } from '@ruvector/e-means'; +import { partition, postingList } from '@ruvector/e-means/ivf'; + +await init(); + +const data = new Float32Array([-10, -9, 10, 9, -9, -10, 9, 10, -11, -9, 11, 9]); +const options = { clusters: 2, epochs: 6, batchSize: 3, seed: 42 }; + +test('IVF partitions use shared fit results and stable exact assignment posting lists', () => { + const before = data.slice(); + const result = partition(data, 2, options); + const shared = fit(data, 2, options); + assert.equal(result.schema, 'ruvector-e-means-ivf-v1'); + assert.deepEqual(result.model.centroids, shared.centroids); + assert.deepEqual(result.model.assignments, predict(data, 2, shared.centroids)); + assert.equal(result.offsets.length, 3); + assert.equal(result.offsets[0], 0); + assert.equal(result.offsets[2], 6); + assert.deepEqual(Array.from(result.indices).sort((a, b) => a - b), [0, 1, 2, 3, 4, 5]); + for (let cluster = 0; cluster < 2; cluster++) { + const expected = Array.from({ length: 6 }, (_, row) => row).filter(row => shared.assignments[row] === cluster); + assert.deepEqual(Array.from(postingList(result, cluster)), expected); + } + assert.deepEqual(data, before); +}); + +test('IVF handles empty clusters and returns independent posting-list storage', () => { + const result = partition(new Float32Array(16).fill(1), 2, { clusters: 4, epochs: 1, batchSize: 8 }); + assert.deepEqual(Array.from(postingList(result, 0)), [0, 1, 2, 3, 4, 5, 6, 7]); + for (let cluster = 1; cluster < 4; cluster++) assert.deepEqual(postingList(result, cluster), new Uint32Array()); + const list = postingList(result, 0); + list.fill(99); + assert.equal(result.indices[0], 0); + assert.deepEqual(postingList(structuredClone(result), 0), postingList(result, 0)); +}); + +test('IVF rejects malformed training data and mutated partition records', () => { + assert.throws(() => partition(new Float32Array([1, NaN]), 2, { clusters: 1 })); + assert.throws(() => partition(new Float32Array([1]), 2, { clusters: 1 })); + assert.throws(() => partition(data, 2, { clusters: 0 })); + const result = partition(data, 2, options); + for (const cluster of [-1, 2, 0.5, NaN]) assert.throws(() => postingList(result, cluster)); + for (const mutation of [ + value => { value.schema = 'unknown'; }, + value => { value.offsets = value.offsets.slice(1); }, + value => { value.offsets[0] = 1; }, + value => { value.offsets[1] = 99; }, + value => { value.indices[0] = 99; }, + value => { value.indices[1] = value.indices[0]; }, + value => { value.model.assignments[value.indices[0]] = 99; }, + value => { value.model.centroids[0] = NaN; }, + value => { value.model.mse = NaN; }, + ]) { + const bad = structuredClone(result); + mutation(bad); + assert.throws(() => postingList(bad, 0)); + } +}); diff --git a/npm/packages/e-means/test/metaharness.test.mjs b/npm/packages/e-means/test/metaharness.test.mjs new file mode 100644 index 0000000000..51a3756bd3 --- /dev/null +++ b/npm/packages/e-means/test/metaharness.test.mjs @@ -0,0 +1,128 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { createClusteringEvaluator, tuneClustering } from '../integrations/metaharness.mjs'; +import { evolveNumeric } from '@metaharness/darwin'; + +function options(extra = {}) { + return { + training: new Float32Array([0, 0, 0, 2, 10, 10, 10, 12]), + holdout: new Float32Array([0, 1, 10, 11]), dimensions: 2, + baseline: { clusters: 2, epochs: 2, batchSize: 2, peakStep: 0.5, seed: 42 }, + domain: { clusters: { min: 1, max: 2, type: 'int', scale: 'linear' } }, + budget: { maxEvaluations: 3, maxDistanceCoordinates: 100_000, timeoutMs: 30_000 }, + ...extra, + }; +} + +test('real WASM measures a separate holdout and flags quality regressions', async () => { + const evaluator = createClusteringEvaluator(options()); + const baseline = await evaluator.evaluate(evaluator.baselineGenome, 'baseline'); + assert.equal(baseline.raw.trainMse, 1); + assert.equal(baseline.raw.holdoutMse, 0); + assert.equal(baseline.primary, 1); + assert.equal(baseline.regressed, false); + const candidate = await evaluator.evaluate({ clusters: 1 }, 'candidate'); + assert.equal(candidate.raw.holdoutMse, 50); + assert.equal(candidate.raw.baselineHoldoutMse, 0); + assert.equal(candidate.primary, 1 / 51); + assert.equal(candidate.regressed, true); + assert.ok(Number.isFinite(candidate.raw.fitMs) && candidate.raw.fitMs >= 0); + assert.ok(Number.isFinite(candidate.raw.holdoutMs) && candidate.raw.holdoutMs >= 0); + assert.equal(evaluator.receipt().sourceApplied, false); +}); + +test('inputs/configuration snapshot before awaits and receipts cannot mutate evidence', async () => { + const supplied = options(); + supplied.training.slice = () => supplied.training; + supplied.holdout.slice = () => supplied.holdout; + const evaluator = createClusteringEvaluator(supplied); + const pending = evaluator.evaluate(evaluator.baselineGenome, 'baseline'); + supplied.training.fill(999); supplied.holdout.fill(999); + supplied.baseline.clusters = 1; supplied.domain.clusters.min = 2; + supplied.budget.maxEvaluations = 0; + const score = await pending; + assert.equal(score.raw.holdoutMse, 0); + score.raw.holdoutMse = 999; + const receipt = evaluator.receipt(); + receipt.trials[0].score.raw.holdoutMse = 999; + receipt.domain.clusters.min = 2; + receipt.baselineConfig.clusters = 99; + assert.equal(evaluator.receipt().baseline.score.raw.holdoutMse, 0); + assert.equal(evaluator.receipt().domain.clusters.min, 1); + assert.equal(evaluator.receipt().baselineConfig.clusters, 2); + assert.equal((await evaluator.evaluate({ clusters: 1 }, 'candidate')).raw.holdoutMse, 50); +}); + +test('admission rejects invalid splits, unbounded domains and oversized work before training', () => { + const badCases = [ + options({ holdout: new Float32Array() }), + options({ holdout: new Float32Array([NaN, 1]) }), + options({ dimensions: 0 }), + options({ domain: {} }), + options({ domain: { clusters: { min: 1, max: Infinity, type: 'int', scale: 'linear' } } }), + options({ domain: { clusters: { min: 1, max: 5, type: 'int', scale: 'linear' } } }), + options({ domain: { batchSize: { min: 1, max: 4, type: 'float', scale: 'linear' } } }), + options({ domain: { peakStep: { min: 0.1, max: 1, type: 'float', scale: 'log', default: 0.1 } } }), + options({ baseline: { clusters: 2, epochs: 2, batchSize: 0 } }), + options({ baseline: { clusters: 2, epochs: null } }), + options({ baseline: { clusters: 2, seed: null } }), + options({ budget: { maxEvaluations: 0, maxDistanceCoordinates: 10_000 } }), + options({ budget: { maxEvaluations: 1, maxDistanceCoordinates: 1 } }), + options({ budget: { maxEvaluations: 1, maxDistanceCoordinates: 1e10 } }), + options({ baseline: { clusters: 2, epochs: 0xffff_ffff, batchSize: 1 } }), + options({ qualityTolerance: -1 }), + options({ qualityTolerance: null }), + ]; + for (const value of badCases) assert.throws(() => createClusteringEvaluator(value)); + const overlapping = options(); overlapping.holdout = overlapping.training.subarray(0, 4); + assert.throws(() => createClusteringEvaluator(overlapping), /nonoverlapping/); + assert.throws(() => createClusteringEvaluator(options({ training: new Float32Array(16_777_217) })), /64 MiB/); +}); + +test('evaluation dispatch, domain, concurrency and cooperative deadline limits are enforced', async () => { + const evaluator = createClusteringEvaluator(options({ budget: { maxEvaluations: 1, maxDistanceCoordinates: 100_000 } })); + const pending = evaluator.evaluate(evaluator.baselineGenome, 'baseline'); + await assert.rejects(evaluator.evaluate(evaluator.baselineGenome, 'parallel'), /Concurrent/); + await pending; + await assert.rejects(evaluator.evaluate({ clusters: 1 }, 'second'), /budget exhausted/); + const firstCandidate = createClusteringEvaluator(options()); + await assert.rejects(firstCandidate.evaluate({ clusters: 1 }, 'wrong-first'), /baselineGenome/); + await assert.rejects(firstCandidate.evaluate({ clusters: 99 }, 'outside'), /clusters/); + const expired = createClusteringEvaluator(options({ budget: { maxEvaluations: 1, maxDistanceCoordinates: 100_000, timeoutMs: 1 } })); + await new Promise(resolve => setTimeout(resolve, 10)); + await assert.rejects(expired.evaluate(expired.baselineGenome, 'expired'), /deadline/); +}); + +test('published Darwin evolveNumeric accepts the real clustering evaluator', async () => { + const evaluator = createClusteringEvaluator(options()); + const archive = await mkdtemp(join(tmpdir(), 'e-means-sdk-test-')); + try { + const result = await evolveNumeric({ genomeSpec: evaluator.domain, baselineGenome: evaluator.baselineGenome, + evaluator, generations: 1, childrenPerGeneration: 2, seed: 7, concurrency: 1, workRoot: archive }); + assert.equal(result.records.length, 3); + assert.equal(result.baseline.score.raw.holdoutMse, 0); + assert.equal(evaluator.receipt().dispatchCount, 3); + assert.ok(result.records.every(record => Number.isFinite(record.score.primary))); + } finally { await rm(archive, { recursive: true, force: true }); } +}); + +test('bounded tuneClustering uses Darwin0.13.0, validates evidence and removes its archive', async () => { + const before = new Set((await readdir(tmpdir())).filter(name => name.startsWith('e-means-darwin-'))); + const supplied = options({ generations: 1, childrenPerGeneration: 2, mutationSeed: 7 }); + const result = await tuneClustering(supplied); + assert.deepEqual(result.engine, { package: '@metaharness/darwin', version: '0.13.0', algorithm: 'evolveNumeric' }); + assert.equal(result.schema, 'e-means.clustering-tuning/1'); + assert.equal(result.trials.length, 3); + assert.equal(result.dispatchCount, 3); + assert.equal(result.baseline.score.raw.holdoutMse, 0); + assert.equal(result.sourceApplied, false); + assert.equal(result.temporaryArchiveRemoved, true); + assert.ok(result.consumedWork <= result.budget.maxDistanceCoordinates); + assert.ok(result.winner.score.primary >= result.baseline.score.primary); + assert.deepEqual(supplied.training, options().training); + assert.equal((await readdir(tmpdir())).filter(name => name.startsWith('e-means-darwin-') && !before.has(name)).length, 0); + await assert.rejects(tuneClustering(options({ generations: 8, childrenPerGeneration: 8 })), /worst-case dispatches/); +}); diff --git a/npm/packages/e-means/test/packed-consumer.mjs b/npm/packages/e-means/test/packed-consumer.mjs new file mode 100644 index 0000000000..5bc838f64e --- /dev/null +++ b/npm/packages/e-means/test/packed-consumer.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const packageDir = fileURLToPath(new URL('../', import.meta.url)); +const temp = await mkdtemp(join(tmpdir(), 'e-means-consumer-')); +function run(command, args, cwd = temp) { + const result = spawnSync(command, args, { cwd, encoding: 'utf8', env: { ...process.env, npm_config_workspaces: 'false' } }); + assert.equal(result.status, 0, `${command} ${args.join(' ')}\n${result.stdout}\n${result.stderr}`); + return result.stdout; +} +try { + const pack = JSON.parse(run('npm', ['pack', '--json', '--pack-destination', temp, '--workspaces=false'], packageDir)); + const files = pack[0].files.map((entry) => entry.path); + assert.ok(files.includes('wasm/ruvector_e_means_bg.wasm')); + assert.ok(files.includes('wasm/ruvector_e_means.js')); + assert.ok(files.includes('e-means.d.ts')); + for (const path of ['runtime.mjs', 'runtime.d.ts', 'integrations/metaharness.mjs', 'integrations/metaharness.d.ts', 'integrations/clustering-validation.mjs']) { + assert.ok(files.includes(path), `archive missing ${path}`); + } + assert.ok(!files.some((path) => path.endsWith('.rs') || path.includes('node_modules'))); + await writeFile(join(temp, 'package.json'), JSON.stringify({ private: true, type: 'module' })); + run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--offline', join(temp, pack[0].filename)]); + await writeFile(join(temp, 'consumer.mjs'), ` + import assert from 'node:assert/strict'; + import { init, fit, predict } from '@ruvector/e-means'; + import { createClusteringEvaluator, tuneClustering } from '@ruvector/e-means/metaharness'; + import { fit as runtimeFit } from '@ruvector/e-means/runtime'; + await init(); + const data = new Float32Array([1, 1, 3, 3]); + const result = fit(data, 2, { clusters: 1 }); + assert.deepEqual([...result.centroids], [2, 2]); + assert.equal(result.mse, 2); + assert.deepEqual([...predict(data, 2, result.centroids)], [0, 0]); + assert.equal(runtimeFit(data, 2, { clusters: 1 }).mse, 2); + const evaluation = { training: new Float32Array([0,0,0,2,10,10,10,12]), holdout: new Float32Array([0,1,10,11]), dimensions:2, + baseline:{clusters:2,epochs:2,batchSize:2,peakStep:0.5}, domain:{clusters:{min:1,max:2,type:'int',scale:'linear'}}, + budget:{maxEvaluations:3,maxDistanceCoordinates:100000} }; + const evaluator = createClusteringEvaluator(evaluation); + assert.equal((await evaluator.evaluate(evaluator.baselineGenome, 'baseline')).raw.holdoutMse, 0); + await assert.rejects(tuneClustering({...evaluation,generations:1,childrenPerGeneration:2}), /optional peer/); + console.log('Installed archive runtime/evaluator passed without the optional SDK'); + `); + process.stdout.write(run(process.execPath, ['consumer.mjs'])); + const installed = JSON.parse(await readFile(join(temp, 'node_modules/@ruvector/e-means/package.json'))); + assert.equal(installed.version, '0.1.0'); + // Cached pinned registry packages only: installation and qualification do not + // execute dependency lifecycle scripts or contact the registry. + run('npm', ['install', '--offline', '--ignore-scripts', '--no-audit', '--no-fund', '@metaharness/darwin@0.13.0', 'typescript@5.9.3']); + await writeFile(join(temp, 'meta-consumer.mjs'), ` + import assert from 'node:assert/strict'; + import { tuneClustering } from '@ruvector/e-means/metaharness'; + import { evolveNumeric } from '@metaharness/darwin'; + assert.equal(typeof evolveNumeric, 'function'); + const result = await tuneClustering({ + training:new Float32Array([0,0,0,2,10,10,10,12]), holdout:new Float32Array([0,1,10,11]), dimensions:2, + baseline:{clusters:2,epochs:2,batchSize:2,peakStep:0.5}, domain:{clusters:{min:1,max:2,type:'int',scale:'linear'}}, + budget:{maxEvaluations:3,maxDistanceCoordinates:100000}, generations:1,childrenPerGeneration:2 }); + assert.equal(result.engine.version,'0.13.0'); + assert.equal(result.trials.length,3); + assert.equal(result.baseline.score.raw.holdoutMse,0); + assert.equal(result.sourceApplied,false); + assert.equal(result.temporaryArchiveRemoved,true); + console.log('Offline installed MetaHarness/WASM consumer passed'); + `); + process.stdout.write(run(process.execPath, ['meta-consumer.mjs'])); + await writeFile(join(temp, 'types.mts'), ` + import {createClusteringEvaluator,tuneClustering,type ClusteringEvaluationOptions} from '@ruvector/e-means/metaharness'; + import {init,fit} from '@ruvector/e-means/runtime'; + const config:ClusteringEvaluationOptions={training:new Float32Array([0,1]),holdout:new Float32Array([2]),dimensions:1, + baseline:{clusters:1,epochs:1},domain:{epochs:{min:1,max:2,type:'int',scale:'linear'}},budget:{maxEvaluations:2,maxDistanceCoordinates:1000}}; + const evaluator=createClusteringEvaluator(config); + const result=await tuneClustering({...config,generations:1,childrenPerGeneration:1}); + const notApplied:false=result.sourceApplied; + await init(new WebAssembly.Module(new Uint8Array())); + const assignments:Uint32Array=fit(config.training,1,config.baseline).assignments; + void [evaluator,notApplied,assignments]; + // @ts-expect-error explicit source required by Node-free runtime + await init(); + `); + run(process.execPath, ['node_modules/typescript/bin/tsc', '--noEmit', '--strict', '--target', 'es2022', '--module', 'nodenext', '--moduleResolution', 'nodenext', '--lib', 'es2022,dom', 'types.mts']); + console.log('Offline installed integration declarations passed'); +} finally { + await rm(temp, { recursive: true, force: true }); +} diff --git a/npm/packages/e-means/test/quantization-types.mts b/npm/packages/e-means/test/quantization-types.mts new file mode 100644 index 0000000000..67fa682ea2 --- /dev/null +++ b/npm/packages/e-means/test/quantization-types.mts @@ -0,0 +1,19 @@ +import { type EMeansOptions } from '@ruvector/e-means'; +import { + trainProductQuantizer, encodeProductQuantizer, decodeProductQuantizer, + productQuantizerToJSON, productQuantizerFromJSON, type ProductQuantizer, +} from '@ruvector/e-means/quantization'; +import { partition, postingList, type IVFPartition } from '@ruvector/e-means/ivf'; + +const data = new Float32Array([1, 2, 3, 4]); +const options: EMeansOptions = { clusters: 1 }; +const pq: ProductQuantizer = trainProductQuantizer(data, 2, 2, options); +const codes: Uint8Array = encodeProductQuantizer(pq, data); +const restored: Float32Array = decodeProductQuantizer(productQuantizerFromJSON(productQuantizerToJSON(pq)), codes); +const ivf: IVFPartition = partition(restored, 2, options); +const posting: Uint32Array = postingList(ivf, 0); +void posting; +// @ts-expect-error byte codes are required +decodeProductQuantizer(pq, new Uint32Array()); +// @ts-expect-error float32 input vectors are required +trainProductQuantizer([1, 2], 2, 1, options); diff --git a/npm/packages/e-means/test/quantization.test.mjs b/npm/packages/e-means/test/quantization.test.mjs new file mode 100644 index 0000000000..65b1f5da53 --- /dev/null +++ b/npm/packages/e-means/test/quantization.test.mjs @@ -0,0 +1,114 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { init, fit, predict } from '@ruvector/e-means'; +import { + trainProductQuantizer, encodeProductQuantizer, decodeProductQuantizer, + productQuantizerToJSON, productQuantizerFromJSON, +} from '@ruvector/e-means/quantization'; + +await init(); + +const data = new Float32Array([1, 2, 10, 11, 1, 2, 20, 21, 3, 4, 10, 11, 3, 4, 20, 21]); +const options = { clusters: 2, epochs: 8, batchSize: 4, peakStep: 0.5, seed: 42 }; + +test('product quantization trains exact shared codebooks and reconstructs representable vectors', () => { + const before = data.slice(); + const model = trainProductQuantizer(data, 4, 2, options); + assert.equal(model.schema, 'ruvector-e-means-pq-v1'); + assert.equal(model.centroids.length, 8); + const codes = encodeProductQuantizer(model, data); + assert.ok(codes instanceof Uint8Array); + assert.equal(codes.length, 8); + assert.deepEqual(decodeProductQuantizer(model, codes), data); + assert.deepEqual(data, before); + assert.deepEqual(trainProductQuantizer(data, 4, 2, options).centroids, model.centroids); + for (let subspace = 0; subspace < 2; subspace++) { + const subvectors = Float32Array.from({ length: 8 }, (_, index) => data[Math.floor(index / 2) * 4 + subspace * 2 + index % 2]); + const shared = fit(subvectors, 2, options); + const start = subspace * 4; + assert.deepEqual(model.centroids.slice(start, start + 4), shared.centroids); + const assignments = predict(subvectors, 2, shared.centroids); + for (let row = 0; row < 4; row++) assert.equal(codes[row * 2 + subspace], assignments[row]); + } +}); + +test('one-cluster subspaces have arithmetic-mean codebooks and support empty codecs', () => { + const model = trainProductQuantizer(data, 4, 2, { clusters: 1, batchSize: 3 }); + assert.deepEqual(Array.from(model.centroids), [2, 3, 15, 16]); + assert.deepEqual(encodeProductQuantizer(model, new Float32Array()), new Uint8Array()); + assert.deepEqual(decodeProductQuantizer(model, new Uint8Array()), new Float32Array()); + assert.deepEqual(Array.from(decodeProductQuantizer(model, new Uint8Array([0, 0]))), [2, 3, 15, 16]); +}); + +test('all subspaces use one snapshot of caller configuration', () => { + let reads = 0; + const model = trainProductQuantizer(data, 4, 2, { + get clusters() { reads++; return reads === 1 ? 1 : 2; }, + batchSize: 3, + }); + assert.equal(reads, 1); + assert.equal(model.clusters, 1); + assert.deepEqual(Array.from(model.centroids), [2, 3, 15, 16]); +}); + +test('codebooks survive JSON and structured clone with independent ownership', () => { + const model = trainProductQuantizer(data, 4, 2, options); + const json = productQuantizerToJSON(model); + const restored = productQuantizerFromJSON(JSON.parse(JSON.stringify(json))); + assert.deepEqual(restored.centroids, model.centroids); + assert.notEqual(restored.centroids, model.centroids); + const clone = structuredClone(model); + assert.deepEqual(encodeProductQuantizer(clone, data), encodeProductQuantizer(model, data)); + restored.centroids.fill(0); + json.centroids.fill(0); + assert.ok(model.centroids.some(value => value !== 0)); +}); + +test('256-centroid codebooks preserve the full byte range', () => { + const model = { + schema: 'ruvector-e-means-pq-v1', dimensions: 1, subspaces: 1, clusters: 256, + centroids: Float32Array.from({ length: 256 }, (_, index) => index), + }; + const input = new Float32Array([0, 255, 254]); + const codes = encodeProductQuantizer(model, input); + assert.deepEqual(Array.from(codes), [0, 255, 254]); + assert.deepEqual(decodeProductQuantizer(model, codes), input); +}); + +test('product quantization rejects invalid shapes, values, configuration and byte codes', () => { + for (const [dimensions, subspaces, clusters] of [[0, 1, 1], [4, 0, 1], [4, 3, 1], [4, 5, 1], [4, 2, 0], [4, 2, 257], [4, 2, 5]]) { + assert.throws(() => trainProductQuantizer(data, dimensions, subspaces, { clusters })); + } + assert.throws(() => trainProductQuantizer(data, 4, 2, { clusters: 2, epochs: 0 })); + assert.throws(() => trainProductQuantizer(data, 4, 2, { ...options, unknown: true })); + assert.throws(() => trainProductQuantizer([1, 2, 3, 4], 4, 2, { clusters: 1 })); + assert.throws(() => trainProductQuantizer(new Float32Array([1, NaN, 3, 4]), 4, 2, { clusters: 1 })); + const model = trainProductQuantizer(data, 4, 2, options); + assert.throws(() => encodeProductQuantizer(model, new Float32Array([1]))); + assert.throws(() => encodeProductQuantizer(model, new Float32Array([1, 2, 3, Infinity]))); + assert.throws(() => decodeProductQuantizer(model, new Uint32Array([0, 0]))); + assert.throws(() => decodeProductQuantizer(model, new Uint8Array([0]))); + assert.throws(() => decodeProductQuantizer(model, new Uint8Array([0, 2]))); +}); + +test('every operation revalidates caller-owned codebooks and JSON values', () => { + const model = trainProductQuantizer(data, 4, 2, options); + for (const mutation of [ + value => { value.schema = 'unknown'; }, + value => { value.subspaces = 3; }, + value => { value.clusters = 257; }, + value => { value.centroids = value.centroids.slice(1); }, + value => { value.centroids[0] = NaN; }, + ]) { + const bad = structuredClone(model); + mutation(bad); + assert.throws(() => encodeProductQuantizer(bad, data)); + assert.throws(() => decodeProductQuantizer(bad, new Uint8Array([0, 0]))); + assert.throws(() => productQuantizerToJSON(bad)); + } + for (const coordinate of ['1', NaN, Infinity, 1e100, undefined]) { + const json = productQuantizerToJSON(model); + json.centroids[0] = coordinate; + assert.throws(() => productQuantizerFromJSON(json)); + } +}); diff --git a/npm/packages/e-means/test/runtime.test.mjs b/npm/packages/e-means/test/runtime.test.mjs new file mode 100644 index 0000000000..70e7a96d5e --- /dev/null +++ b/npm/packages/e-means/test/runtime.test.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; + +test('compiled-module runtime and plain imports load no Node builtins or Darwin SDK', async () => { + const temp = await mkdtemp(join(tmpdir(), 'e-means-runtime-')); + try { + const bytes = await readFile(new URL('../wasm/ruvector_e_means_bg.wasm', import.meta.url)); + const loader = 'data:text/javascript,' + encodeURIComponent(` + export async function resolve(specifier, context, next) { + if (specifier.startsWith('node:') || specifier.includes('@metaharness/darwin')) { + throw new Error('Unexpected Node/SDK import: ' + specifier); + } + return next(specifier, context); + } + `); + const file = join(temp, 'consumer.mjs'); + await writeFile(file, ` + const runtime = await import(${JSON.stringify(new URL('../runtime.mjs', import.meta.url).href)}); + let rejected = false; + try { await runtime.init(); } catch (error) { rejected = /requires WASM bytes/.test(error.message); } + if (!rejected) throw new Error('Runtime must reject an omitted source before initialization'); + const bytes = Uint8Array.from(atob(${JSON.stringify(bytes.toString('base64'))}), character => character.charCodeAt(0)); + const module = new WebAssembly.Module(bytes); + await Promise.all([runtime.init(module), runtime.init(module)]); + const result = runtime.fit(new Float32Array([0, 2]), 1, { clusters: 1 }); + if (result.centroids[0] !== 1 || result.mse !== 1) throw new Error('Compiled module fit differs'); + await import(${JSON.stringify(new URL('../e-means.mjs', import.meta.url).href)}); + const integration = await import(${JSON.stringify(new URL('../integrations/metaharness.mjs', import.meta.url).href)}); + if (typeof integration.createClusteringEvaluator !== 'function') throw new Error('Evaluator export missing'); + console.log('Node-free runtime and lazy SDK imports passed'); + `); + const child = spawnSync(process.execPath, ['--experimental-loader', loader, file], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + assert.match(child.stdout, /Node-free runtime/); + } finally { await rm(temp, { recursive: true, force: true }); } +}); diff --git a/npm/packages/e-means/test/tsconfig.json b/npm/packages/e-means/test/tsconfig.json new file mode 100644 index 0000000000..29ec5a0a8c --- /dev/null +++ b/npm/packages/e-means/test/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "lib": ["ES2022", "DOM"], + "types": [] + }, + "include": ["*.ts", "*.mts"] +} diff --git a/npm/packages/e-means/test/typed-bridge.test.mjs b/npm/packages/e-means/test/typed-bridge.test.mjs new file mode 100644 index 0000000000..11e14a2d4c --- /dev/null +++ b/npm/packages/e-means/test/typed-bridge.test.mjs @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { init, fit } from '../e-means.mjs'; +import initWasm, { fitJson, fitTyped, FittedModel } from '../wasm/ruvector_e_means.js'; + +await init(); +const runtime = await initWasm(); +const samples = new Float32Array([0, 1, 0, 2, 10, 12, 10, 13, -9, -10, -9, -11]); + +test('typed result transport matches JSON transport for both finalizers', () => { + for (const finalization of ['lastEpochMean', 'ema']) { + const options = { clusters: 3, epochs: 3, batchSize: 2, seed: 7, finalization }; + const json = JSON.parse(fitJson(samples, 2, JSON.stringify(options))); + assert.deepEqual(fit(samples, 2, options), { + ...json, + centroids: Float32Array.from(json.centroids), + assignments: Uint32Array.from(json.assignments), + }); + } +}); + +test('consumed outputs are caller-owned and survive handle release and WASM memory growth', () => { + const handle = fitTyped(samples, 2, JSON.stringify({ clusters: 3, epochs: 2 })); + let centroids, assignments; + try { + centroids = handle.takeCentroids(); + assignments = handle.takeAssignments(); + assert.deepEqual(handle.takeCentroids(), new Float32Array()); + assert.deepEqual(handle.takeAssignments(), new Uint32Array()); + assert.notEqual(centroids.buffer, runtime.memory.buffer); + assert.notEqual(assignments.buffer, runtime.memory.buffer); + } finally { + handle.free(); + } + const expectedCentroids = centroids.slice(), expectedAssignments = assignments.slice(); + runtime.memory.grow(1); + fit(samples, 2, { clusters: 2 }); + assert.deepEqual(centroids, expectedCentroids); + assert.deepEqual(assignments, expectedAssignments); +}); + +test('the wrapper frees its model handle after success and output-copy failure', () => { + const originalTake = FittedModel.prototype.takeAssignments; + const originalFree = FittedModel.prototype.free; + let frees = 0; + FittedModel.prototype.free = function () { frees++; return originalFree.call(this); }; + try { + assert.equal(fit(samples, 2, { clusters: 2 }).assignments.length, 6); + assert.equal(frees, 1); + FittedModel.prototype.takeAssignments = function () { throw new Error('injected output-copy failure'); }; + assert.throws(() => fit(samples, 2, { clusters: 2 }), /injected output-copy failure/); + assert.equal(frees, 2); + } finally { + FittedModel.prototype.takeAssignments = originalTake; + FittedModel.prototype.free = originalFree; + } + assert.equal(fit(samples, 2, { clusters: 2 }).assignments.length, 6); +}); + +test('repeated fits release WASM model allocations instead of accumulating them', () => { + const data = Float32Array.from({ length: 4096 }, (_, i) => (i % 17) / 17); + const config = { clusters: 1, epochs: 1, batchSize: 128 }; + for (let i = 0; i < 8; i++) fit(data, 1, config); + const afterWarmup = runtime.memory.buffer.byteLength; + for (let i = 0; i < 128; i++) { + const model = fit(data, 1, config); + assert.equal(model.assignments.length, data.length); + } + assert.equal(runtime.memory.buffer.byteLength, afterWarmup, 'WASM allocation high-water mark must plateau'); +}); diff --git a/npm/packages/e-means/test/types.ts b/npm/packages/e-means/test/types.ts new file mode 100644 index 0000000000..e463a2c7d4 --- /dev/null +++ b/npm/packages/e-means/test/types.ts @@ -0,0 +1,32 @@ +import { init, fit, predict, type EMeansModel, type EMeansOptions } from '@ruvector/e-means'; +import { init as initRuntime } from '@ruvector/e-means/runtime'; +import { createClusteringEvaluator, tuneClustering, type ClusteringEvaluationOptions } from '@ruvector/e-means/metaharness'; + +const options: EMeansOptions = { clusters: 2, epochs: 3, finalization: 'lastEpochMean' }; +await init(); +const model: EMeansModel = fit(new Float32Array([0, 0, 1, 1]), 2, options); +const assignments: Uint32Array = predict(new Float32Array([2, 2]), 2, model.centroids); +void assignments; +// @ts-expect-error missing required cluster count +fit(new Float32Array(), 1, { epochs: 3 }); +// @ts-expect-error row storage must be Float32Array +fit([0, 1], 1, options); +// @ts-expect-error unsupported finalization policy +fit(new Float32Array(), 1, { clusters: 1, finalization: 'slowEma' }); + +await initRuntime(new WebAssembly.Module(new Uint8Array())); +// @ts-expect-error Node-free runtime requires an explicit source +await initRuntime(); +const evaluation: ClusteringEvaluationOptions = { + training: new Float32Array([0, 1]), holdout: new Float32Array([2]), dimensions: 1, + baseline: { clusters: 1, epochs: 1 }, + domain: { epochs: { min: 1, max: 2, type: 'int', scale: 'linear' } }, + budget: { maxEvaluations: 2, maxDistanceCoordinates: 1000 }, +}; +const evaluator = createClusteringEvaluator(evaluation); +await evaluator.evaluate(evaluator.baselineGenome, 'baseline'); +const tuned = await tuneClustering({ ...evaluation, generations: 1, childrenPerGeneration: 1 }); +const neverApplied: false = tuned.sourceApplied; +void neverApplied; +// @ts-expect-error tuning requires a finite explicit evaluation budget +createClusteringEvaluator({ training: new Float32Array(), holdout: new Float32Array(), dimensions: 1, baseline: options, domain: {} }); diff --git a/npm/packages/e-means/wasm-module.d.ts b/npm/packages/e-means/wasm-module.d.ts new file mode 100644 index 0000000000..b439be05df --- /dev/null +++ b/npm/packages/e-means/wasm-module.d.ts @@ -0,0 +1,3 @@ +/** Statically compiled module supplied by a Worker-aware bundler such as Wrangler. */ +declare const module: WebAssembly.Module; +export default module; diff --git a/npm/packages/e-means/wasm/ruvector_e_means.d.ts b/npm/packages/e-means/wasm/ruvector_e_means.d.ts new file mode 100644 index 0000000000..c63fbfba3d --- /dev/null +++ b/npm/packages/e-means/wasm/ruvector_e_means.d.ts @@ -0,0 +1,83 @@ +/* tslint:disable */ +/* eslint-disable */ + +/** + * A temporary owner for one fit result. JavaScript takes independent output + * arrays and then frees this handle, releasing any vectors not taken. + */ +export class FittedModel { + private constructor(); + free(): void; + [Symbol.dispose](): void; + /** + * Cluster indices fit u32 because wasm32 arrays and sample counts do. + */ + takeAssignments(): Uint32Array; + /** + * wasm-bindgen copies the returned vector into a caller-owned Float32Array + * before freeing its Rust allocation. No view into WASM memory escapes. + */ + takeCentroids(): Float32Array; + readonly batchSize: number; + readonly dimensions: number; + readonly mse: number; + readonly peakStep: number; + readonly revivedClusters: number; + readonly updates: number; +} + +/** + * Compatibility transport and an independent parity oracle for typed output. + */ +export function fitJson(data: Float32Array, dimensions: number, config_json: string): string; + +/** + * Only the small configuration crosses JSON; result vectors stay numeric. + */ +export function fitTyped(data: Float32Array, dimensions: number, config_json: string): FittedModel; + +export function predictFlat(data: Float32Array, dimensions: number, centroids: Float32Array): Uint32Array; + +export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; + +export interface InitOutput { + readonly memory: WebAssembly.Memory; + readonly __wbg_fittedmodel_free: (a: number, b: number) => void; + readonly fitJson: (a: number, b: number, c: number, d: number, e: number, f: number) => void; + readonly fitTyped: (a: number, b: number, c: number, d: number, e: number, f: number) => void; + readonly fittedmodel_batchSize: (a: number) => number; + readonly fittedmodel_dimensions: (a: number) => number; + readonly fittedmodel_mse: (a: number) => number; + readonly fittedmodel_peakStep: (a: number) => number; + readonly fittedmodel_revivedClusters: (a: number) => number; + readonly fittedmodel_takeAssignments: (a: number, b: number) => void; + readonly fittedmodel_takeCentroids: (a: number, b: number) => void; + readonly fittedmodel_updates: (a: number) => number; + readonly predictFlat: (a: number, b: number, c: number, d: number, e: number, f: number) => void; + readonly __wbindgen_add_to_stack_pointer: (a: number) => number; + readonly __wbindgen_export: (a: number, b: number) => number; + readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; + readonly __wbindgen_export3: (a: number, b: number, c: number) => void; +} + +export type SyncInitInput = BufferSource | WebAssembly.Module; + +/** + * Instantiates the given `module`, which can either be bytes or + * a precompiled `WebAssembly.Module`. + * + * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated. + * + * @returns {InitOutput} + */ +export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput; + +/** + * If `module_or_path` is {RequestInfo} or {URL}, makes a request and + * for everything else, calls `WebAssembly.instantiate` directly. + * + * @param {{ module_or_path: InitInput | Promise }} module_or_path - Passing `InitInput` directly is deprecated. + * + * @returns {Promise} + */ +export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise } | InitInput | Promise): Promise; diff --git a/npm/packages/e-means/wasm/ruvector_e_means.js b/npm/packages/e-means/wasm/ruvector_e_means.js new file mode 100644 index 0000000000..ba3256a488 --- /dev/null +++ b/npm/packages/e-means/wasm/ruvector_e_means.js @@ -0,0 +1,456 @@ +/* @ts-self-types="./ruvector_e_means.d.ts" */ + +/** + * A temporary owner for one fit result. JavaScript takes independent output + * arrays and then frees this handle, releasing any vectors not taken. + */ +export class FittedModel { + static __wrap(ptr) { + const obj = Object.create(FittedModel.prototype); + obj.__wbg_ptr = ptr; + FittedModelFinalization.register(obj, obj.__wbg_ptr, obj); + return obj; + } + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + FittedModelFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_fittedmodel_free(ptr, 0); + } + /** + * @returns {number} + */ + get batchSize() { + const ret = wasm.fittedmodel_batchSize(this.__wbg_ptr); + return ret >>> 0; + } + /** + * @returns {number} + */ + get dimensions() { + const ret = wasm.fittedmodel_dimensions(this.__wbg_ptr); + return ret >>> 0; + } + /** + * @returns {number} + */ + get mse() { + const ret = wasm.fittedmodel_mse(this.__wbg_ptr); + return ret; + } + /** + * @returns {number} + */ + get peakStep() { + const ret = wasm.fittedmodel_peakStep(this.__wbg_ptr); + return ret; + } + /** + * @returns {number} + */ + get revivedClusters() { + const ret = wasm.fittedmodel_revivedClusters(this.__wbg_ptr); + return ret >>> 0; + } + /** + * Cluster indices fit u32 because wasm32 arrays and sample counts do. + * @returns {Uint32Array} + */ + takeAssignments() { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.fittedmodel_takeAssignments(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var v1 = getArrayU32FromWasm0(r0, r1).slice(); + wasm.__wbindgen_export3(r0, r1 * 4, 4); + return v1; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * wasm-bindgen copies the returned vector into a caller-owned Float32Array + * before freeing its Rust allocation. No view into WASM memory escapes. + * @returns {Float32Array} + */ + takeCentroids() { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.fittedmodel_takeCentroids(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var v1 = getArrayF32FromWasm0(r0, r1).slice(); + wasm.__wbindgen_export3(r0, r1 * 4, 4); + return v1; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * @returns {number} + */ + get updates() { + const ret = wasm.fittedmodel_updates(this.__wbg_ptr); + return ret >>> 0; + } +} +if (Symbol.dispose) FittedModel.prototype[Symbol.dispose] = FittedModel.prototype.free; + +/** + * Compatibility transport and an independent parity oracle for typed output. + * @param {Float32Array} data + * @param {number} dimensions + * @param {string} config_json + * @returns {string} + */ +export function fitJson(data, dimensions, config_json) { + let deferred4_0; + let deferred4_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArrayF32ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(config_json, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len1 = WASM_VECTOR_LEN; + wasm.fitJson(retptr, ptr0, len0, dimensions, ptr1, len1); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + var ptr3 = r0; + var len3 = r1; + if (r3) { + ptr3 = 0; len3 = 0; + throw takeObject(r2); + } + deferred4_0 = ptr3; + deferred4_1 = len3; + return getStringFromWasm0(ptr3, len3); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export3(deferred4_0, deferred4_1, 1); + } +} + +/** + * Only the small configuration crosses JSON; result vectors stay numeric. + * @param {Float32Array} data + * @param {number} dimensions + * @param {string} config_json + * @returns {FittedModel} + */ +export function fitTyped(data, dimensions, config_json) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArrayF32ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(config_json, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len1 = WASM_VECTOR_LEN; + wasm.fitTyped(retptr, ptr0, len0, dimensions, ptr1, len1); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + if (r2) { + throw takeObject(r1); + } + return FittedModel.__wrap(r0); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + +/** + * @param {Float32Array} data + * @param {number} dimensions + * @param {Float32Array} centroids + * @returns {Uint32Array} + */ +export function predictFlat(data, dimensions, centroids) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArrayF32ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passArrayF32ToWasm0(centroids, wasm.__wbindgen_export); + const len1 = WASM_VECTOR_LEN; + wasm.predictFlat(retptr, ptr0, len0, dimensions, ptr1, len1); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + if (r3) { + throw takeObject(r2); + } + var v3 = getArrayU32FromWasm0(r0, r1).slice(); + wasm.__wbindgen_export3(r0, r1 * 4, 4); + return v3; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} +function __wbg_get_imports() { + const import0 = { + __proto__: null, + __wbg_Error_fdd633d4bb5dd76a: function(arg0, arg1) { + const ret = Error(getStringFromWasm0(arg0, arg1)); + return addHeapObject(ret); + }, + __wbg___wbindgen_throw_ea4887a5f8f9a9db: function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }, + }; + return { + __proto__: null, + "./ruvector_e_means_bg.js": import0, + }; +} + +const FittedModelFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_fittedmodel_free(ptr, 1)); + +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1); + const idx = heap_next; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; +} + +function dropObject(idx) { + if (idx < 1028) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function getArrayF32FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getFloat32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); +} + +function getArrayU32FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); +} + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +let cachedFloat32ArrayMemory0 = null; +function getFloat32ArrayMemory0() { + if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) { + cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer); + } + return cachedFloat32ArrayMemory0; +} + +function getStringFromWasm0(ptr, len) { + return decodeText(ptr >>> 0, len); +} + +let cachedUint32ArrayMemory0 = null; +function getUint32ArrayMemory0() { + if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) { + cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer); + } + return cachedUint32ArrayMemory0; +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function getObject(idx) { return heap[idx]; } + +let heap = new Array(1024).fill(undefined); +heap.push(undefined, null, true, false); + +let heap_next = heap.length; + +function passArrayF32ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 4, 4) >>> 0; + getFloat32ArrayMemory0().set(arg, ptr / 4); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +const cachedTextEncoder = new TextEncoder(); + +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + }; +} + +let WASM_VECTOR_LEN = 0; + +let wasmModule, wasmInstance, wasm; +function __wbg_finalize_init(instance, module) { + wasmInstance = instance; + wasm = instance.exports; + wasmModule = module; + cachedDataViewMemory0 = null; + cachedFloat32ArrayMemory0 = null; + cachedUint32ArrayMemory0 = null; + cachedUint8ArrayMemory0 = null; + return wasm; +} + +async function __wbg_load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + } catch (e) { + const validResponse = module.ok && expectedResponseType(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { throw e; } + } + } + + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + } else { + return instance; + } + } + + function expectedResponseType(type) { + switch (type) { + case 'basic': case 'cors': case 'default': return true; + } + return false; + } +} + +function initSync(module) { + if (wasm !== undefined) return wasm; + + + if (module !== undefined) { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({module} = module) + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') + } + } + + const imports = __wbg_get_imports(); + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + const instance = new WebAssembly.Instance(module, imports); + return __wbg_finalize_init(instance, module); +} + +async function __wbg_init(module_or_path) { + if (wasm !== undefined) return wasm; + + + if (module_or_path !== undefined) { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({module_or_path} = module_or_path) + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead') + } + } + + if (module_or_path === undefined) { + module_or_path = new URL('ruvector_e_means_bg.wasm', import.meta.url); + } + const imports = __wbg_get_imports(); + + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { + module_or_path = fetch(module_or_path); + } + + const { instance, module } = await __wbg_load(await module_or_path, imports); + + return __wbg_finalize_init(instance, module); +} + +export { initSync, __wbg_init as default }; diff --git a/npm/packages/e-means/wasm/ruvector_e_means_bg.wasm b/npm/packages/e-means/wasm/ruvector_e_means_bg.wasm new file mode 100644 index 0000000000..b7189ea798 Binary files /dev/null and b/npm/packages/e-means/wasm/ruvector_e_means_bg.wasm differ diff --git a/npm/packages/e-means/wasm/ruvector_e_means_bg.wasm.d.ts b/npm/packages/e-means/wasm/ruvector_e_means_bg.wasm.d.ts new file mode 100644 index 0000000000..1d8bb910ab --- /dev/null +++ b/npm/packages/e-means/wasm/ruvector_e_means_bg.wasm.d.ts @@ -0,0 +1,19 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory; +export const __wbg_fittedmodel_free: (a: number, b: number) => void; +export const fitJson: (a: number, b: number, c: number, d: number, e: number, f: number) => void; +export const fitTyped: (a: number, b: number, c: number, d: number, e: number, f: number) => void; +export const fittedmodel_batchSize: (a: number) => number; +export const fittedmodel_dimensions: (a: number) => number; +export const fittedmodel_mse: (a: number) => number; +export const fittedmodel_peakStep: (a: number) => number; +export const fittedmodel_revivedClusters: (a: number) => number; +export const fittedmodel_takeAssignments: (a: number, b: number) => void; +export const fittedmodel_takeCentroids: (a: number, b: number) => void; +export const fittedmodel_updates: (a: number) => number; +export const predictFlat: (a: number, b: number, c: number, d: number, e: number, f: number) => void; +export const __wbindgen_add_to_stack_pointer: (a: number) => number; +export const __wbindgen_export: (a: number, b: number) => number; +export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_export3: (a: number, b: number, c: number) => void; diff --git a/npm/packages/ruvector/clustering.d.mts b/npm/packages/ruvector/clustering.d.mts new file mode 100644 index 0000000000..2ca0a76f10 --- /dev/null +++ b/npm/packages/ruvector/clustering.d.mts @@ -0,0 +1,2 @@ +export { init, fit, predict } from '@ruvector/e-means'; +export type { EMeansOptions, EMeansModel } from '@ruvector/e-means'; diff --git a/npm/packages/ruvector/clustering.mjs b/npm/packages/ruvector/clustering.mjs new file mode 100644 index 0000000000..b229b6a5a9 --- /dev/null +++ b/npm/packages/ruvector/clustering.mjs @@ -0,0 +1,2 @@ +/** Shared Rust/WASM e-means clustering, loaded explicitly by the caller. */ +export { init, fit, predict } from '@ruvector/e-means'; diff --git a/npm/packages/ruvector/clustering/cloudflare.d.mts b/npm/packages/ruvector/clustering/cloudflare.d.mts new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/npm/packages/ruvector/clustering/cloudflare.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/npm/packages/ruvector/clustering/cloudflare.mjs b/npm/packages/ruvector/clustering/cloudflare.mjs new file mode 100644 index 0000000000..c3a6e7b5f3 --- /dev/null +++ b/npm/packages/ruvector/clustering/cloudflare.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/cloudflare'; diff --git a/npm/packages/ruvector/clustering/ivf.d.mts b/npm/packages/ruvector/clustering/ivf.d.mts new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/npm/packages/ruvector/clustering/ivf.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/npm/packages/ruvector/clustering/ivf.mjs b/npm/packages/ruvector/clustering/ivf.mjs new file mode 100644 index 0000000000..8fcf4814d7 --- /dev/null +++ b/npm/packages/ruvector/clustering/ivf.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/ivf'; diff --git a/npm/packages/ruvector/clustering/metaharness.d.mts b/npm/packages/ruvector/clustering/metaharness.d.mts new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/npm/packages/ruvector/clustering/metaharness.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/npm/packages/ruvector/clustering/metaharness.mjs b/npm/packages/ruvector/clustering/metaharness.mjs new file mode 100644 index 0000000000..79c64e223c --- /dev/null +++ b/npm/packages/ruvector/clustering/metaharness.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/metaharness'; diff --git a/npm/packages/ruvector/clustering/quantization.d.mts b/npm/packages/ruvector/clustering/quantization.d.mts new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/npm/packages/ruvector/clustering/quantization.d.mts @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/npm/packages/ruvector/clustering/quantization.mjs b/npm/packages/ruvector/clustering/quantization.mjs new file mode 100644 index 0000000000..d1e262eeb5 --- /dev/null +++ b/npm/packages/ruvector/clustering/quantization.mjs @@ -0,0 +1 @@ +export * from '@ruvector/e-means/quantization'; diff --git a/npm/packages/ruvector/package.json b/npm/packages/ruvector/package.json index 99a9171646..14355d1efc 100644 --- a/npm/packages/ruvector/package.json +++ b/npm/packages/ruvector/package.json @@ -12,7 +12,8 @@ "verify-dist": "node scripts/verify-dist.js", "prepack": "npm run build && npm run verify-dist", "prepublishOnly": "npm run build && npm run verify-dist", - "test": "node test/integration.js && node test/graph-modularity.js && node test/backend-fallback.js && node test/mincut-wasm.js && node test/graph-cli-e2e.js && node test/metaharness-sdk.js && node test/metaharness-optional-deps.js && node test/cli-commands.js && node test/metaharness-cli.js && node test/db-workflow.js && node test/fast-agentdb-native.js && node test/mcp-stdio.js && node test/metaharness-mcp.js && node test/sigterm-cleanup.js && node test/mcp-policy.js && node test/mcp-command-security.js && node test/mcp-exploit-regression.js && node test/mcp-handshake.js && node test/mcp-intel-durability.js && node test/add-extras.js && node test/rabitq-optional.js && node test/rabitq-recall.js && node test/diskann-cli.js && node test/skills-install.js && node test/manifest-ranges.js && node test/attention-panic.js && node test/mcp-extras.js && node test/rvf-e2e.js && node test/startup-budget.js && node test/rag-context-rerank.js && node test/neural-q.js && node test/learning-import-types.js && node test/weight-space-optional.js && node test/weight-space-cli.js" + "test": "node test/integration.js && node test/graph-modularity.js && node test/backend-fallback.js && node test/mincut-wasm.js && node test/graph-cli-e2e.js && node test/metaharness-sdk.js && node test/metaharness-optional-deps.js && node test/cli-commands.js && node test/metaharness-cli.js && node test/db-workflow.js && node test/fast-agentdb-native.js && node test/mcp-stdio.js && node test/metaharness-mcp.js && node test/sigterm-cleanup.js && node test/mcp-policy.js && node test/mcp-command-security.js && node test/mcp-exploit-regression.js && node test/mcp-handshake.js && node test/mcp-intel-durability.js && node test/add-extras.js && node test/rabitq-optional.js && node test/rabitq-recall.js && node test/diskann-cli.js && node test/skills-install.js && node test/manifest-ranges.js && node test/attention-panic.js && node test/mcp-extras.js && node test/rvf-e2e.js && node test/startup-budget.js && node test/rag-context-rerank.js && node test/neural-q.js && node test/learning-import-types.js && node test/weight-space-optional.js && node test/weight-space-cli.js && node test/e-means.mjs", + "test:e-means": "node test/e-means.mjs" }, "keywords": [ "vector-database", @@ -51,7 +52,8 @@ "@ruvector/sona": "^0.1.11", "chalk": "^4.1.2", "commander": "^11.1.0", - "ora": "^5.4.1" + "ora": "^5.4.1", + "@ruvector/e-means": "0.1.0" }, "optionalDependencies": { "@metaharness/darwin": "^0.13.0", @@ -84,7 +86,10 @@ "wasm/", "skills/", "README.md", - "LICENSE" + "LICENSE", + "clustering.mjs", + "clustering.d.mts", + "clustering/" ], "peerDependencies": { "@metaharness/darwin": "^0.13.0", diff --git a/npm/packages/ruvector/test/e-means.mjs b/npm/packages/ruvector/test/e-means.mjs new file mode 100644 index 0000000000..b1add78cdf --- /dev/null +++ b/npm/packages/ruvector/test/e-means.mjs @@ -0,0 +1,10 @@ +import assert from 'node:assert/strict'; +import { init, fit, predict } from '../clustering.mjs'; + +await init(); +const values = new Float32Array([0, 0, 2, 2]); +const result = fit(values, 2, { clusters: 1, epochs: 1 }); +assert.deepEqual([...result.centroids], [1, 1]); +assert.equal(result.mse, 2); +assert.deepEqual([...predict(values, 2, result.centroids)], [0, 0]); +console.log('RuVector shared e-means dependency passed');