Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ exclude = ["crates/prime-radiant", "patches/candle-core-0.8.4", "patches/candle-
# unify with this workspace. The pure cores it wraps are members below.
"edge", "edge/*"]
members = [
# Shared EMA clustering core and portable npm WASM engine (ADR-359).
"crates/ruvector-e-means",
# ruvector Python SDK (ADR-352): PyO3 + maturin bindings. Salvaged from
# the stale feature/python-sdk-m1 branch (M1 RaBitQ scope) and extended.
"crates/ruvector-py",
Expand Down
2 changes: 2 additions & 0 deletions crates/ruvector-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ hnsw_rs = { workspace = true, optional = true }
# Turbo4 codec + unified encoding plane (ADR-296/297). Dependency-free and
# WASM-safe, so it is unconditional: the codec plane exists on every build.
ruvector-turboquant = { version = "2.3.0", path = "../ruvector-turboquant" }
# Shared CPU/WASM e-means optimizer, used by the explicit PQ training API.
ruvector-e-means = { version = "=0.1.0", path = "../ruvector-e-means" }
simsimd = { workspace = true, optional = true }
rayon = { workspace = true, optional = true }
crossbeam = { workspace = true, optional = true }
Expand Down
99 changes: 94 additions & 5 deletions crates/ruvector-core/src/quantization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,83 @@ pub struct ProductQuantized {
}

impl ProductQuantized {
/// Train product-quantization codebooks with the shared e-means optimizer.
///
/// `config.clusters` selects the number of entries in each codebook (1–256).
/// Every vector must have the same nonzero dimension, evenly divisible by
/// `num_subspaces`, and contain only finite values. The returned codebooks
/// use the same representation as [`Self::train`] and work with
/// [`Self::encode`]. No input vector is encoded until the caller requests it.
///
/// This API is explicit because e-means' automatic hyperparameters were
/// fitted on large embedding datasets; callers training small PQ subspaces
/// can set `batch_size` and `peak_step` in the configuration.
pub fn train_e_means(
vectors: &[Vec<f32>],
num_subspaces: usize,
config: &ruvector_e_means::Config,
) -> Result<Self> {
use crate::error::RuvectorError;

let dimensions = vectors.first().map_or(0, Vec::len);
if vectors.is_empty() || dimensions == 0 {
return Err(RuvectorError::InvalidInput(
"e-means PQ training requires nonempty vectors".into(),
));
}
if num_subspaces == 0 || dimensions % num_subspaces != 0 {
return Err(RuvectorError::InvalidParameter(
"Subspace count must be nonzero and divide vector dimensions".into(),
));
}
if config.clusters == 0 || config.clusters > 256 || config.clusters > vectors.len() {
return Err(RuvectorError::InvalidParameter(
"Codebook size must be in 1..=256 and no greater than the vector count".into(),
));
}
for vector in vectors {
if vector.len() != dimensions {
return Err(RuvectorError::DimensionMismatch {
expected: dimensions,
actual: vector.len(),
});
}
if vector.iter().any(|value| !value.is_finite()) {
return Err(RuvectorError::InvalidInput(
"e-means PQ training vectors must contain only finite values".into(),
));
}
}

let subspace_dim = dimensions / num_subspaces;
let capacity = vectors.len().checked_mul(subspace_dim).ok_or_else(|| {
RuvectorError::InvalidInput("e-means PQ training shape overflows usize".into())
})?;
let mut codebooks = Vec::with_capacity(num_subspaces);
let mut subspace_vectors = Vec::with_capacity(capacity);
for subspace_idx in 0..num_subspaces {
subspace_vectors.clear();
let start = subspace_idx * subspace_dim;
for vector in vectors {
subspace_vectors.extend_from_slice(&vector[start..start + subspace_dim]);
}
let model = ruvector_e_means::fit(&subspace_vectors, subspace_dim, config)
.map_err(|error| RuvectorError::InvalidParameter(error.to_string()))?;
codebooks.push(
model
.centroids
.chunks_exact(subspace_dim)
.map(<[f32]>::to_vec)
.collect(),
);
}

Ok(Self {
codes: Vec::new(),
codebooks,
})
}

/// Train product quantization on a set of vectors
pub fn train(
vectors: &[Vec<f32>],
Expand Down Expand Up @@ -168,16 +245,28 @@ impl ProductQuantized {
let start = subspace_idx * subspace_dim;
let end = start + subspace_dim;
let subvector = &vector[start..end];
// Keep the existing f32 distance, so codes for finite distances are
// unchanged. Only a candidate whose f32 squared distance overflows
// is re-measured in f64, keeping its nearest ordering.
let squared_distance = |centroid: &[f32]| -> f64 {
let fast = euclidean_squared(subvector, centroid);
if fast.is_finite() {
f64::from(fast)
} else {
subvector
.iter()
.zip(centroid)
.map(|(&x, &y)| (f64::from(x) - f64::from(y)).powi(2))
.sum()
}
};

// Find nearest centroid
let code = codebook
.iter()
.enumerate()
.min_by(|(_, a), (_, b)| {
let dist_a = euclidean_squared(subvector, a);
let dist_b = euclidean_squared(subvector, b);
dist_a.partial_cmp(&dist_b).unwrap()
})
.map(|(idx, centroid)| (idx, squared_distance(centroid)))
.min_by(|(_, a), (_, b)| a.total_cmp(b))
.map(|(idx, _)| idx as u8)
.unwrap_or(0);

Expand Down
144 changes: 144 additions & 0 deletions crates/ruvector-core/tests/e_means_quantization.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
//! Integration evidence for the shared e-means PQ training path.

use ruvector_core::error::RuvectorError;
use ruvector_core::ProductQuantized;
use ruvector_e_means::Config;

fn config(clusters: usize) -> Config {
Config {
clusters,
epochs: 3,
seed: 17,
..Config::default()
}
}

#[test]
fn subspace_codebooks_recover_column_means_and_keep_the_existing_format() {
let vectors = vec![
vec![1.0, 2.0, 10.0, 20.0],
vec![3.0, 4.0, 30.0, 40.0],
vec![5.0, 6.0, 50.0, 60.0],
];
let trained = ProductQuantized::train_e_means(&vectors, 2, &config(1)).unwrap();
assert!(trained.codes.is_empty());
assert_eq!(
trained.codebooks,
vec![vec![vec![3.0, 4.0]], vec![vec![30.0, 40.0]]]
);
assert_eq!(trained.encode(&vectors[0]), vec![0, 0]);

// Existing persisted PQ consumers use the same serde representation and
// reconstruct a vector by looking up each subspace's encoded centroid.
let json = serde_json::to_vec(&trained).unwrap();
let restored: ProductQuantized = serde_json::from_slice(&json).unwrap();
let codes = restored.encode(&vectors[2]);
let reconstructed: Vec<f32> = codes
.iter()
.enumerate()
.flat_map(|(subspace, &code)| restored.codebooks[subspace][code as usize].clone())
.collect();
assert_eq!(reconstructed, vec![3.0, 4.0, 30.0, 40.0]);
}

#[test]
fn identical_seed_and_options_produce_identical_codebooks() {
let vectors: Vec<Vec<f32>> = (0..80)
.map(|i| vec![i as f32 / 10.0, (i % 7) as f32, -(i as f32), 2.0])
.collect();
let mut options = config(4);
options.batch_size = Some(7); // Exercise the final short batch.
let first = ProductQuantized::train_e_means(&vectors, 2, &options).unwrap();
let second = ProductQuantized::train_e_means(&vectors, 2, &options).unwrap();
assert_eq!(first.codebooks, second.codebooks);
assert_eq!(first.codebooks.len(), 2);
assert!(first.codebooks.iter().all(|book| book.len() == 4));
assert!(first
.codebooks
.iter()
.flatten()
.all(|centroid| centroid.len() == 2 && centroid.iter().all(|v| v.is_finite())));
}

#[test]
fn maximum_u8_codebook_size_is_usable() {
let vectors: Vec<Vec<f32>> = (0..256).map(|i| vec![i as f32]).collect();
let mut options = config(256);
options.epochs = 1;
options.batch_size = Some(256);
let trained = ProductQuantized::train_e_means(&vectors, 1, &options).unwrap();
assert_eq!(trained.codebooks[0].len(), 256);
let codes: std::collections::BTreeSet<u8> =
vectors.iter().map(|v| trained.encode(v)[0]).collect();
assert_eq!(codes.len(), 256);
}

#[test]
fn large_finite_inputs_keep_the_nearest_centroid_when_squared_f32_distances_overflow() {
let magnitude = f32::MAX / 4.0;
let vectors = vec![vec![-magnitude], vec![magnitude]];
let mut options = config(2);
options.epochs = 1;
options.batch_size = Some(2);
let trained = ProductQuantized::train_e_means(&vectors, 1, &options).unwrap();
// Pick a query past the second centroid. Both candidate squared distances
// overflow f32, but the second remains strictly nearer in f64.
let query = trained.codebooks[0][1][0] * 2.0;
assert!(query.is_finite());
for centroid in &trained.codebooks[0] {
assert!((query - centroid[0]).powi(2).is_infinite());
}
assert_eq!(trained.encode(&[query]), vec![1]);
}

#[test]
fn malformed_training_inputs_return_errors() {
let valid = vec![vec![1.0, 2.0, 3.0, 4.0], vec![2.0, 3.0, 4.0, 5.0]];
for (vectors, subspaces) in [
(Vec::new(), 1),
(vec![Vec::new()], 1),
(valid.clone(), 0),
(valid.clone(), 3),
(valid.clone(), 8),
(vec![valid[0].clone(), vec![1.0]], 2),
(vec![vec![f32::NAN, 0.0]], 1),
(vec![vec![f32::INFINITY, 0.0]], 1),
] {
assert!(ProductQuantized::train_e_means(&vectors, subspaces, &config(1)).is_err());
}
let error =
ProductQuantized::train_e_means(&[vec![1.0, 2.0], vec![1.0]], 1, &config(1)).unwrap_err();
assert!(matches!(error, RuvectorError::DimensionMismatch { .. }));
}

#[test]
fn invalid_optimizer_settings_return_errors() {
let vectors = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
for clusters in [0, 3, 257] {
assert!(ProductQuantized::train_e_means(&vectors, 1, &config(clusters)).is_err());
}
let mut options = config(1);
options.epochs = 0;
assert!(ProductQuantized::train_e_means(&vectors, 1, &options).is_err());
options = config(1);
options.batch_size = Some(0);
assert!(ProductQuantized::train_e_means(&vectors, 1, &options).is_err());
options = config(1);
options.peak_step = Some(f64::NAN);
assert!(ProductQuantized::train_e_means(&vectors, 1, &options).is_err());
}

#[test]
fn finite_distance_codes_keep_the_existing_f32_ordering() {
// These rows tie in f32 but not in f64. Existing encode picks the first
// f32 minimum, and codes for already-trained indexes must not change.
let bits = |v: [u32; 4]| v.map(f32::from_bits).to_vec();
let query = bits([1059241166, 3208695764, 3205419074, 1055842996]);
let first = bits([1063676912, 3210863684, 1060297430, 3212197722]);
let second = bits([1052646232, 3210863675, 1060297430, 3212197722]);
let quantized = ProductQuantized {
codes: Vec::new(),
codebooks: vec![vec![first, second]],
};
assert_eq!(quantized.encode(&query), vec![0]);
}
31 changes: 31 additions & 0 deletions crates/ruvector-e-means/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
[package]
name = "ruvector-e-means"
version = "0.1.0"
edition = "2021"
rust-version = "1.81"
license = "MIT"
description = "Deterministic CPU and WebAssembly e-means clustering with EMA centroid updates"
repository = "https://github.com/ruvnet/ruvector"
homepage = "https://ruv.io"
documentation = "https://docs.rs/ruvector-e-means"
keywords = ["clustering", "k-means", "ema", "quantization", "machine-learning"]
categories = ["algorithms", "science"]
include = ["src/**", "tests/**", "examples/**", "Cargo.toml", "LICENSE", "README.md"]

[lib]
crate-type = ["rlib", "cdylib"]

[features]
default = []
serde = ["dep:serde"]
wasm = ["serde", "dep:serde_json", "dep:wasm-bindgen"]

[dependencies]
rand_chacha = "=0.3.1"
rand_core = "=0.6.4"
serde = { version = "1.0", features = ["derive"], optional = true }
serde_json = { version = "1.0", optional = true }
wasm-bindgen = { version = "=0.2.125", optional = true }

[dev-dependencies]
serde_json = "1.0"
21 changes: 21 additions & 0 deletions crates/ruvector-e-means/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
MIT License

Copyright (c) 2025 rUv

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Loading
Loading