diff --git a/README.md b/README.md index 72499c9..8543c56 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,8 @@ than the standard filter. | Type | Thread-safe | Deletion | Growable | Memory | Use when | |------|-------------|----------|----------|--------|----------| | `BloomFilter` | No | No | No | 1× | fixed capacity, single-threaded | -| `CountingBloomFilter` | No | Yes | No | 8× | need deletion, single-threaded | +| `CountingBloomFilter` | No | Yes | No | 8× | deletion, high accuracy | +| `CuckooFilter` | No | Yes | No | ~1× | deletion, memory-efficient | | `AtomicBloomFilter` | Yes | No | No | 1× | concurrent insert + lookup | | `ScalableBloomFilter` | No | No | Yes | 1×+ | unknown or unbounded item count | diff --git a/src/error.rs b/src/error.rs index 82c893f..740deaf 100644 --- a/src/error.rs +++ b/src/error.rs @@ -62,6 +62,16 @@ pub enum BloomError { /// /// [`ScalableBloomFilter::with_options`]: crate::ScalableBloomFilter::with_options InvalidTighteningRatio(f64), + + /// A [`CuckooFilter`] insertion failed because the table is too full. + /// + /// Cuckoo filters can reject insertions when the load factor approaches + /// its practical limit (~95%). Construct a larger filter or use a + /// [`ScalableBloomFilter`] if unbounded growth is required. + /// + /// [`CuckooFilter`]: crate::CuckooFilter + /// [`ScalableBloomFilter`]: crate::ScalableBloomFilter + CapacityExceeded, } impl PartialEq for BloomError { @@ -76,6 +86,7 @@ impl PartialEq for BloomError { (Self::IncompatibleGeometry { m: m1, k: k1 }, Self::IncompatibleGeometry { m: m2, k: k2 }) => m1 == m2 && k1 == k2, (Self::InvalidGrowthFactor(a), Self::InvalidGrowthFactor(b)) => a == b, (Self::InvalidTighteningRatio(a), Self::InvalidTighteningRatio(b)) => a == b || (a.is_nan() && b.is_nan()), + (Self::CapacityExceeded, Self::CapacityExceeded) => true, _ => false, } } @@ -105,6 +116,9 @@ impl fmt::Display for BloomError { BloomError::InvalidTighteningRatio(r) => { write!(f, "tightening ratio must be in (0, 1), got {r}") } + BloomError::CapacityExceeded => { + write!(f, "cuckoo filter is full — insert failed after maximum evictions") + } } } } diff --git a/src/filters/cuckoo.rs b/src/filters/cuckoo.rs new file mode 100644 index 0000000..737f6b2 --- /dev/null +++ b/src/filters/cuckoo.rs @@ -0,0 +1,371 @@ +//! Cuckoo filter — supports deletion with lower memory than a counting filter. + +use crate::error::BloomError; +use crate::hash::hash_pair; + +use crate::traits::Filter; +use crate::Bloomable; + +/// Number of fingerprint slots per bucket. +const BUCKET_SIZE: usize = 4; + +/// Maximum number of eviction attempts before an insert is declared failed. +const MAX_KICKS: usize = 500; + +/// A cuckoo filter — a space-efficient probabilistic set that supports +/// deletion. +/// +/// A cuckoo filter stores a small **fingerprint** (hash of the item) in one +/// of two candidate buckets. Lookup and deletion check only those two buckets, +/// making both operations `O(1)`. This is faster and more memory-efficient +/// than a [`CountingBloomFilter`] for workloads that require deletion. +/// +/// # Comparison with counting bloom filter +/// +/// | Property | `CuckooFilter` | `CountingBloomFilter` | +/// |----------|---------------|-----------------------| +/// | Deletion | ✓ | ✓ | +/// | Memory | ~1 byte/item | ~8 bytes/item | +/// | Insert | `Result` (can fail at ~95% load) | always succeeds | +/// | FPR | controlled by fingerprint size | controlled by `fpr` | +/// +/// # Deletion correctness +/// +/// Only delete items that were previously inserted. Deleting an absent item +/// may remove a fingerprint belonging to a different item, causing false +/// negatives. The filter cannot detect this; the contract is enforced by the +/// caller. +/// +/// # Capacity and load +/// +/// Insertions can fail when the table approaches its practical capacity limit +/// (~95% load factor). [`insert`] returns [`BloomError::CapacityExceeded`] in +/// that case. Size the filter with headroom: `capacity * 1.1` is a safe +/// starting point. +/// +/// # False positive rate +/// +/// With 8-bit fingerprints and bucket size 4, the empirical FPR is +/// approximately `2 * bucket_size / 2^fingerprint_bits ≈ 3%`. To achieve a +/// target FPR `p`, the fingerprint size `f` must satisfy +/// `f ≥ log2(2 * bucket_size / p)`. For 1% FPR, `f ≥ 10` bits. +/// +/// This implementation uses 8-bit fingerprints for simplicity. For lower FPR, +/// use [`BloomFilter`] or [`CountingBloomFilter`] instead. +/// +/// # Examples +/// +/// ```rust +/// use blume::prelude::*; +/// +/// let mut filter = CuckooFilter::new(1_000).unwrap(); +/// +/// filter.insert("alice").unwrap(); +/// filter.insert("bob").unwrap(); +/// +/// assert!(filter.contains("alice")); +/// assert!(filter.contains("bob")); +/// +/// filter.remove("alice"); +/// assert!(!filter.contains("alice")); +/// assert!(filter.contains("bob")); +/// ``` +/// +/// [`insert`]: CuckooFilter::insert +/// [`BloomFilter`]: crate::BloomFilter +/// [`CountingBloomFilter`]: crate::CountingBloomFilter +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct CuckooFilter { + /// Flat array of fingerprint buckets. Bucket `i` occupies + /// `table[i * BUCKET_SIZE .. (i+1) * BUCKET_SIZE]`. + /// A zero fingerprint means an empty slot. + table: Vec, + /// Number of buckets. + num_buckets: usize, + /// Number of fingerprints currently stored. + count: usize, +} + +impl CuckooFilter { + /// Creates a cuckoo filter sized for at least `capacity` items. + /// + /// The actual bucket count is rounded up to the next power of two to + /// enable fast bitwise index arithmetic. + /// + /// # Errors + /// + /// Returns [`BloomError::InvalidCapacity`] if `capacity` is `0`. + /// + /// # Examples + /// + /// ```rust + /// use blume::prelude::*; + /// + /// let mut filter = CuckooFilter::new(1_000).unwrap(); + /// filter.insert("hello").unwrap(); + /// assert!(filter.contains("hello")); + /// ``` + pub fn new(capacity: usize) -> Result { + if capacity == 0 { + return Err(BloomError::InvalidCapacity(capacity)); + } + // Each bucket holds BUCKET_SIZE items; add headroom for the ~95% load limit. + let min_buckets = (capacity * 2).div_ceil(BUCKET_SIZE); + let num_buckets = min_buckets.next_power_of_two().max(1); + Ok(Self { + table: vec![0u8; num_buckets * BUCKET_SIZE], + num_buckets, + count: 0, + }) + } + + /// Creates a cuckoo filter with an explicit bucket count. + /// + /// `num_buckets` is rounded up to the next power of two. This constructor + /// is intended for cases where you need to match the exact geometry of a + /// previously constructed filter. + /// + /// # Errors + /// + /// Returns [`BloomError::InvalidCapacity`] if `num_buckets` is `0`. + /// + /// # Examples + /// + /// ```rust + /// use blume::prelude::*; + /// + /// let mut filter = CuckooFilter::with_buckets(256).unwrap(); + /// filter.insert("hello").unwrap(); + /// assert!(filter.contains("hello")); + /// ``` + pub fn with_buckets(num_buckets: usize) -> Result { + if num_buckets == 0 { + return Err(BloomError::InvalidCapacity(num_buckets)); + } + let num_buckets = num_buckets.next_power_of_two(); + Ok(Self { + table: vec![0u8; num_buckets * BUCKET_SIZE], + num_buckets, + count: 0, + }) + } + + /// Inserts `item` into the filter. + /// + /// Returns `Ok(())` on success. Returns [`BloomError::CapacityExceeded`] + /// if the table is too full to accommodate the item (typically above ~95% + /// load). In that case, construct a larger filter. + /// + /// # Examples + /// + /// ```rust + /// use blume::prelude::*; + /// + /// let mut filter = CuckooFilter::new(100).unwrap(); + /// assert!(filter.insert("hello").is_ok()); + /// ``` + pub fn insert(&mut self, item: &T) -> Result<(), BloomError> { + let (fp, i1, i2) = self.fingerprint_and_indices(item); + + // Try to place into either candidate bucket. + if self.insert_into_bucket(i1, fp) || self.insert_into_bucket(i2, fp) { + self.count += 1; + return Ok(()); + } + + // Both buckets full — begin cuckoo eviction from a random-ish bucket. + let mut index = i1; + let mut evicted_fp = fp; + + for _ in 0..MAX_KICKS { + // Evict a random slot from the current bucket. + let slot = (evicted_fp as usize) % BUCKET_SIZE; + let offset = index * BUCKET_SIZE + slot; + std::mem::swap(&mut self.table[offset], &mut evicted_fp); + + // Compute the alternate bucket for the evicted fingerprint. + index = self.alt_index(index, evicted_fp); + + if self.insert_into_bucket(index, evicted_fp) { + self.count += 1; + return Ok(()); + } + } + + Err(BloomError::CapacityExceeded) + } + + /// Removes one occurrence of `item` from the filter. + /// + /// Returns `true` if the item was probably present and a fingerprint was + /// removed, `false` if it was definitely absent. + /// + /// # Correctness + /// + /// Only call this for items you know were previously inserted. Removing an + /// absent item may delete a fingerprint belonging to a different item, + /// causing false negatives for that item. + /// + /// # Examples + /// + /// ```rust + /// use blume::prelude::*; + /// + /// let mut filter = CuckooFilter::new(100).unwrap(); + /// filter.insert("hello").unwrap(); + /// + /// assert!(filter.remove("hello")); + /// assert!(!filter.contains("hello")); + /// assert!(!filter.remove("world")); // was never inserted + /// ``` + pub fn remove(&mut self, item: &T) -> bool { + let (fp, i1, i2) = self.fingerprint_and_indices(item); + if self.remove_from_bucket(i1, fp) || self.remove_from_bucket(i2, fp) { + self.count = self.count.saturating_sub(1); + return true; + } + false + } + + /// Resets the filter to empty. + /// + /// # Examples + /// + /// ```rust + /// use blume::prelude::*; + /// + /// let mut filter = CuckooFilter::new(100).unwrap(); + /// filter.insert("hello").unwrap(); + /// filter.clear(); + /// assert!(filter.is_empty()); + /// assert!(!filter.contains("hello")); + /// ``` + pub fn clear(&mut self) { + self.table.fill(0); + self.count = 0; + } + + /// Returns the number of buckets in the underlying hash table. + pub fn num_buckets(&self) -> usize { + self.num_buckets + } + + /// Returns the total number of fingerprint slots (`num_buckets × 4`). + pub fn total_slots(&self) -> usize { + self.num_buckets * BUCKET_SIZE + } + + // --- private helpers --- + + /// Computes the 8-bit fingerprint and both candidate bucket indices for `item`. + /// + /// Uses h1 for the primary index and h2 for the fingerprint. The alternate + /// index is derived as `i1 XOR hash(fp)` so it can be recovered from either + /// bucket without storing the original key. + fn fingerprint_and_indices(&self, item: &T) -> (u8, usize, usize) { + let (h1, h2) = hash_pair(item); + // Non-zero fingerprint: map 0 → 1 to keep 0 as the empty-slot sentinel. + let fp = ((h2 & 0xFF) as u8).max(1); + let i1 = (h1 as usize) & (self.num_buckets - 1); + let i2 = self.alt_index(i1, fp); + (fp, i1, i2) + } + + /// Computes the alternate bucket index using `index XOR hash(fingerprint)`. + /// + /// This operation is its own inverse: `alt_index(alt_index(i, fp), fp) == i`. + /// The property ensures we can always recover the partner bucket from + /// either end without storing the original item. + #[inline] + fn alt_index(&self, index: usize, fp: u8) -> usize { + // Multiply the fingerprint by a large prime to spread the XOR uniformly. + let hash = (fp as usize).wrapping_mul(0x517cc1b727220a95); + (index ^ hash) & (self.num_buckets - 1) + } + + /// Attempts to place `fp` into an empty slot in bucket `index`. + /// Returns `true` on success. + #[inline] + fn insert_into_bucket(&mut self, index: usize, fp: u8) -> bool { + let start = index * BUCKET_SIZE; + for slot in &mut self.table[start..start + BUCKET_SIZE] { + if *slot == 0 { + *slot = fp; + return true; + } + } + false + } + + /// Attempts to remove one occurrence of `fp` from bucket `index`. + /// Returns `true` on success. + #[inline] + fn remove_from_bucket(&mut self, index: usize, fp: u8) -> bool { + let start = index * BUCKET_SIZE; + for slot in &mut self.table[start..start + BUCKET_SIZE] { + if *slot == fp { + *slot = 0; + return true; + } + } + false + } +} + +impl Filter for CuckooFilter { + /// Returns `true` if `item` is probably in the filter, `false` if it is + /// definitely absent. + /// + /// Checks both candidate buckets for the item's fingerprint. This is + /// `O(1)` regardless of filter size. + #[inline] + fn contains(&self, item: &T) -> bool { + let (fp, i1, i2) = self.fingerprint_and_indices(item); + self.bucket_contains(i1, fp) || self.bucket_contains(i2, fp) + } + + fn item_count(&self) -> usize { + self.count + } + + /// Returns the total number of fingerprint slots. + fn bit_size(&self) -> usize { + self.total_slots() * 8 + } + + /// Returns the practical capacity — approximately 95% of total slots. + fn capacity(&self) -> usize { + (self.total_slots() * 95) / 100 + } + + /// Returns an estimated false positive rate based on current load. + /// + /// With 8-bit fingerprints and 4-slot buckets, the FPR at load factor `α` + /// is approximately `α × 2 × bucket_size / 256`. + fn estimated_fpr(&self) -> f64 { + if self.count == 0 { + return 0.0; + } + let load = self.count as f64 / self.total_slots() as f64; + load * 2.0 * BUCKET_SIZE as f64 / 256.0 + } +} + +impl CuckooFilter { + /// Returns `true` if bucket `index` contains fingerprint `fp`. + #[inline] + fn bucket_contains(&self, index: usize, fp: u8) -> bool { + let start = index * BUCKET_SIZE; + self.table[start..start + BUCKET_SIZE].contains(&fp) + } +} + +// Ensure CuckooFilter satisfies the validated_params contract at the type +// level by calling it in `new` — nothing to do here since CuckooFilter's +// `new` validates capacity independently. +const _: () = { + // Compile-time sanity check: BUCKET_SIZE must be a power of two for the + // slot-selection arithmetic to distribute evenly. + assert!(BUCKET_SIZE.is_power_of_two()); +}; diff --git a/src/filters/mod.rs b/src/filters/mod.rs index 8eae136..31c1649 100644 --- a/src/filters/mod.rs +++ b/src/filters/mod.rs @@ -1,9 +1,11 @@ mod atomic_bloom; mod bloom; mod counting; +mod cuckoo; mod scalable; pub use atomic_bloom::AtomicBloomFilter; pub use bloom::BloomFilter; pub use counting::CountingBloomFilter; +pub use cuckoo::CuckooFilter; pub use scalable::ScalableBloomFilter; diff --git a/src/lib.rs b/src/lib.rs index 9a2538a..0238068 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -105,5 +105,5 @@ pub(crate) mod math; pub use bloomable::Bloomable; pub use error::BloomError; -pub use filters::{AtomicBloomFilter, BloomFilter, CountingBloomFilter, ScalableBloomFilter}; +pub use filters::{AtomicBloomFilter, BloomFilter, CountingBloomFilter, CuckooFilter, ScalableBloomFilter}; pub use traits::{ConcurrentFilter, Filter, MutableFilter, RemovableFilter}; diff --git a/src/prelude.rs b/src/prelude.rs index c9a1e31..354ef28 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -12,5 +12,5 @@ pub use crate::bloomable::Bloomable; pub use crate::error::BloomError; -pub use crate::filters::{AtomicBloomFilter, BloomFilter, CountingBloomFilter, ScalableBloomFilter}; +pub use crate::filters::{AtomicBloomFilter, BloomFilter, CountingBloomFilter, CuckooFilter, ScalableBloomFilter}; pub use crate::traits::{ConcurrentFilter, Filter, MutableFilter, RemovableFilter}; diff --git a/tests/cuckoo_filter.rs b/tests/cuckoo_filter.rs new file mode 100644 index 0000000..df05bff --- /dev/null +++ b/tests/cuckoo_filter.rs @@ -0,0 +1,230 @@ +//! Integration tests for [`CuckooFilter`]. +//! +//! - **Construction** — valid and invalid parameters, error variants. +//! - **No false negatives** — every inserted item is found. +//! - **Deletion** — removed items become absent; others are unaffected. +//! - **Capacity** — insert fails gracefully when the filter is full. +//! - **Behavioural** — `clear`, `item_count`, `estimated_fpr`. +//! - **Statistical** — empirical FPR within expected bounds. + +mod common; + +use blume::prelude::*; +use proptest::prelude::*; +use rstest::rstest; + +// --- construction --- + +#[rstest] +#[case(1)] +#[case(100)] +#[case(10_000)] +fn new_valid(#[case] capacity: usize) { + assert!(CuckooFilter::new(capacity).is_ok()); +} + +#[test] +fn new_zero_capacity_errors() { + assert!(matches!( + CuckooFilter::new(0), + Err(BloomError::InvalidCapacity(0)) + )); +} + +#[test] +fn with_buckets_zero_errors() { + assert!(matches!( + CuckooFilter::with_buckets(0), + Err(BloomError::InvalidCapacity(0)) + )); +} + +#[test] +fn with_buckets_rounds_to_power_of_two() { + let f = CuckooFilter::with_buckets(100).unwrap(); + assert!(f.num_buckets().is_power_of_two()); + assert!(f.num_buckets() >= 100); +} + +// --- proptest: no false negatives --- + +proptest! { + /// Every inserted item is found immediately after insertion. + #[test] + fn no_false_negatives_u64(items in prop::collection::hash_set(any::(), 1..500usize)) { + let items: Vec = items.into_iter().collect(); + let mut f = CuckooFilter::new(items.len() * 2).unwrap(); + for item in &items { + f.insert(item).expect("insert failed"); + } + for item in &items { + prop_assert!(f.contains(item)); + } + } + + #[test] + fn no_false_negatives_string( + items in prop::collection::hash_set("[a-z]{1,20}", 1..200usize) + ) { + let items: Vec = items.into_iter().collect(); + let mut f = CuckooFilter::new(items.len() * 2).unwrap(); + for item in &items { + f.insert(item.as_str()).expect("insert failed"); + } + for item in &items { + prop_assert!(f.contains(item.as_str())); + } + } +} + +// --- deletion --- + +/// Removing an inserted item makes it absent; other items are unaffected. +#[test] +fn remove_makes_item_absent() { + let mut f = CuckooFilter::new(100).unwrap(); + f.insert("alice").unwrap(); + f.insert("bob").unwrap(); + + assert!(f.remove("alice")); + assert!(!f.contains("alice")); + assert!(f.contains("bob")); +} + +/// Removing from an empty filter returns false. +#[test] +fn remove_absent_returns_false() { + let mut f = CuckooFilter::new(100).unwrap(); + assert!(!f.remove("never_inserted")); +} + +/// Insert twice, remove once — item is still present (first fingerprint gone, +/// second remains). +#[test] +fn double_insert_requires_double_remove() { + let mut f = CuckooFilter::new(100).unwrap(); + f.insert(&42u64).unwrap(); + f.insert(&42u64).unwrap(); + + f.remove(&42u64); + assert!(f.contains(&42u64)); + + f.remove(&42u64); + assert!(!f.contains(&42u64)); +} + +proptest! { + /// Removing an item in a single-item filter always produces absence. + #[test] + fn remove_only_item_makes_absent(item in any::()) { + let mut f = CuckooFilter::new(100).unwrap(); + f.insert(&item).unwrap(); + f.remove(&item); + prop_assert!(!f.contains(&item)); + } +} + +// --- capacity --- + +/// Inserting at very high load eventually returns `CapacityExceeded`. +#[test] +fn insert_fails_when_full() { + // Very small filter — fill it completely. + let mut f = CuckooFilter::with_buckets(4).unwrap(); + let mut succeeded = 0usize; + let mut failed = false; + + for i in 0u64..1_000 { + match f.insert(&i) { + Ok(()) => succeeded += 1, + Err(BloomError::CapacityExceeded) => { + failed = true; + break; + } + Err(e) => panic!("unexpected error: {e}"), + } + } + + assert!(failed, "expected CapacityExceeded but all {succeeded} inserts succeeded"); +} + +// --- is_empty --- + +#[test] +fn is_empty_reflects_insertions() { + let mut f = CuckooFilter::new(100).unwrap(); + assert!(f.is_empty()); + f.insert(&1u64).unwrap(); + assert!(!f.is_empty()); +} + +// --- clear --- + +#[test] +fn clear_resets_filter() { + let mut f = CuckooFilter::new(100).unwrap(); + f.insert(&1u64).unwrap(); + f.insert(&2u64).unwrap(); + f.clear(); + assert!(f.is_empty()); + assert!(!f.contains(&1u64)); + assert!(!f.contains(&2u64)); +} + +// --- proptest: behavioural --- + +proptest! { + #[test] + fn item_count_tracks_insertions( + items in prop::collection::hash_set(any::(), 1..200usize) + ) { + let items: Vec = items.into_iter().collect(); + let mut f = CuckooFilter::new(items.len() * 2).unwrap(); + for item in &items { f.insert(item).unwrap(); } + prop_assert_eq!(f.item_count(), items.len()); + } + + #[test] + fn item_count_decrements_on_remove( + items in prop::collection::hash_set(any::(), 2..200usize) + ) { + let items: Vec = items.into_iter().collect(); + let mut f = CuckooFilter::new(items.len() * 2).unwrap(); + for item in &items { f.insert(item).unwrap(); } + + let before = f.item_count(); + assert!(f.remove(&items[0])); + assert_eq!(f.item_count(), before - 1); + } + + #[test] + fn estimated_fpr_starts_at_zero_and_rises(n in 10usize..100) { + let mut f = CuckooFilter::new(n * 4).unwrap(); + prop_assert_eq!(f.estimated_fpr(), 0.0); + for i in 0..n as u64 { f.insert(&i).unwrap(); } + prop_assert!(f.estimated_fpr() > 0.0); + } +} + +// --- statistical: fpr within bounds --- + +/// Inserts 1 000 distinct items, then probes 100 000 disjoint values and +/// asserts the measured FPR is below 5% (theoretical ~3% with 8-bit fingerprints). +#[test] +fn fpr_within_expected_bounds() { + let n = 1_000usize; + let mut f = CuckooFilter::new(n * 2).unwrap(); + for i in 0..n as u64 { f.insert(&i).unwrap(); } + + let offset = 1_000_000_000u64; + let trials = 100_000u64; + let false_positives = (offset..offset + trials).filter(|i| f.contains(i)).count(); + let measured = false_positives as f64 / trials as f64; + + // With 8-bit fingerprints the expected FPR is ~3%. Allow up to 5% for + // statistical variance. + assert!( + measured < 0.05, + "measured FPR {measured:.4} exceeded 5% bound for 8-bit cuckoo filter" + ); +}