From 8d1652896422b7d8b89343b71144c4ea2b32a260 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 14:06:56 -0300 Subject: [PATCH 01/34] Initial `aggsigdb.MemDB` implementation - Use Actor model - Implement `store` - Test `store` --- crates/core/src/aggsigdb/memory.rs | 151 +++++++++++++++++++++++++++++ crates/core/src/aggsigdb/mod.rs | 2 + crates/core/src/lib.rs | 6 +- crates/core/src/types.rs | 58 +++++------ 4 files changed, 188 insertions(+), 29 deletions(-) create mode 100644 crates/core/src/aggsigdb/memory.rs create mode 100644 crates/core/src/aggsigdb/mod.rs diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs new file mode 100644 index 00000000..1b4a94a7 --- /dev/null +++ b/crates/core/src/aggsigdb/memory.rs @@ -0,0 +1,151 @@ +use crate::types; +use std::collections::{HashMap, hash_map::Entry}; + +#[derive(Debug, thiserror::Error)] +enum StoreError { + #[error("Mismatching data")] + MismatchingData, + + #[error("Send error: {0}")] + Send(#[from] tokio::sync::mpsc::error::SendError), + + #[error("Recv error: {0}")] + Recv(#[from] tokio::sync::oneshot::error::RecvError), +} + +#[derive(Debug)] +enum MemDBCommand { + Store { + duty: types::Duty, + pub_key: types::PubKey, + signed_data: Box, + + response: tokio::sync::oneshot::Sender>, + }, +} + +#[derive(Debug)] +struct MemDBActor { + receiver: tokio::sync::mpsc::Receiver, + data: HashMap<(types::Duty, types::PubKey), Box>, +} + +impl MemDBActor { + fn new(receiver: tokio::sync::mpsc::Receiver) -> Self { + Self { + receiver, + data: HashMap::new(), + } + } + + async fn run(&mut self) { + while let Some(cmd) = self.receiver.recv().await { + match cmd { + MemDBCommand::Store { + duty, + pub_key, + signed_data, + response, + } => { + let result = self.store(duty, pub_key, signed_data).await; + let _ = response.send(result); + } + } + } + } + + async fn store( + &mut self, + duty: types::Duty, + pub_key: types::PubKey, + signed_data: Box, + ) -> Result<(), StoreError> { + // TODO: Add deadline tracking + // _ = db.deadliner.Add(command.duty) + + match self.data.entry((duty, pub_key)) { + Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { + Err(StoreError::MismatchingData) + } + Entry::Occupied(_) => Ok(()), + Entry::Vacant(slot) => { + slot.insert(signed_data); + Ok(()) + } + } + } +} + +struct MemDBHandle { + sender: tokio::sync::mpsc::Sender, +} + +impl MemDBHandle { + fn new() -> Self { + let (sender, receiver) = tokio::sync::mpsc::channel(100); + let mut actor = MemDBActor::new(receiver); + // TODO: Pass a cancellation token + tokio::spawn(async move { + actor.run().await; + }); + Self { sender } + } + + async fn store( + &self, + duty: types::Duty, + pub_key: types::PubKey, + signed_data: Box, + ) -> Result<(), StoreError> { + let (tx, rx) = tokio::sync::oneshot::channel(); + self.sender + .send(MemDBCommand::Store { + duty, + pub_key, + signed_data, + response: tx, + }) + .await?; + rx.await? + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + signeddata::SignedDataError, + types::{Duty, PubKey, Signature, SignedData, SlotNumber}, + }; + + #[derive(Debug, Clone, PartialEq, Eq)] + struct MockSignedData; + + impl SignedData for MockSignedData { + fn signature(&self) -> Result { + Ok(Signature::new([42u8; 96])) + } + + fn set_signature(&self, _signature: Signature) -> Result { + Ok(self.clone()) + } + + fn message_root(&self) -> Result<[u8; 32], SignedDataError> { + Ok([42u8; 32]) + } + } + + #[tokio::test] + async fn test_single_handle_store() { + let handle = MemDBHandle::new(); + let duty = Duty::new_attester_duty(SlotNumber::new(1)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data: Box = Box::new(MockSignedData); + + let task = tokio::spawn(async move { handle.store(duty, pub_key, signed_data).await }); + + task.await + .expect("store task panicked") + .expect("store returned an error"); + } +} diff --git a/crates/core/src/aggsigdb/mod.rs b/crates/core/src/aggsigdb/mod.rs new file mode 100644 index 00000000..54c50307 --- /dev/null +++ b/crates/core/src/aggsigdb/mod.rs @@ -0,0 +1,2 @@ +/// Memory implementation of the AggSigDB. +pub mod memory; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 1076ade4..91cfd4d0 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -23,10 +23,14 @@ pub mod version; /// Duty deadline tracking and notification. pub mod deadline; -/// parsigdb +/// Implementations of ParSigDB. pub mod parsigdb; +/// Implementations of AggSigDB. +pub mod aggsigdb; + mod parsigex_codec; + // SSZ codec operates on compile-time-constant byte sizes and offsets. // Arithmetic is bounded and casts from `usize` to `u32` are safe because all // sizes are well below `u32::MAX`. diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 8d971f7d..b531ad0a 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -704,51 +704,52 @@ impl TryFrom<(&DutyType, &pbcore::ParSignedDataSet)> for ParSignedDataSet { } } -/// SignedDataSet is a set of signed duty data. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SignedDataSet(HashMap); - -impl Default for SignedDataSet -where - T: SignedData, -{ - fn default() -> Self { - Self(HashMap::default()) - } -} +/// A set of signed duty data. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SignedDataSet(HashMap>); -impl SignedDataSet -where - T: SignedData, -{ +impl SignedDataSet { /// Create a new signed data set. pub fn new() -> Self { Self::default() } /// Get a signed data by public key. - pub fn get(&self, pub_key: &PubKey) -> Option<&T> { - self.0.get(pub_key) + pub fn get(&self, pub_key: &PubKey) -> Option<&dyn SignedData> { + self.0.get(pub_key).map(|b| b.as_ref()) } /// Insert a signed data. - pub fn insert(&mut self, pub_key: PubKey, signed_data: T) { - self.0.insert(pub_key, signed_data); + pub fn insert(&mut self, pub_key: PubKey, signed_data: impl SignedData) { + self.0.insert(pub_key, Box::new(signed_data)); } /// Remove a signed data by public key. - pub fn remove(&mut self, pub_key: &PubKey) -> Option { + pub fn remove(&mut self, pub_key: &PubKey) -> Option> { self.0.remove(pub_key) } - /// Inner signed data set. - pub fn inner(&self) -> &HashMap { - &self.0 + /// Iterate over the signed data set by reference. + pub fn iter(&self) -> std::collections::hash_map::Iter<'_, PubKey, Box> { + self.0.iter() } +} - /// Inner signed data set. - pub fn inner_mut(&mut self) -> &mut HashMap { - &mut self.0 +impl IntoIterator for SignedDataSet { + type IntoIter = std::collections::hash_map::IntoIter>; + type Item = (PubKey, Box); + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<'a> IntoIterator for &'a SignedDataSet { + type IntoIter = std::collections::hash_map::Iter<'a, PubKey, Box>; + type Item = (&'a PubKey, &'a Box); + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() } } @@ -1052,9 +1053,10 @@ mod tests { fn signed_data_set() { let mut signed_data_set = SignedDataSet::new(); signed_data_set.insert(PubKey::new([42u8; PK_LEN]), MockSignedData); + let expected: &dyn SignedData = &MockSignedData; assert_eq!( signed_data_set.get(&PubKey::new([42u8; PK_LEN])), - Some(&MockSignedData) + Some(expected) ); } From 8504da10ea5dc093e1a03b947300fc49694c8802 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 15:43:14 -0300 Subject: [PATCH 02/34] Refactor - Expose top-level `new` - Remove impossible errors - Add inline docs --- crates/core/src/aggsigdb/memory.rs | 65 +++++++++++++++++------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 1b4a94a7..1330acfa 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,37 +1,34 @@ use crate::types; use std::collections::{HashMap, hash_map::Entry}; +/// Errors for the in-memory AggSigDB implementation. #[derive(Debug, thiserror::Error)] -enum StoreError { +pub enum Error { + /// Data for the same duty and public key already exists but does not match + /// the new data. #[error("Mismatching data")] MismatchingData, - - #[error("Send error: {0}")] - Send(#[from] tokio::sync::mpsc::error::SendError), - - #[error("Recv error: {0}")] - Recv(#[from] tokio::sync::oneshot::error::RecvError), } #[derive(Debug)] -enum MemDBCommand { +enum Command { Store { duty: types::Duty, pub_key: types::PubKey, signed_data: Box, - response: tokio::sync::oneshot::Sender>, + response: tokio::sync::oneshot::Sender>, }, } #[derive(Debug)] -struct MemDBActor { - receiver: tokio::sync::mpsc::Receiver, +struct Actor { + receiver: tokio::sync::mpsc::Receiver, data: HashMap<(types::Duty, types::PubKey), Box>, } -impl MemDBActor { - fn new(receiver: tokio::sync::mpsc::Receiver) -> Self { +impl Actor { + fn new(receiver: tokio::sync::mpsc::Receiver) -> Self { Self { receiver, data: HashMap::new(), @@ -41,7 +38,7 @@ impl MemDBActor { async fn run(&mut self) { while let Some(cmd) = self.receiver.recv().await { match cmd { - MemDBCommand::Store { + Command::Store { duty, pub_key, signed_data, @@ -59,13 +56,13 @@ impl MemDBActor { duty: types::Duty, pub_key: types::PubKey, signed_data: Box, - ) -> Result<(), StoreError> { + ) -> Result<(), Error> { // TODO: Add deadline tracking // _ = db.deadliner.Add(command.duty) match self.data.entry((duty, pub_key)) { Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { - Err(StoreError::MismatchingData) + Err(Error::MismatchingData) } Entry::Occupied(_) => Ok(()), Entry::Vacant(slot) => { @@ -76,43 +73,53 @@ impl MemDBActor { } } -struct MemDBHandle { - sender: tokio::sync::mpsc::Sender, +/// Handle to interact with the AggSigDB in-memory actor. +#[derive(Clone)] +pub struct Handle { + sender: tokio::sync::mpsc::Sender, } -impl MemDBHandle { +impl Handle { fn new() -> Self { let (sender, receiver) = tokio::sync::mpsc::channel(100); - let mut actor = MemDBActor::new(receiver); - // TODO: Pass a cancellation token + let mut actor = Actor::new(receiver); tokio::spawn(async move { actor.run().await; }); Self { sender } } - async fn store( + /// Stores aggregated signed duty data set. + pub async fn store( &self, duty: types::Duty, pub_key: types::PubKey, signed_data: Box, - ) -> Result<(), StoreError> { + ) -> Result<(), Error> { let (tx, rx) = tokio::sync::oneshot::channel(); - self.sender - .send(MemDBCommand::Store { + let _ = self + .sender + .send(Command::Store { duty, pub_key, signed_data, response: tx, }) - .await?; - rx.await? + .await; + rx.await.expect("Actor task has been killed") } } +/// Create a new memory AggSigDB implementation and return its handle. +/// +/// Clone this handle to share access to the same AggSigDB instance across +/// multiple tasks. +pub fn new() -> Handle { + Handle::new() +} + #[cfg(test)] mod tests { - use super::*; use crate::{ signeddata::SignedDataError, types::{Duty, PubKey, Signature, SignedData, SlotNumber}, @@ -137,7 +144,7 @@ mod tests { #[tokio::test] async fn test_single_handle_store() { - let handle = MemDBHandle::new(); + let handle = super::new(); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); let signed_data: Box = Box::new(MockSignedData); From 0890f6323899fae4957cfb44e68960e7dbba4287 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 16:31:38 -0300 Subject: [PATCH 03/34] Rewrite from Actor - Use Mutex/Notify pattern - Remove need for handles - Adjust tests and docs --- crates/core/src/aggsigdb/memory.rs | 160 +++++++++++++---------------- 1 file changed, 72 insertions(+), 88 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 1330acfa..e4724860 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,5 +1,10 @@ +use tokio::sync::{Mutex, Notify}; + use crate::types; -use std::collections::{HashMap, hash_map::Entry}; +use std::{ + collections::{HashMap, hash_map::Entry}, + sync::Arc, +}; /// Errors for the in-memory AggSigDB implementation. #[derive(Debug, thiserror::Error)] @@ -10,112 +15,77 @@ pub enum Error { MismatchingData, } -#[derive(Debug)] -enum Command { - Store { - duty: types::Duty, - pub_key: types::PubKey, - signed_data: Box, - - response: tokio::sync::oneshot::Sender>, - }, -} +/// An in-memory implementation of the AggSigDB. +/// +/// Share an instance by cloning. Cloning is cheap and creates a new reference +/// to the same underlying data. +#[derive(Clone)] +pub struct MemDB(Arc); #[derive(Debug)] -struct Actor { - receiver: tokio::sync::mpsc::Receiver, - data: HashMap<(types::Duty, types::PubKey), Box>, +struct MemDBInner { + data: Mutex>>, + notify: Notify, } -impl Actor { - fn new(receiver: tokio::sync::mpsc::Receiver) -> Self { - Self { - receiver, - data: HashMap::new(), - } +impl MemDB { + /// Creates a new in-memory AggSigDB instance. + pub fn new() -> Self { + Self(Arc::new(MemDBInner { + data: Mutex::new(HashMap::new()), + notify: Notify::new(), + })) } - async fn run(&mut self) { - while let Some(cmd) = self.receiver.recv().await { - match cmd { - Command::Store { - duty, - pub_key, - signed_data, - response, - } => { - let result = self.store(duty, pub_key, signed_data).await; - let _ = response.send(result); - } - } - } - } - - async fn store( - &mut self, + /// Stores aggregated signed duty data set. + pub async fn store( + &self, duty: types::Duty, pub_key: types::PubKey, signed_data: Box, ) -> Result<(), Error> { - // TODO: Add deadline tracking - // _ = db.deadliner.Add(command.duty) + let mut data = self.0.data.lock().await; - match self.data.entry((duty, pub_key)) { + match data.entry((duty, pub_key)) { Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { Err(Error::MismatchingData) } Entry::Occupied(_) => Ok(()), Entry::Vacant(slot) => { slot.insert(signed_data); + // TODO: Optimize to only wake those who are waiting for this specific duty and + // pubkey + self.0.notify.notify_waiters(); Ok(()) } } } -} - -/// Handle to interact with the AggSigDB in-memory actor. -#[derive(Clone)] -pub struct Handle { - sender: tokio::sync::mpsc::Sender, -} - -impl Handle { - fn new() -> Self { - let (sender, receiver) = tokio::sync::mpsc::channel(100); - let mut actor = Actor::new(receiver); - tokio::spawn(async move { - actor.run().await; - }); - Self { sender } - } - /// Stores aggregated signed duty data set. - pub async fn store( + /// Blocks and returns the aggregated signed duty data when available. + pub async fn wait_for( &self, duty: types::Duty, pub_key: types::PubKey, - signed_data: Box, - ) -> Result<(), Error> { - let (tx, rx) = tokio::sync::oneshot::channel(); - let _ = self - .sender - .send(Command::Store { - duty, - pub_key, - signed_data, - response: tx, - }) - .await; - rx.await.expect("Actor task has been killed") - } -} + ) -> Box { + let k = (duty, pub_key); + loop { + // Register interest before checking the map so that a concurrent `store` either + // (a) inserts before we check and we observe the value, or (b) inserts after + // our `notified()` is enabled and wakes us. + let notified = self.0.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + + { + let data = self.0.data.lock().await; + if let Some(data) = data.get(&k) { + return data.clone(); + } + } -/// Create a new memory AggSigDB implementation and return its handle. -/// -/// Clone this handle to share access to the same AggSigDB instance across -/// multiple tasks. -pub fn new() -> Handle { - Handle::new() + notified.await; + } + } } #[cfg(test)] @@ -142,17 +112,31 @@ mod tests { } } - #[tokio::test] - async fn test_single_handle_store() { - let handle = super::new(); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn store_and_wait() { + let store = super::MemDB::new(); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); let signed_data: Box = Box::new(MockSignedData); - let task = tokio::spawn(async move { handle.store(duty, pub_key, signed_data).await }); + let reader = { + let store = store.clone(); + let duty = duty.clone(); + let pub_key = pub_key.clone(); + + tokio::spawn(async move { store.wait_for(duty, pub_key).await }) + }; + + // Give the reader a chance to reach `notified.await` before we store, so the + // test actually exercises the notify wakeup path rather than the + // fast-path lookup. + tokio::task::yield_now().await; + assert!(!reader.is_finished(), "wait_for should block until store"); + + let write = store.store(duty, pub_key, signed_data.clone()).await; + let read = reader.await.unwrap(); - task.await - .expect("store task panicked") - .expect("store returned an error"); + assert!(write.is_ok()); + assert_eq!(read, signed_data); } } From df51e51ee8c8185918652b723aa44c89bc793c90 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 18:04:58 -0300 Subject: [PATCH 04/34] Add eviction through Deadliner - Use existing Deadliner API (Arc) - Add tests --- crates/core/src/aggsigdb/memory.rs | 147 ++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 13 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index e4724860..2e9d4369 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,10 +1,9 @@ -use tokio::sync::{Mutex, Notify}; - -use crate::types; +use crate::{deadline::Deadliner, types}; use std::{ collections::{HashMap, hash_map::Entry}, sync::Arc, }; +use tokio::sync::{Mutex, Notify}; /// Errors for the in-memory AggSigDB implementation. #[derive(Debug, thiserror::Error)] @@ -22,19 +21,43 @@ pub enum Error { #[derive(Clone)] pub struct MemDB(Arc); -#[derive(Debug)] struct MemDBInner { data: Mutex>>, + deadliner: Arc, notify: Notify, } impl MemDB { /// Creates a new in-memory AggSigDB instance. - pub fn new() -> Self { - Self(Arc::new(MemDBInner { + pub fn new(deadliner: Arc) -> Self { + let this = Self(Arc::new(MemDBInner { data: Mutex::new(HashMap::new()), + deadliner: Arc::clone(&deadliner), notify: Notify::new(), - })) + })); + + match deadliner.c() { + Some(evictions) => { + tokio::spawn(Self::evict(Arc::downgrade(&this.0), evictions)); + } + None => { + // TODO: In Charon, `deadliner.c()` always returns `Some` + } + } + + this + } + + async fn evict( + inner: std::sync::Weak, + mut evictions: tokio::sync::mpsc::Receiver, + ) { + while let Some(duty) = evictions.recv().await { + let Some(inner) = inner.upgrade() else { + return; + }; + inner.data.lock().await.retain(|(d, _), _| d != &duty); + } } /// Stores aggregated signed duty data set. @@ -44,6 +67,9 @@ impl MemDB { pub_key: types::PubKey, signed_data: Box, ) -> Result<(), Error> { + // TODO(charon): Distinguish between no deadline supported vs already expired. + let _ = self.0.deadliner.add(duty.clone()).await; + let mut data = self.0.data.lock().await; match data.entry((duty, pub_key)) { @@ -91,16 +117,27 @@ impl MemDB { #[cfg(test)] mod tests { use crate::{ + deadline::Deadliner, signeddata::SignedDataError, types::{Duty, PubKey, Signature, SignedData, SlotNumber}, }; + use async_trait::async_trait; + use std::sync::Arc; + use tokio::sync; + /// Some mock signed data type for testing. #[derive(Debug, Clone, PartialEq, Eq)] - struct MockSignedData; + struct MockSignedData(u8); + + impl MockSignedData { + fn for_test(value: u8) -> Box { + Box::new(Self(value)) + } + } impl SignedData for MockSignedData { fn signature(&self) -> Result { - Ok(Signature::new([42u8; 96])) + Ok(Signature::new([self.0; 96])) } fn set_signature(&self, _signature: Signature) -> Result { @@ -108,16 +145,45 @@ mod tests { } fn message_root(&self) -> Result<[u8; 32], SignedDataError> { - Ok([42u8; 32]) + Ok([self.0; 32]) + } + } + + /// Deadliner that hands out a caller-supplied receiver, allowing tests to + /// drive eviction by sending on the paired sender. + struct TestDeadliner(std::sync::Mutex>>); + + impl TestDeadliner { + fn new(receiver: sync::mpsc::Receiver) -> Arc { + Arc::new(Self(std::sync::Mutex::new(Some(receiver)))) + } + + /// Creates a deadliner that never returns any duties to evict, so no + /// eviction will occur. + fn never() -> Arc { + Arc::new(Self(std::sync::Mutex::new(None))) + } + } + + #[async_trait] + impl Deadliner for TestDeadliner { + async fn add(&self, _duty: Duty) -> bool { + true + } + + fn c(&self) -> Option> { + self.0.lock().unwrap().take() } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn store_and_wait() { - let store = super::MemDB::new(); + async fn wait_then_store() { + let deadliner = TestDeadliner::never(); + let store = super::MemDB::new(deadliner); + let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); - let signed_data: Box = Box::new(MockSignedData); + let signed_data: Box = MockSignedData::for_test(0); let reader = { let store = store.clone(); @@ -139,4 +205,59 @@ mod tests { assert!(write.is_ok()); assert_eq!(read, signed_data); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn store_evict_wait_then_store() { + let (evict_tx, evict_rx) = sync::mpsc::channel::(1); + let deadliner = TestDeadliner::new(evict_rx); + + let store = super::MemDB::new(deadliner); + + let duty = Duty::new_attester_duty(SlotNumber::new(1)); + let pub_key = PubKey::new([7u8; 48]); + let first = MockSignedData::for_test(1); + let second = MockSignedData::for_test(2); + + store + .store(duty.clone(), pub_key, first.clone()) + .await + .unwrap(); + + // The eviction task runs concurrently, so we poll until the specific + // data gone, so new readers are guaranteed to not observe it. + evict_tx.send(duty.clone()).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while store + .0 + .data + .lock() + .await + .contains_key(&(duty.clone(), pub_key)) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("eviction was not applied in time"); + + let reader = { + let store = store.clone(); + let duty = duty.clone(); + + tokio::spawn(async move { store.wait_for(duty, pub_key).await }) + }; + + // The eviction has been applied, so wait_for has no entry to return and must + // block. + tokio::task::yield_now().await; + assert!(!reader.is_finished(), "wait_for should block until store"); + + // Store new data for the same duty and pubkey. The reader should wake up and + // return the new data, not the evicted data. + store.store(duty, pub_key, second.clone()).await.unwrap(); + + let read = reader.await.unwrap(); + assert_eq!(read, second); + assert_ne!(read, first); + } } From 453bdcf4303bae02c81ea5dbc5d9fa0f4f7a6c5b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 18:10:56 -0300 Subject: [PATCH 05/34] Port simple test cases --- crates/core/src/aggsigdb/memory.rs | 56 ++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 2e9d4369..f1e31a44 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -260,4 +260,60 @@ mod tests { assert_eq!(read, second); assert_ne!(read, first); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn write_read() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data = MockSignedData::for_test(42); + + store + .store(duty.clone(), pub_key, signed_data.clone()) + .await + .unwrap(); + + let result = store.wait_for(duty, pub_key).await; + assert_eq!(result, signed_data); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cannot_overwrite() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let first = MockSignedData::for_test(1); + let second = MockSignedData::for_test(2); + + store.store(duty.clone(), pub_key, first).await.unwrap(); + + let err = store + .store(duty, pub_key, second) + .await + .expect_err("storing mismatching data should fail"); + assert!(matches!(err, super::Error::MismatchingData)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn write_idempotent() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data = MockSignedData::for_test(42); + + store + .store(duty.clone(), pub_key, signed_data.clone()) + .await + .unwrap(); + store + .store(duty.clone(), pub_key, signed_data.clone()) + .await + .unwrap(); + + let result = store.wait_for(duty, pub_key).await; + assert_eq!(result, signed_data); + } } From be9370f391f439bbe374cf4aa295d5e077fda85b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 18:13:22 -0300 Subject: [PATCH 06/34] Reorder tests and rename - Match Charon --- crates/core/src/aggsigdb/memory.rs | 116 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index f1e31a44..3a6e0ba5 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -177,7 +177,24 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn wait_then_store() { + async fn write_read() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data = MockSignedData::for_test(42); + + store + .store(duty.clone(), pub_key, signed_data.clone()) + .await + .unwrap(); + + let result = store.wait_for(duty, pub_key).await; + assert_eq!(result, signed_data); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn write_unblocks() { let deadliner = TestDeadliner::never(); let store = super::MemDB::new(deadliner); @@ -207,7 +224,46 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn store_evict_wait_then_store() { + async fn cannot_overwrite() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let first = MockSignedData::for_test(1); + let second = MockSignedData::for_test(2); + + store.store(duty.clone(), pub_key, first).await.unwrap(); + + let err = store + .store(duty, pub_key, second) + .await + .expect_err("storing mismatching data should fail"); + assert!(matches!(err, super::Error::MismatchingData)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn write_idempotent() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data = MockSignedData::for_test(42); + + store + .store(duty.clone(), pub_key, signed_data.clone()) + .await + .unwrap(); + store + .store(duty.clone(), pub_key, signed_data.clone()) + .await + .unwrap(); + + let result = store.wait_for(duty, pub_key).await; + assert_eq!(result, signed_data); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn write_evict_wait_then_write() { let (evict_tx, evict_rx) = sync::mpsc::channel::(1); let deadliner = TestDeadliner::new(evict_rx); @@ -260,60 +316,4 @@ mod tests { assert_eq!(read, second); assert_ne!(read, first); } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn write_read() { - let store = super::MemDB::new(TestDeadliner::never()); - - let duty = Duty::new_proposer_duty(SlotNumber::new(10)); - let pub_key = PubKey::new([7u8; 48]); - let signed_data = MockSignedData::for_test(42); - - store - .store(duty.clone(), pub_key, signed_data.clone()) - .await - .unwrap(); - - let result = store.wait_for(duty, pub_key).await; - assert_eq!(result, signed_data); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn cannot_overwrite() { - let store = super::MemDB::new(TestDeadliner::never()); - - let duty = Duty::new_proposer_duty(SlotNumber::new(10)); - let pub_key = PubKey::new([7u8; 48]); - let first = MockSignedData::for_test(1); - let second = MockSignedData::for_test(2); - - store.store(duty.clone(), pub_key, first).await.unwrap(); - - let err = store - .store(duty, pub_key, second) - .await - .expect_err("storing mismatching data should fail"); - assert!(matches!(err, super::Error::MismatchingData)); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn write_idempotent() { - let store = super::MemDB::new(TestDeadliner::never()); - - let duty = Duty::new_proposer_duty(SlotNumber::new(10)); - let pub_key = PubKey::new([7u8; 48]); - let signed_data = MockSignedData::for_test(42); - - store - .store(duty.clone(), pub_key, signed_data.clone()) - .await - .unwrap(); - store - .store(duty.clone(), pub_key, signed_data.clone()) - .await - .unwrap(); - - let result = store.wait_for(duty, pub_key).await; - assert_eq!(result, signed_data); - } } From 6d614914bcd26bae0e7f6892bc85a87390f06adb Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 18:23:07 -0300 Subject: [PATCH 07/34] Add more tests - Rust specific --- crates/core/src/aggsigdb/memory.rs | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 3a6e0ba5..a226f65d 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -316,4 +316,81 @@ mod tests { assert_eq!(read, second); assert_ne!(read, first); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn write_unblocks_many() { + const N: usize = 4; + + let store = super::MemDB::new(TestDeadliner::never()); + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data = MockSignedData::for_test(42); + + let readers: Vec<_> = (0..N) + .map(|_| { + let store = store.clone(); + let duty = duty.clone(); + tokio::spawn(async move { store.wait_for(duty, pub_key).await }) + }) + .collect(); + + // Give readers a chance to reach `notified.await` before the store. + tokio::task::yield_now().await; + for reader in &readers { + assert!( + !reader.is_finished(), + "all readers should block until store" + ); + } + + // A single store unblocks all readers. + store + .store(duty, pub_key, signed_data.clone()) + .await + .unwrap(); + + for reader in readers { + let read = reader.await.unwrap(); + assert_eq!(read, signed_data); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unrelated_write_does_not_unblock() { + let store = super::MemDB::new(TestDeadliner::never()); + + let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); + let data_a = MockSignedData::for_test(1); + + let duty_b = Duty::new_attester_duty(SlotNumber::new(20)); + let data_b = MockSignedData::for_test(2); + + let pub_key = PubKey::new([7u8; 48]); + + let reader = { + let store = store.clone(); + let duty_a = duty_a.clone(); + tokio::spawn(async move { store.wait_for(duty_a, pub_key).await }) + }; + + tokio::task::yield_now().await; + assert!(!reader.is_finished(), "reader should block initially"); + + // Storing an unrelated key wakes readers, which block again since the store is + // unrelated. + store.store(duty_b, pub_key, data_b.clone()).await.unwrap(); + + tokio::task::yield_now().await; + assert!( + !reader.is_finished(), + "reader should re-block after unrelated store" + ); + + // Storing the actual key unblocks the reader. + store.store(duty_a, pub_key, data_a.clone()).await.unwrap(); + + let read = reader.await.unwrap(); + assert_eq!(read, data_a); + assert_ne!(read, data_b); + } } From 6a91cbeb029718c5d309d50bce96e642afa56f56 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 18:23:59 -0300 Subject: [PATCH 08/34] Update lockfile --- Cargo.lock | 434 +++++++++++++++++++++++++++++------------------------ 1 file changed, 241 insertions(+), 193 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 934c8932..9441dcea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -133,7 +133,7 @@ dependencies = [ "k256", "once_cell", "rand 0.8.6", - "secp256k1", + "secp256k1 0.30.0", "serde", "serde_json", "serde_with", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "alloy-core" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e8604b0c092fabc80d075ede181c9b9e596249c70b99253082d7e689836529" +checksum = "62ddde5968de6044d67af107ad835bc0069a7ca245870b94c5958a7d8712b184" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", @@ -192,9 +192,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2db5c583aaef0255aa63a4fe827f826090142528bba48d1bf4119b62780cad" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -203,7 +203,7 @@ dependencies = [ "itoa", "serde", "serde_json", - "winnow 0.7.15", + "winnow 1.0.3", ] [[package]] @@ -246,15 +246,16 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6ae911a2fc304a7cb80a79fb7bed6d1474aed4e7c203df1f8ff538f64fc78d" +checksum = "6b827a6d7784fe3eb3489d40699407a4cdcce74271421a01bdffe60cf573bb16" dependencies = [ "alloy-primitives", "alloy-rlp", "borsh", "once_cell", "serde", + "thiserror 2.0.18", ] [[package]] @@ -297,9 +298,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dbe713da0c737d9e5e387b0ba790eb98b14dd207fe53eef50e19a5a8ec3dac" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -363,9 +364,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3b431b4e72cd8bd0ec7a50b4be18e73dab74de0dba180eef171055e5d5926e" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" dependencies = [ "alloy-rlp", "bytes", @@ -373,7 +374,7 @@ dependencies = [ "const-hex", "derive_more", "foldhash 0.2.0", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itoa", "k256", @@ -384,8 +385,9 @@ dependencies = [ "rapidhash", "ruint", "rustc-hash", + "secp256k1 0.31.1", "serde", - "sha3", + "sha3 0.11.0", ] [[package]] @@ -560,9 +562,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab81bab693da9bb79f7a95b64b394718259fdd7e41dceeced4cad57cb71c4f6a" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -574,9 +576,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489f1620bb7e2483fb5819ed01ab6edc1d2f93939dce35a5695085a1afd1d699" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", @@ -586,16 +588,16 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "sha3", + "sha3 0.11.0", "syn 2.0.117", "syn-solidity", ] [[package]] name = "alloy-sol-macro-input" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56cef806ad22d4392c5fc83cf8f2089f988eb99c7067b4e0c6f1971fc1cca318" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" dependencies = [ "alloy-json-abi", "const-hex", @@ -611,19 +613,19 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6df77fea9d6a2a75c0ef8d2acbdfd92286cc599983d3175ccdc170d3433d249" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" dependencies = [ "serde", - "winnow 0.7.15", + "winnow 1.0.3", ] [[package]] name = "alloy-sol-types" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64612d29379782a5dde6f4b6570d9c756d734d760c0c94c254d361e678a6591f" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -749,7 +751,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -760,7 +762,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -972,9 +974,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -1154,9 +1156,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -1164,9 +1166,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -1355,6 +1357,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blst" version = "0.3.16" @@ -1587,9 +1598,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.61" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -1691,7 +1702,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", "zeroize", ] @@ -2024,6 +2035,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctor" version = "0.2.9" @@ -2153,9 +2173,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -2188,7 +2208,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -2293,12 +2313,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -2312,9 +2342,9 @@ dependencies = [ [[package]] name = "docker_credential" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4564c274ebf369f501de192b02a0b81a5c4bda375abfe526aa70fc702fa6fa0" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ "base64", "serde", @@ -2398,9 +2428,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -2489,7 +2519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2528,9 +2558,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a4a4e4273b0135111fe9464e35465067766a8f664615b5a86338b73864407" +checksum = "e462875ad8693755ea8913d6e905715c76ea4836e2254e18c9cf0f7a8f8c2a13" dependencies = [ "alloy-primitives", "ethereum_serde_utils", @@ -2543,9 +2573,9 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cd82c68120c89361e1a457245cf212f7d9f541bffaffed530c8f2d54a160b2" +checksum = "daf022360bdbe9456eda5f35718a50476d5b2a0d51a97ed4eae27420737a6fba" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -2642,13 +2672,12 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" dependencies = [ "cfg-if", "libc", - "libredox", ] [[package]] @@ -2843,12 +2872,12 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" dependencies = [ - "gloo-timers 0.2.6", - "send_wrapper 0.4.0", + "gloo-timers 0.4.0", + "send_wrapper", ] [[package]] @@ -2968,9 +2997,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "gloo-timers" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" dependencies = [ "futures-channel", "futures-core", @@ -2980,9 +3009,9 @@ dependencies = [ [[package]] name = "gloo-timers" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +checksum = "482ce8a491a501da4cd806bd190275363d674f2845005c6ddbd5d3e1dd54495d" dependencies = [ "futures-channel", "futures-core", @@ -3003,9 +3032,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -3064,15 +3093,18 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", - "serde", - "serde_core", ] [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", + "serde", + "serde_core", +] [[package]] name = "hashlink" @@ -3265,6 +3297,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -3363,7 +3404,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -3617,7 +3658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -3650,16 +3691,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -3760,9 +3791,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.97" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -3794,11 +3825,21 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + [[package]] name = "keccak-asm" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa468878266ad91431012b3e5ef1bf9b170eab22883503a318d46857afa4579a" +checksum = "1766b89733097006f3a1388a02849865d6bc98c89273cb622e29fdd209922183" dependencies = [ "digest 0.10.7", "sha3-asm", @@ -3839,9 +3880,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.4+1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "9b26f66f35e1871b22efcf7191564123d2a446ca0538cde63c23adfefa9b15b7" dependencies = [ "cc", "libc", @@ -4259,7 +4300,7 @@ dependencies = [ "pin-project", "rand 0.8.6", "salsa20", - "sha3", + "sha3 0.10.9", "tracing", ] @@ -4485,7 +4526,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-webrtc-utils", - "send_wrapper 0.6.0", + "send_wrapper", "thiserror 2.0.18", "tracing", "wasm-bindgen", @@ -4524,7 +4565,7 @@ dependencies = [ "futures", "js-sys", "libp2p-core", - "send_wrapper 0.6.0", + "send_wrapper", "thiserror 2.0.18", "tracing", "wasm-bindgen", @@ -4544,7 +4585,7 @@ dependencies = [ "libp2p-noise", "multiaddr", "multihash", - "send_wrapper 0.6.0", + "send_wrapper", "thiserror 2.0.18", "tracing", "wasm-bindgen", @@ -4567,18 +4608,6 @@ dependencies = [ "yamux 0.13.10", ] -[[package]] -name = "libredox" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" -dependencies = [ - "bitflags", - "libc", - "plain", - "redox_syscall 0.7.4", -] - [[package]] name = "libz-sys" version = "1.1.28" @@ -4645,9 +4674,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "macro-string" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" dependencies = [ "proc-macro2", "quote", @@ -4925,7 +4954,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4963,9 +4992,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -5088,7 +5117,7 @@ dependencies = [ "oas3", "prettyplease", "proc-macro2", - "quick-xml", + "quick-xml 0.40.1", "quote", "regex", "reqwest 0.13.3", @@ -5141,9 +5170,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.79" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags", "cfg-if", @@ -5172,9 +5201,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.115" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -5256,7 +5285,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -5361,18 +5390,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -5407,12 +5436,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - [[package]] name = "plotters" version = "0.3.7" @@ -5509,7 +5532,7 @@ dependencies = [ "pluto-ssz", "pluto-testutil", "pluto-tracing", - "quick-xml", + "quick-xml 0.39.4", "rand 0.8.6", "reqwest 0.13.3", "serde", @@ -5727,7 +5750,7 @@ dependencies = [ "serde_json", "serde_with", "sha2", - "sha3", + "sha3 0.10.9", "tempfile", "test-case", "thiserror 2.0.18", @@ -6222,9 +6245,19 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", "serde", @@ -6244,7 +6277,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -6282,7 +6315,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -6494,15 +6527,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" -dependencies = [ - "bitflags", -] - [[package]] name = "ref-cast" version = "1.0.25" @@ -6774,7 +6798,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6833,7 +6857,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6981,10 +7005,21 @@ checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ "bitcoin_hashes", "rand 0.8.6", - "secp256k1-sys", + "secp256k1-sys 0.10.1", "serde", ] +[[package]] +name = "secp256k1" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" +dependencies = [ + "bitcoin_hashes", + "rand 0.9.4", + "secp256k1-sys 0.11.0", +] + [[package]] name = "secp256k1-sys" version = "0.10.1" @@ -6994,6 +7029,15 @@ dependencies = [ "cc", ] +[[package]] +name = "secp256k1-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" +dependencies = [ + "cc", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -7045,12 +7089,6 @@ dependencies = [ "pest", ] -[[package]] -name = "send_wrapper" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" - [[package]] name = "send_wrapper" version = "0.6.0" @@ -7149,11 +7187,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.19.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ "base64", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -7168,9 +7207,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.19.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -7230,14 +7269,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" dependencies = [ "digest 0.10.7", - "keccak", + "keccak 0.1.6", +] + +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.0", ] [[package]] name = "sha3-asm" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59cbb88c189d6352cc8ae96a39d19c7ecad8f7330b29461187f2587fdc2988d5" +checksum = "9f3f15d4e239ebe08413eed880e0f9b5af4b40ee0472543320efa91d488e96a7" dependencies = [ "cc", "cfg-if", @@ -7355,7 +7404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7475,9 +7524,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f425ae0b12e2f5ae65542e00898d500d4d318b4baf09f40fd0d410454e9947" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ "paste", "proc-macro2", @@ -7554,9 +7603,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -7573,7 +7622,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7766,9 +7815,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -7890,7 +7939,7 @@ dependencies = [ "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.2", + "winnow 1.0.3", ] [[package]] @@ -7899,7 +7948,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.2", + "winnow 1.0.3", ] [[package]] @@ -7910,9 +7959,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", @@ -7939,9 +7988,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost 0.14.3", @@ -7969,20 +8018,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", "futures-util", "http", "http-body", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "url", ] [[package]] @@ -8043,9 +8092,9 @@ dependencies = [ [[package]] name = "tracing-loki" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34" +checksum = "a8d1ad78bf74c1790b0825ddc35ad1bb9498736c51c8437796b81cadf4916cd8" dependencies = [ "loki-api", "reqwest 0.12.28", @@ -8053,7 +8102,6 @@ dependencies = [ "serde_json", "snap", "tokio", - "tokio-stream", "tracing", "tracing-core", "tracing-log", @@ -8202,7 +8250,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -8449,9 +8497,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -8462,9 +8510,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.70" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -8472,9 +8520,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8482,9 +8530,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -8495,9 +8543,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -8565,9 +8613,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.97" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -8638,7 +8686,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8988,9 +9036,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -9266,9 +9314,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] From 1ecfe68060e2d3c7e2aa38a694ae9cbb916dfc42 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 18:44:44 -0300 Subject: [PATCH 09/34] Apply clippy suggestions --- crates/core/src/aggsigdb/memory.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index a226f65d..8c3298f1 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -205,7 +205,6 @@ mod tests { let reader = { let store = store.clone(); let duty = duty.clone(); - let pub_key = pub_key.clone(); tokio::spawn(async move { store.wait_for(duty, pub_key).await }) }; From 98993babdf1b490140a8be97439bb9f8b5e437f1 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 21:02:56 -0300 Subject: [PATCH 10/34] Use `SignedDataSet` as input instead of single `SignedData` - Matches Charon API --- crates/core/src/aggsigdb/memory.rs | 125 ++++++++++++++++------------- 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 8c3298f1..c0e6ae3c 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -61,30 +61,36 @@ impl MemDB { } /// Stores aggregated signed duty data set. - pub async fn store( - &self, - duty: types::Duty, - pub_key: types::PubKey, - signed_data: Box, - ) -> Result<(), Error> { + pub async fn store(&self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { // TODO(charon): Distinguish between no deadline supported vs already expired. let _ = self.0.deadliner.add(duty.clone()).await; let mut data = self.0.data.lock().await; - match data.entry((duty, pub_key)) { - Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { - Err(Error::MismatchingData) - } - Entry::Occupied(_) => Ok(()), - Entry::Vacant(slot) => { - slot.insert(signed_data); - // TODO: Optimize to only wake those who are waiting for this specific duty and - // pubkey - self.0.notify.notify_waiters(); - Ok(()) - } - } + let result = set + .into_iter() + .map(|(pub_key, signed_data)| { + let key = (duty.clone(), pub_key); + + match data.entry(key) { + Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { + Err(Error::MismatchingData) + } + Entry::Occupied(_) => Ok(()), + Entry::Vacant(slot) => { + slot.insert(signed_data); + Ok(()) + } + } + }) + .collect::, _>>(); + + // TODO: Optimize wake to only occur if new data was actually inserted, + // and to only wake those who are waiting for the specific duty and pubkey, + // rather than all waiters. + self.0.notify.notify_waiters(); + + result.map(|_| ()) } /// Blocks and returns the aggregated signed duty data when available. @@ -119,7 +125,7 @@ mod tests { use crate::{ deadline::Deadliner, signeddata::SignedDataError, - types::{Duty, PubKey, Signature, SignedData, SlotNumber}, + types::{Duty, PubKey, Signature, SignedData, SignedDataSet, SlotNumber}, }; use async_trait::async_trait; use std::sync::Arc; @@ -129,12 +135,6 @@ mod tests { #[derive(Debug, Clone, PartialEq, Eq)] struct MockSignedData(u8); - impl MockSignedData { - fn for_test(value: u8) -> Box { - Box::new(Self(value)) - } - } - impl SignedData for MockSignedData { fn signature(&self) -> Result { Ok(Signature::new([self.0; 96])) @@ -149,6 +149,18 @@ mod tests { } } + impl MockSignedData { + fn to_set(&self, pub_key: PubKey) -> SignedDataSet { + let mut set = SignedDataSet::new(); + set.insert(pub_key, self.clone()); + set + } + + fn boxed(&self) -> Box { + Box::new(self.clone()) + } + } + /// Deadliner that hands out a caller-supplied receiver, allowing tests to /// drive eviction by sending on the paired sender. struct TestDeadliner(std::sync::Mutex>>); @@ -182,15 +194,15 @@ mod tests { let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); - let signed_data = MockSignedData::for_test(42); + let signed_data = MockSignedData(42); store - .store(duty.clone(), pub_key, signed_data.clone()) + .store(duty.clone(), signed_data.to_set(pub_key)) .await .unwrap(); let result = store.wait_for(duty, pub_key).await; - assert_eq!(result, signed_data); + assert_eq!(result, signed_data.boxed()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -200,7 +212,7 @@ mod tests { let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); - let signed_data: Box = MockSignedData::for_test(0); + let signed_data = MockSignedData(0); let reader = { let store = store.clone(); @@ -215,11 +227,11 @@ mod tests { tokio::task::yield_now().await; assert!(!reader.is_finished(), "wait_for should block until store"); - let write = store.store(duty, pub_key, signed_data.clone()).await; + let write = store.store(duty, signed_data.to_set(pub_key)).await; let read = reader.await.unwrap(); assert!(write.is_ok()); - assert_eq!(read, signed_data); + assert_eq!(read, signed_data.boxed()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -228,13 +240,16 @@ mod tests { let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); - let first = MockSignedData::for_test(1); - let second = MockSignedData::for_test(2); + let first = MockSignedData(1); + let second = MockSignedData(2); - store.store(duty.clone(), pub_key, first).await.unwrap(); + store + .store(duty.clone(), first.to_set(pub_key)) + .await + .unwrap(); let err = store - .store(duty, pub_key, second) + .store(duty, second.to_set(pub_key)) .await .expect_err("storing mismatching data should fail"); assert!(matches!(err, super::Error::MismatchingData)); @@ -246,19 +261,19 @@ mod tests { let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); - let signed_data = MockSignedData::for_test(42); + let signed_data = MockSignedData(42); store - .store(duty.clone(), pub_key, signed_data.clone()) + .store(duty.clone(), signed_data.to_set(pub_key)) .await .unwrap(); store - .store(duty.clone(), pub_key, signed_data.clone()) + .store(duty.clone(), signed_data.to_set(pub_key)) .await .unwrap(); let result = store.wait_for(duty, pub_key).await; - assert_eq!(result, signed_data); + assert_eq!(result, signed_data.boxed()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -270,11 +285,11 @@ mod tests { let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); - let first = MockSignedData::for_test(1); - let second = MockSignedData::for_test(2); + let first = MockSignedData(1); + let second = MockSignedData(2); store - .store(duty.clone(), pub_key, first.clone()) + .store(duty.clone(), first.to_set(pub_key)) .await .unwrap(); @@ -309,11 +324,11 @@ mod tests { // Store new data for the same duty and pubkey. The reader should wake up and // return the new data, not the evicted data. - store.store(duty, pub_key, second.clone()).await.unwrap(); + store.store(duty, second.to_set(pub_key)).await.unwrap(); let read = reader.await.unwrap(); - assert_eq!(read, second); - assert_ne!(read, first); + assert_eq!(read, second.boxed()); + assert_ne!(read, first.boxed()); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -323,7 +338,7 @@ mod tests { let store = super::MemDB::new(TestDeadliner::never()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); - let signed_data = MockSignedData::for_test(42); + let signed_data = MockSignedData(42); let readers: Vec<_> = (0..N) .map(|_| { @@ -344,13 +359,13 @@ mod tests { // A single store unblocks all readers. store - .store(duty, pub_key, signed_data.clone()) + .store(duty, signed_data.to_set(pub_key)) .await .unwrap(); for reader in readers { let read = reader.await.unwrap(); - assert_eq!(read, signed_data); + assert_eq!(read, signed_data.boxed()); } } @@ -359,10 +374,10 @@ mod tests { let store = super::MemDB::new(TestDeadliner::never()); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); - let data_a = MockSignedData::for_test(1); + let data_a = MockSignedData(1); let duty_b = Duty::new_attester_duty(SlotNumber::new(20)); - let data_b = MockSignedData::for_test(2); + let data_b = MockSignedData(2); let pub_key = PubKey::new([7u8; 48]); @@ -377,7 +392,7 @@ mod tests { // Storing an unrelated key wakes readers, which block again since the store is // unrelated. - store.store(duty_b, pub_key, data_b.clone()).await.unwrap(); + store.store(duty_b, data_b.to_set(pub_key)).await.unwrap(); tokio::task::yield_now().await; assert!( @@ -386,10 +401,10 @@ mod tests { ); // Storing the actual key unblocks the reader. - store.store(duty_a, pub_key, data_a.clone()).await.unwrap(); + store.store(duty_a, data_a.to_set(pub_key)).await.unwrap(); let read = reader.await.unwrap(); - assert_eq!(read, data_a); - assert_ne!(read, data_b); + assert_eq!(read, data_a.boxed()); + assert_ne!(read, data_b.boxed()); } } From 33c15ac12c1f9ab247fad5c2ae129c4251e0a415 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 20 May 2026 21:12:32 -0300 Subject: [PATCH 11/34] Refactor - Use `try_for_each` - Proper naming convention --- crates/core/src/aggsigdb/memory.rs | 62 ++++++++++++++++-------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index c0e6ae3c..f361463f 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -67,30 +67,28 @@ impl MemDB { let mut data = self.0.data.lock().await; - let result = set - .into_iter() - .map(|(pub_key, signed_data)| { - let key = (duty.clone(), pub_key); - - match data.entry(key) { - Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { - Err(Error::MismatchingData) - } - Entry::Occupied(_) => Ok(()), - Entry::Vacant(slot) => { - slot.insert(signed_data); - Ok(()) - } + // NOTE: Partial insertions on error match the semantics of Charon. + let result = set.into_iter().try_for_each(|(pub_key, signed_data)| { + let key = (duty.clone(), pub_key); + + match data.entry(key) { + Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { + Err(Error::MismatchingData) } - }) - .collect::, _>>(); + Entry::Occupied(_) => Ok(()), + Entry::Vacant(slot) => { + slot.insert(signed_data); + Ok(()) + } + } + }); // TODO: Optimize wake to only occur if new data was actually inserted, // and to only wake those who are waiting for the specific duty and pubkey, // rather than all waiters. self.0.notify.notify_waiters(); - result.map(|_| ()) + result } /// Blocks and returns the aggregated signed duty data when available. @@ -150,7 +148,7 @@ mod tests { } impl MockSignedData { - fn to_set(&self, pub_key: PubKey) -> SignedDataSet { + fn singleton(&self, pub_key: PubKey) -> SignedDataSet { let mut set = SignedDataSet::new(); set.insert(pub_key, self.clone()); set @@ -197,7 +195,7 @@ mod tests { let signed_data = MockSignedData(42); store - .store(duty.clone(), signed_data.to_set(pub_key)) + .store(duty.clone(), signed_data.singleton(pub_key)) .await .unwrap(); @@ -227,7 +225,7 @@ mod tests { tokio::task::yield_now().await; assert!(!reader.is_finished(), "wait_for should block until store"); - let write = store.store(duty, signed_data.to_set(pub_key)).await; + let write = store.store(duty, signed_data.singleton(pub_key)).await; let read = reader.await.unwrap(); assert!(write.is_ok()); @@ -244,12 +242,12 @@ mod tests { let second = MockSignedData(2); store - .store(duty.clone(), first.to_set(pub_key)) + .store(duty.clone(), first.singleton(pub_key)) .await .unwrap(); let err = store - .store(duty, second.to_set(pub_key)) + .store(duty, second.singleton(pub_key)) .await .expect_err("storing mismatching data should fail"); assert!(matches!(err, super::Error::MismatchingData)); @@ -264,11 +262,11 @@ mod tests { let signed_data = MockSignedData(42); store - .store(duty.clone(), signed_data.to_set(pub_key)) + .store(duty.clone(), signed_data.singleton(pub_key)) .await .unwrap(); store - .store(duty.clone(), signed_data.to_set(pub_key)) + .store(duty.clone(), signed_data.singleton(pub_key)) .await .unwrap(); @@ -289,7 +287,7 @@ mod tests { let second = MockSignedData(2); store - .store(duty.clone(), first.to_set(pub_key)) + .store(duty.clone(), first.singleton(pub_key)) .await .unwrap(); @@ -324,7 +322,7 @@ mod tests { // Store new data for the same duty and pubkey. The reader should wake up and // return the new data, not the evicted data. - store.store(duty, second.to_set(pub_key)).await.unwrap(); + store.store(duty, second.singleton(pub_key)).await.unwrap(); let read = reader.await.unwrap(); assert_eq!(read, second.boxed()); @@ -359,7 +357,7 @@ mod tests { // A single store unblocks all readers. store - .store(duty, signed_data.to_set(pub_key)) + .store(duty, signed_data.singleton(pub_key)) .await .unwrap(); @@ -392,7 +390,10 @@ mod tests { // Storing an unrelated key wakes readers, which block again since the store is // unrelated. - store.store(duty_b, data_b.to_set(pub_key)).await.unwrap(); + store + .store(duty_b, data_b.singleton(pub_key)) + .await + .unwrap(); tokio::task::yield_now().await; assert!( @@ -401,7 +402,10 @@ mod tests { ); // Storing the actual key unblocks the reader. - store.store(duty_a, data_a.to_set(pub_key)).await.unwrap(); + store + .store(duty_a, data_a.singleton(pub_key)) + .await + .unwrap(); let read = reader.await.unwrap(); assert_eq!(read, data_a.boxed()); From b341668fad81921a0268583432afa563e01e1618 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 21 May 2026 02:08:55 -0300 Subject: [PATCH 12/34] Apply Claude suggestions - Annotate `wait_for` abort mechanism - Log when deadliner does not have a channel --- crates/core/src/aggsigdb/memory.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index f361463f..1e6af9f5 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -41,7 +41,7 @@ impl MemDB { tokio::spawn(Self::evict(Arc::downgrade(&this.0), evictions)); } None => { - // TODO: In Charon, `deadliner.c()` always returns `Some` + tracing::warn!("Deadliner channel is not available"); } } @@ -62,12 +62,13 @@ impl MemDB { /// Stores aggregated signed duty data set. pub async fn store(&self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { + let mut data = self.0.data.lock().await; + // TODO(charon): Distinguish between no deadline supported vs already expired. let _ = self.0.deadliner.add(duty.clone()).await; - let mut data = self.0.data.lock().await; - // NOTE: Partial insertions on error match the semantics of Charon. + let mut inserted = false; let result = set.into_iter().try_for_each(|(pub_key, signed_data)| { let key = (duty.clone(), pub_key); @@ -78,20 +79,25 @@ impl MemDB { Entry::Occupied(_) => Ok(()), Entry::Vacant(slot) => { slot.insert(signed_data); + inserted = true; Ok(()) } } }); - // TODO: Optimize wake to only occur if new data was actually inserted, - // and to only wake those who are waiting for the specific duty and pubkey, - // rather than all waiters. - self.0.notify.notify_waiters(); + if inserted { + // TODO: Optimize to only wake those who are waiting for the specific duty and + // pubkey, rather than all waiters. + self.0.notify.notify_waiters(); + } result } /// Blocks and returns the aggregated signed duty data when available. + /// + /// Might block indefinitely if no data is ever stored for the given duty + /// and public key. pub async fn wait_for( &self, duty: types::Duty, From 1161a602122ce064f551c5e1088e19063460bef8 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 21 May 2026 14:08:06 -0300 Subject: [PATCH 13/34] Refactor to nested maps - Faster evictions --- crates/core/src/aggsigdb/memory.rs | 58 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 1e6af9f5..3e6bb807 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -21,8 +21,10 @@ pub enum Error { #[derive(Clone)] pub struct MemDB(Arc); +type SignedDataByPubKey = HashMap>; + struct MemDBInner { - data: Mutex>>, + data: Mutex>, deadliner: Arc, notify: Notify, } @@ -56,7 +58,7 @@ impl MemDB { let Some(inner) = inner.upgrade() else { return; }; - inner.data.lock().await.retain(|(d, _), _| d != &duty); + inner.data.lock().await.remove(&duty); } } @@ -69,21 +71,20 @@ impl MemDB { // NOTE: Partial insertions on error match the semantics of Charon. let mut inserted = false; - let result = set.into_iter().try_for_each(|(pub_key, signed_data)| { - let key = (duty.clone(), pub_key); - - match data.entry(key) { - Entry::Occupied(slot) if slot.get().as_ref() != signed_data.as_ref() => { - Err(Error::MismatchingData) - } - Entry::Occupied(_) => Ok(()), - Entry::Vacant(slot) => { - slot.insert(signed_data); - inserted = true; - Ok(()) - } - } - }); + let for_duty = data.entry(duty).or_default(); + let result = + set.into_iter() + .try_for_each(|(pub_key, signed_data)| match for_duty.entry(pub_key) { + Entry::Vacant(slot) => { + slot.insert(signed_data); + inserted = true; + Ok(()) + } + Entry::Occupied(slot) if slot.get() != &signed_data => { + Err(Error::MismatchingData) + } + Entry::Occupied(_) => Ok(()), + }); if inserted { // TODO: Optimize to only wake those who are waiting for the specific duty and @@ -103,7 +104,6 @@ impl MemDB { duty: types::Duty, pub_key: types::PubKey, ) -> Box { - let k = (duty, pub_key); loop { // Register interest before checking the map so that a concurrent `store` either // (a) inserts before we check and we observe the value, or (b) inserts after @@ -113,8 +113,7 @@ impl MemDB { notified.as_mut().enable(); { - let data = self.0.data.lock().await; - if let Some(data) = data.get(&k) { + if let Some(data) = self.get(&duty, &pub_key).await { return data.clone(); } } @@ -122,6 +121,17 @@ impl MemDB { notified.await; } } + + /// Immediately returns the aggregated signed duty data if available, + /// without blocking. + pub async fn get( + &self, + duty: &types::Duty, + pub_key: &types::PubKey, + ) -> Option> { + let data = self.0.data.lock().await; + data.get(duty).and_then(|inner| inner.get(pub_key)).cloned() + } } #[cfg(test)] @@ -301,13 +311,7 @@ mod tests { // data gone, so new readers are guaranteed to not observe it. evict_tx.send(duty.clone()).await.unwrap(); tokio::time::timeout(std::time::Duration::from_secs(2), async { - while store - .0 - .data - .lock() - .await - .contains_key(&(duty.clone(), pub_key)) - { + while store.get(&duty, &pub_key).await.is_some() { tokio::task::yield_now().await; } }) From 2111bb2bcdcba545f2b236dbb61dc1d275b96495 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 22 May 2026 12:30:51 -0300 Subject: [PATCH 14/34] Scope lock --- crates/core/src/aggsigdb/memory.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 3e6bb807..350dcbff 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -64,29 +64,31 @@ impl MemDB { /// Stores aggregated signed duty data set. pub async fn store(&self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { - let mut data = self.0.data.lock().await; + let mut should_notify = false; - // TODO(charon): Distinguish between no deadline supported vs already expired. - let _ = self.0.deadliner.add(duty.clone()).await; + let result = { + let mut data = self.0.data.lock().await; + // TODO(charon): Distinguish between no deadline supported vs already expired. + let _ = self.0.deadliner.add(duty.clone()).await; + + // NOTE: Partial insertions on error match the semantics of Charon. + let for_duty = data.entry(duty).or_default(); - // NOTE: Partial insertions on error match the semantics of Charon. - let mut inserted = false; - let for_duty = data.entry(duty).or_default(); - let result = set.into_iter() .try_for_each(|(pub_key, signed_data)| match for_duty.entry(pub_key) { Entry::Vacant(slot) => { slot.insert(signed_data); - inserted = true; + should_notify = true; Ok(()) } Entry::Occupied(slot) if slot.get() != &signed_data => { Err(Error::MismatchingData) } Entry::Occupied(_) => Ok(()), - }); + }) + }; - if inserted { + if should_notify { // TODO: Optimize to only wake those who are waiting for the specific duty and // pubkey, rather than all waiters. self.0.notify.notify_waiters(); From c7c4448679de930f29e8e4811f6f63d09892197b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 22 May 2026 12:32:49 -0300 Subject: [PATCH 15/34] Use `RwLock` --- crates/core/src/aggsigdb/memory.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 350dcbff..60d4d2b2 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -3,7 +3,7 @@ use std::{ collections::{HashMap, hash_map::Entry}, sync::Arc, }; -use tokio::sync::{Mutex, Notify}; +use tokio::sync::{Notify, RwLock}; /// Errors for the in-memory AggSigDB implementation. #[derive(Debug, thiserror::Error)] @@ -24,7 +24,7 @@ pub struct MemDB(Arc); type SignedDataByPubKey = HashMap>; struct MemDBInner { - data: Mutex>, + data: RwLock>, deadliner: Arc, notify: Notify, } @@ -33,7 +33,7 @@ impl MemDB { /// Creates a new in-memory AggSigDB instance. pub fn new(deadliner: Arc) -> Self { let this = Self(Arc::new(MemDBInner { - data: Mutex::new(HashMap::new()), + data: RwLock::new(HashMap::new()), deadliner: Arc::clone(&deadliner), notify: Notify::new(), })); @@ -58,7 +58,7 @@ impl MemDB { let Some(inner) = inner.upgrade() else { return; }; - inner.data.lock().await.remove(&duty); + inner.data.write().await.remove(&duty); } } @@ -67,7 +67,7 @@ impl MemDB { let mut should_notify = false; let result = { - let mut data = self.0.data.lock().await; + let mut data = self.0.data.write().await; // TODO(charon): Distinguish between no deadline supported vs already expired. let _ = self.0.deadliner.add(duty.clone()).await; @@ -131,7 +131,7 @@ impl MemDB { duty: &types::Duty, pub_key: &types::PubKey, ) -> Option> { - let data = self.0.data.lock().await; + let data = self.0.data.read().await; data.get(duty).and_then(|inner| inner.get(pub_key)).cloned() } } From db70143fb95a77be21633cf72e5ee63d329bcd80 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Sun, 24 May 2026 02:50:57 -0300 Subject: [PATCH 16/34] Rewrite to actor model - Avoid `notify` and unnecessary awakes - Removes mutexes --- crates/core/src/aggsigdb/memory.rs | 262 +++++++++++++++++------------ 1 file changed, 152 insertions(+), 110 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 60d4d2b2..963779b8 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -3,7 +3,7 @@ use std::{ collections::{HashMap, hash_map::Entry}, sync::Arc, }; -use tokio::sync::{Notify, RwLock}; +use tokio::sync; /// Errors for the in-memory AggSigDB implementation. #[derive(Debug, thiserror::Error)] @@ -14,87 +14,154 @@ pub enum Error { MismatchingData, } -/// An in-memory implementation of the AggSigDB. +struct Actor { + entries: HashMap>>, + waiters: HashMap< + (types::Duty, types::PubKey), + Vec>>, + >, + deadliner: Arc, +} + +impl Actor { + async fn run(&mut self, mut messages: sync::mpsc::Receiver) { + while let Some(msg) = messages.recv().await { + match msg { + Message::Store { + duty, + set, + response, + } => { + let result = self.store(duty, set).await; + let _ = response.send(result); + } + Message::WaitFor { + duty, + pub_key, + response, + } => { + if let Some(found) = self.get(&duty, &pub_key) { + let _ = response.send(found); + } else { + self.waiters + .entry((duty, pub_key)) + .or_default() + .push(response); + } + } + Message::Evict { duty } => { + let _ = self.evict(duty); + } + } + } + } + + async fn store(&mut self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { + // TODO: Improve the `deadliner` API: + // - Return if the duty is already expired. If so, return early. + // - Make `add` sync to avoid an `await` which blocks the actor. + let _ = self.deadliner.add(duty.clone()).await; + + // NOTE: Partial insertions on error match the semantics of Charon. + let for_duty = self.entries.entry(duty.clone()).or_default(); + for (pub_key, signed_data) in set.into_iter() { + match for_duty.entry(pub_key) { + Entry::Vacant(slot) => { + slot.insert(signed_data.clone()); + + let k = (duty.clone(), pub_key); + if let Some((_, waiters)) = self.waiters.remove_entry(&k) { + for w in waiters { + let _ = w.send(signed_data.clone()); + } + }; + } + Entry::Occupied(slot) if slot.get() != &signed_data => { + return Err(Error::MismatchingData); + } + Entry::Occupied(_) => {} + } + } + + Ok(()) + } + + fn get( + &self, + duty: &types::Duty, + pub_key: &types::PubKey, + ) -> Option> { + self.entries + .get(duty) + .and_then(|for_duty| for_duty.get(pub_key)) + .cloned() + } + + fn evict(&mut self, duty: types::Duty) { + self.entries.remove(&duty); + } +} + +enum Message { + Evict { + duty: types::Duty, + }, + Store { + duty: types::Duty, + set: types::SignedDataSet, + response: sync::oneshot::Sender>, + }, + WaitFor { + duty: types::Duty, + pub_key: types::PubKey, + response: sync::oneshot::Sender>, + }, +} + +/// An in-memory implementation of AggSigDB. /// /// Share an instance by cloning. Cloning is cheap and creates a new reference /// to the same underlying data. #[derive(Clone)] -pub struct MemDB(Arc); - -type SignedDataByPubKey = HashMap>; - -struct MemDBInner { - data: RwLock>, - deadliner: Arc, - notify: Notify, +pub struct Handle { + sender: sync::mpsc::Sender, } -impl MemDB { - /// Creates a new in-memory AggSigDB instance. +impl Handle { + /// Creates a new in-memory AggSigDB instance, and get a handle to it. + /// + /// The underlying instance gets dropped when all handles are dropped. pub fn new(deadliner: Arc) -> Self { - let this = Self(Arc::new(MemDBInner { - data: RwLock::new(HashMap::new()), + let (sender, receiver) = sync::mpsc::channel(100); + let mut actor = Actor { + entries: HashMap::new(), + waiters: HashMap::new(), deadliner: Arc::clone(&deadliner), - notify: Notify::new(), - })); + }; + tokio::spawn(async move { actor.run(receiver).await }); - match deadliner.c() { - Some(evictions) => { - tokio::spawn(Self::evict(Arc::downgrade(&this.0), evictions)); - } - None => { - tracing::warn!("Deadliner channel is not available"); + let deadliner_sender = sender.clone(); + tokio::spawn(async move { + if let Some(mut c) = deadliner.c() { + while let Some(duty) = c.recv().await { + let _ = deadliner_sender.send(Message::Evict { duty }).await; + } } - } - - this - } + }); - async fn evict( - inner: std::sync::Weak, - mut evictions: tokio::sync::mpsc::Receiver, - ) { - while let Some(duty) = evictions.recv().await { - let Some(inner) = inner.upgrade() else { - return; - }; - inner.data.write().await.remove(&duty); - } + Self { sender } } /// Stores aggregated signed duty data set. pub async fn store(&self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { - let mut should_notify = false; - - let result = { - let mut data = self.0.data.write().await; - // TODO(charon): Distinguish between no deadline supported vs already expired. - let _ = self.0.deadliner.add(duty.clone()).await; - - // NOTE: Partial insertions on error match the semantics of Charon. - let for_duty = data.entry(duty).or_default(); - - set.into_iter() - .try_for_each(|(pub_key, signed_data)| match for_duty.entry(pub_key) { - Entry::Vacant(slot) => { - slot.insert(signed_data); - should_notify = true; - Ok(()) - } - Entry::Occupied(slot) if slot.get() != &signed_data => { - Err(Error::MismatchingData) - } - Entry::Occupied(_) => Ok(()), - }) + let (response_tx, response_rx) = sync::oneshot::channel(); + let msg = Message::Store { + duty, + set, + response: response_tx, }; - - if should_notify { - // TODO: Optimize to only wake those who are waiting for the specific duty and - // pubkey, rather than all waiters. - self.0.notify.notify_waiters(); - } - - result + let _ = self.sender.send(msg).await; + response_rx.await.unwrap() } /// Blocks and returns the aggregated signed duty data when available. @@ -106,33 +173,14 @@ impl MemDB { duty: types::Duty, pub_key: types::PubKey, ) -> Box { - loop { - // Register interest before checking the map so that a concurrent `store` either - // (a) inserts before we check and we observe the value, or (b) inserts after - // our `notified()` is enabled and wakes us. - let notified = self.0.notify.notified(); - tokio::pin!(notified); - notified.as_mut().enable(); - - { - if let Some(data) = self.get(&duty, &pub_key).await { - return data.clone(); - } - } - - notified.await; - } - } - - /// Immediately returns the aggregated signed duty data if available, - /// without blocking. - pub async fn get( - &self, - duty: &types::Duty, - pub_key: &types::PubKey, - ) -> Option> { - let data = self.0.data.read().await; - data.get(duty).and_then(|inner| inner.get(pub_key)).cloned() + let (response_tx, response_rx) = sync::oneshot::channel(); + let msg = Message::WaitFor { + duty, + pub_key, + response: response_tx, + }; + let _ = self.sender.send(msg).await; + response_rx.await.unwrap() } } @@ -206,7 +254,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_read() { - let store = super::MemDB::new(TestDeadliner::never()); + let store = super::Handle::new(TestDeadliner::never()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -224,7 +272,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_unblocks() { let deadliner = TestDeadliner::never(); - let store = super::MemDB::new(deadliner); + let store = super::Handle::new(deadliner); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -252,7 +300,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cannot_overwrite() { - let store = super::MemDB::new(TestDeadliner::never()); + let store = super::Handle::new(TestDeadliner::never()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -273,7 +321,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_idempotent() { - let store = super::MemDB::new(TestDeadliner::never()); + let store = super::Handle::new(TestDeadliner::never()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -297,7 +345,7 @@ mod tests { let (evict_tx, evict_rx) = sync::mpsc::channel::(1); let deadliner = TestDeadliner::new(evict_rx); - let store = super::MemDB::new(deadliner); + let store = super::Handle::new(deadliner); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -309,17 +357,11 @@ mod tests { .await .unwrap(); - // The eviction task runs concurrently, so we poll until the specific - // data gone, so new readers are guaranteed to not observe it. + // The eviction task runs concurrently, so we wait until the specific + // data is gone, so new readers are guaranteed to not observe it. evict_tx.send(duty.clone()).await.unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while store.get(&duty, &pub_key).await.is_some() { - tokio::task::yield_now().await; - } - }) - .await - .expect("eviction was not applied in time"); - + // TODO: Find a better mechanism to wait for eviction + tokio::time::sleep(std::time::Duration::from_millis(200)).await; let reader = { let store = store.clone(); let duty = duty.clone(); @@ -345,7 +387,7 @@ mod tests { async fn write_unblocks_many() { const N: usize = 4; - let store = super::MemDB::new(TestDeadliner::never()); + let store = super::Handle::new(TestDeadliner::never()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); let signed_data = MockSignedData(42); @@ -381,7 +423,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unrelated_write_does_not_unblock() { - let store = super::MemDB::new(TestDeadliner::never()); + let store = super::Handle::new(TestDeadliner::never()); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); let data_a = MockSignedData(1); From 41b677611878c1442001ba40e4dd7c3720185ead Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Sun, 24 May 2026 03:12:37 -0300 Subject: [PATCH 17/34] Remove deadliner task - Use `tokio::select!` - Apply clippy lints --- crates/core/src/aggsigdb/memory.rs | 89 +++++++++++++++--------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 963779b8..afb30780 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -14,44 +14,54 @@ pub enum Error { MismatchingData, } +type Waiters = + HashMap<(types::Duty, types::PubKey), Vec>>>; + struct Actor { entries: HashMap>>, - waiters: HashMap< - (types::Duty, types::PubKey), - Vec>>, - >, + waiters: Waiters, deadliner: Arc, } impl Actor { - async fn run(&mut self, mut messages: sync::mpsc::Receiver) { - while let Some(msg) = messages.recv().await { - match msg { - Message::Store { - duty, - set, - response, - } => { - let result = self.store(duty, set).await; - let _ = response.send(result); - } - Message::WaitFor { - duty, - pub_key, - response, - } => { - if let Some(found) = self.get(&duty, &pub_key) { - let _ = response.send(found); - } else { - self.waiters - .entry((duty, pub_key)) - .or_default() - .push(response); + async fn run( + &mut self, + mut messages: sync::mpsc::Receiver, + mut evictions: sync::mpsc::Receiver, + ) { + loop { + tokio::select! { + biased; // We want to run evictions first + + Some(duty) = evictions.recv() => self.evict(duty), + + msg = messages.recv() => match msg { + None => break, // All handles have been dropped, so we can stop the actor. + Some(msg) => match msg { + Message::Store { + duty, + set, + response, + } => { + let result = self.store(duty, set).await; + let _ = response.send(result); + } + Message::WaitFor { + duty, + pub_key, + response, + } => { + if let Some(found) = self.get(&duty, &pub_key) { + let _ = response.send(found); + } else { + self.waiters + .entry((duty, pub_key)) + .or_default() + .push(response); + } + } } } - Message::Evict { duty } => { - let _ = self.evict(duty); - } } } } @@ -103,9 +113,6 @@ impl Actor { } enum Message { - Evict { - duty: types::Duty, - }, Store { duty: types::Duty, set: types::SignedDataSet, @@ -138,16 +145,8 @@ impl Handle { waiters: HashMap::new(), deadliner: Arc::clone(&deadliner), }; - tokio::spawn(async move { actor.run(receiver).await }); - - let deadliner_sender = sender.clone(); - tokio::spawn(async move { - if let Some(mut c) = deadliner.c() { - while let Some(duty) = c.recv().await { - let _ = deadliner_sender.send(Message::Evict { duty }).await; - } - } - }); + let (_, never) = sync::mpsc::channel(1); + tokio::spawn(async move { actor.run(receiver, deadliner.c().unwrap_or(never)).await }); Self { sender } } @@ -161,7 +160,7 @@ impl Handle { response: response_tx, }; let _ = self.sender.send(msg).await; - response_rx.await.unwrap() + response_rx.await.expect("Actor panicked") } /// Blocks and returns the aggregated signed duty data when available. @@ -180,7 +179,7 @@ impl Handle { response: response_tx, }; let _ = self.sender.send(msg).await; - response_rx.await.unwrap() + response_rx.await.expect("Actor panicked") } } From b46a8e8dfb02300771b7060c7d97936f77738928 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Sun, 24 May 2026 03:24:59 -0300 Subject: [PATCH 18/34] Improve tests - Do not sleep, wait for eviction to complete --- crates/core/src/aggsigdb/memory.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index afb30780..ac10c8fe 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -356,11 +356,16 @@ mod tests { .await .unwrap(); - // The eviction task runs concurrently, so we wait until the specific - // data is gone, so new readers are guaranteed to not observe it. + // Wait until the eviction is processed and the duty is removed. evict_tx.send(duty.clone()).await.unwrap(); - // TODO: Find a better mechanism to wait for eviction - tokio::time::sleep(std::time::Duration::from_millis(200)).await; + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while evict_tx.capacity() != evict_tx.max_capacity() { + tokio::task::yield_now().await; + } + }) + .await + .expect("Eviction did not complete in time"); + let reader = { let store = store.clone(); let duty = duty.clone(); From 11ee5741716a0783b339f8f236f6b32a40362d92 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Fri, 22 May 2026 22:46:09 +0700 Subject: [PATCH 19/34] refactor(qbft): introduce QbftTypes and some function structs (#431) * test: update qbft test * fix: compare run on retached thread * fix: removed hard coded salt in tests * fix: return error on Context cancelled * fix: hash from string, not magic number * fix: one shot cancel when parent is cancelled * fix: fix make_is_leader test * fix: minors naming and comments * fix: linter * fix: early cancel on the loop * fix: context cancel in compare * fix: validate definition * fix: add check pr < r * fix: test and document on run * fix: compare callback failed should timeout * fix: test use timeout channel instead of sleep * fix: add comment on cancellation poll interval * fix: use enum for invalid defnintion error * fix: add more test on check valid round * fix: using test-case * fix: small fixes * fix: linter * fix: remove unnecessary filter * fix: make_is_leader now 0-based * fix: add cancelled state for fake clock * fix: flanky tests by ordering and add small settle window time * fix: harden fake-clock to avoid flanky scheduling * fix: lint * refactor: introduce QbftTypes * refactor: group QBFT callbacks into typed request/logger structs * refactor: add ClientRecord * refactor: simplify internal_tests * refactor: simplify qbft mod.rs * refactor: remove skip clippy on qbft mod --- crates/core/src/qbft/callbacks.rs | 121 ++++ crates/core/src/qbft/fake_clock.rs | 25 +- crates/core/src/qbft/internal_test.rs | 962 +++++++++++++++----------- crates/core/src/qbft/mod.rs | 702 +++++++++---------- 4 files changed, 1040 insertions(+), 770 deletions(-) create mode 100644 crates/core/src/qbft/callbacks.rs diff --git a/crates/core/src/qbft/callbacks.rs b/crates/core/src/qbft/callbacks.rs new file mode 100644 index 00000000..5cba3815 --- /dev/null +++ b/crates/core/src/qbft/callbacks.rs @@ -0,0 +1,121 @@ +use super::{MessageType, Msg, QbftTypes, Result, UponRule}; +use cancellation::CancellationToken; +use crossbeam::channel as mpmc; +use std::time; + +pub(super) type BroadcastFn = + dyn for<'a> Fn(BroadcastRequest<'a, T>) -> Result<()> + Send + Sync; +pub(super) type CompareFn = dyn for<'a> Fn(CompareRequest<'a, T>) + Send + Sync + 'static; +pub(super) type UponRuleLoggerFn = dyn for<'a> Fn(UponRuleLog<'a, T>) + Send + Sync; +pub(super) type RoundChangeLoggerFn = dyn for<'a> Fn(RoundChangeLog<'a, T>) + Send + Sync; +pub(super) type UnjustLoggerFn = dyn for<'a> Fn(UnjustLog<'a, T>) + Send + Sync; +pub(super) type LeaderFn = dyn for<'a> Fn(LeaderRequest<'a, T>) -> bool + Send + Sync; +pub(super) type DecideFn = dyn for<'a> Fn(DecideRequest<'a, T>) + Send + Sync; + +/// Input passed to `Transport::broadcast`. +pub struct BroadcastRequest<'a, T: QbftTypes> { + /// Parent cancellation token. + pub ct: &'a CancellationToken, + /// Message type to broadcast. + pub type_: MessageType, + /// Consensus instance identifier. + pub instance: &'a T::Instance, + /// Sending process. + pub source: i64, + /// Message round. + pub round: i64, + /// Proposal value. + pub value: &'a T::Value, + /// Prepared round carried by ROUND-CHANGE messages. + pub prepared_round: i64, + /// Prepared value carried by ROUND-CHANGE messages. + pub prepared_value: &'a T::Value, + /// Optional justification piggybacked on the message. + pub justification: Option<&'a Vec>>, +} + +/// Input passed to `Definition::compare`. +pub struct CompareRequest<'a, T: QbftTypes> { + /// Compare-scoped cancellation token. + pub ct: &'a CancellationToken, + /// Proposed commit quorum message. + pub qcommit: &'a Msg, + /// Channel carrying the local compare value if it was not cached yet. + pub input_value_source_ch: &'a mpmc::Receiver, + /// Cached local compare value. + pub input_value_source: &'a T::Compare, + /// Channel used by the callback to return compare status. + pub return_err: &'a mpmc::Sender>, + /// Channel used by the callback to cache the local compare value. + pub return_value: &'a mpmc::Sender, +} + +/// Timer returned by `Definition::new_timer`. +pub struct Timer { + /// Channel that fires when the timer expires. + pub receive: mpmc::Receiver, + /// Stops the timer. + pub stop: Box, +} + +/// Input passed to `Definition::is_leader`. +pub struct LeaderRequest<'a, T: QbftTypes> { + /// Consensus instance identifier. + pub instance: &'a T::Instance, + /// Round being evaluated. + pub round: i64, + /// Process being evaluated. + pub process: i64, +} + +/// Input passed to `Definition::decide`. +pub struct DecideRequest<'a, T: QbftTypes> { + /// Parent cancellation token. + pub ct: &'a CancellationToken, + /// Consensus instance identifier. + pub instance: &'a T::Instance, + /// Decided value. + pub value: &'a T::Value, + /// Commit quorum justifying the decision. + pub qcommit: &'a Vec>, +} + +/// Input passed to `QbftLogger::upon_rule`. +pub struct UponRuleLog<'a, T: QbftTypes> { + /// Consensus instance identifier. + pub instance: &'a T::Instance, + /// Local process. + pub process: i64, + /// Current local round. + pub round: i64, + /// Message that triggered classification. + pub msg: &'a Msg, + /// Rule that fired. + pub upon_rule: UponRule, +} + +/// Input passed to `QbftLogger::round_change`. +pub struct RoundChangeLog<'a, T: QbftTypes> { + /// Consensus instance identifier. + pub instance: &'a T::Instance, + /// Local process. + pub process: i64, + /// Previous local round. + pub round: i64, + /// New local round. + pub new_round: i64, + /// Rule that caused the round change. + pub upon_rule: UponRule, + /// Messages from the previous round. + pub msgs: &'a Vec>, +} + +/// Input passed to `QbftLogger::unjust`. +pub struct UnjustLog<'a, T: QbftTypes> { + /// Consensus instance identifier. + pub instance: &'a T::Instance, + /// Local process. + pub process: i64, + /// Rejected message. + pub msg: Msg, +} diff --git a/crates/core/src/qbft/fake_clock.rs b/crates/core/src/qbft/fake_clock.rs index 6bc32495..c1891fb5 100644 --- a/crates/core/src/qbft/fake_clock.rs +++ b/crates/core/src/qbft/fake_clock.rs @@ -1,3 +1,5 @@ +#![allow(clippy::arithmetic_side_effects)] + use crossbeam::channel as mpmc; use std::{ collections::BTreeMap, @@ -28,7 +30,13 @@ struct FakeClockInner { now: Instant, last_id: usize, cancelled: bool, - clients: BTreeMap, Instant, TimerPriority)>, + clients: BTreeMap, +} + +struct ClientRecord { + sender: mpmc::Sender, + deadline: Instant, + priority: TimerPriority, } impl FakeClock { @@ -80,7 +88,14 @@ impl FakeClock { let deadline = inner.now + duration; inner.last_id += 1; - inner.clients.insert(id, (tx, deadline, priority)); + inner.clients.insert( + id, + ClientRecord { + sender: tx, + deadline, + priority, + }, + ); id }; @@ -124,9 +139,9 @@ impl FakeClock { inner.now += duration; let now = inner.now; - for (&id, (ch, deadline, priority)) in &inner.clients { - if *deadline <= now { - expired.push((id, *deadline, *priority, ch.clone())); + for (&id, record) in &inner.clients { + if record.deadline <= now { + expired.push((id, record.deadline, record.priority, record.sender.clone())); } } diff --git a/crates/core/src/qbft/internal_test.rs b/crates/core/src/qbft/internal_test.rs index 167f2c95..a3113927 100644 --- a/crates/core/src/qbft/internal_test.rs +++ b/crates/core/src/qbft/internal_test.rs @@ -1,3 +1,12 @@ +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_precision_loss, + clippy::cast_sign_loss, + clippy::collapsible_if +)] + use crate::qbft::{ self, fake_clock::{FakeClock, TimerPriority}, @@ -6,7 +15,7 @@ use crate::qbft::{ use cancellation::CancellationTokenSource; use crossbeam::channel as mpmc; use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, VecDeque}, fmt::Write as _, panic::{self, AssertUnwindSafe}, sync::{ @@ -31,13 +40,22 @@ const TEST_STREAM_MSG_ROUND: u64 = 11; const TEST_STREAM_MSG_VALUE: u64 = 12; const TEST_STREAM_MSG_PREPARED_ROUND: u64 = 13; const TEST_STREAM_MSG_PREPARED_VALUE: u64 = 14; +const TRACE_DUMP_LIMIT: usize = 200; const TEST_WAIT_TIMEOUT: Duration = Duration::from_secs(1); // Wall-clock guard catches lack of harness progress. Fake time still controls // protocol progress, so slow-but-progressing parallel runs should not fail. const TEST_STALL_TIMEOUT: Duration = Duration::from_secs(20); type RunOutcome = std::thread::Result>; -type TestMsgRef = Msg; +type TestMsgRef = Msg; + +struct TestQbft; + +impl QbftTypes for TestQbft { + type Compare = i64; + type Instance = i64; + type Value = i64; +} struct PendingCompareGuard { pending_compares: Arc, @@ -114,16 +132,11 @@ fn test_qbft(test: Test) { // Keep peer iteration deterministic. These fake-clock tests assert exact // rounds, and broadcast fanout order affects which node observes quorums // first when tests run in parallel. - let mut receives = BTreeMap::< - i64, - ( - mpmc::Sender>, - mpmc::Receiver>, - ), - >::new(); + let mut receives = + BTreeMap::>, mpmc::Receiver>)>::new(); let (broadcast_tx, broadcast_rx) = mpmc::unbounded::(); let (unjust_tx, unjust_rx) = mpmc::unbounded::(); - let (result_chan_tx, result_chan_rx) = mpmc::bounded::>>(N); + let (result_chan_tx, result_chan_rx) = mpmc::bounded::>>(N); let (run_chan_tx, run_chan_rx) = mpmc::bounded::<(i64, RunOutcome)>(N); let expected_initial_timers = N + test .value_delay @@ -146,84 +159,89 @@ fn test_qbft(test: Test) { Duration::from_secs(u64::pow(2, (round as u32) - 1)) }; - clock.new_timer(d) + let (receive, stop) = clock.new_timer(d); + Timer { receive, stop } }) }, decide: { let result_chan_tx = result_chan_tx.clone(); - Box::new(move |_, _, _, q_commit| { - result_chan_tx.send(q_commit.clone()).expect(WRITE_CHAN_ERR); + Box::new(move |req| { + result_chan_tx + .send(req.qcommit.clone()) + .expect(WRITE_CHAN_ERR); }) }, compare: { let pending_compares = pending_compares.clone(); - Arc::new(move |_, _, _, _, return_err, _| { + Arc::new(move |req| { let _guard = PendingCompareGuard { pending_compares: pending_compares.clone(), }; - return_err.send(Ok(())).expect(WRITE_CHAN_ERR); + req.return_err.send(Ok(())).expect(WRITE_CHAN_ERR); }) }, nodes: N as i64, fifo_limit: FIFO_LIMIT as i64, - log_round_change: { - let clock = clock.clone(); - let trace = trace.clone(); - let pending_timer_actions = pending_timer_actions.clone(); - - Box::new(move |_, process, round, new_round, upon_rule, _| { - if upon_rule == UPON_ROUND_TIMEOUT { - complete_timer_action(&pending_timer_actions); - } + logger: QbftLogger { + round_change: { + let clock = clock.clone(); + let trace = trace.clone(); + let pending_timer_actions = pending_timer_actions.clone(); + + Box::new(move |req| { + if req.upon_rule == UPON_ROUND_TIMEOUT { + complete_timer_action(&pending_timer_actions); + } - trace.push(format!( - "{:?} - {}@{} change to {} ~= {}", - clock.elapsed(), - process, - round, - new_round, - upon_rule, - )); - }) - }, - log_unjust: { - let trace = trace.clone(); - let unjust_tx = unjust_tx.clone(); - let fuzz = test.fuzz; - Box::new(move |_, process, msg| { - let line = format!("Unjust: process={} msg={:?}", process, msg); - trace.push(line.clone()); - if !fuzz { - unjust_tx.send(line).expect(WRITE_CHAN_ERR); - } - }) - }, - log_upon_rule: { - let clock = clock.clone(); - let trace = trace.clone(); - let pending_compares = pending_compares.clone(); - Box::new(move |_, process, round, msg, upon_rule| { - if upon_rule == UPON_JUSTIFIED_PRE_PREPARE { - pending_compares.fetch_add(1, Ordering::SeqCst); - } + trace.push(format!( + "{:?} - {}@{} change to {} ~= {}", + clock.elapsed(), + req.process, + req.round, + req.new_round, + req.upon_rule, + )); + }) + }, + unjust: { + let trace = trace.clone(); + let unjust_tx = unjust_tx.clone(); + let fuzz = test.fuzz; + Box::new(move |req| { + let line = format!("Unjust: process={} msg={:?}", req.process, req.msg); + trace.push(line.clone()); + if !fuzz { + unjust_tx.send(line).expect(WRITE_CHAN_ERR); + } + }) + }, + upon_rule: { + let clock = clock.clone(); + let trace = trace.clone(); + let pending_compares = pending_compares.clone(); + Box::new(move |req| { + if req.upon_rule == UPON_JUSTIFIED_PRE_PREPARE { + pending_compares.fetch_add(1, Ordering::SeqCst); + } - trace.push(format!( - "{:?} {} => {}@{} -> {}@{} ~= {}", - clock.elapsed(), - msg.source(), - msg.type_(), - msg.round(), - process, - round, - upon_rule, - )); - }) + trace.push(format!( + "{:?} {} => {}@{} -> {}@{} ~= {}", + clock.elapsed(), + req.msg.source(), + req.msg.type_(), + req.msg.round(), + req.process, + req.round, + req.upon_rule, + )); + }) + }, }, }); thread::scope(|s| { for i in 1..=N as i64 { - let (sender, receiver) = mpmc::bounded::>(1000); + let (sender, receiver) = mpmc::bounded::>(1000); let broadcast_tx = broadcast_tx.clone(); receives.insert(i, (sender.clone(), receiver.clone())); @@ -232,55 +250,53 @@ fn test_qbft(test: Test) { let clock = clock.clone(); let trace = trace.clone(); - Box::new( - move |_, type_, instance, source, round, value, pr, pv, justification| { - if round > MAX_ROUND as i64 { - return Err(QbftError::MaxRoundReached); - } - - if type_ == MSG_COMMIT && round <= test.commits_after.into() { - trace.push(format!( - "{:?} {} dropping commit for round {}", - clock.elapsed(), - source, - round - )); - return Ok(()); - } + Box::new(move |req| { + if req.round > MAX_ROUND as i64 { + return Err(QbftError::MaxRoundReached); + } + if req.type_ == MSG_COMMIT && req.round <= test.commits_after.into() { trace.push(format!( - "{:?} {} => {}@{}", + "{:?} {} dropping commit for round {}", clock.elapsed(), - source, - type_, - round + req.source, + req.round )); + return Ok(()); + } - let msg = new_msg( - type_, - *instance, - source, - round, - *value, - *value, - pr, - *pv, - justification, - ); - sender.send(msg.clone()).expect(WRITE_CHAN_ERR); - - bcast( - broadcast_tx.clone(), - msg.clone(), - test.bcast_jitter_ms, - clock.clone(), - trace.clone(), - seed, - ); + trace.push(format!( + "{:?} {} => {}@{}", + clock.elapsed(), + req.source, + req.type_, + req.round + )); - Ok(()) - }, - ) + let msg = new_msg( + req.type_, + *req.instance, + req.source, + req.round, + *req.value, + *req.value, + req.prepared_round, + *req.prepared_value, + req.justification, + ); + sender.send(msg.clone()).expect(WRITE_CHAN_ERR); + + bcast( + broadcast_tx.clone(), + msg.clone(), + test.bcast_jitter_ms, + clock.clone(), + trace.clone(), + seed, + ); + + Ok(()) + }) }, receive: receiver.clone(), }; @@ -358,7 +374,11 @@ fn test_qbft(test: Test) { s.spawn(move || { _ = v_chan_tx_send.send(i); }); - } else if is_leader(&test.instance, 1, i) { + } else if is_leader(LeaderRequest { + instance: &test.instance, + round: 1, + process: i, + }) { let v_chan_tx_send = keep_value_sender .as_ref() .expect("value sender kept until run returns") @@ -403,7 +423,7 @@ fn test_qbft(test: Test) { } } - let mut results = BTreeMap::>::new(); + let mut results = BTreeMap::>::new(); let mut count = 0; let mut decided = false; let mut done = 0; @@ -541,7 +561,11 @@ fn test_qbft(test: Test) { ); } } else { // Otherwise check that leader value was used. - if !is_leader(&test.instance, commit.round(), commit.value()) { + if !is_leader(LeaderRequest { + instance: &test.instance, + round: commit.round(), + process: commit.value(), + }) { cts.cancel(); clock.cancel(); panic!( @@ -658,7 +682,7 @@ fn test_qbft(test: Test) { } #[derive(Clone, Default)] -struct Trace(Arc>>); +struct Trace(Arc>>); impl Trace { fn new() -> Self { @@ -666,14 +690,17 @@ impl Trace { } fn push(&self, line: String) { - self.0.lock().unwrap().push(line); + let mut lines = self.0.lock().unwrap(); + if lines.len() == TRACE_DUMP_LIMIT { + lines.pop_front(); + } + lines.push_back(line); } fn dump(&self) -> String { let lines = self.0.lock().unwrap(); - let start = lines.len().saturating_sub(200); let mut out = String::new(); - for line in &lines[start..] { + for line in lines.iter() { let _ = writeln!(out, "{line}"); } out @@ -700,6 +727,10 @@ fn outcome_is_error(outcome: &RunOutcome, expected: fn(&QbftError) -> bool) -> b matches!(outcome, Ok(Err(err)) if expected(err)) } +fn assert_upon_rule(expected: UponRule, actual: UponRule) { + assert!(actual == expected, "want {expected}, got {actual}"); +} + fn test_seed(test: &Test) -> u64 { let mut seed = seed_from_label(TEST_SEED_LABEL); seed ^= test.instance as u64; @@ -722,10 +753,8 @@ fn seed_from_label(label: &str) -> u64 { } /// Construct a leader election function. -fn make_is_leader(n: i64) -> impl Fn(&i64, i64, i64) -> bool + Clone { - move |instance: &i64, round: i64, process: i64| -> bool { - (instance + round).rem_euclid(n) == process - } +fn make_is_leader(n: i64) -> impl for<'a> Fn(LeaderRequest<'a, TestQbft>) -> bool + Clone { + move |req| (*req.instance + req.round).rem_euclid(n) == req.process } /// Returns a new message to be broadcast. @@ -739,8 +768,8 @@ fn new_msg( value_source: i64, pr: i64, pv: i64, - justify: Option<&Vec>>, -) -> Msg { + justify: Option<&Vec>>, +) -> Msg { let msgs = match justify { None => vec![], Some(justify) => justify @@ -790,7 +819,7 @@ fn new_round_change_quorum(round: i64, pr: i64, pv: i64) -> Vec { // messages. fn bcast( broadcast: mpmc::Sender, - msg: Msg, + msg: Msg, jitter_ms: i32, clock: FakeClock, trace: Trace, @@ -837,10 +866,9 @@ fn deliver_ready_broadcasts( .iter() .take_while(|delayed| delayed.deliver_at <= clock.elapsed()) .count(); - let ready = pending.drain(..ready_count).collect::>(); - ready - .into_iter() + pending + .drain(..ready_count) .map(|delayed| fanout_broadcast(receives, drop_prob, seed, trace, clock, delayed.msg)) .sum() } @@ -893,7 +921,7 @@ fn fanout_broadcast( broadcasts } -fn random_msg(instance: i64, peer_idx: i64, seed: u64, counter: u64) -> Msg { +fn random_msg(instance: i64, peer_idx: i64, seed: u64, counter: u64) -> Msg { let message_types = [ MSG_PRE_PREPARE, MSG_PREPARE, @@ -915,12 +943,12 @@ fn random_msg(instance: i64, peer_idx: i64, seed: u64, counter: u64) -> Msg, target: i64, stream_id: u64) -> f64 { +fn deterministic_unit(seed: u64, msg: &Msg, target: i64, stream_id: u64) -> f64 { let value = deterministic_msg_u64(seed, msg, target, stream_id) >> 11; value as f64 / ((1_u64 << 53) as f64) } -fn deterministic_msg_u64(seed: u64, msg: &Msg, target: i64, stream_id: u64) -> u64 { +fn deterministic_msg_u64(seed: u64, msg: &Msg, target: i64, stream_id: u64) -> u64 { let mut value = splitmix64(seed ^ stream_id); value = splitmix64(value ^ i64_to_u64(msg.type_().0)); value = splitmix64(value ^ i64_to_u64(msg.instance())); @@ -967,7 +995,7 @@ struct TestMsg { justify: Option>, } -impl SomeMsg for TestMsg { +impl SomeMsg for TestMsg { fn type_(&self) -> MessageType { self.msg_type } @@ -1000,12 +1028,12 @@ impl SomeMsg for TestMsg { self.pv } - fn justification(&self) -> Vec> { + fn justification(&self) -> Vec> { match self.justify { None => vec![], Some(ref j) => j .iter() - .map(|j| Arc::new(j.clone()) as Msg) + .map(|j| Arc::new(j.clone()) as Msg) .collect(), } } @@ -1206,27 +1234,177 @@ fn fuzzed(start_delay_secs: Option, decide_round: i32, random_round: bool) }); } -fn noop_definition() -> Definition { +fn noop_definition() -> Definition { Definition { - is_leader: Box::new(|_, _, _| false), - new_timer: Box::new(|_| (mpmc::never(), Box::new(|| {}))), - decide: Box::new(|_, _, _, _| {}), - compare: Arc::new(|_, _, _, _, _, _| {}), + is_leader: Box::new(|_| false), + new_timer: Box::new(|_| Timer { + receive: mpmc::never(), + stop: Box::new(|| {}), + }), + decide: Box::new(|_| {}), + compare: Arc::new(|_| {}), nodes: 0, fifo_limit: 0, - log_round_change: Box::new(|_, _, _, _, _, _| {}), - log_unjust: Box::new(|_, _, _| {}), - log_upon_rule: Box::new(|_, _, _, _, _| {}), + logger: QbftLogger { + round_change: Box::new(|_| {}), + unjust: Box::new(|_| {}), + upon_rule: Box::new(|_| {}), + }, } } -fn noop_transport() -> Transport { +fn noop_transport() -> Transport { Transport { - broadcast: Box::new(|_, _, _, _, _, _, _, _, _| Ok(())), + broadcast: Box::new(|_| Ok(())), receive: mpmc::never(), } } +#[derive(Debug, PartialEq, Eq)] +struct BroadcastRecord { + canceled: bool, + type_: MessageType, + instance: i64, + source: i64, + round: i64, + value: i64, + prepared_round: i64, + prepared_value: i64, + justification_len: usize, +} + +#[test] +fn broadcast_request_maps_protocol_fields() { + let (receive_tx, receive_rx) = mpmc::bounded::>(4); + receive_tx + .send(new_msg(MSG_PRE_PREPARE, 0, 1, 1, 7, 7, 0, 0, None)) + .expect(WRITE_CHAN_ERR); + for source in 1..=3 { + receive_tx + .send(new_msg(MSG_PREPARE, 0, source, 1, 7, 7, 0, 0, None)) + .expect(WRITE_CHAN_ERR); + } + + let cts = CancellationTokenSource::new(); + let token = cts.token().clone(); + let (record_tx, record_rx) = mpmc::unbounded(); + let transport = Transport { + broadcast: Box::new(move |req| { + record_tx + .send(BroadcastRecord { + canceled: req.ct.is_canceled(), + type_: req.type_, + instance: *req.instance, + source: req.source, + round: req.round, + value: *req.value, + prepared_round: req.prepared_round, + prepared_value: *req.prepared_value, + justification_len: req.justification.map_or(0, Vec::len), + }) + .expect(WRITE_CHAN_ERR); + if req.type_ == MSG_COMMIT { + cts.cancel(); + } + Ok(()) + }), + receive: receive_rx, + }; + let mut def = noop_definition(); + def.nodes = 4; + def.fifo_limit = 100; + def.is_leader = Box::new(|req| req.process == 1); + def.compare = Arc::new(|req| req.return_err.send(Ok(())).expect(WRITE_CHAN_ERR)); + + let (_input_tx, input_rx) = mpmc::bounded::(1); + let (_source_tx, source_rx) = mpmc::bounded::(1); + assert!(matches!( + qbft::run(&token, &def, &transport, &0, 2, input_rx, source_rx), + Err(QbftError::ContextCanceled) + )); + assert_eq!( + record_rx.try_iter().collect::>(), + vec![ + BroadcastRecord { + canceled: false, + type_: MSG_PREPARE, + instance: 0, + source: 2, + round: 1, + value: 7, + prepared_round: 0, + prepared_value: 0, + justification_len: 0, + }, + BroadcastRecord { + canceled: false, + type_: MSG_COMMIT, + instance: 0, + source: 2, + round: 1, + value: 7, + prepared_round: 0, + prepared_value: 0, + justification_len: 0, + }, + ] + ); + + let (timer_tx, timer_rx) = mpmc::bounded(1); + timer_tx.send(time::Instant::now()).expect(WRITE_CHAN_ERR); + let cts = CancellationTokenSource::new(); + let token = cts.token().clone(); + let (record_tx, record_rx) = mpmc::unbounded(); + let transport = Transport { + broadcast: Box::new(move |req| { + record_tx + .send(BroadcastRecord { + canceled: req.ct.is_canceled(), + type_: req.type_, + instance: *req.instance, + source: req.source, + round: req.round, + value: *req.value, + prepared_round: req.prepared_round, + prepared_value: *req.prepared_value, + justification_len: req.justification.map_or(0, Vec::len), + }) + .expect(WRITE_CHAN_ERR); + cts.cancel(); + Ok(()) + }), + receive: mpmc::never(), + }; + let mut def = noop_definition(); + def.nodes = 4; + def.fifo_limit = 100; + def.new_timer = Box::new(move |_| Timer { + receive: timer_rx.clone(), + stop: Box::new(|| {}), + }); + + let (_input_tx, input_rx) = mpmc::bounded::(1); + let (_source_tx, source_rx) = mpmc::bounded::(1); + assert!(matches!( + qbft::run(&token, &def, &transport, &0, 2, input_rx, source_rx), + Err(QbftError::ContextCanceled) + )); + assert_eq!( + record_rx.try_iter().collect::>(), + vec![BroadcastRecord { + canceled: false, + type_: MSG_ROUND_CHANGE, + instance: 0, + source: 2, + round: 2, + value: 0, + prepared_round: 0, + prepared_value: 0, + justification_len: 0, + }] + ); +} + // Tests quorum/faulty formulas across node counts. // Expect quorum and tolerated-fault counts to match the Charon formula. #[test_case(1, 1, 0 ; "n1")] @@ -1252,7 +1430,7 @@ fn noop_transport() -> Transport { #[test_case(21, 14, 6 ; "n21")] #[test_case(22, 15, 7 ; "n22")] fn formulas(n: i64, q: i64, f: i64) { - let d = Definition:: { + let d = Definition:: { nodes: n, ..noop_definition() }; @@ -1301,7 +1479,7 @@ fn duplicate_pre_prepare_rules() { const NO_LEADER: i64 = 1; const LEADER: i64 = 2; - let new_preprepare = |round: i64| -> Msg { + let new_preprepare = |round: i64| -> Msg { new_msg( MSG_PRE_PREPARE, 0, @@ -1319,33 +1497,37 @@ fn duplicate_pre_prepare_rules() { let mut def = noop_definition(); def.nodes = 4; def.fifo_limit = 100; - def.is_leader = Box::new(|_, _, process| process == LEADER); - def.log_upon_rule = Box::new(move |_, _, round, msg, upon_rule| { - println!("UponRule: rule={} round={} ", upon_rule, msg.round()); + def.is_leader = Box::new(|req| req.process == LEADER); + def.logger.upon_rule = Box::new(move |req| { + println!( + "UponRule: rule={} round={} ", + req.upon_rule, + req.msg.round() + ); - assert!(upon_rule == UPON_JUSTIFIED_PRE_PREPARE); + assert!(req.upon_rule == UPON_JUSTIFIED_PRE_PREPARE); - if msg.round() == 1 { + if req.msg.round() == 1 { return; } - if msg.round() == 2 { + if req.msg.round() == 2 { cts.cancel(); return; } - panic!("unexpected round {}", round); + panic!("unexpected round {}", req.round); }); - def.compare = Arc::new(|_, msg, _, _, return_err, _| { - let result = if msg.round() == 1 { + def.compare = Arc::new(|req| { + let result = if req.qcommit.round() == 1 { Err(QbftError::CompareError) } else { Ok(()) }; - return_err.send(result).expect(WRITE_CHAN_ERR); + req.return_err.send(result).expect(WRITE_CHAN_ERR); }); - let (r_chan_tx, r_chan_rx) = mpmc::bounded::>(2); + let (r_chan_tx, r_chan_rx) = mpmc::bounded::>(2); r_chan_tx.send(new_preprepare(1)).expect(WRITE_CHAN_ERR); r_chan_tx.send(new_preprepare(2)).expect(WRITE_CHAN_ERR); @@ -1407,29 +1589,41 @@ fn idle_run_returns_when_cancelled() { )); } -// Tests definition validation at the `run` boundary. -// Expect invalid node count and FIFO limit to return typed errors. -#[test_case(0, 1, true ; "invalid_nodes")] -#[test_case(4, 0, false ; "invalid_fifo_limit")] -fn invalid_definition_rejected(nodes: i64, fifo_limit: i64, invalid_nodes: bool) { +fn run_with_definition(def: &Definition) -> Result<()> { let cts = CancellationTokenSource::new(); let transport = noop_transport(); let (_input_tx, input_rx) = mpmc::bounded::(1); let (_source_tx, source_rx) = mpmc::bounded::(1); + qbft::run(cts.token(), def, &transport, &0, 1, input_rx, source_rx) +} + +// Tests definition validation at the `run` boundary. +// Expect invalid node count to return a typed error. +#[test] +fn invalid_nodes_rejected() { let mut def = noop_definition(); - def.nodes = nodes; - def.fifo_limit = fifo_limit; - - let result = qbft::run(cts.token(), &def, &transport, &0, 1, input_rx, source_rx); - if invalid_nodes { - assert!(matches!(result, Err(QbftError::InvalidNodes { nodes: 0 }))); - } else { - assert!(matches!( - result, - Err(QbftError::InvalidFifoLimit { fifo_limit: 0 }) - )); - } + def.nodes = 0; + def.fifo_limit = 1; + + assert!(matches!( + run_with_definition(&def), + Err(QbftError::InvalidNodes { nodes: 0 }) + )); +} + +// Tests definition validation at the `run` boundary. +// Expect invalid FIFO limit to return a typed error. +#[test] +fn invalid_fifo_limit_rejected() { + let mut def = noop_definition(); + def.nodes = 4; + def.fifo_limit = 0; + + assert!(matches!( + run_with_definition(&def), + Err(QbftError::InvalidFifoLimit { fifo_limit: 0 }) + )); } // Tests cancellation under a continuously hot receive channel. @@ -1442,7 +1636,7 @@ fn run_cancels_under_hot_receive_stream() { def.nodes = 4; def.fifo_limit = 100; - let (receive_tx, receive_rx) = mpmc::bounded::>(1024); + let (receive_tx, receive_rx) = mpmc::bounded::>(1024); let transport = Transport { receive: receive_rx, ..noop_transport() @@ -1499,15 +1693,17 @@ fn classify_rules() { def.is_leader = Box::new(make_is_leader(4)); let preprepare = new_msg(MSG_PRE_PREPARE, 0, 1, 1, 1, 0, 0, 0, None); - assert!(classify(&def, &0, 1, 2, &HashMap::new(), &preprepare).0 == UPON_JUSTIFIED_PRE_PREPARE); + assert_upon_rule( + UPON_JUSTIFIED_PRE_PREPARE, + classify(&def, &0, 1, 2, &HashMap::new(), &preprepare).0, + ); - let prepares = vec![ - new_msg(MSG_PREPARE, 0, 1, 1, 2, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 2, 1, 2, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 3, 1, 2, 0, 0, 0, None), - ]; + let prepares = new_prepare_quorum(1, 2); let buffer = buffer_by_source(&prepares); - assert!(classify(&def, &0, 1, 2, &buffer, &prepares[2]).0 == UPON_QUORUM_PREPARES); + assert_upon_rule( + UPON_QUORUM_PREPARES, + classify(&def, &0, 1, 2, &buffer, &prepares[2]).0, + ); let commits = vec![ new_msg(MSG_COMMIT, 0, 1, 1, 2, 0, 0, 0, None), @@ -1515,7 +1711,10 @@ fn classify_rules() { new_msg(MSG_COMMIT, 0, 3, 1, 2, 0, 0, 0, None), ]; let buffer = buffer_by_source(&commits); - assert!(classify(&def, &0, 1, 2, &buffer, &commits[2]).0 == UPON_QUORUM_COMMITS); + assert_upon_rule( + UPON_QUORUM_COMMITS, + classify(&def, &0, 1, 2, &buffer, &commits[2]).0, + ); let future_round_changes = vec![ new_msg(MSG_ROUND_CHANGE, 0, 1, 3, 0, 0, 0, 0, None), @@ -1526,15 +1725,11 @@ fn classify_rules() { classify(&def, &0, 1, 2, &buffer, &future_round_changes[1]).0 == UPON_F_PLUS1_ROUND_CHANGES ); - let unjust_round_changes = vec![ - new_msg(MSG_ROUND_CHANGE, 0, 1, 1, 0, 0, 2, 9, None), - new_msg(MSG_ROUND_CHANGE, 0, 2, 1, 0, 0, 2, 9, None), - new_msg(MSG_ROUND_CHANGE, 0, 3, 1, 0, 0, 2, 9, None), - ]; + let unjust_round_changes = new_round_change_quorum(1, 2, 9); let buffer = buffer_by_source(&unjust_round_changes); - assert!( - classify(&def, &0, 1, 2, &buffer, &unjust_round_changes[2]).0 - == UPON_UNJUST_QUORUM_ROUND_CHANGES + assert_upon_rule( + UPON_UNJUST_QUORUM_ROUND_CHANGES, + classify(&def, &0, 1, 2, &buffer, &unjust_round_changes[2]).0, ); } @@ -1545,33 +1740,21 @@ fn classify_rules() { fn justified_qrc_j1_and_j2() { let mut def = noop_definition(); def.nodes = 4; - let j1 = vec![ - new_msg(MSG_ROUND_CHANGE, 0, 1, 2, 0, 0, 0, 0, None), - new_msg(MSG_ROUND_CHANGE, 0, 2, 2, 0, 0, 0, 0, None), - new_msg(MSG_ROUND_CHANGE, 0, 3, 2, 0, 0, 0, 0, None), - ]; + let j1 = new_round_change_quorum(2, 0, 0); assert_eq!(Some(0), contains_justified_qrc(&def, &j1, 2)); assert_eq!(3, get_justified_qrc(&def, &j1, 2).unwrap().len()); - let j2 = vec![ + let mut j2 = vec![ new_msg(MSG_ROUND_CHANGE, 0, 1, 2, 0, 0, 1, 7, None), new_msg(MSG_ROUND_CHANGE, 0, 2, 2, 0, 0, 1, 7, None), new_msg(MSG_ROUND_CHANGE, 0, 3, 2, 0, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 1, 1, 7, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 2, 1, 7, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 3, 1, 7, 0, 0, 0, None), ]; + j2.extend(new_prepare_quorum(1, 7)); assert_eq!(Some(7), contains_justified_qrc(&def, &j2, 2)); assert!(get_justified_qrc(&def, &j2, 2).unwrap().len() >= 6); - let invalid_pr = vec![ - new_msg(MSG_ROUND_CHANGE, 0, 1, 2, 0, 0, 2, 7, None), - new_msg(MSG_ROUND_CHANGE, 0, 2, 2, 0, 0, 2, 7, None), - new_msg(MSG_ROUND_CHANGE, 0, 3, 2, 0, 0, 2, 7, None), - new_msg(MSG_PREPARE, 0, 1, 2, 7, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 2, 2, 7, 0, 0, 0, None), - new_msg(MSG_PREPARE, 0, 3, 2, 7, 0, 0, 0, None), - ]; + let mut invalid_pr = new_round_change_quorum(2, 2, 7); + invalid_pr.extend(new_prepare_quorum(2, 7)); assert_eq!(None, contains_justified_qrc(&def, &invalid_pr, 2)); assert!(get_justified_qrc(&def, &invalid_pr, 2).is_none()); } @@ -1692,8 +1875,8 @@ fn compare_success_error_cached_value_source_and_timeout() { let (_vs_tx, vs_rx) = mpmc::bounded::(1); let timer = mpmc::never(); let mut def = noop_definition(); - def.compare = Arc::new(|_, _, _, _, return_err, _| { - return_err.send(Ok(())).expect(WRITE_CHAN_ERR); + def.compare = Arc::new(|req| { + req.return_err.send(Ok(())).expect(WRITE_CHAN_ERR); }); assert!(matches!( compare(cts.token(), &def, &msg, &vs_rx, 0, &timer), @@ -1701,8 +1884,8 @@ fn compare_success_error_cached_value_source_and_timeout() { )); let mut def = noop_definition(); - def.compare = Arc::new(|_, _, _, _, return_err, _| { - let return_err = return_err.clone(); + def.compare = Arc::new(|req| { + let return_err = req.return_err.clone(); thread::spawn(move || { thread::sleep(Duration::from_millis(10)); return_err.send(Ok(())).expect(WRITE_CHAN_ERR); @@ -1714,8 +1897,8 @@ fn compare_success_error_cached_value_source_and_timeout() { )); let mut def = noop_definition(); - def.compare = Arc::new(|_, _, _, _, return_err, _| { - return_err + def.compare = Arc::new(|req| { + req.return_err .send(Err(QbftError::CompareError)) .expect(WRITE_CHAN_ERR); }); @@ -1727,19 +1910,17 @@ fn compare_success_error_cached_value_source_and_timeout() { let (vs_tx, vs_rx) = mpmc::bounded::(1); vs_tx.send(42).expect(WRITE_CHAN_ERR); let mut def = noop_definition(); - def.compare = Arc::new( - |_, _, input_value_source_ch, input_value_source, return_err, return_value| { - let cached = if *input_value_source == 0 { - let value = input_value_source_ch.recv().expect(READ_CHAN_ERR); - return_value.send(value).expect(WRITE_CHAN_ERR); - value - } else { - *input_value_source - }; - assert_eq!(42, cached); - return_err.send(Ok(())).expect(WRITE_CHAN_ERR); - }, - ); + def.compare = Arc::new(|req| { + let cached = if *req.input_value_source == 0 { + let value = req.input_value_source_ch.recv().expect(READ_CHAN_ERR); + req.return_value.send(value).expect(WRITE_CHAN_ERR); + value + } else { + *req.input_value_source + }; + assert_eq!(42, cached); + req.return_err.send(Ok(())).expect(WRITE_CHAN_ERR); + }); assert!(matches!( compare(cts.token(), &def, &msg, &vs_rx, 0, &timer), (42, Ok(())) @@ -1748,21 +1929,19 @@ fn compare_success_error_cached_value_source_and_timeout() { let (vs_tx, vs_rx) = mpmc::bounded::(1); vs_tx.send(43).expect(WRITE_CHAN_ERR); let mut def = noop_definition(); - def.compare = Arc::new( - |_, _, input_value_source_ch, input_value_source, return_err, return_value| { - let cached = if *input_value_source == 0 { - let value = input_value_source_ch.recv().expect(READ_CHAN_ERR); - return_value.send(value).expect(WRITE_CHAN_ERR); - value - } else { - *input_value_source - }; - assert_eq!(43, cached); - return_err - .send(Err(QbftError::CompareError)) - .expect(WRITE_CHAN_ERR); - }, - ); + def.compare = Arc::new(|req| { + let cached = if *req.input_value_source == 0 { + let value = req.input_value_source_ch.recv().expect(READ_CHAN_ERR); + req.return_value.send(value).expect(WRITE_CHAN_ERR); + value + } else { + *req.input_value_source + }; + assert_eq!(43, cached); + req.return_err + .send(Err(QbftError::CompareError)) + .expect(WRITE_CHAN_ERR); + }); assert!(matches!( compare(cts.token(), &def, &msg, &vs_rx, 0, &timer), (43, Err(QbftError::CompareError)) @@ -1771,9 +1950,9 @@ fn compare_success_error_cached_value_source_and_timeout() { let (timer_tx, timer_rx) = mpmc::bounded(1); timer_tx.send(time::Instant::now()).expect(WRITE_CHAN_ERR); let mut def = noop_definition(); - def.compare = Arc::new(|_, _, _, _, return_err, _| { + def.compare = Arc::new(|req| { thread::sleep(Duration::from_millis(20)); - let _ = return_err.send(Ok(())); + let _ = req.return_err.send(Ok(())); }); assert!(matches!( compare(cts.token(), &def, &msg, &vs_rx, 44, &timer_rx), @@ -1792,11 +1971,11 @@ fn compare_timeout_does_not_wait_for_blocked_callback() { timer_tx.send(time::Instant::now()).expect(WRITE_CHAN_ERR); let mut def = noop_definition(); - def.compare = Arc::new(|ct, _, _, _, return_err, _| { - while !ct.is_canceled() { + def.compare = Arc::new(|req| { + while !req.ct.is_canceled() { thread::sleep(Duration::from_millis(1)); } - let _ = return_err.send(Ok(())); + let _ = req.return_err.send(Ok(())); }); let (result_tx, result_rx) = mpmc::bounded(1); @@ -1826,7 +2005,7 @@ fn compare_callback_exit_without_status_waits_for_timer() { let (callback_done_tx, callback_done_rx) = mpmc::bounded(1); let mut def = noop_definition(); - def.compare = Arc::new(move |_, _, _, _, _, _| { + def.compare = Arc::new(move |_| { callback_done_tx.send(()).expect(WRITE_CHAN_ERR); }); @@ -1868,13 +2047,13 @@ fn compare_parent_cancel_cancels_callback_token() { let (token_cancelled_tx, token_cancelled_rx) = mpmc::bounded(1); let mut def = noop_definition(); - def.compare = Arc::new(move |ct, _, _, _, return_err, _| { + def.compare = Arc::new(move |req| { compare_started_tx.send(()).expect(WRITE_CHAN_ERR); - while !ct.is_canceled() { + while !req.ct.is_canceled() { thread::sleep(Duration::from_millis(1)); } token_cancelled_tx.send(()).expect(WRITE_CHAN_ERR); - let _ = return_err.send(Ok(())); + let _ = req.return_err.send(Ok(())); }); let (result_tx, result_rx) = mpmc::bounded(1); @@ -1919,20 +2098,20 @@ fn run_parent_cancel_during_compare_does_not_prepare() { let mut def = noop_definition(); def.nodes = 4; def.fifo_limit = 100; - def.is_leader = Box::new(|_, _, process| process == LEADER); - def.compare = Arc::new(move |ct, _, _, _, return_err, _| { + def.is_leader = Box::new(|req| req.process == LEADER); + def.compare = Arc::new(move |req| { compare_started_tx.send(()).expect(WRITE_CHAN_ERR); - while !ct.is_canceled() { + while !req.ct.is_canceled() { thread::sleep(Duration::from_millis(1)); } compare_cancelled_tx.send(()).expect(WRITE_CHAN_ERR); - let _ = return_err.send(Ok(())); + let _ = req.return_err.send(Ok(())); }); let (broadcast_tx, broadcast_rx) = mpmc::bounded(1); let transport = Transport { - broadcast: Box::new(move |_, type_, _, _, _, _, _, _, _| { - broadcast_tx.send(type_).expect(WRITE_CHAN_ERR); + broadcast: Box::new(move |req| { + broadcast_tx.send(req.type_).expect(WRITE_CHAN_ERR); Ok(()) }), receive: receive_rx, @@ -1969,7 +2148,7 @@ fn run_parent_cancel_during_compare_does_not_prepare() { ); } -fn buffer_by_source(msgs: &[Msg]) -> HashMap>> { +fn buffer_by_source(msgs: &[Msg]) -> HashMap>> { let mut buffer = HashMap::new(); for msg in msgs { buffer @@ -2026,15 +2205,10 @@ fn test_qbft_chain_split(test: ChainSplitTest) { // Keep peer iteration deterministic. These fake-clock tests assert exact // rounds, and broadcast fanout order affects which node observes quorums // first when tests run in parallel. - let mut receives = BTreeMap::< - i64, - ( - mpmc::Sender>, - mpmc::Receiver>, - ), - >::new(); - let (broadcast_tx, broadcast_rx) = mpmc::unbounded::>(); - let (result_chan_tx, result_chan_rx) = mpmc::bounded::>>(N); + let mut receives = + BTreeMap::>, mpmc::Receiver>)>::new(); + let (broadcast_tx, broadcast_rx) = mpmc::unbounded::>(); + let (result_chan_tx, result_chan_rx) = mpmc::bounded::>>(N); let (run_chan_tx, run_chan_rx) = mpmc::bounded::<(i64, RunOutcome)>(N); let instance = 0; @@ -2043,135 +2217,132 @@ fn test_qbft_chain_split(test: ChainSplitTest) { new_timer: { let clock = clock.clone(); Box::new(move |round| { - clock.new_timer(Duration::from_secs(u64::pow(2, (round as u32) - 1))) + let (receive, stop) = + clock.new_timer(Duration::from_secs(u64::pow(2, (round as u32) - 1))); + Timer { receive, stop } }) }, decide: { let result_chan_tx = result_chan_tx.clone(); - Box::new(move |_, _, _, q_commit| { - result_chan_tx.send(q_commit.clone()).expect(WRITE_CHAN_ERR); + Box::new(move |req| { + result_chan_tx + .send(req.qcommit.clone()) + .expect(WRITE_CHAN_ERR); }) }, compare: { let pending_compares = pending_compares.clone(); - Arc::new( - move |_, - qcommit, - input_value_source_ch, - input_value_source, - return_err, - return_value| { - let _guard = PendingCompareGuard { - pending_compares: pending_compares.clone(), - }; - let leader_value_source = qcommit.value_source().expect("value source"); - let local = if *input_value_source == 0 { - let value = input_value_source_ch.recv().expect(READ_CHAN_ERR); - return_value.send(value).expect(WRITE_CHAN_ERR); - value - } else { - *input_value_source - }; - - if leader_value_source != local { - return_err - .send(Err(QbftError::CompareError)) - .expect(WRITE_CHAN_ERR); - return; - } + Arc::new(move |req| { + let _guard = PendingCompareGuard { + pending_compares: pending_compares.clone(), + }; + let leader_value_source = req.qcommit.value_source().expect("value source"); + let local = if *req.input_value_source == 0 { + let value = req.input_value_source_ch.recv().expect(READ_CHAN_ERR); + req.return_value.send(value).expect(WRITE_CHAN_ERR); + value + } else { + *req.input_value_source + }; - return_err.send(Ok(())).expect(WRITE_CHAN_ERR); - }, - ) - }, - nodes: N as i64, - fifo_limit: FIFO_LIMIT, - log_round_change: { - let clock = clock.clone(); - let trace = trace.clone(); - let pending_timer_actions = pending_timer_actions.clone(); - Box::new(move |_, process, round, new_round, upon_rule, _| { - if upon_rule == UPON_ROUND_TIMEOUT { - complete_timer_action(&pending_timer_actions); + if leader_value_source != local { + req.return_err + .send(Err(QbftError::CompareError)) + .expect(WRITE_CHAN_ERR); + return; } - trace.push(format!( - "{:?} - {}@{} change to {} ~= {}", - clock.elapsed(), - process, - round, - new_round, - upon_rule - )); + req.return_err.send(Ok(())).expect(WRITE_CHAN_ERR); }) }, - log_unjust: { - let trace = trace.clone(); - Box::new(move |_, process, msg| { - trace.push(format!("Unjust: process={} msg={:?}", process, msg)) - }) - }, - log_upon_rule: { - let clock = clock.clone(); - let trace = trace.clone(); - let pending_compares = pending_compares.clone(); - Box::new(move |_, process, round, msg, upon_rule| { - if upon_rule == UPON_JUSTIFIED_PRE_PREPARE { - pending_compares.fetch_add(1, Ordering::SeqCst); - } + nodes: N as i64, + fifo_limit: FIFO_LIMIT, + logger: QbftLogger { + round_change: { + let clock = clock.clone(); + let trace = trace.clone(); + let pending_timer_actions = pending_timer_actions.clone(); + Box::new(move |req| { + if req.upon_rule == UPON_ROUND_TIMEOUT { + complete_timer_action(&pending_timer_actions); + } - trace.push(format!( - "{:?} {} => {}@{} -> {}@{} ~= {}", - clock.elapsed(), - msg.source(), - msg.type_(), - msg.round(), - process, - round, - upon_rule - )); - }) + trace.push(format!( + "{:?} - {}@{} change to {} ~= {}", + clock.elapsed(), + req.process, + req.round, + req.new_round, + req.upon_rule + )); + }) + }, + unjust: { + let trace = trace.clone(); + Box::new(move |req| { + trace.push(format!("Unjust: process={} msg={:?}", req.process, req.msg)) + }) + }, + upon_rule: { + let clock = clock.clone(); + let trace = trace.clone(); + let pending_compares = pending_compares.clone(); + Box::new(move |req| { + if req.upon_rule == UPON_JUSTIFIED_PRE_PREPARE { + pending_compares.fetch_add(1, Ordering::SeqCst); + } + + trace.push(format!( + "{:?} {} => {}@{} -> {}@{} ~= {}", + clock.elapsed(), + req.msg.source(), + req.msg.type_(), + req.msg.round(), + req.process, + req.round, + req.upon_rule + )); + }) + }, }, }); thread::scope(|s| { for i in 1..=N as i64 { - let (sender, receiver) = mpmc::bounded::>(1000); + let (sender, receiver) = mpmc::bounded::>(1000); receives.insert(i, (sender.clone(), receiver.clone())); let broadcast_tx = broadcast_tx.clone(); let trace = trace.clone(); let clock = clock.clone(); let transport = Transport { - broadcast: Box::new( - move |_, type_, instance, source, round, value, pr, pv, justification| { - if round > MAX_ROUND { - return Err(QbftError::MaxRoundReached); - } + broadcast: Box::new(move |req| { + if req.round > MAX_ROUND { + return Err(QbftError::MaxRoundReached); + } - trace.push(format!( - "{:?} {} => {}@{}", - clock.elapsed(), - source, - type_, - round - )); - let msg = new_msg( - type_, - *instance, - source, - round, - *value, - *value, - pr, - *pv, - justification, - ); - sender.send(msg.clone()).expect(WRITE_CHAN_ERR); - broadcast_tx.send(msg).expect(WRITE_CHAN_ERR); - Ok(()) - }, - ), + trace.push(format!( + "{:?} {} => {}@{}", + clock.elapsed(), + req.source, + req.type_, + req.round + )); + let msg = new_msg( + req.type_, + *req.instance, + req.source, + req.round, + *req.value, + *req.value, + req.prepared_round, + *req.prepared_value, + req.justification, + ); + sender.send(msg.clone()).expect(WRITE_CHAN_ERR); + broadcast_tx.send(msg).expect(WRITE_CHAN_ERR); + Ok(()) + }), receive: receiver, }; @@ -2208,11 +2379,33 @@ fn test_qbft_chain_split(test: ChainSplitTest) { } } - let mut results = BTreeMap::>::new(); + let mut results = BTreeMap::>::new(); let mut count = 0; let mut decided = false; let mut done = 0; let mut last_progress = time::Instant::now(); + let chain_split_seed = seed_from_label(CHAIN_SPLIT_SEED_LABEL); + // The no-consensus halt case must reach round 11; using Go's 1ms tick + // here makes this Rust harness exceed its real-time guard, so only that + // halt path fast-forwards fake time. + let tick = if test.should_halt { + Duration::from_millis(100) + } else { + Duration::from_millis(1) + }; + let timeout_limit = if test.should_halt { + let max_round = u32::try_from(MAX_ROUND).expect("MAX_ROUND fits u32"); + let seconds = 1_u64 + .checked_shl( + max_round + .checked_add(1) + .expect("MAX_ROUND permits timeout limit"), + ) + .expect("MAX_ROUND permits timeout limit"); + Duration::from_secs(seconds) + } else { + Duration::from_secs(60) + }; loop { mpmc::select! { @@ -2224,13 +2417,7 @@ fn test_qbft_chain_split(test: ChainSplitTest) { continue; } out_tx.send(msg.clone()).expect(WRITE_CHAN_ERR); - if deterministic_unit( - seed_from_label(CHAIN_SPLIT_SEED_LABEL), - &msg, - *target, - TEST_STREAM_DUPLICATE, - ) < 0.1 - { + if deterministic_unit(chain_split_seed, &msg, *target, TEST_STREAM_DUPLICATE) < 0.1 { out_tx.send(msg.clone()).expect(WRITE_CHAN_ERR); } } @@ -2249,7 +2436,7 @@ fn test_qbft_chain_split(test: ChainSplitTest) { ); } - for commit in q_commit.clone() { + for commit in &q_commit { for previous in results.values() { if previous.value() != commit.value() { cts.cancel(); @@ -2289,7 +2476,7 @@ fn test_qbft_chain_split(test: ChainSplitTest) { ); } } - results.insert(commit.source(), commit); + results.insert(commit.source(), commit.clone()); } count += 1; if count == N { @@ -2347,26 +2534,9 @@ fn test_qbft_chain_split(test: ChainSplitTest) { // Matches the Go harness throttle; ordering correctness comes // from the pending-work barriers, not this duration. thread::sleep(Duration::from_micros(1)); - // The no-consensus halt case must reach round 11; using Go's - // 1ms tick here makes this Rust harness exceed its real-time - // guard, so only that halt path fast-forwards fake time. - let tick = if test.should_halt { - Duration::from_millis(100) - } else { - Duration::from_millis(1) - }; clock.advance_and_wait(tick, &pending_timer_actions); last_progress = time::Instant::now(); - let limit = if test.should_halt { - let max_round = u32::try_from(MAX_ROUND).expect("MAX_ROUND fits u32"); - let seconds = 1_u64 - .checked_shl(max_round.checked_add(1).expect("MAX_ROUND permits timeout limit")) - .expect("MAX_ROUND permits timeout limit"); - Duration::from_secs(seconds) - } else { - Duration::from_secs(60) - }; - if clock.elapsed() > limit { + if clock.elapsed() > timeout_limit { cts.cancel(); clock.cancel(); panic!("chain split hang: decided={decided} done={done} count={count} elapsed={:?}\n{}", clock.elapsed(), trace.dump()); diff --git a/crates/core/src/qbft/mod.rs b/crates/core/src/qbft/mod.rs index 956f17e8..554fcfb6 100644 --- a/crates/core/src/qbft/mod.rs +++ b/crates/core/src/qbft/mod.rs @@ -10,16 +10,6 @@ //! - No domain-specific dependencies. //! - Explicit justifications. -// TODO: Remove these checks -#![allow(missing_docs)] -#![allow(clippy::type_complexity)] -#![allow(clippy::collapsible_if)] -#![allow(clippy::cast_sign_loss)] -#![allow(clippy::cast_precision_loss)] -#![allow(clippy::cast_possible_wrap)] -#![allow(clippy::cast_possible_truncation)] -#![allow(clippy::arithmetic_side_effects)] - use cancellation::{CancellationToken, CancellationTokenSource}; use crossbeam::channel as mpmc; use std::{ @@ -31,6 +21,16 @@ use std::{ sync, thread, time, }; +mod callbacks; +use callbacks::{ + BroadcastFn, CompareFn, DecideFn, LeaderFn, RoundChangeLoggerFn, UnjustLoggerFn, + UponRuleLoggerFn, +}; +pub use callbacks::{ + BroadcastRequest, CompareRequest, DecideRequest, LeaderRequest, RoundChangeLog, Timer, + UnjustLog, UponRuleLog, +}; + type Result = std::result::Result; // The `cancellation` crate is callback-based, not channel-based, so it cannot @@ -38,139 +38,105 @@ type Result = std::result::Result; // does not need sub-millisecond latency, and idle instances should stay cheap. const CANCELLATION_POLL_INTERVAL: time::Duration = time::Duration::from_millis(50); -type CompareFn = dyn Fn( - /* ct */ &CancellationToken, - /* qcommit */ &Msg, - /* input_value_source_ch */ &mpmc::Receiver, - /* input_value_source */ &C, - /* return_err */ &mpmc::Sender>, - /* return_value */ &mpmc::Sender, - ) + Send - + Sync - + 'static; +/// Associated types used by a QBFT instance. +pub trait QbftTypes: 'static { + /// Consensus instance identifier. + type Instance: Send + Sync + 'static; + /// Consensus value. + type Value: Eq + Hash + Default + 'static; + /// Application value used by the compare callback. + type Compare: Clone + Send + Sync + Default + 'static; +} +/// Errors returned by the QBFT core. #[derive(Debug, thiserror::Error)] pub enum QbftError { + /// Round timer expired before compare completed. #[error("Timeout")] TimeoutError, + /// Leader proposal failed application-level comparison. #[error("Compare leader value with local value failed")] CompareError, + /// Compare returned an error variant that core does not expect. #[error("bug: expected only comparison or timeout error, got {0}")] UnexpectedCompareError(Box), + /// Parent cancellation token was canceled. #[error("context canceled")] ContextCanceled, + /// Test or caller configured maximum round was reached. #[error("Maximum round reached")] MaxRoundReached, + /// Own input value was the null/default value. #[error("Zero input value not supported")] ZeroInputValue, + /// Node count must be positive. #[error("invalid node count: must be greater than zero, got {nodes}")] - InvalidNodes { nodes: i64 }, + InvalidNodes { + /// Configured node count. + nodes: i64, + }, + /// Per-source FIFO limit must be positive. #[error("invalid FIFO limit: must be greater than zero, got {fifo_limit}")] - InvalidFifoLimit { fifo_limit: i64 }, + InvalidFifoLimit { + /// Configured FIFO limit. + fifo_limit: i64, + }, + /// Receive channel closed unexpectedly. #[error("Failed to read from channel: {0}")] ChannelError(#[from] mpmc::RecvError), } /// Abstracts the transport layer between processes in the consensus system. -pub struct Transport -where - V: PartialEq, -{ +pub struct Transport { /// Broadcast sends a message with the provided fields to all other /// processes in the system (including this process). /// /// Note that an error exits the algorithm. - pub broadcast: Box< - dyn Fn( - /* ct */ &CancellationToken, - /* type_ */ MessageType, - /* instance */ &I, - /* source */ i64, - /* round */ i64, - /* value */ &V, - /* pr */ i64, - /* pv */ &V, - /* justification */ Option<&Vec>>, - ) -> Result<()> - + Send - + Sync, - >, + pub broadcast: Box>, /// Receive returns a stream of messages received /// from other processes in the system (including this process). - pub receive: mpmc::Receiver>, + pub receive: mpmc::Receiver>, +} + +/// Debug hooks for QBFT state transitions and rejected messages. +pub struct QbftLogger { + /// Called when an upon-rule fires. + pub upon_rule: Box>, + /// Called when the local process changes round. + pub round_change: Box>, + /// Called when an unjustified message is rejected. + pub unjust: Box>, } /// Defines the consensus system parameters that are external to the qbft /// algorithm. This remains constant across multiple instances of consensus /// (calls to `run`). -pub struct Definition -where - V: PartialEq, -{ +pub struct Definition { /// A deterministic leader election function. - pub is_leader: - Box bool + Send + Sync>, + pub is_leader: Box>, /// Returns a new timer channel and stop function for the round - pub new_timer: Box< - dyn Fn(/* round */ i64) -> (mpmc::Receiver, Box) - + Send - + Sync, - >, + pub new_timer: Box Timer + Send + Sync>, /// Charon parity hook called when the leader proposes a value. The core /// algorithm only runs this callback and reacts to its result; any /// value-source comparison policy belongs to the caller. - pub compare: sync::Arc>, + pub compare: sync::Arc>, /// Called when consensus has been reached on a value. - pub decide: Box< - dyn Fn( - /* ct */ &CancellationToken, - /* instance */ &I, - /* value */ &V, - /* qcommit */ &Vec>, - ) + Send - + Sync, - >, - - /// Allows debug logging of triggered upon rules on message receipt. - /// It includes the rule that triggered it and all received round messages. - pub log_upon_rule: Box< - dyn Fn( - /* instance */ &I, - /* process */ i64, - /* round */ i64, - /* msg */ &Msg, - /* upon_rule */ UponRule, - ) + Send - + Sync, - >, - /// Allows debug logging of round changes. - pub log_round_change: Box< - dyn Fn( - /* instance */ &I, - /* process */ i64, - /* round */ i64, - /* new_round */ i64, - /* upon_rule */ UponRule, - /* msgs */ &Vec>, - ) + Send - + Sync, - >, - - /// Allows debug logging of unjust messages. - pub log_unjust: - Box) + Send + Sync>, + pub decide: Box>, + + /// Debug logging callbacks. + pub logger: QbftLogger, /// Total number of nodes/processes participating in consensus. pub nodes: i64, @@ -179,20 +145,36 @@ where pub fifo_limit: i64, } -impl Definition -where - V: PartialEq, -{ +impl Definition { /// Quorum count for the system. /// See IBFT 2.0 paper for correct formula: pub fn quorum(&self) -> i64 { - (self.nodes as u64 * 2).div_ceil(3) as i64 + self.nodes + .checked_mul(2) + .and_then(|nodes| nodes.checked_add(2)) + .and_then(|nodes| nodes.checked_div(3)) + .expect("node count permits quorum calculation") } /// Maximum number of faulty/byzantine nodes supported in the system. /// See IBFT 2.0 paper for correct formula: pub fn faulty(&self) -> i64 { - (self.nodes - 1) / 3 + self.nodes + .checked_sub(1) + .and_then(|nodes| nodes.checked_div(3)) + .expect("node count permits faulty-node calculation") + } + + fn quorum_count(&self) -> usize { + usize::try_from(self.quorum()).expect("quorum fits usize") + } + + fn faulty_plus_one_count(&self) -> usize { + let threshold = self + .faulty() + .checked_add(1) + .expect("faulty-node count permits threshold calculation"); + usize::try_from(threshold).expect("faulty-node threshold fits usize") } } @@ -202,11 +184,17 @@ pub struct MessageType(i64); // NOTE: message type ordering MUST not change, since it breaks backwards // compatibility. +/// Unknown message type. pub const MSG_UNKNOWN: MessageType = MessageType(0); +/// PRE-PREPARE message type. pub const MSG_PRE_PREPARE: MessageType = MessageType(1); +/// PREPARE message type. pub const MSG_PREPARE: MessageType = MessageType(2); +/// COMMIT message type. pub const MSG_COMMIT: MessageType = MessageType(3); +/// ROUND-CHANGE message type. pub const MSG_ROUND_CHANGE: MessageType = MessageType(4); +/// DECIDED catch-up message type. pub const MSG_DECIDED: MessageType = MessageType(5); const MSG_SENTINEL: MessageType = MessageType(6); // intentionally not public @@ -235,48 +223,54 @@ impl Display for MessageType { } /// Defines the inter process messages. -pub trait SomeMsg: Send + Sync + fmt::Debug -where - V: PartialEq, -{ +pub trait SomeMsg: Send + Sync + fmt::Debug { /// Type of the message. fn type_(&self) -> MessageType; /// Consensus instance. - fn instance(&self) -> I; + fn instance(&self) -> T::Instance; /// Process that sent the message. fn source(&self) -> i64; /// The round the message pertains to. fn round(&self) -> i64; /// The value being proposed, usually a hash. - fn value(&self) -> V; + fn value(&self) -> T::Value; /// Usually the value that was hashed and is returned in `value`. - fn value_source(&self) -> Result; + fn value_source(&self) -> Result; /// The justified prepared round. fn prepared_round(&self) -> i64; /// The justified prepared value. - fn prepared_value(&self) -> V; + fn prepared_value(&self) -> T::Value; /// Set of messages that explicitly justifies this message. - fn justification(&self) -> Vec>; + fn justification(&self) -> Vec>; /// Cast as `Any` to allow downcasting. fn as_any(&self) -> &dyn any::Any; } /// Alias for any `Msg` implementation tracked by reference counting. -pub type Msg = sync::Arc>; +pub type Msg = sync::Arc>; /// Defines the event based rules that are triggered when messages are received. #[derive(PartialEq, Eq, Hash, Clone, Copy)] pub struct UponRule(i64); +/// No upon-rule fired. pub const UPON_NOTHING: UponRule = UponRule(0); +/// PRE-PREPARE was justified. pub const UPON_JUSTIFIED_PRE_PREPARE: UponRule = UponRule(1); +/// Quorum PREPARE messages was received. pub const UPON_QUORUM_PREPARES: UponRule = UponRule(2); +/// Quorum COMMIT messages was received. pub const UPON_QUORUM_COMMITS: UponRule = UponRule(3); +/// Quorum ROUND-CHANGE messages was received but not justified. pub const UPON_UNJUST_QUORUM_ROUND_CHANGES: UponRule = UponRule(4); +/// F+1 future ROUND-CHANGE messages was received. pub const UPON_F_PLUS1_ROUND_CHANGES: UponRule = UponRule(5); +/// Quorum ROUND-CHANGE messages was received. pub const UPON_QUORUM_ROUND_CHANGES: UponRule = UponRule(6); +/// DECIDED message was justified. pub const UPON_JUSTIFIED_DECIDED: UponRule = UponRule(7); +/// Round timer expired. pub const UPON_ROUND_TIMEOUT: UponRule = UponRule(8); // This is not triggered by a message, but by a timer. impl Display for UponRule { @@ -311,76 +305,75 @@ struct DedupKey { /// remains active so it can answer later `ROUND_CHANGE` messages with `DECIDED` /// catch-up messages. /// -/// `I` identifies the consensus instance, `V` is the comparable proposed value, -/// and `C` is the application value used by `Definition::compare` to compare a -/// leader proposal with the local input source. -pub fn run( +/// `T::Instance` identifies the consensus instance, `T::Value` is the +/// comparable proposed value, and `T::Compare` is the application value used by +/// `Definition::compare` to compare a leader proposal with the local input +/// source. +pub fn run( ct: &CancellationToken, - d: &Definition, - t: &Transport, - instance: &I, + d: &Definition, + t: &Transport, + instance: &T::Instance, process: i64, - mut input_value_ch: mpmc::Receiver, - input_value_source_ch: mpmc::Receiver, -) -> Result<()> -where - I: Send + Sync + 'static, - V: PartialEq + Eq + Hash + Default + 'static, - C: Clone + Send + Sync + Default + 'static, -{ + mut input_value_ch: mpmc::Receiver, + input_value_source_ch: mpmc::Receiver, +) -> Result<()> { validate_definition(d)?; + let fifo_limit = usize::try_from(d.fifo_limit).expect("validated FIFO limit fits usize"); // === State === let round: Cell = Cell::new(1); - let input_value: RefCell = RefCell::new(Default::default()); - let mut input_value_source: C = Default::default(); - let ppj_cache: RefCell>>> = RefCell::new(None); // Cached pre-prepare justification for the current round (`None` value is unset). + let input_value: RefCell = RefCell::new(Default::default()); + let mut input_value_source: T::Compare = Default::default(); + let ppj_cache: RefCell>>> = RefCell::new(None); // Cached pre-prepare justification for the current round (`None` value is unset). let prepared_round: Cell = Cell::new(0); - let prepared_value: RefCell = RefCell::new(Default::default()); + let prepared_value: RefCell = RefCell::new(Default::default()); let mut compare_failure_round: i64 = 0; - let prepared_justification: RefCell>>> = RefCell::new(None); - let mut q_commit: Option>> = None; - let buffer: RefCell>>> = RefCell::new(HashMap::new()); - let dedup_rules: RefCell> = RefCell::new(HashMap::new()); + let prepared_justification: RefCell>>> = RefCell::new(None); + let mut q_commit: Option>> = None; + let buffer: RefCell>>> = RefCell::new(HashMap::new()); + let dedup_rules: RefCell> = RefCell::new(HashSet::new()); let mut timer_chan: mpmc::Receiver; - let mut stop_timer: Box; + let mut stop_timer: Box; // === Helpers == // Broadcasts a non-ROUND-CHANGE message for current round. let broadcast_msg = - |type_: MessageType, value: &V, justification: Option<&Vec>>| { - (t.broadcast)( + |type_: MessageType, value: &T::Value, justification: Option<&Vec>>| { + let default_value = T::Value::default(); + (t.broadcast)(BroadcastRequest { ct, type_, instance, - process, - round.get(), + source: process, + round: round.get(), value, - 0, - &Default::default(), + prepared_round: 0, + prepared_value: &default_value, justification, - ) + }) }; // Broadcasts a ROUND-CHANGE message with current state. let broadcast_round_change = || { - (t.broadcast)( + let default_value = T::Value::default(); + (t.broadcast)(BroadcastRequest { ct, - MSG_ROUND_CHANGE, + type_: MSG_ROUND_CHANGE, instance, - process, - round.get(), - &Default::default(), - prepared_round.get(), - &prepared_value.borrow(), - prepared_justification.borrow().as_ref(), - ) + source: process, + round: round.get(), + value: &default_value, + prepared_round: prepared_round.get(), + prepared_value: &prepared_value.borrow(), + justification: prepared_justification.borrow().as_ref(), + }) }; // Broadcasts a PRE-PREPARE message with current state // and our own input value if present, otherwise it caches the justification // to be used when the input value becomes available. - let broadcast_own_pre_prepare = |justification: Vec>| { + let broadcast_own_pre_prepare = |justification: Vec>| { if ppj_cache.borrow().is_some() { panic!("bug: justification cache must be none") } @@ -395,13 +388,17 @@ where }; // Adds a message to each process' FIFO queue - let buffer_msg = |msg: &Msg| { + let buffer_msg = |msg: &Msg| { let mut b = buffer.borrow_mut(); let fifo = b.entry(msg.source()).or_default(); fifo.push(msg.clone()); - if fifo.len() as i64 > d.fifo_limit { - fifo.drain(0..(fifo.len() - d.fifo_limit as usize)); + if fifo.len() > fifo_limit { + let expired = fifo + .len() + .checked_sub(fifo_limit) + .expect("FIFO length exceeds limit"); + fifo.drain(0..expired); } }; @@ -409,7 +406,7 @@ where // change. let is_duplicated_rule = |upon_rule: UponRule, round: i64| { let k = DedupKey { upon_rule, round }; - dedup_rules.borrow_mut().insert(k, true).is_some() + !dedup_rules.borrow_mut().insert(k) }; // Updates round and clears the rule dedup state. @@ -418,28 +415,34 @@ where return; } - (d.log_round_change)( + (d.logger.round_change)(RoundChangeLog { instance, process, - round.get(), + round: round.get(), new_round, - rule, - &extract_round_messages(&buffer.borrow(), round.get()), - ); + upon_rule: rule, + msgs: &extract_round_messages(&buffer.borrow(), round.get()), + }); round.set(new_round); - dedup_rules.replace(HashMap::new()); + dedup_rules.replace(HashSet::new()); ppj_cache.replace(None); }; // Algorithm 1:11 { - if (d.is_leader)(instance, round.get(), process) { + if (d.is_leader)(LeaderRequest { + instance, + round: round.get(), + process, + }) { // Note round==1 at this point. broadcast_own_pre_prepare(vec![])?; // Empty justification since round==1 } - (timer_chan, stop_timer) = (d.new_timer)(round.get()); + let timer = (d.new_timer)(round.get()); + timer_chan = timer.receive; + stop_timer = timer.stop; } loop { @@ -468,20 +471,24 @@ where recv(t.receive) -> result => { let msg = result?; - if let Some(v) = q_commit.as_ref() { - if !v.is_empty() { - if msg.source() != process && msg.type_() == MSG_ROUND_CHANGE { - // Algorithm 3:17 - broadcast_msg(MSG_DECIDED, &v[0].value(), Some(v))?; - } - - continue; + if let Some(v) = q_commit.as_ref() + && !v.is_empty() + { + if msg.source() != process && msg.type_() == MSG_ROUND_CHANGE { + // Algorithm 3:17 + broadcast_msg(MSG_DECIDED, &v[0].value(), Some(v))?; } + + continue; } // Drop unjust messages if !is_justified(d, instance, &msg, compare_failure_round) { - (d.log_unjust)(instance, process, msg); + (d.logger.unjust)(UnjustLog { + instance, + process, + msg, + }); continue; } @@ -494,7 +501,13 @@ where continue; } - (d.log_upon_rule)(instance, process, round.get(), &msg, rule); + (d.logger.upon_rule)(UponRuleLog { + instance, + process, + round: round.get(), + msg: &msg, + upon_rule: rule, + }); match rule { // Algorithm 2:1 @@ -502,7 +515,9 @@ where change_round(msg.round(), rule); stop_timer(); - (timer_chan, stop_timer) = (d.new_timer)(round.get()); + let timer = (d.new_timer)(round.get()); + timer_chan = timer.receive; + stop_timer = timer.stop; let (new_input_value_source, compare_result) = compare( ct, @@ -526,10 +541,16 @@ where // might timeout in the meantime. If // this happens, we trigger round change. // Algorithm 3:1 - change_round(round.get() + 1, UPON_ROUND_TIMEOUT); + let next_round = round + .get() + .checked_add(1) + .expect("round permits increment"); + change_round(next_round, UPON_ROUND_TIMEOUT); stop_timer(); - (timer_chan, stop_timer) = (d.new_timer)(round.get()); + let timer = (d.new_timer)(round.get()); + timer_chan = timer.receive; + stop_timer = timer.stop; broadcast_round_change()?; } @@ -562,7 +583,12 @@ where let justification = q_commit.as_ref() .expect("Rules `UPON_QUORUM_COMMITS` and `UPON_JUSTIFIED_DECIDED` always include a justification"); - (d.decide)(ct, instance, &msg.value(), justification); + (d.decide)(DecideRequest { + ct, + instance, + value: &msg.value(), + qcommit: justification, + }); } UPON_F_PLUS1_ROUND_CHANGES => { // Algorithm 3:5 @@ -578,7 +604,9 @@ where ); stop_timer(); - (timer_chan, stop_timer) = (d.new_timer)(round.get()); + let timer = (d.new_timer)(round.get()); + timer_chan = timer.receive; + stop_timer = timer.stop; broadcast_round_change()?; } @@ -606,10 +634,16 @@ where recv(timer_chan) -> result => { result?; - change_round(round.get() + 1, UPON_ROUND_TIMEOUT); + let next_round = round + .get() + .checked_add(1) + .expect("round permits increment"); + change_round(next_round, UPON_ROUND_TIMEOUT); stop_timer(); - (timer_chan, stop_timer) = (d.new_timer)(round.get()); + let timer = (d.new_timer)(round.get()); + timer_chan = timer.receive; + stop_timer = timer.stop; broadcast_round_change()?; } @@ -623,10 +657,7 @@ where } } -fn validate_definition(d: &Definition) -> Result<()> -where - V: PartialEq, -{ +fn validate_definition(d: &Definition) -> Result<()> { if d.nodes <= 0 { return Err(QbftError::InvalidNodes { nodes: d.nodes }); } @@ -643,21 +674,16 @@ where /// The callback may cache the local input source and return success/failure. /// This helper only preserves that callback result and lets the round timer win /// if the callback blocks. -fn compare( +fn compare( ct: &CancellationToken, - d: &Definition, - msg: &Msg, - input_value_source_ch: &mpmc::Receiver, - input_value_source: C, + d: &Definition, + msg: &Msg, + input_value_source_ch: &mpmc::Receiver, + input_value_source: T::Compare, timer_chan: &mpmc::Receiver, -) -> (C, Result<()>) -where - I: Send + Sync + 'static, - V: PartialEq + 'static, - C: Clone + Send + Sync + 'static, -{ +) -> (T::Compare, Result<()>) { let (compare_err_tx, mut compare_err_rx) = mpmc::bounded::>(1); - let (compare_value_tx, mut compare_value_rx) = mpmc::bounded::(1); + let (compare_value_tx, mut compare_value_rx) = mpmc::bounded::(1); // d.Compare has 2 roles: // 1. Read from the `input_value_source_ch` (if `input_value_source` is empty). @@ -680,14 +706,14 @@ where // caller-provided compare callback ignores cancellation and never reports, // it may outlive this call. thread::spawn(move || { - (compare)( - &compare_ct, - &msg, - &input_value_source_ch, - &input_value_source, - &compare_err_tx, - &compare_value_tx, - ); + (compare)(CompareRequest { + ct: &compare_ct, + qcommit: &msg, + input_value_source_ch: &input_value_source_ch, + input_value_source: &input_value_source, + return_err: &compare_err_tx, + return_value: &compare_value_tx, + }); }); loop { @@ -748,13 +774,10 @@ where } /// Returns all messages from the provided round. -fn extract_round_messages( - buffer: &HashMap>>, +fn extract_round_messages( + buffer: &HashMap>>, round: i64, -) -> Vec> -where - V: PartialEq, -{ +) -> Vec> { let mut resp = vec![]; for msgs in buffer.values() { @@ -770,17 +793,14 @@ where /// Returns the rule triggered upon receipt of the last message and its /// justifications. -fn classify( - d: &Definition, - instance: &I, +fn classify( + d: &Definition, + instance: &T::Instance, round: i64, process: i64, - buffer: &HashMap>>, - msg: &Msg, -) -> (UponRule, Option>>) -where - V: Eq + Hash + Default, -{ + buffer: &HashMap>>, + msg: &Msg, +) -> (UponRule, Option>>) { match msg.type_() { MSG_DECIDED => (UPON_JUSTIFIED_DECIDED, Some(msg.justification())), MSG_PRE_PREPARE => { @@ -799,7 +819,7 @@ where let prepares = filter_by_round_and_value(&flatten(buffer), MSG_PREPARE, msg.round(), msg.value()); - if prepares.len() as i64 >= d.quorum() { + if prepares.len() >= d.quorum_count() { (UPON_QUORUM_PREPARES, Some(prepares)) } else { (UPON_NOTHING, None) @@ -813,7 +833,7 @@ where let commits = filter_by_round_and_value(&flatten(buffer), MSG_COMMIT, msg.round(), msg.value()); - if commits.len() as i64 >= d.quorum() { + if commits.len() >= d.quorum_count() { (UPON_QUORUM_COMMITS, Some(commits)) } else { (UPON_NOTHING, None) @@ -839,7 +859,7 @@ where /* else msg.round() == round */ let qrc = filter_round_change(&all, msg.round()); - if (qrc.len() as i64) < d.quorum() { + if qrc.len() < d.quorum_count() { return (UPON_NOTHING, None); } @@ -847,7 +867,11 @@ where return (UPON_UNJUST_QUORUM_ROUND_CHANGES, None); }; - if !(d.is_leader)(instance, msg.round(), process) { + if !(d.is_leader)(LeaderRequest { + instance, + round: msg.round(), + process, + }) { return (UPON_NOTHING, None); } @@ -861,12 +885,9 @@ where /// Implements algorithm 3:6 and returns the next minimum round from received /// round change messages. -fn next_min_round(d: &Definition, frc: &Vec>, round: i64) -> i64 -where - V: PartialEq, -{ +fn next_min_round(d: &Definition, frc: &Vec>, round: i64) -> i64 { // Get all RoundChange messages with round (rj) higher than current round (ri) - if (frc.len() as i64) < d.faulty() + 1 { + if frc.len() < d.faulty_plus_one_count() { panic!("bug: Frc too short"); } @@ -889,15 +910,12 @@ where } /// Returns true if message is justified or if it does not need justification. -fn is_justified( - d: &Definition, - instance: &I, - msg: &Msg, +fn is_justified( + d: &Definition, + instance: &T::Instance, + msg: &Msg, compare_failure_round: i64, -) -> bool -where - V: Eq + Hash + Default, -{ +) -> bool { match msg.type_() { MSG_PRE_PREPARE => is_justified_pre_prepare(d, instance, msg, compare_failure_round), MSG_PREPARE => true, @@ -910,10 +928,7 @@ where /// Returns true if the ROUND_CHANGE message's prepared round and value is /// justified. -fn is_justified_round_change(d: &Definition, msg: &Msg) -> bool -where - V: PartialEq + Default, -{ +fn is_justified_round_change(d: &Definition, msg: &Msg) -> bool { if msg.type_() != MSG_ROUND_CHANGE { panic!("bug: not a round change message"); } @@ -938,11 +953,11 @@ where // No need to check for all possible combinations, since justified should only // contain a one. - if (prepares.len() as i64) < d.quorum() { + if prepares.len() < d.quorum_count() { return false; } - let mut uniq = uniq_source::(vec![]); + let mut uniq = uniq_source(); for prepare in prepares { if !uniq(&prepare) { return false; @@ -964,20 +979,14 @@ where true } -fn valid_round_change_prepared_round(msg: &Msg) -> bool -where - V: PartialEq, -{ +fn valid_round_change_prepared_round(msg: &Msg) -> bool { let pr = msg.prepared_round(); pr >= 0 && pr < msg.round() } /// Returns true if the decided message is justified by quorum COMMIT messages /// of identical round and value. -fn is_justified_decided(d: &Definition, msg: &Msg) -> bool -where - V: PartialEq, -{ +fn is_justified_decided(d: &Definition, msg: &Msg) -> bool { if msg.type_() != MSG_DECIDED { panic!("bug: not a decided message"); } @@ -992,30 +1001,34 @@ where None, ); - (commits.len() as i64) >= d.quorum() + commits.len() >= d.quorum_count() } /// Returns true if the PRE-PREPARE message is justified. -fn is_justified_pre_prepare( - d: &Definition, - instance: &I, - msg: &Msg, +fn is_justified_pre_prepare( + d: &Definition, + instance: &T::Instance, + msg: &Msg, compare_failure_round: i64, -) -> bool -where - V: Eq + Hash + Default, -{ +) -> bool { if msg.type_() != MSG_PRE_PREPARE { panic!("bug: not a preprepare message"); } - if !(d.is_leader)(instance, msg.round(), msg.source()) { + if !(d.is_leader)(LeaderRequest { + instance, + round: msg.round(), + process: msg.source(), + }) { return false; } // Justified if PrePrepare is the first round OR if comparison failed previous // round. - if msg.round() == 1 || (msg.round() == compare_failure_round + 1) { + let next_compare_round = compare_failure_round + .checked_add(1) + .expect("compare failure round permits increment"); + if msg.round() == 1 || (msg.round() == next_compare_round) { return true; } @@ -1032,19 +1045,16 @@ where /// Implements algorithm 4:1 and returns true and pv if the messages contains a /// justified quorum ROUND_CHANGEs (Qrc). -fn contains_justified_qrc( - d: &Definition, - justification: &Vec>, +fn contains_justified_qrc( + d: &Definition, + justification: &Vec>, round: i64, -) -> Option -where - V: Eq + Hash + Default, -{ +) -> Option { let qrc = filter_round_change(justification, round) .into_iter() .filter(valid_round_change_prepared_round) .collect::>(); - if (qrc.len() as i64) < d.quorum() { + if qrc.len() < d.quorum_count() { return None; } @@ -1089,17 +1099,14 @@ where /// Extracts the single justified Pr and Pv from quorum PREPARES in list of /// messages. It expects only one possible combination. -fn get_single_justified_pr_pv( - d: &Definition, - msgs: &Vec>, -) -> Option<(i64, V)> -where - V: Eq + Hash + Default, -{ +fn get_single_justified_pr_pv( + d: &Definition, + msgs: &Vec>, +) -> Option<(i64, T::Value)> { let mut pr: i64 = 0; - let mut pv: V = Default::default(); - let mut count: i64 = 0; - let mut uniq = uniq_source::(vec![]); + let mut pv: T::Value = Default::default(); + let mut count: usize = 0; + let mut uniq = uniq_source(); for msg in msgs { if msg.type_() != MSG_PREPARE { @@ -1117,10 +1124,12 @@ where return None; } - count += 1; + count = count + .checked_add(1) + .expect("prepare count permits increment"); } - if count >= d.quorum() { + if count >= d.quorum_count() { Some((pr, pv)) } else { None @@ -1128,14 +1137,11 @@ where } /// Implements algorithm 4:1 and returns a justified quorum ROUND_CHANGEs (Qrc) -fn get_justified_qrc( - d: &Definition, - all: &Vec>, +fn get_justified_qrc( + d: &Definition, + all: &Vec>, round: i64, -) -> Option>> -where - V: Eq + Hash + Default, -{ +) -> Option>> { if let (qrc, true) = quorum_null_prepared(d, all, round) { // Return any quorum null pv ROUND_CHANGE messages as Qrc. return Some(qrc); @@ -1149,11 +1155,11 @@ where for prepares in get_prepare_quorums(d, all) { // See if we have quorum ROUND-CHANGE with HIGHEST_PREPARED(qrc) == // prepares.Round. - let mut qrc: Vec> = vec![]; + let mut qrc: Vec> = vec![]; let mut has_highest_prepared = false; let pr = prepares[0].round(); let pv = prepares[0].value(); - let mut uniq = uniq_source::(vec![]); + let mut uniq = uniq_source(); for rc in round_changes.iter() { if rc.prepared_round() > pr { @@ -1171,7 +1177,7 @@ where qrc.push(rc.clone()); } - if (qrc.len() as i64) >= d.quorum() && has_highest_prepared { + if qrc.len() >= d.quorum_count() && has_highest_prepared { qrc.extend(prepares); return Some(qrc); } @@ -1183,15 +1189,12 @@ where /// Returns true and Faulty+1 ROUND-CHANGE messages (Frc) with the rounds higher /// than the provided round. It returns the highest round per process in order /// to jump furthest. -fn get_fplus1_round_changes( - d: &Definition, - all: &Vec>, +fn get_fplus1_round_changes( + d: &Definition, + all: &Vec>, round: i64, -) -> Option>> -where - V: PartialEq, -{ - let mut highest_by_source = HashMap::>::new(); +) -> Option>> { + let mut highest_by_source = HashMap::>::new(); for msg in all { if msg.type_() != MSG_ROUND_CHANGE { @@ -1202,20 +1205,20 @@ where continue; } - if let Some(highest) = highest_by_source.get(&msg.source()) { - if highest.round() > msg.round() { - continue; - } + if let Some(highest) = highest_by_source.get(&msg.source()) + && highest.round() > msg.round() + { + continue; } highest_by_source.insert(msg.source(), msg.clone()); - if (highest_by_source.len() as i64) == d.faulty() + 1 { + if highest_by_source.len() == d.faulty_plus_one_count() { break; } } - if (highest_by_source.len() as i64) < d.faulty() + 1 { + if highest_by_source.len() < d.faulty_plus_one_count() { return None; } @@ -1236,14 +1239,8 @@ where /// Returns all unique-source PREPARE quorums grouped by identical round and /// value. -fn get_prepare_quorums( - d: &Definition, - all: &Vec>, -) -> Vec>> -where - V: Eq + Hash, -{ - let mut sets = HashMap::, HashMap>>::new(); +fn get_prepare_quorums(d: &Definition, all: &Vec>) -> Vec>> { + let mut sets = HashMap::, HashMap>>::new(); for msg in all { if msg.type_() != MSG_PREPARE { @@ -1263,16 +1260,11 @@ where let mut quorums = vec![]; for (_, msgs) in sets { - if (msgs.len() as i64) < d.quorum() { + if msgs.len() < d.quorum_count() { continue; } - let mut quorum = vec![]; - for (_, msg) in msgs { - quorum.push(msg); - } - - quorums.push(quorum); + quorums.push(msgs.into_values().collect()); } quorums @@ -1281,61 +1273,47 @@ where /// Implements condition J1 and returns Qrc and true if a quorum /// of round changes messages (Qrc) for the round have null prepared round and /// value. -fn quorum_null_prepared( - d: &Definition, - all: &Vec>, +fn quorum_null_prepared( + d: &Definition, + all: &Vec>, round: i64, -) -> (Vec>, bool) -where - V: PartialEq + Default, -{ +) -> (Vec>, bool) { let null_pr = Default::default(); let null_pv = Some(&Default::default()); let justification = filter_msgs(all, MSG_ROUND_CHANGE, round, None, Some(null_pr), null_pv); + let has_quorum = justification.len() >= d.quorum_count(); - ( - justification.clone(), - justification.len() as i64 >= d.quorum(), - ) + (justification, has_quorum) } /// Returns the messages matching the type and value. -fn filter_by_round_and_value( - msgs: &Vec>, +fn filter_by_round_and_value( + msgs: &Vec>, message_type: MessageType, round: i64, - value: V, -) -> Vec> -where - V: PartialEq, -{ + value: T::Value, +) -> Vec> { filter_msgs(msgs, message_type, round, Some(&value), None, None) } /// Returns all round change messages for the provided round. -fn filter_round_change(msgs: &Vec>, round: i64) -> Vec> -where - V: PartialEq, -{ - filter_msgs::(msgs, MSG_ROUND_CHANGE, round, None, None, None) +fn filter_round_change(msgs: &Vec>, round: i64) -> Vec> { + filter_msgs::(msgs, MSG_ROUND_CHANGE, round, None, None, None) } /// Returns one message per process matching the provided type and round and /// optional value, pr, pv. -fn filter_msgs( - msgs: &Vec>, +fn filter_msgs( + msgs: &Vec>, message_type: MessageType, round: i64, - value: Option<&V>, + value: Option<&T::Value>, pr: Option, - pv: Option<&V>, -) -> Vec> -where - V: PartialEq, -{ + pv: Option<&T::Value>, +) -> Vec> { let mut resp = Vec::new(); - let mut uniq = uniq_source::(vec![]); + let mut uniq = uniq_source(); for msg in msgs { if message_type != msg.type_() { @@ -1374,11 +1352,8 @@ where /// Produce a vector containing all the buffered messages as well as all their /// justifications. -fn flatten(buffer: &HashMap>>) -> Vec> -where - V: PartialEq, -{ - let mut resp: Vec> = Vec::new(); +fn flatten(buffer: &HashMap>>) -> Vec> { + let mut resp: Vec> = Vec::new(); for msgs in buffer.values() { for msg in msgs { @@ -1397,20 +1372,9 @@ where /// Construct a function that returns true if the message is from a unique /// source. -fn uniq_source(vec: Vec>) -> Box) -> bool> -where - V: PartialEq, -{ - let mut s = vec.iter().map(|msg| msg.source()).collect::>(); - Box::new(move |msg: &Msg| { - let source = msg.source(); - if s.contains(&source) { - false - } else { - s.insert(source); - true - } - }) +fn uniq_source() -> impl FnMut(&Msg) -> bool { + let mut sources = HashSet::new(); + move |msg: &Msg| sources.insert(msg.source()) } #[cfg(test)] From dfa5eccaecd75bbcdce6a94cd4707dc7fdef3f66 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 13:09:20 -0300 Subject: [PATCH 20/34] Adjust deadliner API --- crates/core/src/aggsigdb/memory.rs | 92 ++++++++++++++---------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index ac10c8fe..339ee9cb 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,8 +1,5 @@ -use crate::{deadline::Deadliner, types}; -use std::{ - collections::{HashMap, hash_map::Entry}, - sync::Arc, -}; +use crate::{deadline::DeadlinerHandle, types}; +use std::collections::{HashMap, hash_map::Entry}; use tokio::sync; /// Errors for the in-memory AggSigDB implementation. @@ -20,7 +17,7 @@ type Waiters = struct Actor { entries: HashMap>>, waiters: Waiters, - deadliner: Arc, + deadliner: DeadlinerHandle, } impl Actor { @@ -138,15 +135,15 @@ impl Handle { /// Creates a new in-memory AggSigDB instance, and get a handle to it. /// /// The underlying instance gets dropped when all handles are dropped. - pub fn new(deadliner: Arc) -> Self { + pub fn new(deadliner: DeadlinerHandle, evictions: sync::mpsc::Receiver) -> Self { let (sender, receiver) = sync::mpsc::channel(100); let mut actor = Actor { entries: HashMap::new(), waiters: HashMap::new(), - deadliner: Arc::clone(&deadliner), + deadliner, }; - let (_, never) = sync::mpsc::channel(1); - tokio::spawn(async move { actor.run(receiver, deadliner.c().unwrap_or(never)).await }); + + tokio::spawn(async move { actor.run(receiver, evictions).await }); Self { sender } } @@ -186,13 +183,12 @@ impl Handle { #[cfg(test)] mod tests { use crate::{ - deadline::Deadliner, + deadline, signeddata::SignedDataError, types::{Duty, PubKey, Signature, SignedData, SignedDataSet, SlotNumber}, }; - use async_trait::async_trait; - use std::sync::Arc; use tokio::sync; + use tokio_util::sync::CancellationToken; /// Some mock signed data type for testing. #[derive(Debug, Clone, PartialEq, Eq)] @@ -200,13 +196,20 @@ mod tests { impl SignedData for MockSignedData { fn signature(&self) -> Result { - Ok(Signature::new([self.0; 96])) + Ok([self.0; 96]) } fn set_signature(&self, _signature: Signature) -> Result { Ok(self.clone()) } + fn set_signature_boxed( + &self, + signature: Signature, + ) -> Result, SignedDataError> { + Ok(Box::new(self.set_signature(signature)?)) + } + fn message_root(&self) -> Result<[u8; 32], SignedDataError> { Ok([self.0; 32]) } @@ -224,36 +227,28 @@ mod tests { } } - /// Deadliner that hands out a caller-supplied receiver, allowing tests to - /// drive eviction by sending on the paired sender. - struct TestDeadliner(std::sync::Mutex>>); + struct TestDeadliner; impl TestDeadliner { - fn new(receiver: sync::mpsc::Receiver) -> Arc { - Arc::new(Self(std::sync::Mutex::new(Some(receiver)))) + // Creates a deadliner that immediately schedules the Duty for eviction + fn immediate() -> (deadline::DeadlinerHandle, sync::mpsc::Receiver) { + todo!() } - /// Creates a deadliner that never returns any duties to evict, so no - /// eviction will occur. - fn never() -> Arc { - Arc::new(Self(std::sync::Mutex::new(None))) - } - } - - #[async_trait] - impl Deadliner for TestDeadliner { - async fn add(&self, _duty: Duty) -> bool { - true - } - - fn c(&self) -> Option> { - self.0.lock().unwrap().take() + // Creates a deadliner that never returns any duties to evict + fn never() -> (deadline::DeadlinerHandle, sync::mpsc::Receiver) { + deadline::DeadlinerTask::start( + CancellationToken::new(), + "", + deadline::NeverExpiringCalculator, + ) } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_read() { - let store = super::Handle::new(TestDeadliner::never()); + let (deadliner, evictions) = TestDeadliner::never(); + let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -270,8 +265,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_unblocks() { - let deadliner = TestDeadliner::never(); - let store = super::Handle::new(deadliner); + let (deadliner, evictions) = TestDeadliner::never(); + let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -299,7 +294,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cannot_overwrite() { - let store = super::Handle::new(TestDeadliner::never()); + let (deadliner, evictions) = TestDeadliner::never(); + let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -320,7 +316,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_idempotent() { - let store = super::Handle::new(TestDeadliner::never()); + let (deadliner, evictions) = TestDeadliner::never(); + let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -341,10 +338,9 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_evict_wait_then_write() { - let (evict_tx, evict_rx) = sync::mpsc::channel::(1); - let deadliner = TestDeadliner::new(evict_rx); + let (deadliner, evictions) = TestDeadliner::immediate(); - let store = super::Handle::new(deadliner); + let store = super::Handle::new(deadliner.clone(), evictions); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -357,11 +353,9 @@ mod tests { .unwrap(); // Wait until the eviction is processed and the duty is removed. - evict_tx.send(duty.clone()).await.unwrap(); + deadliner.add(duty.clone()).await; tokio::time::timeout(std::time::Duration::from_secs(2), async { - while evict_tx.capacity() != evict_tx.max_capacity() { - tokio::task::yield_now().await; - } + todo!("Find a way to ensure that the eviction has been processed"); }) .await .expect("Eviction did not complete in time"); @@ -391,7 +385,8 @@ mod tests { async fn write_unblocks_many() { const N: usize = 4; - let store = super::Handle::new(TestDeadliner::never()); + let (deadliner, evictions) = TestDeadliner::never(); + let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); let signed_data = MockSignedData(42); @@ -427,7 +422,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unrelated_write_does_not_unblock() { - let store = super::Handle::new(TestDeadliner::never()); + let (deadliner, evictions) = TestDeadliner::never(); + let store = super::Handle::new(deadliner, evictions); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); let data_a = MockSignedData(1); From 187b64e4b263b0c13cfb87743d56ce5237794c74 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 15:57:58 -0300 Subject: [PATCH 21/34] Adjust tests to use new API --- crates/core/src/aggsigdb/memory.rs | 45 +++++++++++++----------------- crates/core/src/deadline/mod.rs | 18 ++++++++++++ 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 339ee9cb..af4c741c 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -188,7 +188,6 @@ mod tests { types::{Duty, PubKey, Signature, SignedData, SignedDataSet, SlotNumber}, }; use tokio::sync; - use tokio_util::sync::CancellationToken; /// Some mock signed data type for testing. #[derive(Debug, Clone, PartialEq, Eq)] @@ -227,27 +226,21 @@ mod tests { } } - struct TestDeadliner; - - impl TestDeadliner { - // Creates a deadliner that immediately schedules the Duty for eviction - fn immediate() -> (deadline::DeadlinerHandle, sync::mpsc::Receiver) { - todo!() - } + /// Create a test deadline handle and an eviction channel. + fn test_deadline() -> ( + sync::mpsc::Sender, + deadline::DeadlinerHandle, + sync::mpsc::Receiver, + ) { + let (tx, rx) = sync::mpsc::channel(1); + let deadliner = deadline::DeadlinerHandle::always(deadline::AddOutcome::Scheduled); - // Creates a deadliner that never returns any duties to evict - fn never() -> (deadline::DeadlinerHandle, sync::mpsc::Receiver) { - deadline::DeadlinerTask::start( - CancellationToken::new(), - "", - deadline::NeverExpiringCalculator, - ) - } + (tx, deadliner, rx) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_read() { - let (deadliner, evictions) = TestDeadliner::never(); + let (_, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); @@ -265,7 +258,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_unblocks() { - let (deadliner, evictions) = TestDeadliner::never(); + let (_, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_attester_duty(SlotNumber::new(1)); @@ -294,7 +287,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cannot_overwrite() { - let (deadliner, evictions) = TestDeadliner::never(); + let (_, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); @@ -316,7 +309,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_idempotent() { - let (deadliner, evictions) = TestDeadliner::never(); + let (_, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); @@ -338,7 +331,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_evict_wait_then_write() { - let (deadliner, evictions) = TestDeadliner::immediate(); + let (evict, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner.clone(), evictions); @@ -353,9 +346,11 @@ mod tests { .unwrap(); // Wait until the eviction is processed and the duty is removed. - deadliner.add(duty.clone()).await; + evict.send(duty.clone()).await.unwrap(); tokio::time::timeout(std::time::Duration::from_secs(2), async { - todo!("Find a way to ensure that the eviction has been processed"); + while evict.capacity() != evict.max_capacity() { + tokio::task::yield_now().await; + } }) .await .expect("Eviction did not complete in time"); @@ -385,7 +380,7 @@ mod tests { async fn write_unblocks_many() { const N: usize = 4; - let (deadliner, evictions) = TestDeadliner::never(); + let (_, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner, evictions); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -422,7 +417,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unrelated_write_does_not_unblock() { - let (deadliner, evictions) = TestDeadliner::never(); + let (_, deadliner, evictions) = test_deadline(); let store = super::Handle::new(deadliner, evictions); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); diff --git a/crates/core/src/deadline/mod.rs b/crates/core/src/deadline/mod.rs index fbf9ed85..dca033c2 100644 --- a/crates/core/src/deadline/mod.rs +++ b/crates/core/src/deadline/mod.rs @@ -144,6 +144,24 @@ impl DeadlinerHandle { // `FailedToCompute` if the task dropped the sender (shutdown race). response_rx.await.unwrap_or(AddOutcome::FailedToCompute) } + + /// Create a handle that always returns the given [`AddOutcome`]. + #[cfg(test)] + pub fn always(expected: AddOutcome) -> Self { + let (tx, mut rx) = mpsc::channel(1); + let handle = DeadlinerHandle { + cancel_token: CancellationToken::new(), + input_tx: tx, + }; + + tokio::spawn(async move { + while let Some(input) = rx.recv().await { + let _ = input.response_tx.send(expected); + } + }); + + handle + } } /// Owned state of the background task that drives a [`DeadlinerHandle`]'s From 970147ec0de6e5579e5c9d432f10ede114bc8211 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 16:01:32 -0300 Subject: [PATCH 22/34] Consistent naming ("eviction", "expiration") --- crates/core/src/aggsigdb/memory.rs | 51 ++++++++++++++++-------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index af4c741c..daac224b 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -23,16 +23,16 @@ struct Actor { impl Actor { async fn run( &mut self, - mut messages: sync::mpsc::Receiver, - mut evictions: sync::mpsc::Receiver, + mut messages_rx: sync::mpsc::Receiver, + mut expired_rx: sync::mpsc::Receiver, ) { loop { tokio::select! { biased; // We want to run evictions first - Some(duty) = evictions.recv() => self.evict(duty), + Some(duty) = expired_rx.recv() => self.evict(duty), - msg = messages.recv() => match msg { + msg = messages_rx.recv() => match msg { None => break, // All handles have been dropped, so we can stop the actor. Some(msg) => match msg { Message::Store { @@ -135,7 +135,10 @@ impl Handle { /// Creates a new in-memory AggSigDB instance, and get a handle to it. /// /// The underlying instance gets dropped when all handles are dropped. - pub fn new(deadliner: DeadlinerHandle, evictions: sync::mpsc::Receiver) -> Self { + pub fn new( + deadliner: DeadlinerHandle, + expiration_rx: sync::mpsc::Receiver, + ) -> Self { let (sender, receiver) = sync::mpsc::channel(100); let mut actor = Actor { entries: HashMap::new(), @@ -143,7 +146,7 @@ impl Handle { deadliner, }; - tokio::spawn(async move { actor.run(receiver, evictions).await }); + tokio::spawn(async move { actor.run(receiver, expiration_rx).await }); Self { sender } } @@ -226,7 +229,7 @@ mod tests { } } - /// Create a test deadline handle and an eviction channel. + /// Create a test deadline handle and an expiration channel. fn test_deadline() -> ( sync::mpsc::Sender, deadline::DeadlinerHandle, @@ -240,8 +243,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_read() { - let (_, deadliner, evictions) = test_deadline(); - let store = super::Handle::new(deadliner, evictions); + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -258,8 +261,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_unblocks() { - let (_, deadliner, evictions) = test_deadline(); - let store = super::Handle::new(deadliner, evictions); + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -287,8 +290,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cannot_overwrite() { - let (_, deadliner, evictions) = test_deadline(); - let store = super::Handle::new(deadliner, evictions); + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -309,8 +312,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_idempotent() { - let (_, deadliner, evictions) = test_deadline(); - let store = super::Handle::new(deadliner, evictions); + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -331,9 +334,9 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_evict_wait_then_write() { - let (evict, deadliner, evictions) = test_deadline(); + let (expiration_tx, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner.clone(), evictions); + let store = super::Handle::new(deadliner.clone(), expiration_rx); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -345,10 +348,10 @@ mod tests { .await .unwrap(); - // Wait until the eviction is processed and the duty is removed. - evict.send(duty.clone()).await.unwrap(); + // Wait until the expiration is processed and the duty is evicted. + expiration_tx.send(duty.clone()).await.unwrap(); tokio::time::timeout(std::time::Duration::from_secs(2), async { - while evict.capacity() != evict.max_capacity() { + while expiration_tx.capacity() != expiration_tx.max_capacity() { tokio::task::yield_now().await; } }) @@ -380,8 +383,8 @@ mod tests { async fn write_unblocks_many() { const N: usize = 4; - let (_, deadliner, evictions) = test_deadline(); - let store = super::Handle::new(deadliner, evictions); + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); let signed_data = MockSignedData(42); @@ -417,8 +420,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unrelated_write_does_not_unblock() { - let (_, deadliner, evictions) = test_deadline(); - let store = super::Handle::new(deadliner, evictions); + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); let data_a = MockSignedData(1); From d748f7bde952a374c93e041171d04f12c8154895 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 16:23:34 -0300 Subject: [PATCH 23/34] Handle `deadliner.add` outcomes - Proceed with store only when appropriate --- crates/core/src/aggsigdb/memory.rs | 90 ++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index daac224b..19d85a67 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,6 +1,7 @@ -use crate::{deadline::DeadlinerHandle, types}; +use crate::{deadline, types}; use std::collections::{HashMap, hash_map::Entry}; use tokio::sync; +use tokio_util::sync::CancellationToken; /// Errors for the in-memory AggSigDB implementation. #[derive(Debug, thiserror::Error)] @@ -9,6 +10,11 @@ pub enum Error { /// the new data. #[error("Mismatching data")] MismatchingData, + + /// The request cannot be processed because the instance has been + /// terminated. + #[error("The instance has been terminated")] + Terminated, } type Waiters = @@ -17,7 +23,7 @@ type Waiters = struct Actor { entries: HashMap>>, waiters: Waiters, - deadliner: DeadlinerHandle, + deadliner: deadline::DeadlinerHandle, } impl Actor { @@ -25,15 +31,18 @@ impl Actor { &mut self, mut messages_rx: sync::mpsc::Receiver, mut expired_rx: sync::mpsc::Receiver, + ct: CancellationToken, ) { loop { tokio::select! { - biased; // We want to run evictions first + biased; // We want to evaluate expirations first + + _ = ct.cancelled() => break, // Stop the actor when the cancellation token is triggered. Some(duty) = expired_rx.recv() => self.evict(duty), msg = messages_rx.recv() => match msg { - None => break, // All handles have been dropped, so we can stop the actor. + None => break, // Stop the actor when all handles have been dropped. Some(msg) => match msg { Message::Store { duty, @@ -64,10 +73,14 @@ impl Actor { } async fn store(&mut self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { - // TODO: Improve the `deadliner` API: - // - Return if the duty is already expired. If so, return early. - // - Make `add` sync to avoid an `await` which blocks the actor. - let _ = self.deadliner.add(duty.clone()).await; + match self.deadliner.add(duty.clone()).await { + // The duty is already expired, so we can ignore the store. + deadline::AddOutcome::AlreadyExpired => return Ok(()), + // Failures are treated as terminated. + deadline::AddOutcome::FailedToCompute => return Err(Error::Terminated), + // In other cases proceed with the store. + _ => {} + } // NOTE: Partial insertions on error match the semantics of Charon. let for_duty = self.entries.entry(duty.clone()).or_default(); @@ -136,8 +149,9 @@ impl Handle { /// /// The underlying instance gets dropped when all handles are dropped. pub fn new( - deadliner: DeadlinerHandle, - expiration_rx: sync::mpsc::Receiver, + deadliner: deadline::DeadlinerHandle, + expired_rx: sync::mpsc::Receiver, + ct: CancellationToken, ) -> Self { let (sender, receiver) = sync::mpsc::channel(100); let mut actor = Actor { @@ -146,7 +160,7 @@ impl Handle { deadliner, }; - tokio::spawn(async move { actor.run(receiver, expiration_rx).await }); + tokio::spawn(async move { actor.run(receiver, expired_rx, ct).await }); Self { sender } } @@ -159,8 +173,8 @@ impl Handle { set, response: response_tx, }; - let _ = self.sender.send(msg).await; - response_rx.await.expect("Actor panicked") + self.sender.send(msg).await.map_err(|_| Error::Terminated)?; + response_rx.await.map_err(|_| Error::Terminated)? } /// Blocks and returns the aggregated signed duty data when available. @@ -171,15 +185,15 @@ impl Handle { &self, duty: types::Duty, pub_key: types::PubKey, - ) -> Box { + ) -> Result, Error> { let (response_tx, response_rx) = sync::oneshot::channel(); let msg = Message::WaitFor { duty, pub_key, response: response_tx, }; - let _ = self.sender.send(msg).await; - response_rx.await.expect("Actor panicked") + self.sender.send(msg).await.map_err(|_| Error::Terminated)?; + response_rx.await.map_err(|_| Error::Terminated) } } @@ -191,6 +205,7 @@ mod tests { types::{Duty, PubKey, Signature, SignedData, SignedDataSet, SlotNumber}, }; use tokio::sync; + use tokio_util::sync::CancellationToken; /// Some mock signed data type for testing. #[derive(Debug, Clone, PartialEq, Eq)] @@ -244,7 +259,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_read() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx); + let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -255,14 +270,14 @@ mod tests { .await .unwrap(); - let result = store.wait_for(duty, pub_key).await; + let result = store.wait_for(duty, pub_key).await.unwrap(); assert_eq!(result, signed_data.boxed()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_unblocks() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx); + let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -282,16 +297,33 @@ mod tests { assert!(!reader.is_finished(), "wait_for should block until store"); let write = store.store(duty, signed_data.singleton(pub_key)).await; - let read = reader.await.unwrap(); + let read = reader.await.unwrap().unwrap(); assert!(write.is_ok()); assert_eq!(read, signed_data.boxed()); } + #[tokio::test] + async fn write_while_cancelled() { + let ct = CancellationToken::new(); + + let (_, deadliner, expiration_rx) = test_deadline(); + let store = super::Handle::new(deadliner, expiration_rx, ct.clone()); + + let duty = Duty::new_proposer_duty(SlotNumber::new(10)); + let pub_key = PubKey::new([7u8; 48]); + let signed_data = MockSignedData(42); + + ct.cancel(); + + let res = store.store(duty, signed_data.singleton(pub_key)).await; + assert!(matches!(res, Err(super::Error::Terminated))); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cannot_overwrite() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx); + let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -313,7 +345,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_idempotent() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx); + let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -328,7 +360,7 @@ mod tests { .await .unwrap(); - let result = store.wait_for(duty, pub_key).await; + let result = store.wait_for(duty, pub_key).await.unwrap(); assert_eq!(result, signed_data.boxed()); } @@ -336,7 +368,7 @@ mod tests { async fn write_evict_wait_then_write() { let (expiration_tx, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner.clone(), expiration_rx); + let store = super::Handle::new(deadliner.clone(), expiration_rx, CancellationToken::new()); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -374,7 +406,7 @@ mod tests { // return the new data, not the evicted data. store.store(duty, second.singleton(pub_key)).await.unwrap(); - let read = reader.await.unwrap(); + let read = reader.await.unwrap().unwrap(); assert_eq!(read, second.boxed()); assert_ne!(read, first.boxed()); } @@ -384,7 +416,7 @@ mod tests { const N: usize = 4; let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx); + let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); let signed_data = MockSignedData(42); @@ -413,7 +445,7 @@ mod tests { .unwrap(); for reader in readers { - let read = reader.await.unwrap(); + let read = reader.await.unwrap().unwrap(); assert_eq!(read, signed_data.boxed()); } } @@ -421,7 +453,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unrelated_write_does_not_unblock() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx); + let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); let data_a = MockSignedData(1); @@ -459,7 +491,7 @@ mod tests { .await .unwrap(); - let read = reader.await.unwrap(); + let read = reader.await.unwrap().unwrap(); assert_eq!(read, data_a.boxed()); assert_ne!(read, data_b.boxed()); } From 73ee2377c84b0b6e5416b9782a881f51437099c3 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 16:38:07 -0300 Subject: [PATCH 24/34] Revert unnecessary change --- crates/core/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 7018da49..5e9fe61d 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -23,7 +23,7 @@ pub mod version; /// Duty deadline tracking and notification. pub mod deadline; -/// Implementations of ParSigDB. +/// parsigdb. pub mod parsigdb; /// DutyDB — in-memory store for unsigned duty data. From 0b8f3fe2a22d68371bb517bc6bd4e34c06ea0e09 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 16:38:27 -0300 Subject: [PATCH 25/34] Simplify `SignedDataSet` - Use type alias instead --- crates/core/src/aggsigdb/memory.rs | 2 +- crates/core/src/types.rs | 60 +++++------------------------- 2 files changed, 11 insertions(+), 51 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 19d85a67..33271d13 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -235,7 +235,7 @@ mod tests { impl MockSignedData { fn singleton(&self, pub_key: PubKey) -> SignedDataSet { let mut set = SignedDataSet::new(); - set.insert(pub_key, self.clone()); + set.insert(pub_key, self.boxed()); set } diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 6e649847..0d85d9c3 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -729,53 +729,7 @@ impl TryFrom<(&DutyType, &pbcore::ParSignedDataSet)> for ParSignedDataSet { } /// A set of signed duty data. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct SignedDataSet(HashMap>); - -impl SignedDataSet { - /// Create a new signed data set. - pub fn new() -> Self { - Self::default() - } - - /// Get a signed data by public key. - pub fn get(&self, pub_key: &PubKey) -> Option<&dyn SignedData> { - self.0.get(pub_key).map(|b| b.as_ref()) - } - - /// Insert a signed data. - pub fn insert(&mut self, pub_key: PubKey, signed_data: impl SignedData) { - self.0.insert(pub_key, Box::new(signed_data)); - } - - /// Remove a signed data by public key. - pub fn remove(&mut self, pub_key: &PubKey) -> Option> { - self.0.remove(pub_key) - } - - /// Iterate over the signed data set by reference. - pub fn iter(&self) -> std::collections::hash_map::Iter<'_, PubKey, Box> { - self.0.iter() - } -} - -impl IntoIterator for SignedDataSet { - type IntoIter = std::collections::hash_map::IntoIter>; - type Item = (PubKey, Box); - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl<'a> IntoIterator for &'a SignedDataSet { - type IntoIter = std::collections::hash_map::Iter<'a, PubKey, Box>; - type Item = (&'a PubKey, &'a Box); - - fn into_iter(self) -> Self::IntoIter { - self.0.iter() - } -} +pub type SignedDataSet = HashMap>; /// Slot struct #[derive(Debug, Clone, PartialEq, Eq)] @@ -1057,6 +1011,12 @@ mod tests { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct MockSignedData; + impl MockSignedData { + fn boxed(&self) -> Box { + Box::new(self.clone()) + } + } + impl SignedData for MockSignedData { fn signature(&self) -> Result { Ok([42u8; SIGNATURE_LENGTH]) @@ -1096,11 +1056,11 @@ mod tests { #[test] fn signed_data_set() { let mut signed_data_set = SignedDataSet::new(); - signed_data_set.insert(PubKey::new([42u8; PK_LEN]), MockSignedData); - let expected: &dyn SignedData = &MockSignedData; + signed_data_set.insert(PubKey::new([42u8; PK_LEN]), MockSignedData.boxed()); + let expected = MockSignedData.boxed(); assert_eq!( signed_data_set.get(&PubKey::new([42u8; PK_LEN])), - Some(expected) + Some(&expected) ); } From fe654a0410c8ebab04e8e414e6ab5df725b864c1 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 16:39:21 -0300 Subject: [PATCH 26/34] Revert change --- crates/core/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 5e9fe61d..6cd84def 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -23,7 +23,7 @@ pub mod version; /// Duty deadline tracking and notification. pub mod deadline; -/// parsigdb. +/// parsigdb pub mod parsigdb; /// DutyDB — in-memory store for unsigned duty data. From 98a6160f4f9d3914447bd43cd729c229ed8abf4f Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 19:08:32 -0300 Subject: [PATCH 27/34] Revert the deadliner behavior --- crates/core/src/aggsigdb/memory.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 33271d13..90153be7 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -73,14 +73,8 @@ impl Actor { } async fn store(&mut self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { - match self.deadliner.add(duty.clone()).await { - // The duty is already expired, so we can ignore the store. - deadline::AddOutcome::AlreadyExpired => return Ok(()), - // Failures are treated as terminated. - deadline::AddOutcome::FailedToCompute => return Err(Error::Terminated), - // In other cases proceed with the store. - _ => {} - } + // TODO(charon): Distinguish between no deadline supported vs already expired. + let _ = self.deadliner.add(duty.clone()).await; // NOTE: Partial insertions on error match the semantics of Charon. let for_duty = self.entries.entry(duty.clone()).or_default(); From f60f53412f415278a223b9ed916242c033279f84 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 19:28:54 -0300 Subject: [PATCH 28/34] Revert wrong change --- crates/eth2util/src/keystore/store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/eth2util/src/keystore/store.rs b/crates/eth2util/src/keystore/store.rs index 7874bbd0..3b4e1466 100644 --- a/crates/eth2util/src/keystore/store.rs +++ b/crates/eth2util/src/keystore/store.rs @@ -230,6 +230,7 @@ async fn write_file(path: impl AsRef, data: &[u8], mode: u32) -> Result<() .open(path.as_ref()) .await?; file.write_all(data).await?; + file.flush().await?; Ok(()) } From 139852dee9c3b24511da65946f56890643dcc0e5 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 19:40:29 -0300 Subject: [PATCH 29/34] Fix Claude issues --- crates/core/src/aggsigdb/memory.rs | 31 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 90153be7..bb9bc3d4 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -69,6 +69,9 @@ impl Actor { } } } + + // After each message, trim waiters in case that the futures are dropped. + self.trim_readers(); } } @@ -114,6 +117,14 @@ impl Actor { fn evict(&mut self, duty: types::Duty) { self.entries.remove(&duty); } + + fn trim_readers(&mut self) { + self.waiters.retain(|_, waiters| { + waiters.retain(|w| !w.is_closed()); + + !waiters.is_empty() + }); + } } enum Message { @@ -374,15 +385,16 @@ mod tests { .await .unwrap(); - // Wait until the expiration is processed and the duty is evicted. + // Queue the expiration. Immediately run a dummy store, and by the time it + // compeltes we know that the expiration has been processed. expiration_tx.send(duty.clone()).await.unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while expiration_tx.capacity() != expiration_tx.max_capacity() { - tokio::task::yield_now().await; - } - }) - .await - .expect("Eviction did not complete in time"); + { + let dummy = Duty::new_attester_duty(SlotNumber::new(u64::MAX)); + store + .store(dummy, MockSignedData(0).singleton(pub_key)) + .await + .unwrap(); + } let reader = { let store = store.clone(); @@ -466,8 +478,7 @@ mod tests { tokio::task::yield_now().await; assert!(!reader.is_finished(), "reader should block initially"); - // Storing an unrelated key wakes readers, which block again since the store is - // unrelated. + // Storing an unrelated key does not affect readers. store .store(duty_b, data_b.singleton(pub_key)) .await From 7923d6761430f80ebf8bd5904faa6b0799a6ec28 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 28 May 2026 19:42:04 -0300 Subject: [PATCH 30/34] Document `wait_for` blocking --- crates/core/src/aggsigdb/memory.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index bb9bc3d4..b03da050 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -186,6 +186,10 @@ impl Handle { /// /// Might block indefinitely if no data is ever stored for the given duty /// and public key. + /// + /// To avoid blocking indefinitely, consider using a timeout, + /// [`CancellationToken`] or racing using `tokio::select!` against other + /// events. pub async fn wait_for( &self, duty: types::Duty, From f87b4b672df8fd15e8a90b403a9544d52e705c6c Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 29 May 2026 09:23:24 -0300 Subject: [PATCH 31/34] Extract to trait - Rename structs to `Memory*` --- crates/core/src/aggsigdb/memory.rs | 64 +++++++++++------------------- crates/core/src/aggsigdb/mod.rs | 37 +++++++++++++++++ 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index b03da050..91a1588d 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,32 +1,21 @@ -use crate::{deadline, types}; +use crate::{ + aggsigdb::{AggSigDB, Error}, + deadline, types, +}; use std::collections::{HashMap, hash_map::Entry}; use tokio::sync; use tokio_util::sync::CancellationToken; -/// Errors for the in-memory AggSigDB implementation. -#[derive(Debug, thiserror::Error)] -pub enum Error { - /// Data for the same duty and public key already exists but does not match - /// the new data. - #[error("Mismatching data")] - MismatchingData, - - /// The request cannot be processed because the instance has been - /// terminated. - #[error("The instance has been terminated")] - Terminated, -} - type Waiters = HashMap<(types::Duty, types::PubKey), Vec>>>; -struct Actor { +struct MemoryDBActor { entries: HashMap>>, waiters: Waiters, deadliner: deadline::DeadlinerHandle, } -impl Actor { +impl MemoryDBActor { async fn run( &mut self, mut messages_rx: sync::mpsc::Receiver, @@ -145,11 +134,11 @@ enum Message { /// Share an instance by cloning. Cloning is cheap and creates a new reference /// to the same underlying data. #[derive(Clone)] -pub struct Handle { +pub struct MemoryDBHandle { sender: sync::mpsc::Sender, } -impl Handle { +impl MemoryDBHandle { /// Creates a new in-memory AggSigDB instance, and get a handle to it. /// /// The underlying instance gets dropped when all handles are dropped. @@ -159,7 +148,7 @@ impl Handle { ct: CancellationToken, ) -> Self { let (sender, receiver) = sync::mpsc::channel(100); - let mut actor = Actor { + let mut actor = MemoryDBActor { entries: HashMap::new(), waiters: HashMap::new(), deadliner, @@ -169,9 +158,11 @@ impl Handle { Self { sender } } +} - /// Stores aggregated signed duty data set. - pub async fn store(&self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { +#[async_trait::async_trait] +impl AggSigDB for MemoryDBHandle { + async fn store(&self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { let (response_tx, response_rx) = sync::oneshot::channel(); let msg = Message::Store { duty, @@ -182,15 +173,7 @@ impl Handle { response_rx.await.map_err(|_| Error::Terminated)? } - /// Blocks and returns the aggregated signed duty data when available. - /// - /// Might block indefinitely if no data is ever stored for the given duty - /// and public key. - /// - /// To avoid blocking indefinitely, consider using a timeout, - /// [`CancellationToken`] or racing using `tokio::select!` against other - /// events. - pub async fn wait_for( + async fn wait_for( &self, duty: types::Duty, pub_key: types::PubKey, @@ -209,6 +192,7 @@ impl Handle { #[cfg(test)] mod tests { use crate::{ + aggsigdb::{AggSigDB, Error, memory::MemoryDBHandle}, deadline, signeddata::SignedDataError, types::{Duty, PubKey, Signature, SignedData, SignedDataSet, SlotNumber}, @@ -268,7 +252,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_read() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -286,7 +270,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_unblocks() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -317,7 +301,7 @@ mod tests { let ct = CancellationToken::new(); let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, ct.clone()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, ct.clone()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -326,13 +310,13 @@ mod tests { ct.cancel(); let res = store.store(duty, signed_data.singleton(pub_key)).await; - assert!(matches!(res, Err(super::Error::Terminated))); + assert!(matches!(res, Err(Error::Terminated))); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cannot_overwrite() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -354,7 +338,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_idempotent() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); @@ -377,7 +361,7 @@ mod tests { async fn write_evict_wait_then_write() { let (expiration_tx, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner.clone(), expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner.clone(), expiration_rx, CancellationToken::new()); let duty = Duty::new_attester_duty(SlotNumber::new(1)); let pub_key = PubKey::new([7u8; 48]); @@ -426,7 +410,7 @@ mod tests { const N: usize = 4; let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, CancellationToken::new()); let duty = Duty::new_proposer_duty(SlotNumber::new(10)); let pub_key = PubKey::new([7u8; 48]); let signed_data = MockSignedData(42); @@ -463,7 +447,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unrelated_write_does_not_unblock() { let (_, deadliner, expiration_rx) = test_deadline(); - let store = super::Handle::new(deadliner, expiration_rx, CancellationToken::new()); + let store = MemoryDBHandle::new(deadliner, expiration_rx, CancellationToken::new()); let duty_a = Duty::new_proposer_duty(SlotNumber::new(10)); let data_a = MockSignedData(1); diff --git a/crates/core/src/aggsigdb/mod.rs b/crates/core/src/aggsigdb/mod.rs index 54c50307..eae11db4 100644 --- a/crates/core/src/aggsigdb/mod.rs +++ b/crates/core/src/aggsigdb/mod.rs @@ -1,2 +1,39 @@ +use crate::types; + +/// Errors for AggSigDB operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Data for the same duty and public key already exists but does not match + /// the new data. + #[error("Mismatching data")] + MismatchingData, + + /// The request cannot be processed because the instance has been + /// terminated. + #[error("The instance has been terminated")] + Terminated, +} + +/// A persistent store for aggregated signed duty data. +#[async_trait::async_trait] +pub trait AggSigDB { + /// Stores aggregated signed duty data set. + async fn store(&self, duty: types::Duty, data: types::SignedDataSet) -> Result<(), Error>; + + /// Blocks and returns the aggregated signed duty data when available. + /// + /// Might block indefinitely if no data is ever stored for the given duty + /// and public key. + /// + /// To avoid blocking indefinitely, consider using a timeout, + /// [`CancellationToken`] or racing using `tokio::select!` against other + /// events. + async fn wait_for( + &self, + duty: types::Duty, + pub_key: types::PubKey, + ) -> Result, Error>; +} + /// Memory implementation of the AggSigDB. pub mod memory; From 4ace069b6e8149c1a39963dce6c0de2b42b15af6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 2 Jun 2026 13:57:35 -0300 Subject: [PATCH 32/34] Early return on `store` with an empty set --- crates/core/src/aggsigdb/memory.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index 91a1588d..cf30298c 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -65,6 +65,10 @@ impl MemoryDBActor { } async fn store(&mut self, duty: types::Duty, set: types::SignedDataSet) -> Result<(), Error> { + if set.is_empty() { + return Ok(()); + } + // TODO(charon): Distinguish between no deadline supported vs already expired. let _ = self.deadliner.add(duty.clone()).await; From fab5556a31b35149f86b1dc7e8aac7dc0adf7484 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 2 Jun 2026 13:59:50 -0300 Subject: [PATCH 33/34] Avoid unnecessary cloning on closed channels --- crates/core/src/aggsigdb/memory.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index cf30298c..fbf74a2b 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -82,7 +82,9 @@ impl MemoryDBActor { let k = (duty.clone(), pub_key); if let Some((_, waiters)) = self.waiters.remove_entry(&k) { for w in waiters { - let _ = w.send(signed_data.clone()); + if !w.is_closed() { + let _ = w.send(signed_data.clone()); + } } }; } From 15371cfd9cb34e5266b69ef29fff125d151c2818 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Tue, 2 Jun 2026 14:05:06 -0300 Subject: [PATCH 34/34] Reorg modules --- crates/core/src/aggsigdb/memory.rs | 7 ++++-- crates/core/src/aggsigdb/mod.rs | 38 ++---------------------------- crates/core/src/aggsigdb/types.rs | 36 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 38 deletions(-) create mode 100644 crates/core/src/aggsigdb/types.rs diff --git a/crates/core/src/aggsigdb/memory.rs b/crates/core/src/aggsigdb/memory.rs index fbf74a2b..b2165d2a 100644 --- a/crates/core/src/aggsigdb/memory.rs +++ b/crates/core/src/aggsigdb/memory.rs @@ -1,5 +1,5 @@ use crate::{ - aggsigdb::{AggSigDB, Error}, + aggsigdb::types::{AggSigDB, Error}, deadline, types, }; use std::collections::{HashMap, hash_map::Entry}; @@ -198,7 +198,10 @@ impl AggSigDB for MemoryDBHandle { #[cfg(test)] mod tests { use crate::{ - aggsigdb::{AggSigDB, Error, memory::MemoryDBHandle}, + aggsigdb::{ + memory::MemoryDBHandle, + types::{AggSigDB, Error}, + }, deadline, signeddata::SignedDataError, types::{Duty, PubKey, Signature, SignedData, SignedDataSet, SlotNumber}, diff --git a/crates/core/src/aggsigdb/mod.rs b/crates/core/src/aggsigdb/mod.rs index eae11db4..1c5a4f3b 100644 --- a/crates/core/src/aggsigdb/mod.rs +++ b/crates/core/src/aggsigdb/mod.rs @@ -1,39 +1,5 @@ -use crate::types; - -/// Errors for AggSigDB operations. -#[derive(Debug, thiserror::Error)] -pub enum Error { - /// Data for the same duty and public key already exists but does not match - /// the new data. - #[error("Mismatching data")] - MismatchingData, - - /// The request cannot be processed because the instance has been - /// terminated. - #[error("The instance has been terminated")] - Terminated, -} - -/// A persistent store for aggregated signed duty data. -#[async_trait::async_trait] -pub trait AggSigDB { - /// Stores aggregated signed duty data set. - async fn store(&self, duty: types::Duty, data: types::SignedDataSet) -> Result<(), Error>; - - /// Blocks and returns the aggregated signed duty data when available. - /// - /// Might block indefinitely if no data is ever stored for the given duty - /// and public key. - /// - /// To avoid blocking indefinitely, consider using a timeout, - /// [`CancellationToken`] or racing using `tokio::select!` against other - /// events. - async fn wait_for( - &self, - duty: types::Duty, - pub_key: types::PubKey, - ) -> Result, Error>; -} +/// Type definitions and traits for the AggSigDB. +pub mod types; /// Memory implementation of the AggSigDB. pub mod memory; diff --git a/crates/core/src/aggsigdb/types.rs b/crates/core/src/aggsigdb/types.rs new file mode 100644 index 00000000..e4ecba6c --- /dev/null +++ b/crates/core/src/aggsigdb/types.rs @@ -0,0 +1,36 @@ +use crate::types; + +/// Errors for AggSigDB operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Data for the same duty and public key already exists but does not match + /// the new data. + #[error("Mismatching data")] + MismatchingData, + + /// The request cannot be processed because the instance has been + /// terminated. + #[error("The instance has been terminated")] + Terminated, +} + +/// A persistent store for aggregated signed duty data. +#[async_trait::async_trait] +pub trait AggSigDB { + /// Stores aggregated signed duty data set. + async fn store(&self, duty: types::Duty, data: types::SignedDataSet) -> Result<(), Error>; + + /// Blocks and returns the aggregated signed duty data when available. + /// + /// Might block indefinitely if no data is ever stored for the given duty + /// and public key. + /// + /// To avoid blocking indefinitely, consider using a timeout, + /// [`CancellationToken`] or racing using `tokio::select!` against other + /// events. + async fn wait_for( + &self, + duty: types::Duty, + pub_key: types::PubKey, + ) -> Result, Error>; +}