From 8ef1e59989a0e2ac12ebb63e389dde0cb6d2296b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 02:50:10 +0000 Subject: [PATCH 01/26] Initial plan From c3b75c708f5b39158ddec9fa51dbbc42266d6103 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 02:55:50 +0000 Subject: [PATCH 02/26] Add ProgressiveMerkleHasher implementation with basic tests Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/lib.rs | 2 + tree_hash/src/progressive_merkle_hasher.rs | 266 +++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 tree_hash/src/progressive_merkle_hasher.rs diff --git a/tree_hash/src/lib.rs b/tree_hash/src/lib.rs index e5ac66f..72d4790 100644 --- a/tree_hash/src/lib.rs +++ b/tree_hash/src/lib.rs @@ -2,10 +2,12 @@ pub mod impls; mod merkle_hasher; mod merkleize_padded; mod merkleize_standard; +mod progressive_merkle_hasher; pub use merkle_hasher::{Error, MerkleHasher}; pub use merkleize_padded::merkleize_padded; pub use merkleize_standard::merkleize_standard; +pub use progressive_merkle_hasher::ProgressiveMerkleHasher; use ethereum_hashing::{hash_fixed, ZERO_HASHES, ZERO_HASHES_MAX_INDEX}; use smallvec::SmallVec; diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs new file mode 100644 index 0000000..f1a19db --- /dev/null +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -0,0 +1,266 @@ +use crate::{get_zero_hash, merkle_root, Hash256, BYTES_PER_CHUNK}; +use ethereum_hashing::hash32_concat; + +#[derive(Clone, Debug, PartialEq)] +pub enum Error { + /// The maximum number of leaves has been exceeded. + MaximumLeavesExceeded { max_leaves: usize }, +} + +/// A progressive Merkle hasher that implements the semantics of `merkleize_progressive` as +/// defined in EIP-7916. +/// +/// The progressive merkle tree has a unique structure where: +/// - At each level, the right child is a binary merkle tree with a specific number of leaves +/// - The left child recursively contains more progressive structure +/// - The number of leaves in each right subtree grows by 4x at each level (1, 4, 16, 64, ...) +/// +/// # Example Tree Structure +/// +/// ```text +/// root +/// /\ +/// / \ +/// /\ 1: chunks[0 ..< 1] +/// / \ +/// /\ 4: chunks[1 ..< 5] +/// / \ +/// /\ 16: chunks[5 ..< 21] +/// / \ +/// 0 64: chunks[21 ..< 85] +/// ``` +/// +/// This structure allows efficient appending and proof generation for growing lists. +pub struct ProgressiveMerkleHasher { + /// All chunks that have been written to the hasher. + chunks: Vec<[u8; BYTES_PER_CHUNK]>, + /// Maximum number of leaves this hasher can accept. + max_leaves: usize, +} + +impl ProgressiveMerkleHasher { + /// Create a new progressive merkle hasher that can accept up to `max_leaves` leaves. + /// + /// # Panics + /// + /// Panics if `max_leaves == 0`. + pub fn with_leaves(max_leaves: usize) -> Self { + assert!(max_leaves > 0, "must have at least one leaf"); + Self { + chunks: Vec::new(), + max_leaves, + } + } + + /// Write bytes to the hasher. + /// + /// The bytes will be split into 32-byte chunks. If the final chunk is incomplete, + /// it will be padded with zeros. + /// + /// # Errors + /// + /// Returns an error if writing these bytes would exceed the maximum number of leaves. + pub fn write(&mut self, bytes: &[u8]) -> Result<(), Error> { + let num_new_leaves = bytes.len().div_ceil(BYTES_PER_CHUNK); + + if self.chunks.len() + num_new_leaves > self.max_leaves { + return Err(Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + }); + } + + // Split bytes into 32-byte chunks + for chunk_bytes in bytes.chunks(BYTES_PER_CHUNK) { + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[..chunk_bytes.len()].copy_from_slice(chunk_bytes); + self.chunks.push(chunk); + } + + Ok(()) + } + + /// Finish the hasher and return the progressive merkle root. + /// + /// This implements the recursive merkleize_progressive algorithm: + /// - If no chunks: return zero hash + /// - Otherwise: hash(merkleize_progressive(left), merkleize(right)) + /// where right contains the first num_leaves chunks as a binary tree, + /// and left recursively contains the rest with num_leaves * 4. + pub fn finish(self) -> Result { + Ok(merkleize_progressive(&self.chunks, 1)) + } +} + +/// Recursively compute the progressive merkle root for the given chunks. +/// +/// # Arguments +/// +/// * `chunks` - The chunks to merkleize +/// * `num_leaves` - The number of leaves for the right (binary tree) subtree at this level +/// +/// # Algorithm +/// +/// Following the spec: +/// ```text +/// merkleize_progressive(chunks, num_leaves=1): Given ordered BYTES_PER_CHUNK-byte chunks: +/// The merkleization depends on the number of input chunks and is defined recursively: +/// If len(chunks) == 0: the root is a zero value, Bytes32(). +/// Otherwise: compute the root using hash(a, b) +/// a: Recursively merkleize chunks beyond num_leaves using +/// merkleize_progressive(chunks[num_leaves:], num_leaves * 4). +/// b: Merkleize the first up to num_leaves chunks as a binary tree using +/// merkleize(chunks[:num_leaves], num_leaves). +/// ``` +fn merkleize_progressive(chunks: &[[u8; BYTES_PER_CHUNK]], num_leaves: usize) -> Hash256 { + if chunks.is_empty() { + // Base case: no chunks, return zero hash + return Hash256::ZERO; + } + + // Split chunks into right (first num_leaves) and left (rest) + let right_chunks = &chunks[..chunks.len().min(num_leaves)]; + let left_chunks = &chunks[chunks.len().min(num_leaves)..]; + + // Compute right subtree: binary merkle tree with num_leaves leaves + let right_root = if right_chunks.is_empty() { + // If no chunks for right, use zero hash + Hash256::from_slice(get_zero_hash(compute_height(num_leaves))) + } else { + // Use merkle_root to compute binary tree root + let bytes: Vec = right_chunks.iter().flat_map(|c| c.iter().copied()).collect(); + merkle_root(&bytes, num_leaves) + }; + + // Compute left subtree: recursive progressive merkle tree with num_leaves * 4 + let left_root = merkleize_progressive(left_chunks, num_leaves * 4); + + // Combine left and right roots + Hash256::from_slice(&hash32_concat(left_root.as_slice(), right_root.as_slice())) +} + +/// Compute the height of a binary tree with the given number of leaves. +fn compute_height(num_leaves: usize) -> usize { + if num_leaves == 0 { + 0 + } else { + // Height is log2(next_power_of_two(num_leaves)) + let power_of_two = num_leaves.next_power_of_two(); + power_of_two.trailing_zeros() as usize + } +} + + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty_tree() { + let hasher = ProgressiveMerkleHasher::with_leaves(1); + let root = hasher.finish().unwrap(); + assert_eq!(root, Hash256::ZERO); + } + + #[test] + fn test_single_chunk() { + let mut hasher = ProgressiveMerkleHasher::with_leaves(1); + let chunk = [1u8; BYTES_PER_CHUNK]; + hasher.write(&chunk).unwrap(); + let root = hasher.finish().unwrap(); + + // For a single chunk, the progressive tree should be: + // hash(merkleize_progressive([], 4), merkleize([chunk], 1)) + // = hash(zero_hash, chunk) + let zero_left = Hash256::ZERO; + let right = Hash256::from_slice(&chunk); + let expected = Hash256::from_slice(&hash32_concat( + zero_left.as_slice(), + right.as_slice() + )); + + assert_eq!(root, expected); + } + + #[test] + fn test_two_chunks() { + let mut hasher = ProgressiveMerkleHasher::with_leaves(5); + let chunk1 = [1u8; BYTES_PER_CHUNK]; + let chunk2 = [2u8; BYTES_PER_CHUNK]; + hasher.write(&chunk1).unwrap(); + hasher.write(&chunk2).unwrap(); + let root = hasher.finish().unwrap(); + + // First chunk goes to right (num_leaves=1) + // Second chunk goes to left recursive call (num_leaves=4) + + // Right: binary tree with 1 leaf = chunk1 + let right = Hash256::from_slice(&chunk1); + + // Left: progressive tree with chunk2 at num_leaves=4 + // At this level: hash(merkleize_progressive([], 16), merkleize([chunk2], 4)) + // = hash(zero_hash, merkle([chunk2], 4)) + let chunk2_padded = merkle_root(&chunk2, 4); + let zero_left_inner = Hash256::ZERO; + let left = Hash256::from_slice(&hash32_concat( + zero_left_inner.as_slice(), + chunk2_padded.as_slice() + )); + + let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), right.as_slice())); + assert_eq!(root, expected); + } + + #[test] + fn test_max_leaves_exceeded() { + let mut hasher = ProgressiveMerkleHasher::with_leaves(2); + let chunk = [1u8; BYTES_PER_CHUNK]; + hasher.write(&chunk).unwrap(); + hasher.write(&chunk).unwrap(); + + // Third write should fail + let result = hasher.write(&chunk); + assert!(matches!(result, Err(Error::MaximumLeavesExceeded { .. }))); + } + + #[test] + fn test_partial_chunk() { + let mut hasher = ProgressiveMerkleHasher::with_leaves(1); + let partial = vec![1u8, 2u8, 3u8]; + hasher.write(&partial).unwrap(); + let root = hasher.finish().unwrap(); + + // Partial chunk should be padded with zeros + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[0] = 1; + chunk[1] = 2; + chunk[2] = 3; + + let zero_left = Hash256::ZERO; + let right = Hash256::from_slice(&chunk); + let expected = Hash256::from_slice(&hash32_concat( + zero_left.as_slice(), + right.as_slice() + )); + + assert_eq!(root, expected); + } + + #[test] + fn test_multiple_writes() { + let mut hasher = ProgressiveMerkleHasher::with_leaves(10); + hasher.write(&[1u8; 16]).unwrap(); + hasher.write(&[2u8; 16]).unwrap(); + hasher.write(&[3u8; 32]).unwrap(); + let root = hasher.finish().unwrap(); + + // Should handle multiple writes correctly + assert_ne!(root, Hash256::ZERO); + } + + #[test] + #[should_panic(expected = "must have at least one leaf")] + fn test_zero_leaves_panics() { + ProgressiveMerkleHasher::with_leaves(0); + } +} From 74331be814a1a24c886993d1f134485a13f22a61 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 02:57:58 +0000 Subject: [PATCH 03/26] Add byte buffering and comprehensive tests for ProgressiveMerkleHasher Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/progressive_merkle_hasher.rs | 166 +++++++++++++++++++-- 1 file changed, 153 insertions(+), 13 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index f1a19db..592e17c 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -36,6 +36,8 @@ pub struct ProgressiveMerkleHasher { chunks: Vec<[u8; BYTES_PER_CHUNK]>, /// Maximum number of leaves this hasher can accept. max_leaves: usize, + /// Buffer for bytes that haven't been completed into a chunk yet. + buffer: Vec, } impl ProgressiveMerkleHasher { @@ -49,31 +51,34 @@ impl ProgressiveMerkleHasher { Self { chunks: Vec::new(), max_leaves, + buffer: Vec::new(), } } /// Write bytes to the hasher. /// - /// The bytes will be split into 32-byte chunks. If the final chunk is incomplete, - /// it will be padded with zeros. + /// The bytes will be split into 32-byte chunks. Bytes are buffered across multiple + /// write calls to ensure proper chunk boundaries. /// /// # Errors /// /// Returns an error if writing these bytes would exceed the maximum number of leaves. pub fn write(&mut self, bytes: &[u8]) -> Result<(), Error> { - let num_new_leaves = bytes.len().div_ceil(BYTES_PER_CHUNK); + // Add bytes to buffer + self.buffer.extend_from_slice(bytes); - if self.chunks.len() + num_new_leaves > self.max_leaves { - return Err(Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - }); - } - - // Split bytes into 32-byte chunks - for chunk_bytes in bytes.chunks(BYTES_PER_CHUNK) { + // Process complete chunks from buffer + while self.buffer.len() >= BYTES_PER_CHUNK { + if self.chunks.len() >= self.max_leaves { + return Err(Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + }); + } + let mut chunk = [0u8; BYTES_PER_CHUNK]; - chunk[..chunk_bytes.len()].copy_from_slice(chunk_bytes); + chunk.copy_from_slice(&self.buffer[..BYTES_PER_CHUNK]); self.chunks.push(chunk); + self.buffer.drain(..BYTES_PER_CHUNK); } Ok(()) @@ -86,7 +91,22 @@ impl ProgressiveMerkleHasher { /// - Otherwise: hash(merkleize_progressive(left), merkleize(right)) /// where right contains the first num_leaves chunks as a binary tree, /// and left recursively contains the rest with num_leaves * 4. - pub fn finish(self) -> Result { + /// + /// Any remaining bytes in the buffer will be padded to form a final chunk. + pub fn finish(mut self) -> Result { + // Process any remaining bytes in the buffer as a final chunk + if !self.buffer.is_empty() { + if self.chunks.len() >= self.max_leaves { + return Err(Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + }); + } + + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[..self.buffer.len()].copy_from_slice(&self.buffer); + self.chunks.push(chunk); + } + Ok(merkleize_progressive(&self.chunks, 1)) } } @@ -263,4 +283,124 @@ mod tests { fn test_zero_leaves_panics() { ProgressiveMerkleHasher::with_leaves(0); } + + #[test] + fn test_five_chunks() { + // Test with 5 chunks as per the problem statement structure: + // chunks[0] goes to right at level 1 (1 leaf) + // chunks[1..5] go to left recursive call (4 leaves at level 2) + let mut hasher = ProgressiveMerkleHasher::with_leaves(5); + for i in 0..5 { + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[0] = i as u8; + hasher.write(&chunk).unwrap(); + } + let root = hasher.finish().unwrap(); + + // Manually compute expected root: + // Right: chunks[0] + let mut chunk0 = [0u8; BYTES_PER_CHUNK]; + chunk0[0] = 0; + let right = Hash256::from_slice(&chunk0); + + // Left: merkleize_progressive(chunks[1..5], 4) + // Which is: hash(merkleize_progressive([], 16), merkleize(chunks[1..5], 4)) + let chunks_1_to_4: Vec = (1..5) + .flat_map(|i| { + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[0] = i; + chunk + }) + .collect(); + let right_inner = merkle_root(&chunks_1_to_4, 4); + let left_inner = Hash256::ZERO; + let left = Hash256::from_slice(&hash32_concat( + left_inner.as_slice(), + right_inner.as_slice() + )); + + let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), right.as_slice())); + assert_eq!(root, expected); + } + + #[test] + fn test_21_chunks() { + // Test with 21 chunks as per problem statement: + // chunks[0] goes to right at level 1 (1 leaf) + // chunks[1..5] go to right at level 2 (4 leaves) + // chunks[5..21] go to right at level 3 (16 leaves) + let mut hasher = ProgressiveMerkleHasher::with_leaves(21); + for i in 0..21 { + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[0] = i as u8; + hasher.write(&chunk).unwrap(); + } + let root = hasher.finish().unwrap(); + + // Root should not be zero + assert_ne!(root, Hash256::ZERO); + } + + #[test] + fn test_85_chunks() { + // Test with 85 chunks as per problem statement structure: + // chunks[0] at level 1 (1 leaf) + // chunks[1..5] at level 2 (4 leaves) + // chunks[5..21] at level 3 (16 leaves) + // chunks[21..85] at level 4 (64 leaves) + let mut hasher = ProgressiveMerkleHasher::with_leaves(85); + for i in 0..85 { + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[0] = (i % 256) as u8; + hasher.write(&chunk).unwrap(); + } + let root = hasher.finish().unwrap(); + + // Root should not be zero + assert_ne!(root, Hash256::ZERO); + } + + #[test] + fn test_consistency_with_manual_calculation() { + // Test that using ProgressiveMerkleHasher gives the same result as + // manually calling merkleize_progressive + let chunks: Vec<[u8; BYTES_PER_CHUNK]> = (0..10) + .map(|i| { + let mut chunk = [0u8; BYTES_PER_CHUNK]; + chunk[0] = i; + chunk + }) + .collect(); + + // Use ProgressiveMerkleHasher + let mut hasher = ProgressiveMerkleHasher::with_leaves(10); + for chunk in &chunks { + hasher.write(chunk).unwrap(); + } + let hasher_root = hasher.finish().unwrap(); + + // Manually call merkleize_progressive + let manual_root = merkleize_progressive(&chunks, 1); + + assert_eq!(hasher_root, manual_root); + } + + #[test] + fn test_byte_streaming() { + // Test that we can write bytes in various chunk sizes + let data = vec![42u8; BYTES_PER_CHUNK * 3 + 10]; + + // Write all at once + let mut hasher1 = ProgressiveMerkleHasher::with_leaves(10); + hasher1.write(&data).unwrap(); + let root1 = hasher1.finish().unwrap(); + + // Write in smaller chunks + let mut hasher2 = ProgressiveMerkleHasher::with_leaves(10); + hasher2.write(&data[0..50]).unwrap(); + hasher2.write(&data[50..]).unwrap(); + let root2 = hasher2.finish().unwrap(); + + assert_eq!(root1, root2); + } } From f5ecf94920c66142a8f6960085c58fc25a24b7f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 03:00:50 +0000 Subject: [PATCH 04/26] Export ProgressiveMerkleHasherError from lib.rs Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tree_hash/src/lib.rs b/tree_hash/src/lib.rs index 72d4790..996a5cc 100644 --- a/tree_hash/src/lib.rs +++ b/tree_hash/src/lib.rs @@ -7,7 +7,9 @@ mod progressive_merkle_hasher; pub use merkle_hasher::{Error, MerkleHasher}; pub use merkleize_padded::merkleize_padded; pub use merkleize_standard::merkleize_standard; -pub use progressive_merkle_hasher::ProgressiveMerkleHasher; +pub use progressive_merkle_hasher::{ + Error as ProgressiveMerkleHasherError, ProgressiveMerkleHasher, +}; use ethereum_hashing::{hash_fixed, ZERO_HASHES, ZERO_HASHES_MAX_INDEX}; use smallvec::SmallVec; From 3e3051213e36dbe92e777402acd5855dabaaca40 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 03:04:53 +0000 Subject: [PATCH 05/26] Add clarifying comment about hash order in progressive merkleization Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/progressive_merkle_hasher.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index 592e17c..d1184b9 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -154,7 +154,8 @@ fn merkleize_progressive(chunks: &[[u8; BYTES_PER_CHUNK]], num_leaves: usize) -> // Compute left subtree: recursive progressive merkle tree with num_leaves * 4 let left_root = merkleize_progressive(left_chunks, num_leaves * 4); - // Combine left and right roots + // Combine left and right roots according to spec: hash(a, b) where + // a = left subtree (recursive progressive), b = right subtree (binary tree) Hash256::from_slice(&hash32_concat(left_root.as_slice(), right_root.as_slice())) } From 07466523c9be0c9478a76e08d1571e28cf16ed3a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 04:40:08 +0000 Subject: [PATCH 06/26] Refactor ProgressiveMerkleHasher for efficiency - hash chunks as they stream in Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/progressive_merkle_hasher.rs | 205 ++++++++++++--------- 1 file changed, 118 insertions(+), 87 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index d1184b9..d838df5 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -1,4 +1,4 @@ -use crate::{get_zero_hash, merkle_root, Hash256, BYTES_PER_CHUNK}; +use crate::{merkle_root, Hash256, BYTES_PER_CHUNK}; use ethereum_hashing::hash32_concat; #[derive(Clone, Debug, PartialEq)] @@ -31,13 +31,27 @@ pub enum Error { /// ``` /// /// This structure allows efficient appending and proof generation for growing lists. +/// +/// # Efficiency +/// +/// This implementation hashes chunks as they are streamed in, storing only the minimum +/// necessary state (completed subtree roots). When a level is filled, its binary merkle +/// root is computed and stored, avoiding the need to keep all chunks in memory. pub struct ProgressiveMerkleHasher { - /// All chunks that have been written to the hasher. - chunks: Vec<[u8; BYTES_PER_CHUNK]>, - /// Maximum number of leaves this hasher can accept. - max_leaves: usize, + /// Completed subtree roots at each level, stored in reverse order. + /// Index 0 = most recent (smallest) completed level, higher indices = larger levels. + /// Each level i contains 4^i leaves. + completed_roots: Vec, + /// Chunks currently being accumulated for the next level to fill. + current_chunks: Vec<[u8; BYTES_PER_CHUNK]>, + /// The number of leaves expected at the current level (1, 4, 16, 64, ...). + current_level_size: usize, /// Buffer for bytes that haven't been completed into a chunk yet. buffer: Vec, + /// Maximum number of leaves this hasher can accept. + max_leaves: usize, + /// Total number of chunks written so far. + total_chunks: usize, } impl ProgressiveMerkleHasher { @@ -49,16 +63,20 @@ impl ProgressiveMerkleHasher { pub fn with_leaves(max_leaves: usize) -> Self { assert!(max_leaves > 0, "must have at least one leaf"); Self { - chunks: Vec::new(), - max_leaves, + completed_roots: Vec::new(), + current_chunks: Vec::new(), + current_level_size: 1, buffer: Vec::new(), + max_leaves, + total_chunks: 0, } } /// Write bytes to the hasher. /// /// The bytes will be split into 32-byte chunks. Bytes are buffered across multiple - /// write calls to ensure proper chunk boundaries. + /// write calls to ensure proper chunk boundaries. Complete subtrees are hashed + /// immediately as chunks are written. /// /// # Errors /// @@ -69,7 +87,7 @@ impl ProgressiveMerkleHasher { // Process complete chunks from buffer while self.buffer.len() >= BYTES_PER_CHUNK { - if self.chunks.len() >= self.max_leaves { + if self.total_chunks >= self.max_leaves { return Err(Error::MaximumLeavesExceeded { max_leaves: self.max_leaves, }); @@ -77,26 +95,46 @@ impl ProgressiveMerkleHasher { let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk.copy_from_slice(&self.buffer[..BYTES_PER_CHUNK]); - self.chunks.push(chunk); self.buffer.drain(..BYTES_PER_CHUNK); + + self.process_chunk(chunk)?; } Ok(()) } + + /// Process a single chunk by adding it to the current level and completing the level if full. + fn process_chunk(&mut self, chunk: [u8; BYTES_PER_CHUNK]) -> Result<(), Error> { + self.current_chunks.push(chunk); + self.total_chunks += 1; + + // Check if current level is complete + if self.current_chunks.len() == self.current_level_size { + // Compute the merkle root for this level + let bytes: Vec = self.current_chunks.iter().flat_map(|c| c.iter().copied()).collect(); + let root = merkle_root(&bytes, self.current_level_size); + + // Store this completed root + self.completed_roots.push(root); + + // Move to next level (4x larger) + self.current_chunks.clear(); + self.current_level_size *= 4; + } + + Ok(()) + } /// Finish the hasher and return the progressive merkle root. /// - /// This implements the recursive merkleize_progressive algorithm: - /// - If no chunks: return zero hash - /// - Otherwise: hash(merkleize_progressive(left), merkleize(right)) - /// where right contains the first num_leaves chunks as a binary tree, - /// and left recursively contains the rest with num_leaves * 4. + /// This completes any partial level and combines all completed subtree roots + /// according to the progressive merkleization algorithm. /// /// Any remaining bytes in the buffer will be padded to form a final chunk. pub fn finish(mut self) -> Result { // Process any remaining bytes in the buffer as a final chunk if !self.buffer.is_empty() { - if self.chunks.len() >= self.max_leaves { + if self.total_chunks >= self.max_leaves { return Err(Error::MaximumLeavesExceeded { max_leaves: self.max_leaves, }); @@ -104,74 +142,57 @@ impl ProgressiveMerkleHasher { let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[..self.buffer.len()].copy_from_slice(&self.buffer); - self.chunks.push(chunk); + self.process_chunk(chunk)?; } - Ok(merkleize_progressive(&self.chunks, 1)) - } -} - -/// Recursively compute the progressive merkle root for the given chunks. -/// -/// # Arguments -/// -/// * `chunks` - The chunks to merkleize -/// * `num_leaves` - The number of leaves for the right (binary tree) subtree at this level -/// -/// # Algorithm -/// -/// Following the spec: -/// ```text -/// merkleize_progressive(chunks, num_leaves=1): Given ordered BYTES_PER_CHUNK-byte chunks: -/// The merkleization depends on the number of input chunks and is defined recursively: -/// If len(chunks) == 0: the root is a zero value, Bytes32(). -/// Otherwise: compute the root using hash(a, b) -/// a: Recursively merkleize chunks beyond num_leaves using -/// merkleize_progressive(chunks[num_leaves:], num_leaves * 4). -/// b: Merkleize the first up to num_leaves chunks as a binary tree using -/// merkleize(chunks[:num_leaves], num_leaves). -/// ``` -fn merkleize_progressive(chunks: &[[u8; BYTES_PER_CHUNK]], num_leaves: usize) -> Hash256 { - if chunks.is_empty() { - // Base case: no chunks, return zero hash - return Hash256::ZERO; + // If we have no chunks at all, return zero hash + if self.total_chunks == 0 { + return Ok(Hash256::ZERO); + } + + // If there are chunks in current_chunks (partial level), compute their root + let current_root = if !self.current_chunks.is_empty() { + let bytes: Vec = self.current_chunks.iter().flat_map(|c| c.iter().copied()).collect(); + Some(merkle_root(&bytes, self.current_level_size)) + } else { + None + }; + + // Build the progressive tree from completed roots and current root + // completed_roots are in order: [smallest level, ..., largest level] + // We need to build from right to left in the tree + Ok(self.build_progressive_root(current_root)) } - - // Split chunks into right (first num_leaves) and left (rest) - let right_chunks = &chunks[..chunks.len().min(num_leaves)]; - let left_chunks = &chunks[chunks.len().min(num_leaves)..]; - - // Compute right subtree: binary merkle tree with num_leaves leaves - let right_root = if right_chunks.is_empty() { - // If no chunks for right, use zero hash - Hash256::from_slice(get_zero_hash(compute_height(num_leaves))) - } else { - // Use merkle_root to compute binary tree root - let bytes: Vec = right_chunks.iter().flat_map(|c| c.iter().copied()).collect(); - merkle_root(&bytes, num_leaves) - }; - - // Compute left subtree: recursive progressive merkle tree with num_leaves * 4 - let left_root = merkleize_progressive(left_chunks, num_leaves * 4); - - // Combine left and right roots according to spec: hash(a, b) where - // a = left subtree (recursive progressive), b = right subtree (binary tree) - Hash256::from_slice(&hash32_concat(left_root.as_slice(), right_root.as_slice())) -} - -/// Compute the height of a binary tree with the given number of leaves. -fn compute_height(num_leaves: usize) -> usize { - if num_leaves == 0 { - 0 - } else { - // Height is log2(next_power_of_two(num_leaves)) - let power_of_two = num_leaves.next_power_of_two(); - power_of_two.trailing_zeros() as usize + + /// Build the final progressive merkle root by combining completed subtree roots. + /// + /// The progressive tree structure: at each node, hash(left=next_levels, right=this_level). + /// Build from the current (largest/partial) level backwards to the first level. + fn build_progressive_root(&self, current_root: Option) -> Hash256 { + // Start from the leftmost (largest) level + // If there's a current partial level, it needs to be wrapped: hash(ZERO, current_root) + // because the spec applies the structure at every level + let mut result = if let Some(curr) = current_root { + Hash256::from_slice(&hash32_concat(Hash256::ZERO.as_slice(), curr.as_slice())) + } else { + Hash256::ZERO + }; + + // Process completed roots from largest to smallest (reverse order) + // At each step: result = hash(result, completed_root) because + // result is the accumulated left subtree, completed_root is the right subtree at this level + for &completed_root in self.completed_roots.iter().rev() { + result = Hash256::from_slice(&hash32_concat( + result.as_slice(), + completed_root.as_slice(), + )); + } + + result } } - #[cfg(test)] mod tests { use super::*; @@ -362,9 +383,8 @@ mod tests { } #[test] - fn test_consistency_with_manual_calculation() { - // Test that using ProgressiveMerkleHasher gives the same result as - // manually calling merkleize_progressive + fn test_consistency_across_write_patterns() { + // Test that different write patterns produce the same result let chunks: Vec<[u8; BYTES_PER_CHUNK]> = (0..10) .map(|i| { let mut chunk = [0u8; BYTES_PER_CHUNK]; @@ -373,17 +393,28 @@ mod tests { }) .collect(); - // Use ProgressiveMerkleHasher - let mut hasher = ProgressiveMerkleHasher::with_leaves(10); + // Write all chunks individually + let mut hasher1 = ProgressiveMerkleHasher::with_leaves(10); for chunk in &chunks { - hasher.write(chunk).unwrap(); + hasher1.write(chunk).unwrap(); } - let hasher_root = hasher.finish().unwrap(); + let root1 = hasher1.finish().unwrap(); - // Manually call merkleize_progressive - let manual_root = merkleize_progressive(&chunks, 1); + // Write all chunks at once + let mut hasher2 = ProgressiveMerkleHasher::with_leaves(10); + let all_bytes: Vec = chunks.iter().flat_map(|c| c.iter().copied()).collect(); + hasher2.write(&all_bytes).unwrap(); + let root2 = hasher2.finish().unwrap(); - assert_eq!(hasher_root, manual_root); + // Write in groups + let mut hasher3 = ProgressiveMerkleHasher::with_leaves(10); + hasher3.write(&all_bytes[..3 * BYTES_PER_CHUNK]).unwrap(); + hasher3.write(&all_bytes[3 * BYTES_PER_CHUNK..7 * BYTES_PER_CHUNK]).unwrap(); + hasher3.write(&all_bytes[7 * BYTES_PER_CHUNK..]).unwrap(); + let root3 = hasher3.finish().unwrap(); + + assert_eq!(root1, root2); + assert_eq!(root1, root3); } #[test] From 16fc4a0368c385f1e116031d589529a775371449 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 04:45:39 +0000 Subject: [PATCH 07/26] Address code review feedback - improve documentation and extract helper method Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/progressive_merkle_hasher.rs | 34 +++++++++++++--------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index d838df5..5cbb1e6 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -38,9 +38,9 @@ pub enum Error { /// necessary state (completed subtree roots). When a level is filled, its binary merkle /// root is computed and stored, avoiding the need to keep all chunks in memory. pub struct ProgressiveMerkleHasher { - /// Completed subtree roots at each level, stored in reverse order. - /// Index 0 = most recent (smallest) completed level, higher indices = larger levels. - /// Each level i contains 4^i leaves. + /// Completed subtree roots at each level, stored in order of completion. + /// Index 0 = first completed level (1 leaf), index 1 = second level (4 leaves), etc. + /// Level i contains 4^i leaves. completed_roots: Vec, /// Chunks currently being accumulated for the next level to fill. current_chunks: Vec<[u8; BYTES_PER_CHUNK]>, @@ -111,8 +111,7 @@ impl ProgressiveMerkleHasher { // Check if current level is complete if self.current_chunks.len() == self.current_level_size { // Compute the merkle root for this level - let bytes: Vec = self.current_chunks.iter().flat_map(|c| c.iter().copied()).collect(); - let root = merkle_root(&bytes, self.current_level_size); + let root = Self::compute_level_root(&self.current_chunks, self.current_level_size); // Store this completed root self.completed_roots.push(root); @@ -124,6 +123,12 @@ impl ProgressiveMerkleHasher { Ok(()) } + + /// Helper to compute the merkle root for a level's chunks. + fn compute_level_root(chunks: &[[u8; BYTES_PER_CHUNK]], num_leaves: usize) -> Hash256 { + let bytes: Vec = chunks.iter().flat_map(|c| c.iter().copied()).collect(); + merkle_root(&bytes, num_leaves) + } /// Finish the hasher and return the progressive merkle root. /// @@ -152,8 +157,7 @@ impl ProgressiveMerkleHasher { // If there are chunks in current_chunks (partial level), compute their root let current_root = if !self.current_chunks.is_empty() { - let bytes: Vec = self.current_chunks.iter().flat_map(|c| c.iter().copied()).collect(); - Some(merkle_root(&bytes, self.current_level_size)) + Some(Self::compute_level_root(&self.current_chunks, self.current_level_size)) } else { None }; @@ -166,12 +170,13 @@ impl ProgressiveMerkleHasher { /// Build the final progressive merkle root by combining completed subtree roots. /// - /// The progressive tree structure: at each node, hash(left=next_levels, right=this_level). - /// Build from the current (largest/partial) level backwards to the first level. + /// The progressive tree structure: at each node, hash(left=deeper_levels, right=this_level). + /// This builds the tree from the largest (leftmost) level backwards to the smallest (rightmost). fn build_progressive_root(&self, current_root: Option) -> Hash256 { - // Start from the leftmost (largest) level - // If there's a current partial level, it needs to be wrapped: hash(ZERO, current_root) - // because the spec applies the structure at every level + // Start from the leftmost (largest/deepest) level + // Per EIP-7916 spec, even partial levels follow the progressive structure: + // merkleize_progressive(chunks, n) = hash(merkleize_progressive(chunks[n:], n*4), merkleize(chunks[:n], n)) + // So a partial level with k chunks becomes: hash(ZERO (no further chunks), merkleize(chunks, n)) let mut result = if let Some(curr) = current_root { Hash256::from_slice(&hash32_concat(Hash256::ZERO.as_slice(), curr.as_slice())) } else { @@ -179,8 +184,9 @@ impl ProgressiveMerkleHasher { }; // Process completed roots from largest to smallest (reverse order) - // At each step: result = hash(result, completed_root) because - // result is the accumulated left subtree, completed_root is the right subtree at this level + // At each step: result = hash(result, completed_root) + // - result accumulates the left subtree (deeper/larger levels) + // - completed_root is the right subtree at this level for &completed_root in self.completed_roots.iter().rev() { result = Hash256::from_slice(&hash32_concat( result.as_slice(), From 733344a8e0f975713da23f4905f7dc1ff8905646 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Dec 2025 02:35:35 +0000 Subject: [PATCH 08/26] Refactor ProgressiveMerkleHasher to use MerkleHasher internally for binary tree hashing Co-authored-by: michaelsproul <4452260+michaelsproul@users.noreply.github.com> --- tree_hash/src/progressive_merkle_hasher.rs | 60 ++++++++++++++-------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index 5cbb1e6..67d93ab 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -1,4 +1,4 @@ -use crate::{merkle_root, Hash256, BYTES_PER_CHUNK}; +use crate::{Hash256, MerkleHasher, BYTES_PER_CHUNK}; use ethereum_hashing::hash32_concat; #[derive(Clone, Debug, PartialEq)] @@ -42,10 +42,12 @@ pub struct ProgressiveMerkleHasher { /// Index 0 = first completed level (1 leaf), index 1 = second level (4 leaves), etc. /// Level i contains 4^i leaves. completed_roots: Vec, - /// Chunks currently being accumulated for the next level to fill. - current_chunks: Vec<[u8; BYTES_PER_CHUNK]>, + /// MerkleHasher for computing the current level's binary tree root. + current_hasher: MerkleHasher, /// The number of leaves expected at the current level (1, 4, 16, 64, ...). current_level_size: usize, + /// Number of chunks written to the current hasher. + current_level_chunks: usize, /// Buffer for bytes that haven't been completed into a chunk yet. buffer: Vec, /// Maximum number of leaves this hasher can accept. @@ -64,8 +66,9 @@ impl ProgressiveMerkleHasher { assert!(max_leaves > 0, "must have at least one leaf"); Self { completed_roots: Vec::new(), - current_chunks: Vec::new(), + current_hasher: MerkleHasher::with_leaves(1), current_level_size: 1, + current_level_chunks: 0, buffer: Vec::new(), max_leaves, total_chunks: 0, @@ -105,30 +108,39 @@ impl ProgressiveMerkleHasher { /// Process a single chunk by adding it to the current level and completing the level if full. fn process_chunk(&mut self, chunk: [u8; BYTES_PER_CHUNK]) -> Result<(), Error> { - self.current_chunks.push(chunk); + // Write the chunk to the current MerkleHasher + self.current_hasher.write(&chunk).map_err(|_| Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + })?; + + self.current_level_chunks += 1; self.total_chunks += 1; // Check if current level is complete - if self.current_chunks.len() == self.current_level_size { - // Compute the merkle root for this level - let root = Self::compute_level_root(&self.current_chunks, self.current_level_size); + if self.current_level_chunks == self.current_level_size { + // Move to next level (4x larger) + let next_level_size = self.current_level_size * 4; + + // Replace the current hasher with a new one for the next level + let completed_hasher = std::mem::replace( + &mut self.current_hasher, + MerkleHasher::with_leaves(next_level_size) + ); + + // Finish the completed hasher to get the root + let root = completed_hasher.finish().map_err(|_| Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + })?; // Store this completed root self.completed_roots.push(root); - // Move to next level (4x larger) - self.current_chunks.clear(); - self.current_level_size *= 4; + self.current_level_size = next_level_size; + self.current_level_chunks = 0; } Ok(()) } - - /// Helper to compute the merkle root for a level's chunks. - fn compute_level_root(chunks: &[[u8; BYTES_PER_CHUNK]], num_leaves: usize) -> Hash256 { - let bytes: Vec = chunks.iter().flat_map(|c| c.iter().copied()).collect(); - merkle_root(&bytes, num_leaves) - } /// Finish the hasher and return the progressive merkle root. /// @@ -155,9 +167,16 @@ impl ProgressiveMerkleHasher { return Ok(Hash256::ZERO); } - // If there are chunks in current_chunks (partial level), compute their root - let current_root = if !self.current_chunks.is_empty() { - Some(Self::compute_level_root(&self.current_chunks, self.current_level_size)) + // If there are chunks in current level (partial level), compute their root + let current_root = if self.current_level_chunks > 0 { + // Create a temporary hasher to replace the current one (since finish() takes ownership) + let temp_hasher = std::mem::replace( + &mut self.current_hasher, + MerkleHasher::with_leaves(1) // dummy value, won't be used + ); + Some(temp_hasher.finish().map_err(|_| Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + })?) } else { None }; @@ -202,6 +221,7 @@ impl ProgressiveMerkleHasher { #[cfg(test)] mod tests { use super::*; + use crate::merkle_root; #[test] fn test_empty_tree() { From 8277f30d61fd5bc5c0bf792bc5dde5d20bcc1359 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 10 Dec 2025 10:02:27 +1100 Subject: [PATCH 09/26] Fmt --- tree_hash/src/progressive_merkle_hasher.rs | 137 +++++++++++---------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index 67d93ab..5cf29a8 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -87,7 +87,7 @@ impl ProgressiveMerkleHasher { pub fn write(&mut self, bytes: &[u8]) -> Result<(), Error> { // Add bytes to buffer self.buffer.extend_from_slice(bytes); - + // Process complete chunks from buffer while self.buffer.len() >= BYTES_PER_CHUNK { if self.total_chunks >= self.max_leaves { @@ -95,50 +95,54 @@ impl ProgressiveMerkleHasher { max_leaves: self.max_leaves, }); } - + let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk.copy_from_slice(&self.buffer[..BYTES_PER_CHUNK]); self.buffer.drain(..BYTES_PER_CHUNK); - + self.process_chunk(chunk)?; } Ok(()) } - + /// Process a single chunk by adding it to the current level and completing the level if full. fn process_chunk(&mut self, chunk: [u8; BYTES_PER_CHUNK]) -> Result<(), Error> { // Write the chunk to the current MerkleHasher - self.current_hasher.write(&chunk).map_err(|_| Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - })?; - + self.current_hasher + .write(&chunk) + .map_err(|_| Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + })?; + self.current_level_chunks += 1; self.total_chunks += 1; - + // Check if current level is complete if self.current_level_chunks == self.current_level_size { // Move to next level (4x larger) let next_level_size = self.current_level_size * 4; - + // Replace the current hasher with a new one for the next level let completed_hasher = std::mem::replace( &mut self.current_hasher, - MerkleHasher::with_leaves(next_level_size) + MerkleHasher::with_leaves(next_level_size), ); - + // Finish the completed hasher to get the root - let root = completed_hasher.finish().map_err(|_| Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - })?; - + let root = completed_hasher + .finish() + .map_err(|_| Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + })?; + // Store this completed root self.completed_roots.push(root); - + self.current_level_size = next_level_size; self.current_level_chunks = 0; } - + Ok(()) } @@ -156,37 +160,41 @@ impl ProgressiveMerkleHasher { max_leaves: self.max_leaves, }); } - + let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[..self.buffer.len()].copy_from_slice(&self.buffer); self.process_chunk(chunk)?; } - + // If we have no chunks at all, return zero hash if self.total_chunks == 0 { return Ok(Hash256::ZERO); } - + // If there are chunks in current level (partial level), compute their root let current_root = if self.current_level_chunks > 0 { // Create a temporary hasher to replace the current one (since finish() takes ownership) let temp_hasher = std::mem::replace( &mut self.current_hasher, - MerkleHasher::with_leaves(1) // dummy value, won't be used + MerkleHasher::with_leaves(1), // dummy value, won't be used ); - Some(temp_hasher.finish().map_err(|_| Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - })?) + Some( + temp_hasher + .finish() + .map_err(|_| Error::MaximumLeavesExceeded { + max_leaves: self.max_leaves, + })?, + ) } else { None }; - + // Build the progressive tree from completed roots and current root // completed_roots are in order: [smallest level, ..., largest level] // We need to build from right to left in the tree Ok(self.build_progressive_root(current_root)) } - + /// Build the final progressive merkle root by combining completed subtree roots. /// /// The progressive tree structure: at each node, hash(left=deeper_levels, right=this_level). @@ -201,23 +209,20 @@ impl ProgressiveMerkleHasher { } else { Hash256::ZERO }; - + // Process completed roots from largest to smallest (reverse order) // At each step: result = hash(result, completed_root) // - result accumulates the left subtree (deeper/larger levels) // - completed_root is the right subtree at this level for &completed_root in self.completed_roots.iter().rev() { - result = Hash256::from_slice(&hash32_concat( - result.as_slice(), - completed_root.as_slice(), - )); + result = + Hash256::from_slice(&hash32_concat(result.as_slice(), completed_root.as_slice())); } - + result } } - #[cfg(test)] mod tests { use super::*; @@ -236,17 +241,14 @@ mod tests { let chunk = [1u8; BYTES_PER_CHUNK]; hasher.write(&chunk).unwrap(); let root = hasher.finish().unwrap(); - + // For a single chunk, the progressive tree should be: // hash(merkleize_progressive([], 4), merkleize([chunk], 1)) // = hash(zero_hash, chunk) let zero_left = Hash256::ZERO; let right = Hash256::from_slice(&chunk); - let expected = Hash256::from_slice(&hash32_concat( - zero_left.as_slice(), - right.as_slice() - )); - + let expected = Hash256::from_slice(&hash32_concat(zero_left.as_slice(), right.as_slice())); + assert_eq!(root, expected); } @@ -258,13 +260,13 @@ mod tests { hasher.write(&chunk1).unwrap(); hasher.write(&chunk2).unwrap(); let root = hasher.finish().unwrap(); - + // First chunk goes to right (num_leaves=1) // Second chunk goes to left recursive call (num_leaves=4) - + // Right: binary tree with 1 leaf = chunk1 let right = Hash256::from_slice(&chunk1); - + // Left: progressive tree with chunk2 at num_leaves=4 // At this level: hash(merkleize_progressive([], 16), merkleize([chunk2], 4)) // = hash(zero_hash, merkle([chunk2], 4)) @@ -272,9 +274,9 @@ mod tests { let zero_left_inner = Hash256::ZERO; let left = Hash256::from_slice(&hash32_concat( zero_left_inner.as_slice(), - chunk2_padded.as_slice() + chunk2_padded.as_slice(), )); - + let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), right.as_slice())); assert_eq!(root, expected); } @@ -285,7 +287,7 @@ mod tests { let chunk = [1u8; BYTES_PER_CHUNK]; hasher.write(&chunk).unwrap(); hasher.write(&chunk).unwrap(); - + // Third write should fail let result = hasher.write(&chunk); assert!(matches!(result, Err(Error::MaximumLeavesExceeded { .. }))); @@ -297,20 +299,17 @@ mod tests { let partial = vec![1u8, 2u8, 3u8]; hasher.write(&partial).unwrap(); let root = hasher.finish().unwrap(); - + // Partial chunk should be padded with zeros let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[0] = 1; chunk[1] = 2; chunk[2] = 3; - + let zero_left = Hash256::ZERO; let right = Hash256::from_slice(&chunk); - let expected = Hash256::from_slice(&hash32_concat( - zero_left.as_slice(), - right.as_slice() - )); - + let expected = Hash256::from_slice(&hash32_concat(zero_left.as_slice(), right.as_slice())); + assert_eq!(root, expected); } @@ -321,7 +320,7 @@ mod tests { hasher.write(&[2u8; 16]).unwrap(); hasher.write(&[3u8; 32]).unwrap(); let root = hasher.finish().unwrap(); - + // Should handle multiple writes correctly assert_ne!(root, Hash256::ZERO); } @@ -344,13 +343,13 @@ mod tests { hasher.write(&chunk).unwrap(); } let root = hasher.finish().unwrap(); - + // Manually compute expected root: // Right: chunks[0] let mut chunk0 = [0u8; BYTES_PER_CHUNK]; chunk0[0] = 0; let right = Hash256::from_slice(&chunk0); - + // Left: merkleize_progressive(chunks[1..5], 4) // Which is: hash(merkleize_progressive([], 16), merkleize(chunks[1..5], 4)) let chunks_1_to_4: Vec = (1..5) @@ -364,9 +363,9 @@ mod tests { let left_inner = Hash256::ZERO; let left = Hash256::from_slice(&hash32_concat( left_inner.as_slice(), - right_inner.as_slice() + right_inner.as_slice(), )); - + let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), right.as_slice())); assert_eq!(root, expected); } @@ -384,7 +383,7 @@ mod tests { hasher.write(&chunk).unwrap(); } let root = hasher.finish().unwrap(); - + // Root should not be zero assert_ne!(root, Hash256::ZERO); } @@ -403,7 +402,7 @@ mod tests { hasher.write(&chunk).unwrap(); } let root = hasher.finish().unwrap(); - + // Root should not be zero assert_ne!(root, Hash256::ZERO); } @@ -418,27 +417,29 @@ mod tests { chunk }) .collect(); - + // Write all chunks individually let mut hasher1 = ProgressiveMerkleHasher::with_leaves(10); for chunk in &chunks { hasher1.write(chunk).unwrap(); } let root1 = hasher1.finish().unwrap(); - + // Write all chunks at once let mut hasher2 = ProgressiveMerkleHasher::with_leaves(10); let all_bytes: Vec = chunks.iter().flat_map(|c| c.iter().copied()).collect(); hasher2.write(&all_bytes).unwrap(); let root2 = hasher2.finish().unwrap(); - + // Write in groups let mut hasher3 = ProgressiveMerkleHasher::with_leaves(10); hasher3.write(&all_bytes[..3 * BYTES_PER_CHUNK]).unwrap(); - hasher3.write(&all_bytes[3 * BYTES_PER_CHUNK..7 * BYTES_PER_CHUNK]).unwrap(); + hasher3 + .write(&all_bytes[3 * BYTES_PER_CHUNK..7 * BYTES_PER_CHUNK]) + .unwrap(); hasher3.write(&all_bytes[7 * BYTES_PER_CHUNK..]).unwrap(); let root3 = hasher3.finish().unwrap(); - + assert_eq!(root1, root2); assert_eq!(root1, root3); } @@ -447,18 +448,18 @@ mod tests { fn test_byte_streaming() { // Test that we can write bytes in various chunk sizes let data = vec![42u8; BYTES_PER_CHUNK * 3 + 10]; - + // Write all at once let mut hasher1 = ProgressiveMerkleHasher::with_leaves(10); hasher1.write(&data).unwrap(); let root1 = hasher1.finish().unwrap(); - + // Write in smaller chunks let mut hasher2 = ProgressiveMerkleHasher::with_leaves(10); hasher2.write(&data[0..50]).unwrap(); hasher2.write(&data[50..]).unwrap(); let root2 = hasher2.finish().unwrap(); - + assert_eq!(root1, root2); } } From 4fb08f86259cf919a3626e828875c7ff32c3c4e9 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 10 Dec 2025 15:36:05 +1100 Subject: [PATCH 10/26] Start implementing derive macro --- tree_hash/src/lib.rs | 7 ++ tree_hash/tests/tests.rs | 17 +++++ tree_hash_derive/src/attrs.rs | 127 ++++++++++++++++++++++++++++++++++ tree_hash_derive/src/lib.rs | 95 +++++++++++++------------ 4 files changed, 202 insertions(+), 44 deletions(-) create mode 100644 tree_hash_derive/src/attrs.rs diff --git a/tree_hash/src/lib.rs b/tree_hash/src/lib.rs index 996a5cc..f9e2278 100644 --- a/tree_hash/src/lib.rs +++ b/tree_hash/src/lib.rs @@ -93,6 +93,13 @@ pub fn mix_in_selector(root: &Hash256, selector: u8) -> Option { Some(Hash256::from_slice(&root)) } +pub fn mix_in_active_fields(root: Hash256, active_fields: [u8; BYTES_PER_CHUNK]) -> Hash256 { + Hash256::from(ethereum_hashing::hash32_concat( + root.as_slice(), + &active_fields, + )) +} + /// Returns a cached padding node for a given height. fn get_zero_hash(height: usize) -> &'static [u8] { if height <= ZERO_HASHES_MAX_INDEX { diff --git a/tree_hash/tests/tests.rs b/tree_hash/tests/tests.rs index 8531548..f9f22e0 100644 --- a/tree_hash/tests/tests.rs +++ b/tree_hash/tests/tests.rs @@ -1,5 +1,6 @@ use alloy_primitives::{Address, U128, U160, U256}; use ssz_derive::Encode; +use std::str::FromStr; use tree_hash::{Hash256, MerkleHasher, PackedEncoding, TreeHash, BYTES_PER_CHUNK}; use tree_hash_derive::TreeHash; @@ -167,3 +168,19 @@ fn packed_encoding_example() { ); } } + +#[derive(TreeHash)] +#[tree_hash(struct_behaviour = "progressive_container", active_fields(1))] +struct ProgressiveContainerOneField { + x: u8, +} + +#[test] +fn progressive_container_one_field() { + let container = ProgressiveContainerOneField { x: 125 }; + assert_eq!( + container.tree_hash_root(), + Hash256::from_str("0xfacc8073916cbe1d3e400f69945fb5b6423d1e8f99be04713bcbe254fad2c94c") + .unwrap() + ); +} diff --git a/tree_hash_derive/src/attrs.rs b/tree_hash_derive/src/attrs.rs new file mode 100644 index 0000000..83426d2 --- /dev/null +++ b/tree_hash_derive/src/attrs.rs @@ -0,0 +1,127 @@ +use darling::{ast::NestedMeta, Error, FromDeriveInput, FromMeta}; +use quote::quote; + +pub const MAX_ACTIVE_FIELDS: usize = 256; +pub const ACTIVE_FIELDS_PACKED_BITS_LEN: usize = MAX_ACTIVE_FIELDS / 8; + +#[derive(Debug, FromDeriveInput)] +#[darling(attributes(tree_hash))] +pub struct StructOpts { + #[darling(default)] + pub enum_behaviour: Option, + #[darling(default)] + pub struct_behaviour: Option, + #[darling(default)] + pub active_fields: Option, +} + +#[derive(Debug, FromMeta)] +pub enum EnumBehaviour { + Transparent, + Union, +} + +#[derive(Debug, Default, FromMeta)] +pub enum StructBehaviour { + #[default] + Container, + ProgressiveContainer, +} + +#[derive(Debug)] +pub struct ActiveFields { + pub active_fields: Vec, +} + +impl FromMeta for ActiveFields { + fn from_list(items: &[NestedMeta]) -> Result { + let active_fields = items + .iter() + .map(|nested_meta| match u8::from_nested_meta(nested_meta) { + Ok(0) => Ok(false), + Ok(1) => Ok(true), + Ok(n) => Err(Error::custom(format!( + "invalid integer in active_fields: {n}" + ))), + Err(e) => Err(Error::custom(format!( + "unable to parse active_fields entry: {e:?}" + ))), + }) + .collect::>()?; + Self::new(active_fields) + } +} + +impl ActiveFields { + fn new(active_fields: Vec) -> Result { + if active_fields.is_empty() { + return Err(Error::custom(format!("active_fields must be non-empty"))); + } + if active_fields.len() > MAX_ACTIVE_FIELDS { + return Err(Error::custom(format!( + "active_fields cannot contain more than {MAX_ACTIVE_FIELDS} entries" + ))); + } + + if let Some(false) = active_fields.last() { + return Err(Error::custom(format!( + "the last entry of active_fields must not be 0" + ))); + } + + Ok(Self { active_fields }) + } + + pub fn packed(&self) -> [u8; ACTIVE_FIELDS_PACKED_BITS_LEN] { + let mut result = [0; ACTIVE_FIELDS_PACKED_BITS_LEN]; + for (i, bit) in self.active_fields.iter().enumerate() { + if *bit { + result[i / 8] |= 1 << (i % 8); + } + } + result + } + + /// Return tokens for the packed representation of these `active_fields`. + /// + /// We compute the packed representation at compile-time, and then inline it via the output + /// of this function. + pub fn packed_tokens(&self) -> proc_macro2::TokenStream { + let packed = self.packed().to_vec(); + quote! { + [ + #(#packed),* + ] + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn active_fields_packed_basic() { + let active_fields = ActiveFields { + active_fields: vec![true], + }; + assert_eq!( + active_fields.packed(), + [ + 0b0000001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + ] + ); + + let active_fields = ActiveFields { + active_fields: vec![true, false, true, false, false, true], + }; + assert_eq!( + active_fields.packed(), + [ + 0b0100101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + ] + ); + } +} diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index 6e6a727..95d98a4 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -1,4 +1,7 @@ #![recursion_limit = "256"] +mod attrs; + +use crate::attrs::{EnumBehaviour, StructBehaviour, StructOpts}; use darling::FromDeriveInput; use proc_macro::TokenStream; use quote::quote; @@ -9,37 +12,6 @@ use syn::{parse_macro_input, DataEnum, DataStruct, DeriveInput, Ident}; /// extensions). const MAX_UNION_SELECTOR: u8 = 127; -#[derive(Debug, FromDeriveInput)] -#[darling(attributes(tree_hash))] -struct StructOpts { - #[darling(default)] - enum_behaviour: Option, -} - -const ENUM_TRANSPARENT: &str = "transparent"; -const ENUM_UNION: &str = "union"; -const ENUM_VARIANTS: &[&str] = &[ENUM_TRANSPARENT, ENUM_UNION]; -const NO_ENUM_BEHAVIOUR_ERROR: &str = "enums require an \"enum_behaviour\" attribute, \ - e.g., #[tree_hash(enum_behaviour = \"transparent\")]"; - -enum EnumBehaviour { - Transparent, - Union, -} - -impl EnumBehaviour { - pub fn new(s: Option) -> Option { - s.map(|s| match s.as_ref() { - ENUM_TRANSPARENT => EnumBehaviour::Transparent, - ENUM_UNION => EnumBehaviour::Union, - other => panic!( - "{} is an invalid enum_behaviour, use either {:?}", - other, ENUM_VARIANTS - ), - }) - } -} - /// Return a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields /// that should not be hashed. /// @@ -82,40 +54,73 @@ fn should_skip_hashing(field: &syn::Field) -> bool { }) } -/// Implements `tree_hash::TreeHash` for some `struct`. +/// Implements `tree_hash::TreeHash` for a type. /// /// Fields are hashed in the order they are defined. #[proc_macro_derive(TreeHash, attributes(tree_hash))] pub fn tree_hash_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); let opts = StructOpts::from_derive_input(&item).unwrap(); - let enum_opt = EnumBehaviour::new(opts.enum_behaviour); + let enum_opt = opts.enum_behaviour; + let struct_opt = opts.struct_behaviour; - match &item.data { - syn::Data::Struct(s) => { + match (&item.data, enum_opt, struct_opt) { + (syn::Data::Struct(s), enum_opt, struct_opt) => { if enum_opt.is_some() { panic!("enum_behaviour is invalid for structs"); } - tree_hash_derive_struct(&item, s) + let struct_behaviour = struct_opt.unwrap_or_default(); + tree_hash_derive_struct(&item, s, struct_behaviour, opts.active_fields) } - syn::Data::Enum(s) => match enum_opt.expect(NO_ENUM_BEHAVIOUR_ERROR) { - EnumBehaviour::Transparent => tree_hash_derive_enum_transparent(&item, s), - EnumBehaviour::Union => tree_hash_derive_enum_union(&item, s), - }, - _ => panic!("tree_hash_derive only supports structs and enums."), + (syn::Data::Enum(s), Some(enum_behaviour), struct_opt) => { + if struct_opt.is_some() { + panic!("struct_behaviour is invalid for enums"); + } + match enum_behaviour { + EnumBehaviour::Transparent => tree_hash_derive_enum_transparent(&item, s), + EnumBehaviour::Union => tree_hash_derive_enum_union(&item, s), + } + } + _ => panic!("tree_hash_derive only supports structs and enums"), } } -fn tree_hash_derive_struct(item: &DeriveInput, struct_data: &DataStruct) -> TokenStream { +fn tree_hash_derive_struct( + item: &DeriveInput, + struct_data: &DataStruct, + struct_behaviour: StructBehaviour, + active_fields_opt: Option, +) -> TokenStream { let name = &item.ident; let (impl_generics, ty_generics, where_clause) = &item.generics.split_for_impl(); let idents = get_hashable_fields(struct_data); let num_leaves = idents.len(); + let hasher_type = if let StructBehaviour::ProgressiveContainer = struct_behaviour { + quote! { tree_hash::ProgressiveMerkleHasher } + } else { + quote! { tree_hash::MerkleHasher } + }; + let mixin_logic = if let StructBehaviour::ProgressiveContainer = struct_behaviour { + let Some(active_fields) = active_fields_opt else { + panic!("active_fields must be provided for progressive_container"); + }; + + let packed_active_fields = active_fields.packed_tokens(); + + quote! { + const ACTIVE_FIELDS: [u8; 32] = #packed_active_fields; + tree_hash::mix_in_active_fields(container_root, ACTIVE_FIELDS) + } + } else { + quote! { container_root } + }; + let output = quote! { impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause { fn tree_hash_type() -> tree_hash::TreeHashType { + // FIXME(sproul): consider adjusting this with active_fields tree_hash::TreeHashType::Container } @@ -128,14 +133,16 @@ fn tree_hash_derive_struct(item: &DeriveInput, struct_data: &DataStruct) -> Toke } fn tree_hash_root(&self) -> tree_hash::Hash256 { - let mut hasher = tree_hash::MerkleHasher::with_leaves(#num_leaves); + let mut hasher = #hasher_type::with_leaves(#num_leaves); #( hasher.write(self.#idents.tree_hash_root().as_slice()) .expect("tree hash derive should not apply too many leaves"); )* - hasher.finish().expect("tree hash derive should not have a remaining buffer") + let container_root = hasher.finish().expect("tree hash derive should not have a remaining buffer"); + + #mixin_logic } } }; From 0a5ed99a872010b42c091c64fe0baa6b1b4d2a5d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 10 Dec 2025 16:44:28 +1100 Subject: [PATCH 11/26] Implement TreeHash for ProgressiveBitList --- Cargo.toml | 3 +++ tree_hash/src/impls.rs | 27 ++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a9e38f8..23dbcc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,6 @@ readme = "README.md" repository = "https://github.com/sigp/tree_hash" keywords = ["ethereum"] categories = ["cryptography::cryptocurrencies"] + +[patch.crates-io] +ethereum_ssz = { path = "../ethereum_ssz/ssz" } diff --git a/tree_hash/src/impls.rs b/tree_hash/src/impls.rs index bc9cd59..c0068d7 100644 --- a/tree_hash/src/impls.rs +++ b/tree_hash/src/impls.rs @@ -1,6 +1,6 @@ use super::*; use alloy_primitives::{Address, FixedBytes, U128, U256}; -use ssz::{Bitfield, Fixed, Variable}; +use ssz::{Bitfield, Fixed, Progressive, Variable}; use std::sync::Arc; use typenum::Unsigned; @@ -208,6 +208,31 @@ impl TreeHash for Bitfield> { } } +impl TreeHash for Bitfield { + fn tree_hash_type() -> TreeHashType { + TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> PackedEncoding { + unreachable!("ProgressiveBitField should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("ProgressiveBitField should never be packed.") + } + + fn tree_hash_root(&self) -> Hash256 { + let mut hasher = ProgressiveMerkleHasher::new(); + hasher + .write(self.as_slice()) + .expect("ProgessiveBitList should not exceed tree hash leaf limit"); + + hasher + .finish() + .expect("ProgressiveBitList tree hash buffer should not exceed leaf limit") + } +} + impl TreeHash for Bitfield> { fn tree_hash_type() -> TreeHashType { TreeHashType::Vector From 26fc12f68ead656f21e4102916a7796342fe4255 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 10 Dec 2025 16:45:14 +1100 Subject: [PATCH 12/26] Remove unnecessary max_leaves from ProgressiveMerkleHasher --- tree_hash/src/progressive_merkle_hasher.rs | 90 +++++----------------- tree_hash_derive/src/lib.rs | 10 +-- 2 files changed, 25 insertions(+), 75 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index 5cf29a8..a6b5ed3 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -3,8 +3,7 @@ use ethereum_hashing::hash32_concat; #[derive(Clone, Debug, PartialEq)] pub enum Error { - /// The maximum number of leaves has been exceeded. - MaximumLeavesExceeded { max_leaves: usize }, + MerkleHasher(crate::merkle_hasher::Error), } /// A progressive Merkle hasher that implements the semantics of `merkleize_progressive` as @@ -50,27 +49,19 @@ pub struct ProgressiveMerkleHasher { current_level_chunks: usize, /// Buffer for bytes that haven't been completed into a chunk yet. buffer: Vec, - /// Maximum number of leaves this hasher can accept. - max_leaves: usize, /// Total number of chunks written so far. total_chunks: usize, } impl ProgressiveMerkleHasher { - /// Create a new progressive merkle hasher that can accept up to `max_leaves` leaves. - /// - /// # Panics - /// - /// Panics if `max_leaves == 0`. - pub fn with_leaves(max_leaves: usize) -> Self { - assert!(max_leaves > 0, "must have at least one leaf"); + /// Create a new progressive merkle hasher that can accept any number of chunks. + pub fn new() -> Self { Self { completed_roots: Vec::new(), current_hasher: MerkleHasher::with_leaves(1), current_level_size: 1, current_level_chunks: 0, buffer: Vec::new(), - max_leaves, total_chunks: 0, } } @@ -90,12 +81,6 @@ impl ProgressiveMerkleHasher { // Process complete chunks from buffer while self.buffer.len() >= BYTES_PER_CHUNK { - if self.total_chunks >= self.max_leaves { - return Err(Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - }); - } - let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk.copy_from_slice(&self.buffer[..BYTES_PER_CHUNK]); self.buffer.drain(..BYTES_PER_CHUNK); @@ -111,9 +96,7 @@ impl ProgressiveMerkleHasher { // Write the chunk to the current MerkleHasher self.current_hasher .write(&chunk) - .map_err(|_| Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - })?; + .map_err(Error::MerkleHasher)?; self.current_level_chunks += 1; self.total_chunks += 1; @@ -130,11 +113,7 @@ impl ProgressiveMerkleHasher { ); // Finish the completed hasher to get the root - let root = completed_hasher - .finish() - .map_err(|_| Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - })?; + let root = completed_hasher.finish().map_err(Error::MerkleHasher)?; // Store this completed root self.completed_roots.push(root); @@ -155,12 +134,6 @@ impl ProgressiveMerkleHasher { pub fn finish(mut self) -> Result { // Process any remaining bytes in the buffer as a final chunk if !self.buffer.is_empty() { - if self.total_chunks >= self.max_leaves { - return Err(Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - }); - } - let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[..self.buffer.len()].copy_from_slice(&self.buffer); self.process_chunk(chunk)?; @@ -174,17 +147,12 @@ impl ProgressiveMerkleHasher { // If there are chunks in current level (partial level), compute their root let current_root = if self.current_level_chunks > 0 { // Create a temporary hasher to replace the current one (since finish() takes ownership) + // FIXME(sproul): get rid of this by making build_progressive_root a static method. let temp_hasher = std::mem::replace( &mut self.current_hasher, MerkleHasher::with_leaves(1), // dummy value, won't be used ); - Some( - temp_hasher - .finish() - .map_err(|_| Error::MaximumLeavesExceeded { - max_leaves: self.max_leaves, - })?, - ) + Some(temp_hasher.finish().map_err(Error::MerkleHasher)?) } else { None }; @@ -230,14 +198,14 @@ mod tests { #[test] fn test_empty_tree() { - let hasher = ProgressiveMerkleHasher::with_leaves(1); + let hasher = ProgressiveMerkleHasher::new(); let root = hasher.finish().unwrap(); assert_eq!(root, Hash256::ZERO); } #[test] fn test_single_chunk() { - let mut hasher = ProgressiveMerkleHasher::with_leaves(1); + let mut hasher = ProgressiveMerkleHasher::new(); let chunk = [1u8; BYTES_PER_CHUNK]; hasher.write(&chunk).unwrap(); let root = hasher.finish().unwrap(); @@ -254,7 +222,7 @@ mod tests { #[test] fn test_two_chunks() { - let mut hasher = ProgressiveMerkleHasher::with_leaves(5); + let mut hasher = ProgressiveMerkleHasher::new(); let chunk1 = [1u8; BYTES_PER_CHUNK]; let chunk2 = [2u8; BYTES_PER_CHUNK]; hasher.write(&chunk1).unwrap(); @@ -281,21 +249,9 @@ mod tests { assert_eq!(root, expected); } - #[test] - fn test_max_leaves_exceeded() { - let mut hasher = ProgressiveMerkleHasher::with_leaves(2); - let chunk = [1u8; BYTES_PER_CHUNK]; - hasher.write(&chunk).unwrap(); - hasher.write(&chunk).unwrap(); - - // Third write should fail - let result = hasher.write(&chunk); - assert!(matches!(result, Err(Error::MaximumLeavesExceeded { .. }))); - } - #[test] fn test_partial_chunk() { - let mut hasher = ProgressiveMerkleHasher::with_leaves(1); + let mut hasher = ProgressiveMerkleHasher::new(); let partial = vec![1u8, 2u8, 3u8]; hasher.write(&partial).unwrap(); let root = hasher.finish().unwrap(); @@ -315,7 +271,7 @@ mod tests { #[test] fn test_multiple_writes() { - let mut hasher = ProgressiveMerkleHasher::with_leaves(10); + let mut hasher = ProgressiveMerkleHasher::new(); hasher.write(&[1u8; 16]).unwrap(); hasher.write(&[2u8; 16]).unwrap(); hasher.write(&[3u8; 32]).unwrap(); @@ -325,18 +281,12 @@ mod tests { assert_ne!(root, Hash256::ZERO); } - #[test] - #[should_panic(expected = "must have at least one leaf")] - fn test_zero_leaves_panics() { - ProgressiveMerkleHasher::with_leaves(0); - } - #[test] fn test_five_chunks() { // Test with 5 chunks as per the problem statement structure: // chunks[0] goes to right at level 1 (1 leaf) // chunks[1..5] go to left recursive call (4 leaves at level 2) - let mut hasher = ProgressiveMerkleHasher::with_leaves(5); + let mut hasher = ProgressiveMerkleHasher::new(); for i in 0..5 { let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[0] = i as u8; @@ -376,7 +326,7 @@ mod tests { // chunks[0] goes to right at level 1 (1 leaf) // chunks[1..5] go to right at level 2 (4 leaves) // chunks[5..21] go to right at level 3 (16 leaves) - let mut hasher = ProgressiveMerkleHasher::with_leaves(21); + let mut hasher = ProgressiveMerkleHasher::new(); for i in 0..21 { let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[0] = i as u8; @@ -395,7 +345,7 @@ mod tests { // chunks[1..5] at level 2 (4 leaves) // chunks[5..21] at level 3 (16 leaves) // chunks[21..85] at level 4 (64 leaves) - let mut hasher = ProgressiveMerkleHasher::with_leaves(85); + let mut hasher = ProgressiveMerkleHasher::new(); for i in 0..85 { let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[0] = (i % 256) as u8; @@ -419,20 +369,20 @@ mod tests { .collect(); // Write all chunks individually - let mut hasher1 = ProgressiveMerkleHasher::with_leaves(10); + let mut hasher1 = ProgressiveMerkleHasher::new(); for chunk in &chunks { hasher1.write(chunk).unwrap(); } let root1 = hasher1.finish().unwrap(); // Write all chunks at once - let mut hasher2 = ProgressiveMerkleHasher::with_leaves(10); + let mut hasher2 = ProgressiveMerkleHasher::new(); let all_bytes: Vec = chunks.iter().flat_map(|c| c.iter().copied()).collect(); hasher2.write(&all_bytes).unwrap(); let root2 = hasher2.finish().unwrap(); // Write in groups - let mut hasher3 = ProgressiveMerkleHasher::with_leaves(10); + let mut hasher3 = ProgressiveMerkleHasher::new(); hasher3.write(&all_bytes[..3 * BYTES_PER_CHUNK]).unwrap(); hasher3 .write(&all_bytes[3 * BYTES_PER_CHUNK..7 * BYTES_PER_CHUNK]) @@ -450,12 +400,12 @@ mod tests { let data = vec![42u8; BYTES_PER_CHUNK * 3 + 10]; // Write all at once - let mut hasher1 = ProgressiveMerkleHasher::with_leaves(10); + let mut hasher1 = ProgressiveMerkleHasher::new(); hasher1.write(&data).unwrap(); let root1 = hasher1.finish().unwrap(); // Write in smaller chunks - let mut hasher2 = ProgressiveMerkleHasher::with_leaves(10); + let mut hasher2 = ProgressiveMerkleHasher::new(); hasher2.write(&data[0..50]).unwrap(); hasher2.write(&data[50..]).unwrap(); let root2 = hasher2.finish().unwrap(); diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index 95d98a4..3a97b27 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -95,12 +95,12 @@ fn tree_hash_derive_struct( let (impl_generics, ty_generics, where_clause) = &item.generics.split_for_impl(); let idents = get_hashable_fields(struct_data); - let num_leaves = idents.len(); - let hasher_type = if let StructBehaviour::ProgressiveContainer = struct_behaviour { - quote! { tree_hash::ProgressiveMerkleHasher } + let hasher_init = if let StructBehaviour::ProgressiveContainer = struct_behaviour { + quote! { tree_hash::ProgressiveMerkleHasher::new() } } else { - quote! { tree_hash::MerkleHasher } + let num_leaves = idents.len(); + quote! { tree_hash::MerkleHasher::with_leaves(#num_leaves) } }; let mixin_logic = if let StructBehaviour::ProgressiveContainer = struct_behaviour { let Some(active_fields) = active_fields_opt else { @@ -133,7 +133,7 @@ fn tree_hash_derive_struct( } fn tree_hash_root(&self) -> tree_hash::Hash256 { - let mut hasher = #hasher_type::with_leaves(#num_leaves); + let mut hasher = #hasher_init; #( hasher.write(self.#idents.tree_hash_root().as_slice()) From f101c0401a5820511f62203fb7d5e0416f60a6a4 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 11 Dec 2025 15:08:41 +1100 Subject: [PATCH 13/26] mix in length --- tree_hash/src/impls.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tree_hash/src/impls.rs b/tree_hash/src/impls.rs index c0068d7..dbe1b03 100644 --- a/tree_hash/src/impls.rs +++ b/tree_hash/src/impls.rs @@ -227,9 +227,10 @@ impl TreeHash for Bitfield { .write(self.as_slice()) .expect("ProgessiveBitList should not exceed tree hash leaf limit"); - hasher + let bitfield_root = hasher .finish() - .expect("ProgressiveBitList tree hash buffer should not exceed leaf limit") + .expect("ProgressiveBitList tree hash buffer should not exceed leaf limit"); + mix_in_length(&bitfield_root, self.len()) } } From f13637a2b81b1a3f70b445768219518c48ecb0d5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 17 Dec 2025 18:58:12 +1100 Subject: [PATCH 14/26] Add workaround for empty progressive bitlist --- tree_hash/src/impls.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tree_hash/src/impls.rs b/tree_hash/src/impls.rs index dbe1b03..bdafd55 100644 --- a/tree_hash/src/impls.rs +++ b/tree_hash/src/impls.rs @@ -222,6 +222,12 @@ impl TreeHash for Bitfield { } fn tree_hash_root(&self) -> Hash256 { + // FIXME(sproul): unclear if this is intended or a bug in the spec tests + // See: https://github.com/ethereum/consensus-specs/issues/4795 + if self.is_empty() { + return mix_in_length(&Hash256::ZERO, 0); + } + let mut hasher = ProgressiveMerkleHasher::new(); hasher .write(self.as_slice()) From 7400510546840a3985474380d934d33f9ba0d417 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 18 Dec 2025 21:50:17 +1100 Subject: [PATCH 15/26] Use active_fields correctly --- tree_hash_derive/src/lib.rs | 58 ++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index 3a97b27..454d0dc 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -102,20 +102,52 @@ fn tree_hash_derive_struct( let num_leaves = idents.len(); quote! { tree_hash::MerkleHasher::with_leaves(#num_leaves) } }; - let mixin_logic = if let StructBehaviour::ProgressiveContainer = struct_behaviour { - let Some(active_fields) = active_fields_opt else { - panic!("active_fields must be provided for progressive_container"); - }; - let packed_active_fields = active_fields.packed_tokens(); + // Compute the field hashes while accounting for inactive fields which hash as 0x0. + // + // The `mixin_logic` is the expression to mix in the `active_fields` in the case of a + // progressive container. + let (field_hashes, mixin_logic) = + if let StructBehaviour::ProgressiveContainer = struct_behaviour { + let Some(active_fields) = active_fields_opt else { + panic!("active_fields must be provided for progressive_container"); + }; - quote! { - const ACTIVE_FIELDS: [u8; 32] = #packed_active_fields; - tree_hash::mix_in_active_fields(container_root, ACTIVE_FIELDS) - } - } else { - quote! { container_root } - }; + let mut active_field_index = 0; + let mut field_hashes: Vec = vec![]; + for active in &active_fields.active_fields { + if *active { + let Some(ident) = idents.get(active_field_index) else { + panic!( + "active_fields is inconsistent with struct fields. \ + index: {active_field_index}, hashable fields: {}", + idents.len() + ) + }; + active_field_index += 1; + field_hashes.push(quote! { self.#ident.tree_hash_root() }); + } else { + field_hashes.push(quote! { tree_hash::Hash256::ZERO }); + } + } + + let packed_active_fields = active_fields.packed_tokens(); + + let mixin_logic = quote! { + const ACTIVE_FIELDS: [u8; 32] = #packed_active_fields; + tree_hash::mix_in_active_fields(container_root, ACTIVE_FIELDS) + }; + + (field_hashes, mixin_logic) + } else { + ( + idents + .into_iter() + .map(|ident| quote! { self.#ident.tree_hash_root() }) + .collect(), + quote! { container_root }, + ) + }; let output = quote! { impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause { @@ -136,7 +168,7 @@ fn tree_hash_derive_struct( let mut hasher = #hasher_init; #( - hasher.write(self.#idents.tree_hash_root().as_slice()) + hasher.write(#field_hashes.as_slice()) .expect("tree hash derive should not apply too many leaves"); )* From 2fd80712e4d85e5f61dc574d8f5837a5c9b355b8 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 12 Jan 2026 15:47:24 +1100 Subject: [PATCH 16/26] Implement compatible union support with manual selectors --- tree_hash_derive/src/attrs.rs | 8 +++ tree_hash_derive/src/lib.rs | 102 ++++++++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/tree_hash_derive/src/attrs.rs b/tree_hash_derive/src/attrs.rs index 83426d2..4a7da1d 100644 --- a/tree_hash_derive/src/attrs.rs +++ b/tree_hash_derive/src/attrs.rs @@ -15,10 +15,18 @@ pub struct StructOpts { pub active_fields: Option, } +/// Variant-level configuration (for enums). +#[derive(Debug, Default, FromMeta)] +pub struct VariantOpts { + #[darling(default)] + pub selector: Option, +} + #[derive(Debug, FromMeta)] pub enum EnumBehaviour { Transparent, Union, + CompatibleUnion, } #[derive(Debug, Default, FromMeta)] diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index 454d0dc..cb998fe 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -1,12 +1,12 @@ #![recursion_limit = "256"] mod attrs; -use crate::attrs::{EnumBehaviour, StructBehaviour, StructOpts}; -use darling::FromDeriveInput; +use crate::attrs::{EnumBehaviour, StructBehaviour, StructOpts, VariantOpts}; +use darling::{FromDeriveInput, FromMeta}; use proc_macro::TokenStream; use quote::quote; use std::convert::TryInto; -use syn::{parse_macro_input, DataEnum, DataStruct, DeriveInput, Ident}; +use syn::{parse_macro_input, Attribute, DataEnum, DataStruct, DeriveInput, Ident}; /// The highest possible union selector value (higher values are reserved for backwards compatible /// extensions). @@ -50,7 +50,7 @@ fn get_hashable_fields_and_their_caches( /// The field attribute is: `#[tree_hash(skip_hashing)]` fn should_skip_hashing(field: &syn::Field) -> bool { field.attrs.iter().any(|attr| { - attr.path().is_ident("tree_hash") && attr.parse_args::().unwrap() == "skip_hashing" + is_tree_hash_attr(attr) && attr.parse_args::().unwrap() == "skip_hashing" }) } @@ -78,7 +78,9 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream { } match enum_behaviour { EnumBehaviour::Transparent => tree_hash_derive_enum_transparent(&item, s), - EnumBehaviour::Union => tree_hash_derive_enum_union(&item, s), + EnumBehaviour::Union | EnumBehaviour::CompatibleUnion => { + tree_hash_derive_enum_union(&item, s, enum_behaviour) + } } } _ => panic!("tree_hash_derive only supports structs and enums"), @@ -259,19 +261,30 @@ fn tree_hash_derive_enum_transparent( output.into() } -/// Derive `TreeHash` for an `enum` following the "union" SSZ spec. +/// Derive `TreeHash` for an `enum` following the compatible union or ordinary union SSZ spec. +/// +/// The union selectors for a compatible union MUST be defined using the variant attribute: /// -/// The union selector will be determined based upon the order in which the enum variants are -/// defined. E.g., the top-most variant in the enum will have a selector of `0`, the variant -/// beneath it will have a selector of `1` and so on. +/// - `tree_hash(selector = "X")` +/// +/// The union selectors for an ordinary union will be determined based upon the order in which the +/// enum variants are defined. E.g., the top-most variant in the enum will have a selector of `0`, +/// the variant beneath it will have a selector of `1` and so on. /// /// # Limitations /// /// Only supports enums where each variant has a single field. -fn tree_hash_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream { +fn tree_hash_derive_enum_union( + derive_input: &DeriveInput, + enum_data: &DataEnum, + enum_behaviour: EnumBehaviour, +) -> TokenStream { let name = &derive_input.ident; let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl(); + // Parse variant-level configuration. + let variant_opts = parse_variant_opts(enum_data); + let patterns: Vec<_> = enum_data .variants .iter() @@ -288,7 +301,18 @@ fn tree_hash_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) }) .collect(); - let union_selectors = compute_union_selectors(patterns.len()); + let union_selectors = match enum_behaviour { + EnumBehaviour::CompatibleUnion => get_compatible_union_selectors(enum_data, &variant_opts), + EnumBehaviour::Union => { + // For now we don't allow selectors in ordinary unions to be set manually. + assert!( + variant_opts.iter().all(|opt| opt.selector.is_none()), + "specifying the selector in a regular union is not supported" + ); + compute_union_selectors(patterns.len()) + } + EnumBehaviour::Transparent => unreachable!("union code called for transparent enum"), + }; let output = quote! { impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause { @@ -321,6 +345,41 @@ fn tree_hash_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) output.into() } +fn parse_variant_opts(enum_data: &DataEnum) -> Vec { + enum_data + .variants + .iter() + .map(|variant| { + let tree_hash_attrs = variant + .attrs + .iter() + .filter(|attr| is_tree_hash_attr(attr)) + .collect::>(); + + if tree_hash_attrs.len() > 1 { + panic!("more than one variant-level \"tree_hash\" attribute provided"); + } + + tree_hash_attrs + .first() + .map(|attr| VariantOpts::from_meta(&attr.meta).unwrap()) + .unwrap_or_default() + }) + .collect() +} + +/// Predicate for determining whether an attribute is a `tree_hash` attribute. +fn is_tree_hash_attr(attr: &Attribute) -> bool { + is_attr_with_ident(attr, "tree_hash") +} + +/// Predicate for determining whether an attribute has the given `ident` as its path. +fn is_attr_with_ident(attr: &Attribute, ident: &str) -> bool { + attr.path() + .get_ident() + .is_some_and(|attr_ident| *attr_ident == ident) +} + fn compute_union_selectors(num_variants: usize) -> Vec { let union_selectors = (0..num_variants) .map(|i| { @@ -343,3 +402,24 @@ fn compute_union_selectors(num_variants: usize) -> Vec { union_selectors } + +fn get_compatible_union_selectors(enum_data: &DataEnum, variant_opts: &[VariantOpts]) -> Vec { + enum_data + .variants + .iter() + .zip(variant_opts.iter()) + .map(|(variant, variant_opt)| { + let variant_name = &variant.ident; + let Some(selector) = variant_opt.selector else { + panic!("you must define a selector for variant \"{variant_name}\""); + }; + if selector == 0 || selector > MAX_UNION_SELECTOR { + panic!( + "selector = {selector} for variant \"{variant_name}\" is illegal in a \ + compatible union" + ); + } + selector + }) + .collect() +} From 4f7349913ba9ac8cce7112b2279364f8fcac1f5d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 13 Jan 2026 11:52:18 +1100 Subject: [PATCH 17/26] Read/verify ssz attributes as well as tree_hash --- tree_hash_derive/src/attrs.rs | 6 +++++- tree_hash_derive/src/lib.rs | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/tree_hash_derive/src/attrs.rs b/tree_hash_derive/src/attrs.rs index 4a7da1d..08021a7 100644 --- a/tree_hash_derive/src/attrs.rs +++ b/tree_hash_derive/src/attrs.rs @@ -16,7 +16,11 @@ pub struct StructOpts { } /// Variant-level configuration (for enums). -#[derive(Debug, Default, FromMeta)] +/// +/// These attributes NEED to be kept in sync with `ethereum_ssz` because both crates try to read +/// each others attributes to avoid mandatory duplication. In future this might mean parsing some +/// SSZ-only attributes here and then ignoring them. +#[derive(Debug, Default, PartialEq, FromMeta)] pub struct VariantOpts { #[darling(default)] pub selector: Option, diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index cb998fe..ef79e59 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -355,15 +355,39 @@ fn parse_variant_opts(enum_data: &DataEnum) -> Vec { .iter() .filter(|attr| is_tree_hash_attr(attr)) .collect::>(); + let ssz_attrs = variant + .attrs + .iter() + .filter(|attr| is_ssz_attr(attr)) + .collect::>(); + // Check for duplicate `tree_hash` attributes. + // Checking duplicate `ssz` attributes is the job of the `ssz_derive` macro. if tree_hash_attrs.len() > 1 { panic!("more than one variant-level \"tree_hash\" attribute provided"); } - tree_hash_attrs + let tree_hash_opts = tree_hash_attrs .first() - .map(|attr| VariantOpts::from_meta(&attr.meta).unwrap()) - .unwrap_or_default() + .map(|attr| VariantOpts::from_meta(&attr.meta).unwrap()); + + let ssz_opts = ssz_attrs + .first() + .map(|attr| VariantOpts::from_meta(&attr.meta).unwrap()); + + // Check consistency with SSZ opts, or fall back to SSZ attribute if tree_hash attribute + // is absent. + match (tree_hash_opts, ssz_opts) { + (Some(tree_hash), Some(ssz)) => { + assert_eq!( + tree_hash, ssz, + "inconsistent \"tree_hash\" and \"ssz\" attributes" + ); + tree_hash + } + (Some(attr), None) | (None, Some(attr)) => attr, + (None, None) => VariantOpts::default(), + } }) .collect() } @@ -373,6 +397,10 @@ fn is_tree_hash_attr(attr: &Attribute) -> bool { is_attr_with_ident(attr, "tree_hash") } +fn is_ssz_attr(attr: &Attribute) -> bool { + is_attr_with_ident(attr, "ssz") +} + /// Predicate for determining whether an attribute has the given `ident` as its path. fn is_attr_with_ident(attr: &Attribute, ident: &str) -> bool { attr.path() From 656b85ecc5380a30c47acf332e9caa41fe52067f Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 13 Jan 2026 13:03:50 +1100 Subject: [PATCH 18/26] Document quirk of bitfield hashing --- tree_hash/src/impls.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tree_hash/src/impls.rs b/tree_hash/src/impls.rs index bdafd55..e12a7e7 100644 --- a/tree_hash/src/impls.rs +++ b/tree_hash/src/impls.rs @@ -222,8 +222,12 @@ impl TreeHash for Bitfield { } fn tree_hash_root(&self) -> Hash256 { - // FIXME(sproul): unclear if this is intended or a bug in the spec tests - // See: https://github.com/ethereum/consensus-specs/issues/4795 + // XXX: This is a workaround for the fact that the internal representation of bitfields is + // misaligned with the spec. + // + // See: + // + // - https://github.com/sigp/ethereum_ssz/pull/68 if self.is_empty() { return mix_in_length(&Hash256::ZERO, 0); } From 9248e51986187cf81f56928538d88e4072c781e7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 13 Jan 2026 13:07:09 +1100 Subject: [PATCH 19/26] Remove inelegant temporary hasher --- tree_hash/src/progressive_merkle_hasher.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index a6b5ed3..a61ad57 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -146,13 +146,7 @@ impl ProgressiveMerkleHasher { // If there are chunks in current level (partial level), compute their root let current_root = if self.current_level_chunks > 0 { - // Create a temporary hasher to replace the current one (since finish() takes ownership) - // FIXME(sproul): get rid of this by making build_progressive_root a static method. - let temp_hasher = std::mem::replace( - &mut self.current_hasher, - MerkleHasher::with_leaves(1), // dummy value, won't be used - ); - Some(temp_hasher.finish().map_err(Error::MerkleHasher)?) + Some(self.current_hasher.finish().map_err(Error::MerkleHasher)?) } else { None }; @@ -160,14 +154,20 @@ impl ProgressiveMerkleHasher { // Build the progressive tree from completed roots and current root // completed_roots are in order: [smallest level, ..., largest level] // We need to build from right to left in the tree - Ok(self.build_progressive_root(current_root)) + Ok(Self::build_progressive_root( + current_root, + self.completed_roots, + )) } /// Build the final progressive merkle root by combining completed subtree roots. /// /// The progressive tree structure: at each node, hash(left=deeper_levels, right=this_level). /// This builds the tree from the largest (leftmost) level backwards to the smallest (rightmost). - fn build_progressive_root(&self, current_root: Option) -> Hash256 { + fn build_progressive_root( + current_root: Option, + completed_roots: Vec, + ) -> Hash256 { // Start from the leftmost (largest/deepest) level // Per EIP-7916 spec, even partial levels follow the progressive structure: // merkleize_progressive(chunks, n) = hash(merkleize_progressive(chunks[n:], n*4), merkleize(chunks[:n], n)) @@ -182,7 +182,7 @@ impl ProgressiveMerkleHasher { // At each step: result = hash(result, completed_root) // - result accumulates the left subtree (deeper/larger levels) // - completed_root is the right subtree at this level - for &completed_root in self.completed_roots.iter().rev() { + for &completed_root in completed_roots.iter().rev() { result = Hash256::from_slice(&hash32_concat(result.as_slice(), completed_root.as_slice())); } From 528a1a7c1dbf6532720dcf9f5a09c51642ffbb0e Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 13 Jan 2026 13:07:49 +1100 Subject: [PATCH 20/26] Remove from_slice --- tree_hash/src/progressive_merkle_hasher.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index a61ad57..e11233f 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -183,8 +183,7 @@ impl ProgressiveMerkleHasher { // - result accumulates the left subtree (deeper/larger levels) // - completed_root is the right subtree at this level for &completed_root in completed_roots.iter().rev() { - result = - Hash256::from_slice(&hash32_concat(result.as_slice(), completed_root.as_slice())); + result = Hash256::from(hash32_concat(result.as_slice(), completed_root.as_slice())); } result From 2f96e24a77a34bda9cd622ade5dd83280bdce183 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 13 Jan 2026 14:46:35 +1100 Subject: [PATCH 21/26] Document TreeHashType choice --- tree_hash_derive/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index ef79e59..28eff58 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -154,7 +154,10 @@ fn tree_hash_derive_struct( let output = quote! { impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause { fn tree_hash_type() -> tree_hash::TreeHashType { - // FIXME(sproul): consider adjusting this with active_fields + // There is no specific type for progressive containers, although we could consider + // adding one in future if it would prove useful. + // + // See: https://github.com/sigp/tree_hash/issues/44 tree_hash::TreeHashType::Container } From 817387ccc16fe8f2ae119da7d33cf12c4925854a Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 3 Feb 2026 20:57:43 +1100 Subject: [PATCH 22/26] Reverse tree order to match spec --- tree_hash/src/progressive_merkle_hasher.rs | 108 ++++++++++----------- tree_hash/tests/tests.rs | 2 +- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index e11233f..7889592 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -10,23 +10,23 @@ pub enum Error { /// defined in EIP-7916. /// /// The progressive merkle tree has a unique structure where: -/// - At each level, the right child is a binary merkle tree with a specific number of leaves -/// - The left child recursively contains more progressive structure -/// - The number of leaves in each right subtree grows by 4x at each level (1, 4, 16, 64, ...) +/// - At each level, the left child is a binary merkle tree with a specific number of leaves +/// - The right child recursively contains more progressive structure +/// - The number of leaves in each left subtree grows by 4x at each level (1, 4, 16, 64, ...) /// /// # Example Tree Structure /// /// ```text -/// root -/// /\ -/// / \ -/// /\ 1: chunks[0 ..< 1] -/// / \ -/// /\ 4: chunks[1 ..< 5] -/// / \ -/// /\ 16: chunks[5 ..< 21] -/// / \ -/// 0 64: chunks[21 ..< 85] +/// root +/// /\ +/// / \ +/// 1: chunks[0 ..< 1] /\ +/// / \ +/// 4: chunks[1 ..< 5] /\ +/// / \ +/// 16: chunks[5 ..< 21] /\ +/// / \ +/// 64: chunks[21 ..< 85] 0 /// ``` /// /// This structure allows efficient appending and proof generation for growing lists. @@ -153,7 +153,7 @@ impl ProgressiveMerkleHasher { // Build the progressive tree from completed roots and current root // completed_roots are in order: [smallest level, ..., largest level] - // We need to build from right to left in the tree + // We need to build from left to right in the tree Ok(Self::build_progressive_root( current_root, self.completed_roots, @@ -162,28 +162,28 @@ impl ProgressiveMerkleHasher { /// Build the final progressive merkle root by combining completed subtree roots. /// - /// The progressive tree structure: at each node, hash(left=deeper_levels, right=this_level). - /// This builds the tree from the largest (leftmost) level backwards to the smallest (rightmost). + /// The progressive tree structure: at each node, hash(left=this_level, right=deeper_levels). + /// This builds the tree from the largest (rightmost) level backwards to the smallest (leftmost). fn build_progressive_root( current_root: Option, completed_roots: Vec, ) -> Hash256 { - // Start from the leftmost (largest/deepest) level + // Start from the rightmost (largest/deepest) level // Per EIP-7916 spec, even partial levels follow the progressive structure: - // merkleize_progressive(chunks, n) = hash(merkleize_progressive(chunks[n:], n*4), merkleize(chunks[:n], n)) - // So a partial level with k chunks becomes: hash(ZERO (no further chunks), merkleize(chunks, n)) + // merkleize_progressive(chunks, n) = hash(merkleize(chunks[:n], n), merkleize_progressive(chunks[n:], n*4)) + // So a partial level with k chunks becomes: hash(merkleize(chunks, n), ZERO (no further chunks)) let mut result = if let Some(curr) = current_root { - Hash256::from_slice(&hash32_concat(Hash256::ZERO.as_slice(), curr.as_slice())) + Hash256::from_slice(&hash32_concat(curr.as_slice(), Hash256::ZERO.as_slice())) } else { Hash256::ZERO }; // Process completed roots from largest to smallest (reverse order) - // At each step: result = hash(result, completed_root) - // - result accumulates the left subtree (deeper/larger levels) - // - completed_root is the right subtree at this level + // At each step: result = hash(completed_root, result) + // - completed_root is the left subtree at this level (binary tree) + // - result accumulates the right subtree (deeper/larger levels) for &completed_root in completed_roots.iter().rev() { - result = Hash256::from(hash32_concat(result.as_slice(), completed_root.as_slice())); + result = Hash256::from(hash32_concat(completed_root.as_slice(), result.as_slice())); } result @@ -210,11 +210,11 @@ mod tests { let root = hasher.finish().unwrap(); // For a single chunk, the progressive tree should be: - // hash(merkleize_progressive([], 4), merkleize([chunk], 1)) - // = hash(zero_hash, chunk) - let zero_left = Hash256::ZERO; - let right = Hash256::from_slice(&chunk); - let expected = Hash256::from_slice(&hash32_concat(zero_left.as_slice(), right.as_slice())); + // hash(merkleize([chunk], 1), merkleize_progressive([], 4)) + // = hash(chunk, zero_hash) + let left = Hash256::from_slice(&chunk); + let zero_right = Hash256::ZERO; + let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), zero_right.as_slice())); assert_eq!(root, expected); } @@ -228,20 +228,20 @@ mod tests { hasher.write(&chunk2).unwrap(); let root = hasher.finish().unwrap(); - // First chunk goes to right (num_leaves=1) - // Second chunk goes to left recursive call (num_leaves=4) + // First chunk goes to left (num_leaves=1) + // Second chunk goes to right recursive call (num_leaves=4) - // Right: binary tree with 1 leaf = chunk1 - let right = Hash256::from_slice(&chunk1); + // Left: binary tree with 1 leaf = chunk1 + let left = Hash256::from_slice(&chunk1); - // Left: progressive tree with chunk2 at num_leaves=4 - // At this level: hash(merkleize_progressive([], 16), merkleize([chunk2], 4)) - // = hash(zero_hash, merkle([chunk2], 4)) + // Right: progressive tree with chunk2 at num_leaves=4 + // At this level: hash(merkleize([chunk2], 4), merkleize_progressive([], 16)) + // = hash(merkle([chunk2], 4), zero_hash) let chunk2_padded = merkle_root(&chunk2, 4); - let zero_left_inner = Hash256::ZERO; - let left = Hash256::from_slice(&hash32_concat( - zero_left_inner.as_slice(), + let zero_right_inner = Hash256::ZERO; + let right = Hash256::from_slice(&hash32_concat( chunk2_padded.as_slice(), + zero_right_inner.as_slice(), )); let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), right.as_slice())); @@ -261,9 +261,9 @@ mod tests { chunk[1] = 2; chunk[2] = 3; - let zero_left = Hash256::ZERO; - let right = Hash256::from_slice(&chunk); - let expected = Hash256::from_slice(&hash32_concat(zero_left.as_slice(), right.as_slice())); + let left = Hash256::from_slice(&chunk); + let zero_right = Hash256::ZERO; + let expected = Hash256::from_slice(&hash32_concat(left.as_slice(), zero_right.as_slice())); assert_eq!(root, expected); } @@ -283,8 +283,8 @@ mod tests { #[test] fn test_five_chunks() { // Test with 5 chunks as per the problem statement structure: - // chunks[0] goes to right at level 1 (1 leaf) - // chunks[1..5] go to left recursive call (4 leaves at level 2) + // chunks[0] goes to left at level 1 (1 leaf) + // chunks[1..5] go to right recursive call (4 leaves at level 2) let mut hasher = ProgressiveMerkleHasher::new(); for i in 0..5 { let mut chunk = [0u8; BYTES_PER_CHUNK]; @@ -294,13 +294,13 @@ mod tests { let root = hasher.finish().unwrap(); // Manually compute expected root: - // Right: chunks[0] + // Left: chunks[0] let mut chunk0 = [0u8; BYTES_PER_CHUNK]; chunk0[0] = 0; - let right = Hash256::from_slice(&chunk0); + let left = Hash256::from_slice(&chunk0); - // Left: merkleize_progressive(chunks[1..5], 4) - // Which is: hash(merkleize_progressive([], 16), merkleize(chunks[1..5], 4)) + // Right: merkleize_progressive(chunks[1..5], 4) + // Which is: hash(merkleize(chunks[1..5], 4), merkleize_progressive([], 16)) let chunks_1_to_4: Vec = (1..5) .flat_map(|i| { let mut chunk = [0u8; BYTES_PER_CHUNK]; @@ -308,9 +308,9 @@ mod tests { chunk }) .collect(); - let right_inner = merkle_root(&chunks_1_to_4, 4); - let left_inner = Hash256::ZERO; - let left = Hash256::from_slice(&hash32_concat( + let left_inner = merkle_root(&chunks_1_to_4, 4); + let right_inner = Hash256::ZERO; + let right = Hash256::from_slice(&hash32_concat( left_inner.as_slice(), right_inner.as_slice(), )); @@ -322,9 +322,9 @@ mod tests { #[test] fn test_21_chunks() { // Test with 21 chunks as per problem statement: - // chunks[0] goes to right at level 1 (1 leaf) - // chunks[1..5] go to right at level 2 (4 leaves) - // chunks[5..21] go to right at level 3 (16 leaves) + // chunks[0] goes to left at level 1 (1 leaf) + // chunks[1..5] go to left at level 2 (4 leaves) + // chunks[5..21] go to left at level 3 (16 leaves) let mut hasher = ProgressiveMerkleHasher::new(); for i in 0..21 { let mut chunk = [0u8; BYTES_PER_CHUNK]; diff --git a/tree_hash/tests/tests.rs b/tree_hash/tests/tests.rs index f9f22e0..aa85283 100644 --- a/tree_hash/tests/tests.rs +++ b/tree_hash/tests/tests.rs @@ -180,7 +180,7 @@ fn progressive_container_one_field() { let container = ProgressiveContainerOneField { x: 125 }; assert_eq!( container.tree_hash_root(), - Hash256::from_str("0xfacc8073916cbe1d3e400f69945fb5b6423d1e8f99be04713bcbe254fad2c94c") + Hash256::from_str("0xb6a2f148c33179dec1bdaa979a11776ff2d881fca93974b286443a8539dc0872") .unwrap() ); } From 4c9b0407f6e9ac3138117246cf9f0bbe828e4faf Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 3 Feb 2026 22:06:53 +1100 Subject: [PATCH 23/26] Fix clippy --- tree_hash/src/progressive_merkle_hasher.rs | 6 ++++++ tree_hash_derive/src/attrs.rs | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index 7889592..0fef7d5 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -190,6 +190,12 @@ impl ProgressiveMerkleHasher { } } +impl Default for ProgressiveMerkleHasher { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tree_hash_derive/src/attrs.rs b/tree_hash_derive/src/attrs.rs index 08021a7..02529fc 100644 --- a/tree_hash_derive/src/attrs.rs +++ b/tree_hash_derive/src/attrs.rs @@ -67,7 +67,7 @@ impl FromMeta for ActiveFields { impl ActiveFields { fn new(active_fields: Vec) -> Result { if active_fields.is_empty() { - return Err(Error::custom(format!("active_fields must be non-empty"))); + return Err(Error::custom("active_fields must be non-empty".to_string())); } if active_fields.len() > MAX_ACTIVE_FIELDS { return Err(Error::custom(format!( @@ -76,9 +76,9 @@ impl ActiveFields { } if let Some(false) = active_fields.last() { - return Err(Error::custom(format!( - "the last entry of active_fields must not be 0" - ))); + return Err(Error::custom( + "the last entry of active_fields must not be 0".to_string(), + )); } Ok(Self { active_fields }) From 858514b0f5ab5b0c4b669e932261d5cd86510d09 Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 3 Feb 2026 22:11:34 +1100 Subject: [PATCH 24/26] Use git dep instead of path dep --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 23dbcc4..113c7f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,5 +14,6 @@ repository = "https://github.com/sigp/tree_hash" keywords = ["ethereum"] categories = ["cryptography::cryptocurrencies"] +# TODO: Remove once ethereum_ssz is published. [patch.crates-io] -ethereum_ssz = { path = "../ethereum_ssz/ssz" } +ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" } From ca07459738c9b1584eec99e37f3a52774fa97dd0 Mon Sep 17 00:00:00 2001 From: Mac L Date: Fri, 6 Feb 2026 01:00:43 +1100 Subject: [PATCH 25/26] Fix typo in expect --- tree_hash/src/impls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree_hash/src/impls.rs b/tree_hash/src/impls.rs index e12a7e7..f2e64d2 100644 --- a/tree_hash/src/impls.rs +++ b/tree_hash/src/impls.rs @@ -235,7 +235,7 @@ impl TreeHash for Bitfield { let mut hasher = ProgressiveMerkleHasher::new(); hasher .write(self.as_slice()) - .expect("ProgessiveBitList should not exceed tree hash leaf limit"); + .expect("ProgressiveBitList should not exceed tree hash leaf limit"); let bitfield_root = hasher .finish() From 5e272f38aa290179f62552820ce6edb0ba43fc20 Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 30 Jun 2026 02:56:21 +1000 Subject: [PATCH 26/26] General tidy up --- Cargo.toml | 1 + tree_hash/src/impls.rs | 12 +- tree_hash/src/lib.rs | 29 ++- tree_hash/src/merkle_hasher.rs | 13 + tree_hash/src/progressive_merkle_hasher.rs | 267 ++++++++++++--------- tree_hash/tests/tests.rs | 173 ++++++++++++- tree_hash_derive/src/attrs.rs | 125 +++++++++- tree_hash_derive/src/lib.rs | 127 ++++++---- 8 files changed, 577 insertions(+), 170 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1e526c4..eb43e66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,4 @@ categories = ["cryptography::cryptocurrencies"] # TODO: Remove once ethereum_ssz is published. [patch.crates-io] ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" } +ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" } diff --git a/tree_hash/src/impls.rs b/tree_hash/src/impls.rs index f2e64d2..0990b31 100644 --- a/tree_hash/src/impls.rs +++ b/tree_hash/src/impls.rs @@ -214,16 +214,18 @@ impl TreeHash for Bitfield { } fn tree_hash_packed_encoding(&self) -> PackedEncoding { - unreachable!("ProgressiveBitField should never be packed.") + unreachable!("ProgressiveBitList should never be packed.") } fn tree_hash_packing_factor() -> usize { - unreachable!("ProgressiveBitField should never be packed.") + unreachable!("ProgressiveBitList should never be packed.") } fn tree_hash_root(&self) -> Hash256 { // XXX: This is a workaround for the fact that the internal representation of bitfields is - // misaligned with the spec. + // misaligned with the spec. An empty bitlist still stores a single zero byte (see + // `bytes_for_bit_len`), which would otherwise be hashed as a (non-zero) zero chunk rather + // than yielding the empty `merkleize_progressive([]) == Bytes32()` root. // // See: // @@ -235,11 +237,11 @@ impl TreeHash for Bitfield { let mut hasher = ProgressiveMerkleHasher::new(); hasher .write(self.as_slice()) - .expect("ProgressiveBitList should not exceed tree hash leaf limit"); + .expect("ProgressiveMerkleHasher has no leaf limit, so write cannot fail"); let bitfield_root = hasher .finish() - .expect("ProgressiveBitList tree hash buffer should not exceed leaf limit"); + .expect("ProgressiveMerkleHasher has no leaf limit, so finish cannot fail"); mix_in_length(&bitfield_root, self.len()) } } diff --git a/tree_hash/src/lib.rs b/tree_hash/src/lib.rs index f9e2278..3d53272 100644 --- a/tree_hash/src/lib.rs +++ b/tree_hash/src/lib.rs @@ -65,7 +65,10 @@ pub fn mix_in_length(root: &Hash256, length: usize) -> Hash256 { let mut length_bytes = [0; BYTES_PER_CHUNK]; length_bytes[0..usize_len].copy_from_slice(&length.to_le_bytes()); - Hash256::from_slice(ðereum_hashing::hash32_concat(root.as_slice(), &length_bytes)[..]) + Hash256::from(ethereum_hashing::hash32_concat( + root.as_slice(), + &length_bytes, + )) } /// Returns `Some(root)` created by hashing `root` and `selector`, if `selector <= @@ -90,10 +93,13 @@ pub fn mix_in_selector(root: &Hash256, selector: u8) -> Option { chunk[0] = selector; let root = ethereum_hashing::hash32_concat(root.as_slice(), &chunk); - Some(Hash256::from_slice(&root)) + Some(Hash256::from(root)) } -pub fn mix_in_active_fields(root: Hash256, active_fields: [u8; BYTES_PER_CHUNK]) -> Hash256 { +/// Returns the node created by hashing `root` and the packed `active_fields` bitvector chunk. +/// +/// Used in `TreeHash` for the progressive container type. +pub fn mix_in_active_fields(root: &Hash256, active_fields: [u8; BYTES_PER_CHUNK]) -> Hash256 { Hash256::from(ethereum_hashing::hash32_concat( root.as_slice(), &active_fields, @@ -213,4 +219,21 @@ mod test { &hash[..] ); } + + #[test] + fn mix_active_fields() { + let root = Hash256::from_slice(&[42; BYTES_PER_CHUNK]); + // A multi-bit packed vector (bits 0 and 2 set) so an argument swap or wrong-bytes + // regression would be visible. + let mut active_fields = [0u8; BYTES_PER_CHUNK]; + active_fields[0] = 0b0000_0101; + + assert_eq!( + mix_in_active_fields(&root, active_fields), + Hash256::from(ethereum_hashing::hash32_concat( + root.as_slice(), + &active_fields + )) + ); + } } diff --git a/tree_hash/src/merkle_hasher.rs b/tree_hash/src/merkle_hasher.rs index 07d4352..526c78d 100644 --- a/tree_hash/src/merkle_hasher.rs +++ b/tree_hash/src/merkle_hasher.rs @@ -11,6 +11,19 @@ pub enum Error { MaximumLeavesExceeded { max_leaves: usize }, } +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::MaximumLeavesExceeded { max_leaves } => write!( + f, + "the maximum number of leaves ({max_leaves}) has been exceeded" + ), + } + } +} + +impl std::error::Error for Error {} + /// Helper struct to store either a hash digest or a slice. /// /// Should be used as a left or right value for some node. diff --git a/tree_hash/src/progressive_merkle_hasher.rs b/tree_hash/src/progressive_merkle_hasher.rs index 0fef7d5..3e8208f 100644 --- a/tree_hash/src/progressive_merkle_hasher.rs +++ b/tree_hash/src/progressive_merkle_hasher.rs @@ -1,11 +1,28 @@ use crate::{Hash256, MerkleHasher, BYTES_PER_CHUNK}; use ethereum_hashing::hash32_concat; +use smallvec::SmallVec; #[derive(Clone, Debug, PartialEq)] pub enum Error { MerkleHasher(crate::merkle_hasher::Error), } +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::MerkleHasher(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::MerkleHasher(e) => Some(e), + } + } +} + /// A progressive Merkle hasher that implements the semantics of `merkleize_progressive` as /// defined in EIP-7916. /// @@ -41,16 +58,17 @@ pub struct ProgressiveMerkleHasher { /// Index 0 = first completed level (1 leaf), index 1 = second level (4 leaves), etc. /// Level i contains 4^i leaves. completed_roots: Vec, - /// MerkleHasher for computing the current level's binary tree root. - current_hasher: MerkleHasher, + /// `MerkleHasher` for computing the current level's binary tree root, created lazily on the + /// first chunk of each level so that a level completed by its final chunk does not allocate a + /// hasher that is never used. + current_hasher: Option, /// The number of leaves expected at the current level (1, 4, 16, 64, ...). current_level_size: usize, /// Number of chunks written to the current hasher. current_level_chunks: usize, - /// Buffer for bytes that haven't been completed into a chunk yet. - buffer: Vec, - /// Total number of chunks written so far. - total_chunks: usize, + /// Carry for bytes that haven't been completed into a chunk yet. Always < 32 bytes, so this + /// never spills to the heap. + buffer: SmallVec<[u8; BYTES_PER_CHUNK]>, } impl ProgressiveMerkleHasher { @@ -58,11 +76,10 @@ impl ProgressiveMerkleHasher { pub fn new() -> Self { Self { completed_roots: Vec::new(), - current_hasher: MerkleHasher::with_leaves(1), + current_hasher: None, current_level_size: 1, current_level_chunks: 0, - buffer: Vec::new(), - total_chunks: 0, + buffer: SmallVec::new(), } } @@ -74,51 +91,63 @@ impl ProgressiveMerkleHasher { /// /// # Errors /// - /// Returns an error if writing these bytes would exceed the maximum number of leaves. + /// This hasher imposes no leaf limit, so under normal use `write` does not fail; the `Result` + /// only forwards an error from the internal per-level [`MerkleHasher`], which is not expected + /// to occur. pub fn write(&mut self, bytes: &[u8]) -> Result<(), Error> { - // Add bytes to buffer - self.buffer.extend_from_slice(bytes); + let mut bytes = bytes; - // Process complete chunks from buffer - while self.buffer.len() >= BYTES_PER_CHUNK { - let mut chunk = [0u8; BYTES_PER_CHUNK]; - chunk.copy_from_slice(&self.buffer[..BYTES_PER_CHUNK]); - self.buffer.drain(..BYTES_PER_CHUNK); + // If a partial carry exists, top it up from the front of `bytes` to complete a chunk. + if !self.buffer.is_empty() { + let take = (BYTES_PER_CHUNK - self.buffer.len()).min(bytes.len()); + self.buffer.extend_from_slice(&bytes[..take]); + bytes = &bytes[take..]; + + // Still short of a full chunk; nothing more to do. + if self.buffer.len() < BYTES_PER_CHUNK { + return Ok(()); + } + let chunk = std::mem::take(&mut self.buffer); + self.process_chunk(&chunk)?; + } + + // Forward each complete 32-byte chunk directly, then carry the remainder. + let mut chunks = bytes.chunks_exact(BYTES_PER_CHUNK); + for chunk in &mut chunks { self.process_chunk(chunk)?; } + self.buffer.extend_from_slice(chunks.remainder()); Ok(()) } - /// Process a single chunk by adding it to the current level and completing the level if full. - fn process_chunk(&mut self, chunk: [u8; BYTES_PER_CHUNK]) -> Result<(), Error> { - // Write the chunk to the current MerkleHasher - self.current_hasher - .write(&chunk) - .map_err(Error::MerkleHasher)?; + /// Process a single 32-byte chunk by adding it to the current level and completing the level if + /// full. + fn process_chunk(&mut self, chunk: &[u8]) -> Result<(), Error> { + // Lazily create the current level's hasher, so a level completed by its final chunk does + // not allocate a hasher that is immediately discarded. + let level_size = self.current_level_size; + let hasher = self + .current_hasher + .get_or_insert_with(|| MerkleHasher::with_leaves(level_size)); + hasher.write(chunk).map_err(Error::MerkleHasher)?; self.current_level_chunks += 1; - self.total_chunks += 1; - // Check if current level is complete + // Once the current level is full, finalize its binary root and advance to the next level + // (4x larger). `current_level_size` grows as 4^i, which cannot overflow `usize` in + // practice: completing level i requires streaming 4^i chunks, far beyond any realizable + // input. if self.current_level_chunks == self.current_level_size { - // Move to next level (4x larger) - let next_level_size = self.current_level_size * 4; - - // Replace the current hasher with a new one for the next level - let completed_hasher = std::mem::replace( - &mut self.current_hasher, - MerkleHasher::with_leaves(next_level_size), - ); - - // Finish the completed hasher to get the root + let completed_hasher = self + .current_hasher + .take() + .expect("current hasher exists after a chunk has been written"); let root = completed_hasher.finish().map_err(Error::MerkleHasher)?; - - // Store this completed root self.completed_roots.push(root); - self.current_level_size = next_level_size; + self.current_level_size *= 4; self.current_level_chunks = 0; } @@ -136,58 +165,55 @@ impl ProgressiveMerkleHasher { if !self.buffer.is_empty() { let mut chunk = [0u8; BYTES_PER_CHUNK]; chunk[..self.buffer.len()].copy_from_slice(&self.buffer); - self.process_chunk(chunk)?; + self.process_chunk(&chunk)?; } - // If we have no chunks at all, return zero hash - if self.total_chunks == 0 { + // With no chunks at all, the root is zero. Because the first level holds a single chunk, + // any processed chunk immediately populates `completed_roots`, so this is exactly the + // empty case. + if self.completed_roots.is_empty() && self.current_level_chunks == 0 { return Ok(Hash256::ZERO); } - // If there are chunks in current level (partial level), compute their root + // If the final level is partially filled, compute its (zero-padded) binary root. let current_root = if self.current_level_chunks > 0 { - Some(self.current_hasher.finish().map_err(Error::MerkleHasher)?) + let hasher = self + .current_hasher + .take() + .expect("current hasher exists while the level is partially filled"); + Some(hasher.finish().map_err(Error::MerkleHasher)?) } else { None }; - // Build the progressive tree from completed roots and current root - // completed_roots are in order: [smallest level, ..., largest level] - // We need to build from left to right in the tree - Ok(Self::build_progressive_root( - current_root, - self.completed_roots, - )) + // completed_roots are in order [smallest level, ..., largest level]; build the progressive + // tree by folding them from the largest level inward. + Ok(build_progressive_root(current_root, &self.completed_roots)) } +} - /// Build the final progressive merkle root by combining completed subtree roots. - /// - /// The progressive tree structure: at each node, hash(left=this_level, right=deeper_levels). - /// This builds the tree from the largest (rightmost) level backwards to the smallest (leftmost). - fn build_progressive_root( - current_root: Option, - completed_roots: Vec, - ) -> Hash256 { - // Start from the rightmost (largest/deepest) level - // Per EIP-7916 spec, even partial levels follow the progressive structure: - // merkleize_progressive(chunks, n) = hash(merkleize(chunks[:n], n), merkleize_progressive(chunks[n:], n*4)) - // So a partial level with k chunks becomes: hash(merkleize(chunks, n), ZERO (no further chunks)) - let mut result = if let Some(curr) = current_root { - Hash256::from_slice(&hash32_concat(curr.as_slice(), Hash256::ZERO.as_slice())) - } else { - Hash256::ZERO - }; - - // Process completed roots from largest to smallest (reverse order) - // At each step: result = hash(completed_root, result) - // - completed_root is the left subtree at this level (binary tree) - // - result accumulates the right subtree (deeper/larger levels) - for &completed_root in completed_roots.iter().rev() { - result = Hash256::from(hash32_concat(completed_root.as_slice(), result.as_slice())); - } - - result +/// Build the final progressive merkle root by combining completed subtree roots. +/// +/// The progressive tree structure: at each node, `hash(left = this level, right = deeper levels)`. +/// This builds the tree from the largest (rightmost) level backwards to the smallest (leftmost). +fn build_progressive_root(current_root: Option, completed_roots: &[Hash256]) -> Hash256 { + // Per EIP-7916, a partial final level still follows the progressive structure: + // merkleize_progressive(chunks, n) + // = hash(merkleize(chunks[:n], n), merkleize_progressive(chunks[n:], n*4)) + // so a partial level with k chunks becomes hash(merkleize(chunks, n), ZERO). + let mut result = if let Some(curr) = current_root { + Hash256::from(hash32_concat(curr.as_slice(), Hash256::ZERO.as_slice())) + } else { + Hash256::ZERO + }; + + // Fold completed roots from largest to smallest: result = hash(completed_root, result), where + // completed_root is the left (binary) subtree and result accumulates the deeper levels. + for &completed_root in completed_roots.iter().rev() { + result = Hash256::from(hash32_concat(completed_root.as_slice(), result.as_slice())); } + + result } impl Default for ProgressiveMerkleHasher { @@ -201,6 +227,50 @@ mod tests { use super::*; use crate::merkle_root; + /// Independent, non-streaming reference implementation of `merkleize_progressive` (EIP-7916), + /// used to cross-check the streaming hasher. + fn reference_progressive(chunks: &[[u8; BYTES_PER_CHUNK]], num_leaves: usize) -> Hash256 { + if chunks.is_empty() { + return Hash256::ZERO; + } + let take = num_leaves.min(chunks.len()); + let left_bytes: Vec = chunks[..take].iter().flatten().copied().collect(); + let left = merkle_root(&left_bytes, num_leaves); + let right = reference_progressive(&chunks[take..], num_leaves * 4); + Hash256::from(hash32_concat(left.as_slice(), right.as_slice())) + } + + fn streaming_root(chunks: &[[u8; BYTES_PER_CHUNK]]) -> Hash256 { + let mut hasher = ProgressiveMerkleHasher::new(); + for chunk in chunks { + hasher.write(chunk).unwrap(); + } + hasher.finish().unwrap() + } + + fn indexed_chunk(i: usize) -> [u8; BYTES_PER_CHUNK] { + let mut c = [0u8; BYTES_PER_CHUNK]; + c[..8].copy_from_slice(&(i as u64 + 1).to_le_bytes()); + c + } + + /// Cross-check the streaming hasher against the recursive reference across a range of chunk + /// counts, including exact level fills (1, 5, 21, 85, 341), partial trailing levels, and a + /// deep level (>256) that crosses the inner `MerkleHasher`'s inline-stack capacity. + #[test] + fn matches_reference_oracle() { + for count in [ + 0usize, 1, 2, 3, 4, 5, 6, 16, 20, 21, 22, 64, 84, 85, 86, 200, 341, 342, + ] { + let chunks: Vec<[u8; BYTES_PER_CHUNK]> = (0..count).map(indexed_chunk).collect(); + assert_eq!( + streaming_root(&chunks), + reference_progressive(&chunks, 1), + "mismatch for {count} chunks" + ); + } + } + #[test] fn test_empty_tree() { let hasher = ProgressiveMerkleHasher::new(); @@ -282,8 +352,12 @@ mod tests { hasher.write(&[3u8; 32]).unwrap(); let root = hasher.finish().unwrap(); - // Should handle multiple writes correctly - assert_ne!(root, Hash256::ZERO); + // The three writes form exactly two chunks: [1; 16] ++ [2; 16], then [3; 32]. + let mut chunk0 = [0u8; BYTES_PER_CHUNK]; + chunk0[..16].fill(1); + chunk0[16..].fill(2); + let chunk1 = [3u8; BYTES_PER_CHUNK]; + assert_eq!(root, reference_progressive(&[chunk0, chunk1], 1)); } #[test] @@ -327,39 +401,16 @@ mod tests { #[test] fn test_21_chunks() { - // Test with 21 chunks as per problem statement: - // chunks[0] goes to left at level 1 (1 leaf) - // chunks[1..5] go to left at level 2 (4 leaves) - // chunks[5..21] go to left at level 3 (16 leaves) - let mut hasher = ProgressiveMerkleHasher::new(); - for i in 0..21 { - let mut chunk = [0u8; BYTES_PER_CHUNK]; - chunk[0] = i as u8; - hasher.write(&chunk).unwrap(); - } - let root = hasher.finish().unwrap(); - - // Root should not be zero - assert_ne!(root, Hash256::ZERO); + // 21 chunks exactly fills levels 1 + 4 + 16. + let chunks: Vec<[u8; BYTES_PER_CHUNK]> = (0..21).map(indexed_chunk).collect(); + assert_eq!(streaming_root(&chunks), reference_progressive(&chunks, 1)); } #[test] fn test_85_chunks() { - // Test with 85 chunks as per problem statement structure: - // chunks[0] at level 1 (1 leaf) - // chunks[1..5] at level 2 (4 leaves) - // chunks[5..21] at level 3 (16 leaves) - // chunks[21..85] at level 4 (64 leaves) - let mut hasher = ProgressiveMerkleHasher::new(); - for i in 0..85 { - let mut chunk = [0u8; BYTES_PER_CHUNK]; - chunk[0] = (i % 256) as u8; - hasher.write(&chunk).unwrap(); - } - let root = hasher.finish().unwrap(); - - // Root should not be zero - assert_ne!(root, Hash256::ZERO); + // 85 chunks exactly fills levels 1 + 4 + 16 + 64. + let chunks: Vec<[u8; BYTES_PER_CHUNK]> = (0..85).map(indexed_chunk).collect(); + assert_eq!(streaming_root(&chunks), reference_progressive(&chunks, 1)); } #[test] diff --git a/tree_hash/tests/tests.rs b/tree_hash/tests/tests.rs index aa85283..9ec63f9 100644 --- a/tree_hash/tests/tests.rs +++ b/tree_hash/tests/tests.rs @@ -1,7 +1,11 @@ use alloy_primitives::{Address, U128, U160, U256}; +use ssz::ProgressiveBitList; use ssz_derive::Encode; use std::str::FromStr; -use tree_hash::{Hash256, MerkleHasher, PackedEncoding, TreeHash, BYTES_PER_CHUNK}; +use tree_hash::{ + mix_in_active_fields, Hash256, MerkleHasher, PackedEncoding, ProgressiveMerkleHasher, TreeHash, + BYTES_PER_CHUNK, +}; use tree_hash_derive::TreeHash; #[derive(Encode)] @@ -184,3 +188,170 @@ fn progressive_container_one_field() { .unwrap() ); } + +/// Reference merkleization of a progressive container's leaves, via the public hasher. +fn progressive_merkle_root(leaves: &[Hash256]) -> Hash256 { + let mut hasher = ProgressiveMerkleHasher::new(); + for leaf in leaves { + hasher.write(leaf.as_slice()).unwrap(); + } + hasher.finish().unwrap() +} + +/// Pack `active_fields` bits LSB-first into a single 32-byte chunk. +fn packed_active_fields(bits: &[bool]) -> [u8; BYTES_PER_CHUNK] { + let mut packed = [0u8; BYTES_PER_CHUNK]; + for (i, bit) in bits.iter().enumerate() { + if *bit { + packed[i / 8] |= 1 << (i % 8); + } + } + packed +} + +#[derive(TreeHash)] +#[tree_hash(struct_behaviour = "progressive_container", active_fields(1, 1, 1))] +struct ProgressiveContainerThreeFields { + a: u8, + b: u64, + c: Hash256, +} + +#[test] +fn progressive_container_multi_field() { + let container = ProgressiveContainerThreeFields { + a: 7, + b: 0x0102_0304_0506_0708, + c: Hash256::repeat_byte(0xcd), + }; + + let leaves = [ + container.a.tree_hash_root(), + container.b.tree_hash_root(), + container.c.tree_hash_root(), + ]; + let expected = mix_in_active_fields( + &progressive_merkle_root(&leaves), + packed_active_fields(&[true, true, true]), + ); + + assert_eq!(container.tree_hash_root(), expected); +} + +#[derive(TreeHash)] +#[tree_hash(struct_behaviour = "progressive_container", active_fields(1, 0, 1))] +struct ProgressiveContainerWithGap { + a: u8, + c: u16, +} + +#[test] +fn progressive_container_inactive_field() { + let container = ProgressiveContainerWithGap { a: 9, c: 0xbeef }; + + // The inactive middle position must contribute a zero leaf so chunk positions stay stable. + let leaves = [ + container.a.tree_hash_root(), + Hash256::ZERO, + container.c.tree_hash_root(), + ]; + let expected = mix_in_active_fields( + &progressive_merkle_root(&leaves), + packed_active_fields(&[true, false, true]), + ); + + assert_eq!(container.tree_hash_root(), expected); +} + +#[derive(TreeHash)] +#[tree_hash(enum_behaviour = "compatible_union")] +enum CompatUnion { + #[tree_hash(selector = "1")] + A(u8), + #[tree_hash(selector = "4")] + B(HashVec), +} + +#[test] +fn compatible_union() { + // The root is `mix_in_selector(inner_root, selector)` using the explicit per-variant selectors. + assert_eq!( + CompatUnion::A(2).tree_hash_root(), + mix_in_selector(u8_hash(2), 1) + ); + assert_eq!( + CompatUnion::B(HashVec::from(vec![2])).tree_hash_root(), + mix_in_selector(u8_hash_concat(2, 1), 4) + ); +} + +#[derive(Encode, TreeHash)] +#[ssz(enum_behaviour = "compatible_union")] +#[tree_hash(enum_behaviour = "compatible_union")] +enum CompatUnionSszSelectors { + #[ssz(selector = "3")] + A(u8), + #[ssz(selector = "5")] + B(u8), +} + +#[test] +fn compatible_union_reads_ssz_selectors() { + // These variants carry no `tree_hash(selector)`, so `tree_hash` must read the selectors from + // the sibling `ssz` attribute (the `(None, Some(ssz))` fallback in `parse_variant_opts`). + assert_eq!( + CompatUnionSszSelectors::A(2).tree_hash_root(), + mix_in_selector(u8_hash(2), 3) + ); + assert_eq!( + CompatUnionSszSelectors::B(2).tree_hash_root(), + mix_in_selector(u8_hash(2), 5) + ); +} + +/// Merkleize raw bytes (zero-padded into chunks) via the public progressive hasher. +fn progressive_merkle_root_bytes(bytes: &[u8]) -> Hash256 { + let mut hasher = ProgressiveMerkleHasher::new(); + hasher.write(bytes).unwrap(); + hasher.finish().unwrap() +} + +#[test] +fn progressive_bitlist_empty() { + // An empty bitlist must hash as mix_in_length(merkleize_progressive([]) = ZERO, 0), exercising + // the empty-list workaround (an empty bitlist still stores a single zero byte internally). + let bitlist = ProgressiveBitList::with_capacity(0).unwrap(); + assert_eq!( + bitlist.tree_hash_root(), + tree_hash::mix_in_length(&Hash256::ZERO, 0) + ); +} + +#[test] +fn progressive_bitlist_matches_packed_progressive() { + // The root must equal mix_in_length(merkleize_progressive(pack_bits(value)), len) for a range + // of lengths, including byte- and chunk-aligned cases and progressive level crossings. + for len in [1usize, 7, 8, 9, 16, 100, 256, 257, 1024] { + let mut bitlist = ProgressiveBitList::with_capacity(len).unwrap(); + for i in (0..len).step_by(3) { + bitlist.set(i, true).unwrap(); + } + + let expected = + tree_hash::mix_in_length(&progressive_merkle_root_bytes(bitlist.as_slice()), len); + assert_eq!(bitlist.tree_hash_root(), expected, "len = {len}"); + } +} + +#[test] +fn progressive_bitlist_nonempty_all_false_differs_from_empty() { + // A non-empty all-false bitlist must NOT short-circuit to the empty value. + let bitlist = ProgressiveBitList::with_capacity(8).unwrap(); + let empty = ProgressiveBitList::with_capacity(0).unwrap(); + + assert_ne!(bitlist.tree_hash_root(), empty.tree_hash_root()); + assert_eq!( + bitlist.tree_hash_root(), + tree_hash::mix_in_length(&progressive_merkle_root_bytes(bitlist.as_slice()), 8) + ); +} diff --git a/tree_hash_derive/src/attrs.rs b/tree_hash_derive/src/attrs.rs index 02529fc..e37c78c 100644 --- a/tree_hash_derive/src/attrs.rs +++ b/tree_hash_derive/src/attrs.rs @@ -2,7 +2,12 @@ use darling::{ast::NestedMeta, Error, FromDeriveInput, FromMeta}; use quote::quote; pub const MAX_ACTIVE_FIELDS: usize = 256; -pub const ACTIVE_FIELDS_PACKED_BITS_LEN: usize = MAX_ACTIVE_FIELDS / 8; +/// The number of bytes in the packed `active_fields` bitvector. +/// +/// `active_fields` is restricted to `MAX_ACTIVE_FIELDS` (256) bits so that it packs into exactly +/// one 32-byte chunk, which must equal `tree_hash::BYTES_PER_CHUNK` (the generated code asserts +/// this at compile time). +pub const ACTIVE_FIELDS_PACKED_BYTES_LEN: usize = MAX_ACTIVE_FIELDS / 8; #[derive(Debug, FromDeriveInput)] #[darling(attributes(tree_hash))] @@ -21,6 +26,10 @@ pub struct StructOpts { /// each others attributes to avoid mandatory duplication. In future this might mean parsing some /// SSZ-only attributes here and then ignoring them. #[derive(Debug, Default, PartialEq, FromMeta)] +// `allow_unknown_fields` ensures that `ssz`-only keys (current or future) on a shared variant +// attribute are tolerated and ignored here, rather than causing a parse error. This mirrors +// `ssz_derive`'s `VariantOpts`, which tolerates `tree_hash`-only keys for the same reason. +#[darling(allow_unknown_fields)] pub struct VariantOpts { #[darling(default)] pub selector: Option, @@ -75,17 +84,22 @@ impl ActiveFields { ))); } + // A trailing inactive field is materialized as a real zero leaf, so it would change the + // root rather than being a no-op. The progressive container's `active_fields` bitvector is + // canonically delimited by its highest active field, so a trailing `0` is not permitted. if let Some(false) = active_fields.last() { return Err(Error::custom( - "the last entry of active_fields must not be 0".to_string(), + "the last entry of active_fields must not be 0 (the bitvector is delimited by the \ + highest active field)" + .to_string(), )); } Ok(Self { active_fields }) } - pub fn packed(&self) -> [u8; ACTIVE_FIELDS_PACKED_BITS_LEN] { - let mut result = [0; ACTIVE_FIELDS_PACKED_BITS_LEN]; + pub fn packed(&self) -> [u8; ACTIVE_FIELDS_PACKED_BYTES_LEN] { + let mut result = [0; ACTIVE_FIELDS_PACKED_BYTES_LEN]; for (i, bit) in self.active_fields.iter().enumerate() { if *bit { result[i / 8] |= 1 << (i % 8); @@ -99,10 +113,11 @@ impl ActiveFields { /// We compute the packed representation at compile-time, and then inline it via the output /// of this function. pub fn packed_tokens(&self) -> proc_macro2::TokenStream { - let packed = self.packed().to_vec(); + let packed = self.packed(); + let bytes = packed.iter(); quote! { [ - #(#packed),* + #(#bytes),* ] } } @@ -120,20 +135,110 @@ mod test { assert_eq!( active_fields.packed(), [ - 0b0000001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, + 0b0000_0001, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, ] ); + // Bits are packed LSB-first, so [0, 2, 5] set yields 1 + 4 + 32 = 0b0010_0101. let active_fields = ActiveFields { active_fields: vec![true, false, true, false, false, true], }; assert_eq!( active_fields.packed(), [ - 0b0100101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, + 0b0010_0101, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, ] ); } + + #[test] + fn active_fields_packed_multi_byte() { + // Bits 0 and 10 set, spanning two bytes: byte 0 has bit 0, byte 1 has bit 2 (10 % 8). + let mut bits = vec![false; 11]; + bits[0] = true; + bits[10] = true; + let packed = ActiveFields::new(bits).unwrap().packed(); + + assert_eq!(packed[0], 0b0000_0001); + assert_eq!(packed[1], 0b0000_0100); + assert!(packed[2..].iter().all(|&b| b == 0)); + } + + #[test] + fn active_fields_new_validation() { + // Empty is rejected. + assert!(ActiveFields::new(vec![]).is_err()); + // A trailing `0` (inactive last field) is rejected. + assert!(ActiveFields::new(vec![false]).is_err()); + assert!(ActiveFields::new(vec![true, false]).is_err()); + // More than MAX_ACTIVE_FIELDS entries is rejected. + assert!(ActiveFields::new(vec![true; MAX_ACTIVE_FIELDS + 1]).is_err()); + + // Valid cases. + assert!(ActiveFields::new(vec![true]).is_ok()); + assert!(ActiveFields::new(vec![false, true]).is_ok()); + assert!(ActiveFields::new(vec![true; MAX_ACTIVE_FIELDS]).is_ok()); + } } diff --git a/tree_hash_derive/src/lib.rs b/tree_hash_derive/src/lib.rs index 28eff58..1e4fddc 100644 --- a/tree_hash_derive/src/lib.rs +++ b/tree_hash_derive/src/lib.rs @@ -5,6 +5,7 @@ use crate::attrs::{EnumBehaviour, StructBehaviour, StructOpts, VariantOpts}; use darling::{FromDeriveInput, FromMeta}; use proc_macro::TokenStream; use quote::quote; +use std::collections::HashSet; use std::convert::TryInto; use syn::{parse_macro_input, Attribute, DataEnum, DataStruct, DeriveInput, Ident}; @@ -50,7 +51,10 @@ fn get_hashable_fields_and_their_caches( /// The field attribute is: `#[tree_hash(skip_hashing)]` fn should_skip_hashing(field: &syn::Field) -> bool { field.attrs.iter().any(|attr| { - is_tree_hash_attr(attr) && attr.parse_args::().unwrap() == "skip_hashing" + is_tree_hash_attr(attr) + && attr + .parse_args::() + .is_ok_and(|ident| ident == "skip_hashing") }) } @@ -76,6 +80,9 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream { if struct_opt.is_some() { panic!("struct_behaviour is invalid for enums"); } + if opts.active_fields.is_some() { + panic!("active_fields is invalid for enums"); + } match enum_behaviour { EnumBehaviour::Transparent => tree_hash_derive_enum_transparent(&item, s), EnumBehaviour::Union | EnumBehaviour::CompatibleUnion => { @@ -109,48 +116,71 @@ fn tree_hash_derive_struct( // // The `mixin_logic` is the expression to mix in the `active_fields` in the case of a // progressive container. - let (field_hashes, mixin_logic) = - if let StructBehaviour::ProgressiveContainer = struct_behaviour { - let Some(active_fields) = active_fields_opt else { - panic!("active_fields must be provided for progressive_container"); - }; + let (field_hashes, mixin_logic) = if let StructBehaviour::ProgressiveContainer = + struct_behaviour + { + let Some(active_fields) = active_fields_opt else { + panic!("active_fields must be provided for progressive_container"); + }; - let mut active_field_index = 0; - let mut field_hashes: Vec = vec![]; - for active in &active_fields.active_fields { - if *active { - let Some(ident) = idents.get(active_field_index) else { - panic!( - "active_fields is inconsistent with struct fields. \ + // Map each `active_fields` entry to a leaf: an active entry consumes the next hashable + // struct field (fields marked `#[tree_hash(skip_hashing)]` are already excluded from + // `idents`), while an inactive entry contributes a zero leaf so that chunk positions + // (and therefore gindices) stay stable. + let mut active_field_index = 0; + let mut field_hashes: Vec = vec![]; + for active in &active_fields.active_fields { + if *active { + let Some(ident) = idents.get(active_field_index) else { + panic!( + "active_fields is inconsistent with struct fields. \ index: {active_field_index}, hashable fields: {}", - idents.len() - ) - }; - active_field_index += 1; - field_hashes.push(quote! { self.#ident.tree_hash_root() }); - } else { - field_hashes.push(quote! { tree_hash::Hash256::ZERO }); - } + idents.len() + ) + }; + active_field_index += 1; + field_hashes.push(quote! { self.#ident.tree_hash_root() }); + } else { + field_hashes.push(quote! { tree_hash::Hash256::ZERO }); } + } - let packed_active_fields = active_fields.packed_tokens(); - - let mixin_logic = quote! { - const ACTIVE_FIELDS: [u8; 32] = #packed_active_fields; - tree_hash::mix_in_active_fields(container_root, ACTIVE_FIELDS) - }; - - (field_hashes, mixin_logic) - } else { - ( - idents - .into_iter() - .map(|ident| quote! { self.#ident.tree_hash_root() }) - .collect(), - quote! { container_root }, - ) + // Every hashable field must be claimed by an active entry. Without this check an + // under-specified `active_fields` would silently drop trailing fields from the hash. + // (The over-specified case is already caught above by `idents.get(..)` returning + // `None`.) + assert_eq!( + active_field_index, + idents.len(), + "active_fields has {active_field_index} active entries but the struct has {} \ + hashable fields (fields marked #[tree_hash(skip_hashing)] are not counted)", + idents.len() + ); + + let packed_active_fields = active_fields.packed_tokens(); + let active_fields_len = attrs::ACTIVE_FIELDS_PACKED_BYTES_LEN; + + let mixin_logic = quote! { + const ACTIVE_FIELDS: [u8; #active_fields_len] = #packed_active_fields; + // The packed active_fields must occupy exactly one chunk for `mix_in_active_fields`. + const _: () = assert!(#active_fields_len == tree_hash::BYTES_PER_CHUNK); + tree_hash::mix_in_active_fields(&container_root, ACTIVE_FIELDS) }; + (field_hashes, mixin_logic) + } else { + if active_fields_opt.is_some() { + panic!("active_fields is only valid with struct_behaviour = \"progressive_container\""); + } + ( + idents + .into_iter() + .map(|ident| quote! { self.#ident.tree_hash_root() }) + .collect(), + quote! { container_root }, + ) + }; + let output = quote! { impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause { fn tree_hash_type() -> tree_hash::TreeHashType { @@ -370,13 +400,18 @@ fn parse_variant_opts(enum_data: &DataEnum) -> Vec { panic!("more than one variant-level \"tree_hash\" attribute provided"); } - let tree_hash_opts = tree_hash_attrs - .first() - .map(|attr| VariantOpts::from_meta(&attr.meta).unwrap()); + let variant_name = &variant.ident; + let parse_opts = |meta: &syn::Meta| { + VariantOpts::from_meta(meta).unwrap_or_else(|e| { + panic!( + "failed to parse variant attribute for \"{variant_name}\": {e}; \ + note that `selector` must be an integer in 1..={MAX_UNION_SELECTOR}" + ) + }) + }; - let ssz_opts = ssz_attrs - .first() - .map(|attr| VariantOpts::from_meta(&attr.meta).unwrap()); + let tree_hash_opts = tree_hash_attrs.first().map(|attr| parse_opts(&attr.meta)); + let ssz_opts = ssz_attrs.first().map(|attr| parse_opts(&attr.meta)); // Check consistency with SSZ opts, or fall back to SSZ attribute if tree_hash attribute // is absent. @@ -435,6 +470,7 @@ fn compute_union_selectors(num_variants: usize) -> Vec { } fn get_compatible_union_selectors(enum_data: &DataEnum, variant_opts: &[VariantOpts]) -> Vec { + let mut seen_selectors = HashSet::new(); enum_data .variants .iter() @@ -450,6 +486,11 @@ fn get_compatible_union_selectors(enum_data: &DataEnum, variant_opts: &[VariantO compatible union" ); } + if !seen_selectors.insert(selector) { + panic!( + "selector = {selector} for variant \"{variant_name}\" is used more than once" + ); + } selector }) .collect()