From 5eb9a14c405158a296262e2b86faa7fc0b138dd3 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 24 Mar 2025 17:00:45 +0100 Subject: [PATCH 1/8] PartialSmt: fix misleading variable names --- miden-crypto/src/merkle/smt/partial.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miden-crypto/src/merkle/smt/partial.rs b/miden-crypto/src/merkle/smt/partial.rs index 93479d3..8bbc4e9 100644 --- a/miden-crypto/src/merkle/smt/partial.rs +++ b/miden-crypto/src/merkle/smt/partial.rs @@ -46,8 +46,8 @@ impl PartialSmt { { let mut partial_smt = Self::new(); - for (leaf, path) in paths.into_iter().map(SmtProof::into_parts) { - partial_smt.add_path(path, leaf)?; + for (path, leaf) in paths.into_iter().map(SmtProof::into_parts) { + partial_smt.add_path(leaf, path)?; } Ok(partial_smt) From 07bda60233daf86ee4187b3c0cc3f64ce91a7f63 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Tue, 25 Feb 2025 21:02:08 +0100 Subject: [PATCH 2/8] smt: refactor MerklePath logic Computation of the correct node indices to get is moved to `NodeIndex::proof_indices()`, and getting a node's index based on its parent is generalized into `SparseMerkleTree::get_hash()`. --- miden-crypto/src/merkle/index.rs | 45 ++++++++++++++++++++++++++++++ miden-crypto/src/merkle/smt/mod.rs | 42 ++++++++++++++++------------ 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/miden-crypto/src/merkle/index.rs b/miden-crypto/src/merkle/index.rs index ae27fec..39554c7 100644 --- a/miden-crypto/src/merkle/index.rs +++ b/miden-crypto/src/merkle/index.rs @@ -164,6 +164,18 @@ impl NodeIndex { self.depth = self.depth.saturating_sub(delta); self.value >>= delta as u32; } + + // ITERATORS + // -------------------------------------------------------------------------------------------- + + /// Return an iterator of the indices required for a Merkle proof of inclusion of a node at + /// `self`. + /// + /// This is *exclusive* on both ends: neither `self` nor the root index are included in the + /// returned iterator. + pub fn proof_indices(&self) -> impl ExactSizeIterator + use<> { + ProofIter { next_index: self.sibling() } + } } impl Display for NodeIndex { @@ -188,6 +200,39 @@ impl Deserializable for NodeIndex { } } +/// Implementation for [`NodeIndex::proof_indices()`]. +#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Hash)] +struct ProofIter { + next_index: NodeIndex, +} + +impl Iterator for ProofIter { + type Item = NodeIndex; + + fn next(&mut self) -> Option { + if self.next_index.is_root() { + return None; + } + + let index = self.next_index; + self.next_index = index.parent().sibling(); + + Some(index) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = ExactSizeIterator::len(self); + + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for ProofIter { + fn len(&self) -> usize { + self.next_index.depth() as usize + } +} + #[cfg(test)] mod tests { use assert_matches::assert_matches; diff --git a/miden-crypto/src/merkle/smt/mod.rs b/miden-crypto/src/merkle/smt/mod.rs index 03f1bed..6369540 100644 --- a/miden-crypto/src/merkle/smt/mod.rs +++ b/miden-crypto/src/merkle/smt/mod.rs @@ -79,28 +79,34 @@ pub(crate) trait SparseMerkleTree { // PROVIDED METHODS // --------------------------------------------------------------------------------------------- + /// Returns a [MerklePath] to the specified key. + /// + /// Mostly this is an implementation detail of [`Self::open()`]. + fn get_path(&self, key: &Self::Key) -> MerklePath { + let index = NodeIndex::from(Self::key_to_leaf_index(key)); + index.proof_indices().map(|index| self.get_hash(index)).collect() + } + + /// Get the hash of a node at an arbitrary index, including the root or leaf hashes. + /// + /// The root index simply returns [`Self::root()`]. Other hashes are retrieved by calling + /// [`Self::get_inner_node()`] on the parent, and returning the respective child hash. + fn get_hash(&self, index: NodeIndex) -> RpoDigest { + if index.is_root() { + return self.root(); + } + + let InnerNode { left, right } = self.get_inner_node(index.parent()); + + let index_is_right = index.is_value_odd(); + if index_is_right { right } else { left } + } + /// Returns an opening of the leaf associated with `key`. Conceptually, an opening is a Merkle /// path to the leaf, as well as the leaf itself. fn open(&self, key: &Self::Key) -> Self::Opening { let leaf = self.get_leaf(key); - - let mut index: NodeIndex = { - let leaf_index: LeafIndex = Self::key_to_leaf_index(key); - leaf_index.into() - }; - - let merkle_path = { - let mut path = Vec::with_capacity(index.depth() as usize); - for _ in 0..index.depth() { - let is_right = index.is_value_odd(); - index.move_up(); - let InnerNode { left, right } = self.get_inner_node(index); - let value = if is_right { left } else { right }; - path.push(value); - } - - MerklePath::new(path) - }; + let merkle_path = self.get_path(key); Self::path_and_leaf_to_opening(merkle_path, leaf) } From aa386c67a4fb57f1df2b4f181e948257c619b280 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 4 Apr 2025 14:52:11 +0200 Subject: [PATCH 3/8] MerkleTree: use new NodeIndex::proof_indices() to resolve TODO --- miden-crypto/src/merkle/merkle_tree.rs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/miden-crypto/src/merkle/merkle_tree.rs b/miden-crypto/src/merkle/merkle_tree.rs index 4d7478e..ff037b6 100644 --- a/miden-crypto/src/merkle/merkle_tree.rs +++ b/miden-crypto/src/merkle/merkle_tree.rs @@ -95,26 +95,16 @@ impl MerkleTree { /// Returns an error if: /// * The specified depth is greater than the depth of the tree. /// * The specified value is not valid for the specified depth. - pub fn get_path(&self, mut index: NodeIndex) -> Result { + pub fn get_path(&self, index: NodeIndex) -> Result { if index.is_root() { return Err(MerkleError::DepthTooSmall(index.depth())); } else if index.depth() > self.depth() { return Err(MerkleError::DepthTooBig(index.depth() as u64)); } - // TODO should we create a helper in `NodeIndex` that will encapsulate traversal to root so - // we always use inlined `for` instead of `while`? the reason to use `for` is because its - // easier for the compiler to vectorize. - let mut path = Vec::with_capacity(index.depth() as usize); - for _ in 0..index.depth() { - let sibling = index.sibling().to_scalar_index() as usize; - path.push(self.nodes[sibling]); - index.move_up(); - } - - debug_assert!(index.is_root(), "the path walk must go all the way to the root"); - - Ok(path.into()) + Ok(MerklePath::from(Vec::from_iter( + index.proof_indices().map(|index| self.get_node(index).unwrap()), + ))) } // ITERATORS From d5245438999af29e540b1903a24298c92a171ae1 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 4 Apr 2025 13:56:29 +0200 Subject: [PATCH 4/8] MerklePath: document indexing order of nodes I've left the iteration order of `MerklePath::inner_nodes()` unspecified, but other methods of iteration and indexing are now specified to be in order of deepest to shallowest. --- miden-crypto/src/merkle/path.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/miden-crypto/src/merkle/path.rs b/miden-crypto/src/merkle/path.rs index ee35c31..8ddf63c 100644 --- a/miden-crypto/src/merkle/path.rs +++ b/miden-crypto/src/merkle/path.rs @@ -11,6 +11,9 @@ use crate::{ // ================================================================================================ /// A merkle path container, composed of a sequence of nodes of a Merkle tree. +/// +/// Indexing into this type starts at the deepest part of the path and gets shallower. That is, +/// the node at index `0` is deeper than the node at index `self.len() - 1`. #[derive(Clone, Debug, Default, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct MerklePath { @@ -22,6 +25,8 @@ impl MerklePath { // -------------------------------------------------------------------------------------------- /// Creates a new Merkle path from a list of nodes. + /// + /// The list is assumed to be in order of deepest to shallowest. pub fn new(nodes: Vec) -> Self { assert!(nodes.len() <= u8::MAX.into(), "MerklePath may have at most 256 items"); Self { nodes } @@ -35,7 +40,7 @@ impl MerklePath { self.nodes.len() as u8 } - /// Returns a reference to the [MerklePath]'s nodes. + /// Returns a reference to the [MerklePath]'s nodes, in order of deepest to shallowest. pub fn nodes(&self) -> &[RpoDigest] { &self.nodes } From 22c19983ace6d84b331165b07e5597b991d38422 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 4 Apr 2025 14:13:37 +0200 Subject: [PATCH 5/8] MerklePath: add clarity getters for API parity with future SparseMerklePath This adds `MerklePath::at_depth()` and `MerklePath::at_idx()`, both for clarity and for API parity with `SparseMerklePath` in the next commit. --- miden-crypto/src/merkle/path.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/miden-crypto/src/merkle/path.rs b/miden-crypto/src/merkle/path.rs index 8ddf63c..53f94c5 100644 --- a/miden-crypto/src/merkle/path.rs +++ b/miden-crypto/src/merkle/path.rs @@ -1,5 +1,8 @@ use alloc::vec::Vec; -use core::ops::{Deref, DerefMut}; +use core::{ + num::NonZero, + ops::{Deref, DerefMut}, +}; use super::{InnerNodeInfo, MerkleError, NodeIndex, Rpo256, RpoDigest}; use crate::{ @@ -35,6 +38,26 @@ impl MerklePath { // PROVIDERS // -------------------------------------------------------------------------------------------- + /// Returns a reference to the path node at the specified depth. + /// + /// The `depth` parameter is defined in terms of `self.depth()`. Merkle paths conventionally do + /// not include the root, so the shallowest depth is `1`, and the deepest depth is + /// `self.depth()`. + pub fn at_depth(&self, depth: NonZero) -> Option<&RpoDigest> { + let index = u8::checked_sub(self.depth(), depth.get())?; + self.nodes.get(index as usize) + } + + /// Returns a reference to the path node at the specified index, or [None] if the index is out + /// of bounds. + /// + /// The node at index 0 is the deepest part of the path. + /// + /// This is a checked version of using the indexing operator `[]`. + pub fn at_idx(&self, index: usize) -> Option<&RpoDigest> { + self.nodes.get(index) + } + /// Returns the depth in which this Merkle path proof is valid. pub fn depth(&self) -> u8 { self.nodes.len() as u8 From 5febc5798f0e6b01d96d2605783db3a7ea638422 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Thu, 27 Feb 2025 17:26:28 +0100 Subject: [PATCH 6/8] smt: add SparseMerklePath, a compact representation of MerklePath --- CHANGELOG.md | 1 + miden-crypto/src/merkle/mod.rs | 3 + miden-crypto/src/merkle/sparse_path.rs | 827 +++++++++++++++++++++++++ 3 files changed, 831 insertions(+) create mode 100644 miden-crypto/src/merkle/sparse_path.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e5161..48247a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Optimized duplicate key detection in `Smt::with_entries_concurrent` (#395). - [BREAKING] Moved `rand` to version `0.9` removing the `try_fill_bytes` method (#398). - [BREAKING] Increment minimum supported Rust version to 1.85 (#399). +- Added `SparseMerklePath`, a compact representation of `MerklePath` which compacts empty nodes into a bitmask (#389). ## 0.13.3 (2025-02-18) diff --git a/miden-crypto/src/merkle/mod.rs b/miden-crypto/src/merkle/mod.rs index 509de41..b6a02bc 100644 --- a/miden-crypto/src/merkle/mod.rs +++ b/miden-crypto/src/merkle/mod.rs @@ -20,6 +20,9 @@ pub use merkle_tree::{MerkleTree, path_to_text, tree_to_text}; mod path; pub use path::{MerklePath, RootPath, ValuePath}; +mod sparse_path; +pub use sparse_path::{SparseMerklePath, SparseValuePath}; + mod smt; pub use smt::{ InnerNode, LeafIndex, MutationSet, NodeMutation, PartialSmt, SMT_DEPTH, SMT_MAX_DEPTH, diff --git a/miden-crypto/src/merkle/sparse_path.rs b/miden-crypto/src/merkle/sparse_path.rs new file mode 100644 index 0000000..65322dd --- /dev/null +++ b/miden-crypto/src/merkle/sparse_path.rs @@ -0,0 +1,827 @@ +use alloc::vec::Vec; +use core::{ + iter::{self, FusedIterator}, + num::NonZero, +}; + +use winter_utils::{Deserializable, DeserializationError, Serializable}; + +use super::{ + EmptySubtreeRoots, MerkleError, MerklePath, RpoDigest, SMT_MAX_DEPTH, ValuePath, Word, +}; + +/// A different representation of [`MerklePath`] designed for memory efficiency for Merkle paths +/// with empty nodes. +/// +/// Empty nodes in the path are stored only as their position, represented with a bitmask. A +/// maximum of 64 nodes in the path can be empty. The number of empty nodes has no effect on memory +/// usage by this struct, but will incur overhead during iteration or conversion to a +/// [`MerklePath`], for each empty node. +/// +/// NOTE: This type assumes that Merkle paths always span from the root of the tree to a leaf. +/// Partial paths are not supported. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct SparseMerklePath { + /// A bitmask representing empty nodes. The set bit corresponds to the depth of an empty node. + empty_nodes: u64, + /// The non-empty nodes, stored in depth-order, but not contiguous across depth. + nodes: Vec, +} + +impl SparseMerklePath { + /// Constructs a sparse Merkle path from an iterator over Merkle nodes that also knows its + /// exact size (such as iterators created with [Vec::into_iter]). The iterator must be in order + /// of deepest to shallowest. + /// + /// Knowing the size is necessary to calculate the depth of the tree, which is needed to detect + /// which nodes are empty nodes. If you know the size but your iterator type is not + /// [ExactSizeIterator], use [`SparseMerklePath::from_iter_with_depth()`]. + pub fn from_sized_iter(iterator: I) -> Result + where + I: IntoIterator, + { + let iterator = iterator.into_iter(); + // `iterator.len() as u8` will truncate, but not below `SMT_MAX_DEPTH`, which + // `from_iter_with_depth` checks for. + Self::from_iter_with_depth(iterator.len() as u8, iterator) + } + + /// Constructs a sparse Merkle path from a manually specified tree depth, and an iterator over + /// Merkle nodes from deepest to shallowest. + /// + /// Knowing the size is necessary to calculate the depth of the tree, which is needed to detect + /// which nodes are empty nodes. + pub fn from_iter_with_depth( + tree_depth: u8, + iter: impl IntoIterator, + ) -> Result { + if tree_depth > SMT_MAX_DEPTH { + return Err(MerkleError::DepthTooBig(tree_depth as u64)); + } + + let path: Self = iter::zip(path_depth_iter(tree_depth), iter) + .map(|(depth, node)| { + let &equivalent_empty_node = EmptySubtreeRoots::entry(tree_depth, depth.get()); + let is_empty = node == equivalent_empty_node; + let node = if is_empty { None } else { Some(node) }; + + (depth, node) + }) + .collect(); + + Ok(path) + } + + /// Returns the total depth of this path, i.e., the number of nodes this path represents. + pub fn depth(&self) -> u8 { + (self.nodes.len() + self.empty_nodes.count_ones() as usize) as u8 + } + + /// Get a specific node in this path at a given depth. + /// + /// The `depth` parameter is defined in terms of `self.depth()`. Merkle paths conventionally do + /// not include the root, so the shallowest depth is `1`, and the deepest depth is + /// `self.depth()`. + /// + /// # Errors + /// Returns [MerkleError::DepthTooBig] if `node_depth` is greater than the total depth of this + /// path. + pub fn at_depth(&self, node_depth: NonZero) -> Result { + let node = self + .at_depth_nonempty(node_depth)? + .unwrap_or_else(|| *EmptySubtreeRoots::entry(self.depth(), node_depth.get())); + + Ok(node) + } + + /// Get a specific non-empty node in this path at a given depth, or `None` if the specified + /// node is an empty node. + /// + /// # Errors + /// Returns [MerkleError::DepthTooBig] if `node_depth` is greater than the total depth of this + /// path. + pub fn at_depth_nonempty( + &self, + node_depth: NonZero, + ) -> Result, MerkleError> { + if node_depth.get() > self.depth() { + return Err(MerkleError::DepthTooBig(node_depth.get().into())); + } + + if self.is_depth_empty(node_depth) { + return Ok(None); + } + + // Our index needs to account for all the empty nodes that aren't in `self.nodes`. + let nonempty_index = self.get_nonempty_index(node_depth); + + Ok(Some(self.nodes[nonempty_index])) + } + + /// Returns the path node at the specified index, or [None] if the index is out of bounds. + /// + /// The node at index 0 is the deepest part of the path. + /// + /// ``` + /// # use core::num::NonZero; + /// # use miden_crypto::{ZERO, ONE, hash::rpo::RpoDigest, merkle::SparseMerklePath}; + /// # let zero = RpoDigest::new([ZERO; 4]); + /// # let one = RpoDigest::new([ONE; 4]); + /// # let sparse_path = SparseMerklePath::from_sized_iter(vec![zero, one, one, zero]).unwrap(); + /// let depth = NonZero::new(sparse_path.depth()).unwrap(); + /// assert_eq!( + /// sparse_path.at_idx(0).unwrap(), + /// sparse_path.at_depth(depth).unwrap(), + /// ); + /// ``` + pub fn at_idx(&self, index: usize) -> Option { + // If this overflows *or* if the depth is zero then the index was out of bounds. + let depth = NonZero::new(u8::checked_sub(self.depth(), index as u8)?)?; + self.at_depth(depth).ok() + } + + // PROVIDERS + // ============================================================================================ + + /// Constructs a borrowing iterator over the nodes in this path. + pub fn iter(&self) -> impl ExactSizeIterator { + self.into_iter() + } + + // PRIVATE HELPERS + // ============================================================================================ + + const fn bitmask_for_depth(node_depth: NonZero) -> u64 { + // - 1 because paths do not include the root. + 1 << (node_depth.get() - 1) + } + + const fn is_depth_empty(&self, node_depth: NonZero) -> bool { + (self.empty_nodes & Self::bitmask_for_depth(node_depth)) != 0 + } + + fn get_nonempty_index(&self, node_depth: NonZero) -> usize { + let bit_index = node_depth.get() - 1; + let without_shallower = self.empty_nodes >> bit_index; + let empty_deeper = without_shallower.count_ones() as usize; + // The vec index we would use if we didn't have any empty nodes to account for... + let normal_index = (self.depth() - node_depth.get()) as usize; + // subtracted by the number of empty nodes that are deeper than us. + normal_index - empty_deeper + } +} + +// CONVERSIONS +// ================================================================================================ + +impl From for MerklePath { + fn from(sparse_path: SparseMerklePath) -> Self { + MerklePath::from_iter(sparse_path) + } +} + +/// # Errors +/// +/// This conversion returns [MerkleError::DepthTooBig] if the path length is greater than +/// [`SMT_MAX_DEPTH`]. +impl TryFrom for SparseMerklePath { + type Error = MerkleError; + + fn try_from(path: MerklePath) -> Result { + SparseMerklePath::from_sized_iter(path) + } +} + +impl From for Vec { + fn from(path: SparseMerklePath) -> Self { + Vec::from_iter(path) + } +} + +// ITERATORS +// ================================================================================================ + +/// Contructs a [SparseMerklePath] out of an iterator of optional nodes, where `None` indicates an +/// empty node. +impl FromIterator<(NonZero, Option)> for SparseMerklePath { + fn from_iter(iter: I) -> SparseMerklePath + where + I: IntoIterator, Option)>, + { + let mut empty_nodes: u64 = 0; + let mut nodes: Vec = Default::default(); + + for (depth, node) in iter { + match node { + Some(node) => nodes.push(node), + None => empty_nodes |= Self::bitmask_for_depth(depth), + } + } + + SparseMerklePath { nodes, empty_nodes } + } +} + +impl<'p> IntoIterator for &'p SparseMerklePath { + type Item = as Iterator>::Item; + type IntoIter = SparseMerkleIter<'p>; + + fn into_iter(self) -> SparseMerkleIter<'p> { + let tree_depth = self.depth(); + SparseMerkleIter { path: self, next_depth: tree_depth } + } +} + +/// Borrowing iterator for [`SparseMerklePath`]. +pub struct SparseMerkleIter<'p> { + /// The "inner" value we're iterating over. + path: &'p SparseMerklePath, + + /// The depth a `next()` call will get. It will only be None if someone calls `next_back()` at + /// depth 0, to indicate that all further `next_back()` calls must also return `None`. + next_depth: u8, +} + +impl Iterator for SparseMerkleIter<'_> { + type Item = RpoDigest; + + fn next(&mut self) -> Option { + let this_depth = self.next_depth; + // Paths don't include the root, so if `this_depth` is 0 then we keep returning `None`. + let this_depth = NonZero::new(this_depth)?; + self.next_depth = this_depth.get() - 1; + + // `this_depth` is only ever decreasing, so it can't ever exceed `self.path.depth()`. + let node = self.path.at_depth(this_depth).unwrap(); + Some(node) + } + + // SparseMerkleIter always knows its exact size. + fn size_hint(&self) -> (usize, Option) { + let remaining = ExactSizeIterator::len(self); + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for SparseMerkleIter<'_> { + fn len(&self) -> usize { + self.next_depth as usize + } +} + +impl FusedIterator for SparseMerkleIter<'_> {} + +// TODO: impl DoubleEndedIterator. + +/// Owning iterator for [SparseMerklePath]. +pub struct IntoIter { + /// The "inner" value we're iterating over. + path: SparseMerklePath, + + /// The depth a `next()` call will get. It will only be None if someone calls `next_back()` at + /// depth 0, to indicate that all further `next_back()` calls must also return `None`. + next_depth: u8, +} + +impl IntoIterator for SparseMerklePath { + type IntoIter = IntoIter; + type Item = ::Item; + + fn into_iter(self) -> IntoIter { + let tree_depth = self.depth(); + IntoIter { path: self, next_depth: tree_depth } + } +} + +impl Iterator for IntoIter { + type Item = RpoDigest; + + fn next(&mut self) -> Option { + let this_depth = self.next_depth; + // Paths don't include the root, so if `this_depth` is 0 then we keep returning `None`. + let this_depth = NonZero::new(this_depth)?; + self.next_depth = this_depth.get() - 1; + + // `this_depth` is only ever decreasing, so it can't ever exceed `self.path.depth()`. + let node = self.path.at_depth(this_depth).unwrap(); + Some(node) + } + + // IntoIter always knows its exact size. + fn size_hint(&self) -> (usize, Option) { + let remaining = ExactSizeIterator::len(self); + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for IntoIter { + fn len(&self) -> usize { + self.next_depth as usize + } +} + +impl FusedIterator for IntoIter {} + +// TODO: impl DoubleEndedIterator. + +// COMPARISONS +// ================================================================================================ +impl PartialEq for SparseMerklePath { + fn eq(&self, rhs: &MerklePath) -> bool { + if self.depth() != rhs.depth() { + return false; + } + + for (node, &rhs_node) in iter::zip(self, rhs.iter()) { + if node != rhs_node { + return false; + } + } + + true + } +} + +impl PartialEq for MerklePath { + fn eq(&self, rhs: &SparseMerklePath) -> bool { + rhs == self + } +} + +// SPARSE MERKLE PATH CONTAINERS +// ================================================================================================ +/// A container for a [crate::Word] value and its [SparseMerklePath] opening. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SparseValuePath { + /// The node value opening for `path`. + pub value: RpoDigest, + /// The path from `value` to `root` (exclusive), using an efficient memory representation for + /// empty nodes. + pub path: SparseMerklePath, +} + +impl SparseValuePath { + /// Convenience function to construct a [SparseValuePath]. + /// + /// `value` is the value `path` leads to, in the tree. + pub fn new(value: RpoDigest, path: SparseMerklePath) -> Self { + Self { value, path } + } +} + +impl From<(SparseMerklePath, Word)> for SparseValuePath { + fn from((path, value): (SparseMerklePath, Word)) -> Self { + SparseValuePath::new(value.into(), path) + } +} + +impl TryFrom for SparseValuePath { + type Error = MerkleError; + + fn try_from(other: ValuePath) -> Result { + let ValuePath { value, path } = other; + let path = SparseMerklePath::try_from(path)?; + Ok(SparseValuePath { value, path }) + } +} + +impl From for ValuePath { + fn from(other: SparseValuePath) -> Self { + let SparseValuePath { value, path } = other; + ValuePath { value, path: path.into() } + } +} + +impl PartialEq for SparseValuePath { + fn eq(&self, rhs: &ValuePath) -> bool { + self.value == rhs.value && self.path == rhs.path + } +} + +impl PartialEq for ValuePath { + fn eq(&self, rhs: &SparseValuePath) -> bool { + rhs == self + } +} + +// SERIALIZATION +// ================================================================================================ + +impl Serializable for SparseMerklePath { + fn write_into(&self, target: &mut W) { + target.write_u8(self.depth()); + target.write_u64(self.empty_nodes); + target.write_many(&self.nodes); + } +} + +impl Deserializable for SparseMerklePath { + fn read_from( + source: &mut R, + ) -> Result { + let depth = source.read_u8()?; + let empty_nodes = source.read_u64()?; + let count = depth as u32 - empty_nodes.count_ones(); + let nodes = source.read_many::(count as usize)?; + Ok(Self { empty_nodes, nodes }) + } +} + +// HELPERS +// ================================================================================================ + +/// Iterator for path depths, which start at the deepest part of the tree and go the shallowest +/// depth before the root (depth 1). +fn path_depth_iter(tree_depth: u8) -> impl ExactSizeIterator> { + let top_down_iter = (1..=tree_depth).map(|depth| { + // SAFETY: `RangeInclusive<1, _>` cannot ever yield 0. Even if `tree_depth` is 0, then the + // range is `RangeInclusive<1, 0>` will simply not yield any values, and this block won't + // even be reached. + unsafe { NonZero::new_unchecked(depth) } + }); + + // Reverse the top-down iterator to get a bottom-up iterator. + top_down_iter.rev() +} + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + use core::{iter, num::NonZero}; + + use super::SparseMerklePath; + use crate::{ + Felt, ONE, Word, + hash::rpo::RpoDigest, + merkle::{ + EmptySubtreeRoots, MerklePath, NodeIndex, SMT_DEPTH, Smt, smt::SparseMerkleTree, + sparse_path::path_depth_iter, + }, + }; + + fn make_smt(pair_count: u64) -> Smt { + let entries: Vec<(RpoDigest, Word)> = (0..pair_count) + .map(|n| { + let leaf_index = ((n as f64 / pair_count as f64) * 255.0) as u64; + let key = RpoDigest::new([ONE, ONE, Felt::new(n), Felt::new(leaf_index)]); + let value = [ONE, ONE, ONE, ONE]; + (key, value) + }) + .collect(); + + Smt::with_entries(entries).unwrap() + } + + #[test] + fn test_sparse_indices() { + let tree = make_smt(8192); + + for (key, _value) in tree.entries() { + let (path, _) = tree.open(key).into_parts(); + let index = NodeIndex::from(Smt::key_to_leaf_index(key)); + + for (path_node, index) in iter::zip(path, index.proof_indices()) { + let tree_node = tree.get_hash(index); + assert_eq!(path_node, tree_node); + } + } + } + + #[test] + fn roundtrip() { + let tree = make_smt(8192); + + for (key, _value) in tree.entries() { + let (control_path, _) = tree.open(key).into_parts(); + assert_eq!(control_path.len(), tree.depth() as usize); + + let sparse_path = SparseMerklePath::try_from(control_path.clone()).unwrap(); + assert_eq!(control_path.depth(), sparse_path.depth()); + assert_eq!(sparse_path.depth(), SMT_DEPTH); + let test_path = MerklePath::from_iter(sparse_path.clone().into_iter()); + + for depth in path_depth_iter(control_path.depth()) { + let &control_node = control_path.at_depth(depth).unwrap(); + let &test_node = control_path.at_depth(depth).unwrap(); + let sparse_node = sparse_path.at_depth(depth).unwrap(); + assert_eq!(control_node, test_node); + assert_eq!(control_node, sparse_node, "at depth {depth}"); + } + + let mut count: u64 = 0; + for (&control_node, sparse_node) in iter::zip(control_path.iter(), sparse_path.iter()) { + count += 1; + assert_eq!(control_node, sparse_node); + } + assert_eq!(count, control_path.depth() as u64); + + for index in 0..=(control_path.len()) { + let control_node = control_path.at_idx(index).copied(); + let sparse_node = sparse_path.at_idx(index); + assert_eq!(control_node, sparse_node); + } + + assert_eq!( + sparse_path.at_depth(NonZero::new(sparse_path.depth()).unwrap()).unwrap(), + control_path[0], + ); + + assert_eq!(control_path, test_path); + } + } + + #[test] + fn test_sparse_bits() { + const DEPTH: u8 = 8; + let raw_nodes: [RpoDigest; DEPTH as usize] = [ + // Depth 8. + ([8u8, 8, 8, 8].into()), + // Depth 7. + *EmptySubtreeRoots::entry(DEPTH, 7), + // Depth 6. + *EmptySubtreeRoots::entry(DEPTH, 6), + // Depth 5. + [5u8, 5, 5, 5].into(), + // Depth 4. + [4u8, 4, 4, 4].into(), + // Depth 3. + *EmptySubtreeRoots::entry(DEPTH, 3), + // Depth 2. + *EmptySubtreeRoots::entry(DEPTH, 2), + // Depth 1. + *EmptySubtreeRoots::entry(DEPTH, 1), + // Root is not included. + ]; + + let sparse_nodes: [Option; DEPTH as usize] = [ + // Depth 8. + Some([8u8, 8, 8, 8].into()), + // Depth 7. + None, + // Depth 6. + None, + // Depth 5. + Some([5u8, 5, 5, 5].into()), + // Depth 4. + Some([4u8, 4, 4, 4].into()), + // Depth 3. + None, + // Depth 2. + None, + // Depth 1. + None, + // Root is not included. + ]; + + const EMPTY_BITS: u64 = 0b0110_0111; + + let sparse_path = SparseMerklePath::from_sized_iter(raw_nodes).unwrap(); + + assert_eq!(sparse_path.empty_nodes, EMPTY_BITS); + + // Depth 8. + { + let depth: u8 = 8; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 0); + let bit = 0b1000_0000; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert!(!is_set); + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + + let control_node = raw_nodes.get(idx).unwrap(); + let nonempty_idx: usize = 0; + assert_eq!(sparse_path.get_nonempty_index(NonZero::new(depth).unwrap()), nonempty_idx); + let test_node = sparse_path.nodes.get(nonempty_idx).unwrap(); + assert_eq!(test_node, control_node); + } + + // Depth 7. + { + let depth: u8 = 7; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 1); + let bit = 0b0100_0000; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert!(is_set); + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + + let &test_node = sparse_nodes.get(idx).unwrap(); + assert_eq!(test_node, None); + } + + // Depth 6. + { + let depth: u8 = 6; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 2); + let bit = 0b0010_0000; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + assert!(is_set); + + let &test_node = sparse_nodes.get(idx).unwrap(); + assert_eq!(test_node, None); + } + + // Depth 5. + { + let depth: u8 = 5; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 3); + let bit = 0b0001_0000; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + assert!(!is_set); + + let control_node = raw_nodes.get(idx).unwrap(); + let nonempty_idx: usize = 1; + assert_eq!(sparse_path.nodes.get(nonempty_idx).unwrap(), control_node); + assert_eq!(sparse_path.get_nonempty_index(NonZero::new(depth).unwrap()), nonempty_idx,); + let test_node = sparse_path.nodes.get(nonempty_idx).unwrap(); + assert_eq!(test_node, control_node); + } + + // Depth 4. + { + let depth: u8 = 4; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 4); + let bit = 0b0000_1000; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + assert!(!is_set); + + let control_node = raw_nodes.get(idx).unwrap(); + let nonempty_idx: usize = 2; + assert_eq!(sparse_path.nodes.get(nonempty_idx).unwrap(), control_node); + assert_eq!(sparse_path.get_nonempty_index(NonZero::new(depth).unwrap()), nonempty_idx,); + let test_node = sparse_path.nodes.get(nonempty_idx).unwrap(); + assert_eq!(test_node, control_node); + } + + // Depth 3. + { + let depth: u8 = 3; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 5); + let bit = 0b0000_0100; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert!(is_set); + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + + let &test_node = sparse_nodes.get(idx).unwrap(); + assert_eq!(test_node, None); + } + + // Depth 2. + { + let depth: u8 = 2; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 6); + let bit = 0b0000_0010; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert!(is_set); + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + + let &test_node = sparse_nodes.get(idx).unwrap(); + assert_eq!(test_node, None); + } + + // Depth 1. + { + let depth: u8 = 1; + let idx = (sparse_path.depth() - depth) as usize; + assert_eq!(idx, 7); + let bit = 0b0000_0001; + assert_eq!(bit, 1 << (depth - 1)); + let is_set = (sparse_path.empty_nodes & bit) != 0; + assert!(is_set); + assert_eq!(is_set, sparse_nodes.get(idx).unwrap().is_none()); + + let &test_node = sparse_nodes.get(idx).unwrap(); + assert_eq!(test_node, None); + } + } + + #[test] + fn from_sized_iter() { + let tree = make_smt(8192); + + for (key, _value) in tree.entries() { + let index = NodeIndex::from(Smt::key_to_leaf_index(key)); + + let control_path = tree.get_path(key); + for (&control_node, proof_index) in iter::zip(&*control_path, index.proof_indices()) { + let proof_node = tree.get_hash(proof_index); + assert_eq!(control_node, proof_node, "WHat"); + } + + let sparse_path = + SparseMerklePath::from_sized_iter(control_path.clone().into_iter()).unwrap(); + for (sparse_node, proof_idx) in iter::zip(sparse_path.clone(), index.proof_indices()) { + let proof_node = tree.get_hash(proof_idx); + assert_eq!(sparse_node, proof_node, "WHat"); + } + + assert_eq!(control_path.depth(), sparse_path.depth()); + for (i, (control, sparse)) in iter::zip(control_path, sparse_path).enumerate() { + assert_eq!(control, sparse, "on iteration {i}"); + } + } + } + + #[test] + fn random_access() { + let tree = make_smt(8192); + + for (i, (key, _value)) in tree.entries().enumerate() { + let control_path = tree.get_path(key); + let sparse_path = SparseMerklePath::try_from(control_path.clone()).unwrap(); + assert_eq!(control_path.depth(), sparse_path.depth()); + assert_eq!(sparse_path.depth(), SMT_DEPTH); + + let depth_iter = path_depth_iter(control_path.depth()); + for (depth, control_node) in iter::zip(depth_iter, control_path.iter()) { + let test_node = sparse_path.at_depth(depth).unwrap(); + assert_eq!(*control_node, test_node, "at depth {depth} for entry {i}"); + } + } + } + + #[test] + fn test_owning_iterator() { + let tree = make_smt(8192); + + for (i, (key, _value)) in tree.entries().enumerate() { + let path = tree.get_path(key); + let sparse_path = SparseMerklePath::try_from(path.clone()).unwrap(); + + // Test owned iterator. + let mut depth_iter = path_depth_iter(sparse_path.depth()); + let mut path_iter = sparse_path.clone().into_iter(); + assert_eq!(path_iter.len(), tree.depth() as usize); + + while let Some(iter_node) = path_iter.next() { + let depth = depth_iter.next().unwrap(); + assert_eq!(depth_iter.len(), path_iter.len()); + let path_index = path.depth() - depth.get(); + let control_node = path[path_index as usize]; + assert_eq!(iter_node, control_node, "at depth {depth} for entry {i}"); + } + assert_eq!(depth_iter.next(), None); + assert_eq!(depth_iter.len(), 0); + assert_eq!(path_iter.next(), None); + assert_eq!(path_iter.len(), 0); + } + } + + #[test] + fn test_borrowing_iterator() { + let tree = make_smt(8192); + + for (i, (key, _value)) in tree.entries().enumerate() { + let path = tree.get_path(key); + let sparse_path = SparseMerklePath::try_from(path.clone()).unwrap(); + assert_eq!(path.depth(), sparse_path.depth()); + assert_eq!(sparse_path.depth(), SMT_DEPTH); + + let mut depth_iter = path_depth_iter(sparse_path.depth()); + let mut path_iter = sparse_path.iter(); + assert_eq!(path_iter.len(), tree.depth() as usize); + + let mut did_iterate = false; + while let Some(iter_node) = path_iter.next() { + did_iterate = true; + let depth = depth_iter.next().unwrap(); + assert_eq!(depth_iter.len(), path_iter.len()); + let path_index = path.depth() - depth.get(); + let control_node = path[path_index as usize]; + assert_eq!(iter_node, control_node, "at depth {depth} for entry {i}"); + } + assert!(did_iterate); + assert_eq!(depth_iter.next(), None); + assert_eq!(depth_iter.len(), 0); + assert_eq!(path_iter.next(), None); + assert_eq!(path_iter.len(), 0); + } + } + + #[test] + fn test_zero_sized() { + let nodes: Vec = Default::default(); + + let sparse_path = SparseMerklePath::from_sized_iter(nodes).unwrap(); + assert_eq!(sparse_path.depth(), 0); + assert_eq!(sparse_path.iter().next(), None); + assert_eq!(sparse_path.into_iter().next(), None); + } +} From b31a50de2ad25ab842b9c68bae0faa22dbf70535 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 4 Apr 2025 15:09:32 +0200 Subject: [PATCH 7/8] SimpleSmt: do not assume that Merkle paths Deref to Vec in tests This is in preparation for the next commit, where we change `SimpleSmt::open()` to return a `SparseMerklePath`, which cannot dereference to a Vec. --- miden-crypto/src/merkle/smt/simple/tests.rs | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/miden-crypto/src/merkle/smt/simple/tests.rs b/miden-crypto/src/merkle/smt/simple/tests.rs index 919fed2..a4a7a6b 100644 --- a/miden-crypto/src/merkle/smt/simple/tests.rs +++ b/miden-crypto/src/merkle/smt/simple/tests.rs @@ -10,8 +10,8 @@ use crate::{ EMPTY_WORD, Word, hash::rpo::Rpo256, merkle::{ - EmptySubtreeRoots, InnerNodeInfo, LeafIndex, MerkleTree, digests_to_words, int_to_leaf, - int_to_node, smt::SparseMerkleTree, + EmptySubtreeRoots, InnerNodeInfo, LeafIndex, MerklePath, MerkleTree, digests_to_words, + int_to_leaf, int_to_node, smt::SparseMerkleTree, }, }; @@ -115,10 +115,22 @@ fn test_depth2_tree() { assert_eq!(VALUES4[3], tree.get_node(NodeIndex::make(2, 3)).unwrap()); // check get_path(): depth 2 - assert_eq!(vec![VALUES4[1], node3], *tree.open(&LeafIndex::<2>::new(0).unwrap()).path); - assert_eq!(vec![VALUES4[0], node3], *tree.open(&LeafIndex::<2>::new(1).unwrap()).path); - assert_eq!(vec![VALUES4[3], node2], *tree.open(&LeafIndex::<2>::new(2).unwrap()).path); - assert_eq!(vec![VALUES4[2], node2], *tree.open(&LeafIndex::<2>::new(3).unwrap()).path); + assert_eq!( + MerklePath::from(vec![VALUES4[1], node3]), + tree.open(&LeafIndex::<2>::new(0).unwrap()).path, + ); + assert_eq!( + MerklePath::from(vec![VALUES4[0], node3]), + tree.open(&LeafIndex::<2>::new(1).unwrap()).path, + ); + assert_eq!( + MerklePath::from(vec![VALUES4[3], node2]), + tree.open(&LeafIndex::<2>::new(2).unwrap()).path, + ); + assert_eq!( + MerklePath::from(vec![VALUES4[2], node2]), + tree.open(&LeafIndex::<2>::new(3).unwrap()).path, + ); } #[test] @@ -239,7 +251,7 @@ fn small_tree_opening_is_consistent() { for (key, path) in cases { let opening = tree.open(&LeafIndex::<3>::new(key).unwrap()); - assert_eq!(path, *opening.path); + assert_eq!(MerklePath::from(path), opening.path); } } From b43afcd2cebb41c51d18cd0736f36d225a4bf0d3 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 4 Apr 2025 15:14:27 +0200 Subject: [PATCH 8/8] smt: change SimpleSmt::open() to return a sparse path --- miden-crypto/src/merkle/smt/simple/mod.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/miden-crypto/src/merkle/smt/simple/mod.rs b/miden-crypto/src/merkle/smt/simple/mod.rs index 6773cd1..054e08e 100644 --- a/miden-crypto/src/merkle/smt/simple/mod.rs +++ b/miden-crypto/src/merkle/smt/simple/mod.rs @@ -1,5 +1,7 @@ use alloc::collections::BTreeSet; +use crate::merkle::{SparseMerklePath, SparseValuePath}; + use super::{ super::ValuePath, EMPTY_WORD, EmptySubtreeRoots, InnerNode, InnerNodeInfo, InnerNodes, LeafIndex, MerkleError, MerklePath, MutationSet, NodeIndex, RpoDigest, SMT_MAX_DEPTH, @@ -169,8 +171,15 @@ impl SimpleSmt { /// Returns an opening of the leaf associated with `key`. Conceptually, an opening is a Merkle /// path to the leaf, as well as the leaf itself. - pub fn open(&self, key: &LeafIndex) -> ValuePath { - >::open(self, key) + pub fn open(&self, key: &LeafIndex) -> SparseValuePath { + let value = RpoDigest::new(self.get_value(key)); + let nodes = key.index.proof_indices().map(|index| self.get_hash(index)); + // `from_sized_iter()` returns an error if there are more nodes than `SMT_MAX_DEPTH`, but + // this could only happen if we have more levels than `SMT_MAX_DEPTH` ourselves, which is + // guarded against in `SimpleSmt::new()`. + let path = SparseMerklePath::from_sized_iter(nodes).unwrap(); + + SparseValuePath { value, path } } /// Returns a boolean value indicating whether the SMT is empty.