From c808c42da74db9db77c9de4bd1426dc496b1ad80 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 21 Mar 2025 13:43:43 +0100 Subject: [PATCH 1/6] SparseMerklePath: implement random access --- miden-crypto/src/merkle/sparse_path.rs | 75 ++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/miden-crypto/src/merkle/sparse_path.rs b/miden-crypto/src/merkle/sparse_path.rs index fc1b202..017e6b0 100644 --- a/miden-crypto/src/merkle/sparse_path.rs +++ b/miden-crypto/src/merkle/sparse_path.rs @@ -1,7 +1,7 @@ use alloc::vec::Vec; use core::iter; -use super::{EmptySubtreeRoots, MerklePath, RpoDigest, SMT_MAX_DEPTH}; +use super::{EmptySubtreeRoots, MerkleError, MerklePath, RpoDigest, SMT_MAX_DEPTH}; /// A different representation of [`MerklePath`] designed for memory efficiency for Merkle paths /// with empty nodes. @@ -89,6 +89,50 @@ impl SparseMerklePath { pub fn depth(&self) -> u8 { (self.nodes.len() + self.empty_nodes.count_ones() as usize) as u8 } + + /// Get a specific node in this path at a given depth. + /// + /// # Errors + /// Returns [MerkleError::DepthTooBig] if `node_depth` is greater than the total depth of this + /// path. + pub fn get(&self, node_depth: u8) -> Result { + let node = self + .get_nonempty(node_depth)? + .unwrap_or_else(|| *EmptySubtreeRoots::entry(self.depth(), node_depth)); + + Ok(node) + } + + /// Get a specific non-emptynode in this path at a given depth, or `None` if the specified node + /// is an empty node. + /// + /// # Errors + /// Returns [MerkleError::DepthTooBig] if `node_depth` is greater than the total depth of this + /// path. + pub fn get_nonempty(&self, node_depth: u8) -> Result, MerkleError> { + if node_depth > self.depth() { + return Err(MerkleError::DepthTooBig(node_depth.into())); + } + + let empty_bit = 1u64 << node_depth; + let is_empty = (self.empty_nodes & empty_bit) != 0; + + if is_empty { + return Ok(None); + } + + // Our index needs to account for all the empty nodes that aren't in `self.nodes`. + let nonempty_index: usize = { + // TODO: this could also be u64::unbounded_shl(1, node_depth + 1).wrapping_sub(1). + // We should check if that has any performance benefits over using 128-bit integers. + let mask: u64 = ((1u128 << (node_depth + 1)) - 1u128).try_into().unwrap(); + + let empty_before = u64::count_ones(self.empty_nodes & mask); + node_depth as usize - empty_before as usize + }; + + Ok(Some(self.nodes[nonempty_index])) + } } #[cfg(test)] @@ -102,9 +146,7 @@ mod tests { merkle::{SMT_DEPTH, Smt, smt::SparseMerkleTree}, }; - #[test] - fn roundtrip() { - let pair_count: u64 = 8192; + fn make_smt(pair_count: u64) -> Smt { let entries: Vec<(RpoDigest, Word)> = (0..pair_count) .map(|n| { let leaf_index = ((n as f64 / pair_count as f64) * 255.0) as u64; @@ -113,7 +155,13 @@ mod tests { (key, value) }) .collect(); - let tree = Smt::with_entries(entries).unwrap(); + + Smt::with_entries(entries).unwrap() + } + + #[test] + fn roundtrip() { + let tree = make_smt(8192); for (key, _value) in tree.entries() { let control_path = tree.path(key); @@ -125,4 +173,21 @@ mod tests { assert_eq!(control_path, test_path); } } + + #[test] + fn random_access() { + let tree = make_smt(8192); + + for (i, (key, _value)) in tree.entries().enumerate() { + let control_path = tree.path(key); + let sparse_path = SparseMerklePath::from_path(control_path.clone()).unwrap(); + assert_eq!(control_path.depth(), sparse_path.depth()); + assert_eq!(sparse_path.depth(), SMT_DEPTH); + + for (depth, control_node) in control_path.iter().enumerate() { + let test_node = sparse_path.get(depth as u8).unwrap(); + assert_eq!(*control_node, test_node, "at depth {depth} for entry {i}"); + } + } + } } From f4fb5533c79db1b910a9a69c3043f397191fa4c9 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 21 Mar 2025 13:59:36 +0100 Subject: [PATCH 2/6] SparseMerklePath: implement iterators --- miden-crypto/src/merkle/sparse_path.rs | 118 +++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/miden-crypto/src/merkle/sparse_path.rs b/miden-crypto/src/merkle/sparse_path.rs index 017e6b0..0b47743 100644 --- a/miden-crypto/src/merkle/sparse_path.rs +++ b/miden-crypto/src/merkle/sparse_path.rs @@ -135,6 +135,99 @@ impl SparseMerklePath { } } +// ITERATORS +// ================================================================================================ + +/// Contructs a [SparseMerklePath] out of an iterator of optional nodes, where `None` indicates an +/// empty node. +impl FromIterator> for SparseMerklePath { + fn from_iter(iter: I) -> SparseMerklePath + where + I: IntoIterator>, + { + let mut empty_nodes: u64 = 0; + let mut nodes: Vec = Default::default(); + for (depth, node) in iter.into_iter().enumerate() { + match node { + Some(node) => nodes.push(node), + None => empty_nodes |= 1 << depth, + } + } + + SparseMerklePath { nodes, empty_nodes } + } +} + +impl IntoIterator for SparseMerklePath { + type Item = ::Item; + type IntoIter = SparseMerkleIter; + + fn into_iter(self) -> SparseMerkleIter { + let tree_depth = self.depth(); + SparseMerkleIter { + path: self, + next_depth: Some(0), + tree_depth, + } + } +} + +/// Owning iterator for [`SparseMerklePath`]. +// TODO: add a non-owning iterator too. +pub struct SparseMerkleIter { + /// The "inner" value we're iterating over. + path: SparseMerklePath, + + /// The depth a `next()` call will get. It will only be None if someone calls `next_back()` at + /// depth 0, to indicate that all further `next_back()` calls must also return `None`. + next_depth: Option, + + /// "Cached" value of `path.depth()`. + tree_depth: u8, +} + +impl Iterator for SparseMerkleIter { + type Item = RpoDigest; + + fn next(&mut self) -> Option { + // If `next_depth` is None, then someone called `next_back()` at depth 0. + let next_depth = self.next_depth.unwrap_or(0); + if next_depth >= self.tree_depth { + return None; + } + + let node = self.path.get(next_depth).unwrap(); + self.next_depth = Some(next_depth + 1); + Some(node) + } + + // SparseMerkleIter always knows its exact size. + fn size_hint(&self) -> (usize, Option) { + let next_depth = self.next_depth.unwrap_or(0); + let len: usize = self.path.depth().into(); + let remaining = len - next_depth as usize; + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for SparseMerkleIter { + fn len(&self) -> usize { + let next_depth = self.next_depth.unwrap_or(0); + (self.path.depth() - next_depth) as usize + } +} + +impl DoubleEndedIterator for SparseMerkleIter { + fn next_back(&mut self) -> Option { + // While `next_depth` is None, all calls to `next_back()` also return `None`. + let next_depth = self.next_depth?; + + let node = self.path.get(next_depth).unwrap(); + self.next_depth = if next_depth == 0 { None } else { Some(next_depth - 1) }; + Some(node) + } +} + #[cfg(test)] mod tests { use alloc::vec::Vec; @@ -190,4 +283,29 @@ mod tests { } } } + + #[test] + fn iterator() { + let tree = make_smt(8192); + + for (i, (key, _value)) in tree.entries().enumerate() { + let path = tree.path(key); + let sparse_path = SparseMerklePath::from_path(path.clone()).unwrap(); + assert_eq!(path.depth(), sparse_path.depth()); + assert_eq!(sparse_path.depth(), SMT_DEPTH); + for (depth, iter_node) in sparse_path.clone().into_iter().enumerate() { + let control_node = sparse_path.get(depth as u8).unwrap(); + assert_eq!(control_node, iter_node, "at depth {depth} for entry {i}"); + } + + let iter = sparse_path.clone().into_iter().enumerate().rev().skip(1); + for (depth, iter_node) in iter { + let control_node = sparse_path.get(depth as u8).unwrap(); + assert_eq!( + control_node, iter_node, + "at depth {depth} for entry {i} during reverse-iteration", + ); + } + } + } } From c65e5de887b7eeb9ad2a1938abd5550e9099e642 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 21 Mar 2025 14:07:54 +0100 Subject: [PATCH 3/6] SparseMerklePath: implement Serializable and Deserializable --- miden-crypto/src/merkle/sparse_path.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/miden-crypto/src/merkle/sparse_path.rs b/miden-crypto/src/merkle/sparse_path.rs index 0b47743..feac1e2 100644 --- a/miden-crypto/src/merkle/sparse_path.rs +++ b/miden-crypto/src/merkle/sparse_path.rs @@ -1,6 +1,8 @@ use alloc::vec::Vec; use core::iter; +use winter_utils::{Deserializable, DeserializationError, Serializable}; + use super::{EmptySubtreeRoots, MerkleError, MerklePath, RpoDigest, SMT_MAX_DEPTH}; /// A different representation of [`MerklePath`] designed for memory efficiency for Merkle paths @@ -228,6 +230,29 @@ impl DoubleEndedIterator for SparseMerkleIter { } } +// SERIALIZATION +// ================================================================================================ + +impl Serializable for SparseMerklePath { + fn write_into(&self, target: &mut W) { + target.write_u8(self.depth()); + target.write_u64(self.empty_nodes); + target.write_many(&self.nodes); + } +} + +impl Deserializable for SparseMerklePath { + fn read_from( + source: &mut R, + ) -> Result { + let depth = source.read_u8()?; + let empty_nodes = source.read_u64()?; + let count = depth as u32 - empty_nodes.count_ones(); + let nodes = source.read_many::(count as usize)?; + Ok(Self { empty_nodes, nodes }) + } +} + #[cfg(test)] mod tests { use alloc::vec::Vec; From 297868c532020a87a8adb255a6acfa4735e97657 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 21 Mar 2025 18:41:04 +0100 Subject: [PATCH 4/6] SparseMerklePath: impl From/Into conversions --- miden-crypto/src/merkle/sparse_path.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/miden-crypto/src/merkle/sparse_path.rs b/miden-crypto/src/merkle/sparse_path.rs index feac1e2..c01faad 100644 --- a/miden-crypto/src/merkle/sparse_path.rs +++ b/miden-crypto/src/merkle/sparse_path.rs @@ -137,6 +137,30 @@ impl SparseMerklePath { } } +// CONVERSIONS +// ================================================================================================ + +impl From for MerklePath { + fn from(sparse_path: SparseMerklePath) -> Self { + sparse_path.into_path() + } +} + +/// # Panics +/// +/// This conversion fails and panics if the path length is greater than [`SMT_MAX_DEPTH`]. +impl From for SparseMerklePath { + fn from(path: MerklePath) -> Self { + SparseMerklePath::from_path(path).unwrap() + } +} + +impl From for Vec { + fn from(path: SparseMerklePath) -> Self { + path.into_path().into() + } +} + // ITERATORS // ================================================================================================ From 90423ca62207289f92873036fb038aee4aacfa92 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Sat, 22 Mar 2025 20:42:07 +0100 Subject: [PATCH 5/6] smt: impl SimpleSmt::get_path() which returns a sparse path --- miden-crypto/src/merkle/smt/simple/mod.rs | 37 ++++++++++++++++++- miden-crypto/src/merkle/smt/simple/tests.rs | 10 ++++- miden-crypto/src/merkle/sparse_path.rs | 41 ++++++++++++++++++++- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/miden-crypto/src/merkle/smt/simple/mod.rs b/miden-crypto/src/merkle/smt/simple/mod.rs index 6773cd1..737b335 100644 --- a/miden-crypto/src/merkle/smt/simple/mod.rs +++ b/miden-crypto/src/merkle/smt/simple/mod.rs @@ -1,10 +1,11 @@ -use alloc::collections::BTreeSet; +use alloc::{collections::BTreeSet, vec::Vec}; use super::{ super::ValuePath, EMPTY_WORD, EmptySubtreeRoots, InnerNode, InnerNodeInfo, InnerNodes, LeafIndex, MerkleError, MerklePath, MutationSet, NodeIndex, RpoDigest, SMT_MAX_DEPTH, SMT_MIN_DEPTH, SparseMerkleTree, Word, }; +use crate::merkle::{SparseMerklePath, sparse_path::SparseValuePath}; #[cfg(test)] mod tests; @@ -173,6 +174,40 @@ impl SimpleSmt { >::open(self, key) } + /// Returns a path (but not an opening) to the leaf associated with `key`. + /// + /// Unlike [`SimpleSmt::open()`], this returns a [SparseValuePath] which has a more efficient + /// memory representation optimized for paths containing empty nodes. See [SparseMerklePath] + /// for more information. + pub fn get_path(&self, key: &LeafIndex) -> SparseValuePath { + let value: RpoDigest = self.get_value(key).into(); + + // This is a partial re-implementation of `SparseMerklePath::from_sized_iter()`, which + // constructs in place instead of cloning and immediately dropping the entire vec returned + // by `self.open()`. + + let mut nodes: Vec = Default::default(); + let mut index = NodeIndex::from(*key); + let mut empty_nodes: u64 = 0; + + for _ in 0..DEPTH { + let is_right = index.is_value_odd(); + index.move_up(); + + match self.inner_nodes.get(&index) { + Some(InnerNode { left, right }) => { + let value = if is_right { left } else { right }; + nodes.push(*value); + }, + None => empty_nodes |= u64::checked_shl(1, index.depth().into()).unwrap(), + } + } + + let path = SparseMerklePath::from_raw_parts(empty_nodes, nodes); + + SparseValuePath { value, path } + } + /// Returns a boolean value indicating whether the SMT is empty. pub fn is_empty(&self) -> bool { debug_assert_eq!(self.leaves.is_empty(), self.root == Self::EMPTY_ROOT); diff --git a/miden-crypto/src/merkle/smt/simple/tests.rs b/miden-crypto/src/merkle/smt/simple/tests.rs index 919fed2..a24ba5b 100644 --- a/miden-crypto/src/merkle/smt/simple/tests.rs +++ b/miden-crypto/src/merkle/smt/simple/tests.rs @@ -1,4 +1,5 @@ use alloc::vec::Vec; +use core::iter; use assert_matches::assert_matches; @@ -237,9 +238,16 @@ fn small_tree_opening_is_consistent() { ]; for (key, path) in cases { - let opening = tree.open(&LeafIndex::<3>::new(key).unwrap()); + let index = LeafIndex::<3>::new(key).unwrap(); + let opening = tree.open(&index); assert_eq!(path, *opening.path); + + // Also check that the sparse versions work the same way. + let sparse_path = tree.get_path(&index); + for (path_node, sparse_path_node) in iter::zip(path, sparse_path.path) { + assert_eq!(path_node, sparse_path_node); + } } } diff --git a/miden-crypto/src/merkle/sparse_path.rs b/miden-crypto/src/merkle/sparse_path.rs index c01faad..a43b139 100644 --- a/miden-crypto/src/merkle/sparse_path.rs +++ b/miden-crypto/src/merkle/sparse_path.rs @@ -3,7 +3,7 @@ use core::iter; use winter_utils::{Deserializable, DeserializationError, Serializable}; -use super::{EmptySubtreeRoots, MerkleError, MerklePath, RpoDigest, SMT_MAX_DEPTH}; +use super::{EmptySubtreeRoots, MerkleError, MerklePath, RpoDigest, SMT_MAX_DEPTH, Word}; /// A different representation of [`MerklePath`] designed for memory efficiency for Merkle paths /// with empty nodes. @@ -87,6 +87,18 @@ impl SparseMerklePath { MerklePath::from(nodes) } + /// Creates a [SparseMerklePath] directly from a bitmask representing empty nodes, and a vec + /// containing all non-empty nodes. + pub fn from_raw_parts(empty_nodes: u64, nodes: Vec) -> Self { + Self { empty_nodes, nodes } + } + + /// Decomposes a [SparseMerklePath] into its raw components: `(empty_nodes, nodes)`. + pub fn into_raw_parts(self) -> (u64, Vec) { + let SparseMerklePath { empty_nodes, nodes } = self; + (empty_nodes, nodes) + } + /// Returns the total depth of this path, i.e., the number of nodes this path represents. pub fn depth(&self) -> u8 { (self.nodes.len() + self.empty_nodes.count_ones() as usize) as u8 @@ -254,6 +266,33 @@ impl DoubleEndedIterator for SparseMerkleIter { } } +// SPARSE MERKLE PATH CONTAINERS +// ================================================================================================ +/// A container for a [crate::Word] value and its [SparseMerklePath] opening. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SparseValuePath { + /// The node value opening for `path`. + pub value: RpoDigest, + /// The path from `value` to `root` (exclusive), using an efficient memory representation for + /// empty nodes. + pub path: SparseMerklePath, +} + +impl SparseValuePath { + /// Convenience function to construct a [SparseValuePath]. + /// + /// `value` is the value `path` leads to, in the tree. + pub fn new(value: RpoDigest, path: SparseMerklePath) -> Self { + Self { value, path } + } +} + +impl From<(SparseMerklePath, Word)> for SparseValuePath { + fn from((path, value): (SparseMerklePath, Word)) -> Self { + SparseValuePath::new(value.into(), path) + } +} + // SERIALIZATION // ================================================================================================ From 9c84bcb2f60d9c3435fc50764dd21d7ae0d23fb5 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 24 Mar 2025 17:00:45 +0100 Subject: [PATCH 6/6] PartialSmt: fix misleading variable names --- miden-crypto/src/merkle/smt/partial.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miden-crypto/src/merkle/smt/partial.rs b/miden-crypto/src/merkle/smt/partial.rs index 93479d3..8bbc4e9 100644 --- a/miden-crypto/src/merkle/smt/partial.rs +++ b/miden-crypto/src/merkle/smt/partial.rs @@ -46,8 +46,8 @@ impl PartialSmt { { let mut partial_smt = Self::new(); - for (leaf, path) in paths.into_iter().map(SmtProof::into_parts) { - partial_smt.add_path(path, leaf)?; + for (path, leaf) in paths.into_iter().map(SmtProof::into_parts) { + partial_smt.add_path(leaf, path)?; } Ok(partial_smt)