diff --git a/CHANGELOG.md b/CHANGELOG.md index ca38d2add..511321edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Configure `NativeFaucet`, which determines the native asset used to pay fees - Configure the base verification fee - Note: fees are not yet activated, and this has no impact beyond setting these values in the block headers +- [BREAKING] `GetAccountProofs` endpoint uses a `PartialSmt` for proofs. ([#1158](https://github.com/0xMiden/miden-node/pull/1158)). ### Fixes diff --git a/Cargo.lock b/Cargo.lock index 2b1ae6cc2..ee59b301d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2467,6 +2467,7 @@ dependencies = [ "miden-node-proto-build", "miden-node-utils", "miden-objects", + "pretty_assertions", "proptest", "prost", "prost-build", diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index ae1677656..597c135ec 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -25,7 +25,8 @@ thiserror = { workspace = true } tonic = { workspace = true } [dev-dependencies] -proptest = { version = "1.7" } +proptest = { version = "1.7" } +pretty_assertions = { workspace = true} [build-dependencies] anyhow = { workspace = true } diff --git a/crates/proto/src/domain/merkle.rs b/crates/proto/src/domain/merkle.rs index ca1ca551a..7e4ecd7f3 100644 --- a/crates/proto/src/domain/merkle.rs +++ b/crates/proto/src/domain/merkle.rs @@ -1,9 +1,14 @@ +use std::collections::{HashMap, HashSet}; + use miden_objects::Word; use miden_objects::crypto::merkle::{ Forest, + InnerNode, LeafIndex, MerklePath, MmrDelta, + NodeIndex, + PartialSmt, SmtLeaf, SmtProof, SparseMerklePath, @@ -206,3 +211,226 @@ impl From for proto::primitives::SmtOpening { } } } + +// NODE INDEX +// ------------------------------------------------------------------------------------------------ +impl From for proto::primitives::NodeIndex { + fn from(value: NodeIndex) -> Self { + proto::primitives::NodeIndex { + depth: value.depth() as u32, + value: value.value(), + } + } +} +impl TryFrom for NodeIndex { + type Error = ConversionError; + fn try_from(index: proto::primitives::NodeIndex) -> Result { + let depth = u8::try_from(index.depth)?; + let value = index.value; + Ok(NodeIndex::new(depth, value)?) + } +} + +// PARTIAL SMT +// ------------------------------------------------------------------------------------------------ + +impl TryFrom for PartialSmt { + type Error = ConversionError; + fn try_from(value: proto::primitives::PartialSmt) -> Result { + let proto::primitives::PartialSmt { root, leaves, nodes } = value; + let root = root + .as_ref() + .ok_or(proto::primitives::PartialSmt::missing_field(stringify!(root)))? + .try_into()?; + // TODO ensure `!leaves.is_empty()` + + // Convert other proto primitives to crypto types + let leaves = Result::, _>::from_iter(try_convert(leaves))?; + let mut inner = + Result::, _>::from_iter(nodes.into_iter().map(|inner| { + let node_index = NodeIndex::try_from( + inner + .index + .ok_or(proto::primitives::NodeIndex::missing_field(stringify!(index)))?, + )?; + let digest = Word::try_from( + inner + .digest + .ok_or(proto::primitives::Digest::missing_field(stringify!(digest)))?, + )?; + Ok::<_, Self::Error>((node_index, digest)) + }))?; + + let leaf_indices = + HashSet::::from_iter(leaves.iter().map(|leaf| leaf.index().into())); + + // Must contain the leaves too + inner.extend(leaves.iter().map(|leaf| (leaf.index().into(), leaf.hash()))); + + // Start constructing the partial SMT + // + // Construct a `MerklePath` per leaf by transcending from leaf digest down to depth 0. + // Then verify the merkle proof holds consistency and completeness checks and all + // required sibling nodes are present to deeduct required intermediate nodes. + let mut partial = PartialSmt::new(); + for leaf in leaves { + // Construct the merkle path: + let leaf_node_index: NodeIndex = leaf.index().into(); + let mut current = leaf_node_index.clone(); + let mut siblings = Vec::new(); + + // If we ever try to trancend beyond this depth level, something is wrong and + // we must stop decoding. + let max_depth = leaf_node_index.depth(); + // root: 00 + // / \ + // 10 11 + // / \ / \ + // 20 21 22 23 + // / \ / \ / \ / \ + // leaves ... x y + // Iterate from the leaf up to the root (exclusive) + // We start by picking the sibling of `x`, `y`, our starting point and + // then moving towards the root `0`. By definition siblings have the same parent. + loop { + let sibling_idx = current.sibling(); + // TODO FIXME for a leaf we get another leaf, we need to ensure those are part of + // the inner set or contained in the inner HashMap + let sibling_digest = if let Some(sibling_digest) = inner.get(&sibling_idx) { + // Previous round already calculated the entry or it was given explicitly + *sibling_digest + } else { + // The entry does not exist, so we need to lazily follow the missing nodes and + // calculate recursively. + + // DFS, build the subtree recursively, starting from the current sibling + let mut stack = Vec::::new(); + stack.push(sibling_idx.clone()); + loop { + let Some(idx) = stack.pop() else { + unreachable!( + "Must be an error, we must have nodes to resolve all questions, otherwise construction is borked" + ) + }; + if let Some(node_digest) = inner.get(&idx) { + if stack.is_empty() && idx == sibling_idx { + // we emptied the stack which means the current one is our desired + // starting point + break *node_digest; + } + // if the digest exists, we don't need to recurse + continue; + } + debug_assert!( + !leaf_indices.contains(&idx), + "For every relevant leaf, we must have the relevant value" + ); + let left = idx.left_child(); + let right = idx.right_child(); + if max_depth < left.depth() || max_depth < right.depth() { + // TODO might happen in case of a missing node, so we must handle this + // gracefully + unreachable!("graceful!") + } + // proceed if the inner nodes are unknown + if !inner.contains_key(&left) { + stack.push(left); + } + if !inner.contains_key(&right) { + stack.push(right); + } + // left and right exist, we can derive the digest for `idx` + if let Some(&left) = inner.get(&left) + && let Some(&right) = inner.get(&right) + { + let node = InnerNode { left, right }; + let node_digest = node.hash(); + + if stack.is_empty() && idx == sibling_idx { + // we emptied the stack which means the current one is our desired + // starting point + break node_digest; + } + inner.insert(idx, node_digest); + } + } + }; + siblings.push(sibling_digest); + + // Move up to the parent level, and repeat + current = current.parent(); + if current.depth() == 0 { + break; + } + } + + let path = MerklePath::new(siblings); + path.verify(leaf_node_index.value(), leaf.hash(), &root).expect("It's fine"); + partial.add_path(leaf, path); + } + assert_eq!(partial.root(), root); // FIXME make error + Ok(partial) + } +} + +impl From for proto::primitives::PartialSmt { + fn from(partial: PartialSmt) -> Self { + // Find all leaf digests, we need to include those, they are POIs + let mut leaves = Vec::new(); + for (key, value) in partial.entries() { + let leaf = partial.get_leaf(key).unwrap(); + leaves.push(crate::generated::primitives::SmtLeaf::from(leaf)); + } + + // Now collect the minimal set of internal nodes to be able to recalc the intermediate nodes + // forming a partial smt + let mut retained = HashMap::::new(); + for (idx, node) in partial.inner_node_indices() { + // if neither of the child keys are tracked, we cannot re-calc the inner node digest + // on-the-fly and hence need to add the node to the set to be transferred + if partial.get_value(node.left).is_err() || partial.get_value(node.left).is_err() { + retained.insert(idx, node.hash()); + continue; + } + } + let nodes = Vec::from_iter(retained.into_iter().map(|(index, digest)| { + crate::generated::primitives::InnerNode { + index: Some(crate::generated::primitives::NodeIndex::from(index)), + digest: Some(crate::generated::primitives::Digest::from(digest)), + } + })); + let root = Some(partial.root().into()); + // Remember: nodes and leaves as mutually exclusive + Self { root, nodes, leaves } + } +} + +#[cfg(test)] +mod tests { + use miden_objects::crypto::merkle::{PartialSmt, Smt}; + use pretty_assertions::assert_eq; + + use super::*; + #[test] + fn partial_smt_roundtrip() { + let mut x = Smt::new(); + + x.insert(Word::from([1_u32, 2, 3, 4]), Word::from([5_u32, 6, 7, 8])); + x.insert(Word::from([10_u32, 11, 12, 13]), Word::from([14_u32, 15, 16, 17])); + x.insert(Word::from([0x00_u32, 0xFF, 0xFF, 0xFF]), Word::from([0x00_u32; 4])); + x.insert(Word::from([0xAA_u32, 0xFF, 0xFF, 0xFF]), Word::from([0xAA_u32; 4])); + x.insert(Word::from([0xBB_u32, 0xFF, 0xFF, 0xFF]), Word::from([0xBB_u32; 4])); + x.insert(Word::from([0xCC_u32, 0xFF, 0xFF, 0xFF]), Word::from([0xCC_u32; 4])); + + let proof = x.open(&Word::from([10_u32, 11, 12, 13])); + + let mut orig = PartialSmt::new(); + orig.add_proof(proof); + let orig = orig; + + let proto = proto::primitives::PartialSmt::from(orig.clone()); + let recovered = PartialSmt::try_from(proto).unwrap(); + + assert_eq!(orig, recovered); + } +} diff --git a/crates/proto/src/generated/primitives.rs b/crates/proto/src/generated/primitives.rs index 39cb70ae2..6351ff434 100644 --- a/crates/proto/src/generated/primitives.rs +++ b/crates/proto/src/generated/primitives.rs @@ -1,4 +1,39 @@ // This file is @generated by prost-build. +/// Representation of a partial sparse merkle tree. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PartialSmt { + /// The sparse merkle tree root. + #[prost(message, optional, tag = "1")] + pub root: ::core::option::Option, + /// Set of leaves of the merkle tree. + #[prost(message, repeated, tag = "2")] + pub leaves: ::prost::alloc::vec::Vec, + /// Unique set of inner merkle tree digest, all belonging to at least one + /// merkle path of a given leave. Note that we skip all inner nodes that do + /// have two children, since we can recalculate them on-the-fly. + #[prost(message, repeated, tag = "3")] + pub nodes: ::prost::alloc::vec::Vec, +} +/// Node index representation. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct NodeIndex { + /// The depth of the index, starting from 0 as root. + #[prost(uint32, tag = "1")] + pub depth: u32, + /// The index within a certain tree depth, left-most being zero. + #[prost(uint64, tag = "2")] + pub value: u64, +} +/// Inner node of a sparse merkle tree +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct InnerNode { + /// The position of the inner node within the tree. + #[prost(message, optional, tag = "1")] + pub index: ::core::option::Option, + /// The digest of the subtree down to the root. + #[prost(message, optional, tag = "2")] + pub digest: ::core::option::Option, +} /// Represents a single SMT leaf entry. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct SmtLeafEntry { diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index f39545395..d9cd260aa 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -99,23 +99,25 @@ pub mod account_proofs { /// the current one. #[prost(bytes = "vec", optional, tag = "3")] pub account_code: ::core::option::Option<::prost::alloc::vec::Vec>, - /// Storage slots information for this account - #[prost(message, repeated, tag = "4")] - pub storage_maps: ::prost::alloc::vec::Vec< - account_state_header::StorageSlotMapProof, + /// A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries. + #[prost(message, repeated, tag = "5")] + pub partial_storage_smts: ::prost::alloc::vec::Vec< + account_state_header::StorageSlotMapPartialSmt, >, } /// Nested message and enum types in `AccountStateHeader`. pub mod account_state_header { /// Represents a single storage slot with the requested keys and their respective values. #[derive(Clone, PartialEq, ::prost::Message)] - pub struct StorageSlotMapProof { + pub struct StorageSlotMapPartialSmt { /// The storage slot index (\[0..255\]). #[prost(uint32, tag = "1")] pub storage_slot: u32, - /// Merkle proof of the map value - #[prost(bytes = "vec", tag = "2")] - pub smt_proof: ::prost::alloc::vec::Vec, + /// Merkle proofs of the map value as partial sparse merkle tree for compression. + #[prost(message, optional, tag = "2")] + pub partial_smt: ::core::option::Option< + super::super::super::super::primitives::PartialSmt, + >, } } } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 2255b5a3b..7d14bf9dc 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -35,7 +35,9 @@ use miden_objects::crypto::merkle::{ MmrDelta, MmrPeaks, MmrProof, + NodeIndex, PartialMmr, + PartialSmt, SmtProof, }; use miden_objects::note::{NoteDetails, NoteId, Nullifier}; @@ -900,8 +902,7 @@ impl State { .expect("retrieved accounts were validated against request"); if let Some(details) = &account_info.details { - let mut storage_slot_map_keys = Vec::new(); - + let mut partials = BTreeMap::::default(); for StorageMapKeysProof { storage_index, storage_keys } in &request.storage_requests { @@ -910,30 +911,24 @@ impl State { { for map_key in storage_keys { let proof = storage_map.open(map_key); - - let slot_map_key = proto::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapProof { - storage_slot: u32::from(*storage_index), - smt_proof: proof.to_bytes(), - }; - storage_slot_map_keys.push(slot_map_key); + partials + .entry(*storage_index) + .or_insert_with(PartialSmt::new) + .add_proof(proof)?; } } else { return Err(AccountError::StorageSlotNotMap(*storage_index).into()); } } - - // Only include unknown account codes - let account_code = known_code_commitments - .contains(&details.code().commitment()) - .not() - .then(|| details.code().to_bytes()); - let state_header = proto::rpc_store::account_proofs::account_proof::AccountStateHeader { header: Some(AccountHeader::from(details).into()), storage_header: details.storage().to_header().to_bytes(), account_code, - storage_maps: storage_slot_map_keys, + partial_storage_smts: Vec::from_iter(partials.into_iter().map(|(slot, partial_smt)| proto::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapPartialSmt { + storage_slot: u32::from(slot), + partial_smt: Some(proto::primitives::PartialSmt::from(partial_smt)), + })), }; headers_map.insert(account_info.summary.account_id, state_header); diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index 48d6e819b..d19365a3f 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -56,7 +56,7 @@ service Rpc { // The response includes each note's metadata and inclusion proof. // // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the - // tip of the chain. + // tip of the chain. rpc SyncNotes(SyncNotesRequest) returns (SyncNotesResponse) {} // Returns info which can be used by the client to sync up to the latest state of the chain @@ -137,12 +137,12 @@ message AccountProofs { // State header for public accounts. message AccountStateHeader { // Represents a single storage slot with the requested keys and their respective values. - message StorageSlotMapProof { + message StorageSlotMapPartialSmt { // The storage slot index ([0..255]). uint32 storage_slot = 1; - // Merkle proof of the map value - bytes smt_proof = 2; + // Merkle proofs of the map value as partial sparse merkle tree for compression. + primitives.PartialSmt partial_smt = 2; } // Account header. @@ -155,8 +155,8 @@ message AccountProofs { // the current one. optional bytes account_code = 3; - // Storage slots information for this account - repeated StorageSlotMapProof storage_maps = 4; + // A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries. + repeated StorageSlotMapPartialSmt partial_storage_smts = 5; } // The account witness for the current state commitment of one account ID. diff --git a/proto/proto/types/primitives.proto b/proto/proto/types/primitives.proto index 28a424981..750e8fb02 100644 --- a/proto/proto/types/primitives.proto +++ b/proto/proto/types/primitives.proto @@ -4,6 +4,34 @@ package primitives; // SMT // ================================================================================================ +// Representation of a partial sparse merkle tree. +message PartialSmt { + // The sparse merkle tree root. + Digest root = 1; + // Set of leaves of the merkle tree. + repeated SmtLeaf leaves = 2; + // Unique set of inner merkle tree digest, all belonging to at least one + // merkle path of a given leave. Note that we skip all inner nodes that do + // have two children, since we can recalculate them on-the-fly. + repeated InnerNode nodes = 3; +} + +// Node index representation. +message NodeIndex { + // The depth of the index, starting from 0 as root. + uint32 depth = 1; + // The index within a certain tree depth, left-most being zero. + uint64 value = 2; +} + +// Inner node of a sparse merkle tree +message InnerNode { + // The position of the inner node within the tree. + NodeIndex index = 1; + // The digest of the subtree down to the root. + Digest digest = 2; +} + // Represents a single SMT leaf entry. message SmtLeafEntry { // The key of the entry.