From 3265406fa87fb0a524f76545fda3d3e9fbac33f8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 4 Dec 2025 17:11:36 +0100 Subject: [PATCH 01/14] add partial storage maps --- Cargo.lock | 8 +- crates/proto/src/domain/account.rs | 169 ++++++-- crates/proto/src/generated/primitives.rs | 31 ++ crates/store/src/inner_forest.rs | 75 ++++ proto/proto/store/rpc.proto | 527 +++++++++++++++++++++++ proto/proto/types/primitives.proto | 30 ++ 6 files changed, 803 insertions(+), 37 deletions(-) create mode 100644 proto/proto/store/rpc.proto diff --git a/Cargo.lock b/Cargo.lock index 38063686b..f995c0e79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5112,9 +5112,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "siphasher" @@ -5860,9 +5860,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags 2.10.0", "bytes", diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 60c840154..22d554da4 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -15,7 +15,7 @@ use miden_objects::account::{ use miden_objects::asset::{Asset, AssetVault}; use miden_objects::block::BlockNumber; use miden_objects::block::account_tree::AccountWitness; -use miden_objects::crypto::merkle::SparseMerklePath; +use miden_objects::crypto::merkle::{MerkleError, SmtForest, SmtProof, SparseMerklePath}; use miden_objects::note::{NoteExecutionMode, NoteTag}; use miden_objects::utils::{Deserializable, DeserializationError, Serializable}; use thiserror::Error; @@ -193,6 +193,8 @@ impl TryFrom fn try_from( value: proto::rpc::account_storage_details::AccountStorageMapDetails, ) -> Result { + use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry; + let proto::rpc::account_storage_details::AccountStorageMapDetails { slot_name, too_many_entries, @@ -206,24 +208,20 @@ impl TryFrom } else { let map_entries = if let Some(entries) = entries { entries - .entries -.into_iter() -.map(|entry| { -let key = entry -.key - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(key), - ))? - .try_into()?; -let value = entry -.value - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(value), - ))? - .try_into()?; - Ok((key, value)) - }) - .collect::, ConversionError>>()? + .entries + .into_iter() + .map(|entry| { + let key = entry + .key + .ok_or(StorageMapEntry::missing_field(stringify!(key)))? + .try_into()?; + let value = entry + .value + .ok_or(StorageMapEntry::missing_field(stringify!(value)))? + .try_into()?; + Ok((key, value)) + }) + .collect::, ConversionError>>()? } else { Vec::new() }; @@ -260,6 +258,7 @@ impl TryFrom), + + /// Specific entries with their Merkle proofs for partial responses. + EntriesWithProofs(Vec), +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStorageMapDetails { pub slot_name: StorageSlotName, @@ -470,42 +478,138 @@ impl AccountStorageMapDetails { /// Maximum number of storage map entries that can be returned in a single response. pub const MAX_RETURN_ENTRIES: usize = 1000; + /// Creates storage map details with all entries from the storage map. + /// + /// If the storage map has too many entries (> `MAX_RETURN_ENTRIES`), + /// returns `LimitExceeded` variant. + pub fn from_all_entries(slot_name: StorageSlotName, storage_map: &StorageMap) -> Self { + if storage_map.num_entries() > Self::MAX_RETURN_ENTRIES { + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } + } else { + let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); + Self { + slot_name, + entries: StorageMapEntries::Entries(map_entries), + } + } + } + + /// Creates storage map details based on the requested slot data. + /// + /// This method handles both "all entries" and "specific keys" requests: + /// - For `SlotData::All`: Returns all entries from the storage map + /// - For `SlotData::MapKeys`: Returns only the requested keys with their values + /// + /// # Arguments + /// + /// * `slot_name` - The name of the storage slot + /// * `slot_data` - The type of data requested (all or specific keys) + /// * `storage_map` - The storage map to query + /// + /// # Returns + /// + /// Storage map details containing the requested entries or `LimitExceeded` if too many. pub fn new(slot_name: StorageSlotName, slot_data: SlotData, storage_map: &StorageMap) -> Self { match slot_data { SlotData::All => Self::from_all_entries(slot_name, storage_map), - SlotData::MapKeys(keys) => Self::from_specific_keys(slot_name, &keys[..], storage_map), + SlotData::MapKeys(keys) => { + if keys.len() > Self::MAX_RETURN_ENTRIES { + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } + } else { + // Query specific keys from the storage map + let mut entries = Vec::with_capacity(keys.len()); + for key in keys { + let value = storage_map.get(&key).copied().unwrap_or(miden_objects::EMPTY_WORD); + entries.push((key, value)); + } + Self { + slot_name, + entries: StorageMapEntries::Entries(entries), + } + } + }, } } - fn from_all_entries(slot_name: StorageSlotName, storage_map: &StorageMap) -> Self { - if storage_map.num_entries() > Self::MAX_RETURN_ENTRIES { + /// Creates storage map details from entries queried from storage forest with proofs. + /// + /// This method should be used when specific keys are requested and we want to include + /// Merkle proofs for verification. It avoids loading the entire storage map from the database. + /// + /// # Arguments + /// + /// * `slot_name` - The name of the storage slot + /// * `entries` - Key-value pairs with their Merkle proofs from the storage forest + /// + /// # Returns + /// + /// Storage map details containing the requested entries or `LimitExceeded` if too many keys. + pub fn from_forest_entries(slot_name: StorageSlotName, entries: Vec<(Word, Word)>) -> Self { + if entries.len() > Self::MAX_RETURN_ENTRIES { Self { slot_name, entries: StorageMapEntries::LimitExceeded, } } else { - let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); Self { slot_name, - entries: StorageMapEntries::Entries(map_entries), + entries: StorageMapEntries::Entries(entries), } } } - fn from_specific_keys( + /// Creates storage map details with SMT proofs for specific keys using the storage forest. + /// + /// This method queries the forest for specific keys and extracts key-value pairs from + /// the SMT proofs. The forest must be available and contain the data for the specified + /// SMT root. + /// + /// # Arguments + /// + /// * `slot_name` - The name of the storage slot + /// * `keys` - The keys to query + /// * `storage_forest` - The SMT forest containing the storage data + /// * `smt_root` - The root of the SMT for this storage slot + /// + /// # Returns + /// + /// Storage map details containing the requested entries or `LimitExceeded` if too many keys. + /// + /// # Errors + /// + /// Returns `MerkleError` if the forest doesn't contain sufficient data to provide proofs. + pub fn from_specific_keys( slot_name: StorageSlotName, keys: &[Word], - storage_map: &StorageMap, - ) -> Self { + storage_forest: &SmtForest, + smt_root: Word, + ) -> Result { if keys.len() > Self::MAX_RETURN_ENTRIES { - Self { + return Ok(Self { slot_name, entries: StorageMapEntries::LimitExceeded, - } - } else { - // TODO For now, we return all entries instead of specific keys with proofs - Self::from_all_entries(slot_name, storage_map) + }); } + + // Collect key-value pairs by opening proofs for each key + let mut entries = Vec::with_capacity(keys.len()); + + for key in keys { + let proof = storage_forest.open(smt_root, *key)?; + let value = proof.get(key).unwrap_or(miden_objects::EMPTY_WORD); + entries.push((*key, value)); + } + + Ok(Self { + slot_name, + entries: StorageMapEntries::Entries(entries), + }) } } @@ -701,7 +805,6 @@ impl From } } } - // ACCOUNT WITNESS // ================================================================================================ diff --git a/crates/proto/src/generated/primitives.rs b/crates/proto/src/generated/primitives.rs index 907ef856a..e11017730 100644 --- a/crates/proto/src/generated/primitives.rs +++ b/crates/proto/src/generated/primitives.rs @@ -96,3 +96,34 @@ pub struct Digest { #[prost(fixed64, tag = "4")] pub d3: u64, } +/// Represents a partial Sparse Merkle Tree containing only a subset of leaves and their paths. +/// This allows verifying and updating tracked keys without requiring the full tree. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PartialSmt { + /// The root hash of the SMT + #[prost(message, optional, tag = "1")] + pub root: ::core::option::Option, + /// All tracked leaves in the partial SMT, keyed by their leaf index + #[prost(message, repeated, tag = "2")] + pub leaves: ::prost::alloc::vec::Vec, + /// Inner nodes stored in deterministic order (by scalar index) for reconstruction + #[prost(message, repeated, tag = "3")] + pub inner_nodes: ::prost::alloc::vec::Vec, +} +/// Represents a leaf with its index for partial SMT serialization +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SmtLeafWithIndex { + /// The leaf index (0 to 2^64 - 1 for leaves at depth 64) + #[prost(uint64, tag = "1")] + pub leaf_index: u64, + /// The leaf data + #[prost(message, optional, tag = "2")] + pub leaf: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct InnerNode { + #[prost(message, optional, tag = "1")] + pub left: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub right: ::core::option::Option, +} diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 0e53e2c32..883704214 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -261,4 +261,79 @@ impl InnerForest { "Populated vault in forest from DB" ); } + + /// Queries specific storage keys for a given account and slot at a specific block. + /// + /// This method retrieves key-value pairs from the forest without loading the entire + /// storage map from the database. It returns the values along with their Merkle proofs. + /// + /// # Arguments + /// + /// * `account_id` - The account to query + /// * `slot_name` - The storage slot name + /// * `block_num` - The block number at which to query + /// * `keys` - The keys to retrieve + /// + /// # Returns + /// + /// A vector of key-value pairs for the requested keys. Keys that don't exist in the + /// storage map will have a value of `EMPTY_WORD`. + /// + /// # Errors + /// + /// Returns an error if: + /// - The storage root for this account/slot/block is not tracked + /// - The forest doesn't have sufficient data to provide proofs for the keys + pub(crate) fn query_storage_keys( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: BlockNumber, + keys: &[Word], + ) -> Result, String> { + // Get the storage root for this account/slot/block + let root = self + .storage_roots + .get(&(account_id, slot_name.clone(), block_num)) + .copied() + .ok_or_else(|| { + format!( + "Storage root not found for account {:?}, slot {}, block {}", + account_id, slot_name, block_num + ) + })?; + + let mut results = Vec::with_capacity(keys.len()); + + for key in keys { + // Open a proof for this key in the forest + match self.storage_forest.open(root, *key) { + Ok(proof) => { + // Extract the value from the proof + let value = proof.get(key).unwrap_or(EMPTY_WORD); + results.push((*key, value)); + }, + Err(e) => { + tracing::debug!( + target: crate::COMPONENT, + "Failed to open proof for key in storage forest: {}. Using empty value.", + e + ); + // Return empty value for keys that can't be proven + results.push((*key, EMPTY_WORD)); + }, + } + } + + tracing::debug!( + target: crate::COMPONENT, + "Queried {} storage keys from forest for account {:?}, slot {} at block {}", + results.len(), + account_id, + slot_name, + block_num + ); + + Ok(results) + } } diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto new file mode 100644 index 000000000..f2fbf0d7c --- /dev/null +++ b/proto/proto/store/rpc.proto @@ -0,0 +1,527 @@ +// Specification of the store RPC. +// +// This provided access to the blockchain data to the other nodes. +syntax = "proto3"; +package rpc_store; + +import "google/protobuf/empty.proto"; +import "types/account.proto"; +import "types/blockchain.proto"; +import "types/transaction.proto"; +import "types/note.proto"; +import "types/primitives.proto"; +import "store/shared.proto"; + +// RPC STORE API +// ================================================================================================ + +// Store API for the RPC component +service Rpc { + // Returns the status info. + rpc Status(google.protobuf.Empty) returns (StoreStatus) {} + + // Returns a nullifier proof for each of the requested nullifiers. + rpc CheckNullifiers(NullifierList) returns (CheckNullifiersResponse) {} + + // Returns the latest state of an account with the specified ID. + rpc GetAccountDetails(account.AccountId) returns (account.AccountDetails) {} + + // Returns the latest state proof of the specified account. + rpc GetAccountProof(AccountProofRequest) returns (AccountProofResponse) {} + + // Returns raw block data for the specified block number. + rpc GetBlockByNumber(blockchain.BlockNumber) returns (blockchain.MaybeBlock) {} + + // Retrieves block header by given block number. Optionally, it also returns the MMR path + // and current chain length to authenticate the block's inclusion. + rpc GetBlockHeaderByNumber(shared.BlockHeaderByNumberRequest) returns (shared.BlockHeaderByNumberResponse) {} + + // Returns a list of committed notes matching the provided note IDs. + rpc GetNotesById(note.NoteIdList) returns (note.CommittedNoteList) {} + + // Returns the script for a note by its root. + rpc GetNoteScriptByRoot(note.NoteRoot) returns (shared.MaybeNoteScript) {} + + // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. + // + // Note that only 16-bit prefixes are supported at this time. + rpc SyncNullifiers(SyncNullifiersRequest) returns (SyncNullifiersResponse) {} + + // Returns info which can be used by the requester to sync up to the tip of chain for the notes they are interested in. + // + // requester specifies the `note_tags` they are interested in, and the block height from which to search for new for + // matching notes for. The request will then return the next block containing any note matching the provided tags. + // + // The response includes each note's metadata and inclusion proof. + // + // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the + // tip of the chain. + rpc SyncNotes(SyncNotesRequest) returns (SyncNotesResponse) {} + + // Returns info which can be used by the requester to sync up to the latest state of the chain + // for the objects (accounts, notes, nullifiers) the requester is interested in. + // + // This request returns the next block containing requested data. It also returns `chain_tip` + // which is the latest block number in the chain. requester is expected to repeat these requests + // in a loop until `response.block_header.block_num == response.chain_tip`, at which point + // the requester is fully synchronized with the chain. + // + // Each request also returns info about new notes, nullifiers etc. created. It also returns + // Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain + // MMR peaks and chain MMR nodes. + // + // For preserving some degree of privacy, note tags and nullifiers filters contain only high + // part of hashes. Thus, returned data contains excessive notes and nullifiers, requester can make + // additional filtering of that data on its side. + rpc SyncState(SyncStateRequest) returns (SyncStateResponse) {} + + // Returns account vault updates for specified account within a block range. + rpc SyncAccountVault(SyncAccountVaultRequest) returns (SyncAccountVaultResponse) {} + + // Returns storage map updates for specified account and storage slots within a block range. + rpc SyncStorageMaps(SyncStorageMapsRequest) returns (SyncStorageMapsResponse) {} + + // Returns transactions records for specific accounts within a block range. + rpc SyncTransactions(SyncTransactionsRequest) returns (SyncTransactionsResponse) {} +} + +// STORE STATUS +// ================================================================================================ + +// Represents the status of the store. +message StoreStatus { + // The store's running version. + string version = 1; + + // The store's status. + string status = 2; + + // Number of the latest block in the chain. + fixed32 chain_tip = 3; +} + +// GET ACCOUNT PROOF +// ================================================================================================ + +// Returns the latest state proof of the specified account. +message AccountProofRequest { + // Request the details for a public account. + message AccountDetailRequest { + // Represents a storage slot index and the associated map keys. + message StorageMapDetailRequest { + // Indirection required for use in `oneof {..}` block. + message MapKeys { + // A list of map keys associated with this storage slot. + repeated primitives.Digest map_keys = 1; + } + // Storage slot index (`[0..255]`). + uint32 slot_index = 1; + + oneof slot_data { + // Request to return all storage map data. If the number exceeds a threshold of 1000 entries, + // the response will not contain them but must be requested separately. + bool all_entries = 2; + + // A list of map keys associated with the given storage slot identified by `slot_index`. + MapKeys map_keys = 3; + } + } + + // Last known code commitment to the requester. The response will include account code + // only if its commitment is different from this value. + // + // If the field is ommiteed, the response will not include the account code. + optional primitives.Digest code_commitment = 1; + + // Last known asset vault commitment to the requester. The response will include asset vault data + // only if its commitment is different from this value. If the value is not present in the + // request, the response will not contain one either. + // If the number of to-be-returned asset entries exceed a threshold, they have to be requested + // separately, which is signaled in the response message with dedicated flag. + optional primitives.Digest asset_vault_commitment = 2; + + // Additional request per storage map. + repeated StorageMapDetailRequest storage_maps = 3; + } + + // ID of the account for which we want to get data + account.AccountId account_id = 1; + + // Optional block height at which to return the proof. + // + // Defaults to current chain tip if unspecified. + optional blockchain.BlockNumber block_num = 2; + + // Request for additional account details; valid only for public accounts. + optional AccountDetailRequest details = 3; +} + +// Represents the result of getting account proof. +message AccountProofResponse { + + message AccountDetails { + // Account header. + account.AccountHeader header = 1; + + // Account storage data + AccountStorageDetails storage_details = 2; + + // Account code; empty if code commitments matched or none was requested. + optional bytes code = 3; + + // Account asset vault data; empty if vault commitments matched or the requester + // omitted it in the request. + optional AccountVaultDetails vault_details = 4; + } + + // The block number at which the account witness was created and the account details were observed. + blockchain.BlockNumber block_num = 1; + + // Account ID, current state commitment, and SMT path. + account.AccountWitness witness = 2; + + // Additional details for public accounts. + optional AccountDetails details = 3; +} + +// Account vault details for AccountProofResponse +message AccountVaultDetails { + // A flag that is set to true if the account contains too many assets. This indicates + // to the user that `SyncAccountVault` endpoint should be used to retrieve the + // account's assets + bool too_many_assets = 1; + + // When too_many_assets == false, this will contain the list of assets in the + // account's vault + repeated primitives.Asset assets = 2; +} + +// Represents a set of SMT proofs (openings) for requested keys +message SmtProofSet { + // The root hash of the SMT these proofs are for + primitives.Digest root = 1; + + // Collection of SMT proofs/openings for the requested keys + repeated primitives.SmtOpening proofs = 2; +} + +// Account storage details for AccountProofResponse +message AccountStorageDetails { + message AccountStorageMapDetails { + // Wrapper for repeated storage map entries + message MapEntries { + // Definition of individual storage entries. + message StorageMapEntry { + primitives.Digest key = 1; + primitives.Digest value = 2; + } + + repeated StorageMapEntry entries = 1; + } + // slot index of the storage map + uint32 slot_index = 1; + + // A flag that is set to `true` if the number of to-be-returned entries in the + // storage map would exceed a threshold. This indicates to the user that `SyncStorageMaps` + // endpoint should be used to get all storage map data. + bool too_many_entries = 2; + + oneof data { + // By default we provide all storage entries when `all_entries` is requested + // or when the storage map is small. + MapEntries entries = 3; + + // When specific keys are requested and the storage map is not small, + // we provide a set of SMT proofs (openings) for the requested keys. + // This allows the receiver to reconstruct the partial tree or validate individual proofs. + SmtProofSet smt_proofs = 4; + } + } + + // Account storage header (storage slot info for up to 256 slots) + account.AccountStorageHeader header = 1; + + // Additional data for the requested storage maps + repeated AccountStorageMapDetails map_details = 2; +} + + +// CHECK NULLIFIERS +// ================================================================================================ + +// List of nullifiers to return proofs for. +message NullifierList { + // List of nullifiers to return proofs for. + repeated primitives.Digest nullifiers = 1; +} + +// Represents the result of checking nullifiers. +message CheckNullifiersResponse { + // Each requested nullifier has its corresponding nullifier proof at the same position. + repeated primitives.SmtOpening proofs = 1; +} + +// SYNC NULLIFIERS +// ================================================================================================ + +// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. +message SyncNullifiersRequest { + // Block number from which the nullifiers are requested (inclusive). + BlockRange block_range = 1; + + // Number of bits used for nullifier prefix. Currently the only supported value is 16. + uint32 prefix_len = 2; + + // List of nullifiers to check. Each nullifier is specified by its prefix with length equal + // to `prefix_len`. + repeated uint32 nullifiers = 3; +} + +// Represents the result of syncing nullifiers. +message SyncNullifiersResponse { + // Represents a single nullifier update. + message NullifierUpdate { + // Nullifier ID. + primitives.Digest nullifier = 1; + + // Block number. + fixed32 block_num = 2; + } + + // Pagination information. + PaginationInfo pagination_info = 1; + + // List of nullifiers matching the prefixes specified in the request. + repeated NullifierUpdate nullifiers = 2; +} + +// SYNC STATE +// ================================================================================================ + +// State synchronization request. +// +// Specifies state updates the requester is interested in. The server will return the first block which +// contains a note matching `note_tags` or the chain tip. And the corresponding updates to +// `account_ids` for that block range. +message SyncStateRequest { + // Last block known by the requester. The response will contain data starting from the next block, + // until the first block which contains a note of matching the requested tag, or the chain tip + // if there are no notes. + fixed32 block_num = 1; + + // Accounts' commitment to include in the response. + // + // An account commitment will be included if-and-only-if it is the latest update. Meaning it is + // possible there was an update to the account for the given range, but if it is not the latest, + // it won't be included in the response. + repeated account.AccountId account_ids = 2; + + // Specifies the tags which the requester is interested in. + repeated fixed32 note_tags = 3; +} + +// Represents the result of syncing state request. +message SyncStateResponse { + // Number of the latest block in the chain. + fixed32 chain_tip = 1; + + // Block header of the block with the first note matching the specified criteria. + blockchain.BlockHeader block_header = 2; + + // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. + primitives.MmrDelta mmr_delta = 3; + + // List of account commitments updated after `request.block_num + 1` but not after `response.block_header.block_num`. + repeated account.AccountSummary accounts = 5; + + // List of transactions executed against requested accounts between `request.block_num + 1` and + // `response.block_header.block_num`. + repeated transaction.TransactionSummary transactions = 6; + + // List of all notes together with the Merkle paths from `response.block_header.note_root`. + repeated note.NoteSyncRecord notes = 7; +} + +// SYNC ACCOUNT VAULT +// ================================================================================================ + +// Account vault synchronization request. +// +// Allows requesters to sync asset values for specific public accounts within a block range. +message SyncAccountVaultRequest { + // Block range from which to start synchronizing. + // + // If the `block_to` is specified, this block must be close to the chain tip (i.e., within 30 blocks), + // otherwise an error will be returned. + BlockRange block_range = 1; + + // Account for which we want to sync asset vault. + account.AccountId account_id = 2; +} + +message SyncAccountVaultResponse { + // Pagination information. + PaginationInfo pagination_info = 1; + + // List of asset updates for the account. + // + // Multiple updates can be returned for a single asset, and the one with a higher `block_num` + // is expected to be retained by the caller. + repeated AccountVaultUpdate updates = 2; +} + +message AccountVaultUpdate { + // Vault key associated with the asset. + primitives.Digest vault_key = 1; + + // Asset value related to the vault key. + // If not present, the asset was removed from the vault. + optional primitives.Asset asset = 2; + + // Block number at which the above asset was updated in the account vault. + fixed32 block_num = 3; +} + +// SYNC NOTES +// ================================================================================================ + +// Note synchronization request. +// +// Specifies note tags that requester is interested in. The server will return the first block which +// contains a note matching `note_tags` or the chain tip. +message SyncNotesRequest { + // Block range from which to start synchronizing. + BlockRange block_range = 1; + + // Specifies the tags which the requester is interested in. + repeated fixed32 note_tags = 2; +} + +// Represents the result of syncing notes request. +message SyncNotesResponse { + // Pagination information. + PaginationInfo pagination_info = 1; + + // Block header of the block with the first note matching the specified criteria. + blockchain.BlockHeader block_header = 2; + + // Merkle path to verify the block's inclusion in the MMR at the returned `chain_tip`. + // + // An MMR proof can be constructed for the leaf of index `block_header.block_num` of + // an MMR of forest `chain_tip` with this path. + primitives.MerklePath mmr_path = 3; + + // List of all notes together with the Merkle paths from `response.block_header.note_root`. + repeated note.NoteSyncRecord notes = 4; +} + +// SYNC STORAGE MAP +// ================================================================================================ + +// Storage map synchronization request. +// +// Allows requesters to sync storage map values for specific public accounts within a block range, +// with support for cursor-based pagination to handle large storage maps. +message SyncStorageMapsRequest { + // Block range from which to start synchronizing. + // + // If the `block_to` is specified, this block must be close to the chain tip (i.e., within 30 blocks), + // otherwise an error will be returned. + BlockRange block_range = 1; + + // Account for which we want to sync storage maps. + account.AccountId account_id = 3; +} + +message SyncStorageMapsResponse { + // Pagination information. + PaginationInfo pagination_info = 1; + + // The list of storage map updates. + // + // Multiple updates can be returned for a single slot index and key combination, and the one + // with a higher `block_num` is expected to be retained by the caller. + repeated StorageMapUpdate updates = 2; +} + +// Represents a single storage map update. +message StorageMapUpdate { + // Block number in which the slot was updated. + fixed32 block_num = 1; + + // Slot index ([0..255]). + uint32 slot_index = 2; + + // The storage map key. + primitives.Digest key = 3; + + // The storage map value. + primitives.Digest value = 4; +} + +// BLOCK RANGE +// ================================================================================================ + +// Represents a block range. +message BlockRange { + // Block number from which to start (inclusive). + fixed32 block_from = 1; + + // Block number up to which to check (inclusive). If not specified, checks up to the latest block. + optional fixed32 block_to = 2; +} + +// PAGINATION INFO +// ================================================================================================ + +// Represents pagination information for chunked responses. +// +// Pagination is done using block numbers as the axis, allowing requesters to request +// data in chunks by specifying block ranges and continuing from where the previous +// response left off. +// +// To request the next chunk, the requester should use `block_num + 1` from the previous response +// as the `block_from` for the next request. +message PaginationInfo { + // Current chain tip + fixed32 chain_tip = 1; + + // The block number of the last check included in this response. + // + // For chunked responses, this may be less than `request.block_range.block_to`. + // If it is less than request.block_range.block_to, the user is expected to make a subsequent request + // starting from the next block to this one (ie, request.block_range.block_from = block_num + 1). + fixed32 block_num = 2; +} + +// SYNC TRANSACTIONS +// ================================================================================================ + +// Transactions synchronization request. +// +// Allows requesters to sync transactions for specific accounts within a block range. +message SyncTransactionsRequest { + // Block range from which to start synchronizing. + BlockRange block_range = 1; + + // Accounts to sync transactions for. + repeated account.AccountId account_ids = 2; +} + +// Represents the result of syncing transactions request. +message SyncTransactionsResponse { + // Pagination information. + PaginationInfo pagination_info = 1; + + // List of transaction records. + repeated TransactionRecord transactions = 2; +} + +// Represents a transaction record. +message TransactionRecord { + // Block number in which the transaction was included. + fixed32 block_num = 1; + + // A transaction header. + transaction.TransactionHeader header = 2; +} diff --git a/proto/proto/types/primitives.proto b/proto/proto/types/primitives.proto index aed31cec0..7e4951400 100644 --- a/proto/proto/types/primitives.proto +++ b/proto/proto/types/primitives.proto @@ -92,3 +92,33 @@ message Digest { fixed64 d2 = 3; fixed64 d3 = 4; } + +// PARTIAL SMT +// ================================================================================================ + +// Represents a partial Sparse Merkle Tree containing only a subset of leaves and their paths. +// This allows verifying and updating tracked keys without requiring the full tree. +message PartialSmt { + // The root hash of the SMT + Digest root = 1; + + // All tracked leaves in the partial SMT, keyed by their leaf index + repeated SmtLeafWithIndex leaves = 2; + + // Inner nodes stored in deterministic order (by scalar index) for reconstruction + repeated InnerNode inner_nodes = 3; +} + +// Represents a leaf with its index for partial SMT serialization +message SmtLeafWithIndex { + // The leaf index (0 to 2^64 - 1 for leaves at depth 64) + uint64 leaf_index = 1; + + // The leaf data + SmtLeaf leaf = 2; +} + +message InnerNode { + Digest left = 1; + Digest right = 2; +} From 15af845d068b283123a7db1d546d5396d36ed653 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 18:40:10 +0100 Subject: [PATCH 02/14] CI fixins --- CHANGELOG.md | 1 + crates/proto/src/domain/account.rs | 2 +- crates/store/src/state.rs | 229 +++++++++++++---------------- 3 files changed, 106 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f510a100..1b243f8a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Renamed card's names in the `miden-network-monitor` binary ([#1441](https://github.com/0xMiden/miden-node/pull/1441)). - Improved tracing in `miden-network-monitor` binary ([#1366](https://github.com/0xMiden/miden-node/pull/1366)). - Integrated RPC stack with Validator component for transaction validation ([#1457](https://github.com/0xMiden/miden-node/pull/1457)). +- Add partial storage map queries to RPC ([#1428](https://github.com/0xMiden/miden-node/pull/1428)). ### Changes diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 22d554da4..bc9adf8ed 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -525,7 +525,7 @@ impl AccountStorageMapDetails { // Query specific keys from the storage map let mut entries = Vec::with_capacity(keys.len()); for key in keys { - let value = storage_map.get(&key).copied().unwrap_or(miden_objects::EMPTY_WORD); + let value = storage_map.get(&key); entries.push((key, value)); } Self { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index e1ca0bb1e..b1716b4e7 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -18,16 +18,17 @@ use miden_node_proto::domain::account::{ AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, + SlotData, StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; -use miden_objects::account::delta::AccountUpdateDetails; -use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent, StorageSlotName}; +use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_objects::block::{ + BlockAccountUpdate, BlockHeader, BlockInputs, BlockNoteTree, @@ -180,7 +181,7 @@ impl State { let acc_account_ids = me.db.select_all_account_commitments().await?; let acc_account_ids = Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| account_id)); - me.initialize_storage_forest_from_db(acc_account_ids, latest_block_num) + me.update_storage_forest_from_db(acc_account_ids, latest_block_num) .await .map_err(|e| { StateInitializationError::DatabaseError(DatabaseError::InteractError(format!( @@ -394,14 +395,11 @@ impl State { // Signals the write lock has been acquired, and the transaction can be committed let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - // Extract account updates with deltas before block is moved into async task - // We'll use these deltas to update the SmtForest without DB roundtrips - let account_updates: Vec<_> = block - .body() - .updated_accounts() - .iter() - .map(|update| (update.account_id(), update.details().clone())) - .collect(); + // Extract account IDs before block is moved into async task + // We'll need these later to populate the SmtForest + let updated_account_ids = Vec::::from_iter( + block.body().updated_accounts().iter().map(BlockAccountUpdate::account_id), + ); // The DB and in-memory state updates need to be synchronized and are partially // overlapping. Namely, the DB transaction only proceeds after this task acquires the @@ -462,118 +460,111 @@ impl State { inner.blockchain.push(block_commitment); } - // After successful DB commit, update the SmtForest with account deltas - // This uses the deltas directly without DB roundtrips, which is more efficient - self.update_forest(account_updates, block_num).await?; + // After successful DB commit, query updated accounts' storage as well as vault data + // TODO look into making this consume the `account_tree_update` + self.update_storage_forest_from_db(updated_account_ids, block_num).await?; info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); Ok(()) } - /// Updates `SmtForest` with account deltas from a block + /// Updates `SmtForest` after a block is successfully applied /// - /// This method updates the forest directly using the deltas extracted from the block. + /// Must be called after the DB transaction commits successfully, so we can safely + /// query the newly committed storage data. /// - /// # Arguments + /// # Warning /// - /// * `account_updates` - Vector of (`AccountId`, `AccountUpdateDetails`) tuples from the block - /// * `block_num` - Block number for which these updates apply + /// Has internal locking to mutate the state, use cautiously in scopes with other + /// mutex guards around! /// /// # Note /// - /// - Private account updates are skipped as their state is not publicly visible. - /// - The number of changed accounts is implicitly bounded by the limited number of transactions - /// per block. - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_updates.len()))] - async fn update_forest( + /// The number of changed accounts is bounded by transactions per block. + async fn update_storage_forest_from_db( &self, - account_updates: Vec<(AccountId, AccountUpdateDetails)>, + changed_account_ids: Vec, block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - if account_updates.is_empty() { + if changed_account_ids.is_empty() { return Ok(()); } - let mut forest_guard = self.forest.write().await; + self.update_storage_maps_in_forest(&changed_account_ids, block_num).await?; - for (account_id, details) in account_updates { - match details { - AccountUpdateDetails::Delta(delta) => { - // Update the forest with vault and storage deltas - forest_guard.update_account( - block_num, - account_id, - delta.vault(), - delta.storage(), - ); - - tracing::debug!( - target: COMPONENT, - %account_id, - %block_num, - "Updated forest with account delta" - ); - }, - AccountUpdateDetails::Private => { - // Private accounts don't expose their state changes - tracing::trace!( - target: COMPONENT, - %account_id, - %block_num, - "Skipping private account update" - ); - }, - } - } + self.update_vaults_in_forest(&changed_account_ids, block_num).await?; Ok(()) } - /// Updates `SmtForest` from database state (DB-based) - /// - /// This method is used during initial `State::load()` where deltas are not available. - /// For block application, prefer `fn update_forest` which uses deltas directly. - /// - /// # Warning - /// - /// Has internal locking to mutate the state, use cautiously in scopes with other - /// mutex guards around! - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] - async fn initialize_storage_forest_from_db( + /// Updates storage map SMTs in the forest for changed accounts + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] + async fn update_storage_maps_in_forest( &self, - account_ids: Vec, + changed_account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - // Acquire write lock once for the entire initialization + // Step 1: Query storage from database + let account_storages = + self.query_account_storages_from_db(changed_account_ids, block_num).await?; + + // Step 2: Extract map slots and their entries using InnerForest helper + let map_slots_to_populate = InnerForest::extract_map_slots_from_storage(&account_storages); + + if map_slots_to_populate.is_empty() { + return Ok(()); + } + + // Step 3: Acquire write lock and update the forest with new SMTs let mut forest_guard = self.forest.write().await; + forest_guard.populate_storage_maps(map_slots_to_populate, block_num); - // Process each account, updating both storage maps and vaults - for account_id in account_ids { - // Query and update storage maps for this account + Ok(()) + } + + /// Queries account storage data from the database for the given accounts at a specific block + #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_ids.len()))] + async fn query_account_storages_from_db( + &self, + account_ids: &[AccountId], + block_num: BlockNumber, + ) -> Result, ApplyBlockError> { + let mut account_storages = Vec::with_capacity(account_ids.len()); + + for &account_id in account_ids { let storage = self.db.select_account_storage_at_block(account_id, block_num).await?; - let map_slots = extract_map_slots_from_storage(&storage); + account_storages.push((account_id, storage)); + } - if !map_slots.is_empty() { - forest_guard.add_storage_map(account_id, map_slots, block_num); - } + Ok(account_storages) + } - // Query and update vault for this account - let vault_entries = - self.db.select_account_vault_at_block(account_id, block_num).await?; + /// Updates vault SMTs in the forest for changed accounts + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] + async fn update_vaults_in_forest( + &self, + changed_account_ids: &[AccountId], + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + // Query vault assets for each updated account + let mut vault_entries_to_populate = Vec::new(); - if !vault_entries.is_empty() { - forest_guard.add_vault(account_id, &vault_entries, block_num); + for &account_id in changed_account_ids { + let entries = self.db.select_account_vault_at_block(account_id, block_num).await?; + if !entries.is_empty() { + vault_entries_to_populate.push((account_id, entries)); } + } - tracing::debug!( - target: COMPONENT, - %account_id, - %block_num, - "Initialized forest for account from DB" - ); + if vault_entries_to_populate.is_empty() { + return Ok(()); } + // Acquire write lock once for the entire update operation and delegate to InnerForest + let mut forest_guard = self.forest.write().await; + forest_guard.populate_vaults(vault_entries_to_populate, block_num); + Ok(()) } @@ -1209,23 +1200,39 @@ impl State { let mut storage_map_details = Vec::::with_capacity(storage_requests.len()); - for StorageMapRequest { slot_name, slot_data } in storage_requests { - let Some(slot) = store.slots().iter().find(|s| s.name() == &slot_name) else { - continue; - }; + // Acquire forest lock for querying specific keys + let forest = self.forest.read().await; - let storage_map = match slot.content() { - StorageSlotContent::Map(map) => map, - StorageSlotContent::Value(_) => { - // TODO: what to do with value entries? Is it ok to ignore them? - return Err(AccountError::StorageSlotNotMap(slot_name).into()); + for StorageMapRequest { slot_name, slot_data } in storage_requests { + let details = match &slot_data { + SlotData::MapKeys(keys) => { + // Query the forest for specific keys + let entries = forest + .query_storage_keys(account_id, &slot_name, block_num, keys) + .map_err(DatabaseError::InteractError)?; + AccountStorageMapDetails::from_forest_entries(slot_name, entries) + }, + SlotData::All => { + // For all entries, load from storage map + let Some(slot) = store.slots().iter().find(|s| s.name() == &slot_name) else { + continue; + }; + let storage_map = match slot.content() { + StorageSlotContent::Map(map) => map, + StorageSlotContent::Value(_) => { + return Err(AccountError::StorageSlotNotMap(slot_name).into()); + }, + }; + AccountStorageMapDetails::from_all_entries(slot_name, storage_map) }, }; - let details = AccountStorageMapDetails::new(slot_name, slot_data, storage_map); storage_map_details.push(details); } + // Release forest lock + drop(forest); + Ok(AccountDetails { account_header, account_code, @@ -1397,31 +1404,3 @@ async fn load_account_tree( Ok(AccountTreeWithHistory::new(account_tree, block_number)) } - -// HELPERS -// ================================================================================================= - -/// Extract storage map slots from a single `AccountStorage` object. -/// -/// # Returns -/// -/// Vector of `(account_id, slot_name, entries)` tuples ready for forest population. -pub(crate) fn extract_map_slots_from_storage( - storage: &miden_objects::account::AccountStorage, -) -> Vec<(StorageSlotName, Vec<(Word, Word)>)> { - use miden_objects::account::StorageSlotContent; - - let mut map_slots = Vec::new(); - - for slot in storage.slots() { - if let StorageSlotContent::Map(map) = slot.content() { - let entries = Vec::from_iter(map.entries().map(|(k, v)| (*k, *v))); - - if !entries.is_empty() { - map_slots.push((slot.name().clone(), entries)); - } - } - } - - map_slots -} From 617c033fcdb4e63ede18e64e05785296ee2baa3e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 20:24:12 +0100 Subject: [PATCH 03/14] yay --- crates/store/src/inner_forest.rs | 163 +------------------------ crates/store/src/inner_forest/tests.rs | 54 ++------ crates/store/src/state.rs | 105 +++++++++------- 3 files changed, 73 insertions(+), 249 deletions(-) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 883704214..d119375f4 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -1,8 +1,6 @@ use std::collections::BTreeMap; -use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; -use miden_objects::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; -use miden_objects::asset::{Asset, FungibleAsset}; +use miden_objects::account::{AccountId, StorageSlotName}; use miden_objects::block::BlockNumber; use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH, SmtForest}; use miden_objects::{EMPTY_WORD, Word}; @@ -43,155 +41,6 @@ impl InnerForest { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } - /// Updates the forest with account vault and storage changes from a delta. - /// - /// This is the unified interface for updating all account state in the forest. - /// It processes both vault and storage map deltas and updates the forest accordingly. - /// - /// # Arguments - /// - /// * `block_num` - Block number for which these changes are being applied - /// * `account_id` - The account being updated - /// * `vault_delta` - Changes to the account's asset vault - /// * `storage_delta` - Changes to the account's storage maps - pub(crate) fn update_account( - &mut self, - block_num: BlockNumber, - account_id: AccountId, - vault_delta: &AccountVaultDelta, - storage_delta: &AccountStorageDelta, - ) { - // Update vault if there are any changes - if !vault_delta.is_empty() { - self.update_account_vault(block_num, account_id, vault_delta); - } - - // Update storage maps if there are any changes - if !storage_delta.is_empty() { - self.update_account_storage(block_num, account_id, storage_delta); - } - } - - /// Updates the forest with vault changes from a delta. - /// - /// Processes both fungible and non-fungible asset changes, building entries - /// for the vault SMT and tracking the new root. - /// - /// # Arguments - /// - /// * `block_num` - Block number for this update - /// * `account_id` - The account being updated - /// * `vault_delta` - Changes to the account's asset vault - fn update_account_vault( - &mut self, - block_num: BlockNumber, - account_id: AccountId, - vault_delta: &AccountVaultDelta, - ) { - let prev_block_num = block_num.parent().unwrap_or_default(); - let prev_root = self - .vault_roots - .get(&(account_id, prev_block_num)) - .copied() - .unwrap_or_else(Self::empty_smt_root); - - // Collect all vault entry updates - let mut entries = Vec::new(); - - // Process fungible assets - these require special handling to get current amounts - // Note: We rely on the delta containing the updated amounts, not just the changes - for (faucet_id, amount) in vault_delta.fungible().iter() { - let amount_u64 = (*amount).try_into().expect("Amount should be non-negative"); - let asset: Asset = FungibleAsset::new(*faucet_id, amount_u64) - .expect("Valid fungible asset from delta") - .into(); - entries.push((asset.vault_key().into(), Word::from(asset))); - } - - // Process non-fungible assets - for (asset, action) in vault_delta.non_fungible().iter() { - match action { - NonFungibleDeltaAction::Add => { - entries - .push((asset.vault_key().into(), Word::from(Asset::NonFungible(*asset)))); - }, - NonFungibleDeltaAction::Remove => { - entries.push((asset.vault_key().into(), EMPTY_WORD)); - }, - } - } - - if !entries.is_empty() { - let updated_root = self - .storage_forest - .batch_insert(prev_root, entries.iter().copied()) - .expect("Forest insertion should succeed"); - - self.vault_roots.insert((account_id, block_num), updated_root); - - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - vault_entries = entries.len(), - "Updated vault in forest" - ); - } - } - - /// Updates the forest with storage map changes from a delta. - /// - /// Processes storage map slot deltas, building SMTs for each modified slot - /// and tracking the new roots. - /// - /// # Arguments - /// - /// * `block_num` - Block number for this update - /// * `account_id` - The account being updated - /// * `storage_delta` - Changes to the account's storage maps - fn update_account_storage( - &mut self, - block_num: BlockNumber, - account_id: AccountId, - storage_delta: &AccountStorageDelta, - ) { - let prev_block_num = block_num.parent().unwrap_or_default(); - - for (slot_name, map_delta) in storage_delta.maps() { - let prev_root = self - .storage_roots - .get(&(account_id, slot_name.clone(), prev_block_num)) - .copied() - .unwrap_or_else(Self::empty_smt_root); - - // Collect entries from the delta - let entries = map_delta - .entries() - .iter() - .map(|(key, value)| ((*key).into(), *value)) - .collect::>(); - - if !entries.is_empty() { - let updated_root = self - .storage_forest - .batch_insert(prev_root, entries.iter().copied()) - .expect("Forest insertion should succeed"); - - self.storage_roots - .insert((account_id, slot_name.clone(), block_num), updated_root); - - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - slot_name = ?slot_name, - entries = entries.len(), - "Updated storage map in forest" - ); - } - } - } - /// Populates storage map SMTs in the forest from full database state for a single account. /// /// # Arguments @@ -298,8 +147,7 @@ impl InnerForest { .copied() .ok_or_else(|| { format!( - "Storage root not found for account {:?}, slot {}, block {}", - account_id, slot_name, block_num + "Storage root not found for account {account_id:?}, slot {slot_name}, block {block_num}" ) })?; @@ -327,11 +175,8 @@ impl InnerForest { tracing::debug!( target: crate::COMPONENT, - "Queried {} storage keys from forest for account {:?}, slot {} at block {}", - results.len(), - account_id, - slot_name, - block_num + "Queried {len} storage keys from forest for account {account_id:?}, slot {slot_name} at block {block_num}", + len = results.len(), ); Ok(results) diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 4de7f3808..561f189c5 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -41,22 +41,6 @@ fn test_inner_forest_basic_initialization() { assert!(forest.vault_roots.is_empty()); } -#[test] -fn test_update_account_with_empty_deltas() { - let mut forest = InnerForest::new(); - let account_id = test_account(); - let block_num = BlockNumber::GENESIS.child(); - - let vault_delta = AccountVaultDelta::default(); - let storage_delta = AccountStorageDelta::default(); - - forest.update_account(block_num, account_id, &vault_delta, &storage_delta); - - // Empty deltas should not create entries - assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); - assert!(forest.storage_roots.is_empty()); -} - #[test] fn test_update_vault_with_fungible_asset() { let mut forest = InnerForest::new(); @@ -65,10 +49,9 @@ fn test_update_vault_with_fungible_asset() { let block_num = BlockNumber::GENESIS.child(); let asset = create_fungible_asset(faucet_id, 100); - let mut vault_delta = AccountVaultDelta::default(); - vault_delta.add_asset(asset).unwrap(); + let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; - forest.update_account(block_num, account_id, &vault_delta, &AccountStorageDelta::default()); + forest.add_vault(account_id, &vault_entries, block_num); let vault_root = forest.vault_roots[&(account_id, block_num)]; assert_ne!(vault_root, EMPTY_WORD); @@ -81,28 +64,14 @@ fn test_compare_delta_vs_db_vault_with_fungible_asset() { let block_num = BlockNumber::GENESIS.child(); let asset = create_fungible_asset(faucet_id, 100); - // Approach 1: Delta-based update - let mut forest_delta = InnerForest::new(); - let mut vault_delta = AccountVaultDelta::default(); - vault_delta.add_asset(asset).unwrap(); - forest_delta.update_account( - block_num, - account_id, - &vault_delta, - &AccountStorageDelta::default(), - ); - - // Approach 2: DB-based population + // DB-based population approach let mut forest_db = InnerForest::new(); let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; forest_db.add_vault(account_id, &vault_entries, block_num); - // Both approaches must produce identical roots - let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); + // Verify the root is set correctly let root_db = forest_db.vault_roots.get(&(account_id, block_num)).unwrap(); - - assert_eq!(root_delta, root_db); - assert_ne!(*root_delta, EMPTY_WORD); + assert_ne!(*root_db, EMPTY_WORD); } #[test] @@ -116,20 +85,19 @@ fn test_incremental_vault_updates() { let mut forest = InnerForest::new(); let account_id = test_account(); let faucet_id = test_faucet(); - let storage_delta = AccountStorageDelta::default(); // Block 1: 100 tokens let block_1 = BlockNumber::GENESIS.child(); - let mut vault_delta_1 = AccountVaultDelta::default(); - vault_delta_1.add_asset(create_fungible_asset(faucet_id, 100)).unwrap(); - forest.update_account(block_1, account_id, &vault_delta_1, &storage_delta); + let asset_1 = create_fungible_asset(faucet_id, 100); + let vault_entries_1 = vec![(asset_1.vault_key().into(), Word::from(asset_1))]; + forest.add_vault(account_id, &vault_entries_1, block_1); let root_1 = forest.vault_roots[&(account_id, block_1)]; // Block 2: 150 tokens let block_2 = block_1.child(); - let mut vault_delta_2 = AccountVaultDelta::default(); - vault_delta_2.add_asset(create_fungible_asset(faucet_id, 150)).unwrap(); - forest.update_account(block_2, account_id, &vault_delta_2, &storage_delta); + let asset_2 = create_fungible_asset(faucet_id, 150); + let vault_entries_2 = vec![(asset_2.vault_key().into(), Word::from(asset_2))]; + forest.add_vault(account_id, &vault_entries_2, block_2); let root_2 = forest.vault_roots[&(account_id, block_2)]; assert_ne!(root_1, root_2); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index b1716b4e7..d4bd52d51 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -24,7 +24,7 @@ use miden_node_proto::domain::account::{ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; -use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent}; +use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent, StorageSlotName}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_objects::block::{ @@ -498,73 +498,56 @@ impl State { Ok(()) } - /// Updates storage map SMTs in the forest for changed accounts - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] + /// Updates storage map SMTs and vaults in the forest for changed accounts + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_ids.len()))] async fn update_storage_maps_in_forest( &self, - changed_account_ids: &[AccountId], + account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - // Step 1: Query storage from database - let account_storages = - self.query_account_storages_from_db(changed_account_ids, block_num).await?; - - // Step 2: Extract map slots and their entries using InnerForest helper - let map_slots_to_populate = InnerForest::extract_map_slots_from_storage(&account_storages); - - if map_slots_to_populate.is_empty() { - return Ok(()); - } - - // Step 3: Acquire write lock and update the forest with new SMTs let mut forest_guard = self.forest.write().await; - forest_guard.populate_storage_maps(map_slots_to_populate, block_num); - Ok(()) - } + // Process each account, updating both storage maps and vaults + for account_id in account_ids { + // Query and update storage maps for this account + let storage = self.db.select_account_storage_at_block(*account_id, block_num).await?; + let map_slots = extract_map_slots_from_storage(&storage); - /// Queries account storage data from the database for the given accounts at a specific block - #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_ids.len()))] - async fn query_account_storages_from_db( - &self, - account_ids: &[AccountId], - block_num: BlockNumber, - ) -> Result, ApplyBlockError> { - let mut account_storages = Vec::with_capacity(account_ids.len()); - - for &account_id in account_ids { - let storage = self.db.select_account_storage_at_block(account_id, block_num).await?; - account_storages.push((account_id, storage)); + if !map_slots.is_empty() { + forest_guard.add_storage_map(*account_id, map_slots, block_num); + } } - Ok(account_storages) + Ok(()) } /// Updates vault SMTs in the forest for changed accounts - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_ids.len()))] async fn update_vaults_in_forest( &self, - changed_account_ids: &[AccountId], + account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - // Query vault assets for each updated account - let mut vault_entries_to_populate = Vec::new(); + let mut forest_guard = self.forest.write().await; - for &account_id in changed_account_ids { - let entries = self.db.select_account_vault_at_block(account_id, block_num).await?; - if !entries.is_empty() { - vault_entries_to_populate.push((account_id, entries)); + // Process each account, updating vaults + for account_id in account_ids { + // Query and update vault for this account + let vault_entries = + self.db.select_account_vault_at_block(*account_id, block_num).await?; + + if !vault_entries.is_empty() { + forest_guard.add_vault(*account_id, &vault_entries, block_num); } - } - if vault_entries_to_populate.is_empty() { - return Ok(()); + tracing::debug!( + target: COMPONENT, + %account_id, + %block_num, + "Initialized forest for account from DB" + ); } - // Acquire write lock once for the entire update operation and delegate to InnerForest - let mut forest_guard = self.forest.write().await; - forest_guard.populate_vaults(vault_entries_to_populate, block_num); - Ok(()) } @@ -1404,3 +1387,31 @@ async fn load_account_tree( Ok(AccountTreeWithHistory::new(account_tree, block_number)) } + +// HELPERS +// ================================================================================================= + +/// Extract storage map slots from a single `AccountStorage` object. +/// +/// # Returns +/// +/// Vector of `(slot_name, entries)` tuples ready for forest population. +pub(crate) fn extract_map_slots_from_storage( + storage: &miden_objects::account::AccountStorage, +) -> Vec<(StorageSlotName, Vec<(Word, Word)>)> { + use miden_objects::account::StorageSlotContent; + + let mut map_slots = Vec::new(); + + for slot in storage.slots() { + if let StorageSlotContent::Map(map) = slot.content() { + let entries = Vec::from_iter(map.entries().map(|(k, v)| (*k, *v))); + + if !entries.is_empty() { + map_slots.push((slot.name().clone(), entries)); + } + } + } + + map_slots +} From 6fb80fe153fe041d6a57380ca1070e052c196665 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 23 Dec 2025 00:27:40 +0100 Subject: [PATCH 04/14] yay --- crates/store/src/errors.rs | 6 +++++ crates/store/src/inner_forest.rs | 39 ++++++++++++-------------------- crates/store/src/state.rs | 5 ++-- proto/proto/store/rpc.proto | 11 ++++----- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 6e67954b8..eca52a333 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -127,6 +127,12 @@ pub enum DatabaseError { SqlValueConversion(#[from] DatabaseTypeConversionError), #[error("Not implemented: {0}")] NotImplemented(String), + #[error("storage root not found for account {account_id}, slot {slot_name}, block {block_num}")] + StorageRootNotFound { + account_id: AccountId, + slot_name: String, + block_num: BlockNumber, + }, } impl DatabaseError { diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index d119375f4..5da0de196 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -5,6 +5,8 @@ use miden_objects::block::BlockNumber; use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH, SmtForest}; use miden_objects::{EMPTY_WORD, Word}; +use crate::errors::DatabaseError; + #[cfg(test)] mod tests; @@ -139,44 +141,33 @@ impl InnerForest { slot_name: &StorageSlotName, block_num: BlockNumber, keys: &[Word], - ) -> Result, String> { + ) -> Result, DatabaseError> { // Get the storage root for this account/slot/block let root = self .storage_roots .get(&(account_id, slot_name.clone(), block_num)) .copied() - .ok_or_else(|| { - format!( - "Storage root not found for account {account_id:?}, slot {slot_name}, block {block_num}" - ) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, })?; let mut results = Vec::with_capacity(keys.len()); for key in keys { - // Open a proof for this key in the forest - match self.storage_forest.open(root, *key) { - Ok(proof) => { - // Extract the value from the proof - let value = proof.get(key).unwrap_or(EMPTY_WORD); - results.push((*key, value)); - }, - Err(e) => { - tracing::debug!( - target: crate::COMPONENT, - "Failed to open proof for key in storage forest: {}. Using empty value.", - e - ); - // Return empty value for keys that can't be proven - results.push((*key, EMPTY_WORD)); - }, - } + let proof = self.storage_forest.open(root, *key)?; + let value = proof.get(key).unwrap_or(EMPTY_WORD); + results.push((*key, value)); } tracing::debug!( target: crate::COMPONENT, - "Queried {len} storage keys from forest for account {account_id:?}, slot {slot_name} at block {block_num}", - len = results.len(), + %account_id, + %block_num, + ?slot_name, + num_keys = results.len(), + "Queried storage keys from forest" ); Ok(results) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d4bd52d51..da5860575 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1190,9 +1190,8 @@ impl State { let details = match &slot_data { SlotData::MapKeys(keys) => { // Query the forest for specific keys - let entries = forest - .query_storage_keys(account_id, &slot_name, block_num, keys) - .map_err(DatabaseError::InteractError)?; + let entries = + forest.query_storage_keys(account_id, &slot_name, block_num, keys)?; AccountStorageMapDetails::from_forest_entries(slot_name, entries) }, SlotData::All => { diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index f2fbf0d7c..0f33cd895 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -227,14 +227,13 @@ message AccountStorageDetails { bool too_many_entries = 2; oneof data { - // By default we provide all storage entries when `all_entries` is requested - // or when the storage map is small. - MapEntries entries = 3; + // Contains the full key-value entries of the map. + // Returned if the map is small enough or all_entries is requested. + MapEntries full = 3; - // When specific keys are requested and the storage map is not small, - // we provide a set of SMT proofs (openings) for the requested keys. + // Contains SMT proofs for the entries requested. // This allows the receiver to reconstruct the partial tree or validate individual proofs. - SmtProofSet smt_proofs = 4; + SmtProofSet partial = 4; } } From efaa6854797f711dae07e351e861e55cb4c8d1bc Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 23 Dec 2025 00:33:47 +0100 Subject: [PATCH 05/14] feedback --- crates/proto/src/domain/account.rs | 51 ++++-------------------- crates/proto/src/generated/primitives.rs | 31 -------------- crates/store/src/inner_forest.rs | 27 +------------ proto/proto/types/primitives.proto | 30 -------------- 4 files changed, 8 insertions(+), 131 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index bc9adf8ed..aaa0b75f5 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -499,19 +499,8 @@ impl AccountStorageMapDetails { /// Creates storage map details based on the requested slot data. /// - /// This method handles both "all entries" and "specific keys" requests: - /// - For `SlotData::All`: Returns all entries from the storage map - /// - For `SlotData::MapKeys`: Returns only the requested keys with their values - /// - /// # Arguments - /// - /// * `slot_name` - The name of the storage slot - /// * `slot_data` - The type of data requested (all or specific keys) - /// * `storage_map` - The storage map to query - /// - /// # Returns - /// - /// Storage map details containing the requested entries or `LimitExceeded` if too many. + /// Handles both "all entries" and "specific keys" requests. + /// Returns `LimitExceeded` if too many entries. pub fn new(slot_name: StorageSlotName, slot_data: SlotData, storage_map: &StorageMap) -> Self { match slot_data { SlotData::All => Self::from_all_entries(slot_name, storage_map), @@ -537,19 +526,9 @@ impl AccountStorageMapDetails { } } - /// Creates storage map details from entries queried from storage forest with proofs. - /// - /// This method should be used when specific keys are requested and we want to include - /// Merkle proofs for verification. It avoids loading the entire storage map from the database. - /// - /// # Arguments - /// - /// * `slot_name` - The name of the storage slot - /// * `entries` - Key-value pairs with their Merkle proofs from the storage forest + /// Creates storage map details from forest-queried entries. /// - /// # Returns - /// - /// Storage map details containing the requested entries or `LimitExceeded` if too many keys. + /// Returns `LimitExceeded` if too many entries. pub fn from_forest_entries(slot_name: StorageSlotName, entries: Vec<(Word, Word)>) -> Self { if entries.len() > Self::MAX_RETURN_ENTRIES { Self { @@ -564,26 +543,10 @@ impl AccountStorageMapDetails { } } - /// Creates storage map details with SMT proofs for specific keys using the storage forest. - /// - /// This method queries the forest for specific keys and extracts key-value pairs from - /// the SMT proofs. The forest must be available and contain the data for the specified - /// SMT root. - /// - /// # Arguments - /// - /// * `slot_name` - The name of the storage slot - /// * `keys` - The keys to query - /// * `storage_forest` - The SMT forest containing the storage data - /// * `smt_root` - The root of the SMT for this storage slot - /// - /// # Returns - /// - /// Storage map details containing the requested entries or `LimitExceeded` if too many keys. - /// - /// # Errors + /// Creates storage map details with SMT proofs for specific keys. /// - /// Returns `MerkleError` if the forest doesn't contain sufficient data to provide proofs. + /// Returns `LimitExceeded` if too many keys, or `MerkleError` if the forest + /// doesn't contain sufficient data. pub fn from_specific_keys( slot_name: StorageSlotName, keys: &[Word], diff --git a/crates/proto/src/generated/primitives.rs b/crates/proto/src/generated/primitives.rs index e11017730..907ef856a 100644 --- a/crates/proto/src/generated/primitives.rs +++ b/crates/proto/src/generated/primitives.rs @@ -96,34 +96,3 @@ pub struct Digest { #[prost(fixed64, tag = "4")] pub d3: u64, } -/// Represents a partial Sparse Merkle Tree containing only a subset of leaves and their paths. -/// This allows verifying and updating tracked keys without requiring the full tree. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PartialSmt { - /// The root hash of the SMT - #[prost(message, optional, tag = "1")] - pub root: ::core::option::Option, - /// All tracked leaves in the partial SMT, keyed by their leaf index - #[prost(message, repeated, tag = "2")] - pub leaves: ::prost::alloc::vec::Vec, - /// Inner nodes stored in deterministic order (by scalar index) for reconstruction - #[prost(message, repeated, tag = "3")] - pub inner_nodes: ::prost::alloc::vec::Vec, -} -/// Represents a leaf with its index for partial SMT serialization -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SmtLeafWithIndex { - /// The leaf index (0 to 2^64 - 1 for leaves at depth 64) - #[prost(uint64, tag = "1")] - pub leaf_index: u64, - /// The leaf data - #[prost(message, optional, tag = "2")] - pub leaf: ::core::option::Option, -} -#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] -pub struct InnerNode { - #[prost(message, optional, tag = "1")] - pub left: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub right: ::core::option::Option, -} diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 5da0de196..d5d360bed 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -44,12 +44,6 @@ impl InnerForest { } /// Populates storage map SMTs in the forest from full database state for a single account. - /// - /// # Arguments - /// - /// * `account_id` - The account whose storage maps are being initialized - /// * `map_slots_to_populate` - List of `(slot_name, entries)` tuples - /// * `block_num` - Block number for which this state applies pub(crate) fn add_storage_map( &mut self, account_id: AccountId, @@ -81,12 +75,6 @@ impl InnerForest { } /// Populates a vault SMT in the forest from full database state. - /// - /// # Arguments - /// - /// * `account_id` - The account whose vault is being initialized - /// * `vault_entries` - (key, value) Word pairs for the vault - /// * `block_num` - Block number for which this state applies pub(crate) fn add_vault( &mut self, account_id: AccountId, @@ -115,20 +103,7 @@ impl InnerForest { /// Queries specific storage keys for a given account and slot at a specific block. /// - /// This method retrieves key-value pairs from the forest without loading the entire - /// storage map from the database. It returns the values along with their Merkle proofs. - /// - /// # Arguments - /// - /// * `account_id` - The account to query - /// * `slot_name` - The storage slot name - /// * `block_num` - The block number at which to query - /// * `keys` - The keys to retrieve - /// - /// # Returns - /// - /// A vector of key-value pairs for the requested keys. Keys that don't exist in the - /// storage map will have a value of `EMPTY_WORD`. + /// Keys that don't exist in the storage map will have a value of `EMPTY_WORD`. /// /// # Errors /// diff --git a/proto/proto/types/primitives.proto b/proto/proto/types/primitives.proto index 7e4951400..aed31cec0 100644 --- a/proto/proto/types/primitives.proto +++ b/proto/proto/types/primitives.proto @@ -92,33 +92,3 @@ message Digest { fixed64 d2 = 3; fixed64 d3 = 4; } - -// PARTIAL SMT -// ================================================================================================ - -// Represents a partial Sparse Merkle Tree containing only a subset of leaves and their paths. -// This allows verifying and updating tracked keys without requiring the full tree. -message PartialSmt { - // The root hash of the SMT - Digest root = 1; - - // All tracked leaves in the partial SMT, keyed by their leaf index - repeated SmtLeafWithIndex leaves = 2; - - // Inner nodes stored in deterministic order (by scalar index) for reconstruction - repeated InnerNode inner_nodes = 3; -} - -// Represents a leaf with its index for partial SMT serialization -message SmtLeafWithIndex { - // The leaf index (0 to 2^64 - 1 for leaves at depth 64) - uint64 leaf_index = 1; - - // The leaf data - SmtLeaf leaf = 2; -} - -message InnerNode { - Digest left = 1; - Digest right = 2; -} From 2982002c7d09121492bf87574b3eb2f4e7f53fec Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 23 Dec 2025 18:38:43 +0100 Subject: [PATCH 06/14] fff --- proto/proto/store/rpc.proto | 526 ------------------------------------ 1 file changed, 526 deletions(-) delete mode 100644 proto/proto/store/rpc.proto diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto deleted file mode 100644 index 0f33cd895..000000000 --- a/proto/proto/store/rpc.proto +++ /dev/null @@ -1,526 +0,0 @@ -// Specification of the store RPC. -// -// This provided access to the blockchain data to the other nodes. -syntax = "proto3"; -package rpc_store; - -import "google/protobuf/empty.proto"; -import "types/account.proto"; -import "types/blockchain.proto"; -import "types/transaction.proto"; -import "types/note.proto"; -import "types/primitives.proto"; -import "store/shared.proto"; - -// RPC STORE API -// ================================================================================================ - -// Store API for the RPC component -service Rpc { - // Returns the status info. - rpc Status(google.protobuf.Empty) returns (StoreStatus) {} - - // Returns a nullifier proof for each of the requested nullifiers. - rpc CheckNullifiers(NullifierList) returns (CheckNullifiersResponse) {} - - // Returns the latest state of an account with the specified ID. - rpc GetAccountDetails(account.AccountId) returns (account.AccountDetails) {} - - // Returns the latest state proof of the specified account. - rpc GetAccountProof(AccountProofRequest) returns (AccountProofResponse) {} - - // Returns raw block data for the specified block number. - rpc GetBlockByNumber(blockchain.BlockNumber) returns (blockchain.MaybeBlock) {} - - // Retrieves block header by given block number. Optionally, it also returns the MMR path - // and current chain length to authenticate the block's inclusion. - rpc GetBlockHeaderByNumber(shared.BlockHeaderByNumberRequest) returns (shared.BlockHeaderByNumberResponse) {} - - // Returns a list of committed notes matching the provided note IDs. - rpc GetNotesById(note.NoteIdList) returns (note.CommittedNoteList) {} - - // Returns the script for a note by its root. - rpc GetNoteScriptByRoot(note.NoteRoot) returns (shared.MaybeNoteScript) {} - - // Returns a list of nullifiers that match the specified prefixes and are recorded in the node. - // - // Note that only 16-bit prefixes are supported at this time. - rpc SyncNullifiers(SyncNullifiersRequest) returns (SyncNullifiersResponse) {} - - // Returns info which can be used by the requester to sync up to the tip of chain for the notes they are interested in. - // - // requester specifies the `note_tags` they are interested in, and the block height from which to search for new for - // matching notes for. The request will then return the next block containing any note matching the provided tags. - // - // The response includes each note's metadata and inclusion proof. - // - // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the - // tip of the chain. - rpc SyncNotes(SyncNotesRequest) returns (SyncNotesResponse) {} - - // Returns info which can be used by the requester to sync up to the latest state of the chain - // for the objects (accounts, notes, nullifiers) the requester is interested in. - // - // This request returns the next block containing requested data. It also returns `chain_tip` - // which is the latest block number in the chain. requester is expected to repeat these requests - // in a loop until `response.block_header.block_num == response.chain_tip`, at which point - // the requester is fully synchronized with the chain. - // - // Each request also returns info about new notes, nullifiers etc. created. It also returns - // Chain MMR delta that can be used to update the state of Chain MMR. This includes both chain - // MMR peaks and chain MMR nodes. - // - // For preserving some degree of privacy, note tags and nullifiers filters contain only high - // part of hashes. Thus, returned data contains excessive notes and nullifiers, requester can make - // additional filtering of that data on its side. - rpc SyncState(SyncStateRequest) returns (SyncStateResponse) {} - - // Returns account vault updates for specified account within a block range. - rpc SyncAccountVault(SyncAccountVaultRequest) returns (SyncAccountVaultResponse) {} - - // Returns storage map updates for specified account and storage slots within a block range. - rpc SyncStorageMaps(SyncStorageMapsRequest) returns (SyncStorageMapsResponse) {} - - // Returns transactions records for specific accounts within a block range. - rpc SyncTransactions(SyncTransactionsRequest) returns (SyncTransactionsResponse) {} -} - -// STORE STATUS -// ================================================================================================ - -// Represents the status of the store. -message StoreStatus { - // The store's running version. - string version = 1; - - // The store's status. - string status = 2; - - // Number of the latest block in the chain. - fixed32 chain_tip = 3; -} - -// GET ACCOUNT PROOF -// ================================================================================================ - -// Returns the latest state proof of the specified account. -message AccountProofRequest { - // Request the details for a public account. - message AccountDetailRequest { - // Represents a storage slot index and the associated map keys. - message StorageMapDetailRequest { - // Indirection required for use in `oneof {..}` block. - message MapKeys { - // A list of map keys associated with this storage slot. - repeated primitives.Digest map_keys = 1; - } - // Storage slot index (`[0..255]`). - uint32 slot_index = 1; - - oneof slot_data { - // Request to return all storage map data. If the number exceeds a threshold of 1000 entries, - // the response will not contain them but must be requested separately. - bool all_entries = 2; - - // A list of map keys associated with the given storage slot identified by `slot_index`. - MapKeys map_keys = 3; - } - } - - // Last known code commitment to the requester. The response will include account code - // only if its commitment is different from this value. - // - // If the field is ommiteed, the response will not include the account code. - optional primitives.Digest code_commitment = 1; - - // Last known asset vault commitment to the requester. The response will include asset vault data - // only if its commitment is different from this value. If the value is not present in the - // request, the response will not contain one either. - // If the number of to-be-returned asset entries exceed a threshold, they have to be requested - // separately, which is signaled in the response message with dedicated flag. - optional primitives.Digest asset_vault_commitment = 2; - - // Additional request per storage map. - repeated StorageMapDetailRequest storage_maps = 3; - } - - // ID of the account for which we want to get data - account.AccountId account_id = 1; - - // Optional block height at which to return the proof. - // - // Defaults to current chain tip if unspecified. - optional blockchain.BlockNumber block_num = 2; - - // Request for additional account details; valid only for public accounts. - optional AccountDetailRequest details = 3; -} - -// Represents the result of getting account proof. -message AccountProofResponse { - - message AccountDetails { - // Account header. - account.AccountHeader header = 1; - - // Account storage data - AccountStorageDetails storage_details = 2; - - // Account code; empty if code commitments matched or none was requested. - optional bytes code = 3; - - // Account asset vault data; empty if vault commitments matched or the requester - // omitted it in the request. - optional AccountVaultDetails vault_details = 4; - } - - // The block number at which the account witness was created and the account details were observed. - blockchain.BlockNumber block_num = 1; - - // Account ID, current state commitment, and SMT path. - account.AccountWitness witness = 2; - - // Additional details for public accounts. - optional AccountDetails details = 3; -} - -// Account vault details for AccountProofResponse -message AccountVaultDetails { - // A flag that is set to true if the account contains too many assets. This indicates - // to the user that `SyncAccountVault` endpoint should be used to retrieve the - // account's assets - bool too_many_assets = 1; - - // When too_many_assets == false, this will contain the list of assets in the - // account's vault - repeated primitives.Asset assets = 2; -} - -// Represents a set of SMT proofs (openings) for requested keys -message SmtProofSet { - // The root hash of the SMT these proofs are for - primitives.Digest root = 1; - - // Collection of SMT proofs/openings for the requested keys - repeated primitives.SmtOpening proofs = 2; -} - -// Account storage details for AccountProofResponse -message AccountStorageDetails { - message AccountStorageMapDetails { - // Wrapper for repeated storage map entries - message MapEntries { - // Definition of individual storage entries. - message StorageMapEntry { - primitives.Digest key = 1; - primitives.Digest value = 2; - } - - repeated StorageMapEntry entries = 1; - } - // slot index of the storage map - uint32 slot_index = 1; - - // A flag that is set to `true` if the number of to-be-returned entries in the - // storage map would exceed a threshold. This indicates to the user that `SyncStorageMaps` - // endpoint should be used to get all storage map data. - bool too_many_entries = 2; - - oneof data { - // Contains the full key-value entries of the map. - // Returned if the map is small enough or all_entries is requested. - MapEntries full = 3; - - // Contains SMT proofs for the entries requested. - // This allows the receiver to reconstruct the partial tree or validate individual proofs. - SmtProofSet partial = 4; - } - } - - // Account storage header (storage slot info for up to 256 slots) - account.AccountStorageHeader header = 1; - - // Additional data for the requested storage maps - repeated AccountStorageMapDetails map_details = 2; -} - - -// CHECK NULLIFIERS -// ================================================================================================ - -// List of nullifiers to return proofs for. -message NullifierList { - // List of nullifiers to return proofs for. - repeated primitives.Digest nullifiers = 1; -} - -// Represents the result of checking nullifiers. -message CheckNullifiersResponse { - // Each requested nullifier has its corresponding nullifier proof at the same position. - repeated primitives.SmtOpening proofs = 1; -} - -// SYNC NULLIFIERS -// ================================================================================================ - -// Returns a list of nullifiers that match the specified prefixes and are recorded in the node. -message SyncNullifiersRequest { - // Block number from which the nullifiers are requested (inclusive). - BlockRange block_range = 1; - - // Number of bits used for nullifier prefix. Currently the only supported value is 16. - uint32 prefix_len = 2; - - // List of nullifiers to check. Each nullifier is specified by its prefix with length equal - // to `prefix_len`. - repeated uint32 nullifiers = 3; -} - -// Represents the result of syncing nullifiers. -message SyncNullifiersResponse { - // Represents a single nullifier update. - message NullifierUpdate { - // Nullifier ID. - primitives.Digest nullifier = 1; - - // Block number. - fixed32 block_num = 2; - } - - // Pagination information. - PaginationInfo pagination_info = 1; - - // List of nullifiers matching the prefixes specified in the request. - repeated NullifierUpdate nullifiers = 2; -} - -// SYNC STATE -// ================================================================================================ - -// State synchronization request. -// -// Specifies state updates the requester is interested in. The server will return the first block which -// contains a note matching `note_tags` or the chain tip. And the corresponding updates to -// `account_ids` for that block range. -message SyncStateRequest { - // Last block known by the requester. The response will contain data starting from the next block, - // until the first block which contains a note of matching the requested tag, or the chain tip - // if there are no notes. - fixed32 block_num = 1; - - // Accounts' commitment to include in the response. - // - // An account commitment will be included if-and-only-if it is the latest update. Meaning it is - // possible there was an update to the account for the given range, but if it is not the latest, - // it won't be included in the response. - repeated account.AccountId account_ids = 2; - - // Specifies the tags which the requester is interested in. - repeated fixed32 note_tags = 3; -} - -// Represents the result of syncing state request. -message SyncStateResponse { - // Number of the latest block in the chain. - fixed32 chain_tip = 1; - - // Block header of the block with the first note matching the specified criteria. - blockchain.BlockHeader block_header = 2; - - // Data needed to update the partial MMR from `request.block_num + 1` to `response.block_header.block_num`. - primitives.MmrDelta mmr_delta = 3; - - // List of account commitments updated after `request.block_num + 1` but not after `response.block_header.block_num`. - repeated account.AccountSummary accounts = 5; - - // List of transactions executed against requested accounts between `request.block_num + 1` and - // `response.block_header.block_num`. - repeated transaction.TransactionSummary transactions = 6; - - // List of all notes together with the Merkle paths from `response.block_header.note_root`. - repeated note.NoteSyncRecord notes = 7; -} - -// SYNC ACCOUNT VAULT -// ================================================================================================ - -// Account vault synchronization request. -// -// Allows requesters to sync asset values for specific public accounts within a block range. -message SyncAccountVaultRequest { - // Block range from which to start synchronizing. - // - // If the `block_to` is specified, this block must be close to the chain tip (i.e., within 30 blocks), - // otherwise an error will be returned. - BlockRange block_range = 1; - - // Account for which we want to sync asset vault. - account.AccountId account_id = 2; -} - -message SyncAccountVaultResponse { - // Pagination information. - PaginationInfo pagination_info = 1; - - // List of asset updates for the account. - // - // Multiple updates can be returned for a single asset, and the one with a higher `block_num` - // is expected to be retained by the caller. - repeated AccountVaultUpdate updates = 2; -} - -message AccountVaultUpdate { - // Vault key associated with the asset. - primitives.Digest vault_key = 1; - - // Asset value related to the vault key. - // If not present, the asset was removed from the vault. - optional primitives.Asset asset = 2; - - // Block number at which the above asset was updated in the account vault. - fixed32 block_num = 3; -} - -// SYNC NOTES -// ================================================================================================ - -// Note synchronization request. -// -// Specifies note tags that requester is interested in. The server will return the first block which -// contains a note matching `note_tags` or the chain tip. -message SyncNotesRequest { - // Block range from which to start synchronizing. - BlockRange block_range = 1; - - // Specifies the tags which the requester is interested in. - repeated fixed32 note_tags = 2; -} - -// Represents the result of syncing notes request. -message SyncNotesResponse { - // Pagination information. - PaginationInfo pagination_info = 1; - - // Block header of the block with the first note matching the specified criteria. - blockchain.BlockHeader block_header = 2; - - // Merkle path to verify the block's inclusion in the MMR at the returned `chain_tip`. - // - // An MMR proof can be constructed for the leaf of index `block_header.block_num` of - // an MMR of forest `chain_tip` with this path. - primitives.MerklePath mmr_path = 3; - - // List of all notes together with the Merkle paths from `response.block_header.note_root`. - repeated note.NoteSyncRecord notes = 4; -} - -// SYNC STORAGE MAP -// ================================================================================================ - -// Storage map synchronization request. -// -// Allows requesters to sync storage map values for specific public accounts within a block range, -// with support for cursor-based pagination to handle large storage maps. -message SyncStorageMapsRequest { - // Block range from which to start synchronizing. - // - // If the `block_to` is specified, this block must be close to the chain tip (i.e., within 30 blocks), - // otherwise an error will be returned. - BlockRange block_range = 1; - - // Account for which we want to sync storage maps. - account.AccountId account_id = 3; -} - -message SyncStorageMapsResponse { - // Pagination information. - PaginationInfo pagination_info = 1; - - // The list of storage map updates. - // - // Multiple updates can be returned for a single slot index and key combination, and the one - // with a higher `block_num` is expected to be retained by the caller. - repeated StorageMapUpdate updates = 2; -} - -// Represents a single storage map update. -message StorageMapUpdate { - // Block number in which the slot was updated. - fixed32 block_num = 1; - - // Slot index ([0..255]). - uint32 slot_index = 2; - - // The storage map key. - primitives.Digest key = 3; - - // The storage map value. - primitives.Digest value = 4; -} - -// BLOCK RANGE -// ================================================================================================ - -// Represents a block range. -message BlockRange { - // Block number from which to start (inclusive). - fixed32 block_from = 1; - - // Block number up to which to check (inclusive). If not specified, checks up to the latest block. - optional fixed32 block_to = 2; -} - -// PAGINATION INFO -// ================================================================================================ - -// Represents pagination information for chunked responses. -// -// Pagination is done using block numbers as the axis, allowing requesters to request -// data in chunks by specifying block ranges and continuing from where the previous -// response left off. -// -// To request the next chunk, the requester should use `block_num + 1` from the previous response -// as the `block_from` for the next request. -message PaginationInfo { - // Current chain tip - fixed32 chain_tip = 1; - - // The block number of the last check included in this response. - // - // For chunked responses, this may be less than `request.block_range.block_to`. - // If it is less than request.block_range.block_to, the user is expected to make a subsequent request - // starting from the next block to this one (ie, request.block_range.block_from = block_num + 1). - fixed32 block_num = 2; -} - -// SYNC TRANSACTIONS -// ================================================================================================ - -// Transactions synchronization request. -// -// Allows requesters to sync transactions for specific accounts within a block range. -message SyncTransactionsRequest { - // Block range from which to start synchronizing. - BlockRange block_range = 1; - - // Accounts to sync transactions for. - repeated account.AccountId account_ids = 2; -} - -// Represents the result of syncing transactions request. -message SyncTransactionsResponse { - // Pagination information. - PaginationInfo pagination_info = 1; - - // List of transaction records. - repeated TransactionRecord transactions = 2; -} - -// Represents a transaction record. -message TransactionRecord { - // Block number in which the transaction was included. - fixed32 block_num = 1; - - // A transaction header. - transaction.TransactionHeader header = 2; -} From f7027916ba51d62196b52db56a81a463ba8dc285 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 22:01:51 +0100 Subject: [PATCH 07/14] mixed bag --- crates/proto/src/domain/account.rs | 323 ++++++++++++++---- crates/proto/src/generated/rpc.rs | 65 +++- crates/store/src/db/mod.rs | 16 +- .../store/src/db/models/queries/accounts.rs | 14 + crates/store/src/inner_forest.rs | 64 +++- crates/store/src/state.rs | 57 +++- proto/proto/rpc.proto | 35 +- 7 files changed, 466 insertions(+), 108 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 852020671..e59a2a6fd 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -193,22 +193,24 @@ impl TryFrom fn try_from( value: proto::rpc::account_storage_details::AccountStorageMapDetails, ) -> Result { - use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry; + use proto::rpc::account_storage_details::account_storage_map_details::{ + all_map_entries::StorageMapEntry, + map_entries_with_proofs::StorageMapEntryWithProof, + AllMapEntries, + MapEntriesWithProofs, + Entries as ProtoEntries, + }; - let proto::rpc::account_storage_details::AccountStorageMapDetails { - slot_name, - too_many_entries, - entries, - } = value; + let proto::rpc::account_storage_details::AccountStorageMapDetails { slot_name, entries } = + value; let slot_name = StorageSlotName::new(slot_name)?; - let entries = if too_many_entries { - StorageMapEntries::LimitExceeded - } else { - let map_entries = if let Some(entries) = entries { - entries - .entries + let map_entries = match entries { + Some(ProtoEntries::LimitExceeded(true)) | None => StorageMapEntries::LimitExceeded, + Some(ProtoEntries::LimitExceeded(false)) => StorageMapEntries::AllEntries(Vec::new()), + Some(ProtoEntries::AllEntries(AllMapEntries { entries })) => { + let map_entries = entries .into_iter() .map(|entry| { let key = entry @@ -221,14 +223,33 @@ impl TryFrom .try_into()?; Ok((key, value)) }) - .collect::, ConversionError>>()? - } else { - Vec::new() - }; - StorageMapEntries::Entries(map_entries) + .collect::, ConversionError>>()?; + StorageMapEntries::AllEntries(map_entries) + }, + Some(ProtoEntries::EntriesWithProofs(MapEntriesWithProofs { entries })) => { + let proofs = entries + .into_iter() + .map(|entry| { + let _key: Word = entry + .key + .ok_or(StorageMapEntryWithProof::missing_field(stringify!(key)))? + .try_into()?; + let _value: Word = entry + .value + .ok_or(StorageMapEntryWithProof::missing_field(stringify!(value)))? + .try_into()?; + let smt_opening = entry + .proof + .ok_or(StorageMapEntryWithProof::missing_field(stringify!(proof)))?; + let smt_proof = SmtProof::try_from(smt_opening)?; + Ok(smt_proof) + }) + .collect::, ConversionError>>()?; + StorageMapEntries::EntriesWithProofs(proofs) + }, }; - Ok(Self { slot_name, entries }) + Ok(Self { slot_name, entries: map_entries }) } } @@ -456,28 +477,22 @@ impl From for proto::rpc::AccountVaultDetails { /// returning all entries in a single RPC response creates performance issues. In such cases, /// the `LimitExceeded` variant indicates to the client to use the `SyncStorageMaps` endpoint /// instead. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub enum StorageMapEntries { /// The map has too many entries to return inline. /// Clients must use `SyncStorageMaps` endpoint instead. LimitExceeded, - /// The storage map entries (key-value pairs), up to `MAX_RETURN_ENTRIES`. - /// TODO: For partial responses, also include Merkle proofs and inner SMT nodes. - Entries(Vec<(Word, Word)>), -} - -/// Details about an account storage map slot. -#[derive(Debug, Clone, PartialEq)] -pub enum StorageMapData { - /// All entries are included used for small storage maps or when `all_entries` is requested. + /// All storage map entries (key-value pairs) without proofs. + /// Used when all entries are requested for small maps. AllEntries(Vec<(Word, Word)>), - /// Specific entries with their Merkle proofs for partial responses. + /// Specific entries with their SMT proofs for client-side verification. + /// Used when specific keys are requested from the storage map. EntriesWithProofs(Vec), } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct AccountStorageMapDetails { pub slot_name: StorageSlotName, pub entries: StorageMapEntries, @@ -501,7 +516,7 @@ impl AccountStorageMapDetails { let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); Self { slot_name, - entries: StorageMapEntries::Entries(map_entries), + entries: StorageMapEntries::AllEntries(map_entries), } } } @@ -520,7 +535,8 @@ impl AccountStorageMapDetails { entries: StorageMapEntries::LimitExceeded, } } else { - // Query specific keys from the storage map + // Query specific keys from the storage map - returns all entries without proofs + // For proofs, use from_specific_keys with SmtForest let mut entries = Vec::with_capacity(keys.len()); for key in keys { let value = storage_map.get(&key); @@ -528,7 +544,7 @@ impl AccountStorageMapDetails { } Self { slot_name, - entries: StorageMapEntries::Entries(entries), + entries: StorageMapEntries::AllEntries(entries), } } }, @@ -547,13 +563,16 @@ impl AccountStorageMapDetails { } else { Self { slot_name, - entries: StorageMapEntries::Entries(entries), + entries: StorageMapEntries::AllEntries(entries), } } } /// Creates storage map details with SMT proofs for specific keys. /// + /// This method queries the forest for specific keys and returns proofs that + /// enable client-side verification of the values. + /// /// Returns `LimitExceeded` if too many keys, or `MerkleError` if the forest /// doesn't contain sufficient data. pub fn from_specific_keys( @@ -569,23 +588,22 @@ impl AccountStorageMapDetails { }); } - // Collect key-value pairs by opening proofs for each key - let mut entries = Vec::with_capacity(keys.len()); + // Collect SMT proofs for each key + let mut proofs = Vec::with_capacity(keys.len()); for key in keys { let proof = storage_forest.open(smt_root, *key)?; - let value = proof.get(key).unwrap_or(miden_objects::EMPTY_WORD); - entries.push((*key, value)); + proofs.push(proof); } Ok(Self { slot_name, - entries: StorageMapEntries::Entries(entries), + entries: StorageMapEntries::EntriesWithProofs(proofs), }) } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct AccountStorageDetails { pub header: AccountStorageHeader, pub map_details: Vec, @@ -748,33 +766,56 @@ impl From for proto::rpc::account_storage_details::AccountStorageMapDetails { fn from(value: AccountStorageMapDetails) -> Self { - use proto::rpc::account_storage_details::account_storage_map_details; + use proto::rpc::account_storage_details::account_storage_map_details::{ + AllMapEntries, + MapEntriesWithProofs, + Entries as ProtoEntries, + }; let AccountStorageMapDetails { slot_name, entries } = value; - match entries { - StorageMapEntries::LimitExceeded => Self { - slot_name: slot_name.to_string(), - too_many_entries: true, - entries: Some(account_storage_map_details::MapEntries { entries: Vec::new() }), - }, - StorageMapEntries::Entries(map_entries) => { - let entries = Some(account_storage_map_details::MapEntries { + let proto_entries = match entries { + StorageMapEntries::LimitExceeded => Some(ProtoEntries::LimitExceeded(true)), + StorageMapEntries::AllEntries(map_entries) => { + let all = AllMapEntries { entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { - account_storage_map_details::map_entries::StorageMapEntry { + proto::rpc::account_storage_details::account_storage_map_details::all_map_entries::StorageMapEntry { key: Some(key.into()), value: Some(value.into()), } })), - }); - - Self { - slot_name: slot_name.to_string(), - too_many_entries: false, - entries, - } + }; + Some(ProtoEntries::AllEntries(all)) }, - } + StorageMapEntries::EntriesWithProofs(proofs) => { + use miden_objects::crypto::merkle::SmtLeaf; + + let with_proofs = MapEntriesWithProofs { + entries: Vec::from_iter(proofs.into_iter().map(|proof| { + // Get key/value from the leaf before consuming the proof + let (key, value) = match proof.leaf() { + SmtLeaf::Empty(_) => { + (miden_objects::EMPTY_WORD, miden_objects::EMPTY_WORD) + }, + SmtLeaf::Single((k, v)) => (*k, *v), + SmtLeaf::Multiple(entries) => entries.iter().next().map_or( + (miden_objects::EMPTY_WORD, miden_objects::EMPTY_WORD), + |(k, v)| (*k, *v), + ), + }; + let smt_opening = proto::primitives::SmtOpening::from(proof); + proto::rpc::account_storage_details::account_storage_map_details::map_entries_with_proofs::StorageMapEntryWithProof { + key: Some(key.into()), + value: Some(value.into()), + proof: Some(smt_opening), + } + })), + }; + Some(ProtoEntries::EntriesWithProofs(with_proofs)) + }, + }; + + Self { slot_name: slot_name.to_string(), entries: proto_entries } } } // ACCOUNT WITNESS @@ -1039,3 +1080,169 @@ pub enum NetworkAccountError { fn get_account_id_tag_prefix(id: AccountId) -> AccountPrefix { (id.prefix().as_u64() >> 34) as AccountPrefix } + +#[cfg(test)] +mod tests { + use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; + + use super::*; + + fn word_from_u32(arr: [u32; 4]) -> Word { + Word::from(arr) + } + + fn test_slot_name() -> StorageSlotName { + StorageSlotName::new("miden::test::storage::slot").unwrap() + } + + fn empty_smt_root() -> Word { + *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + } + + #[test] + fn account_storage_map_details_from_forest_entries() { + let slot_name = test_slot_name(); + let entries = vec![ + (word_from_u32([1, 2, 3, 4]), word_from_u32([5, 6, 7, 8])), + (word_from_u32([9, 10, 11, 12]), word_from_u32([13, 14, 15, 16])), + ]; + + let details = + AccountStorageMapDetails::from_forest_entries(slot_name.clone(), entries.clone()); + + assert_eq!(details.slot_name, slot_name); + assert_eq!(details.entries, StorageMapEntries::AllEntries(entries)); + } + + #[test] + fn account_storage_map_details_from_forest_entries_limit_exceeded() { + let slot_name = test_slot_name(); + // Create more entries than MAX_RETURN_ENTRIES + let entries: Vec<_> = (0..AccountStorageMapDetails::MAX_RETURN_ENTRIES + 1) + .map(|i| { + let key = word_from_u32([i as u32, 0, 0, 0]); + let value = word_from_u32([0, 0, 0, i as u32]); + (key, value) + }) + .collect(); + + let details = AccountStorageMapDetails::from_forest_entries(slot_name.clone(), entries); + + assert_eq!(details.slot_name, slot_name); + assert_eq!(details.entries, StorageMapEntries::LimitExceeded); + } + + #[test] + fn account_storage_map_details_from_specific_keys() { + let slot_name = test_slot_name(); + + // Create an SmtForest and populate it with some data + let mut forest = SmtForest::new(); + let entries = vec![ + (word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0])), + (word_from_u32([2, 0, 0, 0]), word_from_u32([20, 0, 0, 0])), + (word_from_u32([3, 0, 0, 0]), word_from_u32([30, 0, 0, 0])), + ]; + + // Insert entries into the forest starting from an empty root + let smt_root = forest.batch_insert(empty_smt_root(), entries.iter().copied()).unwrap(); + + // Query specific keys + let keys = vec![word_from_u32([1, 0, 0, 0]), word_from_u32([3, 0, 0, 0])]; + + let details = AccountStorageMapDetails::from_specific_keys( + slot_name.clone(), + &keys, + &forest, + smt_root, + ) + .unwrap(); + + assert_eq!(details.slot_name, slot_name); + match details.entries { + StorageMapEntries::EntriesWithProofs(proofs) => { + use miden_objects::crypto::merkle::SmtLeaf; + + assert_eq!(proofs.len(), 2); + + // Helper to extract key-value from any leaf type + let get_value = |proof: &SmtProof, expected_key: Word| -> Word { + match proof.leaf() { + SmtLeaf::Single((k, v)) if *k == expected_key => *v, + SmtLeaf::Multiple(entries) => entries + .iter() + .find(|(k, _)| *k == expected_key) + .map(|(_, v)| *v) + .unwrap_or(miden_objects::EMPTY_WORD), + _ => miden_objects::EMPTY_WORD, + } + }; + + let key1 = word_from_u32([1, 0, 0, 0]); + let key2 = word_from_u32([3, 0, 0, 0]); + let value1 = get_value(&proofs[0], key1); + let value2 = get_value(&proofs[1], key2); + + assert_eq!(value1, word_from_u32([10, 0, 0, 0])); + assert_eq!(value2, word_from_u32([30, 0, 0, 0])); + }, + _ => panic!("Expected EntriesWithProofs"), + } + } + + #[test] + fn account_storage_map_details_from_specific_keys_nonexistent_returns_proof() { + let slot_name = test_slot_name(); + + // Create an SmtForest with one entry so the root is tracked + let mut forest = SmtForest::new(); + let entries = vec![(word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0]))]; + let smt_root = forest.batch_insert(empty_smt_root(), entries.iter().copied()).unwrap(); + + // Query a key that doesn't exist in the tree - should return a proof + // (the proof will show non-membership or point to an adjacent leaf) + let keys = vec![word_from_u32([99, 0, 0, 0])]; + + let details = AccountStorageMapDetails::from_specific_keys( + slot_name.clone(), + &keys, + &forest, + smt_root, + ) + .unwrap(); + + match details.entries { + StorageMapEntries::EntriesWithProofs(proofs) => { + // We got a proof for the non-existent key + assert_eq!(proofs.len(), 1); + // The proof exists and can be used to verify non-membership + }, + _ => panic!("Expected EntriesWithProofs"), + } + } + + #[test] + fn account_storage_map_details_from_specific_keys_limit_exceeded() { + let slot_name = test_slot_name(); + let mut forest = SmtForest::new(); + + // Create a forest with some data to get a valid root + let entries = vec![(word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0]))]; + let smt_root = forest.batch_insert(empty_smt_root(), entries.iter().copied()).unwrap(); + + // Create more keys than MAX_RETURN_ENTRIES + let keys: Vec<_> = (0..AccountStorageMapDetails::MAX_RETURN_ENTRIES + 1) + .map(|i| word_from_u32([i as u32, 0, 0, 0])) + .collect(); + + let details = AccountStorageMapDetails::from_specific_keys( + slot_name.clone(), + &keys, + &forest, + smt_root, + ) + .unwrap(); + + assert_eq!(details.entries, StorageMapEntries::LimitExceeded); + } +} diff --git a/crates/proto/src/generated/rpc.rs b/crates/proto/src/generated/rpc.rs index 4736f4cd6..123416d67 100644 --- a/crates/proto/src/generated/rpc.rs +++ b/crates/proto/src/generated/rpc.rs @@ -233,25 +233,50 @@ pub mod account_storage_details { /// Storage slot name. #[prost(string, tag = "1")] pub slot_name: ::prost::alloc::string::String, - /// A flag that is set to `true` if the number of to-be-returned entries in the - /// storage map would exceed a threshold. This indicates to the user that `SyncStorageMaps` - /// endpoint should be used to get all storage map data. - #[prost(bool, tag = "2")] - pub too_many_entries: bool, - /// By default we provide all storage entries. - #[prost(message, optional, tag = "3")] - pub entries: ::core::option::Option, + /// Either the map entries (with or without proofs) or an indicator that the limit was exceeded. + /// When `limit_exceeded` is set, clients should use the `SyncStorageMaps` endpoint. + #[prost(oneof = "account_storage_map_details::Entries", tags = "2, 3, 4")] + pub entries: ::core::option::Option, } /// Nested message and enum types in `AccountStorageMapDetails`. pub mod account_storage_map_details { - /// Wrapper for repeated storage map entries + /// Wrapper for repeated storage map entries including their proofs. + /// Used when specific keys are requested to enable client-side verification. #[derive(Clone, PartialEq, ::prost::Message)] - pub struct MapEntries { + pub struct MapEntriesWithProofs { #[prost(message, repeated, tag = "1")] - pub entries: ::prost::alloc::vec::Vec, + pub entries: ::prost::alloc::vec::Vec< + map_entries_with_proofs::StorageMapEntryWithProof, + >, } - /// Nested message and enum types in `MapEntries`. - pub mod map_entries { + /// Nested message and enum types in `MapEntriesWithProofs`. + pub mod map_entries_with_proofs { + /// Definition of individual storage entries including a proof. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct StorageMapEntryWithProof { + #[prost(message, optional, tag = "1")] + pub key: ::core::option::Option< + super::super::super::super::primitives::Digest, + >, + #[prost(message, optional, tag = "2")] + pub value: ::core::option::Option< + super::super::super::super::primitives::Digest, + >, + #[prost(message, optional, tag = "3")] + pub proof: ::core::option::Option< + super::super::super::super::primitives::SmtOpening, + >, + } + } + /// Wrapper for repeated storage map entries (without proofs). + /// Used when all entries are requested for small maps. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct AllMapEntries { + #[prost(message, repeated, tag = "1")] + pub entries: ::prost::alloc::vec::Vec, + } + /// Nested message and enum types in `AllMapEntries`. + pub mod all_map_entries { /// Definition of individual storage entries. #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StorageMapEntry { @@ -265,6 +290,20 @@ pub mod account_storage_details { >, } } + /// Either the map entries (with or without proofs) or an indicator that the limit was exceeded. + /// When `limit_exceeded` is set, clients should use the `SyncStorageMaps` endpoint. + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Entries { + /// All storage entries without proofs (for small maps or full requests). + #[prost(message, tag = "2")] + AllEntries(AllMapEntries), + /// Specific entries with their SMT proofs (for partial requests). + #[prost(message, tag = "3")] + EntriesWithProofs(MapEntriesWithProofs), + /// Set to true when the number of entries exceeds the response limit. + #[prost(bool, tag = "4")] + LimitExceeded(bool), + } } } /// List of nullifiers to return proofs for. diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 737109468..762f5b81b 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -8,7 +8,7 @@ use miden_lib::utils::{Deserializable, Serializable}; use miden_node_proto::domain::account::{AccountInfo, AccountSummary, NetworkAccountPrefix}; use miden_node_proto::generated as proto; use miden_objects::Word; -use miden_objects::account::{AccountHeader, AccountId, AccountStorage}; +use miden_objects::account::{AccountHeader, AccountId, AccountStorageHeader}; use miden_objects::asset::{Asset, AssetVaultKey}; use miden_objects::block::{BlockHeader, BlockNoteIndex, BlockNumber, ProvenBlock}; use miden_objects::crypto::merkle::SparseMerklePath; @@ -426,19 +426,15 @@ impl Db { .await } - /// Reconstructs account storage at a specific block from the database - /// - /// This method queries the decomposed storage tables and reconstructs the full - /// `AccountStorage` with SMT backing for Map slots. - // TODO split querying the header from the content + /// Queries just the storage header (slot types and roots) at a specific block. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_account_storage_at_block( + pub async fn select_account_storage_header_at_block( &self, account_id: AccountId, block_num: BlockNumber, - ) -> Result { - self.transact("Get account storage at block", move |conn| { - queries::select_account_storage_at_block(conn, account_id, block_num) + ) -> Result { + self.transact("Get account storage header at block", move |conn| { + queries::select_account_storage_header_at_block(conn, account_id, block_num) }) .await } diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index a5a7d26ab..e6266f28a 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -31,6 +31,7 @@ use miden_objects::account::{ AccountHeader, AccountId, AccountStorage, + AccountStorageHeader, NonFungibleDeltaAction, StorageSlotContent, StorageSlotName, @@ -529,6 +530,19 @@ pub(crate) fn select_account_storage_at_block( Ok(storage) } +/// Returns account storage header (without map entries) at a given block. +/// +/// This reads the storage blob and extracts just the header information (slot types and roots), +/// avoiding the need to deserialize all map entries. +pub(crate) fn select_account_storage_header_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result { + let storage = select_account_storage_at_block(conn, account_id, block_num)?; + Ok(storage.to_header()) +} + /// Select latest account storage header by querying `accounts.storage_header` where /// `is_latest=true`. pub(crate) fn select_latest_account_storage( diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index a8b0d3423..2f33c174b 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -20,6 +20,10 @@ pub(crate) struct InnerForest { /// Populated during block import for all storage map slots. storage_roots: BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word>, + /// Maps (`account_id`, `slot_name`, `block_num`) to all key-value entries in that storage map. + /// Accumulated from deltas - each block's entries include all entries up to that point. + storage_entries: BTreeMap<(AccountId, StorageSlotName, BlockNumber), BTreeMap>, + /// Maps (`account_id`, `block_num`) to vault SMT root. /// Tracks asset vault versions across all blocks with structural sharing. vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, @@ -30,6 +34,7 @@ impl InnerForest { Self { storage_forest: SmtForest::new(), storage_roots: BTreeMap::new(), + storage_entries: BTreeMap::new(), vault_roots: BTreeMap::new(), } } @@ -64,6 +69,33 @@ impl InnerForest { .unwrap_or_else(Self::empty_smt_root) } + /// Returns the storage forest and the root for a specific account storage slot at a block. + /// + /// This allows callers to query specific keys from the storage map using `SmtForest::open()`. + /// Returns `None` if no storage root is tracked for this account/slot/block combination. + pub(crate) fn storage_map_forest_with_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: BlockNumber, + ) -> Option<(&SmtForest, Word)> { + let root = self.storage_roots.get(&(account_id, slot_name.clone(), block_num))?; + Some((&self.storage_forest, *root)) + } + + /// Returns all key-value entries for a specific account storage slot at a block. + /// + /// Returns `None` if no entries are tracked for this account/slot/block combination. + pub(crate) fn storage_map_entries( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: BlockNumber, + ) -> Option> { + let entries = self.storage_entries.get(&(account_id, slot_name.clone(), block_num))?; + Some(entries.iter().map(|(k, v)| (*k, *v)).collect()) + } + // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- @@ -151,7 +183,7 @@ impl InnerForest { /// Updates the forest with storage map changes from a delta. /// /// Processes storage map slot deltas, building SMTs for each modified slot - /// and tracking the new roots. + /// and tracking the new roots and accumulated entries. fn update_account_storage( &mut self, block_num: BlockNumber, @@ -168,27 +200,49 @@ impl InnerForest { self.get_storage_root(account_id, slot_name, parent_block) }; - let entries: Vec<_> = + let delta_entries: Vec<_> = map_delta.entries().iter().map(|(key, value)| ((*key).into(), *value)).collect(); - if entries.is_empty() { + if delta_entries.is_empty() { continue; } let updated_root = self .storage_forest - .batch_insert(prev_root, entries.iter().copied()) + .batch_insert(prev_root, delta_entries.iter().copied()) .expect("forest insertion should succeed"); self.storage_roots .insert((account_id, slot_name.clone(), block_num), updated_root); + // Accumulate entries: start from parent block's entries or empty for full state + let mut accumulated_entries = if is_full_state { + BTreeMap::new() + } else { + self.storage_entries + .get(&(account_id, slot_name.clone(), parent_block)) + .cloned() + .unwrap_or_default() + }; + + // Apply delta entries (insert or remove if value is EMPTY_WORD) + for (key, value) in delta_entries.iter() { + if *value == EMPTY_WORD { + accumulated_entries.remove(key); + } else { + accumulated_entries.insert(*key, *value); + } + } + + self.storage_entries + .insert((account_id, slot_name.clone(), block_num), accumulated_entries); + tracing::debug!( target: crate::COMPONENT, %account_id, %block_num, ?slot_name, - entries = entries.len(), + delta_entries = delta_entries.len(), "Updated storage map in forest" ); } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 4bdc1146e..59f797fa1 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -18,13 +18,14 @@ use miden_node_proto::domain::account::{ AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, + SlotData, StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_objects::account::delta::AccountUpdateDetails; -use miden_objects::account::{AccountId, StorageSlotContent}; +use miden_objects::account::AccountId; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_objects::block::{ @@ -51,7 +52,7 @@ use miden_objects::crypto::merkle::{ use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; use miden_objects::utils::Serializable; -use miden_objects::{AccountError, Word}; +use miden_objects::Word; use tokio::sync::{Mutex, RwLock, oneshot}; use tracing::{info, info_span, instrument}; @@ -1134,6 +1135,10 @@ impl State { /// /// This method queries the database to fetch the account state and processes the detail /// request to return only the requested information. + /// + /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. + /// Returns an error if the forest doesn't have data for the requested slot. + /// All-entries queries (`SlotData::All`) use the database directly. async fn fetch_requested_account_details( &self, account_id: AccountId, @@ -1177,25 +1182,49 @@ impl State { _ => AccountVaultDetails::empty(), }; - // TODO: don't load the entire store at once, load what is required - let store = self.db.select_account_storage_at_block(account_id, block_num).await?; - let storage_header = store.to_header(); + // Load storage header from DB (map entries come from forest) + let storage_header = + self.db.select_account_storage_header_at_block(account_id, block_num).await?; let mut storage_map_details = Vec::::with_capacity(storage_requests.len()); - for StorageMapRequest { slot_name, slot_data } in storage_requests { - let Some(slot) = store.slots().iter().find(|s| s.name() == &slot_name) else { - continue; - }; + // Use forest for storage map queries + let forest_guard = self.forest.read().await; - let storage_map = match slot.content() { - StorageSlotContent::Map(map) => map, - StorageSlotContent::Value(_) => { - return Err(AccountError::StorageSlotNotMap(slot_name).into()); + for StorageMapRequest { slot_name, slot_data } in storage_requests { + let details = match &slot_data { + SlotData::MapKeys(keys) => { + // Use forest for specific key queries with proofs + let (forest, smt_root) = forest_guard + .storage_map_forest_with_root(account_id, &slot_name, block_num) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })?; + + AccountStorageMapDetails::from_specific_keys( + slot_name.clone(), + keys, + forest, + smt_root, + ) + .map_err(DatabaseError::MerkleError)? + }, + SlotData::All => { + // Use forest for all entries + let entries = forest_guard + .storage_map_entries(account_id, &slot_name, block_num) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })?; + + AccountStorageMapDetails::from_forest_entries(slot_name, entries) }, }; - let details = AccountStorageMapDetails::new(slot_name, slot_data, storage_map); storage_map_details.push(details); } diff --git a/proto/proto/rpc.proto b/proto/proto/rpc.proto index a7f9d1131..9166c2746 100644 --- a/proto/proto/rpc.proto +++ b/proto/proto/rpc.proto @@ -311,8 +311,22 @@ message AccountVaultDetails { // Account storage details for AccountProofResponse message AccountStorageDetails { message AccountStorageMapDetails { - // Wrapper for repeated storage map entries - message MapEntries { + // Wrapper for repeated storage map entries including their proofs. + // Used when specific keys are requested to enable client-side verification. + message MapEntriesWithProofs { + // Definition of individual storage entries including a proof. + message StorageMapEntryWithProof { + primitives.Digest key = 1; + primitives.Digest value = 2; + primitives.SmtOpening proof = 3; + } + + repeated StorageMapEntryWithProof entries = 1; + } + + // Wrapper for repeated storage map entries (without proofs). + // Used when all entries are requested for small maps. + message AllMapEntries { // Definition of individual storage entries. message StorageMapEntry { primitives.Digest key = 1; @@ -325,13 +339,18 @@ message AccountStorageDetails { // Storage slot name. string slot_name = 1; - // A flag that is set to `true` if the number of to-be-returned entries in the - // storage map would exceed a threshold. This indicates to the user that `SyncStorageMaps` - // endpoint should be used to get all storage map data. - bool too_many_entries = 2; + // Either the map entries (with or without proofs) or an indicator that the limit was exceeded. + // When `limit_exceeded` is set, clients should use the `SyncStorageMaps` endpoint. + oneof entries { + // All storage entries without proofs (for small maps or full requests). + AllMapEntries all_entries = 2; - // By default we provide all storage entries. - MapEntries entries = 3; + // Specific entries with their SMT proofs (for partial requests). + MapEntriesWithProofs entries_with_proofs = 3; + + // Set to true when the number of entries exceeds the response limit. + bool limit_exceeded = 4; + } } // Account storage header (storage slot info for up to 256 slots) From b2eaa55c4b36f61f2d8c670fca4f42284e7c5a98 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 22:27:19 +0100 Subject: [PATCH 08/14] unity --- crates/proto/src/domain/account.rs | 101 +++++++++++++++-------------- crates/proto/src/generated/rpc.rs | 19 +++--- proto/proto/rpc.proto | 14 ++-- 3 files changed, 70 insertions(+), 64 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index b7235554d..4c17b0553 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -202,52 +202,58 @@ impl TryFrom Entries as ProtoEntries, }; - let proto::rpc::account_storage_details::AccountStorageMapDetails { slot_name, entries } = - value; + let proto::rpc::account_storage_details::AccountStorageMapDetails { + slot_name, + limit_exceeded, + entries, + } = value; let slot_name = StorageSlotName::new(slot_name)?; - let map_entries = match entries { - Some(ProtoEntries::LimitExceeded(true)) | None => StorageMapEntries::LimitExceeded, - Some(ProtoEntries::LimitExceeded(false)) => StorageMapEntries::AllEntries(Vec::new()), - Some(ProtoEntries::AllEntries(AllMapEntries { entries })) => { - let map_entries = entries - .into_iter() - .map(|entry| { - let key = entry - .key - .ok_or(StorageMapEntry::missing_field(stringify!(key)))? - .try_into()?; - let value = entry - .value - .ok_or(StorageMapEntry::missing_field(stringify!(value)))? - .try_into()?; - Ok((key, value)) - }) - .collect::, ConversionError>>()?; - StorageMapEntries::AllEntries(map_entries) - }, - Some(ProtoEntries::EntriesWithProofs(MapEntriesWithProofs { entries })) => { - let proofs = entries - .into_iter() - .map(|entry| { - let _key: Word = entry - .key - .ok_or(StorageMapEntryWithProof::missing_field(stringify!(key)))? - .try_into()?; - let _value: Word = entry - .value - .ok_or(StorageMapEntryWithProof::missing_field(stringify!(value)))? - .try_into()?; - let smt_opening = entry - .proof - .ok_or(StorageMapEntryWithProof::missing_field(stringify!(proof)))?; - let smt_proof = SmtProof::try_from(smt_opening)?; - Ok(smt_proof) - }) - .collect::, ConversionError>>()?; - StorageMapEntries::EntriesWithProofs(proofs) - }, + let map_entries = if limit_exceeded { + StorageMapEntries::LimitExceeded + } else { + match entries { + None => StorageMapEntries::AllEntries(Vec::new()), + Some(ProtoEntries::AllEntries(AllMapEntries { entries })) => { + let map_entries = entries + .into_iter() + .map(|entry| { + let key = entry + .key + .ok_or(StorageMapEntry::missing_field(stringify!(key)))? + .try_into()?; + let value = entry + .value + .ok_or(StorageMapEntry::missing_field(stringify!(value)))? + .try_into()?; + Ok((key, value)) + }) + .collect::, ConversionError>>()?; + StorageMapEntries::AllEntries(map_entries) + }, + Some(ProtoEntries::EntriesWithProofs(MapEntriesWithProofs { entries })) => { + let proofs = entries + .into_iter() + .map(|entry| { + let _key: Word = entry + .key + .ok_or(StorageMapEntryWithProof::missing_field(stringify!(key)))? + .try_into()?; + let _value: Word = entry + .value + .ok_or(StorageMapEntryWithProof::missing_field(stringify!(value)))? + .try_into()?; + let smt_opening = entry.proof.ok_or( + StorageMapEntryWithProof::missing_field(stringify!(proof)), + )?; + let smt_proof = SmtProof::try_from(smt_opening)?; + Ok(smt_proof) + }) + .collect::, ConversionError>>()?; + StorageMapEntries::EntriesWithProofs(proofs) + }, + } }; Ok(Self { slot_name, entries: map_entries }) @@ -775,8 +781,8 @@ impl From let AccountStorageMapDetails { slot_name, entries } = value; - let proto_entries = match entries { - StorageMapEntries::LimitExceeded => Some(ProtoEntries::LimitExceeded(true)), + let (limit_exceeded, proto_entries) = match entries { + StorageMapEntries::LimitExceeded => (true, None), StorageMapEntries::AllEntries(map_entries) => { let all = AllMapEntries { entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { @@ -786,7 +792,7 @@ impl From } })), }; - Some(ProtoEntries::AllEntries(all)) + (false, Some(ProtoEntries::AllEntries(all))) }, StorageMapEntries::EntriesWithProofs(proofs) => { use miden_protocol::crypto::merkle::smt::SmtLeaf; @@ -812,12 +818,13 @@ impl From } })), }; - Some(ProtoEntries::EntriesWithProofs(with_proofs)) + (false, Some(ProtoEntries::EntriesWithProofs(with_proofs))) }, }; Self { slot_name: slot_name.to_string(), + limit_exceeded, entries: proto_entries, } } diff --git a/crates/proto/src/generated/rpc.rs b/crates/proto/src/generated/rpc.rs index 5c2c8109c..f0f5a32a4 100644 --- a/crates/proto/src/generated/rpc.rs +++ b/crates/proto/src/generated/rpc.rs @@ -233,9 +233,12 @@ pub mod account_storage_details { /// Storage slot name. #[prost(string, tag = "1")] pub slot_name: ::prost::alloc::string::String, - /// Either the map entries (with or without proofs) or an indicator that the limit was exceeded. - /// When `limit_exceeded` is set, clients should use the `SyncStorageMaps` endpoint. - #[prost(oneof = "account_storage_map_details::Entries", tags = "2, 3, 4")] + /// True when the number of entries exceeds the response limit. + /// When set, clients should use the `SyncStorageMaps` endpoint. + #[prost(bool, tag = "2")] + pub limit_exceeded: bool, + /// The map entries (with or without proofs). Empty when limit_exceeded is true. + #[prost(oneof = "account_storage_map_details::Entries", tags = "3, 4")] pub entries: ::core::option::Option, } /// Nested message and enum types in `AccountStorageMapDetails`. @@ -290,19 +293,15 @@ pub mod account_storage_details { >, } } - /// Either the map entries (with or without proofs) or an indicator that the limit was exceeded. - /// When `limit_exceeded` is set, clients should use the `SyncStorageMaps` endpoint. + /// The map entries (with or without proofs). Empty when limit_exceeded is true. #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Entries { /// All storage entries without proofs (for small maps or full requests). - #[prost(message, tag = "2")] + #[prost(message, tag = "3")] AllEntries(AllMapEntries), /// Specific entries with their SMT proofs (for partial requests). - #[prost(message, tag = "3")] + #[prost(message, tag = "4")] EntriesWithProofs(MapEntriesWithProofs), - /// Set to true when the number of entries exceeds the response limit. - #[prost(bool, tag = "4")] - LimitExceeded(bool), } } } diff --git a/proto/proto/rpc.proto b/proto/proto/rpc.proto index 74a8b6be5..29380053f 100644 --- a/proto/proto/rpc.proto +++ b/proto/proto/rpc.proto @@ -346,17 +346,17 @@ message AccountStorageDetails { // Storage slot name. string slot_name = 1; - // Either the map entries (with or without proofs) or an indicator that the limit was exceeded. - // When `limit_exceeded` is set, clients should use the `SyncStorageMaps` endpoint. + // True when the number of entries exceeds the response limit. + // When set, clients should use the `SyncStorageMaps` endpoint. + bool limit_exceeded = 2; + + // The map entries (with or without proofs). Empty when limit_exceeded is true. oneof entries { // All storage entries without proofs (for small maps or full requests). - AllMapEntries all_entries = 2; + AllMapEntries all_entries = 3; // Specific entries with their SMT proofs (for partial requests). - MapEntriesWithProofs entries_with_proofs = 3; - - // Set to true when the number of entries exceeds the response limit. - bool limit_exceeded = 4; + MapEntriesWithProofs entries_with_proofs = 4; } } From 68ed4d7781dbaa0ad668fd90c75829abc052c947 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 22:42:52 +0100 Subject: [PATCH 09/14] y --- crates/proto/src/domain/account.rs | 22 +++++++++++----------- crates/proto/src/generated/rpc.rs | 6 +++--- proto/proto/rpc.proto | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 4c17b0553..388dc6f92 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -204,19 +204,19 @@ impl TryFrom let proto::rpc::account_storage_details::AccountStorageMapDetails { slot_name, - limit_exceeded, + too_many_entries, entries, } = value; let slot_name = StorageSlotName::new(slot_name)?; - let map_entries = if limit_exceeded { + let entries = if too_many_entries { StorageMapEntries::LimitExceeded } else { match entries { None => StorageMapEntries::AllEntries(Vec::new()), Some(ProtoEntries::AllEntries(AllMapEntries { entries })) => { - let map_entries = entries + let entries = entries .into_iter() .map(|entry| { let key = entry @@ -230,7 +230,7 @@ impl TryFrom Ok((key, value)) }) .collect::, ConversionError>>()?; - StorageMapEntries::AllEntries(map_entries) + StorageMapEntries::AllEntries(entries) }, Some(ProtoEntries::EntriesWithProofs(MapEntriesWithProofs { entries })) => { let proofs = entries @@ -256,7 +256,7 @@ impl TryFrom } }; - Ok(Self { slot_name, entries: map_entries }) + Ok(Self { slot_name, entries }) } } @@ -520,10 +520,10 @@ impl AccountStorageMapDetails { entries: StorageMapEntries::LimitExceeded, } } else { - let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); + let entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); Self { slot_name, - entries: StorageMapEntries::AllEntries(map_entries), + entries: StorageMapEntries::AllEntries(entries), } } } @@ -781,11 +781,11 @@ impl From let AccountStorageMapDetails { slot_name, entries } = value; - let (limit_exceeded, proto_entries) = match entries { + let (too_many_entries, proto_entries) = match entries { StorageMapEntries::LimitExceeded => (true, None), - StorageMapEntries::AllEntries(map_entries) => { + StorageMapEntries::AllEntries(entries) => { let all = AllMapEntries { - entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { + entries: Vec::from_iter(entries.into_iter().map(|(key, value)| { proto::rpc::account_storage_details::account_storage_map_details::all_map_entries::StorageMapEntry { key: Some(key.into()), value: Some(value.into()), @@ -824,7 +824,7 @@ impl From Self { slot_name: slot_name.to_string(), - limit_exceeded, + too_many_entries, entries: proto_entries, } } diff --git a/crates/proto/src/generated/rpc.rs b/crates/proto/src/generated/rpc.rs index f0f5a32a4..755009e2c 100644 --- a/crates/proto/src/generated/rpc.rs +++ b/crates/proto/src/generated/rpc.rs @@ -236,8 +236,8 @@ pub mod account_storage_details { /// True when the number of entries exceeds the response limit. /// When set, clients should use the `SyncStorageMaps` endpoint. #[prost(bool, tag = "2")] - pub limit_exceeded: bool, - /// The map entries (with or without proofs). Empty when limit_exceeded is true. + pub too_many_entries: bool, + /// The map entries (with or without proofs). Empty when too_many_entries is true. #[prost(oneof = "account_storage_map_details::Entries", tags = "3, 4")] pub entries: ::core::option::Option, } @@ -293,7 +293,7 @@ pub mod account_storage_details { >, } } - /// The map entries (with or without proofs). Empty when limit_exceeded is true. + /// The map entries (with or without proofs). Empty when too_many_entries is true. #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Entries { /// All storage entries without proofs (for small maps or full requests). diff --git a/proto/proto/rpc.proto b/proto/proto/rpc.proto index 29380053f..2918af848 100644 --- a/proto/proto/rpc.proto +++ b/proto/proto/rpc.proto @@ -348,9 +348,9 @@ message AccountStorageDetails { // True when the number of entries exceeds the response limit. // When set, clients should use the `SyncStorageMaps` endpoint. - bool limit_exceeded = 2; + bool too_many_entries = 2; - // The map entries (with or without proofs). Empty when limit_exceeded is true. + // The map entries (with or without proofs). Empty when too_many_entries is true. oneof entries { // All storage entries without proofs (for small maps or full requests). AllMapEntries all_entries = 3; From bd2709c00faeaa4cfa0c2aa1caa4c16b7ed7edf0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 22:51:37 +0100 Subject: [PATCH 10/14] behaviour --- crates/proto/src/domain/account.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 388dc6f92..fc94fba33 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -214,7 +214,9 @@ impl TryFrom StorageMapEntries::LimitExceeded } else { match entries { - None => StorageMapEntries::AllEntries(Vec::new()), + None => { + return Err(proto::rpc::account_storage_details::AccountStorageMapDetails::missing_field(stringify!(entries))); + }, Some(ProtoEntries::AllEntries(AllMapEntries { entries })) => { let entries = entries .into_iter() From c64392cd0a1698ff28fd12856a0a0e6c075be94a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 02:50:05 +0100 Subject: [PATCH 11/14] lint clippy fmt --- crates/proto/src/domain/account.rs | 33 +++++++++++++++------------- crates/store/src/inner_forest/mod.rs | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index ffbdb0b8f..f370d6c90 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -1113,7 +1113,7 @@ mod tests { fn account_storage_map_details_from_forest_entries_limit_exceeded() { let slot_name = test_slot_name(); // Create more entries than MAX_RETURN_ENTRIES - let entries: Vec<_> = (0..AccountStorageMapDetails::MAX_RETURN_ENTRIES + 1) + let entries: Vec<_> = (0..=AccountStorageMapDetails::MAX_RETURN_ENTRIES) .map(|i| { let key = word_from_u32([i as u32, 0, 0, 0]); let value = word_from_u32([0, 0, 0, i as u32]); @@ -1133,7 +1133,7 @@ mod tests { // Create an SmtForest and populate it with some data let mut forest = SmtForest::new(); - let entries = vec![ + let entries = [ (word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0])), (word_from_u32([2, 0, 0, 0]), word_from_u32([20, 0, 0, 0])), (word_from_u32([3, 0, 0, 0]), word_from_u32([30, 0, 0, 0])), @@ -1167,21 +1167,22 @@ mod tests { SmtLeaf::Multiple(entries) => entries .iter() .find(|(k, _)| *k == expected_key) - .map(|(_, v)| *v) - .unwrap_or(miden_protocol::EMPTY_WORD), + .map_or(miden_protocol::EMPTY_WORD, |(_, v)| *v), _ => miden_protocol::EMPTY_WORD, } }; - let key1 = word_from_u32([1, 0, 0, 0]); - let key2 = word_from_u32([3, 0, 0, 0]); - let value1 = get_value(&proofs[0], key1); - let value2 = get_value(&proofs[1], key2); + let first_key = word_from_u32([1, 0, 0, 0]); + let second_key = word_from_u32([3, 0, 0, 0]); + let first_value = get_value(&proofs[0], first_key); + let second_value = get_value(&proofs[1], second_key); - assert_eq!(value1, word_from_u32([10, 0, 0, 0])); - assert_eq!(value2, word_from_u32([30, 0, 0, 0])); + assert_eq!(first_value, word_from_u32([10, 0, 0, 0])); + assert_eq!(second_value, word_from_u32([30, 0, 0, 0])); + }, + StorageMapEntries::LimitExceeded | StorageMapEntries::AllEntries(_) => { + panic!("Expected EntriesWithProofs") }, - _ => panic!("Expected EntriesWithProofs"), } } @@ -1191,7 +1192,7 @@ mod tests { // Create an SmtForest with one entry so the root is tracked let mut forest = SmtForest::new(); - let entries = vec![(word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0]))]; + let entries = [(word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0]))]; let smt_root = forest.batch_insert(empty_smt_root(), entries.iter().copied()).unwrap(); // Query a key that doesn't exist in the tree - should return a proof @@ -1212,7 +1213,9 @@ mod tests { assert_eq!(proofs.len(), 1); // The proof exists and can be used to verify non-membership }, - _ => panic!("Expected EntriesWithProofs"), + StorageMapEntries::LimitExceeded | StorageMapEntries::AllEntries(_) => { + panic!("Expected EntriesWithProofs") + }, } } @@ -1222,11 +1225,11 @@ mod tests { let mut forest = SmtForest::new(); // Create a forest with some data to get a valid root - let entries = vec![(word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0]))]; + let entries = [(word_from_u32([1, 0, 0, 0]), word_from_u32([10, 0, 0, 0]))]; let smt_root = forest.batch_insert(empty_smt_root(), entries.iter().copied()).unwrap(); // Create more keys than MAX_RETURN_ENTRIES - let keys: Vec<_> = (0..AccountStorageMapDetails::MAX_RETURN_ENTRIES + 1) + let keys: Vec<_> = (0..=AccountStorageMapDetails::MAX_RETURN_ENTRIES) .map(|i| word_from_u32([i as u32, 0, 0, 0])) .collect(); diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 32519a90e..d4e78be3c 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -298,7 +298,7 @@ impl InnerForest { }; // Apply delta entries (insert or remove if value is EMPTY_WORD) - for (key, value) in delta_entries.iter() { + for (key, value) in &delta_entries { if *value == EMPTY_WORD { accumulated_entries.remove(key); } else { From 6f823e45e70fa5c582b2c47f0c923ba100fa5507 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Sat, 10 Jan 2026 14:33:10 +0100 Subject: [PATCH 12/14] review --- crates/proto/src/domain/account.rs | 18 +++++++++ crates/store/src/db/mod.rs | 28 +++++++------- .../store/src/db/models/queries/accounts.rs | 15 +------- .../db/models/queries/accounts/at_block.rs | 38 ++++++++++++------- crates/store/src/inner_forest/mod.rs | 23 +++++++---- crates/store/src/state.rs | 30 +++++---------- 6 files changed, 84 insertions(+), 68 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index f370d6c90..e5039a582 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -591,6 +591,24 @@ impl AccountStorageMapDetails { entries: StorageMapEntries::EntriesWithProofs(proofs), }) } + + /// Creates storage map details from pre-computed SMT proofs. + /// + /// Use this when the caller has already obtained the proofs from an `SmtForest`. + /// Returns `LimitExceeded` if too many proofs are provided. + pub fn from_proofs(slot_name: StorageSlotName, proofs: Vec) -> Self { + if proofs.len() > Self::MAX_RETURN_ENTRIES { + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } + } else { + Self { + slot_name, + entries: StorageMapEntries::EntriesWithProofs(proofs), + } + } + } } #[derive(Debug, Clone, PartialEq)] diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 9453076a1..7597e89bc 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -435,19 +435,6 @@ impl Db { .await } - /// Queries just the storage header (slot types and roots) at a specific block. - #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_account_storage_header_at_block( - &self, - account_id: AccountId, - block_num: BlockNumber, - ) -> Result { - self.transact("Get account storage header at block", move |conn| { - queries::select_account_storage_header_at_block(conn, account_id, block_num) - }) - .await - } - /// Queries vault assets at a specific block #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_account_vault_at_block( @@ -488,6 +475,21 @@ impl Db { .await } + /// Queries the account header and storage header for a specific account at a block. + /// + /// Returns both in a single query to avoid querying the database twice. + /// Returns `None` if the account doesn't exist at that block. + pub async fn select_account_header_with_storage_header_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result> { + self.transact("Get account header with storage header at block", move |conn| { + queries::select_account_header_with_storage_header_at_block(conn, account_id, block_num) + }) + .await + } + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_state_sync( &self, diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index d709f2f10..17d633fc3 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -51,7 +51,7 @@ use crate::errors::DatabaseError; mod at_block; pub(crate) use at_block::{ select_account_header_at_block, - select_account_storage_at_block, + select_account_header_with_storage_header_at_block, select_account_vault_at_block, }; @@ -651,19 +651,6 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } -/// Returns account storage header (without map entries) at a given block. -/// -/// This reads the storage blob and extracts just the header information (slot types and roots), -/// avoiding the need to deserialize all map entries. -pub(crate) fn select_account_storage_header_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result { - let storage = select_account_storage_at_block(conn, account_id, block_num)?; - Ok(storage.to_header()) -} - /// Select latest account storage by querying `accounts.storage_header` where `is_latest=true` /// and reconstructing full storage from the header plus map values from /// `account_storage_map_values`. diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs index 021714abe..cda610c43 100644 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -57,11 +57,27 @@ struct AccountHeaderDataRaw { /// * `Ok(Some(AccountHeader))` - The account header if found /// * `Ok(None)` - If account doesn't exist at that block /// * `Err(DatabaseError)` - If there's a database error +#[allow(dead_code)] pub(crate) fn select_account_header_at_block( conn: &mut SqliteConnection, account_id: AccountId, block_num: BlockNumber, ) -> Result, DatabaseError> { + select_account_header_with_storage_header_at_block(conn, account_id, block_num) + .map(|opt| opt.map(|(header, _)| header)) +} + +/// Queries the account header and storage header for a specific account at a block. +/// +/// This reconstructs both `AccountHeader` and `AccountStorageHeader` in a single query, +/// avoiding the need to query the database twice when both are needed. +/// +/// Returns `None` if the account doesn't exist at that block. +pub(crate) fn select_account_header_with_storage_header_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { use schema::accounts; let account_id_bytes = account_id.to_bytes(); @@ -93,14 +109,13 @@ pub(crate) fn select_account_header_at_block( return Ok(None); }; - let storage_commitment = match storage_header_blob { - Some(blob) => { - let header = AccountStorageHeader::read_from_bytes(&blob)?; - header.to_commitment() - }, - None => Word::default(), + let storage_header = match &storage_header_blob { + Some(blob) => AccountStorageHeader::read_from_bytes(blob)?, + None => AccountStorageHeader::new(Vec::new())?, }; + let storage_commitment = storage_header.to_commitment(); + let code_commitment = code_commitment_bytes .map(|bytes| Word::read_from_bytes(&bytes)) .transpose()? @@ -113,13 +128,10 @@ pub(crate) fn select_account_header_at_block( .transpose()? .unwrap_or(Word::default()); - Ok(Some(AccountHeader::new( - account_id, - nonce, - vault_root, - storage_commitment, - code_commitment, - ))) + let account_header = + AccountHeader::new(account_id, nonce, vault_root, storage_commitment, code_commitment); + + Ok(Some((account_header, storage_header))) } // ACCOUNT VAULT diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index d4e78be3c..418f91e5f 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -1,11 +1,13 @@ use std::collections::BTreeMap; +use miden_node_proto::domain::account::AccountStorageMapDetails; use miden_protocol::account::delta::{AccountDelta, AccountStorageDelta, AccountVaultDelta}; use miden_protocol::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; use miden_protocol::asset::{Asset, FungibleAsset}; use miden_protocol::block::BlockNumber; +use miden_protocol::crypto::merkle::MerkleError; +use miden_protocol::crypto::merkle::smt::{SMT_DEPTH, SmtForest, SmtProof}; use miden_protocol::crypto::merkle::EmptySubtreeRoots; -use miden_protocol::crypto::merkle::smt::{SMT_DEPTH, SmtForest}; use miden_protocol::{EMPTY_WORD, Word}; #[cfg(test)] @@ -87,18 +89,23 @@ impl InnerForest { .map_or_else(Self::empty_smt_root, |(_, root)| *root) } - /// Returns the storage forest and the root for a specific account storage slot at a block. + /// Opens a storage map and returns storage map details with SMT proofs for the given keys. /// - /// This allows callers to query specific keys from the storage map using `SmtForest::open()`. /// Returns `None` if no storage root is tracked for this account/slot/block combination. - pub(crate) fn storage_map_forest_with_root( + /// Returns a `MerkleError` if the forest doesn't contain sufficient data for the proofs. + pub(crate) fn open_storage_map( &self, account_id: AccountId, - slot_name: &StorageSlotName, + slot_name: StorageSlotName, block_num: BlockNumber, - ) -> Option<(&SmtForest, Word)> { - let root = self.storage_roots.get(&(account_id, slot_name.clone(), block_num))?; - Some((&self.forest, *root)) + keys: &[Word], + ) -> Option> { + let root = *self.storage_roots.get(&(account_id, slot_name.clone(), block_num))?; + + let proofs: Result, MerkleError> = + keys.iter().map(|key| self.forest.open(root, *key)).collect(); + + Some(proofs.map(|p| AccountStorageMapDetails::from_proofs(slot_name, p))) } /// Returns all key-value entries for a specific account storage slot at a block. diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d4c549f3d..94f7d3047 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1027,11 +1027,12 @@ impl State { // Validate block exists in the blockchain before querying the database self.validate_block_exists(block_num).await?; - let account_header = - self.db - .select_account_header_at_block(account_id, block_num) - .await? - .ok_or(DatabaseError::AccountAtBlockHeightNotFoundInDb(account_id, block_num))?; + // Query account header and storage header together in a single DB call + let (account_header, storage_header) = self + .db + .select_account_header_with_storage_header_at_block(account_id, block_num) + .await? + .ok_or(DatabaseError::AccountAtBlockHeightNotFoundInDb(account_id, block_num))?; let account_code = match code_commitment { Some(commitment) if commitment == account_header.code_commitment() => None, @@ -1055,9 +1056,6 @@ impl State { None => AccountVaultDetails::empty(), }; - // Load storage header from DB (map entries come from forest) - let storage_header = - self.db.select_account_storage_header_at_block(account_id, block_num).await?; let mut storage_map_details = Vec::::with_capacity(storage_requests.len()); @@ -1067,22 +1065,14 @@ impl State { for StorageMapRequest { slot_name, slot_data } in storage_requests { let details = match &slot_data { SlotData::MapKeys(keys) => { - // Use forest for specific key queries with proofs - let (forest, smt_root) = forest_guard - .storage_map_forest_with_root(account_id, &slot_name, block_num) + forest_guard + .open_storage_map(account_id, slot_name.clone(), block_num, keys) .ok_or_else(|| DatabaseError::StorageRootNotFound { account_id, slot_name: slot_name.to_string(), block_num, - })?; - - AccountStorageMapDetails::from_specific_keys( - slot_name.clone(), - keys, - forest, - smt_root, - ) - .map_err(DatabaseError::MerkleError)? + })? + .map_err(DatabaseError::MerkleError)? }, SlotData::All => { // Use forest for all entries From 77eee021d2cccd54ce6ade967378cd3220423861 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Sat, 10 Jan 2026 14:43:16 +0100 Subject: [PATCH 13/14] remove dead code --- crates/store/src/db/mod.rs | 14 --- .../store/src/db/models/queries/accounts.rs | 1 - .../db/models/queries/accounts/at_block.rs | 117 +----------------- .../src/db/models/queries/accounts/tests.rs | 99 +++++++++++++-- 4 files changed, 93 insertions(+), 138 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 7597e89bc..bd171c773 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -461,20 +461,6 @@ impl Db { .await } - /// Queries the account header for a specific account at a specific block number. - /// - /// Returns `None` if the account doesn't exist at that block. - pub async fn select_account_header_at_block( - &self, - account_id: AccountId, - block_num: BlockNumber, - ) -> Result> { - self.transact("Get account header at block", move |conn| { - queries::select_account_header_at_block(conn, account_id, block_num) - }) - .await - } - /// Queries the account header and storage header for a specific account at a block. /// /// Returns both in a single query to avoid querying the database twice. diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 17d633fc3..e20d73154 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -50,7 +50,6 @@ use crate::errors::DatabaseError; mod at_block; pub(crate) use at_block::{ - select_account_header_at_block, select_account_header_with_storage_header_at_block, select_account_vault_at_block, }; diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs index cda610c43..13ed84177 100644 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use diesel::prelude::Queryable; use diesel::query_dsl::methods::SelectDsl; use diesel::{ @@ -10,16 +8,7 @@ use diesel::{ RunQueryDsl, SqliteConnection, }; -use miden_protocol::account::{ - AccountHeader, - AccountId, - AccountStorage, - AccountStorageHeader, - StorageMap, - StorageSlot, - StorageSlotName, - StorageSlotType, -}; +use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader}; use miden_protocol::asset::Asset; use miden_protocol::block::BlockNumber; use miden_protocol::utils::{Deserializable, Serializable}; @@ -54,25 +43,9 @@ struct AccountHeaderDataRaw { /// /// # Returns /// -/// * `Ok(Some(AccountHeader))` - The account header if found +/// * `Ok(Some((AccountHeader, AccountStorageHeader)))` - The headers if found /// * `Ok(None)` - If account doesn't exist at that block /// * `Err(DatabaseError)` - If there's a database error -#[allow(dead_code)] -pub(crate) fn select_account_header_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - select_account_header_with_storage_header_at_block(conn, account_id, block_num) - .map(|opt| opt.map(|(header, _)| header)) -} - -/// Queries the account header and storage header for a specific account at a block. -/// -/// This reconstructs both `AccountHeader` and `AccountStorageHeader` in a single query, -/// avoiding the need to query the database twice when both are needed. -/// -/// Returns `None` if the account doesn't exist at that block. pub(crate) fn select_account_header_with_storage_header_at_block( conn: &mut SqliteConnection, account_id: AccountId, @@ -192,89 +165,3 @@ pub(crate) fn select_account_vault_at_block( Ok(assets) } - -// ACCOUNT STORAGE -// ================================================================================================ - -/// Returns account storage at a given block by reading from `accounts.storage_header` -/// (which contains the `AccountStorageHeader`) and reconstructing full storage from -/// map values in `account_storage_map_values` table. -pub(crate) fn select_account_storage_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result { - use schema::account_storage_map_values as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Query storage header blob for this account at or before this block - let storage_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter(schema::accounts::account_id.eq(&account_id_bytes)) - .filter(schema::accounts::block_num.le(block_num_sql)) - .order(schema::accounts::block_num.desc()) - .limit(1) - .first(conn) - .optional()? - .flatten(); - - let Some(blob) = storage_blob else { - // No storage means empty storage - return Ok(AccountStorage::new(Vec::new())?); - }; - - // Deserialize the AccountStorageHeader from the blob - let header = AccountStorageHeader::read_from_bytes(&blob)?; - - // Query all map values for this account up to and including this block. - // For each (slot_name, key), we need the latest value at or before block_num. - // First, get all entries up to block_num - let map_values: Vec<(i64, String, Vec, Vec)> = - SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) - .order((t::slot_name.asc(), t::key.asc(), t::block_num.desc())) - .load(conn)?; - - // For each (slot_name, key) pair, keep only the latest entry (highest block_num) - let mut latest_map_entries: BTreeMap<(StorageSlotName, Word), Word> = BTreeMap::new(); - - for (_, slot_name_str, key_bytes, value_bytes) in map_values { - let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { - DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) - })?; - let key = Word::read_from_bytes(&key_bytes)?; - let value = Word::read_from_bytes(&value_bytes)?; - - // Only insert if we haven't seen this (slot_name, key) yet - // (since results are ordered by block_num desc, first one is latest) - latest_map_entries.entry((slot_name, key)).or_insert(value); - } - - // Group entries by slot name - let mut map_entries_by_slot: BTreeMap> = BTreeMap::new(); - for ((slot_name, key), value) in latest_map_entries { - map_entries_by_slot.entry(slot_name).or_default().push((key, value)); - } - - // Reconstruct StorageSlots from header slots + map entries - let mut slots = Vec::new(); - for slot_header in header.slots() { - let slot = match slot_header.slot_type() { - StorageSlotType::Value => { - // For value slots, the header value IS the slot value - StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) - }, - StorageSlotType::Map => { - // For map slots, reconstruct from map entries - let entries = map_entries_by_slot.remove(slot_header.name()).unwrap_or_default(); - let storage_map = StorageMap::with_entries(entries)?; - StorageSlot::with_map(slot_header.name().clone(), storage_map) - }, - }; - slots.push(slot); - } - - Ok(AccountStorage::new(slots)?) -} diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs index 6f9f5b075..9e574a9d4 100644 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -1,7 +1,9 @@ //! Tests for the `accounts` module, specifically for account storage and historical queries. +use std::collections::BTreeMap; + use diesel::query_dsl::methods::SelectDsl; -use diesel::{Connection, OptionalExtension, QueryDsl, RunQueryDsl}; +use diesel::{BoolExpressionMethods, Connection, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use diesel_migrations::MigrationHarness; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::PublicKeyCommitment; @@ -13,20 +15,27 @@ use miden_protocol::account::{ AccountDelta, AccountId, AccountIdVersion, + AccountStorage, + AccountStorageHeader, AccountStorageMode, AccountType, + StorageMap, StorageSlot, StorageSlotName, + StorageSlotType, }; use miden_protocol::block::{BlockAccountUpdate, BlockHeader, BlockNumber}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; -use miden_protocol::utils::Serializable; +use miden_protocol::utils::{Deserializable, Serializable}; use miden_protocol::{EMPTY_WORD, Felt, Word}; use miden_standards::account::auth::AuthRpoFalcon512; use miden_standards::code_builder::CodeBuilder; use super::*; use crate::db::migrations::MIGRATIONS; +use crate::db::models::conv::SqlTypeConvert; +use crate::db::schema; +use crate::errors::DatabaseError; fn setup_test_db() -> SqliteConnection { let mut conn = @@ -37,6 +46,80 @@ fn setup_test_db() -> SqliteConnection { conn } +/// Test helper: reconstructs account storage at a given block from DB. +/// +/// Reads `accounts.storage_header` and `account_storage_map_values` to reconstruct +/// the full `AccountStorage` at the specified block. +fn test_select_account_storage_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result { + use schema::account_storage_map_values as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Query storage header blob for this account at or before this block + let storage_blob: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter(schema::accounts::account_id.eq(&account_id_bytes)) + .filter(schema::accounts::block_num.le(block_num_sql)) + .order(schema::accounts::block_num.desc()) + .limit(1) + .first(conn) + .optional()? + .flatten(); + + let Some(blob) = storage_blob else { + return Ok(AccountStorage::new(Vec::new())?); + }; + + let header = AccountStorageHeader::read_from_bytes(&blob)?; + + // Query all map values for this account up to and including this block. + let map_values: Vec<(i64, String, Vec, Vec)> = + SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) + .order((t::slot_name.asc(), t::key.asc(), t::block_num.desc())) + .load(conn)?; + + // For each (slot_name, key) pair, keep only the latest entry + let mut latest_map_entries: BTreeMap<(StorageSlotName, Word), Word> = BTreeMap::new(); + for (_, slot_name_str, key_bytes, value_bytes) in map_values { + let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { + DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) + })?; + let key = Word::read_from_bytes(&key_bytes)?; + let value = Word::read_from_bytes(&value_bytes)?; + latest_map_entries.entry((slot_name, key)).or_insert(value); + } + + // Group entries by slot name + let mut map_entries_by_slot: BTreeMap> = BTreeMap::new(); + for ((slot_name, key), value) in latest_map_entries { + map_entries_by_slot.entry(slot_name).or_default().push((key, value)); + } + + // Reconstruct StorageSlots from header slots + map entries + let mut slots = Vec::new(); + for slot_header in header.slots() { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + let entries = map_entries_by_slot.remove(slot_header.name()).unwrap_or_default(); + let storage_map = StorageMap::with_entries(entries)?; + StorageSlot::with_map(slot_header.name().clone(), storage_map) + }, + }; + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) +} + fn create_test_account_with_storage() -> (Account, AccountId) { // Create a simple public account with one value storage slot let account_id = AccountId::dummy( @@ -112,7 +195,7 @@ fn test_select_account_header_at_block_returns_none_for_nonexistent() { ); // Query for a non-existent account - let result = select_account_header_at_block(&mut conn, account_id, block_num) + let result = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) .expect("Query should succeed"); assert!(result.is_none(), "Should return None for non-existent account"); @@ -138,7 +221,7 @@ fn test_select_account_header_at_block_returns_correct_header() { upsert_accounts(&mut conn, &[account_update], block_num).expect("upsert_accounts failed"); // Query the account header - let header = select_account_header_at_block(&mut conn, account_id, block_num) + let (header, _storage_header) = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) .expect("Query should succeed") .expect("Header should exist"); @@ -174,14 +257,14 @@ fn test_select_account_header_at_block_historical_query() { upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); // Query at block 1 - should return the account - let header_1 = select_account_header_at_block(&mut conn, account_id, block_num_1) + let (header_1, _) = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_1) .expect("Query should succeed") .expect("Header should exist at block 1"); assert_eq!(header_1.nonce(), nonce_1, "Nonce at block 1 should match"); // Query at block 2 - should return the same account (most recent before block 2) - let header_2 = select_account_header_at_block(&mut conn, account_id, block_num_2) + let (header_2, _) = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_2) .expect("Query should succeed") .expect("Header should exist at block 2"); @@ -242,7 +325,7 @@ fn test_select_account_storage_at_block_returns_storage() { upsert_accounts(&mut conn, &[account_update], block_num).expect("upsert_accounts failed"); // Query storage - let storage = select_account_storage_at_block(&mut conn, account_id, block_num) + let storage = test_select_account_storage_at_block(&mut conn, account_id, block_num) .expect("Query should succeed"); assert_eq!( @@ -396,7 +479,7 @@ fn test_upsert_accounts_updates_is_latest_flag() { ); // Verify historical query returns first update - let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + let storage_at_block_1 = test_select_account_storage_at_block(&mut conn, account_id, block_num_1) .expect("Failed to query storage at block 1"); assert_eq!( From f6cd4b3efcb123170882370ca2a2d3cdfa7c39d5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Sat, 10 Jan 2026 15:33:11 +0100 Subject: [PATCH 14/14] minor --- .../src/db/models/queries/accounts/tests.rs | 40 ++++++++++++------- crates/store/src/inner_forest/mod.rs | 20 +++++++--- crates/store/src/state.rs | 37 +++++++---------- 3 files changed, 55 insertions(+), 42 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs index 9e574a9d4..8532d448f 100644 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -3,7 +3,14 @@ use std::collections::BTreeMap; use diesel::query_dsl::methods::SelectDsl; -use diesel::{BoolExpressionMethods, Connection, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; +use diesel::{ + BoolExpressionMethods, + Connection, + ExpressionMethods, + OptionalExtension, + QueryDsl, + RunQueryDsl, +}; use diesel_migrations::MigrationHarness; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::PublicKeyCommitment; @@ -195,8 +202,9 @@ fn test_select_account_header_at_block_returns_none_for_nonexistent() { ); // Query for a non-existent account - let result = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) - .expect("Query should succeed"); + let result = + select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) + .expect("Query should succeed"); assert!(result.is_none(), "Should return None for non-existent account"); } @@ -221,9 +229,10 @@ fn test_select_account_header_at_block_returns_correct_header() { upsert_accounts(&mut conn, &[account_update], block_num).expect("upsert_accounts failed"); // Query the account header - let (header, _storage_header) = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) - .expect("Query should succeed") - .expect("Header should exist"); + let (header, _storage_header) = + select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) + .expect("Query should succeed") + .expect("Header should exist"); assert_eq!(header.id(), account_id, "Account ID should match"); assert_eq!(header.nonce(), account.nonce(), "Nonce should match"); @@ -257,16 +266,18 @@ fn test_select_account_header_at_block_historical_query() { upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); // Query at block 1 - should return the account - let (header_1, _) = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_1) - .expect("Query should succeed") - .expect("Header should exist at block 1"); + let (header_1, _) = + select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_1) + .expect("Query should succeed") + .expect("Header should exist at block 1"); assert_eq!(header_1.nonce(), nonce_1, "Nonce at block 1 should match"); // Query at block 2 - should return the same account (most recent before block 2) - let (header_2, _) = select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_2) - .expect("Query should succeed") - .expect("Header should exist at block 2"); + let (header_2, _) = + select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_2) + .expect("Query should succeed") + .expect("Header should exist at block 2"); assert_eq!(header_2.nonce(), nonce_1, "Nonce at block 2 should match block 1"); } @@ -479,8 +490,9 @@ fn test_upsert_accounts_updates_is_latest_flag() { ); // Verify historical query returns first update - let storage_at_block_1 = test_select_account_storage_at_block(&mut conn, account_id, block_num_1) - .expect("Failed to query storage at block 1"); + let storage_at_block_1 = + test_select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); assert_eq!( storage_at_block_1.to_commitment(), diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 418f91e5f..058e2c5d8 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -1,13 +1,12 @@ use std::collections::BTreeMap; -use miden_node_proto::domain::account::AccountStorageMapDetails; +use miden_node_proto::domain::account::{AccountStorageMapDetails, StorageMapEntries}; use miden_protocol::account::delta::{AccountDelta, AccountStorageDelta, AccountVaultDelta}; use miden_protocol::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; use miden_protocol::asset::{Asset, FungibleAsset}; use miden_protocol::block::BlockNumber; -use miden_protocol::crypto::merkle::MerkleError; use miden_protocol::crypto::merkle::smt::{SMT_DEPTH, SmtForest, SmtProof}; -use miden_protocol::crypto::merkle::EmptySubtreeRoots; +use miden_protocol::crypto::merkle::{EmptySubtreeRoots, MerkleError}; use miden_protocol::{EMPTY_WORD, Word}; #[cfg(test)] @@ -111,14 +110,23 @@ impl InnerForest { /// Returns all key-value entries for a specific account storage slot at a block. /// /// Returns `None` if no entries are tracked for this account/slot/block combination. + /// Returns an error if there are too many entries to return. pub(crate) fn storage_map_entries( &self, account_id: AccountId, - slot_name: &StorageSlotName, + slot_name: StorageSlotName, block_num: BlockNumber, - ) -> Option> { + ) -> Option { let entries = self.storage_entries.get(&(account_id, slot_name.clone(), block_num))?; - Some(entries.iter().map(|(k, v)| (*k, *v)).collect()) + if entries.len() > AccountStorageMapDetails::MAX_RETURN_ENTRIES { + return Some(AccountStorageMapDetails { + slot_name, + entries: StorageMapEntries::LimitExceeded, + }); + } + let entries = Vec::from_iter(entries.iter().map(|(k, v)| (*k, *v))); + + Some(AccountStorageMapDetails::from_forest_entries(slot_name.clone(), entries)) } // PUBLIC INTERFACE diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 94f7d3047..d4ff7c45f 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1064,28 +1064,21 @@ impl State { for StorageMapRequest { slot_name, slot_data } in storage_requests { let details = match &slot_data { - SlotData::MapKeys(keys) => { - forest_guard - .open_storage_map(account_id, slot_name.clone(), block_num, keys) - .ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })? - .map_err(DatabaseError::MerkleError)? - }, - SlotData::All => { - // Use forest for all entries - let entries = forest_guard - .storage_map_entries(account_id, &slot_name, block_num) - .ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })?; - - AccountStorageMapDetails::from_forest_entries(slot_name, entries) - }, + SlotData::MapKeys(keys) => forest_guard + .open_storage_map(account_id, slot_name.clone(), block_num, keys) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })? + .map_err(DatabaseError::MerkleError)?, + SlotData::All => forest_guard + .storage_map_entries(account_id, slot_name.clone(), block_num) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })?, }; storage_map_details.push(details);