feat: [2/4] integrate smtforest, avoid ser/de of full account/vault data in database

CHANGELOG.md

@@ -26,13 +26,14 @@
 - Remove `trait AccountTreeStorage` ([#1352](https://github.com/0xMiden/miden-node/issues/1352)).
 - [BREAKING] `SubmitProvenTransaction` now **requires** that the network's genesis commitment is set in the request's `ACCEPT` header ([#1298](https://github.com/0xMiden/miden-node/pull/1298), [#1436](https://github.com/0xMiden/miden-node/pull/1436)).
 - Add `S` generic to `NullifierTree` to allow usage with `LargeSmt`s ([#1353](https://github.com/0xMiden/miden-node/issues/1353)).
-- Removed internal errors from the `miden-network-monitor` ([#1424](https://github.com/0xMiden/miden-node/pull/1424)).
-- Track network transactions latency in `miden-network-monitor` ([#1430](https://github.com/0xMiden/miden-node/pull/1430)).
+- Refactor account table and introduce tracking forest ([#1394](https://github.com/0xMiden/miden-node/pull/1394)).
 - [BREAKING] Re-organized RPC protobuf schema to be independent of internal schema ([#1401](https://github.com/0xMiden/miden-node/pull/1401)).
-- Increased the maximum query limit for the store ([#1443](https://github.com/0xMiden/miden-node/pull/1443)).
+- Removed internal errors from the `miden-network-monitor` ([#1424](https://github.com/0xMiden/miden-node/pull/1424)).
 - [BREAKING] Added block signing capabilities to Validator component and updated gensis bootstrap to sign blocks with configured signer ([#1426](https://github.com/0xMiden/miden-node/pull/1426)).
+- Track network transactions latency in `miden-network-monitor` ([#1430](https://github.com/0xMiden/miden-node/pull/1430)).
 - Reduced default block interval from `5s` to `2s` ([#1438](https://github.com/0xMiden/miden-node/pull/1438)).
 - Increased retained account tree history from 33 to 100 blocks to account for the reduced block interval ([#1438](https://github.com/0xMiden/miden-node/pull/1438)).
+- Increased the maximum query limit for the store ([#1443](https://github.com/0xMiden/miden-node/pull/1443)).
 - [BREAKING] Migrated to version `v0.20` of the VM ([#1476](https://github.com/0xMiden/miden-node/pull/1476)).
 
 ### Fixes

crates/proto/src/domain/account.rs

@@ -1,7 +1,6 @@
 use std::fmt::{Debug, Display, Formatter};
 
 use miden_node_utils::formatting::format_opt;
-use miden_protocol::Word;
 use miden_protocol::account::{
     Account,
     AccountHeader,
@@ -18,6 +17,7 @@ use miden_protocol::block::account_tree::AccountWitness;
 use miden_protocol::crypto::merkle::SparseMerklePath;
 use miden_protocol::note::{NoteExecutionMode, NoteTag};
 use miden_protocol::utils::{Deserializable, DeserializationError, Serializable};
+use miden_protocol::{AssetError, Word};
 use thiserror::Error;
 
 use super::try_convert;
@@ -72,6 +72,7 @@ impl From<AccountId> for proto::account::AccountId {
 // ACCOUNT UPDATE
 // ================================================================================================
 
+// TODO should be called `AccountStateRef` or so
 #[derive(Debug, PartialEq)]
 pub struct AccountSummary {
     pub account_id: AccountId,
@@ -99,7 +100,7 @@ impl From<&AccountInfo> for proto::account::AccountDetails {
     fn from(AccountInfo { summary, details }: &AccountInfo) -> Self {
         Self {
             summary: Some(summary.into()),
-            details: details.as_ref().map(miden_protocol::utils::Serializable::to_bytes),
+            details: details.as_ref().map(Serializable::to_bytes),
         }
     }
 }
@@ -192,6 +193,7 @@ impl TryFrom<proto::rpc::account_storage_details::AccountStorageMapDetails>
     fn try_from(
         value: proto::rpc::account_storage_details::AccountStorageMapDetails,
     ) -> Result<Self, Self::Error> {
+        use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry;
         let proto::rpc::account_storage_details::AccountStorageMapDetails {
             slot_name,
             too_many_entries,
@@ -200,32 +202,32 @@ impl TryFrom<proto::rpc::account_storage_details::AccountStorageMapDetails>
 
         let slot_name = StorageSlotName::new(slot_name)?;
 
-        // Extract map_entries from the MapEntries message
-        let map_entries = if let Some(entries) = entries {
-            entries
-                .entries
-                .into_iter()
-                .map(|entry| {
-                    let key = entry
-                        .key
-                        .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field(
-                            stringify!(key),
-                        ))?
-                        .try_into()?;
-                    let value = entry
-                        .value
-                        .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field(
-                            stringify!(value),
-                        ))?
-                        .try_into()?;
-                    Ok((key, value))
-                })
-                .collect::<Result<Vec<_>, ConversionError>>()?
+        let entries = if too_many_entries {
+            StorageMapEntries::LimitExceeded
         } else {
-            Vec::new()
+            let map_entries = if let Some(entries) = entries {
+                entries
+                    .entries
+                    .into_iter()
+                    .map(|entry| {
+                        let key = entry
+                            .key
+                            .ok_or(StorageMapEntry::missing_field(stringify!(key)))?
+                            .try_into()?;
+                        let value = entry
+                            .value
+                            .ok_or(StorageMapEntry::missing_field(stringify!(value)))?
+                            .try_into()?;
+                        Ok((key, value))
+                    })
+                    .collect::<Result<Vec<_>, ConversionError>>()?
+            } else {
+                Vec::new()
+            };
+            StorageMapEntries::Entries(map_entries)
         };
 
-        Ok(Self { slot_name, too_many_entries, map_entries })
+        Ok(Self { slot_name, entries })
     }
 }
 
@@ -346,37 +348,64 @@ impl From<AccountStorageHeader> for proto::account::AccountStorageHeader {
     }
 }
 
+/// Account vault details
+///
+/// When an account contains a large number of assets (>
+/// [`AccountVaultDetails::MAX_RETURN_ENTRIES`]), including all assets in a single RPC response
+/// creates performance issues. In such cases, the `LimitExceeded` variant indicates to the client
+/// to use the `SyncAccountVault` endpoint instead.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub struct AccountVaultDetails {
-    pub too_many_assets: bool,
-    pub assets: Vec<Asset>,
+pub enum AccountVaultDetails {
+    /// The vault has too many assets to return inline.
+    /// Clients must use `SyncAccountVault` endpoint instead.
+    LimitExceeded,
+
+    /// The assets in the vault (up to `MAX_RETURN_ENTRIES`).
+    Assets(Vec<Asset>),
 }
+
 impl AccountVaultDetails {
-    const MAX_RETURN_ENTRIES: usize = 1000;
+    /// Maximum number of vault entries that can be returned in a single response.
+    /// Accounts with more assets will have `LimitExceeded` variant.
+    pub const MAX_RETURN_ENTRIES: usize = 1000;
 
     pub fn new(vault: &AssetVault) -> Self {
         if vault.assets().nth(Self::MAX_RETURN_ENTRIES).is_some() {
-            Self::too_many()
+            Self::LimitExceeded
         } else {
-            Self {
-                too_many_assets: false,
-                assets: Vec::from_iter(vault.assets()),
-            }
+            Self::Assets(Vec::from_iter(vault.assets()))
         }
     }
 
     pub fn empty() -> Self {
-        Self {
-            too_many_assets: false,
-            assets: Vec::new(),
+        Self::Assets(Vec::new())
+    }
+
+    /// Creates `AccountVaultDetails` from a list of assets.
+    pub fn from_assets(assets: Vec<Asset>) -> Self {
+        if assets.len() > Self::MAX_RETURN_ENTRIES {
+            Self::LimitExceeded
+        } else {
+            Self::Assets(assets)
         }
     }
 
-    fn too_many() -> Self {
-        Self {
-            too_many_assets: true,
-            assets: Vec::new(),
+    /// Creates `AccountVaultDetails` from vault entries (key-value pairs).
+    ///
+    /// This is useful when entries have been fetched directly from the database
+    /// rather than extracted from an `AssetVault`.
+    ///
+    /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation.
+    pub fn from_entries(entries: Vec<(Word, Word)>) -> Result<Self, AssetError> {
+        if entries.len() > Self::MAX_RETURN_ENTRIES {
+            return Ok(Self::LimitExceeded);
         }
+
+        let assets = Result::<Vec<_>, _>::from_iter(
+            entries.into_iter().map(|(_key, asset_word)| Asset::try_from(asset_word)),
+        )?;
+
+        Ok(Self::Assets(assets))
     }
 }
 
@@ -386,40 +415,66 @@ impl TryFrom<proto::rpc::AccountVaultDetails> for AccountVaultDetails {
     fn try_from(value: proto::rpc::AccountVaultDetails) -> Result<Self, Self::Error> {
         let proto::rpc::AccountVaultDetails { too_many_assets, assets } = value;
 
-        let assets =
-            Result::<Vec<_>, ConversionError>::from_iter(assets.into_iter().map(|asset| {
-                let asset = asset
-                    .asset
-                    .ok_or(proto::primitives::Asset::missing_field(stringify!(asset)))?;
-                let asset = Word::try_from(asset)?;
-                Asset::try_from(asset).map_err(ConversionError::AssetError)
-            }))?;
-        Ok(Self { too_many_assets, assets })
+        if too_many_assets {
+            Ok(Self::LimitExceeded)
+        } else {
+            let parsed_assets =
+                Result::<Vec<_>, ConversionError>::from_iter(assets.into_iter().map(|asset| {
+                    let asset = asset
+                        .asset
+                        .ok_or(proto::primitives::Asset::missing_field(stringify!(asset)))?;
+                    let asset = Word::try_from(asset)?;
+                    Asset::try_from(asset).map_err(ConversionError::AssetError)
+                }))?;
+            Ok(Self::Assets(parsed_assets))
+        }
     }
 }
 
 impl From<AccountVaultDetails> for proto::rpc::AccountVaultDetails {
     fn from(value: AccountVaultDetails) -> Self {
-        let AccountVaultDetails { too_many_assets, assets } = value;
-
-        Self {
-            too_many_assets,
-            assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset {
-                asset: Some(proto::primitives::Digest::from(Word::from(asset))),
-            })),
+        match value {
+            AccountVaultDetails::LimitExceeded => Self {
+                too_many_assets: true,
+                assets: Vec::new(),
+            },
+            AccountVaultDetails::Assets(assets) => Self {
+                too_many_assets: false,
+                assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset {
+                    asset: Some(proto::primitives::Digest::from(Word::from(asset))),
+                })),
+            },
         }
     }
 }
 
+/// Storage map entries for an account storage slot.
+///
+/// When a storage map contains many entries (> [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`]),
+/// returning all entries in a single RPC response creates performance issues. In such cases,
+/// the `LimitExceeded` variant indicates to the client to use the `SyncStorageMaps` endpoint
+/// instead.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum StorageMapEntries {
+    /// The map has too many entries to return inline.
+    /// Clients must use `SyncStorageMaps` endpoint instead.
+    LimitExceeded,
+
+    /// The storage map entries (key-value pairs), up to `MAX_RETURN_ENTRIES`.
+    /// TODO: For partial responses, also include Merkle proofs and inner SMT nodes.
+    Entries(Vec<(Word, Word)>),
+}
+
+/// Details about an account storage map slot.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct AccountStorageMapDetails {
     pub slot_name: StorageSlotName,
-    pub too_many_entries: bool,
-    pub map_entries: Vec<(Word, Word)>,
+    pub entries: StorageMapEntries,
 }
 
 impl AccountStorageMapDetails {
-    const MAX_RETURN_ENTRIES: usize = 1000;
+    /// Maximum number of storage map entries that can be returned in a single response.
+    pub const MAX_RETURN_ENTRIES: usize = 1000;
 
     pub fn new(slot_name: StorageSlotName, slot_data: SlotData, storage_map: &StorageMap) -> Self {
         match slot_data {
@@ -430,13 +485,15 @@ impl AccountStorageMapDetails {
 
     fn from_all_entries(slot_name: StorageSlotName, storage_map: &StorageMap) -> Self {
         if storage_map.num_entries() > Self::MAX_RETURN_ENTRIES {
-            Self::too_many_entries(slot_name)
+            Self {
+                slot_name,
+                entries: StorageMapEntries::LimitExceeded,
+            }
         } else {
             let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v)));
             Self {
                 slot_name,
-                too_many_entries: false,
-                map_entries,
+                entries: StorageMapEntries::Entries(map_entries),
             }
         }
     }
@@ -447,20 +504,15 @@ impl AccountStorageMapDetails {
         storage_map: &StorageMap,
     ) -> Self {
         if keys.len() > Self::MAX_RETURN_ENTRIES {
-            Self::too_many_entries(slot_name)
+            Self {
+                slot_name,
+                entries: StorageMapEntries::LimitExceeded,
+            }
         } else {
             // TODO For now, we return all entries instead of specific keys with proofs
             Self::from_all_entries(slot_name, storage_map)
         }
     }
-
-    pub fn too_many_entries(slot_name: StorageSlotName) -> Self {
-        Self {
-            slot_name,
-            too_many_entries: true,
-            map_entries: Vec::new(),
-        }
-    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -498,16 +550,16 @@ impl From<AccountStorageDetails> for proto::rpc::AccountStorageDetails {
 
 const fn storage_slot_type_from_raw(slot_type: u32) -> Result<StorageSlotType, ConversionError> {
     Ok(match slot_type {
-        0 => StorageSlotType::Map,
-        1 => StorageSlotType::Value,
+        0 => StorageSlotType::Value,
+        1 => StorageSlotType::Map,
         _ => return Err(ConversionError::EnumDiscriminantOutOfRange),
     })
 }
 
 const fn storage_slot_type_to_raw(slot_type: StorageSlotType) -> u32 {
     match slot_type {
-        StorageSlotType::Map => 0,
-        StorageSlotType::Value => 1,
+        StorageSlotType::Value => 0,
+        StorageSlotType::Map => 1,
     }
 }
 
@@ -628,21 +680,30 @@ impl From<AccountStorageMapDetails>
     fn from(value: AccountStorageMapDetails) -> Self {
         use proto::rpc::account_storage_details::account_storage_map_details;
 
-        let AccountStorageMapDetails { slot_name, too_many_entries, map_entries } = value;
+        let AccountStorageMapDetails { slot_name, entries } = value;
 
-        let entries = Some(account_storage_map_details::MapEntries {
-            entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| {
-                account_storage_map_details::map_entries::StorageMapEntry {
-                    key: Some(key.into()),
-                    value: Some(value.into()),
+        match entries {
+            StorageMapEntries::LimitExceeded => Self {
+                slot_name: slot_name.to_string(),
+                too_many_entries: true,
+                entries: Some(account_storage_map_details::MapEntries { entries: Vec::new() }),
+            },
+            StorageMapEntries::Entries(map_entries) => {
+                let entries = Some(account_storage_map_details::MapEntries {
+                    entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| {
+                        account_storage_map_details::map_entries::StorageMapEntry {
+                            key: Some(key.into()),
+                            value: Some(value.into()),
+                        }
+                    })),
+                });
+
+                Self {
+                    slot_name: slot_name.to_string(),
+                    too_many_entries: false,
+                    entries,
                 }
-            })),
-        });
-
-        Self {
-            slot_name: slot_name.to_string(),
-            too_many_entries,
-            entries,
+            },
         }
     }
 }