feat: [2/4] integrate smtforest, avoid ser/de of full account/vault data in database

crates/proto/src/domain/account.rs

@@ -69,6 +69,7 @@ impl From<AccountId> for proto::account::AccountId {
 // ACCOUNT UPDATE
 // ================================================================================================
 
+// TODO should be called `AccountStateRef` or so
 #[derive(Debug, PartialEq)]
 pub struct AccountSummary {
     pub account_id: AccountId,
@@ -86,6 +87,7 @@ impl From<&AccountSummary> for proto::account::AccountSummary {
     }
 }
 
+// TODO #[deprecated(note = "avoid this type, details will be `None` always!")]
 #[derive(Debug, PartialEq)]
 pub struct AccountInfo {
     pub summary: AccountSummary,
@@ -345,12 +347,31 @@ impl From<AccountStorageHeader> for proto::account::AccountStorageHeader {
     }
 }
 
+/// Account vault assets
+///
+/// Represents a list of assets, if the number of assets is reasonably small, which
+/// is currently set to 1000 for no particular reason.
+///
+/// When an account contains a large number of assets, including all assets
+/// in a single RPC response would create performance issues on client and server as
+/// and consume quite a bit of bandwidth, besides requiring additional memory on
+/// possibly low powered clients.
+///
+/// Hence `too_many_assets` is returned, which is indicating to the client to use the dedicated
+/// `SyncAccountVault` RPC endpoint and do incremental retrieval
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct AccountVaultDetails {
+    /// Flag indicating whether the vault has too many assets to return inline.
+    /// If `true`, clients must use `SyncAccountVault` endpoint instead.
     pub too_many_assets: bool,
+
+    /// The assets in the vault. Empty if `too_many_assets` is `true`.
     pub assets: Vec<Asset>,
 }
+
 impl AccountVaultDetails {
+    /// Maximum number of vault entries that can be returned in a single response.
+    /// Accounts with more assets will have `too_many_assets = true` and empty `assets`.
     const MAX_RETURN_ENTRIES: usize = 1000;
 
     pub fn new(vault: &AssetVault) -> Self {
@@ -371,6 +392,27 @@ impl AccountVaultDetails {
         }
     }
 
+    /// Creates `AccountVaultDetails` from vault entries (key-value pairs).
+    ///
+    /// This is useful when entries have been fetched directly from the database
+    /// rather than extracted from an `AssetVault`.
+    ///
+    /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation.
+    pub fn from_entries(entries: Vec<(Word, Word)>) -> Result<Self, miden_objects::AssetError> {
+        let too_many_assets = entries.len() > Self::MAX_RETURN_ENTRIES;
+
+        if too_many_assets {
+            return Ok(Self::too_many());
+        }
+
+        let assets = entries
+            .into_iter()
+            .map(|(_key, asset_word)| Asset::try_from(asset_word))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(Self { too_many_assets: false, assets })
+    }
+
     fn too_many() -> Self {
         Self {
             too_many_assets: true,
@@ -410,20 +452,42 @@ impl From<AccountVaultDetails> for proto::rpc::AccountVaultDetails {
     }
 }
 
+/// Details about an account storage map slot, including overflow handling.
+///
+/// ## Rationale for "Too Many Entries" Flag
+///
+/// Similar to `AccountVaultDetails`, when a storage map contains many entries (> 1000),
+/// returning all entries in a single RPC response creates performance issues:
+/// - Large serialization/deserialization costs
+/// - Network bandwidth saturation
+/// - Client memory pressure
+///
+/// When `too_many_entries` is `true`:
+/// - The `map_entries` field is empty (no data included)
+/// - Clients should use the dedicated `SyncStorageMaps` RPC endpoint
+/// - That endpoint supports pagination and block range filtering
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct AccountStorageMapDetails {
     pub slot_index: u8,
+
+    /// Flag indicating whether the storage map has too many entries to return inline.
+    /// If `true`, clients must use `SyncStorageMaps` endpoint instead.
     pub too_many_entries: bool,
+
+    /// The storage map entries (key-value pairs). Empty if `too_many_entries` is `true`.
+    /// TODO: For partial responses, also include Merkle proofs and inner SMT nodes.
     pub map_entries: Vec<(Word, Word)>,
 }
 
 impl AccountStorageMapDetails {
-    const MAX_RETURN_ENTRIES: usize = 1000;
+    /// Maximum number of storage map entries that can be returned in a single response.
+    /// Maps with more entries will have `too_many_entries = true` and empty `map_entries`.
+    pub const MAX_RETURN_ENTRIES: usize = 1000;
 
     pub fn new(slot_index: u8, slot_data: SlotData, storage_map: &StorageMap) -> Self {
         match slot_data {
             SlotData::All => Self::from_all_entries(slot_index, storage_map),
-            SlotData::MapKeys(keys) => Self::from_specific_keys(slot_index, &keys[..], storage_map),
+            SlotData::MapKeys(_keys) => Self::from_all_entries(slot_index, storage_map), /* TODO use from_specific_keys */
         }
     }
 
@@ -440,12 +504,17 @@ impl AccountStorageMapDetails {
         }
     }
 
-    fn from_specific_keys(slot_index: u8, keys: &[Word], storage_map: &StorageMap) -> Self {
-        if keys.len() > Self::MAX_RETURN_ENTRIES {
-            Self::too_many_entries(slot_index)
-        } else {
-            // TODO For now, we return all entries instead of specific keys with proofs
-            Self::from_all_entries(slot_index, storage_map)
+    /// Creates an `AccountStorageMapDetails` from already-queried entries (e.g., from database).
+    /// This is useful when entries have been fetched directly rather than extracted from a
+    /// `StorageMap`.
+    pub fn from_entries(slot_index: u8, map_entries: Vec<(Word, Word)>) -> Self {
+        let too_many_entries = map_entries.len() > Self::MAX_RETURN_ENTRIES;
+        let map_entries = if too_many_entries { Vec::new() } else { map_entries };
+
+        Self {
+            slot_index,
+            too_many_entries,
+            map_entries,
         }
     }
 

crates/store/src/db/migrations/2025062000000_setup/up.sql

@@ -18,17 +18,16 @@ CREATE TABLE accounts (
     block_num                               INTEGER NOT NULL,
     account_commitment                      BLOB NOT NULL,
     code_commitment                         BLOB,
-    storage                                 BLOB,
-    vault                                   BLOB,
     nonce                                   INTEGER,
+    storage_header                          BLOB, -- Serialized AccountStorage from miden-objects
     is_latest                               BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id
 
     PRIMARY KEY (account_id, block_num),
     CONSTRAINT all_null_or_none_null CHECK
         (
-            (code_commitment IS NOT NULL AND storage IS NOT NULL AND vault IS NOT NULL AND nonce IS NOT NULL)
+            (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL)
             OR
-            (code_commitment IS NULL AND storage IS NULL AND vault IS NULL AND nonce IS NULL)
+            (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL)
         )
 ) WITHOUT ROWID;
 
@@ -122,6 +121,22 @@ CREATE INDEX idx_vault_assets_account_block ON account_vault_assets(account_id,
 -- Index for querying latest assets
 CREATE INDEX idx_vault_assets_latest ON account_vault_assets(account_id, is_latest) WHERE is_latest = 1;
 
+-- Table to store vault headers (vault root commitments)
+CREATE TABLE account_vault_headers (
+    account_id          BLOB NOT NULL,
+    block_num           INTEGER NOT NULL,
+    vault_root          BLOB    NOT NULL,
+    is_latest           BOOLEAN NOT NULL DEFAULT 0,
+
+    PRIMARY KEY (account_id, block_num),
+    FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE
+) WITHOUT ROWID;
+
+-- Index for joining with accounts table
+CREATE INDEX idx_account_vault_headers_account_block ON account_vault_headers(account_id, block_num);
+-- Index for querying latest state
+CREATE INDEX idx_account_vault_headers_latest ON account_vault_headers(account_id, is_latest) WHERE is_latest = 1;
+
 CREATE TABLE nullifiers (
     nullifier        BLOB    NOT NULL,
     nullifier_prefix INTEGER NOT NULL,

crates/store/src/db/mod.rs

@@ -392,7 +392,7 @@ impl Db {
         .await
     }
 
-    /// Loads all the account commitments from the DB.
+    /// TODO marked for removal, replace with paged version
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
     pub async fn select_all_account_commitments(&self) -> Result<Vec<(AccountId, Word)>> {
         self.transact("read all account commitments", move |conn| {
@@ -408,27 +408,86 @@ impl Db {
             .await
     }
 
-    /// Loads account details at a specific block number from the DB.
+    /// Loads public account details from the DB based on the account ID's prefix.
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
-    pub async fn select_historical_account_at(
+    pub async fn select_network_account_by_prefix(
         &self,
-        id: AccountId,
+        id_prefix: u32,
+    ) -> Result<Option<AccountInfo>> {
+        self.transact("Get account by id prefix", move |conn| {
+            queries::select_account_by_id_prefix(conn, id_prefix)
+        })
+        .await
+    }
+
+    /// Reconstructs account storage at a specific block from the database
+    ///
+    /// This method queries the decomposed storage tables and reconstructs the full
+    /// `AccountStorage` with SMT backing for Map slots.
+    // TODO split querying the header from the content
+    #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
+    pub async fn select_account_storage_at_block(
+        &self,
+        account_id: AccountId,
         block_num: BlockNumber,
-    ) -> Result<AccountInfo> {
-        self.transact("Get historical account details", move |conn| {
-            queries::select_historical_account_at(conn, id, block_num)
+    ) -> Result<miden_objects::account::AccountStorage> {
+        self.transact("Get account storage at block", move |conn| {
+            queries::select_account_storage_at_block(conn, account_id, block_num)
         })
         .await
     }
 
-    /// Loads public account details from the DB based on the account ID's prefix.
+    /// Gets the latest account storage from the database
+    ///
+    /// Uses the `is_latest` flag for efficient querying.
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
-    pub async fn select_network_account_by_prefix(
+    pub async fn select_latest_account_storage(
         &self,
-        id_prefix: u32,
-    ) -> Result<Option<AccountInfo>> {
-        self.transact("Get account by id prefix", move |conn| {
-            queries::select_account_by_id_prefix(conn, id_prefix)
+        account_id: AccountId,
+    ) -> Result<miden_objects::account::AccountStorage> {
+        self.transact("Get latest account storage", move |conn| {
+            queries::select_latest_account_storage(conn, account_id)
+        })
+        .await
+    }
+
+    /// Queries vault assets at a specific block
+    #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
+    pub async fn select_account_vault_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<Vec<(Word, Word)>> {
+        self.transact("Get account vault at block", move |conn| {
+            queries::select_account_vault_at_block(conn, account_id, block_num)
+        })
+        .await
+    }
+
+    /// Queries the account code for a specific account at a specific block number.
+    ///
+    /// Returns `None` if the account doesn't exist at that block or has no code.
+    pub async fn select_account_code_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<Option<Vec<u8>>> {
+        self.transact("Get account code at block", move |conn| {
+            queries::select_account_code_at_block(conn, account_id, block_num)
+        })
+        .await
+    }
+
+    /// Queries the account header for a specific account at a specific block number.
+    ///
+    /// Returns `None` if the account doesn't exist at that block.
+    pub async fn select_account_header_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<Option<miden_objects::account::AccountHeader>> {
+        self.transact("Get account header at block", move |conn| {
+            queries::select_account_header_at_block(conn, account_id, block_num)
         })
         .await
     }
@@ -531,7 +590,7 @@ impl Db {
         .await
     }
 
-    /// Selects storage map values for syncing storage maps for a specific account ID.
+    /// Selects storage map values for syncing storage maps for a specific account ID
     ///
     /// The returned values are the latest known values up to `block_range.end()`, and no values
     /// earlier than `block_range.start()` are returned.

crates/store/src/db/models/conv.rs

@@ -36,6 +36,7 @@ use std::any::type_name;
 
 use miden_node_proto::domain::account::{NetworkAccountError, NetworkAccountPrefix};
 use miden_objects::Felt;
+use miden_objects::account::StorageSlotType;
 use miden_objects::block::BlockNumber;
 use miden_objects::note::{NoteExecutionMode, NoteTag};
 
@@ -116,6 +117,28 @@ impl SqlTypeConvert for NoteTag {
     }
 }
 
+impl SqlTypeConvert for StorageSlotType {
+    type Raw = i32;
+    type Error = DatabaseTypeConversionError;
+
+    #[inline(always)]
+    fn from_raw_sql(raw: Self::Raw) -> Result<Self, Self::Error> {
+        match raw {
+            0 => Ok(StorageSlotType::Map),
+            1 => Ok(StorageSlotType::Value),
+            _ => Err(DatabaseTypeConversionError(type_name::<StorageSlotType>())),
+        }
+    }
+
+    #[inline(always)]
+    fn to_raw_sql(self) -> Self::Raw {
+        match self {
+            StorageSlotType::Map => 0,
+            StorageSlotType::Value => 1,
+        }
+    }
+}
+
 // Raw type conversions - eventually introduce wrapper types
 // ===========================================================
 
@@ -130,9 +153,9 @@ pub(crate) fn nullifier_prefix_to_raw_sql(prefix: u16) -> i32 {
 }
 
 #[inline(always)]
-pub(crate) fn raw_sql_to_nonce(raw: i64) -> u64 {
+pub(crate) fn raw_sql_to_nonce(raw: i64) -> Felt {
     debug_assert!(raw >= 0);
-    raw as u64
+    Felt::new(raw as u64)
 }
 #[inline(always)]
 pub(crate) fn nonce_to_raw_sql(nonce: Felt) -> i64 {