feat: [2/4] integrate smtforest, avoid ser/de of full account/vault data in database

crates/proto/src/domain/account.rs

@@ -69,6 +69,7 @@ impl From<AccountId> for proto::account::AccountId {
 // ACCOUNT UPDATE
 // ================================================================================================
 
+// TODO should be called `AccountStateRef` or so
 #[derive(Debug, PartialEq)]
 pub struct AccountSummary {
     pub account_id: AccountId,
@@ -86,6 +87,7 @@ impl From<&AccountSummary> for proto::account::AccountSummary {
     }
 }
 
+// TODO #[deprecated(note = "avoid this type, details will be `None` always!")]
 #[derive(Debug, PartialEq)]
 pub struct AccountInfo {
     pub summary: AccountSummary,
@@ -375,6 +377,27 @@ impl AccountVaultDetails {
         }
     }
 
+    /// Creates `AccountVaultDetails` from vault entries (key-value pairs).
+    ///
+    /// This is useful when entries have been fetched directly from the database
+    /// rather than extracted from an `AssetVault`.
+    ///
+    /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation.
+    pub fn from_entries(entries: Vec<(Word, Word)>) -> Result<Self, miden_objects::AssetError> {
+        let too_many_assets = entries.len() > Self::MAX_RETURN_ENTRIES;
+
+        if too_many_assets {
+            return Ok(Self::too_many());
+        }
+
+        let assets = entries
+            .into_iter()
+            .map(|(_key, asset_word)| Asset::try_from(asset_word))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(Self { too_many_assets: false, assets })
+    }
+
     fn too_many() -> Self {
         Self {
             too_many_assets: true,
@@ -418,16 +441,20 @@ impl From<AccountVaultDetails> for proto::rpc_store::AccountVaultDetails {
 pub struct AccountStorageMapDetails {
     pub slot_index: u8,
     pub too_many_entries: bool,
+    // TODO the following is only for the case when _all_ entries are included
+    // TODO for partials, we also need to provide merkle proofs / a partial SMT with inner nodes
+    // Reason: if all leaf values are included, one can reconstruct the entire SMT, if just one
+    // is missing one cannot
     pub map_entries: Vec<(Word, Word)>,
 }
 
 impl AccountStorageMapDetails {
-    const MAX_RETURN_ENTRIES: usize = 1000;
+    pub const MAX_RETURN_ENTRIES: usize = 1000;
 
     pub fn new(slot_index: u8, slot_data: SlotData, storage_map: &StorageMap) -> Self {
         match slot_data {
             SlotData::All => Self::from_all_entries(slot_index, storage_map),
-            SlotData::MapKeys(keys) => Self::from_specific_keys(slot_index, &keys[..], storage_map),
+            SlotData::MapKeys(_keys) => Self::from_all_entries(slot_index, storage_map), /* TODO use from_specific_keys */
         }
     }
 
@@ -444,12 +471,27 @@ impl AccountStorageMapDetails {
         }
     }
 
-    fn from_specific_keys(slot_index: u8, keys: &[Word], storage_map: &StorageMap) -> Self {
+    // TODO this is
+    #[allow(dead_code)]
+    fn from_specific_keys(slot_index: u8, keys: &[Word], _storage_map: &StorageMap) -> Self {
         if keys.len() > Self::MAX_RETURN_ENTRIES {
             Self::too_many_entries(slot_index)
         } else {
-            // TODO For now, we return all entries instead of specific keys with proofs
-            Self::from_all_entries(slot_index, storage_map)
+            todo!("construct a partial SMT / set of key values")
+        }
+    }
+
+    /// Creates an `AccountStorageMapDetails` from already-queried entries (e.g., from database).
+    /// This is useful when entries have been fetched directly rather than extracted from a
+    /// `StorageMap`.
+    pub fn from_entries(slot_index: u8, map_entries: Vec<(Word, Word)>) -> Self {
+        let too_many_entries = map_entries.len() > Self::MAX_RETURN_ENTRIES;
+        let map_entries = if too_many_entries { Vec::new() } else { map_entries };
+
+        Self {
+            slot_index,
+            too_many_entries,
+            map_entries,
         }
     }
 

crates/store/src/db/migrations/2025062000000_setup/down.sql

@@ -0,0 +1,12 @@
+-- Drop all tables in reverse order of creation (respecting foreign key dependencies)
+DROP TABLE IF EXISTS transactions;
+DROP TABLE IF EXISTS nullifiers;
+DROP TABLE IF EXISTS account_vault_headers;
+DROP TABLE IF EXISTS account_vault_assets;
+DROP TABLE IF EXISTS account_storage_map_values;
+DROP TABLE IF EXISTS note_scripts;
+DROP TABLE IF EXISTS notes;
+DROP TABLE IF EXISTS account_storage_headers;
+DROP TABLE IF EXISTS accounts;
+DROP TABLE IF EXISTS account_codes;
+DROP TABLE IF EXISTS block_headers;

crates/store/src/db/migrations/2025062000000_setup/up.sql

@@ -18,17 +18,15 @@ CREATE TABLE accounts (
     block_num                               INTEGER NOT NULL,
     account_commitment                      BLOB NOT NULL,
     code_commitment                         BLOB,
-    storage                                 BLOB,
-    vault                                   BLOB,
     nonce                                   INTEGER,
     is_latest                               BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id
 
     PRIMARY KEY (account_id, block_num),
     CONSTRAINT all_null_or_none_null CHECK
         (
-            (code_commitment IS NOT NULL AND storage IS NOT NULL AND vault IS NOT NULL AND nonce IS NOT NULL)
+            (code_commitment IS NOT NULL AND nonce IS NOT NULL)
             OR
-            (code_commitment IS NULL AND storage IS NULL AND vault IS NULL AND nonce IS NULL)
+            (code_commitment IS NULL AND nonce IS NULL)
         )
 ) WITHOUT ROWID;
 
@@ -40,6 +38,26 @@ CREATE INDEX idx_accounts_block_num ON accounts(block_num);
 -- Index for joining with account_codes
 CREATE INDEX idx_accounts_code_commitment ON accounts(code_commitment) WHERE code_commitment IS NOT NULL;
 
+-- Table to store storage slot headers (slot types and commitments)
+CREATE TABLE account_storage_headers (
+    account_id          BLOB NOT NULL,
+    block_num           INTEGER NOT NULL,
+    slot_index          INTEGER NOT NULL,
+    slot_type           INTEGER NOT NULL, -- 0=Map, 1=Value (as per StorageSlotType)
+    slot_commitment     BLOB    NOT NULL,
+    is_latest           BOOLEAN NOT NULL DEFAULT 0,
+
+    PRIMARY KEY (account_id, block_num, slot_index),
+    CONSTRAINT slot_index_is_u8 CHECK (slot_index BETWEEN 0 AND 0xFF),
+    CONSTRAINT slot_type_in_enum CHECK (slot_type BETWEEN 0 AND 1),
+    FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE
+) WITHOUT ROWID;
+
+-- Index for joining with accounts table
+CREATE INDEX idx_account_storage_headers_account_block ON account_storage_headers(account_id, block_num);
+-- Index for querying latest state
+CREATE INDEX idx_account_storage_headers_latest ON account_storage_headers(account_id, is_latest) WHERE is_latest = 1;
+
 CREATE TABLE notes (
     committed_at             INTEGER NOT NULL, -- Block number when the note was committed
     batch_index              INTEGER NOT NULL, -- Index of batch in block, starting from 0
@@ -122,6 +140,22 @@ CREATE INDEX idx_vault_assets_account_block ON account_vault_assets(account_id,
 -- Index for querying latest assets
 CREATE INDEX idx_vault_assets_latest ON account_vault_assets(account_id, is_latest) WHERE is_latest = 1;
 
+-- Table to store vault headers (vault root commitments)
+CREATE TABLE account_vault_headers (
+    account_id          BLOB NOT NULL,
+    block_num           INTEGER NOT NULL,
+    vault_root          BLOB    NOT NULL,
+    is_latest           BOOLEAN NOT NULL DEFAULT 0,
+
+    PRIMARY KEY (account_id, block_num),
+    FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE
+) WITHOUT ROWID;
+
+-- Index for joining with accounts table
+CREATE INDEX idx_account_vault_headers_account_block ON account_vault_headers(account_id, block_num);
+-- Index for querying latest state
+CREATE INDEX idx_account_vault_headers_latest ON account_vault_headers(account_id, is_latest) WHERE is_latest = 1;
+
 CREATE TABLE nullifiers (
     nullifier        BLOB    NOT NULL,
     nullifier_prefix INTEGER NOT NULL,

crates/store/src/db/mod.rs

@@ -393,6 +393,7 @@ impl Db {
     }
 
     /// Loads all the account commitments from the DB.
+    // TODO add a variant with block_num as arg
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
     pub async fn select_all_account_commitments(&self) -> Result<Vec<(AccountId, Word)>> {
         self.transact("read all account commitments", move |conn| {
@@ -433,6 +434,77 @@ impl Db {
         .await
     }
 
+    /// Reconstructs account storage at a specific block from the database
+    ///
+    /// This method queries the decomposed storage tables and reconstructs the full
+    /// `AccountStorage` with SMT backing for Map slots.
+    #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
+    pub async fn select_account_storage_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<miden_objects::account::AccountStorage> {
+        self.transact("Get account storage at block", move |conn| {
+            queries::select_account_storage_at_block(conn, account_id, block_num)
+        })
+        .await
+    }
+
+    /// Gets the latest account storage from the database
+    ///
+    /// Uses the `is_latest` flag for efficient querying.
+    #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
+    pub async fn select_latest_account_storage(
+        &self,
+        account_id: AccountId,
+    ) -> Result<miden_objects::account::AccountStorage> {
+        self.transact("Get latest account storage", move |conn| {
+            queries::select_latest_account_storage(conn, account_id)
+        })
+        .await
+    }
+
+    /// Queries vault assets at a specific block
+    #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
+    pub async fn select_account_vault_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<Vec<(Word, Word)>> {
+        self.transact("Get account vault at block", move |conn| {
+            queries::select_account_vault_at_block(conn, account_id, block_num)
+        })
+        .await
+    }
+
+    /// Queries the account code for a specific account at a specific block number.
+    ///
+    /// Returns `None` if the account doesn't exist at that block or has no code.
+    pub async fn select_account_code_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<Option<Vec<u8>>> {
+        self.transact("Get account code at block", move |conn| {
+            queries::select_account_code_at_block(conn, account_id, block_num)
+        })
+        .await
+    }
+
+    /// Queries the account header for a specific account at a specific block number.
+    ///
+    /// Returns `None` if the account doesn't exist at that block.
+    pub async fn select_account_header_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+    ) -> Result<Option<miden_objects::account::AccountHeader>> {
+        self.transact("Get account header at block", move |conn| {
+            queries::select_account_header_at_block(conn, account_id, block_num)
+        })
+        .await
+    }
+
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
     pub async fn get_state_sync(
         &self,
@@ -531,7 +603,7 @@ impl Db {
         .await
     }
 
-    /// Selects storage map values for syncing storage maps for a specific account ID.
+    /// Selects storage map values for syncing storage maps for a specific account ID
     ///
     /// The returned values are the latest known values up to `block_range.end()`, and no values
     /// earlier than `block_range.start()` are returned.
@@ -546,6 +618,26 @@ impl Db {
         .await
     }
 
+    /// Selects specific storage map keys at a specific block from the DB
+    ///
+    /// This method is optimized for querying specific keys without deserializing the entire
+    /// account, which is much faster for historical queries.
+    #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
+    pub async fn select_storage_map_keys_at_block(
+        &self,
+        account_id: AccountId,
+        block_num: BlockNumber,
+        slot_index: u8,
+        keys: Vec<Word>,
+    ) -> Result<Vec<(Word, Word)>> {
+        self.transact("select storage map keys at block", move |conn| {
+            models::queries::select_storage_map_keys_at_block(
+                conn, account_id, block_num, slot_index, &keys,
+            )
+        })
+        .await
+    }
+
     /// Runs database optimization.
     #[instrument(level = "debug", target = COMPONENT, skip_all, err)]
     pub async fn optimize(&self) -> Result<(), DatabaseError> {

crates/store/src/db/models/conv.rs

@@ -36,6 +36,7 @@ use std::any::type_name;
 
 use miden_node_proto::domain::account::{NetworkAccountError, NetworkAccountPrefix};
 use miden_objects::Felt;
+use miden_objects::account::StorageSlotType;
 use miden_objects::block::BlockNumber;
 use miden_objects::note::{NoteExecutionMode, NoteTag};
 
@@ -116,6 +117,28 @@ impl SqlTypeConvert for NoteTag {
     }
 }
 
+impl SqlTypeConvert for StorageSlotType {
+    type Raw = i32;
+    type Error = DatabaseTypeConversionError;
+
+    #[inline(always)]
+    fn from_raw_sql(raw: Self::Raw) -> Result<Self, Self::Error> {
+        match raw {
+            0 => Ok(StorageSlotType::Map),
+            1 => Ok(StorageSlotType::Value),
+            _ => Err(DatabaseTypeConversionError(type_name::<StorageSlotType>())),
+        }
+    }
+
+    #[inline(always)]
+    fn to_raw_sql(self) -> Self::Raw {
+        match self {
+            StorageSlotType::Map => 0,
+            StorageSlotType::Value => 1,
+        }
+    }
+}
+
 // Raw type conversions - eventually introduce wrapper types
 // ===========================================================
 
@@ -130,9 +153,9 @@ pub(crate) fn nullifier_prefix_to_raw_sql(prefix: u16) -> i32 {
 }
 
 #[inline(always)]
-pub(crate) fn raw_sql_to_nonce(raw: i64) -> u64 {
+pub(crate) fn raw_sql_to_nonce(raw: i64) -> Felt {
     debug_assert!(raw >= 0);
-    raw as u64
+    Felt::new(raw as u64)
 }
 #[inline(always)]
 pub(crate) fn nonce_to_raw_sql(nonce: Felt) -> i64 {