feat: [2/4] integrate smtforest, avoid ser/de of full account/vault data in database

CHANGELOG.md

@@ -30,13 +30,14 @@
 - Remove `trait AccountTreeStorage` ([#1352](https://github.com/0xMiden/miden-node/issues/1352)).
 - [BREAKING] `SubmitProvenTransaction` now **requires** that the network's genesis commitment is set in the request's `ACCEPT` header ([#1298](https://github.com/0xMiden/miden-node/pull/1298), [#1436](https://github.com/0xMiden/miden-node/pull/1436)).
 - Add `S` generic to `NullifierTree` to allow usage with `LargeSmt`s ([#1353](https://github.com/0xMiden/miden-node/issues/1353)).
-- Removed internal errors from the `miden-network-monitor` ([#1424](https://github.com/0xMiden/miden-node/pull/1424)).
-- Track network transactions latency in `miden-network-monitor` ([#1430](https://github.com/0xMiden/miden-node/pull/1430)).
+- Refactor account table and introduce tracking forest ([#1394](https://github.com/0xMiden/miden-node/pull/1394)).
 - [BREAKING] Re-organized RPC protobuf schema to be independent of internal schema ([#1401](https://github.com/0xMiden/miden-node/pull/1401)).
-- Increased the maximum query limit for the store ([#1443](https://github.com/0xMiden/miden-node/pull/1443)).
+- Removed internal errors from the `miden-network-monitor` ([#1424](https://github.com/0xMiden/miden-node/pull/1424)).
 - [BREAKING] Added block signing capabilities to Validator component and updated gensis bootstrap to sign blocks with configured signer ([#1426](https://github.com/0xMiden/miden-node/pull/1426)).
+- Track network transactions latency in `miden-network-monitor` ([#1430](https://github.com/0xMiden/miden-node/pull/1430)).
 - Reduced default block interval from `5s` to `2s` ([#1438](https://github.com/0xMiden/miden-node/pull/1438)).
 - Increased retained account tree history from 33 to 100 blocks to account for the reduced block interval ([#1438](https://github.com/0xMiden/miden-node/pull/1438)).
+- Increased the maximum query limit for the store ([#1443](https://github.com/0xMiden/miden-node/pull/1443)).
 - [BREAKING] Migrated to version `v0.20` of the VM ([#1476](https://github.com/0xMiden/miden-node/pull/1476)).
 - [BREAKING] Change account in database representation ([#1481](https://github.com/0xMiden/miden-node/pull/1481)).
 - Remove the cyclic database optimization ([#1497](https://github.com/0xMiden/miden-node/pull/1497)). 

crates/proto/src/domain/account.rs

@@ -187,6 +187,52 @@ impl TryFrom<proto::rpc::account_proof_request::AccountDetailRequest> for Accoun
     }
 }
 
+impl TryFrom<proto::rpc::account_storage_details::AccountStorageMapDetails>
+    for AccountStorageMapDetails
+{
+    type Error = ConversionError;
+
+    fn try_from(
+        value: proto::rpc::account_storage_details::AccountStorageMapDetails,
+    ) -> Result<Self, Self::Error> {
+        use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry;
+        let proto::rpc::account_storage_details::AccountStorageMapDetails {
+            slot_name,
+            too_many_entries,
+            entries,
+        } = value;
+
+        let slot_name = StorageSlotName::new(slot_name)?;
+
+        let entries = if too_many_entries {
+            StorageMapEntries::LimitExceeded
+        } else {
+            let map_entries = if let Some(entries) = entries {
+                entries
+                    .entries
+                    .into_iter()
+                    .map(|entry| {
+                        let key = entry
+                            .key
+                            .ok_or(StorageMapEntry::missing_field(stringify!(key)))?
+                            .try_into()?;
+                        let value = entry
+                            .value
+                            .ok_or(StorageMapEntry::missing_field(stringify!(value)))?
+                            .try_into()?;
+                        Ok((key, value))
+                    })
+                    .collect::<Result<Vec<_>, ConversionError>>()?
+            } else {
+                Vec::new()
+            };
+            StorageMapEntries::Entries(map_entries)
+        };
+
+        Ok(Self { slot_name, entries })
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct StorageMapRequest {
     pub slot_name: StorageSlotName,
@@ -544,52 +590,6 @@ const fn storage_slot_type_to_raw(slot_type: StorageSlotType) -> u32 {
     }
 }
 
-impl TryFrom<proto::rpc::account_storage_details::AccountStorageMapDetails>
-    for AccountStorageMapDetails
-{
-    type Error = ConversionError;
-
-    fn try_from(
-        value: proto::rpc::account_storage_details::AccountStorageMapDetails,
-    ) -> Result<Self, Self::Error> {
-        use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry;
-        let proto::rpc::account_storage_details::AccountStorageMapDetails {
-            slot_name,
-            too_many_entries,
-            entries,
-        } = value;
-
-        let slot_name = StorageSlotName::new(slot_name)?;
-
-        let entries = if too_many_entries {
-            StorageMapEntries::LimitExceeded
-        } else {
-            let map_entries = if let Some(entries) = entries {
-                entries
-                    .entries
-                    .into_iter()
-                    .map(|entry| {
-                        let key = entry
-                            .key
-                            .ok_or(StorageMapEntry::missing_field(stringify!(key)))?
-                            .try_into()?;
-                        let value = entry
-                            .value
-                            .ok_or(StorageMapEntry::missing_field(stringify!(value)))?
-                            .try_into()?;
-                        Ok((key, value))
-                    })
-                    .collect::<Result<Vec<_>, ConversionError>>()?
-            } else {
-                Vec::new()
-            };
-            StorageMapEntries::Entries(map_entries)
-        };
-
-        Ok(Self { slot_name, entries })
-    }
-}
-
 // ACCOUNT PROOF RESPONSE
 //================================================================================================
 

crates/store/src/accounts/tests.rs

@@ -18,7 +18,7 @@ mod account_tree_with_history_tests {
     /// Helper function to create an `AccountTree` from entries using the new API
     fn create_account_tree(
         entries: impl IntoIterator<Item = (AccountId, Word)>,
-    ) -> AccountTree<LargeSmt<MemoryStorage>> {
+    ) -> InMemoryAccountTree {
         let smt_entries = entries
             .into_iter()
             .map(|(id, commitment)| (account_id_to_smt_key(id), commitment));

crates/store/src/db/mod.rs

@@ -113,8 +113,7 @@ impl TransactionRecord {
         self,
         note_records: Vec<NoteRecord>,
     ) -> proto::rpc::TransactionRecord {
-        let output_notes: Vec<proto::note::NoteSyncRecord> =
-            note_records.into_iter().map(Into::into).collect();
+        let output_notes = Vec::from_iter(note_records.into_iter().map(Into::into));
 
         proto::rpc::TransactionRecord {
             header: Some(proto::transaction::TransactionHeader {
@@ -324,7 +323,7 @@ impl Db {
 
     /// Loads all the nullifiers from the DB.
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
-    pub async fn select_all_nullifiers(&self) -> Result<Vec<NullifierInfo>> {
+    pub(crate) async fn select_all_nullifiers(&self) -> Result<Vec<NullifierInfo>> {
         self.transact("all nullifiers", move |conn| {
             let nullifiers = queries::select_all_nullifiers(conn)?;
             Ok(nullifiers)
@@ -403,7 +402,6 @@ impl Db {
     }
 
     /// Returns all account IDs that have public state.
-    #[allow(dead_code)] // Will be used by InnerForest in next PR
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
     pub async fn select_all_public_account_ids(&self) -> Result<Vec<AccountId>> {
         self.transact("read all public account IDs", move |conn| {

crates/store/src/db/models/queries/accounts.rs

@@ -299,7 +299,6 @@ pub(crate) fn select_all_account_commitments(
 /// ORDER BY
 ///     block_num ASC
 /// ```
-#[allow(dead_code)] // Will be used by InnerForest in next PR
 pub(crate) fn select_all_public_account_ids(
     conn: &mut SqliteConnection,
 ) -> Result<Vec<AccountId>, DatabaseError> {

crates/store/src/errors.rs

@@ -31,6 +31,7 @@ use tonic::Status;
 
 use crate::db::manager::ConnectionManagerError;
 use crate::db::models::conv::DatabaseTypeConversionError;
+use crate::inner_forest::InnerForestError;
 
 // DATABASE ERRORS
 // =================================================================================================
@@ -197,6 +198,8 @@ pub enum StateInitializationError {
     BlockStoreLoadError(#[source] std::io::Error),
     #[error("failed to load database")]
     DatabaseLoadError(#[from] DatabaseSetupError),
+    #[error("inner forest error")]
+    InnerForestError(#[from] InnerForestError),
 }
 
 #[derive(Debug, Error)]
@@ -274,6 +277,8 @@ pub enum ApplyBlockError {
     TokioJoinError(#[from] tokio::task::JoinError),
     #[error("invalid block error")]
     InvalidBlockError(#[from] InvalidBlockError),
+    #[error("inner forest error")]
+    InnerForestError(#[from] InnerForestError),
 
     // OTHER ERRORS
     // ---------------------------------------------------------------------------------------------