diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b85c441..f93bb299d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,13 +30,14 @@ - Remove `trait AccountTreeStorage` ([#1352](https://github.com/0xMiden/miden-node/issues/1352)). - [BREAKING] `SubmitProvenTransaction` now **requires** that the network's genesis commitment is set in the request's `ACCEPT` header ([#1298](https://github.com/0xMiden/miden-node/pull/1298), [#1436](https://github.com/0xMiden/miden-node/pull/1436)). - Add `S` generic to `NullifierTree` to allow usage with `LargeSmt`s ([#1353](https://github.com/0xMiden/miden-node/issues/1353)). -- Removed internal errors from the `miden-network-monitor` ([#1424](https://github.com/0xMiden/miden-node/pull/1424)). -- Track network transactions latency in `miden-network-monitor` ([#1430](https://github.com/0xMiden/miden-node/pull/1430)). +- Refactor account table and introduce tracking forest ([#1394](https://github.com/0xMiden/miden-node/pull/1394)). - [BREAKING] Re-organized RPC protobuf schema to be independent of internal schema ([#1401](https://github.com/0xMiden/miden-node/pull/1401)). -- Increased the maximum query limit for the store ([#1443](https://github.com/0xMiden/miden-node/pull/1443)). +- Removed internal errors from the `miden-network-monitor` ([#1424](https://github.com/0xMiden/miden-node/pull/1424)). - [BREAKING] Added block signing capabilities to Validator component and updated gensis bootstrap to sign blocks with configured signer ([#1426](https://github.com/0xMiden/miden-node/pull/1426)). +- Track network transactions latency in `miden-network-monitor` ([#1430](https://github.com/0xMiden/miden-node/pull/1430)). - Reduced default block interval from `5s` to `2s` ([#1438](https://github.com/0xMiden/miden-node/pull/1438)). - Increased retained account tree history from 33 to 100 blocks to account for the reduced block interval ([#1438](https://github.com/0xMiden/miden-node/pull/1438)). +- Increased the maximum query limit for the store ([#1443](https://github.com/0xMiden/miden-node/pull/1443)). - [BREAKING] Migrated to version `v0.20` of the VM ([#1476](https://github.com/0xMiden/miden-node/pull/1476)). - [BREAKING] Change account in database representation ([#1481](https://github.com/0xMiden/miden-node/pull/1481)). - Remove the cyclic database optimization ([#1497](https://github.com/0xMiden/miden-node/pull/1497)). diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 4330a82de..5bc6b4ecc 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -187,6 +187,52 @@ impl TryFrom for Accoun } } +impl TryFrom + for AccountStorageMapDetails +{ + type Error = ConversionError; + + fn try_from( + value: proto::rpc::account_storage_details::AccountStorageMapDetails, + ) -> Result { + use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry; + let proto::rpc::account_storage_details::AccountStorageMapDetails { + slot_name, + too_many_entries, + entries, + } = value; + + let slot_name = StorageSlotName::new(slot_name)?; + + let entries = if too_many_entries { + StorageMapEntries::LimitExceeded + } else { + let map_entries = if let Some(entries) = entries { + entries + .entries + .into_iter() + .map(|entry| { + let key = entry + .key + .ok_or(StorageMapEntry::missing_field(stringify!(key)))? + .try_into()?; + let value = entry + .value + .ok_or(StorageMapEntry::missing_field(stringify!(value)))? + .try_into()?; + Ok((key, value)) + }) + .collect::, ConversionError>>()? + } else { + Vec::new() + }; + StorageMapEntries::Entries(map_entries) + }; + + Ok(Self { slot_name, entries }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct StorageMapRequest { pub slot_name: StorageSlotName, @@ -544,52 +590,6 @@ const fn storage_slot_type_to_raw(slot_type: StorageSlotType) -> u32 { } } -impl TryFrom - for AccountStorageMapDetails -{ - type Error = ConversionError; - - fn try_from( - value: proto::rpc::account_storage_details::AccountStorageMapDetails, - ) -> Result { - use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry; - let proto::rpc::account_storage_details::AccountStorageMapDetails { - slot_name, - too_many_entries, - entries, - } = value; - - let slot_name = StorageSlotName::new(slot_name)?; - - let entries = if too_many_entries { - StorageMapEntries::LimitExceeded - } else { - let map_entries = if let Some(entries) = entries { - entries - .entries - .into_iter() - .map(|entry| { - let key = entry - .key - .ok_or(StorageMapEntry::missing_field(stringify!(key)))? - .try_into()?; - let value = entry - .value - .ok_or(StorageMapEntry::missing_field(stringify!(value)))? - .try_into()?; - Ok((key, value)) - }) - .collect::, ConversionError>>()? - } else { - Vec::new() - }; - StorageMapEntries::Entries(map_entries) - }; - - Ok(Self { slot_name, entries }) - } -} - // ACCOUNT PROOF RESPONSE //================================================================================================ diff --git a/crates/store/src/accounts/tests.rs b/crates/store/src/accounts/tests.rs index 5880d3982..f70928946 100644 --- a/crates/store/src/accounts/tests.rs +++ b/crates/store/src/accounts/tests.rs @@ -18,7 +18,7 @@ mod account_tree_with_history_tests { /// Helper function to create an `AccountTree` from entries using the new API fn create_account_tree( entries: impl IntoIterator, - ) -> AccountTree> { + ) -> InMemoryAccountTree { let smt_entries = entries .into_iter() .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 8a5a835a4..3d54a501f 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -113,8 +113,7 @@ impl TransactionRecord { self, note_records: Vec, ) -> proto::rpc::TransactionRecord { - let output_notes: Vec = - note_records.into_iter().map(Into::into).collect(); + let output_notes = Vec::from_iter(note_records.into_iter().map(Into::into)); proto::rpc::TransactionRecord { header: Some(proto::transaction::TransactionHeader { @@ -324,7 +323,7 @@ impl Db { /// Loads all the nullifiers from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_all_nullifiers(&self) -> Result> { + pub(crate) async fn select_all_nullifiers(&self) -> Result> { self.transact("all nullifiers", move |conn| { let nullifiers = queries::select_all_nullifiers(conn)?; Ok(nullifiers) @@ -403,7 +402,6 @@ impl Db { } /// Returns all account IDs that have public state. - #[allow(dead_code)] // Will be used by InnerForest in next PR #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_public_account_ids(&self) -> Result> { self.transact("read all public account IDs", move |conn| { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index c1ad88d2b..3c615c51b 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -299,7 +299,6 @@ pub(crate) fn select_all_account_commitments( /// ORDER BY /// block_num ASC /// ``` -#[allow(dead_code)] // Will be used by InnerForest in next PR pub(crate) fn select_all_public_account_ids( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 7ac836ed3..7e0c326a2 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -31,6 +31,7 @@ use tonic::Status; use crate::db::manager::ConnectionManagerError; use crate::db::models::conv::DatabaseTypeConversionError; +use crate::inner_forest::InnerForestError; // DATABASE ERRORS // ================================================================================================= @@ -197,6 +198,8 @@ pub enum StateInitializationError { BlockStoreLoadError(#[source] std::io::Error), #[error("failed to load database")] DatabaseLoadError(#[from] DatabaseSetupError), + #[error("inner forest error")] + InnerForestError(#[from] InnerForestError), } #[derive(Debug, Error)] @@ -274,6 +277,8 @@ pub enum ApplyBlockError { TokioJoinError(#[from] tokio::task::JoinError), #[error("invalid block error")] InvalidBlockError(#[from] InvalidBlockError), + #[error("inner forest error")] + InnerForestError(#[from] InnerForestError), // OTHER ERRORS // --------------------------------------------------------------------------------------------- diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs new file mode 100644 index 000000000..d368896f2 --- /dev/null +++ b/crates/store/src/inner_forest/mod.rs @@ -0,0 +1,341 @@ +use std::collections::BTreeMap; + +use miden_protocol::account::delta::{AccountDelta, AccountStorageDelta, AccountVaultDelta}; +use miden_protocol::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; +use miden_protocol::asset::{Asset, FungibleAsset}; +use miden_protocol::block::BlockNumber; +use miden_protocol::crypto::merkle::EmptySubtreeRoots; +use miden_protocol::crypto::merkle::smt::{SMT_DEPTH, SmtForest}; +use miden_protocol::{EMPTY_WORD, Word}; +use thiserror::Error; + +#[cfg(test)] +mod tests; + +// ERRORS +// ================================================================================================ + +#[derive(Debug, Error)] +pub enum InnerForestError { + #[error( + "balance underflow: account {account_id}, faucet {faucet_id}, \ + previous balance {prev_balance}, delta {delta}" + )] + BalanceUnderflow { + account_id: AccountId, + faucet_id: AccountId, + prev_balance: u64, + delta: i64, + }, +} + +// INNER FOREST +// ================================================================================================ + +/// Container for forest-related state that needs to be updated atomically. +pub(crate) struct InnerForest { + /// `SmtForest` for efficient account storage reconstruction. + /// Populated during block import with storage and vault SMTs. + forest: SmtForest, + + /// Maps (`account_id`, `slot_name`, `block_num`) to SMT root. + /// Populated during block import for all storage map slots. + storage_map_roots: BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word>, + + /// Maps (`account_id`, `block_num`) to vault SMT root. + /// Tracks asset vault versions across all blocks with structural sharing. + vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, +} + +impl InnerForest { + pub(crate) fn new() -> Self { + Self { + forest: SmtForest::new(), + storage_map_roots: BTreeMap::new(), + vault_roots: BTreeMap::new(), + } + } + + // HELPERS + // -------------------------------------------------------------------------------------------- + + /// Returns the root of an empty SMT. + const fn empty_smt_root() -> Word { + *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + } + + /// Retrieves the most recent vault SMT root for an account. + /// + /// Returns the latest vault root entry regardless of block number. + /// Used when applying incremental deltas where we always want the previous state. + /// + /// If no vault root is found for the account, returns an empty SMT root. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, returns an empty SMT root (for new accounts or DB + /// reconstruction where delta values are absolute). If `false`, looks up the previous state + /// (for incremental updates where delta values are relative changes). + fn get_latest_vault_root(&self, account_id: AccountId, is_full_state: bool) -> Word { + if is_full_state { + return Self::empty_smt_root(); + } + self.vault_roots + .range((account_id, BlockNumber::GENESIS)..) + .take_while(|((id, _), _)| *id == account_id) + .last() + .map_or_else(Self::empty_smt_root, |(_, root)| *root) + } + + /// Retrieves the most recent storage map SMT root for an account slot. + /// + /// Returns the latest storage root entry regardless of block number. + /// Used when applying incremental deltas where we always want the previous state. + /// + /// If no storage root is found for the slot, returns an empty SMT root. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, returns an empty SMT root (for new accounts or DB + /// reconstruction where delta values are absolute). If `false`, looks up the previous state + /// (for incremental updates where delta values are relative changes). + fn get_latest_storage_map_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + is_full_state: bool, + ) -> Word { + if is_full_state { + return Self::empty_smt_root(); + } + + self.storage_map_roots + .range((account_id, slot_name.clone(), BlockNumber::GENESIS)..) + .take_while(|((id, name, _), _)| *id == account_id && name == slot_name) + .last() + .map_or_else(Self::empty_smt_root, |(_, root)| *root) + } + + /// Retrieves the vault SMT root for an account at or before the given block. + /// + /// Finds the most recent vault root entry for the account, since vault state persists + /// across blocks where no changes occur. + // + // TODO: a fallback to DB lookup is required once pruning lands. + // Currently returns empty root which would be incorrect + #[cfg(test)] + fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { + self.vault_roots + .range((account_id, BlockNumber::GENESIS)..=(account_id, block_num)) + .next_back() + .map_or_else(Self::empty_smt_root, |(_, root)| *root) + } + + // PUBLIC INTERFACE + // -------------------------------------------------------------------------------------------- + + /// Applies account updates from a block to the forest. + /// + /// Iterates through account updates and applies each delta to the forest. + /// Private accounts should be filtered out before calling this method. + /// + /// # Arguments + /// + /// * `block_num` - Block number for which these updates apply + /// * `account_updates` - Iterator of `AccountDelta` for public accounts + /// + /// # Errors + /// + /// Returns an error if applying a vault delta results in a negative balance. + pub(crate) fn apply_block_updates( + &mut self, + block_num: BlockNumber, + account_updates: impl IntoIterator, + ) -> Result<(), InnerForestError> { + for delta in account_updates { + self.update_account(block_num, &delta)?; + + tracing::debug!( + target: crate::COMPONENT, + account_id = %delta.id(), + %block_num, + is_full_state = delta.is_full_state(), + "Updated forest with account delta" + ); + } + Ok(()) + } + + /// Updates the forest with account vault and storage changes from a delta. + /// + /// Unified interface for updating all account state in the forest, handling both full-state + /// deltas (new accounts or reconstruction from DB) and partial deltas (incremental updates + /// during block application). + /// + /// Full-state deltas (`delta.is_full_state() == true`) populate the forest from scratch using + /// an empty SMT root. Partial deltas apply changes on top of the previous block's state. + /// + /// # Errors + /// + /// Returns an error if applying a vault delta results in a negative balance. + pub(crate) fn update_account( + &mut self, + block_num: BlockNumber, + delta: &AccountDelta, + ) -> Result<(), InnerForestError> { + let account_id = delta.id(); + let is_full_state = delta.is_full_state(); + + if !delta.vault().is_empty() { + self.update_account_vault(block_num, account_id, delta.vault(), is_full_state)?; + } + + if !delta.storage().is_empty() { + self.update_account_storage(block_num, account_id, delta.storage(), is_full_state); + } + Ok(()) + } + + // PRIVATE METHODS + // -------------------------------------------------------------------------------------------- + + /// Updates the forest with vault changes from a delta. + /// + /// Processes both fungible and non-fungible asset changes, building entries for the vault SMT + /// and tracking the new root. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, delta values are absolute (new account or DB reconstruction). + /// If `false`, delta values are relative changes applied to previous state. + /// + /// # Errors + /// + /// Returns an error if applying a delta results in a negative balance. + fn update_account_vault( + &mut self, + block_num: BlockNumber, + account_id: AccountId, + vault_delta: &AccountVaultDelta, + is_full_state: bool, + ) -> Result<(), InnerForestError> { + let prev_root = self.get_latest_vault_root(account_id, is_full_state); + + let mut entries = Vec::new(); + + // Process fungible assets + for (faucet_id, amount_delta) in vault_delta.fungible().iter() { + let key: Word = + FungibleAsset::new(*faucet_id, 0).expect("valid faucet id").vault_key().into(); + + let new_amount = if is_full_state { + // For full-state deltas, amount is the absolute value + (*amount_delta).try_into().expect("full-state amount should be non-negative") + } else { + // For partial deltas, amount is a change that must be applied to previous balance. + // + // TODO: SmtForest only exposes `fn open()` which computes a full Merkle + // proof. We only need the leaf, so a direct `fn get()` method would be faster. + let prev_amount = self + .forest + .open(prev_root, key) + .ok() + .and_then(|proof| proof.get(&key)) + .and_then(|word| FungibleAsset::try_from(word).ok()) + .map_or(0, |asset| asset.amount()); + + let new_balance = i128::from(prev_amount) + i128::from(*amount_delta); + u64::try_from(new_balance).map_err(|_| InnerForestError::BalanceUnderflow { + account_id, + faucet_id: *faucet_id, + prev_balance: prev_amount, + delta: *amount_delta, + })? + }; + + let value = if new_amount == 0 { + EMPTY_WORD + } else { + let asset: Asset = FungibleAsset::new(*faucet_id, new_amount) + .expect("valid fungible asset") + .into(); + Word::from(asset) + }; + entries.push((key, value)); + } + + // Process non-fungible assets + for (asset, action) in vault_delta.non_fungible().iter() { + let value = match action { + NonFungibleDeltaAction::Add => Word::from(Asset::NonFungible(*asset)), + NonFungibleDeltaAction::Remove => EMPTY_WORD, + }; + entries.push((asset.vault_key().into(), value)); + } + + if entries.is_empty() { + return Ok(()); + } + + let updated_root = self + .forest + .batch_insert(prev_root, entries.iter().copied()) + .expect("forest insertion should succeed"); + + self.vault_roots.insert((account_id, block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + %account_id, + %block_num, + vault_entries = entries.len(), + "Updated vault in forest" + ); + Ok(()) + } + + /// Updates the forest with storage map changes from a delta. + /// + /// Processes storage map slot deltas, building SMTs for each modified slot + /// and tracking the new roots. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, delta values are absolute (new account or DB reconstruction). + /// If `false`, delta values are relative changes applied to previous state. + fn update_account_storage( + &mut self, + block_num: BlockNumber, + account_id: AccountId, + storage_delta: &AccountStorageDelta, + is_full_state: bool, + ) { + for (slot_name, map_delta) in storage_delta.maps() { + let prev_root = self.get_latest_storage_map_root(account_id, slot_name, is_full_state); + + let entries: Vec<_> = + map_delta.entries().iter().map(|(key, value)| ((*key).into(), *value)).collect(); + + if entries.is_empty() { + continue; + } + + let updated_root = self + .forest + .batch_insert(prev_root, entries.iter().copied()) + .expect("forest insertion should succeed"); + + self.storage_map_roots + .insert((account_id, slot_name.clone(), block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + %account_id, + %block_num, + ?slot_name, + entries = entries.len(), + "Updated storage map in forest" + ); + } + } +} diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs new file mode 100644 index 000000000..fb6ceb917 --- /dev/null +++ b/crates/store/src/inner_forest/tests.rs @@ -0,0 +1,421 @@ +use miden_protocol::account::AccountCode; +use miden_protocol::asset::{Asset, AssetVault, FungibleAsset}; +use miden_protocol::testing::account_id::{ + ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, + ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, +}; +use miden_protocol::{Felt, FieldElement}; + +use super::*; + +fn dummy_account() -> AccountId { + AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap() +} + +fn dummy_faucet() -> AccountId { + AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap() +} + +fn dummy_fungible_asset(faucet_id: AccountId, amount: u64) -> Asset { + FungibleAsset::new(faucet_id, amount).unwrap().into() +} + +/// Creates a partial `AccountDelta` (without code) for testing incremental updates. +fn dummy_partial_delta( + account_id: AccountId, + vault_delta: AccountVaultDelta, + storage_delta: AccountStorageDelta, +) -> AccountDelta { + // For partial deltas, nonce_delta must be > 0 if there are changes + let nonce_delta = if vault_delta.is_empty() && storage_delta.is_empty() { + Felt::ZERO + } else { + Felt::ONE + }; + AccountDelta::new(account_id, storage_delta, vault_delta, nonce_delta).unwrap() +} + +/// Creates a full-state `AccountDelta` (with code) for testing DB reconstruction. +fn dummy_full_state_delta(account_id: AccountId, assets: &[Asset]) -> AccountDelta { + use miden_protocol::account::{Account, AccountStorage}; + + // Create a minimal account with the given assets + let vault = AssetVault::new(assets).unwrap(); + let storage = AccountStorage::new(vec![]).unwrap(); + let code = AccountCode::mock(); + let nonce = Felt::ONE; + + let account = Account::new(account_id, vault, storage, code, nonce, None).unwrap(); + + // Convert to delta - this will be a full-state delta because it has code + AccountDelta::try_from(account).unwrap() +} + +#[test] +fn test_empty_smt_root_is_recognized() { + use miden_protocol::crypto::merkle::smt::Smt; + + let empty_root = InnerForest::empty_smt_root(); + + // Verify an empty SMT has the expected root + assert_eq!(Smt::default().root(), empty_root); + + // Test that SmtForest accepts this root in batch_insert + let mut forest = SmtForest::new(); + let entries = vec![(Word::from([1u32, 2, 3, 4]), Word::from([5u32, 6, 7, 8]))]; + + assert!(forest.batch_insert(empty_root, entries).is_ok()); +} + +#[test] +fn test_inner_forest_basic_initialization() { + let forest = InnerForest::new(); + assert!(forest.storage_map_roots.is_empty()); + assert!(forest.vault_roots.is_empty()); +} + +#[test] +fn test_update_account_with_empty_deltas() { + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let block_num = BlockNumber::GENESIS.child(); + + let delta = dummy_partial_delta( + account_id, + AccountVaultDelta::default(), + AccountStorageDelta::default(), + ); + + forest.update_account(block_num, &delta).unwrap(); + + // Empty deltas should not create entries + assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); + assert!(forest.storage_map_roots.is_empty()); +} + +#[test] +fn test_update_vault_with_fungible_asset() { + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + let block_num = BlockNumber::GENESIS.child(); + + let asset = dummy_fungible_asset(faucet_id, 100); + let mut vault_delta = AccountVaultDelta::default(); + vault_delta.add_asset(asset).unwrap(); + + let delta = dummy_partial_delta(account_id, vault_delta, AccountStorageDelta::default()); + forest.update_account(block_num, &delta).unwrap(); + + let vault_root = forest.vault_roots[&(account_id, block_num)]; + assert_ne!(vault_root, EMPTY_WORD); +} + +#[test] +fn test_compare_partial_vs_full_state_delta_vault() { + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + let block_num = BlockNumber::GENESIS.child(); + let asset = dummy_fungible_asset(faucet_id, 100); + + // Approach 1: Partial delta (simulates block application) + let mut forest_partial = InnerForest::new(); + let mut vault_delta = AccountVaultDelta::default(); + vault_delta.add_asset(asset).unwrap(); + let partial_delta = + dummy_partial_delta(account_id, vault_delta, AccountStorageDelta::default()); + forest_partial.update_account(block_num, &partial_delta).unwrap(); + + // Approach 2: Full-state delta (simulates DB reconstruction) + let mut forest_full = InnerForest::new(); + let full_delta = dummy_full_state_delta(account_id, &[asset]); + forest_full.update_account(block_num, &full_delta).unwrap(); + + // Both approaches must produce identical vault roots + let root_partial = forest_partial.vault_roots.get(&(account_id, block_num)).unwrap(); + let root_full = forest_full.vault_roots.get(&(account_id, block_num)).unwrap(); + + assert_eq!(root_partial, root_full); + assert_ne!(*root_partial, EMPTY_WORD); +} + +#[test] +fn test_incremental_vault_updates() { + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: 100 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1).unwrap(); + let root_1 = forest.vault_roots[&(account_id, block_1)]; + + // Block 2: 150 tokens (update) + let block_2 = block_1.child(); + let mut vault_delta_2 = AccountVaultDelta::default(); + vault_delta_2.add_asset(dummy_fungible_asset(faucet_id, 150)).unwrap(); + let delta_2 = dummy_partial_delta(account_id, vault_delta_2, AccountStorageDelta::default()); + forest.update_account(block_2, &delta_2).unwrap(); + let root_2 = forest.vault_roots[&(account_id, block_2)]; + + assert_ne!(root_1, root_2); +} + +#[test] +fn test_full_state_delta_starts_from_empty_root() { + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + let block_num = BlockNumber::GENESIS.child(); + + // Simulate a pre-existing vault state that should be ignored for full-state deltas + let mut vault_delta_pre = AccountVaultDelta::default(); + vault_delta_pre.add_asset(dummy_fungible_asset(faucet_id, 999)).unwrap(); + let delta_pre = + dummy_partial_delta(account_id, vault_delta_pre, AccountStorageDelta::default()); + forest.update_account(block_num, &delta_pre).unwrap(); + assert!(forest.vault_roots.contains_key(&(account_id, block_num))); + + // Now create a full-state delta at the same block + // A full-state delta should start from an empty root, not from the previous state + let asset = dummy_fungible_asset(faucet_id, 100); + let full_delta = dummy_full_state_delta(account_id, &[asset]); + + // Create a fresh forest to compare + let mut fresh_forest = InnerForest::new(); + fresh_forest.update_account(block_num, &full_delta).unwrap(); + let fresh_root = fresh_forest.vault_roots[&(account_id, block_num)]; + + // Update the original forest with the full-state delta + forest.update_account(block_num, &full_delta).unwrap(); + let updated_root = forest.vault_roots[&(account_id, block_num)]; + + // The full-state delta should produce the same root regardless of prior state + assert_eq!(updated_root, fresh_root); +} + +#[test] +fn test_vault_state_persists_across_blocks_without_changes() { + // Regression test for issue #7: vault state should persist across blocks + // where no changes occur, not reset to empty. + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: Add 100 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1).unwrap(); + let root_after_block_1 = forest.vault_roots[&(account_id, block_1)]; + + // Blocks 2-5: No changes to this account (simulated by not calling update_account) + // This means no entries are added to vault_roots for these blocks. + + // Block 6: Add 50 more tokens + // The previous root lookup should find block_1's root, not return empty. + let block_6 = BlockNumber::from(6); + let mut vault_delta_6 = AccountVaultDelta::default(); + vault_delta_6.add_asset(dummy_fungible_asset(faucet_id, 150)).unwrap(); // 100 + 50 = 150 + let delta_6 = dummy_partial_delta(account_id, vault_delta_6, AccountStorageDelta::default()); + forest.update_account(block_6, &delta_6).unwrap(); + + // The root at block 6 should be different from block 1 (we added more tokens) + let root_after_block_6 = forest.vault_roots[&(account_id, block_6)]; + assert_ne!(root_after_block_1, root_after_block_6); + + // Verify get_vault_root finds the correct previous root for intermediate blocks + // Block 3 should return block 1's root (most recent before block 3) + let root_at_block_3 = forest.get_vault_root(account_id, BlockNumber::from(3)); + assert_eq!(root_at_block_3, root_after_block_1); + + // Block 5 should also return block 1's root + let root_at_block_5 = forest.get_vault_root(account_id, BlockNumber::from(5)); + assert_eq!(root_at_block_5, root_after_block_1); + + // Block 6 should return block 6's root + let root_at_block_6 = forest.get_vault_root(account_id, block_6); + assert_eq!(root_at_block_6, root_after_block_6); +} + +#[test] +fn test_partial_delta_applies_fungible_changes_correctly() { + // Regression test for issue #8: partial deltas should apply changes to previous balance, + // not treat amounts as absolute values. + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: Add 100 tokens (partial delta with +100) + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1).unwrap(); + let root_after_100 = forest.vault_roots[&(account_id, block_1)]; + + // Block 2: Add 50 more tokens (partial delta with +50) + // Result should be 150 tokens, not 50 tokens + let block_2 = block_1.child(); + let mut vault_delta_2 = AccountVaultDelta::default(); + vault_delta_2.add_asset(dummy_fungible_asset(faucet_id, 50)).unwrap(); + let delta_2 = dummy_partial_delta(account_id, vault_delta_2, AccountStorageDelta::default()); + forest.update_account(block_2, &delta_2).unwrap(); + let root_after_150 = forest.vault_roots[&(account_id, block_2)]; + + // Roots should be different (100 tokens vs 150 tokens) + assert_ne!(root_after_100, root_after_150); + + // Block 3: Remove 30 tokens (partial delta with -30) + // Result should be 120 tokens + let block_3 = block_2.child(); + let mut vault_delta_3 = AccountVaultDelta::default(); + vault_delta_3.remove_asset(dummy_fungible_asset(faucet_id, 30)).unwrap(); + let delta_3 = dummy_partial_delta(account_id, vault_delta_3, AccountStorageDelta::default()); + forest.update_account(block_3, &delta_3).unwrap(); + let root_after_120 = forest.vault_roots[&(account_id, block_3)]; + + // Root should change again + assert_ne!(root_after_150, root_after_120); + + // Verify by creating a fresh forest with a full-state delta of 120 tokens + // The roots should match + let mut fresh_forest = InnerForest::new(); + let full_delta = dummy_full_state_delta(account_id, &[dummy_fungible_asset(faucet_id, 120)]); + fresh_forest.update_account(block_3, &full_delta).unwrap(); + let root_full_state_120 = fresh_forest.vault_roots[&(account_id, block_3)]; + + assert_eq!(root_after_120, root_full_state_120); +} + +#[test] +fn test_partial_delta_across_long_block_range() { + // Validation test: partial deltas should work across 101+ blocks. + // + // This test passes now because InnerForest keeps all history. Once pruning is implemented + // (estimated ~50 blocks), this test will fail unless DB fallback is also implemented. + // When that happens, the test should be updated to use DB fallback or converted to an + // integration test that has DB access. + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: Add 1000 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 1000)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1).unwrap(); + let root_after_1000 = forest.vault_roots[&(account_id, block_1)]; + + // Blocks 2-100: No changes to this account (simulating long gap) + + // Block 101: Add 500 more tokens (partial delta with +500) + // This requires looking up block 1's state across a 100-block gap. + let block_101 = BlockNumber::from(101); + let mut vault_delta_101 = AccountVaultDelta::default(); + vault_delta_101.add_asset(dummy_fungible_asset(faucet_id, 500)).unwrap(); + let delta_101 = + dummy_partial_delta(account_id, vault_delta_101, AccountStorageDelta::default()); + forest.update_account(block_101, &delta_101).unwrap(); + let root_after_1500 = forest.vault_roots[&(account_id, block_101)]; + + // Roots should be different (1000 tokens vs 1500 tokens) + assert_ne!(root_after_1000, root_after_1500); + + // Verify the final state matches a fresh forest with 1500 tokens + let mut fresh_forest = InnerForest::new(); + let full_delta = dummy_full_state_delta(account_id, &[dummy_fungible_asset(faucet_id, 1500)]); + fresh_forest.update_account(block_101, &full_delta).unwrap(); + let root_full_state_1500 = fresh_forest.vault_roots[&(account_id, block_101)]; + + assert_eq!(root_after_1500, root_full_state_1500); +} + +#[test] +fn test_update_storage_map() { + use std::collections::BTreeMap; + + use miden_protocol::account::delta::{StorageMapDelta, StorageSlotDelta}; + + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let block_num = BlockNumber::GENESIS.child(); + + let slot_name = StorageSlotName::mock(3); + let key = Word::from([1u32, 2, 3, 4]); + let value = Word::from([5u32, 6, 7, 8]); + + let mut map_delta = StorageMapDelta::default(); + map_delta.insert(key, value); + let raw = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta))]); + let storage_delta = AccountStorageDelta::from_raw(raw); + + let delta = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta); + forest.update_account(block_num, &delta).unwrap(); + + // Verify storage root was created + assert!( + forest + .storage_map_roots + .contains_key(&(account_id, slot_name.clone(), block_num)) + ); + let storage_root = forest.storage_map_roots[&(account_id, slot_name, block_num)]; + assert_ne!(storage_root, InnerForest::empty_smt_root()); +} + +#[test] +fn test_storage_map_incremental_updates() { + use std::collections::BTreeMap; + + use miden_protocol::account::delta::{StorageMapDelta, StorageSlotDelta}; + + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + + let slot_name = StorageSlotName::mock(3); + let key1 = Word::from([1u32, 0, 0, 0]); + let key2 = Word::from([2u32, 0, 0, 0]); + let value1 = Word::from([10u32, 0, 0, 0]); + let value2 = Word::from([20u32, 0, 0, 0]); + let value3 = Word::from([30u32, 0, 0, 0]); + + // Block 1: Insert key1 -> value1 + let block_1 = BlockNumber::GENESIS.child(); + let mut map_delta_1 = StorageMapDelta::default(); + map_delta_1.insert(key1, value1); + let raw_1 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_1))]); + let storage_delta_1 = AccountStorageDelta::from_raw(raw_1); + let delta_1 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_1); + forest.update_account(block_1, &delta_1).unwrap(); + let root_1 = forest.storage_map_roots[&(account_id, slot_name.clone(), block_1)]; + + // Block 2: Insert key2 -> value2 (key1 should persist) + let block_2 = block_1.child(); + let mut map_delta_2 = StorageMapDelta::default(); + map_delta_2.insert(key2, value2); + let raw_2 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_2))]); + let storage_delta_2 = AccountStorageDelta::from_raw(raw_2); + let delta_2 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_2); + forest.update_account(block_2, &delta_2).unwrap(); + let root_2 = forest.storage_map_roots[&(account_id, slot_name.clone(), block_2)]; + + // Block 3: Update key1 -> value3 + let block_3 = block_2.child(); + let mut map_delta_3 = StorageMapDelta::default(); + map_delta_3.insert(key1, value3); + let raw_3 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_3))]); + let storage_delta_3 = AccountStorageDelta::from_raw(raw_3); + let delta_3 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_3); + forest.update_account(block_3, &delta_3).unwrap(); + let root_3 = forest.storage_map_roots[&(account_id, slot_name, block_3)]; + + // All roots should be different + assert_ne!(root_1, root_2); + assert_ne!(root_2, root_3); + assert_ne!(root_1, root_3); +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 636225da1..5a9dc5ee2 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -3,6 +3,7 @@ mod blocks; mod db; mod errors; pub mod genesis; +mod inner_forest; mod server; pub mod state; diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index 6974b8dcb..324a9dde3 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -163,5 +163,5 @@ pub fn validate_note_commitments(notes: &[proto::primitives::Digest]) -> Result< #[instrument(level = "debug",target = COMPONENT, skip_all)] pub fn read_block_numbers(block_numbers: &[u32]) -> BTreeSet { - block_numbers.iter().map(|raw_number| BlockNumber::from(*raw_number)).collect() + BTreeSet::from_iter(block_numbers.iter().map(|raw_number| BlockNumber::from(*raw_number))) } diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index ba8e82b4f..40f1ae5b3 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -181,8 +181,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { let (account_ids, mut last_block_included) = self.state.get_all_network_accounts(block_range).await.map_err(internal_error)?; - let account_ids: Vec = - account_ids.into_iter().map(Into::into).collect(); + let account_ids = Vec::from_iter(account_ids.into_iter().map(Into::into)); if last_block_included > chain_tip { last_block_included = chain_tip; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 7c72ae818..43e653dcf 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -23,6 +23,7 @@ use miden_node_proto::domain::account::{ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; +use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{AccountId, StorageSlotContent}; use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; @@ -42,6 +43,7 @@ use miden_protocol::{AccountError, Word}; use tokio::sync::{Mutex, RwLock, oneshot}; use tracing::{info, info_span, instrument}; +use crate::accounts::{AccountTreeWithHistory, HistoricalError}; use crate::blocks::BlockStore; use crate::db::models::Page; use crate::db::models::queries::StorageMapValuesPage; @@ -65,7 +67,8 @@ use crate::errors::{ StateInitializationError, StateSyncError, }; -use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; +use crate::inner_forest::InnerForest; +use crate::{COMPONENT, DataDirectory}; // STRUCTURES // ================================================================================================ @@ -117,6 +120,9 @@ pub struct State { /// The lock is writer-preferring, meaning the writer won't be starved. inner: RwLock, + /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. + forest: RwLock, + /// To allow readers to access the tree data while an update in being performed, and prevent /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. writer: Mutex<()>, @@ -146,13 +152,15 @@ impl State { let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); let account_tree = load_account_tree(&mut db, latest_block_num).await?; let nullifier_tree = load_nullifier_tree(&mut db).await?; + let forest = load_smt_forest(&mut db, latest_block_num).await?; let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); + let forest = RwLock::new(forest); let writer = Mutex::new(()); let db = Arc::new(db); - Ok(Self { db, block_store, inner, writer }) + Ok(Self { db, block_store, inner, forest, writer }) } // STATE MUTATOR @@ -291,10 +299,10 @@ impl State { .map(|update| (update.account_id(), update.final_state_commitment())), ) .map_err(|e| match e { - crate::HistoricalError::AccountTreeError(err) => { + HistoricalError::AccountTreeError(err) => { InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) }, - crate::HistoricalError::MerkleError(_) => { + HistoricalError::MerkleError(_) => { panic!("Unexpected MerkleError during account tree mutation computation") }, })?; @@ -354,6 +362,16 @@ impl State { // Signals the write lock has been acquired, and the transaction can be committed let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); + // Extract public account updates with deltas before block is moved into async task. + // Private accounts are filtered out since they don't expose their state changes. + let account_deltas = + Vec::from_iter(block.body().updated_accounts().iter().filter_map(|update| { + match update.details() { + AccountUpdateDetails::Delta(delta) => Some(delta.clone()), + AccountUpdateDetails::Private => None, + } + })); + // The DB and in-memory state updates need to be synchronized and are partially // overlapping. Namely, the DB transaction only proceeds after this task acquires the // in-memory write lock. This requires the DB update to run concurrently, so a new task is @@ -413,6 +431,8 @@ impl State { inner.blockchain.push(block_commitment); } + self.forest.write().await.apply_block_updates(block_num, account_deltas)?; + info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); Ok(()) @@ -1007,11 +1027,11 @@ impl State { // Validate block exists in the blockchain before querying the database self.validate_block_exists(block_num).await?; - let account_header = self - .db - .select_account_header_at_block(account_id, block_num) - .await? - .ok_or_else(|| DatabaseError::AccountNotPublic(account_id))?; + let account_header = + self.db + .select_account_header_at_block(account_id, block_num) + .await? + .ok_or(DatabaseError::AccountAtBlockHeightNotFoundInDb(account_id, block_num))?; let account_code = match code_commitment { Some(commitment) if commitment == account_header.code_commitment() => None, @@ -1121,7 +1141,6 @@ impl State { ) -> Result<(BlockNumber, Vec), DatabaseError> { self.db.get_account_vault_sync(account_id, block_range).await } - /// Returns the network notes for an account that are unconsumed by a specified block number, /// along with the next pagination token. pub async fn get_unconsumed_network_notes_for_account( @@ -1215,3 +1234,40 @@ async fn load_account_tree( Ok(AccountTreeWithHistory::new(account_tree, block_number)) } + +/// Loads SMT forest with storage map and vault Merkle paths for all public accounts. +#[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] +async fn load_smt_forest( + db: &mut Db, + block_num: BlockNumber, +) -> Result { + use miden_protocol::account::delta::AccountDelta; + + let public_account_ids = db.select_all_public_account_ids().await?; + + // Acquire write lock once for the entire initialization + let mut forest = InnerForest::new(); + + // Process each account + for account_id in public_account_ids { + // Get the full account from the database + let account_info = db.select_account(account_id).await?; + let account = account_info.details.expect("public accounts always have details in DB"); + + // Convert the full account to a full-state delta + let delta = + AccountDelta::try_from(account).expect("accounts from DB should not have seeds"); + + // Use the unified update method (will recognize it's a full-state delta) + forest.update_account(block_num, &delta)?; + + tracing::debug!( + target: COMPONENT, + %account_id, + %block_num, + "Initialized forest for account from DB" + ); + } + + Ok(forest) +} diff --git a/crates/utils/src/limiter.rs b/crates/utils/src/limiter.rs index 4e580d302..03b8aeb54 100644 --- a/crates/utils/src/limiter.rs +++ b/crates/utils/src/limiter.rs @@ -10,6 +10,7 @@ //! //! Add new limits here so callers share the same values and rationale. +/// Basic request limit. pub const GENERAL_REQUEST_LIMIT: usize = 1000; #[allow(missing_docs)]