From 60f4db128ce421ff7ec5a6b99e2351463eca6b63 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 3 Oct 2025 16:37:06 -0400 Subject: [PATCH 1/3] fix(engine): flatten storage cache --- crates/engine/tree/src/tree/cached_state.rs | 122 +++++++------------- 1 file changed, 40 insertions(+), 82 deletions(-) diff --git a/crates/engine/tree/src/tree/cached_state.rs b/crates/engine/tree/src/tree/cached_state.rs index 9f4eb8398df..4c0fd58a623 100644 --- a/crates/engine/tree/src/tree/cached_state.rs +++ b/crates/engine/tree/src/tree/cached_state.rs @@ -300,65 +300,69 @@ pub(crate) struct ExecutionCache { /// Cache for contract bytecode, keyed by code hash. code_cache: Cache>, - /// Per-account storage cache: outer cache keyed by Address, inner cache tracks that account’s - /// storage slots. - storage_cache: Cache, + /// Flattened storage cache: composite key of (Address, StorageKey) maps directly to values. + storage_cache: Cache<(Address, StorageKey), Option>, /// Cache for basic account information (nonce, balance, code hash). account_cache: Cache>, } impl ExecutionCache { - /// Get storage value from hierarchical cache. + /// Get storage value from flattened cache. /// /// Returns a `SlotStatus` indicating whether: - /// - `NotCached`: The account's storage cache doesn't exist - /// - `Empty`: The slot exists in the account's cache but is empty + /// - `NotCached`: The storage slot is not in the cache + /// - `Empty`: The slot exists in the cache but is empty /// - `Value`: The slot exists and has a specific value pub(crate) fn get_storage(&self, address: &Address, key: &StorageKey) -> SlotStatus { - match self.storage_cache.get(address) { + match self.storage_cache.get(&(*address, *key)) { None => SlotStatus::NotCached, - Some(account_cache) => account_cache.get_storage(key), + Some(None) => SlotStatus::Empty, + Some(Some(value)) => SlotStatus::Value(value), } } - /// Insert storage value into hierarchical cache + /// Insert storage value into flattened cache pub(crate) fn insert_storage( &self, address: Address, key: StorageKey, value: Option, ) { - self.insert_storage_bulk(address, [(key, value)]); + self.storage_cache.insert((address, key), value); } - /// Insert multiple storage values into hierarchical cache for a single account + /// Insert multiple storage values into flattened cache for a single account /// - /// This method is optimized for inserting multiple storage values for the same address - /// by doing the account cache lookup only once instead of for each key-value pair. + /// This method inserts multiple storage values for the same address directly + /// into the flattened cache. pub(crate) fn insert_storage_bulk(&self, address: Address, storage_entries: I) where I: IntoIterator)>, { - let account_cache = self.storage_cache.get(&address).unwrap_or_else(|| { - let account_cache = AccountStorageCache::default(); - self.storage_cache.insert(address, account_cache.clone()); - account_cache - }); - for (key, value) in storage_entries { - account_cache.insert_storage(key, value); + self.storage_cache.insert((address, key), value); } } /// Invalidate storage for specific account pub(crate) fn invalidate_account_storage(&self, address: &Address) { - self.storage_cache.invalidate(address); + // With flattened cache, we need to remove all entries for this address + // Collect all keys for this address and invalidate them + let keys_to_invalidate: Vec<_> = self + .storage_cache + .iter() + .filter_map(|entry| if entry.key().0 == *address { Some(*entry.key()) } else { None }) + .collect(); + + for key in keys_to_invalidate { + self.storage_cache.invalidate(&key); + } } /// Returns the total number of storage slots cached across all accounts pub(crate) fn total_storage_slots(&self) -> usize { - self.storage_cache.iter().map(|addr| addr.len()).sum() + self.storage_cache.entry_count() as usize } /// Inserts the post-execution state changes into the cache. @@ -452,11 +456,11 @@ impl ExecutionCacheBuilder { const TIME_TO_IDLE: Duration = Duration::from_secs(3600); // 1 hour let storage_cache = CacheBuilder::new(self.storage_cache_entries) - .weigher(|_key: &Address, value: &AccountStorageCache| -> u32 { - // values based on results from measure_storage_cache_overhead test - let base_weight = 39_000; - let slots_weight = value.len() * 218; - (base_weight + slots_weight) as u32 + .weigher(|_key: &(Address, StorageKey), _value: &Option| -> u32 { + // Size of composite key (Address + StorageKey) + Option + // Address: 20 bytes, StorageKey: 32 bytes, Option: 33 bytes + // Plus some overhead for the hash map entry + 120 as u32 }) .max_capacity(storage_cache_size) .time_to_live(EXPIRY_TIME) @@ -573,56 +577,6 @@ impl SavedCache { } } -/// Cache for an individual account's storage slots. -/// -/// This represents the second level of the hierarchical storage cache. -/// Each account gets its own `AccountStorageCache` to store accessed storage slots. -#[derive(Debug, Clone)] -pub(crate) struct AccountStorageCache { - /// Map of storage keys to their cached values. - slots: Cache>, -} - -impl AccountStorageCache { - /// Create a new [`AccountStorageCache`] - pub(crate) fn new(max_slots: u64) -> Self { - Self { - slots: CacheBuilder::new(max_slots).build_with_hasher(DefaultHashBuilder::default()), - } - } - - /// Get a storage value from this account's cache. - /// - `NotCached`: The slot is not in the cache - /// - `Empty`: The slot is empty - /// - `Value`: The slot has a specific value - pub(crate) fn get_storage(&self, key: &StorageKey) -> SlotStatus { - match self.slots.get(key) { - None => SlotStatus::NotCached, - Some(None) => SlotStatus::Empty, - Some(Some(value)) => SlotStatus::Value(value), - } - } - - /// Insert a storage value - pub(crate) fn insert_storage(&self, key: StorageKey, value: Option) { - self.slots.insert(key, value); - } - - /// Returns the number of slots in the cache - pub(crate) fn len(&self) -> usize { - self.slots.entry_count() as usize - } -} - -impl Default for AccountStorageCache { - fn default() -> Self { - // With weigher and max_capacity in place, this number represents - // the maximum number of entries that can be stored, not the actual - // memory usage which is controlled by storage cache's max_capacity. - Self::new(1_000_000) - } -} - #[cfg(test)] mod tests { use super::*; @@ -697,32 +651,36 @@ mod tests { #[test] fn measure_storage_cache_overhead() { - let (base_overhead, cache) = measure_allocation(|| AccountStorageCache::new(1000)); - println!("Base AccountStorageCache overhead: {base_overhead} bytes"); + let (base_overhead, cache) = + measure_allocation(|| ExecutionCacheBuilder::default().build_caches(1000)); + println!("Base ExecutionCache overhead: {base_overhead} bytes"); let mut rng = rand::rng(); + let address = Address::random(); let key = StorageKey::random(); let value = StorageValue::from(rng.random::()); let (first_slot, _) = measure_allocation(|| { - cache.insert_storage(key, Some(value)); + cache.insert_storage(address, key, Some(value)); }); println!("First slot insertion overhead: {first_slot} bytes"); const TOTAL_SLOTS: usize = 10_000; let (test_slots, _) = measure_allocation(|| { for _ in 0..TOTAL_SLOTS { + let addr = Address::random(); let key = StorageKey::random(); let value = StorageValue::from(rng.random::()); - cache.insert_storage(key, Some(value)); + cache.insert_storage(addr, key, Some(value)); } }); println!("Average overhead over {} slots: {} bytes", TOTAL_SLOTS, test_slots / TOTAL_SLOTS); println!("\nTheoretical sizes:"); + println!("Address size: {} bytes", size_of::
()); println!("StorageKey size: {} bytes", size_of::()); println!("StorageValue size: {} bytes", size_of::()); println!("Option size: {} bytes", size_of::>()); - println!("Option size: {} bytes", size_of::>()); + println!("(Address, StorageKey) size: {} bytes", size_of::<(Address, StorageKey)>()); } #[test] From 415c3b7136483b13d83a1bc3401f49ff08955919 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:04:07 -0400 Subject: [PATCH 2/3] perf: invalidate all destroyed accounts at once --- crates/engine/tree/src/tree/cached_state.rs | 38 +++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/crates/engine/tree/src/tree/cached_state.rs b/crates/engine/tree/src/tree/cached_state.rs index 4c0fd58a623..78b80f86555 100644 --- a/crates/engine/tree/src/tree/cached_state.rs +++ b/crates/engine/tree/src/tree/cached_state.rs @@ -1,5 +1,8 @@ //! Execution cache implementation for block processing. -use alloy_primitives::{Address, StorageKey, StorageValue, B256}; +use alloy_primitives::{ + map::{DefaultHashBuilder, HashSet}, + Address, StorageKey, StorageValue, B256, +}; use metrics::Gauge; use mini_moka::sync::CacheBuilder; use reth_errors::ProviderResult; @@ -14,7 +17,6 @@ use reth_trie::{ updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, }; -use revm_primitives::map::DefaultHashBuilder; use std::{sync::Arc, time::Duration}; use tracing::trace; @@ -345,26 +347,22 @@ impl ExecutionCache { } } - /// Invalidate storage for specific account - pub(crate) fn invalidate_account_storage(&self, address: &Address) { - // With flattened cache, we need to remove all entries for this address - // Collect all keys for this address and invalidate them - let keys_to_invalidate: Vec<_> = self - .storage_cache - .iter() - .filter_map(|entry| if entry.key().0 == *address { Some(*entry.key()) } else { None }) - .collect(); - - for key in keys_to_invalidate { - self.storage_cache.invalidate(&key); - } - } - /// Returns the total number of storage slots cached across all accounts pub(crate) fn total_storage_slots(&self) -> usize { self.storage_cache.entry_count() as usize } + /// Invalidates the storage for all addresses in the set + pub(crate) fn invalidate_storages(&self, addresses: HashSet<&Address>) { + let storage_entries = self + .storage_cache + .iter() + .filter_map(|entry| addresses.contains(&entry.key().0).then_some(*entry.key())); + for key in storage_entries { + self.storage_cache.invalidate(&key) + } + } + /// Inserts the post-execution state changes into the cache. /// /// This method is called after transaction execution to update the cache with @@ -389,6 +387,7 @@ impl ExecutionCache { self.code_cache.insert(*code_hash, Some(Bytecode(bytecode.clone()))); } + let mut invalidated_accounts = HashSet::default(); for (addr, account) in &state_updates.state { // If the account was not modified, as in not changed and not destroyed, then we have // nothing to do w.r.t. this particular account and can move on @@ -401,7 +400,7 @@ impl ExecutionCache { // Invalidate the account cache entry if destroyed self.account_cache.invalidate(addr); - self.invalidate_account_storage(addr); + invalidated_accounts.insert(addr); continue } @@ -428,6 +427,9 @@ impl ExecutionCache { self.account_cache.insert(*addr, Some(Account::from(account_info))); } + // invalidate storage for all destroyed acocunts + self.invalidate_storages(invalidated_accounts); + Ok(()) } } From ea9affc223670d264b89fd3c74f52493e2b0ab68 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:09:28 -0400 Subject: [PATCH 3/3] chore: make clippy happy --- crates/engine/tree/src/tree/cached_state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/tree/cached_state.rs b/crates/engine/tree/src/tree/cached_state.rs index 78b80f86555..2864858c3bb 100644 --- a/crates/engine/tree/src/tree/cached_state.rs +++ b/crates/engine/tree/src/tree/cached_state.rs @@ -302,7 +302,7 @@ pub(crate) struct ExecutionCache { /// Cache for contract bytecode, keyed by code hash. code_cache: Cache>, - /// Flattened storage cache: composite key of (Address, StorageKey) maps directly to values. + /// Flattened storage cache: composite key of (`Address`, `StorageKey`) maps directly to values. storage_cache: Cache<(Address, StorageKey), Option>, /// Cache for basic account information (nonce, balance, code hash). @@ -462,7 +462,7 @@ impl ExecutionCacheBuilder { // Size of composite key (Address + StorageKey) + Option // Address: 20 bytes, StorageKey: 32 bytes, Option: 33 bytes // Plus some overhead for the hash map entry - 120 as u32 + 120_u32 }) .max_capacity(storage_cache_size) .time_to_live(EXPIRY_TIME)