use a partial SMT to represent storage entries in accounts query

CHANGELOG.md

@@ -23,6 +23,7 @@
 * [BREAKING] Fixed `createP2IDNote` and `createP2IDENote` convenience functions in the Web Client ([#1142](https://github.com/0xMiden/miden-client/pull/1142)).
 * Reexported utils to parse token amounts as base units ([#1161](https://github.com/0xMiden/miden-client/pull/1161)).
 * [BREAKING] Rename `export/importNote` to `export/importNoteFile`, expose serialization functions for `Note` in Web Client ([#1159](https://github.com/0xMiden/miden-client/pull/1159)).
+* [BREAKING] Change protobuf `AccountStateHeader` definition for reduced redundancy ([#1167](https://github.com/0xMiden/miden-client/pull/1167)).
 * Added support for `MockRpcApi` to web client ([#1096](https://github.com/0xMiden/miden-client/pull/1096)).
 
 ### Features

Cargo.toml

@@ -23,15 +23,17 @@ version      = "0.11.0"
 [workspace.dependencies]
 # Miden dependencies
 miden-lib = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base" }
-miden-node-block-producer = { branch = "next", git = "https://github.com/0xMiden/miden-node" }
-miden-node-ntx-builder = { branch = "next", git = "https://github.com/0xMiden/miden-node" }
-miden-node-proto-build = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-node" }
-miden-node-rpc = { branch = "next", git = "https://github.com/0xMiden/miden-node" }
-miden-node-store = { branch = "next", git = "https://github.com/0xMiden/miden-node" }
-miden-node-utils = { branch = "next", git = "https://github.com/0xMiden/miden-node" }
+miden-node-block-producer = { branch = "bernhard-617-batch-proof", git = "https://github.com/0xMiden/miden-node" }
+miden-node-ntx-builder = { branch = "bernhard-617-batch-proof", git = "https://github.com/0xMiden/miden-node" }
+miden-node-proto-build = { branch = "bernhard-617-batch-proof", default-features = false, git = "https://github.com/0xMiden/miden-node" }
+miden-node-rpc = { branch = "bernhard-617-batch-proof", git = "https://github.com/0xMiden/miden-node" }
+miden-node-store = { branch = "bernhard-617-batch-proof", git = "https://github.com/0xMiden/miden-node" }
+miden-node-utils = { branch = "bernhard-617-batch-proof", git = "https://github.com/0xMiden/miden-node" }
 miden-objects = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base" }
-miden-remote-prover = { branch = "next", features = ["concurrent"], git = "https://github.com/0xMiden/miden-node" }
-miden-remote-prover-client = { branch = "next", default-features = false, features = [
+miden-remote-prover = { branch = "bernhard-617-batch-proof", features = [
+  "concurrent",
+], git = "https://github.com/0xMiden/miden-node" }
+miden-remote-prover-client = { branch = "bernhard-617-batch-proof", default-features = false, features = [
   "tx-prover",
 ], git = "https://github.com/0xMiden/miden-node" }
 miden-testing = { branch = "next", default-features = false, features = [
@@ -52,10 +54,11 @@ tracing     = { version = "0.1" }
 # Pedantic lints are set to a lower priority which allows lints in the group to be selectively enabled.
 pedantic = { level = "warn", priority = -1 }
 # cast_possible_truncation =   "allow"  # Overly many instances especially regarding indices.
-ignored_unit_patterns       = "allow" # Stylistic choice.
-missing_errors_doc          = "allow" # TODO: fixup and enable this.
-missing_panics_doc          = "allow" # TODO: fixup and enable this.
-module_name_repetitions     = "allow" # Many triggers, and is a stylistic choice.
-must_use_candidate          = "allow" # This marks many fn's which isn't helpful.
-should_panic_without_expect = "allow" # We don't care about the specific panic message.
+from_iter_instead_of_collect = "allow" # from_iter oftentimes is more expressive
+ignored_unit_patterns        = "allow" # Stylistic choice.
+missing_errors_doc           = "allow" # TODO: fixup and enable this.
+missing_panics_doc           = "allow" # TODO: fixup and enable this.
+module_name_repetitions      = "allow" # Many triggers, and is a stylistic choice.
+must_use_candidate           = "allow" # This marks many fn's which isn't helpful.
+should_panic_without_expect  = "allow" # We don't care about the specific panic message.
 # End of pedantic lints.

crates/rust-client/src/rpc/domain/account.rs

@@ -11,7 +11,7 @@ use miden_objects::account::{
     AccountStorageHeader,
 };
 use miden_objects::block::{AccountWitness, BlockNumber};
-use miden_objects::crypto::merkle::{MerklePath, SmtProof};
+use miden_objects::crypto::merkle::{MerklePath, PartialSmt};
 use miden_tx::utils::{Deserializable, Serializable, ToHex};
 use thiserror::Error;
 
@@ -175,16 +175,18 @@ impl proto::rpc_store::account_proofs::account_proof::AccountStateHeader {
         account_id: AccountId,
         known_account_codes: &BTreeMap<Word, AccountCode>,
     ) -> Result<StateHeaders, crate::rpc::RpcError> {
+        use miden_objects::crypto::merkle::PartialSmt;
+
         use crate::rpc::RpcError;
         use crate::rpc::domain::MissingFieldHelper;
-        use crate::rpc::generated::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapProof;
 
         let proto::rpc_store::account_proofs::account_proof::AccountStateHeader {
             header,
             storage_header,
             account_code,
-            storage_maps,
+            partial_storage_smts,
         } = self;
+
         let account_header = header
             .ok_or(
                 proto::rpc_store::account_proofs::account_proof::AccountStateHeader::missing_field(
@@ -213,28 +215,28 @@ impl proto::rpc_store::account_proofs::account_proof::AccountStateHeader {
             }
         };
 
-        // Get map values into slot |-> (key, value, proof) mapping
-        let mut storage_slot_proofs: BTreeMap<u8, Vec<SmtProof>> = BTreeMap::new();
-        for StorageSlotMapProof { storage_slot, smt_proof } in storage_maps {
-            let proof = SmtProof::read_from_bytes(&smt_proof)?;
-            match storage_slot_proofs
-                .get_mut(&(u8::try_from(storage_slot).expect("there are no more than 256 slots")))
-            {
-                Some(list) => list.push(proof),
-                None => {
-                    _ = storage_slot_proofs.insert(
-                        u8::try_from(storage_slot).expect("only 256 storage slots"),
-                        vec![proof],
-                    );
-                },
+        let partial_storage_smts = Result::<BTreeMap<u8, PartialSmt>, _>::from_iter(
+            partial_storage_smts.into_iter().map(|entry| {
+                let slot = u8::try_from(entry.storage_slot)
+                    .map_err(crate::rpc::RpcError::SlotOutOfBounds)?;
+                let partial_smt = PartialSmt::read_from_bytes(&entry.partial_smt)?;
+                Ok::<_, crate::rpc::RpcError>((slot, partial_smt))
+            }),
+        )?;
+
+        partial_storage_smts.iter().try_for_each(|(_slot, partial_smt)| {
+            for (key, _value) in partial_smt.entries() {
+                // validate all required inner nodes of the merkle path for the specific leaf exist
+                let _proof = partial_smt.open(key)?;
             }
-        }
+            Ok::<_, crate::rpc::RpcError>(())
+        })?;
 
         Ok(StateHeaders {
             account_header,
             storage_header,
             code,
-            storage_slots: storage_slot_proofs,
+            storage_slots: partial_storage_smts,
         })
     }
 }
@@ -252,7 +254,7 @@ pub struct StateHeaders {
     pub account_header: AccountHeader,
     pub storage_header: AccountStorageHeader,
     pub code: AccountCode,
-    pub storage_slots: BTreeMap<StorageSlotIndex, Vec<SmtProof>>,
+    pub storage_slots: BTreeMap<StorageSlotIndex, PartialSmt>,
 }
 
 /// Represents a proof of existence of an account's state at a specific block number.

crates/rust-client/src/rpc/errors.rs

@@ -29,6 +29,10 @@ pub enum RpcError {
     NoteNotFound(NoteId),
     #[error("rpc request failed for {0}: {1}")]
     RequestError(String, String),
+    #[error("merkle proof is not contained")]
+    MerkleError(#[from] MerkleError),
+    #[error("slot index out of bounds")]
+    SlotOutOfBounds(#[source] TryFromIntError),
 }
 
 impl From<DeserializationError> for RpcError {

crates/rust-client/src/rpc/generated/nostd/rpc_store.rs

@@ -99,14 +99,26 @@ pub mod account_proofs {
             /// the current one.
             #[prost(bytes = "vec", optional, tag = "3")]
             pub account_code: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
-            /// Storage slots information for this account
-            #[prost(message, repeated, tag = "4")]
-            pub storage_maps: ::prost::alloc::vec::Vec<
-                account_state_header::StorageSlotMapProof,
+            /// A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries.
+            #[prost(message, repeated, tag = "5")]
+            pub partial_storage_smts: ::prost::alloc::vec::Vec<
+                account_state_header::StorageSlotMapPartialSmt,
             >,
         }
         /// Nested message and enum types in `AccountStateHeader`.
         pub mod account_state_header {
+            /// Represents a single storage slot with the requested keys and their respective values.
+            #[derive(Clone, PartialEq, ::prost::Message)]
+            pub struct StorageSlotMapPartialSmt {
+                /// The storage slot index (\[0..255\]).
+                #[prost(uint32, tag = "1")]
+                pub storage_slot: u32,
+                /// Merkle proofs of the map value as partial sparse merkle tree for compression.
+                /// The respective rust types is `SparseMerkleTree` and the transformation to and from
+                /// bytes is done via the traits `Serializable::to_bytes` and `Deserializable::from_bytes`.
+                #[prost(bytes = "vec", tag = "2")]
+                pub partial_smt: ::prost::alloc::vec::Vec<u8>,
+            }
             /// Represents a single storage slot with the requested keys and their respective values.
             #[derive(Clone, PartialEq, ::prost::Message)]
             pub struct StorageSlotMapProof {

crates/rust-client/src/rpc/generated/std/rpc_store.rs

@@ -99,14 +99,26 @@ pub mod account_proofs {
             /// the current one.
             #[prost(bytes = "vec", optional, tag = "3")]
             pub account_code: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
-            /// Storage slots information for this account
-            #[prost(message, repeated, tag = "4")]
-            pub storage_maps: ::prost::alloc::vec::Vec<
-                account_state_header::StorageSlotMapProof,
+            /// A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries.
+            #[prost(message, repeated, tag = "5")]
+            pub partial_storage_smts: ::prost::alloc::vec::Vec<
+                account_state_header::StorageSlotMapPartialSmt,
             >,
         }
         /// Nested message and enum types in `AccountStateHeader`.
         pub mod account_state_header {
+            /// Represents a single storage slot with the requested keys and their respective values.
+            #[derive(Clone, PartialEq, ::prost::Message)]
+            pub struct StorageSlotMapPartialSmt {
+                /// The storage slot index (\[0..255\]).
+                #[prost(uint32, tag = "1")]
+                pub storage_slot: u32,
+                /// Merkle proofs of the map value as partial sparse merkle tree for compression.
+                /// The respective rust types is `SparseMerkleTree` and the transformation to and from
+                /// bytes is done via the traits `Serializable::to_bytes` and `Deserializable::from_bytes`.
+                #[prost(bytes = "vec", tag = "2")]
+                pub partial_smt: ::prost::alloc::vec::Vec<u8>,
+            }
             /// Represents a single storage slot with the requested keys and their respective values.
             #[derive(Clone, PartialEq, ::prost::Message)]
             pub struct StorageSlotMapProof {

crates/rust-client/src/test_utils/mock.rs

@@ -17,7 +17,7 @@ use miden_objects::account::{
 };
 use miden_objects::asset::Asset;
 use miden_objects::block::{BlockHeader, BlockNumber, ProvenBlock};
-use miden_objects::crypto::merkle::{Forest, Mmr, MmrProof, SmtProof};
+use miden_objects::crypto::merkle::{Forest, Mmr, MmrProof, PartialSmt, SmtProof};
 use miden_objects::note::{NoteId, NoteTag, Nullifier};
 use miden_objects::transaction::ProvenTransaction;
 use miden_objects::{LexicographicWord, Word};
@@ -389,7 +389,7 @@ impl NodeRpcClient for MockRpcApi {
                                 .iter()
                                 .map(|map_key| storage_map.open(map_key))
                                 .collect::<Vec<_>>();
-                            storage_slots.insert(*index, proofs);
+                            storage_slots.insert(*index, PartialSmt::from_proofs(proofs)?);
                         } else {
                             panic!("Storage slot at index {} is not a map", index);
                         }

crates/rust-client/src/transaction/request/foreign.rs

@@ -136,8 +136,8 @@ impl TryFrom<AccountProof> for AccountInputs {
         {
             // discard slot indices - not needed for execution
             let mut storage_map_proofs = Vec::with_capacity(storage_slots.len());
-            for (_, slots) in storage_slots {
-                let storage_map = PartialStorageMap::new(PartialSmt::from_proofs(slots)?);
+            for (_, partial_smt) in storage_slots {
+                let storage_map = PartialStorageMap::new(partial_smt);
                 storage_map_proofs.push(storage_map);
             }