From a8f3c0b78d2910bb5ba8d6213b43a8ba24ca6d43 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Fri, 12 Dec 2025 16:29:50 -0300 Subject: [PATCH 1/9] draft: add pagination to GetNetworkAccountIds --- crates/ntx-builder/src/builder.rs | 2 +- crates/ntx-builder/src/store.rs | 21 ++++++++++++++++--- crates/proto/src/generated/store.rs | 18 +++++++++++----- crates/store/src/db/mod.rs | 11 +++++++--- .../store/src/db/models/queries/accounts.rs | 9 ++++---- crates/store/src/server/ntx_builder.rs | 15 ++++++++++--- crates/store/src/state.rs | 7 +++++-- proto/proto/internal/store.proto | 8 ++++--- 8 files changed, 67 insertions(+), 24 deletions(-) diff --git a/crates/ntx-builder/src/builder.rs b/crates/ntx-builder/src/builder.rs index c74f2dacd..d239cf7a6 100644 --- a/crates/ntx-builder/src/builder.rs +++ b/crates/ntx-builder/src/builder.rs @@ -153,7 +153,7 @@ impl NetworkTransactionBuilder { }; // Create initial set of actors based on all known network accounts. - let account_ids = store.get_network_account_ids().await?; + let account_ids = store.get_network_account_ids(None).await?; for account_id in account_ids { if let Ok(account_prefix) = NetworkAccountPrefix::try_from(account_id) { self.coordinator diff --git a/crates/ntx-builder/src/store.rs b/crates/ntx-builder/src/store.rs index 9073ed3df..44aacb6e0 100644 --- a/crates/ntx-builder/src/store.rs +++ b/crates/ntx-builder/src/store.rs @@ -1,14 +1,16 @@ +use std::ops::RangeInclusive; use std::time::Duration; use miden_node_proto::clients::{Builder, StoreNtxBuilderClient}; use miden_node_proto::domain::account::NetworkAccountPrefix; use miden_node_proto::domain::note::NetworkNote; use miden_node_proto::errors::ConversionError; +use miden_node_proto::generated::rpc::BlockRange; use miden_node_proto::generated::{self as proto}; use miden_node_proto::try_convert; use miden_objects::Word; use miden_objects::account::{Account, AccountId}; -use miden_objects::block::BlockHeader; +use miden_objects::block::{BlockHeader, BlockNumber}; use miden_objects::crypto::merkle::{Forest, MmrPeaks, PartialMmr}; use miden_objects::note::NoteScript; use miden_tx::utils::Deserializable; @@ -174,8 +176,21 @@ impl StoreClient { // TODO: add pagination. #[instrument(target = COMPONENT, name = "store.client.get_network_account_ids", skip_all, err)] - pub async fn get_network_account_ids(&self) -> Result, StoreError> { - let response = self.inner.clone().get_network_account_ids(()).await?.into_inner(); + pub async fn get_network_account_ids( + &self, + block_range: Option>, + ) -> Result, StoreError> { + let block_range = match block_range { + Some(range) => range, + None => BlockNumber::from(0)..=BlockNumber::from(u32::MAX), + }; + + let response = self + .inner + .clone() + .get_network_account_ids(Into::::into(block_range)) + .await? + .into_inner(); let accounts: Result, ConversionError> = response .account_ids diff --git a/crates/proto/src/generated/store.rs b/crates/proto/src/generated/store.rs index 33703e88a..1e664784d 100644 --- a/crates/proto/src/generated/store.rs +++ b/crates/proto/src/generated/store.rs @@ -206,8 +206,11 @@ pub struct UnconsumedNetworkNotes { /// Represents the result of getting the network account ids. #[derive(Clone, PartialEq, ::prost::Message)] pub struct NetworkAccountIdList { + /// Pagination information. + #[prost(message, optional, tag = "1")] + pub pagination_info: ::core::option::Option, /// The list of network account ids. - #[prost(message, repeated, tag = "1")] + #[prost(message, repeated, tag = "2")] pub account_ids: ::prost::alloc::vec::Vec, } /// Current blockchain data based on the requested block number. @@ -2408,7 +2411,7 @@ pub mod ntx_builder_client { /// Returns a list of all network account ids. pub async fn get_network_account_ids( &mut self, - request: impl tonic::IntoRequest<()>, + request: impl tonic::IntoRequest, ) -> std::result::Result< tonic::Response, tonic::Status, @@ -2508,7 +2511,7 @@ pub mod ntx_builder_server { /// Returns a list of all network account ids. async fn get_network_account_ids( &self, - request: tonic::Request<()>, + request: tonic::Request, ) -> std::result::Result< tonic::Response, tonic::Status, @@ -2806,14 +2809,19 @@ pub mod ntx_builder_server { "/store.NtxBuilder/GetNetworkAccountIds" => { #[allow(non_camel_case_types)] struct GetNetworkAccountIdsSvc(pub Arc); - impl tonic::server::UnaryService<()> + impl< + T: NtxBuilder, + > tonic::server::UnaryService for GetNetworkAccountIdsSvc { type Response = super::NetworkAccountIdList; type Future = BoxFuture< tonic::Response, tonic::Status, >; - fn call(&mut self, request: tonic::Request<()>) -> Self::Future { + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { ::get_network_account_ids(&inner, request) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 7b48684ed..ebcd46469 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -435,9 +435,14 @@ impl Db { /// Loads all network account IDs from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_all_network_account_ids(&self) -> Result> { - self.transact("Get all network account IDs", queries::select_all_network_account_ids) - .await + pub async fn select_all_network_account_ids( + &self, + block_range: RangeInclusive, + ) -> Result> { + self.transact("Get all network account IDs", move |conn| { + queries::select_all_network_account_ids(conn, block_range) + }) + .await } #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index fe5197359..67035aec0 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -414,11 +414,12 @@ pub(crate) fn select_all_accounts( /// A vector with network account IDs, or an error. pub(crate) fn select_all_network_account_ids( conn: &mut SqliteConnection, + block_range: RangeInclusive, ) -> Result, DatabaseError> { - let account_ids_raw: Vec> = QueryDsl::select( - schema::accounts::table.filter(schema::accounts::network_account_id_prefix.is_not_null()), - schema::accounts::account_id, - ) + let account_ids_raw: Vec> = Box::new(QueryDsl::select( + schema::accounts::table.filter(schema::accounts::network_account_id_prefix.is_not_null()), + schema::accounts::account_id, + ).filter(schema::accounts::block_number.between(block_range.start(), block_range.end()))) .load::>(conn)?; let account_ids = account_ids_raw diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 91bc5a648..ab7e37ab1 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -1,6 +1,7 @@ use std::num::{NonZero, TryFromIntError}; use miden_node_proto::domain::account::{AccountInfo, NetworkAccountPrefix}; +use miden_node_proto::generated::rpc::BlockRange; use miden_node_proto::generated::store::ntx_builder_server; use miden_node_proto::generated::{self as proto}; use miden_node_utils::ErrorReport; @@ -161,14 +162,22 @@ impl ntx_builder_server::NtxBuilder for StoreApi { )] async fn get_network_account_ids( &self, - _request: Request<()>, + request: Request, ) -> Result, Status> { - let account_ids = self.state.get_all_network_accounts().await.map_err(internal_error)?; + let block_range = request.into_inner(); + let block_range = BlockNumber::from(block_range.block_from) + ..=BlockNumber::from(block_range.block_to.unwrap_or(0)); + + let account_ids = + self.state.get_all_network_accounts(block_range).await.map_err(internal_error)?; let account_ids: Vec = account_ids.into_iter().map(Into::into).collect(); - Ok(Response::new(proto::store::NetworkAccountIdList { account_ids })) + Ok(Response::new(proto::store::NetworkAccountIdList { + account_ids, + pagination_info: None, + })) } #[instrument( diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 2dd281301..500ae8303 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -919,8 +919,11 @@ impl State { } /// Returns account IDs for all public (on-chain) network accounts. - pub async fn get_all_network_accounts(&self) -> Result, DatabaseError> { - self.db.select_all_network_account_ids().await + pub async fn get_all_network_accounts( + &self, + block_range: RangeInclusive, + ) -> Result, DatabaseError> { + self.db.select_all_network_account_ids(block_range).await } /// Returns the respective account proof with optional details, such as asset and storage diff --git a/proto/proto/internal/store.proto b/proto/proto/internal/store.proto index 05f515ccf..a06279acc 100644 --- a/proto/proto/internal/store.proto +++ b/proto/proto/internal/store.proto @@ -258,7 +258,7 @@ service NtxBuilder { rpc GetNetworkAccountDetailsByPrefix(AccountIdPrefix) returns (MaybeAccountDetails) {} // Returns a list of all network account ids. - rpc GetNetworkAccountIds(google.protobuf.Empty) returns (NetworkAccountIdList) {} + rpc GetNetworkAccountIds(rpc.BlockRange) returns (NetworkAccountIdList) {} // Returns the script for a note by its root. rpc GetNoteScriptByRoot(note.NoteRoot) returns (rpc.MaybeNoteScript) {} @@ -323,11 +323,13 @@ message UnconsumedNetworkNotes { // Represents the result of getting the network account ids. message NetworkAccountIdList { + // Pagination information. + rpc.PaginationInfo pagination_info = 1; + // The list of network account ids. - repeated account.AccountId account_ids = 1; + repeated account.AccountId account_ids = 2; } - // GET CURRENT BLOCKCHAIN DATA // ================================================================================================ From d356cb42f2d260b879101f14e94f1845cc097702 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Tue, 16 Dec 2025 16:43:49 -0300 Subject: [PATCH 2/9] chore: add created_at_block field to accounts table --- .../db/migrations/2025062000000_setup/up.sql | 2 + crates/store/src/db/mod.rs | 2 +- .../store/src/db/models/queries/accounts.rs | 88 +++++++++++++++---- crates/store/src/db/schema.rs | 1 + crates/store/src/server/ntx_builder.rs | 14 ++- crates/store/src/state.rs | 2 +- 6 files changed, 86 insertions(+), 23 deletions(-) diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index aaafb91a8..a526af1e0 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -22,6 +22,7 @@ CREATE TABLE accounts ( vault BLOB, nonce INTEGER, is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id + created_at_block INTEGER NOT NULL, PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK @@ -35,6 +36,7 @@ CREATE TABLE accounts ( CREATE INDEX idx_accounts_network_prefix ON accounts(network_account_id_prefix) WHERE network_account_id_prefix IS NOT NULL; CREATE INDEX idx_accounts_id_block ON accounts(account_id, block_num DESC); CREATE INDEX idx_accounts_latest ON accounts(account_id, is_latest) WHERE is_latest = 1; +CREATE INDEX idx_accounts_created_at_block ON accounts(created_at_block); -- Index for joining with block_headers CREATE INDEX idx_accounts_block_num ON accounts(block_num); -- Index for joining with account_codes diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index ebcd46469..67f0cddf6 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -438,7 +438,7 @@ impl Db { pub async fn select_all_network_account_ids( &self, block_range: RangeInclusive, - ) -> Result> { + ) -> Result<(Vec, BlockNumber)> { self.transact("Get all network account IDs", move |conn| { queries::select_all_network_account_ids(conn, block_range) }) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 67035aec0..178c13cb5 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -415,21 +415,57 @@ pub(crate) fn select_all_accounts( pub(crate) fn select_all_network_account_ids( conn: &mut SqliteConnection, block_range: RangeInclusive, -) -> Result, DatabaseError> { - let account_ids_raw: Vec> = Box::new(QueryDsl::select( - schema::accounts::table.filter(schema::accounts::network_account_id_prefix.is_not_null()), - schema::accounts::account_id, - ).filter(schema::accounts::block_number.between(block_range.start(), block_range.end()))) - .load::>(conn)?; - - let account_ids = account_ids_raw - .into_iter() - .map(|id_bytes| { - AccountId::read_from_bytes(&id_bytes).map_err(DatabaseError::DeserializationError) - }) - .collect::, DatabaseError>>()?; +) -> Result<(Vec, BlockNumber), DatabaseError> { + const ROW_OVERHEAD_BYTES: usize = AccountId::SERIALIZED_SIZE; + const MAX_ROWS: usize = MAX_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; + + if block_range.is_empty() { + return Err(DatabaseError::InvalidBlockRange { + from: *block_range.start(), + to: *block_range.end(), + }); + } + + let account_ids_raw: Vec<(Vec, i64)> = Box::new( + QueryDsl::select( + schema::accounts::table + .filter(schema::accounts::network_account_id_prefix.is_not_null()), + (schema::accounts::account_id, schema::accounts::created_at_block), + ) + .filter( + schema::accounts::block_num + .between(block_range.start().to_raw_sql(), block_range.end().to_raw_sql()), + ) + .order(schema::accounts::created_at_block.asc()) + .limit(i64::try_from(MAX_ROWS + 1).expect("limit fits within i64")), + ) + .load::<(Vec, i64)>(conn)?; - Ok(account_ids) + if let Some(&(_, last_created_at_block)) = account_ids_raw.last() + && account_ids_raw.len() > MAX_ROWS + { + let account_ids = account_ids_raw + .into_iter() + .take_while(|(_, created_at_block)| *created_at_block != last_created_at_block) + .map(|(id_bytes, _)| { + AccountId::read_from_bytes(&id_bytes).map_err(DatabaseError::DeserializationError) + }) + .collect::, DatabaseError>>()?; + + let last_block_included = + BlockNumber::from_raw_sql(last_created_at_block.saturating_sub(1))?; + + Ok((account_ids, last_block_included)) + } else { + let account_ids = account_ids_raw + .into_iter() + .map(|(id_bytes, _)| { + AccountId::read_from_bytes(&id_bytes).map_err(DatabaseError::DeserializationError) + }) + .collect::, DatabaseError>>()?; + + Ok((account_ids, *block_range.end())) + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -801,6 +837,8 @@ pub(crate) fn upsert_accounts( let mut count = 0; for update in accounts { let account_id = update.account_id(); + let account_id_bytes = account_id.to_bytes(); + let block_num_raw = block_num.to_raw_sql(); let network_account_id_prefix = if account_id.is_network() { Some(NetworkAccountPrefix::try_from(account_id)?) @@ -808,6 +846,20 @@ pub(crate) fn upsert_accounts( None }; + // Preserve the original creation block when updating existing accounts. + let created_at_block = QueryDsl::select( + schema::accounts::table.filter( + schema::accounts::account_id + .eq(&account_id_bytes) + .and(schema::accounts::is_latest.eq(true)), + ), + schema::accounts::created_at_block, + ) + .first::(conn) + .optional() + .map_err(DatabaseError::Diesel)? + .unwrap_or(block_num_raw); + // NOTE: we collect storage / asset inserts to apply them only after the account row is // written. The storage and vault tables have FKs pointing to `accounts (account_id, // block_num)`, so inserting them earlier would violate those constraints when inserting a @@ -901,18 +953,18 @@ pub(crate) fn upsert_accounts( diesel::update(schema::accounts::table) .filter( schema::accounts::account_id - .eq(&account_id.to_bytes()) + .eq(&account_id_bytes) .and(schema::accounts::is_latest.eq(true)), ) .set(schema::accounts::is_latest.eq(false)) .execute(conn)?; let account_value = AccountRowInsert { - account_id: account_id.to_bytes(), + account_id: account_id_bytes, network_account_id_prefix: network_account_id_prefix .map(NetworkAccountPrefix::to_raw_sql), account_commitment: update.final_state_commitment().to_bytes(), - block_num: block_num.to_raw_sql(), + block_num: block_num_raw, nonce: full_account.as_ref().map(|account| nonce_to_raw_sql(account.nonce())), storage: full_account.as_ref().map(|account| account.storage().to_bytes()), vault: full_account.as_ref().map(|account| account.vault().to_bytes()), @@ -920,6 +972,7 @@ pub(crate) fn upsert_accounts( .as_ref() .map(|account| account.code().commitment().to_bytes()), is_latest: true, + created_at_block, }; diesel::insert_into(schema::accounts::table) @@ -980,6 +1033,7 @@ pub(crate) struct AccountRowInsert { pub(crate) vault: Option>, pub(crate) nonce: Option, pub(crate) is_latest: bool, + pub(crate) created_at_block: i64, } #[derive(Insertable, AsChangeset, Debug, Clone)] diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 6f36594b9..4c021ef95 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -32,6 +32,7 @@ diesel::table! { nonce -> Nullable, block_num -> BigInt, is_latest -> Bool, + created_at_block -> BigInt, } } diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index ab7e37ab1..3b8208d0d 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -165,10 +165,13 @@ impl ntx_builder_server::NtxBuilder for StoreApi { request: Request, ) -> Result, Status> { let block_range = request.into_inner(); - let block_range = BlockNumber::from(block_range.block_from) - ..=BlockNumber::from(block_range.block_to.unwrap_or(0)); + let chain_tip = self.state.latest_block_num().await; - let account_ids = + let block_from = BlockNumber::from(block_range.block_from); + let block_to = block_range.block_to.map_or(chain_tip, BlockNumber::from); + let block_range = block_from..=block_to; + + let (account_ids, last_block_included) = self.state.get_all_network_accounts(block_range).await.map_err(internal_error)?; let account_ids: Vec = @@ -176,7 +179,10 @@ impl ntx_builder_server::NtxBuilder for StoreApi { Ok(Response::new(proto::store::NetworkAccountIdList { account_ids, - pagination_info: None, + pagination_info: Some(proto::rpc::PaginationInfo { + chain_tip: chain_tip.as_u32(), + block_num: last_block_included.as_u32(), + }), })) } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 500ae8303..fe7a9eb9c 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -922,7 +922,7 @@ impl State { pub async fn get_all_network_accounts( &self, block_range: RangeInclusive, - ) -> Result, DatabaseError> { + ) -> Result<(Vec, BlockNumber), DatabaseError> { self.db.select_all_network_account_ids(block_range).await } From c1ca1e925cb6079ed63f93958f7659442dc54047 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Tue, 16 Dec 2025 16:51:34 -0300 Subject: [PATCH 3/9] docs: add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9df3c1b3..5be2f8e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Added chain tip to the block producer status ([#1419](https://github.com/0xMiden/miden-node/pull/1419)). - The mempool's transaction capacity is now configurable ([#1433](https://github.com/0xMiden/miden-node/pull/1433)). - Renamed card's names in the `miden-network-monitor` binary ([#1441](https://github.com/0xMiden/miden-node/pull/1441)). +- Added pagination to `GetNetworkAccountIds` endpoint ([#1452](https://github.com/0xMiden/miden-node/pull/1452)). ### Changes From e3bf41899683c4352d9baf0c30500b90e77d008d Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Tue, 23 Dec 2025 16:52:43 -0300 Subject: [PATCH 4/9] review: add loop to fetch all account, add docs --- crates/ntx-builder/src/builder.rs | 2 +- crates/ntx-builder/src/store.rs | 77 ++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/crates/ntx-builder/src/builder.rs b/crates/ntx-builder/src/builder.rs index d239cf7a6..c74f2dacd 100644 --- a/crates/ntx-builder/src/builder.rs +++ b/crates/ntx-builder/src/builder.rs @@ -153,7 +153,7 @@ impl NetworkTransactionBuilder { }; // Create initial set of actors based on all known network accounts. - let account_ids = store.get_network_account_ids(None).await?; + let account_ids = store.get_network_account_ids().await?; for account_id in account_ids { if let Ok(account_prefix) = NetworkAccountPrefix::try_from(account_id) { self.coordinator diff --git a/crates/ntx-builder/src/store.rs b/crates/ntx-builder/src/store.rs index 44aacb6e0..852f7c9f7 100644 --- a/crates/ntx-builder/src/store.rs +++ b/crates/ntx-builder/src/store.rs @@ -1,4 +1,3 @@ -use std::ops::RangeInclusive; use std::time::Duration; use miden_node_proto::clients::{Builder, StoreNtxBuilderClient}; @@ -174,34 +173,56 @@ impl StoreClient { Ok(all_notes) } - // TODO: add pagination. + /// Get all network account IDs. + /// + /// Since the `GetNetworkAccountIds` method is paginated, we loop through all pages until we + /// reach the end. #[instrument(target = COMPONENT, name = "store.client.get_network_account_ids", skip_all, err)] - pub async fn get_network_account_ids( - &self, - block_range: Option>, - ) -> Result, StoreError> { - let block_range = match block_range { - Some(range) => range, - None => BlockNumber::from(0)..=BlockNumber::from(u32::MAX), - }; + pub async fn get_network_account_ids(&self) -> Result, StoreError> { + const MAX_ITERATIONS: u32 = 1000; - let response = self - .inner - .clone() - .get_network_account_ids(Into::::into(block_range)) - .await? - .into_inner(); - - let accounts: Result, ConversionError> = response - .account_ids - .into_iter() - .map(|account_id| { - AccountId::read_from_bytes(&account_id.id) - .map_err(|err| ConversionError::deserialization_error("account_id", err)) - }) - .collect(); - - Ok(accounts?) + let block_range = BlockNumber::from(0)..=BlockNumber::from(u32::MAX); + + let mut ids = Vec::new(); + let mut iterations_count = 0; + + loop { + let response = self + .inner + .clone() + .get_network_account_ids(Into::::into(block_range.clone())) + .await? + .into_inner(); + + let accounts: Result, ConversionError> = response + .account_ids + .into_iter() + .map(|account_id| { + AccountId::read_from_bytes(&account_id.id) + .map_err(|err| ConversionError::deserialization_error("account_id", err)) + }) + .collect(); + + let pagination_info = response.pagination_info.ok_or( + ConversionError::MissingFieldInProtobufRepresentation { + entity: "NetworkAccountIdList", + field_name: "pagination_info", + }, + )?; + + ids.extend(accounts?); + iterations_count += 1; + + if pagination_info.block_num == pagination_info.chain_tip { + break; + } + + if iterations_count >= MAX_ITERATIONS { + return Err(StoreError::MaxIterationsReached("GetNetworkAccountIds".to_string())); + } + } + + Ok(ids) } #[instrument(target = COMPONENT, name = "store.client.get_note_script_by_root", skip_all, err)] @@ -241,4 +262,6 @@ pub enum StoreError { MalformedResponse(String), #[error("failed to parse response")] DeserializationError(#[from] ConversionError), + #[error("max iterations reached: {0}")] + MaxIterationsReached(String), } From c1561c60f58dacbb82cd56326973cb6b958b113a Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Tue, 23 Dec 2025 16:53:06 -0300 Subject: [PATCH 5/9] review: add static check for accounts limit --- crates/store/src/db/models/queries/accounts.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 178c13cb5..da8c3a7db 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -419,6 +419,11 @@ pub(crate) fn select_all_network_account_ids( const ROW_OVERHEAD_BYTES: usize = AccountId::SERIALIZED_SIZE; const MAX_ROWS: usize = MAX_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; + const _: () = assert!( + MAX_ROWS > miden_objects::MAX_ACCOUNTS_PER_BLOCK, + "Block pagination limit must exceed maximum block capacity" + ); + if block_range.is_empty() { return Err(DatabaseError::InvalidBlockRange { from: *block_range.start(), From 88b139e1f27604bce5f3ed0c59cbb82ff225bbd9 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Tue, 23 Dec 2025 16:59:52 -0300 Subject: [PATCH 6/9] fix: lint --- crates/store/src/db/models/queries/accounts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 85dc43aa0..59464ecd3 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -420,7 +420,7 @@ pub(crate) fn select_all_network_account_ids( block_range: RangeInclusive, ) -> Result<(Vec, BlockNumber), DatabaseError> { const ROW_OVERHEAD_BYTES: usize = AccountId::SERIALIZED_SIZE; - const MAX_ROWS: usize = MAX_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; + const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; const _: () = assert!( MAX_ROWS > miden_objects::MAX_ACCOUNTS_PER_BLOCK, From a4a7fea62ec7fce80746f257121243c1b5ceb207 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sun, 28 Dec 2025 02:24:48 -0800 Subject: [PATCH 7/9] chore: fix merge error --- crates/store/src/db/models/queries/accounts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 561b5978c..2089e5443 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -423,7 +423,7 @@ pub(crate) fn select_all_network_account_ids( const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; const _: () = assert!( - MAX_ROWS > miden_objects::MAX_ACCOUNTS_PER_BLOCK, + MAX_ROWS > miden_protocol::MAX_ACCOUNTS_PER_BLOCK, "Block pagination limit must exceed maximum block capacity" ); From 83b0bb4ae468139fd998254bc7747d2c19141ef4 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Mon, 5 Jan 2026 11:46:32 -0300 Subject: [PATCH 8/9] review: improve assert comment --- crates/store/src/db/models/queries/accounts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 2089e5443..95358c3c4 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -424,7 +424,7 @@ pub(crate) fn select_all_network_account_ids( const _: () = assert!( MAX_ROWS > miden_protocol::MAX_ACCOUNTS_PER_BLOCK, - "Block pagination limit must exceed maximum block capacity" + "Block pagination limit must exceed maximum block capacity to uphold assumed logic invariant" ); if block_range.is_empty() { From cd46c299380036b084f9951f673047e0a9c6c140 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Mon, 5 Jan 2026 13:45:56 -0300 Subject: [PATCH 9/9] review: add pagination to docs --- crates/ntx-builder/src/store.rs | 4 ++++ crates/store/src/db/mod.rs | 14 +++++++++++++- .../store/src/db/models/queries/accounts.rs | 19 ++++++++++++++----- crates/store/src/server/ntx_builder.rs | 9 ++++++++- crates/store/src/state.rs | 9 ++++++++- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/crates/ntx-builder/src/store.rs b/crates/ntx-builder/src/store.rs index 6723563b1..3b6a565b6 100644 --- a/crates/ntx-builder/src/store.rs +++ b/crates/ntx-builder/src/store.rs @@ -177,6 +177,10 @@ impl StoreClient { /// /// Since the `GetNetworkAccountIds` method is paginated, we loop through all pages until we /// reach the end. + /// + /// Each page can return up to `MAX_RESPONSE_PAYLOAD_BYTES / AccountId::SERIALIZED_SIZE` + /// accounts (~289,000). With 1000 iterations, this supports up to ~524 million network + /// accounts, which is assumed to be sufficient for the foreseeable future. #[instrument(target = COMPONENT, name = "store.client.get_network_account_ids", skip_all, err)] pub async fn get_network_account_ids(&self) -> Result, StoreError> { const MAX_ITERATIONS: u32 = 1000; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 3de01c405..f928fc345 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -433,7 +433,19 @@ impl Db { .await } - /// Loads all network account IDs from the DB. + /// Returns network account IDs within the specified block range (based on account creation + /// block). + /// + /// The function may return fewer accounts than exist in the range if the result would exceed + /// `MAX_RESPONSE_PAYLOAD_BYTES / AccountId::SERIALIZED_SIZE` rows. In this case, the result is + /// truncated at a block boundary to ensure all accounts from included blocks are returned. + /// + /// # Returns + /// + /// A tuple containing: + /// - A vector of network account IDs. + /// - The last block number that was fully included in the result. When truncated, this will be + /// less than the requested range end. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_network_account_ids( &self, diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 95358c3c4..167ebdd63 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -410,11 +410,19 @@ pub(crate) fn select_all_accounts( Ok(account_infos) } -/// Returns all network account IDs. +/// Returns network account IDs within the specified block range (based on account creation +/// block). +/// +/// The function may return fewer accounts than exist in the range if the result would exceed +/// `MAX_RESPONSE_PAYLOAD_BYTES / AccountId::SERIALIZED_SIZE` rows. In this case, the result is +/// truncated at a block boundary to ensure all accounts from included blocks are returned. /// /// # Returns /// -/// A vector with network account IDs, or an error. +/// A tuple containing: +/// - A vector of network account IDs. +/// - The last block number that was fully included in the result. When truncated, this will be less +/// than the requested range end. pub(crate) fn select_all_network_account_ids( conn: &mut SqliteConnection, block_range: RangeInclusive, @@ -449,9 +457,10 @@ pub(crate) fn select_all_network_account_ids( ) .load::<(Vec, i64)>(conn)?; - if let Some(&(_, last_created_at_block)) = account_ids_raw.last() - && account_ids_raw.len() > MAX_ROWS - { + if account_ids_raw.len() > MAX_ROWS { + // SAFETY: We just checked that len > MAX_ROWS, so the vec is not empty. + let last_created_at_block = account_ids_raw.last().expect("vec is not empty").1; + let account_ids = account_ids_raw .into_iter() .take_while(|(_, created_at_block)| *created_at_block != last_created_at_block) diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index cd20b5afd..4106a121d 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -151,7 +151,14 @@ impl ntx_builder_server::NtxBuilder for StoreApi { })) } - // TODO: add pagination. + /// Returns network account IDs within the specified block range (based on account creation + /// block). + /// + /// The function may return fewer accounts than exist in the range if the result would exceed + /// `MAX_RESPONSE_PAYLOAD_BYTES / AccountId::SERIALIZED_SIZE` rows. In this case, the result is + /// truncated at a block boundary to ensure all accounts from included blocks are returned. + /// + /// The response includes pagination info with the last block number that was fully included. #[instrument( parent = None, target = COMPONENT, diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 33aa9ec7f..02e3bb60d 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -906,7 +906,14 @@ impl State { self.db.select_network_account_by_prefix(id_prefix).await } - /// Returns account IDs for all public (on-chain) network accounts. + /// Returns network account IDs within the specified block range (based on account creation + /// block). + /// + /// The function may return fewer accounts than exist in the range if the result would exceed + /// `MAX_RESPONSE_PAYLOAD_BYTES / AccountId::SERIALIZED_SIZE` rows. In this case, the result is + /// truncated at a block boundary to ensure all accounts from included blocks are returned. + /// + /// The response includes the last block number that was fully included in the result. pub async fn get_all_network_accounts( &self, block_range: RangeInclusive,