feat: add pagination to GetNetworkAccountIds

CHANGELOG.md

@@ -12,6 +12,7 @@
 - Added chain tip to the block producer status ([#1419](https://github.com/0xMiden/miden-node/pull/1419)).
 - The mempool's transaction capacity is now configurable ([#1433](https://github.com/0xMiden/miden-node/pull/1433)).
 - Renamed card's names in the `miden-network-monitor` binary ([#1441](https://github.com/0xMiden/miden-node/pull/1441)).
+- Added pagination to `GetNetworkAccountIds` endpoint ([#1452](https://github.com/0xMiden/miden-node/pull/1452)).
 - Improved tracing in `miden-network-monitor` binary ([#1366](https://github.com/0xMiden/miden-node/pull/1366)).
 - Integrated RPC stack with Validator component for transaction validation ([#1457](https://github.com/0xMiden/miden-node/pull/1457)).
 - Added explorer status to the `miden-network-monitor` binary ([#1450](https://github.com/0xMiden/miden-node/pull/1450)).

crates/ntx-builder/src/store.rs

@@ -4,11 +4,12 @@ use miden_node_proto::clients::{Builder, StoreNtxBuilderClient};
 use miden_node_proto::domain::account::NetworkAccountPrefix;
 use miden_node_proto::domain::note::NetworkNote;
 use miden_node_proto::errors::ConversionError;
+use miden_node_proto::generated::rpc::BlockRange;
 use miden_node_proto::generated::{self as proto};
 use miden_node_proto::try_convert;
 use miden_objects::Word;
 use miden_objects::account::{Account, AccountId};
-use miden_objects::block::BlockHeader;
+use miden_objects::block::{BlockHeader, BlockNumber};
 use miden_objects::crypto::merkle::{Forest, MmrPeaks, PartialMmr};
 use miden_objects::note::NoteScript;
 use miden_tx::utils::Deserializable;
@@ -172,21 +173,56 @@ impl StoreClient {
         Ok(all_notes)
     }
 
-    // TODO: add pagination.
+    /// Get all network account IDs.
+    ///
+    /// Since the `GetNetworkAccountIds` method is paginated, we loop through all pages until we
+    /// reach the end.
     #[instrument(target = COMPONENT, name = "store.client.get_network_account_ids", skip_all, err)]
     pub async fn get_network_account_ids(&self) -> Result<Vec<AccountId>, StoreError> {
-        let response = self.inner.clone().get_network_account_ids(()).await?.into_inner();
-
-        let accounts: Result<Vec<AccountId>, ConversionError> = response
-            .account_ids
-            .into_iter()
-            .map(|account_id| {
-                AccountId::read_from_bytes(&account_id.id)
-                    .map_err(|err| ConversionError::deserialization_error("account_id", err))
-            })
-            .collect();
-
-        Ok(accounts?)
+        const MAX_ITERATIONS: u32 = 1000;
+
+        let block_range = BlockNumber::from(0)..=BlockNumber::from(u32::MAX);
+
+        let mut ids = Vec::new();
+        let mut iterations_count = 0;
+
+        loop {
+            let response = self
+                .inner
+                .clone()
+                .get_network_account_ids(Into::<BlockRange>::into(block_range.clone()))
+                .await?
+                .into_inner();
+
+            let accounts: Result<Vec<AccountId>, ConversionError> = response
+                .account_ids
+                .into_iter()
+                .map(|account_id| {
+                    AccountId::read_from_bytes(&account_id.id)
+                        .map_err(|err| ConversionError::deserialization_error("account_id", err))
+                })
+                .collect();
+
+            let pagination_info = response.pagination_info.ok_or(
+                ConversionError::MissingFieldInProtobufRepresentation {
+                    entity: "NetworkAccountIdList",
+                    field_name: "pagination_info",
+                },
+            )?;
+
+            ids.extend(accounts?);
+            iterations_count += 1;
+
+            if pagination_info.block_num == pagination_info.chain_tip {
+                break;
+            }
+
+            if iterations_count >= MAX_ITERATIONS {
+                return Err(StoreError::MaxIterationsReached("GetNetworkAccountIds".to_string()));
+            }
+        }
+
+        Ok(ids)
     }
 
     #[instrument(target = COMPONENT, name = "store.client.get_note_script_by_root", skip_all, err)]
@@ -226,4 +262,6 @@ pub enum StoreError {
     MalformedResponse(String),
     #[error("failed to parse response")]
     DeserializationError(#[from] ConversionError),
+    #[error("max iterations reached: {0}")]
+    MaxIterationsReached(String),
 }

crates/proto/src/generated/store.rs

@@ -206,8 +206,11 @@ pub struct UnconsumedNetworkNotes {
 /// Represents the result of getting the network account ids.
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct NetworkAccountIdList {
+    /// Pagination information.
+    #[prost(message, optional, tag = "1")]
+    pub pagination_info: ::core::option::Option<super::rpc::PaginationInfo>,
     /// The list of network account ids.
-    #[prost(message, repeated, tag = "1")]
+    #[prost(message, repeated, tag = "2")]
     pub account_ids: ::prost::alloc::vec::Vec<super::account::AccountId>,
 }
 /// Current blockchain data based on the requested block number.
@@ -2432,7 +2435,7 @@ pub mod ntx_builder_client {
         /// Returns a list of all network account ids.
         pub async fn get_network_account_ids(
             &mut self,
-            request: impl tonic::IntoRequest<()>,
+            request: impl tonic::IntoRequest<super::super::rpc::BlockRange>,
         ) -> std::result::Result<
             tonic::Response<super::NetworkAccountIdList>,
             tonic::Status,
@@ -2532,7 +2535,7 @@ pub mod ntx_builder_server {
         /// Returns a list of all network account ids.
         async fn get_network_account_ids(
             &self,
-            request: tonic::Request<()>,
+            request: tonic::Request<super::super::rpc::BlockRange>,
         ) -> std::result::Result<
             tonic::Response<super::NetworkAccountIdList>,
             tonic::Status,
@@ -2830,14 +2833,19 @@ pub mod ntx_builder_server {
                 "/store.NtxBuilder/GetNetworkAccountIds" => {
                     #[allow(non_camel_case_types)]
                     struct GetNetworkAccountIdsSvc<T: NtxBuilder>(pub Arc<T>);
-                    impl<T: NtxBuilder> tonic::server::UnaryService<()>
+                    impl<
+                        T: NtxBuilder,
+                    > tonic::server::UnaryService<super::super::rpc::BlockRange>
                     for GetNetworkAccountIdsSvc<T> {
                         type Response = super::NetworkAccountIdList;
                         type Future = BoxFuture<
                             tonic::Response<Self::Response>,
                             tonic::Status,
                         >;
-                        fn call(&mut self, request: tonic::Request<()>) -> Self::Future {
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::super::rpc::BlockRange>,
+                        ) -> Self::Future {
                             let inner = Arc::clone(&self.0);
                             let fut = async move {
                                 <T as NtxBuilder>::get_network_account_ids(&inner, request)

crates/store/src/db/migrations/2025062000000_setup/up.sql

@@ -22,6 +22,7 @@ CREATE TABLE accounts (
     vault                                   BLOB,
     nonce                                   INTEGER,
     is_latest                               BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id
+    created_at_block                        INTEGER NOT NULL,
 
     PRIMARY KEY (account_id, block_num),
     CONSTRAINT all_null_or_none_null CHECK
@@ -35,6 +36,7 @@ CREATE TABLE accounts (
 CREATE INDEX idx_accounts_network_prefix ON accounts(network_account_id_prefix) WHERE network_account_id_prefix IS NOT NULL;
 CREATE INDEX idx_accounts_id_block ON accounts(account_id, block_num DESC);
 CREATE INDEX idx_accounts_latest ON accounts(account_id, is_latest) WHERE is_latest = 1;
+CREATE INDEX idx_accounts_created_at_block ON accounts(created_at_block);
 -- Index for joining with block_headers
 CREATE INDEX idx_accounts_block_num ON accounts(block_num);
 -- Index for joining with account_codes

crates/store/src/db/mod.rs

@@ -435,9 +435,14 @@ impl Db {
 
     /// Loads all network account IDs from the DB.
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]
-    pub async fn select_all_network_account_ids(&self) -> Result<Vec<AccountId>> {
-        self.transact("Get all network account IDs", queries::select_all_network_account_ids)
-            .await
+    pub async fn select_all_network_account_ids(
+        &self,
+        block_range: RangeInclusive<BlockNumber>,
+    ) -> Result<(Vec<AccountId>, BlockNumber)> {
+        self.transact("Get all network account IDs", move |conn| {
+            queries::select_all_network_account_ids(conn, block_range)
+        })
+        .await
     }
 
     #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)]

crates/store/src/db/models/queries/accounts.rs

@@ -417,21 +417,63 @@ pub(crate) fn select_all_accounts(
 /// A vector with network account IDs, or an error.
 pub(crate) fn select_all_network_account_ids(
     conn: &mut SqliteConnection,
-) -> Result<Vec<AccountId>, DatabaseError> {
-    let account_ids_raw: Vec<Vec<u8>> = QueryDsl::select(
-        schema::accounts::table.filter(schema::accounts::network_account_id_prefix.is_not_null()),
-        schema::accounts::account_id,
+    block_range: RangeInclusive<BlockNumber>,
+) -> Result<(Vec<AccountId>, BlockNumber), DatabaseError> {
+    const ROW_OVERHEAD_BYTES: usize = AccountId::SERIALIZED_SIZE;
+    const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES;
+
+    const _: () = assert!(
+        MAX_ROWS > miden_objects::MAX_ACCOUNTS_PER_BLOCK,
+        "Block pagination limit must exceed maximum block capacity"
+    );
+
+    if block_range.is_empty() {
+        return Err(DatabaseError::InvalidBlockRange {
+            from: *block_range.start(),
+            to: *block_range.end(),
+        });
+    }
+
+    let account_ids_raw: Vec<(Vec<u8>, i64)> = Box::new(
+        QueryDsl::select(
+            schema::accounts::table
+                .filter(schema::accounts::network_account_id_prefix.is_not_null()),
+            (schema::accounts::account_id, schema::accounts::created_at_block),
+        )
+        .filter(
+            schema::accounts::block_num
+                .between(block_range.start().to_raw_sql(), block_range.end().to_raw_sql()),
+        )
+        .order(schema::accounts::created_at_block.asc())
+        .limit(i64::try_from(MAX_ROWS + 1).expect("limit fits within i64")),
     )
-    .load::<Vec<u8>>(conn)?;
+    .load::<(Vec<u8>, i64)>(conn)?;
 
-    let account_ids = account_ids_raw
-        .into_iter()
-        .map(|id_bytes| {
-            AccountId::read_from_bytes(&id_bytes).map_err(DatabaseError::DeserializationError)
-        })
-        .collect::<Result<Vec<AccountId>, DatabaseError>>()?;
+    if let Some(&(_, last_created_at_block)) = account_ids_raw.last()
+        && account_ids_raw.len() > MAX_ROWS
+    {
+        let account_ids = account_ids_raw
+            .into_iter()
+            .take_while(|(_, created_at_block)| *created_at_block != last_created_at_block)
+            .map(|(id_bytes, _)| {
+                AccountId::read_from_bytes(&id_bytes).map_err(DatabaseError::DeserializationError)
+            })
+            .collect::<Result<Vec<AccountId>, DatabaseError>>()?;
+
+        let last_block_included =
+            BlockNumber::from_raw_sql(last_created_at_block.saturating_sub(1))?;
+
+        Ok((account_ids, last_block_included))
+    } else {
+        let account_ids = account_ids_raw
+            .into_iter()
+            .map(|(id_bytes, _)| {
+                AccountId::read_from_bytes(&id_bytes).map_err(DatabaseError::DeserializationError)
+            })
+            .collect::<Result<Vec<AccountId>, DatabaseError>>()?;
 
-    Ok(account_ids)
+        Ok((account_ids, *block_range.end()))
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -803,13 +845,29 @@ pub(crate) fn upsert_accounts(
     let mut count = 0;
     for update in accounts {
         let account_id = update.account_id();
+        let account_id_bytes = account_id.to_bytes();
+        let block_num_raw = block_num.to_raw_sql();
 
         let network_account_id_prefix = if account_id.is_network() {
             Some(NetworkAccountPrefix::try_from(account_id)?)
         } else {
             None
         };
 
+        // Preserve the original creation block when updating existing accounts.
+        let created_at_block = QueryDsl::select(
+            schema::accounts::table.filter(
+                schema::accounts::account_id
+                    .eq(&account_id_bytes)
+                    .and(schema::accounts::is_latest.eq(true)),
+            ),
+            schema::accounts::created_at_block,
+        )
+        .first::<i64>(conn)
+        .optional()
+        .map_err(DatabaseError::Diesel)?
+        .unwrap_or(block_num_raw);
+
         // NOTE: we collect storage / asset inserts to apply them only after the  account row is
         // written. The storage and vault tables have FKs pointing to `accounts (account_id,
         // block_num)`, so inserting them earlier would violate those constraints when inserting a
@@ -916,25 +974,26 @@ pub(crate) fn upsert_accounts(
         diesel::update(schema::accounts::table)
             .filter(
                 schema::accounts::account_id
-                    .eq(&account_id.to_bytes())
+                    .eq(&account_id_bytes)
                     .and(schema::accounts::is_latest.eq(true)),
             )
             .set(schema::accounts::is_latest.eq(false))
             .execute(conn)?;
 
         let account_value = AccountRowInsert {
-            account_id: account_id.to_bytes(),
+            account_id: account_id_bytes,
             network_account_id_prefix: network_account_id_prefix
                 .map(NetworkAccountPrefix::to_raw_sql),
             account_commitment: update.final_state_commitment().to_bytes(),
-            block_num: block_num.to_raw_sql(),
+            block_num: block_num_raw,
             nonce: full_account.as_ref().map(|account| nonce_to_raw_sql(account.nonce())),
             storage: full_account.as_ref().map(|account| account.storage().to_bytes()),
             vault: full_account.as_ref().map(|account| account.vault().to_bytes()),
             code_commitment: full_account
                 .as_ref()
                 .map(|account| account.code().commitment().to_bytes()),
             is_latest: true,
+            created_at_block,
         };
 
         diesel::insert_into(schema::accounts::table)
@@ -995,6 +1054,7 @@ pub(crate) struct AccountRowInsert {
     pub(crate) vault: Option<Vec<u8>>,
     pub(crate) nonce: Option<i64>,
     pub(crate) is_latest: bool,
+    pub(crate) created_at_block: i64,
 }
 
 #[derive(Insertable, AsChangeset, Debug, Clone)]

crates/store/src/db/schema.rs

@@ -32,6 +32,7 @@ diesel::table! {
         nonce -> Nullable<BigInt>,
         block_num -> BigInt,
         is_latest -> Bool,
+        created_at_block -> BigInt,
     }
 }
 

crates/store/src/server/ntx_builder.rs

@@ -1,6 +1,7 @@
 use std::num::{NonZero, TryFromIntError};
 
 use miden_node_proto::domain::account::{AccountInfo, NetworkAccountPrefix};
+use miden_node_proto::generated::rpc::BlockRange;
 use miden_node_proto::generated::store::ntx_builder_server;
 use miden_node_proto::generated::{self as proto};
 use miden_node_utils::ErrorReport;
@@ -161,14 +162,28 @@ impl ntx_builder_server::NtxBuilder for StoreApi {
     )]
     async fn get_network_account_ids(
         &self,
-        _request: Request<()>,
+        request: Request<BlockRange>,
     ) -> Result<Response<proto::store::NetworkAccountIdList>, Status> {
-        let account_ids = self.state.get_all_network_accounts().await.map_err(internal_error)?;
+        let block_range = request.into_inner();
+        let chain_tip = self.state.latest_block_num().await;
+
+        let block_from = BlockNumber::from(block_range.block_from);
+        let block_to = block_range.block_to.map_or(chain_tip, BlockNumber::from);
+        let block_range = block_from..=block_to;
+
+        let (account_ids, last_block_included) =
+            self.state.get_all_network_accounts(block_range).await.map_err(internal_error)?;
 
         let account_ids: Vec<proto::account::AccountId> =
             account_ids.into_iter().map(Into::into).collect();
 
-        Ok(Response::new(proto::store::NetworkAccountIdList { account_ids }))
+        Ok(Response::new(proto::store::NetworkAccountIdList {
+            account_ids,
+            pagination_info: Some(proto::rpc::PaginationInfo {
+                chain_tip: chain_tip.as_u32(),
+                block_num: last_block_included.as_u32(),
+            }),
+        }))
     }
 
     #[instrument(

crates/store/src/state.rs

@@ -911,8 +911,11 @@ impl State {
     }
 
     /// Returns account IDs for all public (on-chain) network accounts.
-    pub async fn get_all_network_accounts(&self) -> Result<Vec<AccountId>, DatabaseError> {
-        self.db.select_all_network_account_ids().await
+    pub async fn get_all_network_accounts(
+        &self,
+        block_range: RangeInclusive<BlockNumber>,
+    ) -> Result<(Vec<AccountId>, BlockNumber), DatabaseError> {
+        self.db.select_all_network_account_ids(block_range).await
     }
 
     /// Returns the respective account proof with optional details, such as asset and storage