feat: Deferred block proving

CHANGELOG.md

@@ -12,6 +12,7 @@
 - Introduce `SyncChainMmr` RPC endpoint to sync chain MMR deltas within specified block ranges ([#1591](https://github.com/0xMiden/node/issues/1591)).
 - Fixed `TransactionHeader` serialization for row insertion on database & fixed transaction cursor on retrievals ([#1701](https://github.com/0xMiden/node/issues/1701)).
 - Added KMS signing support in validator ([#1677](https://github.com/0xMiden/node/pull/1677)).
+- Added finality field for `SyncChainMmr` requests ([#1725](https://github.com/0xMiden/miden-node/pull/1725)).
 - Added per-IP gRPC rate limiting across services as well as global concurrent connection limit ([#1746](https://github.com/0xMiden/node/issues/1746)).
 
 ### Changes

bin/node/src/commands/bundled.rs

@@ -4,7 +4,7 @@ use std::path::PathBuf;
 use anyhow::Context;
 use miden_node_block_producer::BlockProducer;
 use miden_node_rpc::Rpc;
-use miden_node_store::Store;
+use miden_node_store::{DEFAULT_MAX_CONCURRENT_PROOFS, Store};
 use miden_node_utils::clap::GrpcOptionsExternal;
 use miden_node_utils::grpc::UrlExt;
 use miden_node_validator::{Validator, ValidatorSigner};
@@ -82,6 +82,14 @@ pub enum BundledCommand {
         #[arg(long = "enable-otel", default_value_t = false, env = ENV_ENABLE_OTEL, value_name = "BOOL")]
         enable_otel: bool,
 
+        /// Maximum number of concurrent block proofs to be scheduled.
+        #[arg(
+            long = "max-concurrent-proofs",
+            default_value_t = DEFAULT_MAX_CONCURRENT_PROOFS,
+            value_name = "NUM"
+        )]
+        max_concurrent_proofs: usize,
+
         #[command(flatten)]
         grpc_options: GrpcOptionsExternal,
     },
@@ -121,6 +129,7 @@ impl BundledCommand {
                 validator,
                 enable_otel: _,
                 grpc_options,
+                max_concurrent_proofs,
             } => {
                 Self::start(
                     rpc_url,
@@ -130,13 +139,14 @@ impl BundledCommand {
                     ntx_builder,
                     validator,
                     grpc_options,
+                    max_concurrent_proofs,
                 )
                 .await
             },
         }
     }
 
-    #[expect(clippy::too_many_lines)]
+    #[expect(clippy::too_many_lines, clippy::too_many_arguments)]
     async fn start(
         rpc_url: Url,
         block_prover_url: Option<Url>,
@@ -145,6 +155,7 @@ impl BundledCommand {
         ntx_builder: NtxBuilderConfig,
         validator: BundledValidatorConfig,
         grpc_options: GrpcOptionsExternal,
+        max_concurrent_proofs: usize,
     ) -> anyhow::Result<()> {
         // Start listening on all gRPC urls so that inter-component connections can be created
         // before each component is fully started up.
@@ -202,6 +213,7 @@ impl BundledCommand {
                     data_directory: data_directory_clone,
                     block_prover_url,
                     grpc_options: grpc_options.into(),
+                    max_concurrent_proofs,
                 }
                 .serve()
                 .await

bin/node/src/commands/store.rs

@@ -1,8 +1,8 @@
 use std::path::{Path, PathBuf};
 
 use anyhow::Context;
-use miden_node_store::Store;
 use miden_node_store::genesis::GenesisBlock;
+use miden_node_store::{DEFAULT_MAX_CONCURRENT_PROOFS, Store};
 use miden_node_utils::clap::GrpcOptionsInternal;
 use miden_node_utils::fs::ensure_empty_directory;
 use miden_node_utils::grpc::UrlExt;
@@ -65,6 +65,13 @@ pub enum StoreCommand {
         #[arg(long = "enable-otel", default_value_t = false, env = ENV_ENABLE_OTEL, value_name = "BOOL")]
         enable_otel: bool,
 
+        /// Maximum number of concurrent block proofs to be scheduled.
+        #[arg(
+            long = "max-concurrent-proofs",
+            default_value_t = DEFAULT_MAX_CONCURRENT_PROOFS,
+            value_name = "NUM"
+        )]
+        max_concurrent_proofs: usize,
         #[command(flatten)]
         grpc_options: GrpcOptionsInternal,
     },
@@ -86,6 +93,7 @@ impl StoreCommand {
                 data_directory,
                 enable_otel: _,
                 grpc_options,
+                max_concurrent_proofs,
             } => {
                 Self::start(
                     rpc_url,
@@ -94,6 +102,7 @@ impl StoreCommand {
                     block_prover_url,
                     data_directory,
                     grpc_options,
+                    max_concurrent_proofs,
                 )
                 .await
             },
@@ -115,6 +124,7 @@ impl StoreCommand {
         block_prover_url: Option<Url>,
         data_directory: PathBuf,
         grpc_options: GrpcOptionsInternal,
+        max_concurrent_proofs: usize,
     ) -> anyhow::Result<()> {
         let rpc_listener = rpc_url
             .to_socket()
@@ -144,6 +154,7 @@ impl StoreCommand {
             block_producer_listener,
             data_directory,
             grpc_options,
+            max_concurrent_proofs,
         }
         .serve()
         .await

bin/remote-prover/src/server/tests.rs

@@ -3,7 +3,6 @@ use std::num::NonZeroUsize;
 use std::sync::Arc;
 use std::time::Duration;
 
-use assert_matches::assert_matches;
 use miden_protocol::MIN_PROOF_SECURITY_LEVEL;
 use miden_protocol::account::auth::AuthScheme;
 use miden_protocol::asset::{Asset, FungibleAsset};
@@ -239,7 +238,7 @@ async fn capacity_is_respected() {
     result.sort_unstable();
     assert_eq!(expected, result);
 
-    assert_matches!(first.err().or(second.err()).or(third.err()), Some(err) => {
+    assert_matches::assert_matches!(first.err().or(second.err()).or(third.err()), Some(err) => {
         assert_eq!(err.code(), tonic::Code::ResourceExhausted);
     });
 

bin/stress-test/src/seeding/mod.rs

@@ -93,7 +93,7 @@ pub async fn seed_store(
     let faucet = create_faucet();
     let fee_params = FeeParameters::new(faucet.id(), 0).unwrap();
     let signer = EcdsaSecretKey::new();
-    let genesis_state = GenesisState::new(vec![faucet.clone()], fee_params, 1, 1, signer);
+    let genesis_state = GenesisState::new(vec![faucet.clone()], fee_params, 1, 1, signer.clone());
     let genesis_block = genesis_state
         .clone()
         .into_block()
@@ -118,6 +118,7 @@ pub async fn seed_store(
         &store_client,
         data_directory,
         accounts_filepath,
+        &signer,
     )
     .await;
 
@@ -129,6 +130,7 @@ pub async fn seed_store(
 ///
 /// The first transaction in each batch sends assets from the faucet to 255 accounts.
 /// The rest of the transactions consume the notes created by the faucet in the previous block.
+#[expect(clippy::too_many_arguments)]
 async fn generate_blocks(
     num_accounts: usize,
     public_accounts_percentage: u8,
@@ -137,6 +139,7 @@ async fn generate_blocks(
     store_client: &StoreClient,
     data_directory: DataDirectory,
     accounts_filepath: PathBuf,
+    signer: &EcdsaSecretKey,
 ) -> SeedingMetrics {
     // Each block is composed of [`BATCHES_PER_BLOCK`] batches, and each batch is composed of
     // [`TRANSACTIONS_PER_BATCH`] txs. The first note of the block is always a send assets tx
@@ -215,7 +218,8 @@ async fn generate_blocks(
         let block_inputs = get_block_inputs(store_client, &batches, &mut metrics).await;
 
         // update blocks
-        prev_block_header = apply_block(batches, block_inputs, store_client, &mut metrics).await;
+        prev_block_header =
+            apply_block(batches, block_inputs, store_client, &mut metrics, signer).await;
         if current_anchor_header.block_epoch() != prev_block_header.block_epoch() {
             current_anchor_header = prev_block_header.clone();
         }
@@ -250,11 +254,12 @@ async fn apply_block(
     block_inputs: BlockInputs,
     store_client: &StoreClient,
     metrics: &mut SeedingMetrics,
+    signer: &EcdsaSecretKey,
 ) -> BlockHeader {
     let proposed_block = ProposedBlock::new(block_inputs, batches).unwrap();
     let (header, body) = proposed_block.clone().into_header_and_body().unwrap();
     let block_size: usize = header.to_bytes().len() + body.to_bytes().len();
-    let signature = EcdsaSecretKey::new().sign(header.commitment());
+    let signature = signer.sign(header.commitment());
     // SAFETY: The header, body, and signature are known to correspond to each other.
     let signed_block = SignedBlock::new_unchecked(header, body, signature);
     let ordered_batches = proposed_block.batches().clone();
@@ -555,6 +560,7 @@ pub async fn start_store(
             block_producer_listener,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::bench(),
+            max_concurrent_proofs: miden_node_store::DEFAULT_MAX_CONCURRENT_PROOFS,
         }
         .serve()
         .await

bin/stress-test/src/store/mod.rs

@@ -469,6 +469,7 @@ async fn sync_chain_mmr(
 ) -> SyncChainMmrRun {
     let sync_request = proto::rpc::SyncChainMmrRequest {
         block_range: Some(proto::rpc::BlockRange { block_from, block_to: Some(block_to) }),
+        finality: proto::rpc::Finality::Committed.into(),
     };
 
     let start = Instant::now();

crates/block-producer/src/server/tests.rs

@@ -2,7 +2,7 @@ use std::num::NonZeroUsize;
 use std::time::Duration;
 
 use miden_node_proto::generated::block_producer::api_client as block_producer_client;
-use miden_node_store::{GenesisState, Store};
+use miden_node_store::{DEFAULT_MAX_CONCURRENT_PROOFS, GenesisState, Store};
 use miden_node_utils::clap::GrpcOptionsInternal;
 use miden_node_utils::fee::test_fee_params;
 use miden_node_validator::{Validator, ValidatorSigner};
@@ -159,6 +159,7 @@ async fn start_store(
             block_prover_url: None,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::bench(),
+            max_concurrent_proofs: DEFAULT_MAX_CONCURRENT_PROOFS,
         }
         .serve()
         .await

crates/rpc/src/tests.rs

@@ -7,8 +7,8 @@ use http::{HeaderMap, HeaderValue};
 use miden_node_proto::clients::{Builder, GrpcClient, Interceptor, RpcClient};
 use miden_node_proto::generated::rpc::api_client::ApiClient as ProtoClient;
 use miden_node_proto::generated::{self as proto};
-use miden_node_store::Store;
 use miden_node_store::genesis::config::GenesisConfig;
+use miden_node_store::{DEFAULT_MAX_CONCURRENT_PROOFS, Store};
 use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal};
 use miden_node_utils::fee::test_fee;
 use miden_node_utils::limiter::{
@@ -480,6 +480,7 @@ async fn start_store(store_listener: TcpListener) -> (Runtime, TempDir, Word, So
             block_producer_listener,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::test(),
+            max_concurrent_proofs: DEFAULT_MAX_CONCURRENT_PROOFS,
         }
         .serve()
         .await
@@ -522,6 +523,7 @@ async fn restart_store(store_addr: SocketAddr, data_directory: &std::path::Path)
             block_producer_listener,
             data_directory: dir,
             grpc_options: GrpcOptionsInternal::test(),
+            max_concurrent_proofs: DEFAULT_MAX_CONCURRENT_PROOFS,
         }
         .serve()
         .await
@@ -597,6 +599,7 @@ async fn sync_chain_mmr_returns_delta() {
 
     let request = proto::rpc::SyncChainMmrRequest {
         block_range: Some(proto::rpc::BlockRange { block_from: 0, block_to: None }),
+        finality: proto::rpc::Finality::Committed.into(),
     };
     let response = rpc_client.sync_chain_mmr(request).await.expect("sync_chain_mmr should succeed");
     let response = response.into_inner();

crates/store/Cargo.toml

@@ -21,7 +21,6 @@ deadpool-diesel                 = { features = ["sqlite"], version = "0.6" }
 diesel                          = { features = ["numeric", "sqlite"], version = "2.3" }
 diesel_migrations               = { features = ["sqlite"], version = "2.3" }
 fs-err                          = { workspace = true }
-futures                         = { workspace = true }
 hex                             = { version = "0.4" }
 indexmap                        = { workspace = true }
 libsqlite3-sys                  = { workspace = true }

crates/store/src/blocks.rs

@@ -107,6 +107,41 @@ impl BlockStore {
         fs_err::write(block_path, data)
     }
 
+    // PROOF STORAGE
+    // --------------------------------------------------------------------------------------------
+
+    #[expect(dead_code)]
+    pub async fn load_proof(
+        &self,
+        block_num: BlockNumber,
+    ) -> Result<Option<Vec<u8>>, std::io::Error> {
+        match tokio::fs::read(self.proof_path(block_num)).await {
+            Ok(data) => Ok(Some(data)),
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
+            Err(err) => Err(err),
+        }
+    }
+
+    #[instrument(
+        target = COMPONENT,
+        name = "store.block_store.save_proof",
+        skip(self, data),
+        err,
+        fields(proof_size = data.len())
+    )]
+    pub async fn save_proof(
+        &self,
+        block_num: BlockNumber,
+        data: &[u8],
+    ) -> Result<(), std::io::Error> {
+        let (epoch_path, proof_path) = self.epoch_proof_path(block_num)?;
+        if !epoch_path.exists() {
+            tokio::fs::create_dir_all(epoch_path).await?;
+        }
+
+        tokio::fs::write(proof_path, data).await
+    }
+
     // HELPER FUNCTIONS
     // --------------------------------------------------------------------------------------------
 
@@ -117,6 +152,13 @@ impl BlockStore {
         epoch_dir.join(format!("block_{block_num:08x}.dat"))
     }
 
+    fn proof_path(&self, block_num: BlockNumber) -> PathBuf {
+        let block_num = block_num.as_u32();
+        let epoch = block_num >> 16;
+        let epoch_dir = self.store_dir.join(format!("{epoch:04x}"));
+        epoch_dir.join(format!("proof_{block_num:08x}.dat"))
+    }
+
     fn epoch_block_path(
         &self,
         block_num: BlockNumber,
@@ -127,6 +169,16 @@ impl BlockStore {
         Ok((epoch_path.to_path_buf(), block_path))
     }
 
+    fn epoch_proof_path(
+        &self,
+        block_num: BlockNumber,
+    ) -> Result<(PathBuf, PathBuf), std::io::Error> {
+        let proof_path = self.proof_path(block_num);
+        let epoch_path = proof_path.parent().ok_or(std::io::Error::from(ErrorKind::NotFound))?;
+
+        Ok((epoch_path.to_path_buf(), proof_path))
+    }
+
     pub fn display(&self) -> std::path::Display<'_> {
         self.store_dir.display()
     }

crates/store/src/db/migrations/2025062000000_setup/up.sql

@@ -1,13 +1,16 @@
 CREATE TABLE block_headers (
-    block_num    INTEGER NOT NULL,
-    block_header BLOB    NOT NULL,
-    signature    BLOB    NOT NULL,
-    commitment   BLOB    NOT NULL,
+    block_num      INTEGER NOT NULL,
+    block_header   BLOB    NOT NULL,
+    signature      BLOB    NOT NULL,
+    commitment     BLOB    NOT NULL,
+    proving_inputs BLOB,             -- Serialized BlockProofRequest needed for deferred proving. NULL if it has been proven or never proven (genesis block).
 
     PRIMARY KEY (block_num),
     CONSTRAINT block_header_block_num_is_u32 CHECK (block_num BETWEEN 0 AND 0xFFFFFFFF)
 );
 
+CREATE INDEX block_headers_proven_desc ON block_headers(block_num DESC) WHERE proving_inputs IS NULL;
+
 CREATE TABLE account_codes (
     code_commitment BLOB NOT NULL,
     code            BLOB NOT NULL,