Generic provider (#218)

Closes #216 

Replaces all concrete usage of `ProviderFactory` and
`ProviderFactoryReopener` with generics bounded by [reth provider
traits](https://github.com/paradigmxyz/reth/blob/661b260f6172c047e8530e2331b2c84141e03c2b/crates/storage/provider/src/traits/full.rs#L14-L24).
This allows running rbuilder in-process using a node provider and eases
swapping to a more modern provider like `BlockchainProvider` in the
future.

Note that not all trait methods can be implemented because
`ProviderFactoryReopener` has no access to the blockchain tree (pending
state etc.).

## DB Consistency Checks

Consistency checks have been moved out of the rbuilder core into the
trait implementation for `ProviderFactoryReopener`. Inside the trait
implementation, the current head block number is not known, so the
reopener cannot assume that the state of the block given - 256 blocks
will be available.

To not redundantly re-check the consistency of the same block,
`ProviderFactoryReopener` tracks the last block verified to have
consistent state and will only re-check it when it changes. This adds 1
additional read per provider call to get the head of the chain, however
I believe the cost will be insignificant.

Author: liamaharon

crates/rbuilder/src/backtest/backtest_build_block.rs

@@ -8,14 +8,14 @@
 use ahash::HashMap;
 use alloy_primitives::utils::format_ether;
 
-use crate::backtest::restore_landed_orders::{
-    restore_landed_orders, sim_historical_block, ExecutedBlockTx, ExecutedTxs, SimplifiedOrder,
-};
-use crate::backtest::OrdersWithTimestamp;
 use crate::{
     backtest::{
         execute::{backtest_prepare_ctx_for_block, BacktestBlockInput},
-        BlockData, HistoricalDataStorage,
+        restore_landed_orders::{
+            restore_landed_orders, sim_historical_block, ExecutedBlockTx, ExecutedTxs,
+            SimplifiedOrder,
+        },
+        BlockData, HistoricalDataStorage, OrdersWithTimestamp,
     },
     building::builders::BacktestSimulateBlockInput,
     live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig},
@@ -58,7 +58,10 @@ struct Cli {
     block: u64,
 }
 
-pub async fn run_backtest_build_block<ConfigType: LiveBuilderConfig>() -> eyre::Result<()> {
+pub async fn run_backtest_build_block<ConfigType>() -> eyre::Result<()>
+where
+    ConfigType: LiveBuilderConfig,
+{
     let cli = Cli::parse();
 
     let config: ConfigType = load_config_toml_and_env(cli.config)?;
@@ -85,10 +88,7 @@ pub async fn run_backtest_build_block<ConfigType: LiveBuilderConfig>() -> eyre::
         print_order_and_timestamp(&block_data.available_orders, &block_data);
     }
 
-    let provider_factory = config
-        .base_config()
-        .provider_factory()?
-        .provider_factory_unchecked();
+    let provider_factory = config.base_config().create_provider_factory()?;
     let chain_spec = config.base_config().chain_spec()?;
     let sbundle_mergeabe_signers = config.base_config().sbundle_mergeabe_signers();
 
@@ -126,7 +126,7 @@ pub async fn run_backtest_build_block<ConfigType: LiveBuilderConfig>() -> eyre::
                     builder_name: builder_name.clone(),
                     sbundle_mergeabe_signers: sbundle_mergeabe_signers.clone(),
                     sim_orders: &sim_orders,
-                    provider_factory: provider_factory.clone(),
+                    provider: provider_factory.clone(),
                     cached_reads: None,
                 };
                 let build_res = config.build_backtest_block(builder_name, input);

crates/rbuilder/src/backtest/backtest_build_range.rs

@@ -24,8 +24,7 @@ use clap::Parser;
 use rayon::prelude::*;
 use std::{
     fs::File,
-    io,
-    io::Write,
+    io::{self, Write},
     path::{Path, PathBuf},
 };
 use time::format_description::well_known::Rfc3339;
@@ -63,8 +62,10 @@ struct Cli {
     blocks: Vec<u64>,
 }
 
-pub async fn run_backtest_build_range<ConfigType: LiveBuilderConfig + Send + Sync>(
-) -> eyre::Result<()> {
+pub async fn run_backtest_build_range<ConfigType>() -> eyre::Result<()>
+where
+    ConfigType: LiveBuilderConfig,
+{
     let cli = Cli::parse();
 
     if cli.store_backtest && cli.compare_backtest {
@@ -110,10 +111,7 @@ pub async fn run_backtest_build_range<ConfigType: LiveBuilderConfig + Send + Syn
         result
     };
 
-    let provider_factory = config
-        .base_config()
-        .provider_factory()?
-        .provider_factory_unchecked();
+    let provider_factory = config.base_config().create_provider_factory()?;
     let chain_spec = config.base_config().chain_spec()?;
 
     let mut profits = Vec::new();

crates/rbuilder/src/backtest/execute.rs

@@ -1,21 +1,19 @@
-use crate::primitives::OrderId;
-use crate::utils::Signer;
 use crate::{
     backtest::BlockData,
     building::{
         builders::BacktestSimulateBlockInput, sim::simulate_all_orders_with_sim_tree,
         BlockBuildingContext, BundleErr, OrderErr, TransactionErr,
     },
     live_builder::cli::LiveBuilderConfig,
-    primitives::SimulatedOrder,
-    utils::clean_extradata,
+    primitives::{OrderId, SimulatedOrder},
+    utils::{clean_extradata, Signer},
 };
 use ahash::HashSet;
 use alloy_primitives::{Address, U256};
-use reth::providers::ProviderFactory;
 use reth_chainspec::ChainSpec;
-use reth_db::{database::Database, DatabaseEnv};
+use reth_db::Database;
 use reth_payload_builder::database::CachedReads;
+use reth_provider::{DatabaseProviderFactory, HeaderProvider, StateProviderFactory};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
@@ -57,14 +55,17 @@ pub struct BacktestBlockInput {
     pub sim_errors: Vec<OrderErr>,
 }
 
-pub fn backtest_prepare_ctx_for_block<DB: Database + Clone>(
+pub fn backtest_prepare_ctx_for_block<P>(
     block_data: BlockData,
-    provider_factory: ProviderFactory<DB>,
+    provider: P,
     chain_spec: Arc<ChainSpec>,
     build_block_lag_ms: i64,
     blocklist: HashSet<Address>,
     builder_signer: Signer,
-) -> eyre::Result<BacktestBlockInput> {
+) -> eyre::Result<BacktestBlockInput>
+where
+    P: StateProviderFactory + Clone + 'static,
+{
     let orders = block_data
         .available_orders
         .iter()
@@ -87,7 +88,7 @@ pub fn backtest_prepare_ctx_for_block<DB: Database + Clone>(
         Some(builder_signer),
     );
     let (sim_orders, sim_errors) =
-        simulate_all_orders_with_sim_tree(provider_factory.clone(), &ctx, &orders, false)?;
+        simulate_all_orders_with_sim_tree(provider.clone(), &ctx, &orders, false)?;
     Ok(BacktestBlockInput {
         ctx,
         sim_orders,
@@ -96,23 +97,28 @@ pub fn backtest_prepare_ctx_for_block<DB: Database + Clone>(
 }
 
 #[allow(clippy::too_many_arguments)]
-pub fn backtest_simulate_block<ConfigType: LiveBuilderConfig>(
+pub fn backtest_simulate_block<P, DB, ConfigType>(
     block_data: BlockData,
-    provider_factory: ProviderFactory<Arc<DatabaseEnv>>,
+    provider: P,
     chain_spec: Arc<ChainSpec>,
     build_block_lag_ms: i64,
     builders_names: Vec<String>,
     config: &ConfigType,
     blocklist: HashSet<Address>,
     sbundle_mergeabe_signers: &[Address],
-) -> eyre::Result<BlockBacktestValue> {
+) -> eyre::Result<BlockBacktestValue>
+where
+    DB: Database + Clone + 'static,
+    P: DatabaseProviderFactory<DB> + StateProviderFactory + HeaderProvider + Clone + 'static,
+    ConfigType: LiveBuilderConfig,
+{
     let BacktestBlockInput {
         ctx,
         sim_orders,
         sim_errors,
     } = backtest_prepare_ctx_for_block(
         block_data.clone(),
-        provider_factory.clone(),
+        provider.clone(),
         chain_spec.clone(),
         build_block_lag_ms,
         blocklist,
@@ -158,7 +164,7 @@ pub fn backtest_simulate_block<ConfigType: LiveBuilderConfig>(
             builder_name: building_algorithm_name.clone(),
             sbundle_mergeabe_signers: sbundle_mergeabe_signers.to_vec(),
             sim_orders: &sim_orders,
-            provider_factory: provider_factory.clone(),
+            provider: provider.clone(),
             cached_reads,
         };
 

crates/rbuilder/src/backtest/redistribute/cli/mod.rs

@@ -1,18 +1,18 @@
 mod csv_output;
 
-use crate::backtest::redistribute::{calc_redistributions, RedistributionBlockOutput};
-use crate::backtest::BlockData;
-use crate::live_builder::base_config::load_config_toml_and_env;
-use crate::live_builder::cli::LiveBuilderConfig;
-use crate::{backtest::HistoricalDataStorage, live_builder::config::Config};
+use crate::{
+    backtest::{
+        redistribute::{calc_redistributions, RedistributionBlockOutput},
+        BlockData, HistoricalDataStorage,
+    },
+    live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig},
+};
 use alloy_primitives::utils::format_ether;
 use clap::Parser;
 use csv_output::{CSVOutputRow, CSVResultWriter};
-use reth_db::DatabaseEnv;
-use reth_provider::ProviderFactory;
-use std::io;
-use std::path::PathBuf;
-use std::sync::Arc;
+use reth_db::Database;
+use reth_provider::{DatabaseProviderFactory, HeaderProvider, StateProviderFactory};
+use std::{io, path::PathBuf};
 use tracing::info;
 
 #[derive(Parser, Debug)]
@@ -45,19 +45,19 @@ enum Commands {
     },
 }
 
-pub async fn run_backtest_redistribute<ConfigType: LiveBuilderConfig>() -> eyre::Result<()> {
+pub async fn run_backtest_redistribute<ConfigType>() -> eyre::Result<()>
+where
+    ConfigType: LiveBuilderConfig,
+{
     let cli = Cli::parse();
 
-    let config: Config = load_config_toml_and_env(cli.config)?;
-    config.base_config.setup_tracing_subsriber()?;
+    let config: ConfigType = load_config_toml_and_env(cli.config)?;
+    config.base_config().setup_tracing_subsriber()?;
 
     let mut historical_data_storage =
-        HistoricalDataStorage::new_from_path(&config.base_config.backtest_fetch_output_file)
+        HistoricalDataStorage::new_from_path(&config.base_config().backtest_fetch_output_file)
             .await?;
-    let provider_factory = config
-        .base_config
-        .provider_factory()?
-        .provider_factory_unchecked();
+    let provider = config.base_config().create_provider_factory()?;
     let mut csv_writer = cli
         .csv
         .map(|path| -> io::Result<_> { CSVResultWriter::new(path) })
@@ -74,7 +74,7 @@ pub async fn run_backtest_redistribute<ConfigType: LiveBuilderConfig>() -> eyre:
                 block_data,
                 csv_writer.as_mut(),
                 json_accum.as_mut(),
-                provider_factory.clone(),
+                provider.clone(),
                 &config,
                 cli.distribute_to_mempool_txs,
             )?;
@@ -91,7 +91,7 @@ pub async fn run_backtest_redistribute<ConfigType: LiveBuilderConfig>() -> eyre:
                     block_data,
                     csv_writer.as_mut(),
                     json_accum.as_mut(),
-                    provider_factory.clone(),
+                    provider.clone(),
                     &config,
                     cli.distribute_to_mempool_txs,
                 )?;
@@ -107,19 +107,24 @@ pub async fn run_backtest_redistribute<ConfigType: LiveBuilderConfig>() -> eyre:
     Ok(())
 }
 
-fn process_redisribution<ConfigType: LiveBuilderConfig + Send + Sync>(
+fn process_redisribution<P, DB, ConfigType>(
     block_data: BlockData,
     csv_writer: Option<&mut CSVResultWriter>,
     json_accum: Option<&mut Vec<RedistributionBlockOutput>>,
-    provider_factory: ProviderFactory<Arc<DatabaseEnv>>,
+    provider: P,
     config: &ConfigType,
     distribute_to_mempool_txs: bool,
-) -> eyre::Result<()> {
+) -> eyre::Result<()>
+where
+    DB: Database + Clone + 'static,
+    P: DatabaseProviderFactory<DB> + StateProviderFactory + HeaderProvider + Clone + 'static,
+    ConfigType: LiveBuilderConfig,
+{
     let block_number = block_data.block_number;
     let block_hash = block_data.onchain_block.header.hash;
     info!(block_number, "Calculating redistribution for a block");
     let redistribution_values = match calc_redistributions(
-        provider_factory.clone(),
+        provider.clone(),
         config,
         block_data,
         distribute_to_mempool_txs,