diff --git a/benches/pbs/bench-config.toml b/benches/pbs/bench-config.toml index 0e10e08f..5c6aed55 100644 --- a/benches/pbs/bench-config.toml +++ b/benches/pbs/bench-config.toml @@ -1,6 +1,6 @@ chain = "Holesky" -[pbs] +[[pbs]] port = 18750 late_in_slot_time_ms = 1000000000000 # skip late in slot checks diff --git a/config.example.toml b/config.example.toml index bb217b35..5a6e6ba9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -8,7 +8,7 @@ chain = "Holesky" # Configuration for the PBS module -[pbs] +[[pbs]] # Docker image to use for the PBS module. # OPTIONAL, DEFAULT: ghcr.io/commit-boost/pbs:latest docker_image = "ghcr.io/commit-boost/pbs:latest" diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 5d257365..7050759a 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -227,89 +227,102 @@ pub async fn handle_docker_init(config_path: String, output_dir: String) -> Resu } }; - // setup pbs service - if metrics_enabled { - targets.push(PrometheusTargetConfig { - targets: vec![format!("cb_pbs:{metrics_port}")], - labels: PrometheusLabelsConfig { job: "pbs".to_owned() }, - }); + let mut pbs_configs = vec![cb_config.pbs]; + if let Some(extended_pbses) = cb_config.extended_pbses { + pbs_configs.extend(extended_pbses); } - let mut pbs_envs = IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT)]); - let mut pbs_volumes = vec![config_volume.clone()]; + let mut general_pbs_envs = IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT)]); + let mut general_pbs_volumes = vec![config_volume.clone()]; if let Some(mux_config) = cb_config.muxes { for mux in mux_config.muxes.iter() { if let Some((env_name, actual_path, internal_path)) = mux.loader_env() { let (key, val) = get_env_val(&env_name, &internal_path); - pbs_envs.insert(key, val); - pbs_volumes.push(Volumes::Simple(format!("{}:{}:ro", actual_path, internal_path))); + general_pbs_envs.insert(key, val); + general_pbs_volumes + .push(Volumes::Simple(format!("{}:{}:ro", actual_path, internal_path))); } } } - if let Some((key, val)) = chain_spec_env.clone() { - pbs_envs.insert(key, val); - } - if metrics_enabled { - let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); - pbs_envs.insert(key, val); - } - if log_to_file { - let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); + for (pbs_id, pbs_config) in pbs_configs.iter().enumerate() { + let pbs_container_name = + if pbs_id == 0 { "cb_pbs".to_owned() } else { format!("cb_pbs_ext_{}", pbs_id) }; + // setup pbs service + if metrics_enabled { + targets.push(PrometheusTargetConfig { + targets: vec![format!("{}:{metrics_port}", pbs_container_name.clone())], + labels: PrometheusLabelsConfig { job: "pbs".to_owned() }, + }); + } + + let mut pbs_envs = general_pbs_envs.clone(); + let mut pbs_volumes = general_pbs_volumes.clone(); + + if let Some((key, val)) = chain_spec_env.clone() { + pbs_envs.insert(key, val); + } + if metrics_enabled { + let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); + pbs_envs.insert(key, val); + } + if log_to_file { + let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); + pbs_envs.insert(key, val); + } + if !builder_events_modules.is_empty() { + let env = builder_events_modules.join(","); + let (k, v) = get_env_val(BUILDER_URLS_ENV, &env); + pbs_envs.insert(k, v); + } + + // ports + let host_endpoint = + SocketAddr::from((pbs_config.pbs_config.host, pbs_config.pbs_config.port)); + let ports = Ports::Short(vec![format!("{}:{}", host_endpoint, pbs_config.pbs_config.port)]); + warnings.push(format!("pbs has an exported port on {}", pbs_config.pbs_config.port)); + + // inside the container expose on 0.0.0.0 + let container_endpoint = + SocketAddr::from((Ipv4Addr::UNSPECIFIED, pbs_config.pbs_config.port)); + let (key, val) = get_env_val(PBS_ENDPOINT_ENV, &container_endpoint.to_string()); pbs_envs.insert(key, val); - } - if !builder_events_modules.is_empty() { - let env = builder_events_modules.join(","); - let (k, v) = get_env_val(BUILDER_URLS_ENV, &env); - pbs_envs.insert(k, v); - } - // ports - let host_endpoint = - SocketAddr::from((cb_config.pbs.pbs_config.host, cb_config.pbs.pbs_config.port)); - let ports = Ports::Short(vec![format!("{}:{}", host_endpoint, cb_config.pbs.pbs_config.port)]); - warnings.push(format!("pbs has an exported port on {}", cb_config.pbs.pbs_config.port)); - - // inside the container expose on 0.0.0.0 - let container_endpoint = - SocketAddr::from((Ipv4Addr::UNSPECIFIED, cb_config.pbs.pbs_config.port)); - let (key, val) = get_env_val(PBS_ENDPOINT_ENV, &container_endpoint.to_string()); - pbs_envs.insert(key, val); - - // volumes - pbs_volumes.extend(chain_spec_volume.clone()); - pbs_volumes.extend(get_log_volume(&cb_config.logs, PBS_MODULE_NAME)); - - // networks - let pbs_networs = if metrics_enabled { - Networks::Simple(vec![METRICS_NETWORK.to_owned()]) - } else { - Networks::default() - }; + // volumes + pbs_volumes.extend(chain_spec_volume.clone()); + pbs_volumes.extend(get_log_volume(&cb_config.logs, PBS_MODULE_NAME)); - let pbs_service = Service { - container_name: Some("cb_pbs".to_owned()), - image: Some(cb_config.pbs.docker_image), - ports, - networks: pbs_networs, - volumes: pbs_volumes, - environment: Environment::KvPair(pbs_envs), - healthcheck: Some(Healthcheck { - test: Some(HealthcheckTest::Single(format!( - "curl -f http://localhost:{}{}{}", - cb_config.pbs.pbs_config.port, BUILDER_API_PATH, GET_STATUS_PATH - ))), - interval: Some("30s".into()), - timeout: Some("5s".into()), - retries: 3, - start_period: Some("5s".into()), - disable: false, - }), - ..Service::default() - }; + // networks + let pbs_networks = if metrics_enabled { + Networks::Simple(vec![METRICS_NETWORK.to_owned()]) + } else { + Networks::default() + }; - services.insert("cb_pbs".to_owned(), Some(pbs_service)); + let pbs_service = Service { + container_name: Some(pbs_container_name.clone()), + image: Some(pbs_config.docker_image.clone()), + ports, + networks: pbs_networks, + volumes: pbs_volumes, + environment: Environment::KvPair(pbs_envs), + healthcheck: Some(Healthcheck { + test: Some(HealthcheckTest::Single(format!( + "curl -f http://localhost:{}{}{}", + pbs_config.pbs_config.port, BUILDER_API_PATH, GET_STATUS_PATH + ))), + interval: Some("30s".into()), + timeout: Some("5s".into()), + retries: 3, + start_period: Some("5s".into()), + disable: false, + }), + ..Service::default() + }; + + services.insert(pbs_container_name.clone(), Some(pbs_service)); + } // setup signer service if let Some(SignerConfig::Local { docker_image, loader, store }) = cb_config.signer { diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index b097d02a..9b071682 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -27,6 +27,7 @@ pub use utils::*; pub struct CommitBoostConfig { pub chain: Chain, pub relays: Vec, + #[serde(skip)] pub pbs: StaticPbsConfig, #[serde(flatten)] pub muxes: Option, @@ -34,6 +35,8 @@ pub struct CommitBoostConfig { pub signer: Option, pub metrics: Option, pub logs: Option, + #[serde(rename = "pbs")] + pub extended_pbses: Option>, } impl CommitBoostConfig { @@ -74,12 +77,13 @@ impl CommitBoostConfig { let config = CommitBoostConfig { chain, relays: helper_config.relays, - pbs: helper_config.pbs, + pbs: Default::default(), muxes: helper_config.muxes, modules: helper_config.modules, signer: helper_config.signer, metrics: helper_config.metrics, logs: helper_config.logs, + extended_pbses: helper_config.pbses, }; Ok(config) @@ -111,7 +115,8 @@ struct ChainConfig { struct HelperConfig { chain: ChainLoader, relays: Vec, - pbs: StaticPbsConfig, + #[serde(rename = "pbs")] + pbses: Option>, #[serde(flatten)] muxes: Option, modules: Option>, diff --git a/crates/common/src/config/module.rs b/crates/common/src/config/module.rs index 2476079b..6101520a 100644 --- a/crates/common/src/config/module.rs +++ b/crates/common/src/config/module.rs @@ -15,7 +15,7 @@ use crate::{ types::{Chain, Jwt, ModuleId}, }; -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub enum ModuleKind { #[serde(alias = "commit")] Commit, @@ -24,7 +24,7 @@ pub enum ModuleKind { } /// Static module config from config file -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct StaticModuleConfig { /// Unique id of the module pub id: ModuleId, diff --git a/crates/common/src/config/mux.rs b/crates/common/src/config/mux.rs index 32ff8e97..df40ab7f 100644 --- a/crates/common/src/config/mux.rs +++ b/crates/common/src/config/mux.rs @@ -37,6 +37,7 @@ impl PbsMuxes { self, chain: Chain, default_pbs: &PbsConfig, + extended_pbses: &Option>, ) -> eyre::Result> { let mut muxes = self.muxes; @@ -44,8 +45,17 @@ impl PbsMuxes { ensure!(!mux.relays.is_empty(), "mux config {} must have at least one relay", mux.id); if let Some(loader) = &mux.loader { - let extra_keys = loader.load(&mux.id, chain, default_pbs.rpc_url.clone()).await?; - mux.validator_pubkeys.extend(extra_keys); + // TODO: this need to be handled with tokio::task::JoinSet for performance + if let Some(extended_pbses) = extended_pbses { + for pbs in extended_pbses { + let extra_keys = loader.load(&mux.id, chain, pbs.rpc_url.clone()).await?; + mux.validator_pubkeys.extend(extra_keys); + } + } else { + let extra_keys = + loader.load(&mux.id, chain, default_pbs.rpc_url.clone()).await?; + mux.validator_pubkeys.extend(extra_keys); + } } ensure!( @@ -80,20 +90,39 @@ impl PbsMuxes { relay_clients.push(RelayClient::new(config)?); } - let config = PbsConfig { - timeout_get_header_ms: mux - .timeout_get_header_ms - .unwrap_or(default_pbs.timeout_get_header_ms), - late_in_slot_time_ms: mux - .late_in_slot_time_ms - .unwrap_or(default_pbs.late_in_slot_time_ms), - ..default_pbs.clone() + let configs_with_extended_pbs = if let Some(extended_pbses) = extended_pbses { + extended_pbses + .iter() + .map(|pbs_config| PbsConfig { + timeout_get_header_ms: mux + .timeout_get_header_ms + .unwrap_or(pbs_config.timeout_get_header_ms), + late_in_slot_time_ms: mux + .late_in_slot_time_ms + .unwrap_or(pbs_config.late_in_slot_time_ms), + ..default_pbs.clone() + }) + .collect() + } else { + vec![PbsConfig { + timeout_get_header_ms: mux + .timeout_get_header_ms + .unwrap_or(default_pbs.timeout_get_header_ms), + late_in_slot_time_ms: mux + .late_in_slot_time_ms + .unwrap_or(default_pbs.late_in_slot_time_ms), + ..default_pbs.clone() + }] }; - let config = Arc::new(config); - let runtime_config = RuntimeMuxConfig { id: mux.id, config, relays: relay_clients }; - for pubkey in mux.validator_pubkeys.iter() { - configs.insert(*pubkey, runtime_config.clone()); + for config in configs_with_extended_pbs { + let config = Arc::new(config); + + let runtime_config = + RuntimeMuxConfig { id: mux.id.clone(), config, relays: relay_clients.clone() }; + for pubkey in mux.validator_pubkeys.iter() { + configs.insert(*pubkey, runtime_config.clone()); + } } } diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 68688109..9b4dfdfa 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -172,6 +172,26 @@ impl PbsConfig { } } +impl Default for PbsConfig { + fn default() -> Self { + Self { + host: default_host(), + port: DEFAULT_PBS_PORT, + relay_check: true, + wait_all_registrations: true, + timeout_get_header_ms: DefaultTimeout::GET_HEADER_MS, + timeout_get_payload_ms: DefaultTimeout::GET_PAYLOAD_MS, + timeout_register_validator_ms: DefaultTimeout::REGISTER_VALIDATOR_MS, + skip_sigverify: false, + min_bid_wei: U256::default(), + relay_monitors: Vec::new(), + late_in_slot_time_ms: LATE_IN_SLOT_TIME_MS, + extra_validation_enabled: false, + rpc_url: None, + } + } +} + /// Static pbs config from config file #[derive(Debug, Deserialize, Serialize)] pub struct StaticPbsConfig { @@ -186,6 +206,12 @@ pub struct StaticPbsConfig { pub with_signer: bool, } +impl Default for StaticPbsConfig { + fn default() -> Self { + Self { docker_image: default_pbs(), pbs_config: Default::default(), with_signer: false } + } +} + /// Runtime config for the pbs module #[derive(Debug, Clone)] pub struct PbsModuleConfig { @@ -225,9 +251,15 @@ pub async fn load_pbs_config() -> Result { SocketAddr::from((config.pbs.pbs_config.host, config.pbs.pbs_config.port)) }; + let extended_pbses = config + .extended_pbses + .map(|pbses| pbses.into_iter().map(|pbs| pbs.pbs_config).collect::>()); + let muxes = match config.muxes { Some(muxes) => { - let mux_configs = muxes.validate_and_fill(config.chain, &config.pbs.pbs_config).await?; + let mux_configs = muxes + .validate_and_fill(config.chain, &config.pbs.pbs_config, &extended_pbses) + .await?; Some(mux_configs) } None => None, @@ -270,40 +302,37 @@ pub async fn load_pbs_config() -> Result { /// Loads a custom pbs config, i.e. with signer client and/or custom data pub async fn load_pbs_custom_config() -> Result<(PbsModuleConfig, T)> { - #[derive(Debug, Deserialize)] - struct CustomPbsConfig { - #[serde(flatten)] - static_config: StaticPbsConfig, - #[serde(flatten)] - extra: U, - } - #[derive(Deserialize, Debug)] struct StubConfig { chain: Chain, relays: Vec, - pbs: CustomPbsConfig, + #[serde(skip)] + pbs: StaticPbsConfig, + extended_pbses: Option>, muxes: Option, + #[serde(flatten)] + extra: U, } // load module config including the extra data (if any) let cb_config: StubConfig = load_file_from_env(CONFIG_ENV)?; - cb_config.pbs.static_config.pbs_config.validate(cb_config.chain).await?; + cb_config.pbs.pbs_config.validate(cb_config.chain).await?; // use endpoint from env if set, otherwise use default host and port let endpoint = if let Some(endpoint) = load_optional_env_var(PBS_ENDPOINT_ENV) { endpoint.parse()? } else { - SocketAddr::from(( - cb_config.pbs.static_config.pbs_config.host, - cb_config.pbs.static_config.pbs_config.port, - )) + SocketAddr::from((cb_config.pbs.pbs_config.host, cb_config.pbs.pbs_config.port)) }; + let extended_pbses = cb_config + .extended_pbses + .map(|pbses| pbses.into_iter().map(|pbs| pbs.pbs_config).collect::>()); + let muxes = match cb_config.muxes { Some(muxes) => Some( muxes - .validate_and_fill(cb_config.chain, &cb_config.pbs.static_config.pbs_config) + .validate_and_fill(cb_config.chain, &cb_config.pbs.pbs_config, &extended_pbses) .await?, ), None => None, @@ -332,7 +361,7 @@ pub async fn load_pbs_custom_config() -> Result<(PbsModuleC let all_relays = all_relays.into_values().collect(); - let signer_client = if cb_config.pbs.static_config.with_signer { + let signer_client = if cb_config.pbs.with_signer { // if custom pbs requires a signer client, load jwt let module_jwt = load_env_var(MODULE_JWT_ENV)?; let signer_server_url = load_env_var(SIGNER_URL_ENV)?.parse()?; @@ -345,13 +374,13 @@ pub async fn load_pbs_custom_config() -> Result<(PbsModuleC PbsModuleConfig { chain: cb_config.chain, endpoint, - pbs_config: Arc::new(cb_config.pbs.static_config.pbs_config), + pbs_config: Arc::new(cb_config.pbs.pbs_config), relays: relay_clients, all_relays, signer_client, event_publisher: maybe_publiher, muxes, }, - cb_config.pbs.extra, + cb_config.extra, )) } diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index 0f78b4d6..833b9f76 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -13,7 +13,7 @@ Commit-Boost needs a configuration file detailing all the services that you want ```toml chain = "Holesky" -[pbs] +[[pbs]] port = 18550 [[relays]] @@ -247,7 +247,7 @@ This will create a Docker image called `test_da_commit` that periodically reques The `cb-config.toml` file needs to be updated as follows: ```toml -[pbs] +[[pbs]] port = 18550 [[relays]] diff --git a/examples/configs/custom_chain.toml b/examples/configs/custom_chain.toml index 58c516ad..3d81c1a0 100644 --- a/examples/configs/custom_chain.toml +++ b/examples/configs/custom_chain.toml @@ -3,7 +3,7 @@ # genesis time in seconds needs to be specified chain = { genesis_time_secs = 100, path = "tests/data/holesky_spec.json" } -[pbs] +[[pbs]] port = 18550 [[relays]] diff --git a/examples/configs/minimal.toml b/examples/configs/minimal.toml index 74217f9c..996a976e 100644 --- a/examples/configs/minimal.toml +++ b/examples/configs/minimal.toml @@ -2,7 +2,7 @@ chain = "Holesky" -[pbs] +[[pbs]] port = 18550 [[relays]] diff --git a/examples/configs/pbs_metrics.toml b/examples/configs/pbs_metrics.toml index b8013c15..51291ebe 100644 --- a/examples/configs/pbs_metrics.toml +++ b/examples/configs/pbs_metrics.toml @@ -2,7 +2,7 @@ chain = "Holesky" -[pbs] +[[pbs]] port = 18550 [[relays]] diff --git a/examples/configs/pbs_mux.toml b/examples/configs/pbs_mux.toml index 3ea9f355..868362a6 100644 --- a/examples/configs/pbs_mux.toml +++ b/examples/configs/pbs_mux.toml @@ -2,7 +2,7 @@ chain = "Holesky" -[pbs] +[[pbs]] port = 18550 timeout_get_header_ms = 950 late_in_slot_time_ms = 2000