Ban solvers based on the settlements success rate

crates/autopilot/src/arguments.rs

@@ -253,22 +253,63 @@ pub struct Arguments {
 
 #[derive(Debug, clap::Parser)]
 pub struct DbBasedSolverParticipationGuardConfig {
-    /// Enables or disables the solver participation guard
+    /// The time-to-live for the solver participation blacklist cache.
+    #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)]
+    pub solver_blacklist_cache_ttl: Duration,
+
+    #[clap(flatten)]
+    pub non_settling_solvers_finder_config: NonSettlingSolversFinderConfig,
+
+    #[clap(flatten)]
+    pub low_settling_solvers_finder_config: LowSettlingSolversFinderConfig,
+}
+
+#[derive(Debug, clap::Parser)]
+pub struct NonSettlingSolversFinderConfig {
+    /// Enables search of non-settling solvers.
     #[clap(
-        id = "db_enabled",
-        long = "db-based-solver-participation-guard-enabled",
-        env = "DB_BASED_SOLVER_PARTICIPATION_GUARD_ENABLED",
+        id = "non_settling_solvers_blacklisting_enabled",
+        long = "non-settling-solvers-blacklisting-enabled",
+        env = "NON_SETTLING_SOLVERS_BLACKLISTING_ENABLED",
         default_value = "true"
     )]
     pub enabled: bool,
 
-    /// The time-to-live for the solver participation blacklist cache.
-    #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)]
-    pub solver_blacklist_cache_ttl: Duration,
+    /// The number of last auctions to check solver participation eligibility.
+    #[clap(
+        id = "non_settling_last_auctions_participation_count",
+        long = "non-settling-last-auctions-participation-count",
+        env = "NON_SETTLING_LAST_AUCTIONS_PARTICIPATION_COUNT",
+        default_value = "3"
+    )]
+    pub last_auctions_participation_count: u32,
+}
+
+#[derive(Debug, clap::Parser)]
+pub struct LowSettlingSolversFinderConfig {
+    /// Enables search of non-settling solvers.
+    #[clap(
+        id = "low_settling_solvers_blacklisting_enabled",
+        long = "low-settling-solvers-blacklisting-enabled",
+        env = "LOW_SETTLING_SOLVERS_BLACKLISTING_ENABLED",
+        default_value = "true"
+    )]
+    pub enabled: bool,
 
     /// The number of last auctions to check solver participation eligibility.
-    #[clap(long, env, default_value = "3")]
-    pub solver_last_auctions_participation_count: u32,
+    #[clap(
+        id = "low_settling_last_auctions_participation_count",
+        long = "low-settling-last-auctions-participation-count",
+        env = "LOW_SETTLING_LAST_AUCTIONS_PARTICIPATION_COUNT",
+        default_value = "100"
+    )]
+    pub last_auctions_participation_count: u32,
+
+    /// A max failure rate for a solver to remain eligible for
+    /// participation in the competition. Otherwise, the solver will be
+    /// banned.
+    #[clap(long, env, default_value = "0.9")]
+    pub solver_max_settlement_failure_rate: f64,
 }
 
 impl std::fmt::Display for Arguments {

crates/autopilot/src/domain/competition/participation_guard/db.rs

@@ -1,44 +1,52 @@
 use {
     crate::{
+        arguments::{
+            DbBasedSolverParticipationGuardConfig,
+            LowSettlingSolversFinderConfig,
+            NonSettlingSolversFinderConfig,
+        },
         domain::{eth, Metrics},
-        infra,
+        infra::{self, solvers::dto},
     },
     ethrpc::block_stream::CurrentBlockWatcher,
     model::time::now_in_epoch_seconds,
     std::{
-        collections::HashMap,
+        collections::{HashMap, HashSet},
         sync::Arc,
         time::{Duration, Instant},
     },
+    tokio::join,
 };
 
 /// Checks the DB by searching for solvers that won N last consecutive auctions
-/// but never settled any of them.
+/// and either never settled any of them or their settlement success rate is
+/// lower than `min_settlement_success_rate`.
 #[derive(Clone)]
-pub(super) struct Validator(Arc<Inner>);
+pub(super) struct SolverValidator(Arc<Inner>);
 
 struct Inner {
     persistence: infra::Persistence,
     banned_solvers: dashmap::DashMap<eth::Address, Instant>,
     ttl: Duration,
-    last_auctions_count: u32,
+    non_settling_config: NonSettlingSolversFinderConfig,
+    low_settling_config: LowSettlingSolversFinderConfig,
     drivers_by_address: HashMap<eth::Address, Arc<infra::Driver>>,
 }
 
-impl Validator {
+impl SolverValidator {
     pub fn new(
         persistence: infra::Persistence,
         current_block: CurrentBlockWatcher,
         competition_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>,
-        ttl: Duration,
-        last_auctions_count: u32,
+        db_based_validator_config: DbBasedSolverParticipationGuardConfig,
         drivers_by_address: HashMap<eth::Address, Arc<infra::Driver>>,
     ) -> Self {
         let self_ = Self(Arc::new(Inner {
             persistence,
             banned_solvers: Default::default(),
-            ttl,
-            last_auctions_count,
+            ttl: db_based_validator_config.solver_blacklist_cache_ttl,
+            non_settling_config: db_based_validator_config.non_settling_solvers_finder_config,
+            low_settling_config: db_based_validator_config.low_settling_solvers_finder_config,
             drivers_by_address,
         }));
 
@@ -59,66 +67,140 @@ impl Validator {
         tokio::spawn(async move {
             while competition_updates_receiver.recv().await.is_some() {
                 let current_block = current_block.borrow().number;
-                match self_
-                    .0
-                    .persistence
-                    .find_non_settling_solvers(self_.0.last_auctions_count, current_block)
-                    .await
-                {
-                    Ok(non_settling_solvers) => {
-                        let non_settling_drivers = non_settling_solvers
-                            .into_iter()
-                            .filter_map(|solver| {
-                                self_.0.drivers_by_address.get(&solver).map(|driver| {
-                                    Metrics::get()
-                                        .non_settling_solver
-                                        .with_label_values(&[&driver.name]);
-
-                                    driver.clone()
-                                })
-                            })
-                            .collect::<Vec<_>>();
-
-                        let non_settling_solver_names = non_settling_drivers
-                            .iter()
-                            .map(|driver| driver.name.clone())
-                            .collect::<Vec<_>>();
-
-                        tracing::debug!(solvers = ?non_settling_solver_names, "found non-settling solvers");
-
-                        let non_settling_drivers = non_settling_drivers
-                            .into_iter()
-                            // Check if solver accepted this feature. This should be removed once a CIP is
-                            // approved.
-                            .filter(|driver| driver.accepts_unsettled_blocking)
-                            .collect::<Vec<_>>();
-
-                        let now = Instant::now();
-                        let banned_until_timestamp =
-                            u64::from(now_in_epoch_seconds()) + self_.0.ttl.as_secs();
-                        infra::notify_non_settling_solvers(
-                            &non_settling_drivers,
-                            banned_until_timestamp,
-                        );
-
-                        for driver in non_settling_drivers {
-                            self_
-                                .0
-                                .banned_solvers
-                                .insert(driver.submission_address, now);
-                        }
-                    }
-                    Err(err) => {
-                        tracing::warn!(?err, "error while searching for non-settling solvers")
-                    }
-                }
+
+                let (non_settling_solvers, mut low_settling_solvers) = join!(
+                    self_.find_non_settling_solvers(current_block),
+                    self_.find_low_settling_solvers(current_block)
+                );
+                // Non-settling issue has a higher priority, remove duplicates from low-settling
+                // solvers.
+                low_settling_solvers.retain(|solver| !non_settling_solvers.contains(solver));
+
+                let found_at = Instant::now();
+                let banned_until_timestamp =
+                    u64::from(now_in_epoch_seconds()) + self_.0.ttl.as_secs();
+
+                self_.post_process(
+                    &non_settling_solvers,
+                    &dto::notify::BanReason::UnsettledConsecutiveAuctions,
+                    found_at,
+                    banned_until_timestamp,
+                );
+                self_.post_process(
+                    &low_settling_solvers,
+                    &dto::notify::BanReason::HighSettleFailureRate,
+                    found_at,
+                    banned_until_timestamp,
+                );
             }
         });
     }
+
+    async fn find_non_settling_solvers(&self, current_block: u64) -> HashSet<eth::Address> {
+        if !self.0.non_settling_config.enabled {
+            return Default::default();
+        }
+
+        match self
+            .0
+            .persistence
+            .find_non_settling_solvers(
+                self.0.non_settling_config.last_auctions_participation_count,
+                current_block,
+            )
+            .await
+        {
+            Ok(solvers) => solvers.into_iter().collect(),
+            Err(err) => {
+                tracing::warn!(?err, "error while searching for non-settling solvers");
+                Default::default()
+            }
+        }
+    }
+
+    async fn find_low_settling_solvers(&self, current_block: u64) -> HashSet<eth::Address> {
+        if !self.0.low_settling_config.enabled {
+            return Default::default();
+        }
+
+        match self
+            .0
+            .persistence
+            .find_low_settling_solvers(
+                self.0.low_settling_config.last_auctions_participation_count,
+                current_block,
+                self.0
+                    .low_settling_config
+                    .solver_max_settlement_failure_rate,
+            )
+            .await
+        {
+            Ok(solvers) => solvers.into_iter().collect(),
+            Err(err) => {
+                tracing::warn!(?err, "error while searching for low-settling solvers");
+                Default::default()
+            }
+        }
+    }
+
+    /// Updates the cache and notifies the solvers.
+    fn post_process(
+        &self,
+        solvers: &HashSet<eth::Address>,
+        ban_reason: &dto::notify::BanReason,
+        found_at: Instant,
+        banned_until_timestamp: u64,
+    ) {
+        if solvers.is_empty() {
+            return;
+        }
+
+        let drivers = solvers
+            .iter()
+            .filter_map(|solver| self.0.drivers_by_address.get(solver).cloned())
+            .collect::<Vec<_>>();
+
+        let log_message = match ban_reason {
+            dto::notify::BanReason::UnsettledConsecutiveAuctions => "found non-settling solvers",
+            dto::notify::BanReason::HighSettleFailureRate => {
+                "found high-failure-settlement solvers"
+            }
+        };
+        let solver_names = drivers
+            .iter()
+            .map(|driver| driver.name.clone())
+            .collect::<Vec<_>>();
+        tracing::debug!(solvers = ?solver_names, log_message);
+
+        let reason = match ban_reason {
+            dto::notify::BanReason::UnsettledConsecutiveAuctions => "non_settling",
+            dto::notify::BanReason::HighSettleFailureRate => "high_settle_failure_rate",
+        };
+
+        for solver in solver_names {
+            Metrics::get()
+                .banned_solver
+                .with_label_values(&[&solver, reason]);
+        }
+
+        let banned_drivers = drivers
+            .into_iter()
+            // Notify and block only solvers that accept unsettled blocking feature. This should be removed once a CIP is approved.
+            .filter(|driver| driver.accepts_unsettled_blocking)
+            .collect::<Vec<_>>();
+
+        infra::notify_banned_solvers(&banned_drivers, ban_reason, banned_until_timestamp);
+
+        for driver in banned_drivers {
+            self.0
+                .banned_solvers
+                .insert(driver.submission_address, found_at);
+        }
+    }
 }
 
 #[async_trait::async_trait]
-impl super::Validator for Validator {
+impl super::SolverValidator for SolverValidator {
     async fn is_allowed(&self, solver: &eth::Address) -> anyhow::Result<bool> {
         if let Some(entry) = self.0.banned_solvers.get(solver) {
             if Instant::now().duration_since(*entry.value()) < self.0.ttl {

crates/autopilot/src/domain/competition/participation_guard/mod.rs

@@ -13,7 +13,7 @@ pub struct SolverParticipationGuard(Arc<Inner>);
 
 struct Inner {
     /// Stores the validators in order they will be called.
-    validators: Vec<Box<dyn Validator + Send + Sync>>,
+    validators: Vec<Box<dyn SolverValidator + Send + Sync>>,
 }
 
 impl SolverParticipationGuard {
@@ -24,20 +24,17 @@ impl SolverParticipationGuard {
         db_based_validator_config: DbBasedSolverParticipationGuardConfig,
         drivers_by_address: HashMap<eth::Address, Arc<infra::Driver>>,
     ) -> Self {
-        let mut validators: Vec<Box<dyn Validator + Send + Sync>> = Vec::new();
+        let mut validators: Vec<Box<dyn SolverValidator + Send + Sync>> = Vec::new();
 
-        if db_based_validator_config.enabled {
-            let current_block = eth.current_block().clone();
-            let database_solver_participation_validator = db::Validator::new(
-                persistence,
-                current_block,
-                competition_updates_receiver,
-                db_based_validator_config.solver_blacklist_cache_ttl,
-                db_based_validator_config.solver_last_auctions_participation_count,
-                drivers_by_address,
-            );
-            validators.push(Box::new(database_solver_participation_validator));
-        }
+        let current_block = eth.current_block().clone();
+        let database_solver_participation_validator = db::SolverValidator::new(
+            persistence,
+            current_block,
+            competition_updates_receiver,
+            db_based_validator_config,
+            drivers_by_address,
+        );
+        validators.push(Box::new(database_solver_participation_validator));
 
         let onchain_solver_participation_validator = onchain::Validator { eth };
         validators.push(Box::new(onchain_solver_participation_validator));
@@ -62,6 +59,6 @@ impl SolverParticipationGuard {
 }
 
 #[async_trait::async_trait]
-trait Validator: Send + Sync {
+trait SolverValidator: Send + Sync {
     async fn is_allowed(&self, solver: &eth::Address) -> anyhow::Result<bool>;
 }

crates/autopilot/src/domain/competition/participation_guard/onchain.rs

@@ -7,7 +7,7 @@ pub(super) struct Validator {
 }
 
 #[async_trait::async_trait]
-impl super::Validator for Validator {
+impl super::SolverValidator for Validator {
     async fn is_allowed(&self, solver: &eth::Address) -> anyhow::Result<bool> {
         Ok(self
             .eth