Updated sweep calculation

src/constants.py

@@ -1,41 +1,44 @@
 from src.types import Gwei
 
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc
-FAR_FUTURE_EPOCH = 2 ** 64 - 1
+FAR_FUTURE_EPOCH = 2**64 - 1
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#time-parameters-1
-MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2 ** 8
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8
 SHARD_COMMITTEE_PERIOD = 256
 MAX_SEED_LOOKAHEAD = 4
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#state-list-lengths
-EPOCHS_PER_SLASHINGS_VECTOR = 2 ** 13
+EPOCHS_PER_SLASHINGS_VECTOR = 2**13
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#rewards-and-penalties
 PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX = 3
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#gwei-values
-EFFECTIVE_BALANCE_INCREMENT = 2 ** 0 * 10 ** 9
-MAX_EFFECTIVE_BALANCE = Gwei(32 * 10 ** 9)
+EFFECTIVE_BALANCE_INCREMENT = 2**0 * 10**9
+MAX_EFFECTIVE_BALANCE = Gwei(32 * 10**9)
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#execution
-MAX_WITHDRAWALS_PER_PAYLOAD = 2 ** 4
+MAX_WITHDRAWALS_PER_PAYLOAD = 2**4
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#withdrawal-prefixes
 ETH1_ADDRESS_WITHDRAWAL_PREFIX = '0x01'
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#withdrawal-prefixes
 COMPOUNDING_WITHDRAWAL_PREFIX = '0x02'
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#validator-cycle
-MIN_PER_EPOCH_CHURN_LIMIT = 2 ** 2
-CHURN_LIMIT_QUOTIENT = 2 ** 16
+MIN_PER_EPOCH_CHURN_LIMIT = 2**2
+CHURN_LIMIT_QUOTIENT = 2**16
+# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#validator-cycle
+MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA = Gwei(2**7 * 10**9)
+MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT = Gwei(2**8 * 10**9)
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#time-parameters
 SLOTS_PER_HISTORICAL_ROOT = 8192
 
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#gwei-values
-MIN_ACTIVATION_BALANCE = Gwei(2 ** 5 * 10 ** 9)
-MAX_EFFECTIVE_BALANCE_ELECTRA = Gwei(2 ** 11 * 10 ** 9)
+MIN_ACTIVATION_BALANCE = Gwei(2**5 * 10**9)
+MAX_EFFECTIVE_BALANCE_ELECTRA = Gwei(2**11 * 10**9)
 
 LIDO_DEPOSIT_AMOUNT = MIN_ACTIVATION_BALANCE
 
 # Local constants
-GWEI_TO_WEI = 10 ** 9
-SHARE_RATE_PRECISION_E27 = 10 ** 27
+GWEI_TO_WEI = 10**9
+SHARE_RATE_PRECISION_E27 = 10**27
 TOTAL_BASIS_POINTS = 10000
 
 MAX_BLOCK_GAS_LIMIT = 30_000_000
 
-UINT64_MAX = 2 ** 64 - 1
+UINT64_MAX = 2**64 - 1

src/main.py

@@ -24,7 +24,7 @@
     KeysAPIClientModule,
     LidoValidatorsProvider,
     FallbackProviderModule,
-    LazyCSM
+    LazyCSM,
 )
 from src.web3py.middleware import metrics_collector
 from src.web3py.types import Web3

src/modules/ejector/ejector.py

@@ -1,25 +1,25 @@
 import logging
+import math
 from functools import reduce
 
-from more_itertools import ilen
 from web3.exceptions import ContractCustomError
 from web3.types import Wei
 
 from src.constants import (
     FAR_FUTURE_EPOCH,
-    MAX_EFFECTIVE_BALANCE,
     MAX_WITHDRAWALS_PER_PAYLOAD,
+    MIN_ACTIVATION_BALANCE,
     MIN_VALIDATOR_WITHDRAWABILITY_DELAY,
 )
 from src.metrics.prometheus.business import CONTRACT_ON_PAUSE
+from src.metrics.prometheus.duration_meter import duration_meter
 from src.metrics.prometheus.ejector import (
-    EJECTOR_VALIDATORS_COUNT_TO_EJECT,
-    EJECTOR_TO_WITHDRAW_WEI_AMOUNT,
     EJECTOR_MAX_WITHDRAWAL_EPOCH,
+    EJECTOR_TO_WITHDRAW_WEI_AMOUNT,
+    EJECTOR_VALIDATORS_COUNT_TO_EJECT,
 )
-from src.metrics.prometheus.duration_meter import duration_meter
 from src.modules.ejector.data_encode import encode_data
-from src.modules.ejector.types import ReportData, EjectorProcessingState
+from src.modules.ejector.types import EjectorProcessingState, ReportData
 from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert
 from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay
 from src.modules.submodules.types import ZERO_HASH
@@ -29,19 +29,18 @@
 from src.services.exit_order_v2.iterator import ValidatorExitIteratorV2
 from src.services.prediction import RewardsPredictionService
 from src.services.validator_state import LidoValidatorStateService
-from src.types import BlockStamp, EpochNumber, ReferenceBlockStamp, NodeOperatorGlobalIndex
+from src.types import BlockStamp, EpochNumber, NodeOperatorGlobalIndex, ReferenceBlockStamp
 from src.utils.cache import global_lru_cache as lru_cache
 from src.utils.validator_state import (
+    compute_activation_exit_epoch,
+    compute_exit_churn_limit,
     is_active_validator,
     is_fully_withdrawable_validator,
     is_partially_withdrawable_validator,
-    compute_activation_exit_epoch,
-    compute_exit_churn_limit,
 )
 from src.web3py.extensions.lido_validators import LidoValidator
 from src.web3py.types import Web3
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -62,6 +61,7 @@ class Ejector(BaseModule, ConsensusModule):
 
     3. Decode lido validators into bytes and send report transaction
     """
+
     COMPATIBLE_ONCHAIN_VERSIONS = [(1, 1), (1, 2)]
 
     AVG_EXPECTING_WITHDRAWALS_SWEEP_DURATION_MULTIPLIER = 0.5
@@ -75,7 +75,7 @@ def __init__(self, w3: Web3):
         self.validators_state_service = LidoValidatorStateService(w3)
 
     def refresh_contracts(self):
-        self.report_contract = self.w3.lido_contracts.validators_exit_bus_oracle
+        self.report_contract = self.w3.lido_contracts.validators_exit_bus_oracle  # type: ignore
 
     def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecuteDelay:
         report_blockstamp = self.get_blockstamp_for_report(last_finalized_blockstamp)
@@ -119,7 +119,7 @@ def get_validators_to_eject(self, blockstamp: ReferenceBlockStamp) -> list[tuple
 
         expected_balance = self._get_total_expected_balance(0, blockstamp)
 
-        consensus_version = self.w3.lido_contracts.validators_exit_bus_oracle.get_consensus_version(blockstamp.block_hash)
+        consensus_version = self.get_consensus_version(blockstamp)
         validators_iterator = iter(self.get_validators_iterator(consensus_version, blockstamp))
 
         validators_to_eject: list[tuple[NodeOperatorGlobalIndex, LidoValidator]] = []
@@ -141,7 +141,7 @@ def get_validators_to_eject(self, blockstamp: ReferenceBlockStamp) -> list[tuple
             'validators_to_eject_count': len(validators_to_eject),
         })
 
-        if consensus_version != 1:
+        if self.get_consensus_version(blockstamp) != 1:
             forced_validators = validators_iterator.get_remaining_forced_validators()
             if forced_validators:
                 logger.info({'msg': 'Eject forced to exit validators.', 'len': len(forced_validators)})
@@ -203,23 +203,16 @@ def is_reporting_allowed(self, blockstamp: ReferenceBlockStamp) -> bool:
     @lru_cache(maxsize=1)
     def _get_withdrawable_lido_validators_balance(self, on_epoch: EpochNumber, blockstamp: BlockStamp) -> Wei:
         lido_validators = self.w3.lido_validators.get_lido_validators(blockstamp=blockstamp)
-
-        def get_total_withdrawable_balance(balance: Wei, validator: Validator) -> Wei:
-            if is_fully_withdrawable_validator(validator, on_epoch):
-                return Wei(balance + self._get_predicted_withdrawable_balance(validator))
-
-            return balance
-
-        result = reduce(
-            get_total_withdrawable_balance,
-            lido_validators,
-            Wei(0),
+        return Wei(
+            sum(
+                self._get_predicted_withdrawable_balance(v)
+                for v in lido_validators
+                if is_fully_withdrawable_validator(v, on_epoch)
+            )
         )
 
-        return result
-
     def _get_predicted_withdrawable_balance(self, validator: Validator) -> Wei:
-        return self.w3.to_wei(min(int(validator.balance), MAX_EFFECTIVE_BALANCE), 'gwei')
+        return self.w3.to_wei(min(int(validator.balance), MIN_ACTIVATION_BALANCE), 'gwei')
 
     @lru_cache(maxsize=1)
     def _get_total_el_balance(self, blockstamp: BlockStamp) -> Wei:
@@ -285,20 +278,39 @@ def _get_latest_exit_epoch(self, blockstamp: ReferenceBlockStamp) -> tuple[Epoch
     @lru_cache(maxsize=1)
     def _get_sweep_delay_in_epochs(self, blockstamp: ReferenceBlockStamp) -> int:
         """Returns amount of epochs that will take to sweep all validators in chain."""
+
+        if self.get_consensus_version(blockstamp) in (1, 2):
+            return self._get_sweep_delay_in_epochs_pre_pectra(blockstamp)
+        return self._get_sweep_delay_in_epochs_post_pectra(blockstamp)
+
+    def _get_sweep_delay_in_epochs_pre_pectra(self, blockstamp: ReferenceBlockStamp) -> int:
         chain_config = self.get_chain_config(blockstamp)
-        total_withdrawable_validators = self._get_total_withdrawable_validators(blockstamp)
+
+        total_withdrawable_validators = len(self._get_withdrawable_validators(blockstamp))
+        logger.info({'msg': 'Calculate total withdrawable validators.', 'value': total_withdrawable_validators})
 
         full_sweep_in_epochs = total_withdrawable_validators / MAX_WITHDRAWALS_PER_PAYLOAD / chain_config.slots_per_epoch
         return int(full_sweep_in_epochs * self.AVG_EXPECTING_WITHDRAWALS_SWEEP_DURATION_MULTIPLIER)
 
-    def _get_total_withdrawable_validators(self, blockstamp: ReferenceBlockStamp) -> int:
-        total_withdrawable_validators = ilen(filter(lambda validator: (
-            is_partially_withdrawable_validator(validator) or
-            is_fully_withdrawable_validator(validator, blockstamp.ref_epoch)
-        ), self.w3.cc.get_validators(blockstamp)))
-
+    def _get_sweep_delay_in_epochs_post_pectra(self, blockstamp: ReferenceBlockStamp) -> int:
+        # This version is intended for use with Pectra, but we do not currently take into account pending withdrawal
+        # requests. It would require a large amount of pending withdrawal requests to significantly impact sweep
+        # duration. Roughly every 512 requests adds one more epoch to sweep duration in the current state.
+        # On the other side, to consider pending withdrawals it is necessary to fetch the beacon state and query the
+        # EIP-7002 predeployed contract, which adds complexity with limited improvement for predictions.
+        chain_config = self.get_chain_config(blockstamp)
+        total_withdrawable_validators = len(self._get_withdrawable_validators(blockstamp))
         logger.info({'msg': 'Calculate total withdrawable validators.', 'value': total_withdrawable_validators})
-        return total_withdrawable_validators
+        slots_to_sweep = math.ceil(total_withdrawable_validators / MAX_WITHDRAWALS_PER_PAYLOAD)
+        full_sweep_in_epochs = math.ceil(slots_to_sweep / chain_config.slots_per_epoch)
+        return math.ceil(full_sweep_in_epochs * self.AVG_EXPECTING_WITHDRAWALS_SWEEP_DURATION_MULTIPLIER)
+
+    def _get_withdrawable_validators(self, blockstamp: ReferenceBlockStamp) -> list[Validator]:
+        return [
+            v
+            for v in self.w3.cc.get_validators(blockstamp)
+            if is_partially_withdrawable_validator(v) or is_fully_withdrawable_validator(v, blockstamp.ref_epoch)
+        ]
 
     @lru_cache(maxsize=1)
     def _get_churn_limit(self, blockstamp: ReferenceBlockStamp) -> int:

src/modules/submodules/consensus.py

@@ -87,6 +87,10 @@ def _get_consensus_contract_members(self, blockstamp: BlockStamp):
         consensus_contract = self._get_consensus_contract(blockstamp)
         return consensus_contract.get_members(blockstamp.block_hash)
 
+    @lru_cache(maxsize=1)
+    def get_consensus_version(self, blockstamp: BlockStamp):
+        return self.report_contract.get_consensus_version(blockstamp.block_hash)
+
     @lru_cache(maxsize=1)
     def get_chain_config(self, blockstamp: BlockStamp) -> ChainConfig:
         consensus_contract = self._get_consensus_contract(blockstamp)

src/web3py/types.py

@@ -1,14 +1,13 @@
 from web3 import Web3 as _Web3
 
-
 from src.providers.ipfs import IPFSProvider
 from src.web3py.extensions import (
-    LidoContracts,
-    TransactionUtils,
+    CSM,
     ConsensusClientModule,
     KeysAPIClientModule,
+    LidoContracts,
     LidoValidatorsProvider,
-    CSM
+    TransactionUtils,
 )
 
 

tests/factory/no_registry.py

@@ -6,18 +6,20 @@
 from hexbytes import HexBytes
 from pydantic_factories import Use
 
-from src.constants import FAR_FUTURE_EPOCH
+from src.constants import EFFECTIVE_BALANCE_INCREMENT, FAR_FUTURE_EPOCH, MIN_ACTIVATION_BALANCE
 from src.providers.consensus.types import Validator, ValidatorState
 from src.providers.keys.types import LidoKey
+from src.types import Gwei
+from src.web3py.extensions.lido_validators import LidoValidator, NodeOperator, StakingModule
 from tests.factory.web3_factory import Web3Factory
-from src.web3py.extensions.lido_validators import StakingModule, LidoValidator, NodeOperator
 
 faker = Faker()
 
 
 class ValidatorStateFactory(Web3Factory):
     __model__ = ValidatorState
 
+    withdrawal_credentials = "0x01"
     exit_epoch = FAR_FUTURE_EPOCH
 
     @classmethod
@@ -40,7 +42,7 @@ def build_pending_deposit_vals(cls, **kwargs: Any):
                 exit_epoch=str(FAR_FUTURE_EPOCH),
                 effective_balance=str(0),
             ),
-            **kwargs
+            **kwargs,
         )
 
 
@@ -83,7 +85,7 @@ def build_pending_deposit_vals(cls, **kwargs: Any):
                 exit_epoch=str(FAR_FUTURE_EPOCH),
                 effective_balance=str(0),
             ),
-            **kwargs
+            **kwargs,
         )
 
     @classmethod
@@ -93,7 +95,7 @@ def build_not_active_vals(cls, epoch, **kwargs: Any):
                 activation_epoch=str(faker.pyint(min_value=epoch + 1, max_value=FAR_FUTURE_EPOCH)),
                 exit_epoch=str(FAR_FUTURE_EPOCH),
             ),
-            **kwargs
+            **kwargs,
         )
 
     @classmethod
@@ -103,7 +105,7 @@ def build_active_vals(cls, epoch, **kwargs: Any):
                 activation_epoch=str(faker.pyint(min_value=0, max_value=epoch - 1)),
                 exit_epoch=str(faker.pyint(min_value=epoch + 1, max_value=FAR_FUTURE_EPOCH)),
             ),
-            **kwargs
+            **kwargs,
         )
 
     @classmethod
@@ -113,7 +115,17 @@ def build_exit_vals(cls, epoch, **kwargs: Any):
                 activation_epoch='0',
                 exit_epoch=str(faker.pyint(min_value=1, max_value=epoch)),
             ),
-            **kwargs
+            **kwargs,
+        )
+
+    @classmethod
+    def build_with_balance(cls, balance: Gwei, **kwargs: Any):
+        return cls.build(
+            balance=balance,
+            validator=ValidatorStateFactory.build(
+                effective_balance=min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MIN_ACTIVATION_BALANCE),
+            ),
+            **kwargs,
         )