diff --git a/neurons/validator.py b/neurons/validator.py index fa65a54b..227d7026 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -209,17 +209,31 @@ def refresh_metagraph(self, interval_minutes: int): An empty result (degenerate chain response) is applied — the shared metagraph object was just emptied by the sync, so stale uids must not outlive it — but not stamped, so the next cycle retries. + + A sync that raises (e.g. the chain endpoint rate-limiting with + HTTP 429) is swallowed: the previous miner set stays in use and, + because the timer is not stamped, the next cycle retries. This + keeps a transient chain outage from killing the process — the + boot call and the scoring loop have no other safety net. """ now = get_current_time() if not metagraph_refresh_due( self.last_metagraph_refresh_at, now, interval_minutes ): return - self.miner_uids = get_available_miners_and_update_metagraph_history( - base_neuron=self, - miner_data_handler=self.miner_data_handler, - save_snapshot=self.cycle_name == CYCLE_SCORING, - ) + try: + self.miner_uids = ( + get_available_miners_and_update_metagraph_history( + base_neuron=self, + miner_data_handler=self.miner_data_handler, + save_snapshot=self.cycle_name == CYCLE_SCORING, + ) + ) + except Exception: + bt.logging.exception( + "Metagraph refresh failed; keeping the previous miner set" + ) + return if self.miner_uids: self.last_metagraph_refresh_at = now diff --git a/synth/base/neuron.py b/synth/base/neuron.py index f0061139..6d1c8ad5 100644 --- a/synth/base/neuron.py +++ b/synth/base/neuron.py @@ -19,6 +19,11 @@ import bittensor as bt from bittensor.core.metagraph import MetagraphMixin +from tenacity import ( + retry, + stop_after_attempt, + wait_random_exponential, +) from abc import ABC, abstractmethod @@ -80,7 +85,7 @@ def __init__(self, config=None): # The wallet holds the cryptographic key pairs for the miner. self.wallet = bt.Wallet(config=self.config) self.subtensor = bt.Subtensor(config=self.config) - self.metagraph = self.subtensor.metagraph(self.config.netuid) + self.metagraph = self._initial_metagraph_fetch() bt.logging.info(f"Wallet: {self.wallet}") bt.logging.info(f"Subtensor: {self.subtensor}") @@ -98,6 +103,14 @@ def __init__(self, config=None): ) self.step = 0 + @retry( + stop=stop_after_attempt(10), + wait=wait_random_exponential(multiplier=2, max=120), + reraise=True, + ) + def _initial_metagraph_fetch(self) -> MetagraphMixin: + return self.subtensor.metagraph(self.config.netuid) + @abstractmethod async def forward_miner(self, synapse: bt.Synapse) -> bt.Synapse: ... diff --git a/tests/test_refresh_metagraph.py b/tests/test_refresh_metagraph.py new file mode 100644 index 00000000..5796375e --- /dev/null +++ b/tests/test_refresh_metagraph.py @@ -0,0 +1,46 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from neurons.validator import Validator + + +def _fake_validator(): + return SimpleNamespace( + last_metagraph_refresh_at=None, + miner_uids=[1, 2], + cycle_name="high_frequency", + miner_data_handler=None, + ) + + +class TestRefreshMetagraph(unittest.TestCase): + def test_sync_failure_keeps_previous_miner_set(self): + # A chain sync failure (e.g. endpoint rate limit) must not + # propagate — the boot call and the scoring loop have no other + # safety net — and must not stamp the timer, so the next cycle + # retries. + fake = _fake_validator() + with patch( + "neurons.validator." + "get_available_miners_and_update_metagraph_history", + side_effect=ConnectionError("server rejected: HTTP 429"), + ): + Validator.refresh_metagraph(fake, 15) + self.assertEqual(fake.miner_uids, [1, 2]) + self.assertIsNone(fake.last_metagraph_refresh_at) + + def test_successful_sync_stamps_timer(self): + fake = _fake_validator() + with patch( + "neurons.validator." + "get_available_miners_and_update_metagraph_history", + return_value=[3, 4], + ): + Validator.refresh_metagraph(fake, 15) + self.assertEqual(fake.miner_uids, [3, 4]) + self.assertIsNotNone(fake.last_metagraph_refresh_at) + + +if __name__ == "__main__": + unittest.main()