validator: survive chain-endpoint rate limiting instead of crashing

neurons/validator.py

@@ -209,17 +209,31 @@ def refresh_metagraph(self, interval_minutes: int):
         An empty result (degenerate chain response) is applied — the shared
         metagraph object was just emptied by the sync, so stale uids must
         not outlive it — but not stamped, so the next cycle retries.
+
+        A sync that raises (e.g. the chain endpoint rate-limiting with
+        HTTP 429) is swallowed: the previous miner set stays in use and,
+        because the timer is not stamped, the next cycle retries. This
+        keeps a transient chain outage from killing the process — the
+        boot call and the scoring loop have no other safety net.
         """
         now = get_current_time()
         if not metagraph_refresh_due(
             self.last_metagraph_refresh_at, now, interval_minutes
         ):
             return
-        self.miner_uids = get_available_miners_and_update_metagraph_history(
-            base_neuron=self,
-            miner_data_handler=self.miner_data_handler,
-            save_snapshot=self.cycle_name == CYCLE_SCORING,
-        )
+        try:
+            self.miner_uids = (
+                get_available_miners_and_update_metagraph_history(
+                    base_neuron=self,
+                    miner_data_handler=self.miner_data_handler,
+                    save_snapshot=self.cycle_name == CYCLE_SCORING,
+                )
+            )
+        except Exception:
+            bt.logging.exception(
+                "Metagraph refresh failed; keeping the previous miner set"
+            )
+            return
         if self.miner_uids:
             self.last_metagraph_refresh_at = now
 

synth/base/neuron.py

@@ -19,6 +19,11 @@
 
 import bittensor as bt
 from bittensor.core.metagraph import MetagraphMixin
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
 
 from abc import ABC, abstractmethod
 
@@ -80,7 +85,7 @@ def __init__(self, config=None):
         # The wallet holds the cryptographic key pairs for the miner.
         self.wallet = bt.Wallet(config=self.config)
         self.subtensor = bt.Subtensor(config=self.config)
-        self.metagraph = self.subtensor.metagraph(self.config.netuid)
+        self.metagraph = self._initial_metagraph_fetch()
 
         bt.logging.info(f"Wallet: {self.wallet}")
         bt.logging.info(f"Subtensor: {self.subtensor}")
@@ -98,6 +103,14 @@ def __init__(self, config=None):
         )
         self.step = 0
 
+    @retry(
+        stop=stop_after_attempt(10),
+        wait=wait_random_exponential(multiplier=2, max=120),
+        reraise=True,
+    )
+    def _initial_metagraph_fetch(self) -> MetagraphMixin:
+        return self.subtensor.metagraph(self.config.netuid)
+
     @abstractmethod
     async def forward_miner(self, synapse: bt.Synapse) -> bt.Synapse: ...
 

tests/test_refresh_metagraph.py

@@ -0,0 +1,46 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from neurons.validator import Validator
+
+
+def _fake_validator():
+    return SimpleNamespace(
+        last_metagraph_refresh_at=None,
+        miner_uids=[1, 2],
+        cycle_name="high_frequency",
+        miner_data_handler=None,
+    )
+
+
+class TestRefreshMetagraph(unittest.TestCase):
+    def test_sync_failure_keeps_previous_miner_set(self):
+        # A chain sync failure (e.g. endpoint rate limit) must not
+        # propagate — the boot call and the scoring loop have no other
+        # safety net — and must not stamp the timer, so the next cycle
+        # retries.
+        fake = _fake_validator()
+        with patch(
+            "neurons.validator."
+            "get_available_miners_and_update_metagraph_history",
+            side_effect=ConnectionError("server rejected: HTTP 429"),
+        ):
+            Validator.refresh_metagraph(fake, 15)
+        self.assertEqual(fake.miner_uids, [1, 2])
+        self.assertIsNone(fake.last_metagraph_refresh_at)
+
+    def test_successful_sync_stamps_timer(self):
+        fake = _fake_validator()
+        with patch(
+            "neurons.validator."
+            "get_available_miners_and_update_metagraph_history",
+            return_value=[3, 4],
+        ):
+            Validator.refresh_metagraph(fake, 15)
+        self.assertEqual(fake.miner_uids, [3, 4])
+        self.assertIsNotNone(fake.last_metagraph_refresh_at)
+
+
+if __name__ == "__main__":
+    unittest.main()