diff --git a/lib/features/relays/relay_health_monitor.dart b/lib/features/relays/relay_health_monitor.dart index 5330e883..7fbe47b1 100644 --- a/lib/features/relays/relay_health_monitor.dart +++ b/lib/features/relays/relay_health_monitor.dart @@ -23,17 +23,44 @@ class RelayHealthMonitor { Timer? _timer; bool _recovering = false; + /// Exponential backoff between recovery attempts while an outage persists: + /// each attempt is a full CLOSE+REQ fan-out, and re-running it on every + /// 6-second tick against relays that stay down is a resubscription storm. + static const Duration initialBackoff = Config.relayDiscoveryTimeout; + static const Duration maxBackoff = Duration(minutes: 5); + Duration _backoff = initialBackoff; + Duration? _nextAttemptAfter; + + /// Monotonic time source for the backoff deadline. The wall clock is not + /// usable here: a backward correction (manual change or an NTP sync during + /// the outage) would park the deadline in the future and suppress recovery + /// for far longer than [maxBackoff]. + final Stopwatch _elapsed = Stopwatch()..start(); + RelayHealthMonitor(this.ref) { _timer = Timer.periodic(Config.relayDiscoveryTimeout, (_) => _check()); ref.onDispose(() => _timer?.cancel()); } /// Runs a single health check synchronously. Exposed for tests so the - /// periodic timer does not need to be awaited. + /// periodic timer does not need to be awaited; [elapsed] injects the + /// monotonic clock reading. @visibleForTesting - Future checkNow() => _check(); + Future checkNow({Duration? elapsed}) => _check(elapsed: elapsed); + + /// Re-arms the backoff so the next check recovers immediately. + /// + /// A healthy tick is the only other reset, and it cannot fire while the + /// outage lasts. After a long stretch in the background with no network the + /// backoff sits at [maxBackoff], so without this a foreground return with + /// working network could wait up to five minutes for the safety net to try + /// again. Called from the foreground transition. + void resetBackoff() { + _backoff = initialBackoff; + _nextAttemptAfter = null; + } - Future _check() async { + Future _check({Duration? elapsed}) async { if (_recovering) return; final nostrService = ref.read(nostrServiceProvider); @@ -46,7 +73,20 @@ class RelayHealthMonitor { final operatingRelays = ref.read(settingsProvider).relays.toSet(); final hasLiveOperatingRelay = nostrService.connectedRelays.any(operatingRelays.contains); - if (hasLiveOperatingRelay) return; + if (hasLiveOperatingRelay) { + // Healthy: arm the next outage for an immediate first attempt. Note this + // exit is unreachable while `settings.relays` is empty (cold start before + // kind-10002 discovery): there is no operating relay to be alive, so the + // backoff only grows. That is why [resetBackoff] exists. + resetBackoff(); + return; + } + + final tick = elapsed ?? _elapsed.elapsed; + if (_nextAttemptAfter != null && tick < _nextAttemptAfter!) return; + _nextAttemptAfter = tick + _backoff; + final doubled = _backoff * 2; + _backoff = doubled > maxBackoff ? maxBackoff : doubled; _recovering = true; try { diff --git a/lib/services/lifecycle_manager.dart b/lib/services/lifecycle_manager.dart index 5a771b98..77daef02 100644 --- a/lib/services/lifecycle_manager.dart +++ b/lib/services/lifecycle_manager.dart @@ -9,6 +9,7 @@ import 'package:mostro_mobile/data/models/enums/storage_keys.dart'; import 'package:mostro_mobile/services/logger_service.dart'; import 'package:mostro_mobile/features/chat/providers/chat_room_providers.dart'; import 'package:mostro_mobile/features/disputes/notifiers/dispute_chat_notifier.dart'; +import 'package:mostro_mobile/features/relays/relay_health_monitor.dart'; import 'package:mostro_mobile/features/subscriptions/subscription_type.dart'; import 'package:mostro_mobile/shared/providers/background_service_provider.dart'; import 'package:mostro_mobile/shared/providers/mostro_service_provider.dart'; @@ -102,6 +103,11 @@ class LifecycleManager extends WidgetsBindingObserver { // back after _switchToBackground() tore it down. subscriptionManager.resume(); + // A long background stretch without network leaves the relay health + // monitor's backoff at its cap, so its safety net would be up to five + // minutes away right when the app is coming back. + ref.read(relayHealthMonitorProvider).resetBackoff(); + // Reinitialize the mostro service logger.i("Reinitializing MostroService"); ref.read(mostroServiceProvider).init(); diff --git a/test/features/relays/relay_health_monitor_test.dart b/test/features/relays/relay_health_monitor_test.dart index d7f9924f..92b09664 100644 --- a/test/features/relays/relay_health_monitor_test.dart +++ b/test/features/relays/relay_health_monitor_test.dart @@ -65,6 +65,142 @@ void main() { verify(subscriptionManager.subscribeToMostroRelayList('test')).called(1); }); + test('holds an exponential backoff while the outage persists', () async { + when(nostrService.connectedRelays).thenReturn({}); + final monitor = buildContainer(relays: ['wss://discovered.example.com']) + .read(relayHealthMonitorProvider); + const t0 = Duration.zero; + + await monitor.checkNow(elapsed: t0); + verify(subscriptionManager.subscribeAll()).called(1); + clearInteractions(subscriptionManager); + clearInteractions(nostrService); + when(nostrService.isInitialized).thenReturn(true); + when(nostrService.connectedRelays).thenReturn({}); + when(nostrService.ensureBootstrapConnectivity()) + .thenAnswer((_) async {}); + + // Next periodic tick, still down: within the backoff window, no re-run. + await monitor.checkNow(elapsed: t0 + const Duration(seconds: 1)); + verifyNever(subscriptionManager.subscribeAll()); + + // After the first backoff interval elapses, it retries once... + await monitor.checkNow( + elapsed: t0 + + RelayHealthMonitor.initialBackoff + + const Duration(seconds: 1)); + verify(subscriptionManager.subscribeAll()).called(1); + clearInteractions(subscriptionManager); + clearInteractions(nostrService); + when(nostrService.isInitialized).thenReturn(true); + when(nostrService.connectedRelays).thenReturn({}); + when(nostrService.ensureBootstrapConnectivity()) + .thenAnswer((_) async {}); + + // ...and the second window is wider than the first. + await monitor.checkNow( + elapsed: t0 + + RelayHealthMonitor.initialBackoff * 2 + + const Duration(seconds: 2)); + verifyNever(subscriptionManager.subscribeAll()); + }); + + test('a healthy check resets the backoff', () async { + when(nostrService.connectedRelays).thenReturn({}); + final monitor = buildContainer(relays: ['wss://discovered.example.com']) + .read(relayHealthMonitorProvider); + const t0 = Duration.zero; + await monitor.checkNow(elapsed: t0); + clearInteractions(subscriptionManager); + clearInteractions(nostrService); + when(nostrService.isInitialized).thenReturn(true); + when(nostrService.ensureBootstrapConnectivity()) + .thenAnswer((_) async {}); + + // Relay comes back: healthy tick resets the backoff... + when(nostrService.connectedRelays) + .thenReturn({'wss://discovered.example.com'}); + await monitor.checkNow(elapsed: t0 + const Duration(seconds: 1)); + + // ...so a new outage right after recovers immediately. + when(nostrService.connectedRelays).thenReturn({}); + await monitor.checkNow(elapsed: t0 + const Duration(seconds: 2)); + verify(subscriptionManager.subscribeAll()).called(1); + }); + + test('drives the backoff from its own monotonic clock, not the wall clock', + () async { + // Without an injected reading the monitor must use its internal + // Stopwatch: two back-to-back checks are one backoff window apart in + // monotonic terms, so the second is skipped regardless of what the + // device wall clock does in between (a backward correction during an + // outage must not park the deadline in the future). + when(nostrService.connectedRelays).thenReturn({}); + final monitor = buildContainer(relays: ['wss://discovered.example.com']) + .read(relayHealthMonitorProvider); + + await monitor.checkNow(); + verify(subscriptionManager.subscribeAll()).called(1); + clearInteractions(subscriptionManager); + + await monitor.checkNow(); + verifyNever(subscriptionManager.subscribeAll()); + }); + + test('caps the backoff at maxBackoff', () async { + when(nostrService.connectedRelays).thenReturn({}); + final monitor = buildContainer(relays: ['wss://discovered.example.com']) + .read(relayHealthMonitorProvider); + + // Ten attempts, each far past its window, so the backoff saturates. + var t = Duration.zero; + var lastAttempt = t; + for (var i = 0; i < 10; i++) { + await monitor.checkNow(elapsed: t); + lastAttempt = t; + t += const Duration(hours: 1); + } + clearInteractions(subscriptionManager); + + // Uncapped, doubling would put the next window ~1.7h out. It must be + // exactly maxBackoff: just under is skipped... + await monitor.checkNow( + elapsed: lastAttempt + + RelayHealthMonitor.maxBackoff - + const Duration(seconds: 1)); + verifyNever(subscriptionManager.subscribeAll()); + + // ...and just over retries. + await monitor.checkNow( + elapsed: lastAttempt + + RelayHealthMonitor.maxBackoff + + const Duration(seconds: 1)); + verify(subscriptionManager.subscribeAll()).called(1); + }); + + test('resetBackoff re-arms an immediate attempt', () async { + // The healthy tick is the only other reset, and it is unreachable while + // the outage lasts: after a long background stretch with no network the + // backoff sits at the cap, so a foreground return would otherwise wait + // up to five minutes for the safety net to try again. + when(nostrService.connectedRelays).thenReturn({}); + final monitor = buildContainer(relays: ['wss://discovered.example.com']) + .read(relayHealthMonitorProvider); + const t0 = Duration.zero; + + await monitor.checkNow(elapsed: t0); + clearInteractions(subscriptionManager); + + // Inside the backoff window: nothing happens... + await monitor.checkNow(elapsed: t0 + const Duration(seconds: 1)); + verifyNever(subscriptionManager.subscribeAll()); + + // ...until the reset, which recovers on the very next check. + monitor.resetBackoff(); + await monitor.checkNow(elapsed: t0 + const Duration(seconds: 2)); + verify(subscriptionManager.subscribeAll()).called(1); + }); + test('stays idle while an operating relay is alive', () async { when(nostrService.connectedRelays) .thenReturn({'wss://discovered.example.com'});