From 683d33ca5d2d32da8f04b7ccaa6f604b0a81cc84 Mon Sep 17 00:00:00 2001 From: Djuri Baars Date: Tue, 30 Jun 2026 20:57:44 +0200 Subject: [PATCH] fix(server): survive transient relay crashes + add restart policy A relay reconnect storm made nostr-tools throw SendingOnClosedConnection asynchronously from a WebSocket 'open' handler (it re-fires a subscription on a socket that closed mid-reconnect). The throw escapes our try/catch and surfaced as an uncaughtException that killed the bunker process; with no restart policy the container then stayed down for days. - Add process guards (uncaughtException/unhandledRejection): swallow the transient relay error (the watchdog re-subscribes shortly after) and exit on any genuine fault so the container restarts cleanly. - Add `restart: unless-stopped` to all docker-compose services so a crash or host reboot auto-recovers instead of leaving the bunker offline. --- apps/server/src/common/process-guards.spec.ts | 51 +++++++++++++++++++ apps/server/src/common/process-guards.ts | 46 +++++++++++++++++ apps/server/src/main.ts | 5 ++ docker-compose.yml | 4 ++ 4 files changed, 106 insertions(+) create mode 100644 apps/server/src/common/process-guards.spec.ts create mode 100644 apps/server/src/common/process-guards.ts diff --git a/apps/server/src/common/process-guards.spec.ts b/apps/server/src/common/process-guards.spec.ts new file mode 100644 index 0000000..bce4653 --- /dev/null +++ b/apps/server/src/common/process-guards.spec.ts @@ -0,0 +1,51 @@ +import { describe, it, expect, vi } from 'vitest'; +import { isTransientRelayError, createFatalHandler } from './process-guards.js'; + +describe('process-guards', () => { + describe('isTransientRelayError', () => { + it('matches nostr-tools SendingOnClosedConnection by name', () => { + const err = new Error('Tried to send on a closed connection to wss://relay.example/'); + err.name = 'SendingOnClosedConnection'; + expect(isTransientRelayError(err)).toBe(true); + }); + + it('rejects unrelated errors and non-Error values', () => { + expect(isTransientRelayError(new Error('real bug'))).toBe(false); + expect(isTransientRelayError('SendingOnClosedConnection')).toBe(false); + expect(isTransientRelayError(null)).toBe(false); + expect(isTransientRelayError(undefined)).toBe(false); + expect(isTransientRelayError({ name: 'SendingOnClosedConnection' })).toBe(false); + }); + }); + + describe('createFatalHandler', () => { + function setup() { + const logger = { warn: vi.fn(), error: vi.fn() }; + const exit = vi.fn(); + return { logger, exit, handle: createFatalHandler(logger, exit) }; + } + + it('swallows transient relay errors without exiting', () => { + const { logger, exit, handle } = setup(); + const err = new Error('closed'); + err.name = 'SendingOnClosedConnection'; + handle('exception', err); + expect(exit).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledOnce(); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('exits(1) on any other uncaught error so the restart policy takes over', () => { + const { logger, exit, handle } = setup(); + handle('exception', new Error('genuine crash')); + expect(exit).toHaveBeenCalledWith(1); + expect(logger.error).toHaveBeenCalledOnce(); + }); + + it('exits(1) on a non-transient unhandled rejection', () => { + const { exit, handle } = setup(); + handle('rejection', 'some string reason'); + expect(exit).toHaveBeenCalledWith(1); + }); + }); +}); diff --git a/apps/server/src/common/process-guards.ts b/apps/server/src/common/process-guards.ts new file mode 100644 index 0000000..9f3b0f3 --- /dev/null +++ b/apps/server/src/common/process-guards.ts @@ -0,0 +1,46 @@ +import { Logger } from '@nestjs/common'; + +/** + * nostr-tools' relay pool throws `SendingOnClosedConnection` asynchronously from a WebSocket + * 'open' handler when a subscription re-fires on a socket that closed mid-reconnect. The throw + * escapes our `try/catch` (it happens in the ws event loop, not our call stack) and surfaces as an + * `uncaughtException`. In production a relay reconnect storm triggered this and killed the whole + * bunker process, which — with no restart policy — then stayed down for days. These errors are + * transient and safe to ignore: the connection watchdog re-subscribes the dropped relay shortly + * after. + */ +export function isTransientRelayError(err: unknown): boolean { + return err instanceof Error && err.name === 'SendingOnClosedConnection'; +} + +type ExitFn = (code: number) => void; +type GuardLogger = Pick; + +/** + * Builds the handler shared by `uncaughtException` and `unhandledRejection`: transient relay errors + * are logged and swallowed so a flaky relay can't take the process down; anything else is logged + * and the process exits, so the container restart policy gives a clean restart rather than letting + * the process limp on in an undefined state. + */ +export function createFatalHandler(logger: GuardLogger, exit: ExitFn) { + return (label: string, err: unknown): void => { + if (isTransientRelayError(err)) { + logger.warn(`Ignoring transient relay ${label}: ${(err as Error).message}`); + return; + } + logger.error( + `Fatal ${label} — exiting for a clean restart`, + err instanceof Error ? err.stack : String(err), + ); + exit(1); + }; +} + +export function installProcessGuards( + logger: GuardLogger = new Logger('ProcessGuards'), + exit: ExitFn = (code) => process.exit(code), +): void { + const handle = createFatalHandler(logger, exit); + process.on('uncaughtException', (err) => handle('exception', err)); + process.on('unhandledRejection', (reason) => handle('rejection', reason)); +} diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 162e138..d329e83 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -18,8 +18,13 @@ import fastifyHelmet from '@fastify/helmet'; import fastifyCookie from '@fastify/cookie'; import { AppModule } from './app.module.js'; import { serverEnvSchema } from '@bunker46/config'; +import { installProcessGuards } from './common/process-guards.js'; async function bootstrap() { + // Survive transient relay errors thrown asynchronously by nostr-tools (otherwise a flaky relay + // can crash the whole bunker); exit on any genuine fault so the container restarts cleanly. + installProcessGuards(); + const env = serverEnvSchema.parse(process.env); const app = await NestFactory.create( diff --git a/docker-compose.yml b/docker-compose.yml index 8fb97f1..06e86d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: db: image: postgres:17-alpine + restart: unless-stopped environment: POSTGRES_USER: bunker46 POSTGRES_PASSWORD: bunker46 @@ -17,6 +18,7 @@ services: redis: image: redis:8-alpine + restart: unless-stopped # Not published to the host: only reachable over the internal Docker network (redis:6379). healthcheck: test: ["CMD", "redis-cli", "ping"] @@ -25,6 +27,7 @@ services: retries: 5 server: + restart: unless-stopped build: context: . dockerfile: apps/server/Dockerfile @@ -66,6 +69,7 @@ services: condition: service_healthy web: + restart: unless-stopped build: context: . dockerfile: apps/web/Dockerfile