Survive transient relay crashes + add restart policy

apps/server/src/common/process-guards.spec.ts

@@ -0,0 +1,51 @@
+import { describe, it, expect, vi } from 'vitest';
+import { isTransientRelayError, createFatalHandler } from './process-guards.js';
+
+describe('process-guards', () => {
+  describe('isTransientRelayError', () => {
+    it('matches nostr-tools SendingOnClosedConnection by name', () => {
+      const err = new Error('Tried to send on a closed connection to wss://relay.example/');
+      err.name = 'SendingOnClosedConnection';
+      expect(isTransientRelayError(err)).toBe(true);
+    });
+
+    it('rejects unrelated errors and non-Error values', () => {
+      expect(isTransientRelayError(new Error('real bug'))).toBe(false);
+      expect(isTransientRelayError('SendingOnClosedConnection')).toBe(false);
+      expect(isTransientRelayError(null)).toBe(false);
+      expect(isTransientRelayError(undefined)).toBe(false);
+      expect(isTransientRelayError({ name: 'SendingOnClosedConnection' })).toBe(false);
+    });
+  });
+
+  describe('createFatalHandler', () => {
+    function setup() {
+      const logger = { warn: vi.fn(), error: vi.fn() };
+      const exit = vi.fn();
+      return { logger, exit, handle: createFatalHandler(logger, exit) };
+    }
+
+    it('swallows transient relay errors without exiting', () => {
+      const { logger, exit, handle } = setup();
+      const err = new Error('closed');
+      err.name = 'SendingOnClosedConnection';
+      handle('exception', err);
+      expect(exit).not.toHaveBeenCalled();
+      expect(logger.warn).toHaveBeenCalledOnce();
+      expect(logger.error).not.toHaveBeenCalled();
+    });
+
+    it('exits(1) on any other uncaught error so the restart policy takes over', () => {
+      const { logger, exit, handle } = setup();
+      handle('exception', new Error('genuine crash'));
+      expect(exit).toHaveBeenCalledWith(1);
+      expect(logger.error).toHaveBeenCalledOnce();
+    });
+
+    it('exits(1) on a non-transient unhandled rejection', () => {
+      const { exit, handle } = setup();
+      handle('rejection', 'some string reason');
+      expect(exit).toHaveBeenCalledWith(1);
+    });
+  });
+});

apps/server/src/common/process-guards.ts

@@ -0,0 +1,46 @@
+import { Logger } from '@nestjs/common';
+
+/**
+ * nostr-tools' relay pool throws `SendingOnClosedConnection` asynchronously from a WebSocket
+ * 'open' handler when a subscription re-fires on a socket that closed mid-reconnect. The throw
+ * escapes our `try/catch` (it happens in the ws event loop, not our call stack) and surfaces as an
+ * `uncaughtException`. In production a relay reconnect storm triggered this and killed the whole
+ * bunker process, which — with no restart policy — then stayed down for days. These errors are
+ * transient and safe to ignore: the connection watchdog re-subscribes the dropped relay shortly
+ * after.
+ */
+export function isTransientRelayError(err: unknown): boolean {
+  return err instanceof Error && err.name === 'SendingOnClosedConnection';
+}
+
+type ExitFn = (code: number) => void;
+type GuardLogger = Pick<Logger, 'warn' | 'error'>;
+
+/**
+ * Builds the handler shared by `uncaughtException` and `unhandledRejection`: transient relay errors
+ * are logged and swallowed so a flaky relay can't take the process down; anything else is logged
+ * and the process exits, so the container restart policy gives a clean restart rather than letting
+ * the process limp on in an undefined state.
+ */
+export function createFatalHandler(logger: GuardLogger, exit: ExitFn) {
+  return (label: string, err: unknown): void => {
+    if (isTransientRelayError(err)) {
+      logger.warn(`Ignoring transient relay ${label}: ${(err as Error).message}`);
+      return;
+    }
+    logger.error(
+      `Fatal ${label} — exiting for a clean restart`,
+      err instanceof Error ? err.stack : String(err),
+    );
+    exit(1);
+  };
+}
+
+export function installProcessGuards(
+  logger: GuardLogger = new Logger('ProcessGuards'),
+  exit: ExitFn = (code) => process.exit(code),
+): void {
+  const handle = createFatalHandler(logger, exit);
+  process.on('uncaughtException', (err) => handle('exception', err));
+  process.on('unhandledRejection', (reason) => handle('rejection', reason));
+}

apps/server/src/main.ts

@@ -18,8 +18,13 @@ import fastifyHelmet from '@fastify/helmet';
 import fastifyCookie from '@fastify/cookie';
 import { AppModule } from './app.module.js';
 import { serverEnvSchema } from '@bunker46/config';
+import { installProcessGuards } from './common/process-guards.js';
 
 async function bootstrap() {
+  // Survive transient relay errors thrown asynchronously by nostr-tools (otherwise a flaky relay
+  // can crash the whole bunker); exit on any genuine fault so the container restarts cleanly.
+  installProcessGuards();
+
   const env = serverEnvSchema.parse(process.env);
 
   const app = await NestFactory.create<NestFastifyApplication>(

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   db:
     image: postgres:17-alpine
+    restart: unless-stopped
     environment:
       POSTGRES_USER: bunker46
       POSTGRES_PASSWORD: bunker46
@@ -17,6 +18,7 @@ services:
 
   redis:
     image: redis:8-alpine
+    restart: unless-stopped
     # Not published to the host: only reachable over the internal Docker network (redis:6379).
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
@@ -25,6 +27,7 @@ services:
       retries: 5
 
   server:
+    restart: unless-stopped
     build:
       context: .
       dockerfile: apps/server/Dockerfile
@@ -66,6 +69,7 @@ services:
         condition: service_healthy
 
   web:
+    restart: unless-stopped
     build:
       context: .
       dockerfile: apps/web/Dockerfile