feat(backend): harden reliability — timeouts, circuit-breaker, retry,…

backend/.env.example

@@ -157,10 +157,37 @@ OG_RENDER_TIMEOUT_MS=5000
 # X (Twitter) API — credit score signals
 X_API_BEARER_TOKEN=
 X_API_BASE_URL=https://api.twitter.com/2
+# X API timeout (ms) — explicit per-request timeout for X API calls (issue #090)
+X_API_TIMEOUT_MS=10000
 
 # IPFS (profile images)
 IPFS_API_URL=
 IPFS_GATEWAY_URL=https://ipfs.io/ipfs/
+# IPFS pinning timeout (ms) — explicit timeout for IPFS HTTP pin (issue #090)
+IPFS_TIMEOUT_MS=15000
+
+# ── Timeouts & resilience (issues #090, #091, #092, #077) ──────────────────
+# Outbound HTTP timeouts — explicit per-dependency, configurable via env (issue #090)
+SOROBAN_RPC_TIMEOUT_MS=10000
+HORIZON_TIMEOUT_MS=8000
+REQUEST_TIMEOUT_MS=30000
+# Circuit breaker — threshold / reset timeout per dependency (issue #091)
+CIRCUIT_BREAKER_THRESHOLD=5
+CIRCUIT_BREAKER_RESET_TIMEOUT_MS=30000
+RPC_CIRCUIT_BREAKER_THRESHOLD=5
+RPC_CIRCUIT_BREAKER_RESET_TIMEOUT_MS=30000
+HORIZON_CIRCUIT_BREAKER_THRESHOLD=5
+HORIZON_CIRCUIT_BREAKER_RESET_TIMEOUT_MS=30000
+# Retry with jitter — exponential backoff (issue #092)
+RETRY_MAX_ATTEMPTS=3
+RETRY_INITIAL_DELAY_MS=100
+RETRY_MAX_DELAY_MS=5000
+RETRY_FACTOR=2
+# Payload limits — right-sized per-route (issue #077)
+JSON_BODY_LIMIT=100kb
+MULTER_FILE_SIZE_LIMIT=5242880
+MULTER_FILES_LIMIT=1
+MULTER_FIELDS_LIMIT=10
 
 # Email / notifications (optional providers)
 SMTP_URL=

backend/src/app.ts

@@ -11,7 +11,9 @@ import { metricsController, metricsMiddleware } from './common/observability/met
 import { getSentryRequestHandler, getSentryErrorHandler } from './common/observability/sentry.js';
 import { logger } from './common/utils/logger.js';
 import { requestId } from './common/middleware/requestId.js';
+import { requestTimeoutAndSignal } from './common/middleware/requestTimeout.js';
 import { healthRouter } from './modules/health/health.routes.js';
+import { config } from './config/index.js';
 
 /**
  * HSTS max-age: 1 year (31536000 seconds).
@@ -164,9 +166,25 @@ export function createApp(): Express {
   );
   app.use(globalRateLimiter);
   app.use(requestId);
+  // Server-level timeout + client-disconnect AbortSignal (issue #090)
+  app.use(requestTimeoutAndSignal);
   app.use(metricsMiddleware);
-  // Must parse both JSON and CSP report bodies (browsers send application/csp-report)
-  app.use(express.json({ limit: '1mb', type: ['application/json', 'application/csp-report'] }));
+  /**
+   * Right-sized JSON limits (issue #077).
+   * Default is tight (100kb, configurable via JSON_BODY_LIMIT) — far below the old 1mb blanket.
+   * Routes that legitimately need more (tip/profile writes) get a larger explicit limit via
+   * adaptiveJsonLimit below. Oversized bodies are mapped to 413 PAYLOAD_TOO_LARGE in errorHandler.
+   */
+  const defaultJsonLimit = (config as unknown as { payload?: { jsonLimit: string } })?.payload?.jsonLimit ?? '100kb'; // e.g. '100kb'
+  const largeJsonLimit = '500kb';
+  // Paths that need larger JSON bodies (documented per-route override)
+  const largeJsonPrefixes = [`${(config as unknown as { server?: { apiBasePath: string } })?.server?.apiBasePath ?? '/api/v1'}/tips`, `${(config as unknown as { server?: { apiBasePath: string } })?.server?.apiBasePath ?? '/api/v1'}/profiles`, `${(config as unknown as { server?: { apiBasePath: string } })?.server?.apiBasePath ?? '/api/v1'}/auth`];
+  const adaptiveJsonLimit = (req: express.Request, _res: express.Response, next: express.NextFunction) => {
+    const needsLarge = largeJsonPrefixes.some((prefix) => req.path.startsWith(prefix) || req.originalUrl.startsWith(prefix));
+    const limit = needsLarge ? largeJsonLimit : defaultJsonLimit;
+    return express.json({ limit, type: ['application/json', 'application/csp-report'] })(req, _res, next);
+  };
+  app.use(adaptiveJsonLimit);
   app.use(pinoHttp({ logger }));
 
   app.get('/metrics', metricsController);

backend/src/common/errors/AppError.ts

@@ -59,3 +59,15 @@ export class TooManyRequestsError extends AppError {
   }
 }
 
+export class PayloadTooLargeError extends AppError {
+  constructor(message = 'Payload too large', details?: unknown) {
+    super(413, message, 'PAYLOAD_TOO_LARGE', details);
+  }
+}
+
+export class RequestTimeoutError extends AppError {
+  constructor(message = 'Request timeout', details?: unknown) {
+    super(503, message, 'REQUEST_TIMEOUT', details);
+  }
+}
+

backend/src/common/middleware/errorHandler.ts

@@ -23,6 +23,48 @@ export function errorHandler(
   _next: NextFunction,
 ): void {
   const requestId = requestIdOf(req);
+
+  // Multer errors (file size/count) — return 413 in app error format, not HTML
+  const multerErr = err as { code?: string; status?: number; type?: string; limit?: string };
+  if (multerErr?.code === 'LIMIT_FILE_SIZE' || multerErr?.code === 'LIMIT_FILE_COUNT' || multerErr?.code === 'LIMIT_FIELD_COUNT' || multerErr?.code === 'LIMIT_UNEXPECTED_FILE') {
+    res.status(413).json({
+      error: { code: 'PAYLOAD_TOO_LARGE', message: 'Payload too large', details: { limit: multerErr.code }, requestId },
+    });
+    return;
+  }
+
+  // Express json entity.too.large (body > limit) — also 413 with JSON envelope
+  if (
+    (err as { type?: string; status?: number })?.type === 'entity.too.large' ||
+    (err as { status?: number })?.status === 413 ||
+    (err as { statusCode?: number })?.statusCode === 413
+  ) {
+    res.status(413).json({
+      error: { code: 'PAYLOAD_TOO_LARGE', message: 'Payload too large', requestId },
+    });
+    return;
+  }
+
+  // TimeoutError from AbortSignal.timeout (upstream) mapped to 503/504? unify to 503 REQUEST_TIMEOUT
+  if (err instanceof DOMException && err.name === 'TimeoutError') {
+    res.status(503).json({
+      error: { code: 'REQUEST_TIMEOUT', message: err.message || 'Upstream request timed out', requestId },
+    });
+    return;
+  }
+  if (err instanceof DOMException && err.name === 'AbortError') {
+    // Client disconnect abort — if headers already sent, ignore; otherwise map to 499 or 503
+    if (res.headersSent) return;
+    // If request was aborted by server timeout, upstream already responded 503; avoid double response
+    if ((req as unknown as { signal?: AbortSignal })?.signal?.aborted) {
+      // If client disconnected, log and return 503 with cancellation code
+      res.status(503).json({
+        error: { code: 'REQUEST_CANCELLED', message: 'Request cancelled', requestId },
+      });
+      return;
+    }
+  }
+
   if (err instanceof ZodError) {
     res.status(400).json({
       error: {

backend/src/common/middleware/requestTimeout.ts

@@ -0,0 +1,63 @@
+import type { NextFunction, Request, Response } from "express";
+import { config } from "../../config/index.js";
+import { logger } from "../utils/logger.js";
+
+/**
+ * Request timeout + client-disconnect AbortSignal middleware (issue #090).
+ *
+ * - Creates a per-request AbortController and exposes its signal as `req.signal`.
+ *   Upstream fetch/RPC calls should pass this signal via fetchWithTimeout({ parentSignal: req.signal })
+ *   or withTimeoutAndSignal(..., req.signal). When the client disconnects, the signal aborts,
+ *   cancelling in-flight work.
+ * - Enforces a server-level request timeout (config.timeouts.requestMs, default 30s).
+ *   If the timeout fires before the response is sent, it responds 503 with code REQUEST_TIMEOUT
+ *   and aborts the controller so upstream work is cancelled.
+ */
+
+export function requestTimeoutAndSignal(req: Request, res: Response, next: NextFunction): void {
+  const timeoutMs = (config as unknown as { timeouts?: { requestMs: number } })?.timeouts?.requestMs ?? 30_000;
+  const controller = new AbortController();
+  // Expose signal on request for downstream handlers
+  (req as unknown as { signal: AbortSignal }).signal = controller.signal;
+
+  let timedOut = false;
+  const timeout = setTimeout(() => {
+    if (res.headersSent || res.writableEnded) return;
+    timedOut = true;
+    logger.warn({ path: req.path, method: req.method, timeoutMs }, "Request timed out — returning 503");
+    controller.abort(new DOMException("Request timeout", "TimeoutError"));
+    if (!res.headersSent) {
+      res.status(503).json({
+        error: {
+          code: "REQUEST_TIMEOUT",
+          message: `Request timed out after ${timeoutMs}ms`,
+          requestId: (req as unknown as { id?: string }).id,
+        },
+      });
+    }
+  }, timeoutMs);
+
+  // Client disconnect -> abort upstream
+  const onClose = () => {
+    if (!res.writableEnded && !timedOut) {
+      logger.debug({ path: req.path, method: req.method }, "Client disconnected — aborting upstream");
+      controller.abort(new DOMException("Client disconnected", "AbortError"));
+    }
+    cleanup();
+  };
+
+  const cleanup = () => {
+    clearTimeout(timeout);
+    req.removeListener("close", onClose);
+    res.removeListener("finish", cleanup);
+    res.removeListener("close", cleanup);
+  };
+
+  req.on("close", onClose);
+  res.on("finish", cleanup);
+  res.on("close", cleanup);
+
+  // If response already wants to abort upstream on its own close, ensure controller abort doesn't double-send
+  // Pass through
+  next();
+}

backend/src/common/observability/metrics.ts

@@ -41,13 +41,22 @@ export interface MetricsData {
   retention: {
     rows_pruned_total: Record<string, number>;
   };
+  circuitBreaker?: Record<string, { state: string; failures: number; opens: number }>;
+  timeouts?: {
+    request_timeout_ms: number;
+    rpc_timeout_ms: number;
+    horizon_timeout_ms: number;
+    ipfs_timeout_ms: number;
+    x_api_timeout_ms: number;
+  };
 }
 
 let requestCount = 0;
 let errorCount = 0;
 let latencySum = 0;
 let latencyCount = 0;
 let slowQueryCount = 0;
+let poolSaturationCount = 0;
 const retentionPrunedCounts: Record<string, number> = {};
 
 export function recordRequest(duration: number) {
@@ -65,6 +74,10 @@ export function recordSlowQuery() {
   slowQueryCount++;
 }
 
+export function recordPoolSaturation(): void {
+  poolSaturationCount++;
+}
+
 /** Records rows removed by one completed retention batch. */
 export function recordRetentionPruned(model: string, count: number): void {
   retentionPrunedCounts[model] = (retentionPrunedCounts[model] ?? 0) + count;
@@ -88,6 +101,15 @@ export async function getMetrics(): Promise<MetricsData> {
     }
   }
 
+  // Circuit breaker states (issue #091) — lazy import to avoid cycle
+  let circuitBreaker: Record<string, { state: string; failures: number; opens: number }> | undefined;
+  try {
+    const { getCircuitBreakerMetrics } = await import('../utils/circuitBreaker.js');
+    circuitBreaker = getCircuitBreakerMetrics();
+  } catch {
+    circuitBreaker = undefined;
+  }
+
   return {
     timestamp: new Date().toISOString(),
     service: 'stellar-tipz-backend',
@@ -123,6 +145,14 @@ export async function getMetrics(): Promise<MetricsData> {
     retention: {
       rows_pruned_total: { ...retentionPrunedCounts },
     },
+    circuitBreaker,
+    timeouts: {
+      request_timeout_ms: env.REQUEST_TIMEOUT_MS,
+      rpc_timeout_ms: env.SOROBAN_RPC_TIMEOUT_MS,
+      horizon_timeout_ms: env.HORIZON_TIMEOUT_MS,
+      ipfs_timeout_ms: env.IPFS_TIMEOUT_MS,
+      x_api_timeout_ms: env.X_API_TIMEOUT_MS,
+    },
   };
 }
 

backend/src/common/stellar/rpcClient.ts

@@ -0,0 +1,81 @@
+import { SorobanRpc } from "@stellar/stellar-sdk";
+import { config } from "../../config/index.js";
+import { CircuitBreaker } from "../utils/circuitBreaker.js";
+import { withTimeoutAndSignal } from "../utils/fetchWithTimeout.js";
+import { logger } from "../utils/logger.js";
+
+/**
+ * Centralized Soroban RPC and Horizon resilience layer (issues #090, #091).
+ * - Timeouts via withTimeoutAndSignal (configurable SOROBAN_RPC_TIMEOUT_MS / HORIZON_TIMEOUT_MS)
+ * - Circuit breaker fast-fails when upstream degrades
+ * - Client disconnect cancellation via parentSignal (req.signal)
+ */
+
+// Shared breaker instances — state exposed via metrics endpoint (issue #091)
+// Use optional chaining with defaults so mocked config in tests (e.g. securityHeaders.test) doesn't crash at import time
+export const rpcCircuitBreaker = new CircuitBreaker(
+  (config as unknown as { circuitBreaker?: { rpcThreshold: number } })?.circuitBreaker?.rpcThreshold ?? 5,
+  (config as unknown as { circuitBreaker?: { rpcResetTimeoutMs: number } })?.circuitBreaker?.rpcResetTimeoutMs ?? 30_000,
+  "Soroban RPC",
+);
+
+export const horizonCircuitBreaker = new CircuitBreaker(
+  (config as unknown as { circuitBreaker?: { horizonThreshold: number } })?.circuitBreaker?.horizonThreshold ?? 5,
+  (config as unknown as { circuitBreaker?: { horizonResetTimeoutMs: number } })?.circuitBreaker?.horizonResetTimeoutMs ?? 30_000,
+  "Horizon",
+);
+
+export function getRpcServer(): SorobanRpc.Server {
+  const rpcUrl = (config as unknown as { stellar?: { rpcUrl: string } })?.stellar?.rpcUrl ?? "https://soroban-testnet.stellar.org";
+  return new SorobanRpc.Server(rpcUrl, {
+    allowHttp: rpcUrl.startsWith("http://"),
+  });
+}
+
+/**
+ * Wraps an RPC operation with circuit breaker and timeout.
+ * Use this for every SorobanRpc.Server call (getAccount, simulateTransaction, sendTransaction, getHealth).
+ */
+export async function rpcCall<T>(
+  operation: (server: SorobanRpc.Server) => Promise<T>,
+  opts: { signal?: AbortSignal; timeoutMs?: number; operationName?: string } = {},
+): Promise<T> {
+  const timeoutMs = opts.timeoutMs ?? (config as unknown as { timeouts?: { sorobanRpcMs: number } })?.timeouts?.sorobanRpcMs ?? 10_000;
+  const name = opts.operationName ?? "RPC call";
+
+  return rpcCircuitBreaker.call(async () => {
+    const server = getRpcServer();
+    const promise = operation(server);
+    try {
+      return await withTimeoutAndSignal(promise, timeoutMs, opts.signal, name);
+    } catch (err) {
+      if (err instanceof DOMException && err.name === "TimeoutError") {
+        logger.warn({ operation: name, timeoutMs }, "Soroban RPC timeout");
+      }
+      throw err;
+    }
+  });
+}
+
+/**
+ * Horizon fetch wrapper with same resilience properties.
+ */
+export async function horizonFetch(
+  path: string,
+  options: RequestInit & { signal?: AbortSignal; timeoutMs?: number } = {},
+): Promise<Response> {
+  const timeoutMs = (options as unknown as { timeoutMs?: number }).timeoutMs ?? (config as unknown as { timeouts?: { horizonMs: number } })?.timeouts?.horizonMs ?? 8_000;
+  const signal = (options as unknown as { signal?: AbortSignal }).signal;
+  const horizonUrl = (config as unknown as { stellar?: { horizonUrl: string } })?.stellar?.horizonUrl ?? "https://horizon-testnet.stellar.org";
+  const url = `${horizonUrl.replace(/\/+$/, "")}${path}`;
+
+  return horizonCircuitBreaker.call(async () => {
+    // Import here to avoid cycle
+    const { fetchWithTimeout } = await import("../utils/fetchWithTimeout.js");
+    return fetchWithTimeout(url, {
+      ...options,
+      timeoutMs,
+      parentSignal: signal,
+    });
+  });
+}

backend/src/common/types/express.d.ts

@@ -4,6 +4,8 @@ declare global {
   namespace Express {
     interface Request {
       user?: AuthUser;
+      id?: string;
+      signal?: AbortSignal;
     }
   }
 }