Swagger docs (timeout, pagination) + oracle key validation + pool exhaustion monitoring

.env.example

@@ -119,6 +119,12 @@ NODE_ENV="development"
 # degraded. Queries pg_stat_replication / pg_last_wal_receive_lsn.
 # DB_REPLICATION_LAG_WARN_BYTES=52428800  # 50 MB default
 
+# ─── Database connection pool exhaustion (#471) ───────────────────────────────
+# Percentage of DATABASE_CONNECTION_LIMIT's active connections above which the
+# health check flags the database as degraded, so pool exhaustion is caught
+# before queries start queuing or timing out.
+# DB_POOL_EXHAUSTION_WARN_PERCENT=90  # 90% default
+
 # ─── CORS (#382) ──────────────────────────────────────────────────────────────
 # CORS allowed origin — REQUIRED. Set to your frontend URL in production.
 # Accepts a single origin or a comma-separated list of origins.

src/claims/claims.controller.ts

@@ -1,4 +1,5 @@
-import { Body, Controller, ForbiddenException, Get, Param, Post, Query, Req, UseGuards, UseInterceptors, UnauthorizedException, NotFoundException, Throttle } from '@nestjs/common';
+import { Body, Controller, ForbiddenException, Get, Param, Post, Query, Req, UseGuards, UseInterceptors, UnauthorizedException, NotFoundException } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import { StreamingInterceptor } from '../common/interceptors/streaming.interceptor';
 import {
   ApiTags,
@@ -55,8 +56,8 @@ export class ClaimsController {
   @ApiBearerAuth()
   @ApiOperation({ summary: 'Get claim history for a wallet address (query param)' })
   @ApiQuery({ name: 'wallet', required: true, description: 'Stellar wallet address' })
-  @ApiQuery({ name: 'page', required: false, description: 'Page number' })
-  @ApiQuery({ name: 'limit', required: false, description: 'Items per page' })
+  @ApiQuery({ name: 'page', required: false, description: 'Page number (default 1)', example: 1 })
+  @ApiQuery({ name: 'limit', required: false, description: 'Items per page, max 100 (default 20)', example: 20 })
   @ApiQuery({
     name: 'stream',
     required: false,
@@ -128,8 +129,8 @@ export class ClaimsController {
   @ApiBearerAuth()
   @ApiOperation({ summary: 'Get all claims for a wallet address' })
   @ApiParam({ name: 'wallet', description: 'Stellar wallet address' })
-  @ApiQuery({ name: 'page', required: false, description: 'Page number' })
-  @ApiQuery({ name: 'limit', required: false, description: 'Items per page' })
+  @ApiQuery({ name: 'page', required: false, description: 'Page number (default 1)', example: 1 })
+  @ApiQuery({ name: 'limit', required: false, description: 'Items per page, max 100 (default 20)', example: 20 })
   @ApiQuery({
     name: 'stream',
     required: false,

src/common/events/webhooks.service.ts

@@ -1,5 +1,5 @@
 import { Injectable, Logger, BadRequestException } from '@nestjs/common';
-import { PrismaService } from '../prisma/prisma.service';
+import { PrismaService } from '../../prisma/prisma.service';
 
 export interface WebhookRegistration {
   id: string;

src/common/swagger/error-response.dto.ts

@@ -102,7 +102,7 @@ function errorExample(
 
 export const ERROR_EXAMPLES = {
   400: errorExample(ErrorCode.VALIDATION_ERROR, 400, 'wallet must be a string; productId should not be empty'),
-  400_bad: errorExample(ErrorCode.BAD_REQUEST, 400, 'Malformed request parameter'),
+  '400_bad': errorExample(ErrorCode.BAD_REQUEST, 400, 'Malformed request parameter'),
   401: errorExample(ErrorCode.UNAUTHORIZED, 401, 'Missing or invalid JWT — include Authorization: Bearer <token>'),
   403: errorExample(ErrorCode.FORBIDDEN, 403, 'Policy belongs to a different wallet'),
   404: errorExample(ErrorCode.NOT_FOUND, 404, 'Resource not found'),

src/common/webhooks/webhooks.controller.ts

@@ -2,7 +2,7 @@ import { Controller, Post, Body, Get } from '@nestjs/common';
 import { ApiTags, ApiOperation, ApiResponse, ApiBearerAuth, ApiExtraModels, getSchemaPath } from '@nestjs/swagger';
 import { ApiErrorResponse } from '../swagger/api-error-responses';
 import { WebhooksService } from '../events/webhooks.service';
-import { PrismaService } from '../prisma/prisma.service';
+import { PrismaService } from '../../prisma/prisma.service';
 import {
   RegisterWebhookDto,
   WebhookRegistrationResponseDto,

src/common/webhooks/webhooks.module.ts

@@ -1,7 +1,7 @@
 import { Module } from '@nestjs/common';
 import { WebhooksController } from './webhooks.controller';
 import { WebhooksService } from '../events/webhooks.service';
-import { PrismaModule } from '../prisma/prisma.module';
+import { PrismaModule } from '../../prisma/prisma.module';
 
 @Module({
   imports: [PrismaModule],

src/config/env.d.ts

@@ -53,6 +53,9 @@ export interface EnvironmentVariables {
   DATABASE_CONNECTION_LIMIT?: string;
   DATABASE_POOL_TIMEOUT?: string;
   DATABASE_CONNECT_TIMEOUT?: string;
+
+  // #471 — connection pool exhaustion warning threshold (see src/health/health.controller.ts)
+  DB_POOL_EXHAUSTION_WARN_PERCENT?: string;
 }
 
 declare global {

src/health/dto/health-response.dto.ts

@@ -9,6 +9,18 @@ export class DatabasePoolDto {
 
   @ApiProperty({ description: 'Number of waiting connections' })
   waiting: number;
+
+  @ApiProperty({ description: 'Configured maximum pool size (DATABASE_CONNECTION_LIMIT, default 10)', example: 10 })
+  max: number;
+
+  @ApiProperty({ description: 'Active connections as a percentage of the configured max pool size', example: 30 })
+  utilizationPercent: number;
+
+  @ApiProperty({
+    description: 'Whether utilization has met or exceeded the exhaustion warning threshold (DB_POOL_EXHAUSTION_WARN_PERCENT, default 90%)',
+    example: false,
+  })
+  exhausted: boolean;
 }
 
 export class DatabaseThroughputDto {

src/health/health.controller.spec.ts

@@ -4,23 +4,57 @@ import { HealthController } from './health.controller';
 describe('HealthController', () => {
   const KEEPER_ADDRESS = 'GAHJJJKMOKYE4RVPZEWZTKH5FVI4PA3VL7GK2LFNUBSGBKQTRB7KXQZ';
 
-  function build(overrides?: { balance?: string; dbFails?: boolean; rpcFails?: boolean; minBalance?: string }) {
+  function build(overrides?: {
+    balance?: string;
+    dbFails?: boolean;
+    rpcFails?: boolean;
+    minBalance?: string;
+    poolActive?: number;
+    connectionLimit?: string;
+    poolWarnPercent?: string;
+  }) {
     const prisma = {
       $queryRaw: overrides?.dbFails
         ? jest.fn().mockRejectedValue(new Error('connection refused'))
-        : jest.fn().mockResolvedValue([{ '?column?': 1 }]),
+        : jest.fn((strings: TemplateStringsArray) => {
+            // Route pg_stat_activity (connection pool) queries to a shape
+            // with `active` set from the override; every other query (the
+            // initial SELECT 1, throughput, replication) gets the default
+            // single-row placeholder — none of them read `state`/`count`.
+            if (strings.join('').includes('pg_stat_activity')) {
+              return Promise.resolve([
+                { state: 'active', count: overrides?.poolActive ?? 0 },
+                { state: 'idle', count: 0 },
+              ]);
+            }
+            return Promise.resolve([{ '?column?': 1 }]);
+          }),
     };
     const stellar = {
       keeperKeypair: { publicKey: () => KEEPER_ADDRESS },
       getAccountBalance: overrides?.rpcFails
         ? jest.fn().mockRejectedValue(new Error('RPC unreachable'))
         : jest.fn().mockResolvedValue(overrides?.balance ?? '100.0000000'),
+      checkRpcConnectivity: jest.fn().mockResolvedValue({ latencyMs: 10, ledger: 1 }),
     };
     const config = {
-      get: jest.fn((key: string) => (key === 'KEEPER_MIN_BALANCE_XLM' ? overrides?.minBalance : undefined)),
+      get: jest.fn((key: string) => {
+        if (key === 'KEEPER_MIN_BALANCE_XLM') return overrides?.minBalance;
+        if (key === 'DATABASE_CONNECTION_LIMIT') return overrides?.connectionLimit;
+        if (key === 'DB_POOL_EXHAUSTION_WARN_PERCENT') return overrides?.poolWarnPercent;
+        return undefined;
+      }),
+    };
+    const redis = {
+      ping: jest.fn().mockResolvedValue('PONG'),
+      llen: jest.fn().mockResolvedValue(0),
+      // Resolve every requested key with a fresh timestamp so worker
+      // heartbeats read as 'ok' rather than 'stale' by default.
+      mget: jest.fn((...keys: string[]) => Promise.resolve(keys.map(() => new Date().toISOString()))),
+      info: jest.fn().mockResolvedValue(''),
     };
 
-    return new HealthController(prisma as any, stellar as any, config as any);
+    return new HealthController(prisma as any, stellar as any, config as any, redis as any);
   }
 
   it('returns 200/ok when DB and Stellar RPC are both healthy with sufficient keeper balance', async () => {
@@ -96,4 +130,60 @@ describe('HealthController', () => {
       }),
     });
   });
+
+  // #471 — connection pool exhaustion monitoring
+  describe('connection pool exhaustion (#471)', () => {
+    it('reports utilization and exhausted:false when active connections are well below the configured max', async () => {
+      const controller = build({ poolActive: 2, connectionLimit: '10' });
+
+      const body = await controller.check();
+
+      expect(body.status).toBe('ok');
+      expect(body.checks.database.pool).toMatchObject({
+        active: 2,
+        max: 10,
+        utilizationPercent: 20,
+        exhausted: false,
+      });
+    });
+
+    it('reports degraded database status once utilization meets the exhaustion warning threshold', async () => {
+      const controller = build({ poolActive: 9, connectionLimit: '10', poolWarnPercent: '90' });
+
+      await expect(controller.check()).rejects.toThrow(HttpException);
+      await expect(controller.check()).rejects.toMatchObject({
+        response: expect.objectContaining({
+          checks: expect.objectContaining({
+            database: expect.objectContaining({
+              status: 'error',
+              pool: expect.objectContaining({ active: 9, max: 10, utilizationPercent: 90, exhausted: true }),
+              error: expect.stringContaining('exhaustion warning threshold'),
+            }),
+          }),
+        }),
+      });
+    });
+
+    it('stays ok when utilization is just below a custom warning threshold', async () => {
+      const controller = build({ poolActive: 7, connectionLimit: '10', poolWarnPercent: '80' });
+
+      const body = await controller.check();
+
+      expect(body.checks.database.pool).toMatchObject({ utilizationPercent: 70, exhausted: false });
+    });
+
+    it('uses the default 10-connection max and 90% threshold when not configured', async () => {
+      const controller = build({ poolActive: 9 });
+
+      await expect(controller.check()).rejects.toMatchObject({
+        response: expect.objectContaining({
+          checks: expect.objectContaining({
+            database: expect.objectContaining({
+              pool: expect.objectContaining({ max: 10, utilizationPercent: 90, exhausted: true }),
+            }),
+          }),
+        }),
+      });
+    });
+  });
 });

src/health/health.controller.ts

@@ -23,6 +23,15 @@ const HEALTH_CHECK_RPC_TIMEOUT_MS = 3000;
 // DB_REPLICATION_LAG_WARN_BYTES env var (bytes, integer).
 const DEFAULT_REPLICATION_LAG_WARN_BYTES = 50 * 1024 * 1024; // 50 MB
 
+// #471 — connection pool exhaustion monitoring. The max pool size mirrors
+// PrismaService's own DATABASE_CONNECTION_LIMIT default (see
+// src/prisma/prisma.service.ts) so the two stay in sync without duplicating
+// config; operators overriding one should override the other identically.
+// At >=90% of the pool's active connections, new queries are likely to start
+// queuing for a free connection, so we flag it before requests actually fail.
+const DEFAULT_DB_POOL_MAX_CONNECTIONS = 10;
+const DEFAULT_DB_POOL_EXHAUSTION_WARN_PERCENT = 90;
+
 // #426 — Lightweight probe URLs for external data providers.
 // Open-Meteo: free API, no key — a minimal forecast request with a 1-day
 //   window for the equator verifies HTTP reachability without side effects.
@@ -64,7 +73,7 @@ export class HealthController {
   async check(): Promise<HealthResponseDto> {
     let dbStatus: 'ok' | 'error' = 'ok';
     let dbError: string | undefined;
-    let dbPool: { active: number; idle: number; waiting: number } | undefined;
+    let dbPool: { active: number; idle: number; waiting: number; max: number; utilizationPercent: number; exhausted: boolean } | undefined;
     let stellarStatus: 'ok' | 'error' = 'ok';
     let stellarError: string | undefined;
     let keeperBalanceXlm: string | undefined;
@@ -89,20 +98,41 @@ export class HealthController {
       this.logger.error(`Health check DB query failed: ${err instanceof Error ? err.message : String(err)}`);
     }
 
-    // #444 — connection pool health: query pg_stat_activity so load balancers
-    // can alert on pool exhaustion before queries start queuing or timing out.
+    // #444/#471 — connection pool health: query pg_stat_activity so load
+    // balancers can alert on pool exhaustion before queries start queuing or
+    // timing out. `active` is compared against the pool's configured max size
+    // to compute a utilization percentage; crossing the warning threshold
+    // marks the database check degraded so it surfaces the same way other
+    // dependency failures do, instead of only being visible as raw counts an
+    // operator has to interpret themselves.
     try {
       const rows = await this.prisma.$queryRaw<Array<{ state: string; count: bigint }>>`
         SELECT state, COUNT(*)::int AS count
         FROM pg_stat_activity
         WHERE datname = current_database()
         GROUP BY state
       `;
-      dbPool = {
-        active:  Number(rows.find(r => r.state === 'active')?.count  ?? 0),
-        idle:    Number(rows.find(r => r.state === 'idle')?.count    ?? 0),
-        waiting: Number(rows.find(r => r.state === 'idle in transaction (aborted)')?.count ?? 0),
-      };
+      const active  = Number(rows.find(r => r.state === 'active')?.count  ?? 0);
+      const idle    = Number(rows.find(r => r.state === 'idle')?.count    ?? 0);
+      const waiting = Number(rows.find(r => r.state === 'idle in transaction (aborted)')?.count ?? 0);
+
+      const maxConnections = Number(
+        this.config.get<string>('DATABASE_CONNECTION_LIMIT') ?? DEFAULT_DB_POOL_MAX_CONNECTIONS,
+      );
+      const warnPercent = Number(
+        this.config.get<string>('DB_POOL_EXHAUSTION_WARN_PERCENT') ?? DEFAULT_DB_POOL_EXHAUSTION_WARN_PERCENT,
+      );
+      const utilizationPercent = maxConnections > 0 ? Math.round((active / maxConnections) * 100) : 0;
+      const exhausted = utilizationPercent >= warnPercent;
+
+      dbPool = { active, idle, waiting, max: maxConnections, utilizationPercent, exhausted };
+
+      if (exhausted) {
+        dbStatus = 'error';
+        const poolMsg = `Connection pool utilization ${utilizationPercent}% (${active}/${maxConnections} active) meets or exceeds the exhaustion warning threshold of ${warnPercent}%`;
+        dbError = dbError ? `${dbError}; ${poolMsg}` : poolMsg;
+        this.logger.error(`Health check: ${poolMsg}`);
+      }
     } catch {
       // Non-fatal: pg_stat_activity may be restricted on managed databases.
     }

src/main.ts

@@ -213,6 +213,59 @@ async function bootstrap() {
       '  "retryAfter": 42\n' +
       '}\n' +
       '```\n\n' +
+      '## Request Timeouts\n\n' +
+      'Every request is subject to a fixed processing timeout, enforced at two layers that share ' +
+      'the same 30-second value:\n\n' +
+      '| Layer | Behavior |\n' +
+      '|-------|----------|\n' +
+      '| Application middleware | Starts a timer when the request enters the handler chain. If no response has been sent after 30 seconds, it responds **408 Request Timeout** and closes the underlying connection so the server slot is freed immediately. |\n' +
+      '| HTTP server socket | A 30-second idle timeout on the underlying Node HTTP server, as a fallback in case the application-level timer is bypassed. |\n\n' +
+      'The timeout window is currently a fixed 30 seconds — it is not configurable via an environment ' +
+      'variable — and applies uniformly to every route; there is no per-endpoint override.\n\n' +
+      '### 408 response body\n\n' +
+      'The 408 response is written directly by the timeout middleware before the request reaches ' +
+      'route handling, so — unlike every other error response documented here — it does **not** use ' +
+      'the standard error envelope (no `success`, `errorCode`, `path`, or `timestamp` fields):\n\n' +
+      '```json\n' +
+      '{\n' +
+      '  "statusCode": 408,\n' +
+      '  "error": "Request Timeout",\n' +
+      '  "message": "The request exceeded the maximum allowed processing time."\n' +
+      '}\n' +
+      '```\n\n' +
+      '### Handling 408s\n\n' +
+      'A 408 means the handler was still running past the 30-second window, not that the request was ' +
+      'necessarily rejected before doing any work. Treat it as retryable following the same guidance as ' +
+      '`SERVICE_UNAVAILABLE` above (exponential backoff; for mutating `POST`/`PUT`/`PATCH` requests, ' +
+      'resend the same `Idempotency-Key` so a request that actually completed server-side is not ' +
+      're-executed).\n\n' +
+      '## Pagination\n\n' +
+      'List endpoints that return more than a handful of rows accept `page` and `limit` query ' +
+      'parameters and return a paginated envelope instead of a bare array:\n\n' +
+      '| Parameter | Type | Default | Notes |\n' +
+      '|-----------|------|---------|-------|\n' +
+      '| `page`    | integer | `1`  | 1-based. Values below 1 are clamped up to 1. |\n' +
+      '| `limit`   | integer | `20` | Clamped to the range `1`-`100`; values outside that range are clamped, not rejected. |\n\n' +
+      '```json\n' +
+      '{\n' +
+      '  "success": true,\n' +
+      '  "data": [ /* items for this page */ ],\n' +
+      '  "total": 42,\n' +
+      '  "page": 1,\n' +
+      '  "limit": 20\n' +
+      '}\n' +
+      '```\n\n' +
+      '| Field   | Description |\n' +
+      '|---------|-------------|\n' +
+      '| `data`  | Array of items for the requested page |\n' +
+      '| `total` | Total number of items across all pages |\n' +
+      '| `page`  | The page number this response corresponds to |\n' +
+      '| `limit` | The page size this response used |\n\n' +
+      'Paginated endpoints: `GET /products`, `GET /policies/me`, `GET /claims`, `GET /claims/history/{wallet}`. ' +
+      'Some of these also support `?stream=true` (NDJSON, one item per line) as an alternative to the ' +
+      'paginated JSON envelope for large result sets — see that endpoint\'s own `stream` parameter ' +
+      'description for details; pagination metadata is then returned via the `X-Total-Count`, `X-Page`, ' +
+      'and `X-Limit` response headers instead of the JSON body.\n\n' +
       '## Error Response Structure\n\n' +
       'All error responses follow a consistent envelope format for reliable parsing:\n\n' +
       '```json\n' +