Add an incident response runbook for stellar rpc or ipf gateway outages

DEPLOYMENT.md

@@ -60,6 +60,8 @@ Always back up the database file before running migrations in production.
 Configure your load balancer or orchestrator to poll `/health` every 30 seconds.  
 Alert on consecutive failures (≥ 2) to catch Stellar RPC or IPFS outages early.
 
+In the event of an outage, refer to the [Dependency Outages Runbook](docs/runbooks/dependency-outages.md) for mitigation and recovery procedures.
+
 Recommended metrics to track:
 - HTTP 5xx error rate
 - Event indexer lag (gap between latest on-chain event and last indexed event)

README.md

@@ -325,7 +325,7 @@ See [DEPLOYMENT.md](DEPLOYMENT.md) for complete deployment instructions.
 
 ## Health Endpoints
 
-The backend exposes two health check endpoints for monitoring and orchestration probes.
+The backend exposes two health check endpoints for monitoring and orchestration probes. For detailed instructions on handling external dependency outages (Stellar RPC or IPFS/Pinata), refer to the [Dependency Outages Runbook](docs/runbooks/dependency-outages.md).
 
 | Method | Path | Auth | Description |
 |--------|------|------|-------------|

__mocks__/better-sqlite3.js

@@ -17,10 +17,42 @@ class Statement {
       if (!this._db._events.find((e) => e.tx_hash === txHash)) {
         this._db._events.push({ type, ledger, tx_hash: txHash, payload });
       }
-    } else if (sql.startsWith('INSERT INTO INDEXER_STATE') || sql.startsWith('INSERT OR REPLACE INTO INDEXER_STATE')) {
+      return { changes: 1, lastInsertRowid: 0 };
+    }
+
+    if (sql.startsWith('INSERT INTO INDEXER_STATE') || sql.startsWith('INSERT OR REPLACE INTO INDEXER_STATE')) {
       const [key, value] = args;
       this._db._state.set(key, value);
+      return { changes: 1, lastInsertRowid: 0 };
+    }
+
+    if (sql.startsWith('INSERT INTO IDEMPOTENCY_KEYS')) {
+      const [key, expiresAt, requestHash, method, path, statusCode, responseBody, createdAt] = args;
+      this._db._idempotencyRows.set(key, {
+        key,
+        expires_at: expiresAt,
+        request_hash: requestHash,
+        method,
+        path,
+        status_code: statusCode,
+        response_body: responseBody,
+        created_at: createdAt,
+      });
+      return { changes: 1, lastInsertRowid: 0 };
     }
+
+    if (sql.startsWith('DELETE FROM IDEMPOTENCY_KEYS')) {
+      const threshold = args[0];
+      let deleted = 0;
+      for (const [key, row] of Array.from(this._db._idempotencyRows.entries())) {
+        if (row.expires_at <= threshold) {
+          this._db._idempotencyRows.delete(key);
+          deleted += 1;
+        }
+      }
+      return { changes: deleted, lastInsertRowid: 0 };
+    }
+
     return { changes: 1, lastInsertRowid: 0 };
   }
 
@@ -31,6 +63,25 @@ class Statement {
       const value = this._db._state.get(key);
       return value !== undefined ? { value } : undefined;
     }
+
+    if (sql.includes('FROM IDEMPOTENCY_KEYS')) {
+      const [key, now] = args;
+      const row = this._db._idempotencyRows.get(key);
+      if (row && row.expires_at > now) {
+        return {
+          key: row.key,
+          expiresAt: row.expires_at,
+          requestHash: row.request_hash,
+          method: row.method,
+          path: row.path,
+          statusCode: row.status_code,
+          responseBody: row.response_body,
+          createdAt: row.created_at,
+        };
+      }
+      return undefined;
+    }
+
     return undefined;
   }
 
@@ -42,6 +93,13 @@ class Statement {
       }
       return [...this._db._events];
     }
+
+    if (sql.includes('FROM IDEMPOTENCY_KEYS')) {
+      return Array.from(this._db._idempotencyRows.values()).map((row) => ({
+        key: row.key,
+      }));
+    }
+
     return [];
   }
 }
@@ -50,6 +108,7 @@ class Database {
   constructor(_path) {
     this._events = [];
     this._state = new Map();
+    this._idempotencyRows = new Map();
   }
 
   exec(_sql) {

db/003_idempotency_keys.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS idempotency_keys (
+  key TEXT PRIMARY KEY,
+  expires_at INTEGER NOT NULL,
+  request_hash TEXT NOT NULL,
+  method TEXT NOT NULL,
+  path TEXT NOT NULL,
+  status_code INTEGER NOT NULL DEFAULT 0,
+  response_body TEXT NOT NULL DEFAULT '',
+  created_at INTEGER NOT NULL
+);

docs/runbooks/dependency-outages.md

@@ -0,0 +1,217 @@
+# Runbook: External Dependency Outages (Stellar RPC & IPFS/Pinata)
+
+This document provides detection, mitigation, recovery, and communication instructions for on-call engineers managing outages of external dependencies in the ScoutOff platform. 
+
+The ScoutOff backend relies on two major external systems:
+1. **Stellar Network / Soroban RPC**: For indexing contract events, verifying milestones, registrations, and pay-to-contact settlements.
+2. **IPFS / Pinata Gateway**: For pinning and storing player profile metadata, photos, highlight reels, and validator evidence.
+
+---
+
+## 1. Stellar RPC Outage
+
+> [!WARNING]
+> During a Stellar RPC outage, write operations (player registration, milestone submissions, contact payments) will fail. However, read operations (browsing profiles, filtering, search caching) will continue to work normally because they read from the local SQLite index.
+
+### Detection Signals
+
+#### Health & Readiness Endpoints
+- **Liveness probe (`GET /health`)**:
+  Returns HTTP `200 OK` but contains `"stellar": "error"` in the response body.
+  ```json
+  {
+    "status": "ok",
+    "healthStatus": {
+      "stellar": "error"
+    }
+  }
+  ```
+- **Readiness probe (`GET /ready` or `GET /health/readiness`)**:
+  Returns HTTP `503 Service Unavailable` with `status: "degraded"`.
+  ```json
+  {
+    "status": "degraded",
+    "services": {
+      "ipfs": "ok",
+      "stellar": "unavailable"
+    }
+  }
+  ```
+
+#### Log Patterns
+Check system logs (`stderr`/`stdout`) for the following patterns:
+- **Event Indexer errors** (emitted every 5 seconds by the indexer loop):
+  `[error] Indexer error: <reason>`
+  Common messages:
+  - `[error] Indexer error: fetch failed`
+  - `[error] Indexer error: request failed with status code 503`
+  - `[error] Indexer error: getaddrinfo ENOTFOUND soroban-testnet.stellar.org`
+- **Route / Controller errors** (logged by global Express error handler):
+  `console.error` logs from failed transactions or signature checks:
+  - `[error] network error` or `PaymentError: NETWORK_ERROR`
+
+### Immediate Mitigation Options
+
+#### Option A: Bypass Stellar Health Check (Keep service marked ready)
+By default, an RPC outage causes `/ready` to return `503`, which may cause Kubernetes or your cloud load balancer to kill/route traffic away from the backend container, resulting in a full service outage.
+To keep the server marked healthy for read-only traffic (non-chain features):
+1. Locate the environment variables or `.env` file on the server.
+2. Set or update:
+   ```env
+   STELLAR_HEALTH_CHECK=false
+   ```
+3. Restart the backend process:
+   ```bash
+   # If running via PM2:
+   pm2 restart scout-off-backend
+   # If running via systemd:
+   systemctl restart scout-off-backend
+   # If running in Docker:
+   docker restart <container_id>
+   ```
+4. Verify `/ready` now returns `200 OK` with `"stellar": "disabled"`:
+   ```json
+   {
+     "status": "ok",
+     "services": {
+       "ipfs": "ok",
+       "stellar": "disabled"
+     }
+   }
+   ```
+
+#### Option B: Failover to Backup RPC Nodes
+If the public SDF RPC endpoint (`https://soroban-testnet.stellar.org`) is offline but other RPC endpoints are healthy (e.g., QuickNode or a private node):
+1. Update `.env` with a backup URL:
+   ```env
+   SOROBAN_RPC_URL=https://<backup-stellar-rpc-provider-url>
+   # Update Horizon if Horizon is also down
+   HORIZON_URL=https://<backup-horizon-url>
+   ```
+2. Restart the backend process.
+3. Check the startup health logs:
+   `[info] Startup health: {"ipfs":"ok","stellar":"ok"}`
+
+### Recovery Verification
+Before reverting any mitigation (like setting `STELLAR_HEALTH_CHECK` back to `true`), verify the RPC network has fully recovered:
+1. Manually query the configured RPC url using curl:
+   ```bash
+   curl -X POST -H "Content-Type: application/json" \
+     -d '{"jsonrpc":"2.0","id":1,"method":"getLatestLedger"}' \
+     https://soroban-testnet.stellar.org
+   ```
+   Verify you receive a valid JSON response containing `sequence` and `protocolVersion`.
+2. Once the RPC responds, restore the config in `.env`:
+   ```env
+   STELLAR_HEALTH_CHECK=true
+   ```
+3. Restart the backend process and verify `GET /ready` returns:
+   ```json
+   {
+     "status": "ok",
+     "services": {
+       "ipfs": "ok",
+       "stellar": "ok"
+     }
+   }
+   ```
+
+---
+
+## 2. IPFS / Pinata Outage
+
+> [!IMPORTANT]
+> When IPFS/Pinata is down, players cannot complete registration because metadata JSON pinning fails, and validators cannot submit new milestones (evidence upload fails).
+
+### Detection Signals
+
+#### Health & Readiness Endpoints
+- **Readiness probe (`GET /ready` or `GET /health/readiness`)**:
+  Returns HTTP `503 Service Unavailable` with `status: "degraded"` and `services.ipfs` marked `unavailable`.
+  ```json
+  {
+    "status": "degraded",
+    "services": {
+      "ipfs": "unavailable",
+      "stellar": "ok"
+    }
+  }
+  ```
+
+#### Log Patterns
+Check system logs for errors thrown during pinning:
+- **Axios error logs** from IPFS service:
+  - `console.error` logs with messages:
+    - `request failed with status code 503` (or 502/504 Bad Gateway from Pinata API)
+    - `getaddrinfo ENOTFOUND api.pinata.cloud`
+    - `Error: IPFS connection refused`
+
+### Immediate Mitigation Options
+
+#### Option A: Enable IPFS Mock/Stub Mode
+If Pinata is experiencing a genuine prolonged outage, you can temporarily enable **IPFS Stub Mode**. This bypasses Axios network requests to Pinata and returns a valid static CID (`QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG`), allowing registrations and milestone submissions to proceed (using stubbed data).
+1. Open the server's `.env` configuration file.
+2. Add or update the following environment variable:
+   ```env
+   IPFS_STUB_MODE=true
+   ```
+3. Restart the backend process:
+   ```bash
+   pm2 restart scout-off-backend
+   ```
+4. Verify `/ready` now returns `200 OK` (with `"ipfs": "ok"` mocked):
+   ```json
+   {
+     "status": "ok",
+     "services": {
+       "ipfs": "ok",
+       "stellar": "ok"
+     }
+   }
+   ```
+5. Test a registration or milestone submission. It should succeed immediately, returning the mock CID.
+
+### Recovery Verification
+1. To check if the Pinata service is back, manually test the authentication API endpoint using curl:
+   ```bash
+   curl -H "pinata_api_key: <YOUR_PINATA_API_KEY>" \
+        -H "pinata_secret_api_key: <YOUR_PINATA_SECRET>" \
+        https://api.pinata.cloud/data/testAuthentication
+   ```
+   If it returns `{"message":"Congratulations! You are communicating with the Pinata API!"}`, Pinata has recovered.
+2. Disable the IPFS stub mode in `.env`:
+   ```env
+   IPFS_STUB_MODE=false
+   ```
+3. Restart the backend process.
+4. Verify `GET /ready` returns HTTP `200 OK` with all actual services listed as `"ok"`.
+
+---
+
+## 3. Communication Playbook
+
+In the event of an outage, communicate the status promptly to platform users and stakeholders:
+
+### Pre-written Notification Templates
+
+#### For Stellar RPC Outage (Degraded/Read-Only Mode)
+* **Channel**: Twitter/X, Discord Announcement, or Banner in Frontend
+* **Message**:
+  > **ScoutOff Infrastructure Notice** ⚠️
+  > The Stellar network node we use is currently experiencing connection issues. 
+  > 
+  > * **What is working**: You can still log in, browse player profiles, view validator history, and search positions.
+  > * **What is paused**: New player registrations, validator approvals, and pay-to-contact transactions are temporarily unavailable.
+  > 
+  > Our engineers are monitoring the situation and will restore full transaction capability as soon as the RPC node is back online. Thank you for your patience!
+
+#### For IPFS/Pinata Outage (Degraded Mode)
+* **Channel**: Twitter/X, Discord Announcement, or Banner in Frontend
+* **Message**:
+  > **ScoutOff Storage Service Interruption** ⚠️
+  > Our media storage provider (Pinata/IPFS) is currently experiencing an outage. 
+  > 
+  > * **What is working**: You can search profiles, view existing cached vitals, and initiate scout contacts.
+  > * **What is paused**: Uploading new highlight videos, pinning profile updates, and submitting new milestone evidence.
+  > 
+  > We have enabled a temporary fallback service so player registration forms can still submit, but files/images will not preview until our storage partner recovers.

package.json

@@ -8,7 +8,8 @@
     "build": "tsc",
     "start": "node dist/index.js",
     "test": "jest --runInBand",
-    "lint": "eslint 'src/**/*.ts' 'tests/**/*.ts' --ext .ts"
+    "lint": "eslint 'src/**/*.ts' 'tests/**/*.ts' --ext .ts",
+    "purge:idempotency-keys": "npm run build && node dist/scripts/purgeIdempotencyKeys.js"
   },
   "dependencies": {
     "@stellar/stellar-sdk": "12.1.0",

src/config.ts

@@ -19,6 +19,14 @@ const ConfigSchema = z.object({
   dbPath: z.string().default('scout-off.db'),
 });
 
+function required(key: string): string {
+  const value = process.env[key];
+  if (!value) {
+    throw new Error(`Missing required environment variable: ${key}`);
+  }
+  return value;
+}
+
 const config = {
   port: parseInt(process.env.PORT ?? '4000', 10),
   network: (process.env.NETWORK ?? 'testnet') as 'testnet' | 'mainnet',
@@ -39,14 +47,14 @@ const config = {
   dbPath: process.env.DB_PATH ?? 'scout-off.db',
   logLevel: (process.env.LOG_LEVEL ?? 'info') as 'debug' | 'info' | 'warn' | 'error',
   stellarHealthCheckEnabled: process.env.STELLAR_HEALTH_CHECK !== 'false',
+  ipfsStubMode: process.env.IPFS_STUB_MODE === 'true',
   adminWallet: process.env.ADMIN_WALLET ?? '',
   securityHeaders: {
     hsts: process.env.SECURITY_HSTS ?? 'max-age=31536000; includeSubDomains',
     xContentTypeOptions: process.env.SECURITY_X_CONTENT_TYPE_OPTIONS ?? 'nosniff',
     xFrameOptions: process.env.SECURITY_X_FRAME_OPTIONS ?? 'DENY',
     referrerPolicy: process.env.SECURITY_REFERRER_POLICY ?? 'no-referrer',
   },
-  logLevel: (process.env.LOG_LEVEL ?? 'info') as 'debug' | 'info' | 'warn' | 'error',
   webhook: {
     enabled: process.env.WEBHOOK_ENABLED === 'true',
     url: process.env.WEBHOOK_URL ?? ''
@@ -56,6 +64,10 @@ const config = {
     windowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS ?? '60000', 10),
     max: parseInt(process.env.RATE_LIMIT_MAX ?? '60', 10),
   },
+  idempotency: {
+    ttlSeconds: parseInt(process.env.IDEMPOTENCY_TTL_SECONDS ?? '86400', 10),
+    purgeIntervalMs: parseInt(process.env.IDEMPOTENCY_PURGE_INTERVAL_MS ?? '60000', 10),
+  },
 };
 
 export default config;