Merge pull request #415 from Depo-dev/feat/stellar-wave-observability-security-loadtest-runbook

Load/soak testing, security regression CI, observability, and ops docs

Author: codebestia

.github/workflows/loadtest-nightly.yml

@@ -0,0 +1,121 @@
+# Issue #385 — nightly WebSocket gateway load/soak test.
+#
+# Boots postgres + redis + two backend instances (ports 3001/3002) sharing
+# the same Redis, so fan-out and presence churn cross node boundaries via
+# @socket.io/redis-adapter. Seeds fixture data directly through Drizzle,
+# then runs scripts/loadtest/run.ts, which exits non-zero if latency,
+# memory, or error-rate thresholds (or the regression baseline in
+# scripts/loadtest/baseline.json) are violated.
+#
+# Runs nightly only — this is a soak test, not a per-PR gate. Trigger
+# manually via workflow_dispatch to validate before merging test changes.
+
+name: Nightly Load Test
+
+on:
+  schedule:
+    - cron: '0 3 * * *' # 03:00 UTC nightly
+  workflow_dispatch:
+
+jobs:
+  loadtest:
+    name: WebSocket gateway load/soak test
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    services:
+      postgres:
+        image: postgres:15-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: clicked
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d clicked"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 5
+      redis:
+        image: redis:7-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 5
+
+    env:
+      DATABASE_URL: postgres://postgres:password@localhost:5432/clicked
+      REDIS_URL: redis://localhost:6379
+      JWT_SECRET: loadtest-secret
+      TOKEN_TRANSFER_CONTRACT_ID: loadtest-placeholder
+      OBJECT_STORE_ENDPOINT: http://localhost:9000
+      OBJECT_STORE_BUCKET: clicked
+      OBJECT_STORE_ACCESS_KEY: clicked
+      OBJECT_STORE_SECRET_KEY: clickedsecret
+      OBJECT_STORE_REGION: us-east-1
+      OBJECT_STORE_FORCE_PATH_STYLE: 'true'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run DB migrations
+        run: pnpm db:migrate
+        working-directory: apps/backend
+
+      - name: Start backend node 1 (port 3001)
+        run: pnpm dev &
+        working-directory: apps/backend
+        env:
+          PORT: '3001'
+
+      - name: Start backend node 2 (port 3002)
+        run: pnpm dev &
+        working-directory: apps/backend
+        env:
+          PORT: '3002'
+
+      - name: Wait for both nodes to be healthy
+        run: |
+          for port in 3001 3002; do
+            for i in $(seq 1 30); do
+              if curl -sf "http://localhost:$port/health" > /dev/null; then
+                echo "node on $port healthy"
+                break
+              fi
+              sleep 1
+            done
+          done
+
+      - name: Seed load-test fixture (200 devices)
+        run: npx tsx scripts/loadtest/seed.ts --devices 200 > fixture.json
+
+      - name: Run load/soak test
+        run: >-
+          npx tsx scripts/loadtest/run.ts
+          --fixture fixture.json
+          --nodes http://localhost:3001,http://localhost:3002
+          --baseline scripts/loadtest/baseline.json
+          --out loadtest-result.json
+
+      - name: Upload result
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: loadtest-result
+          path: loadtest-result.json

.github/workflows/security-ci.yml

@@ -0,0 +1,69 @@
+# Issue #388 — Security regression checks.
+#
+# Runs on every PR and push to main. Two jobs:
+#   - regression: the ciphertext-only guard and the private-key/session-state
+#     field scan (apps/backend/src/__tests__/security.regression.test.ts).
+#   - dependency-audit: `pnpm audit` scoped to the crypto-relevant backend
+#     dependencies (ioredis, jsonwebtoken, web-push, @stellar/stellar-sdk,
+#     drizzle-orm, socket.io), surfacing CVEs without failing on advisories
+#     in unrelated transitive deps.
+
+name: Security CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  regression:
+    name: Ciphertext-only guard + secret-field scan
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: apps/backend
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+        working-directory: .
+
+      - name: Run security regression tests
+        run: pnpm test -- security.regression.test.ts
+        env:
+          JWT_SECRET: ci-test-secret
+          REDIS_URL: redis://localhost:6379
+
+  dependency-audit:
+    name: Crypto dependency CVE audit
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Audit crypto-relevant dependencies
+        run: node scripts/audit-crypto-deps.mjs

apps/backend/package.json

@@ -38,7 +38,9 @@
     "ioredis": "^5.11.0",
     "jsonwebtoken": "^9.0.3",
     "morgan": "^1.10.1",
+    "pino": "^9.6.0",
     "postgres": "^3.4.9",
+    "prom-client": "^15.1.3",
     "redis": "^6.0.0",
     "socket.io": "^4.8.3",
     "web-push": "^3.6.7",

apps/backend/src/__tests__/metrics.test.ts

@@ -0,0 +1,76 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import request from 'supertest';
+
+const mockExecute = vi.fn();
+
+vi.mock('../db/index.js', () => ({
+  db: {
+    execute: mockExecute,
+    query: {
+      conversations: { findFirst: vi.fn() },
+      conversationMembers: { findFirst: vi.fn(), findMany: vi.fn() },
+      messages: { findFirst: vi.fn() },
+      tokenTransfers: { findFirst: vi.fn(), findMany: vi.fn() },
+      users: { findFirst: vi.fn() },
+      wallets: { findFirst: vi.fn() },
+    },
+  },
+}));
+
+vi.mock('../services/pushNotification.js', () => ({
+  dispatchOfflinePush: vi.fn().mockResolvedValue(undefined),
+  reenableExpiredBackoffs: vi.fn().mockResolvedValue(undefined),
+  FILE_CONTENT_TYPES: new Set<string>(),
+}));
+
+vi.mock('../services/deliveryPipeline.js', () => ({
+  deliverMessage: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock('../services/deviceDelivery.js', () => ({
+  publishToDevice: vi.fn().mockResolvedValue(undefined),
+}));
+
+const { app } = await import('../app.js');
+const {
+  messagesPersistedTotal,
+  fanoutSize,
+  pushResultTotal,
+  presenceChurnTotal,
+  backpressureEventsTotal,
+} = await import('../lib/metrics.js');
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('GET /metrics', () => {
+  it('exposes Prometheus-format metrics', async () => {
+    const res = await request(app).get('/metrics');
+
+    expect(res.status).toBe(200);
+    expect(res.headers['content-type']).toContain('text/plain');
+    expect(res.text).toContain('clicked_messages_persisted_total');
+    expect(res.text).toContain('clicked_fanout_size');
+    expect(res.text).toContain('clicked_delivery_latency_seconds');
+    expect(res.text).toContain('clicked_prekey_consumed_total');
+    expect(res.text).toContain('clicked_push_result_total');
+    expect(res.text).toContain('clicked_presence_churn_total');
+    expect(res.text).toContain('clicked_backpressure_events_total');
+    expect(res.text).toContain('clicked_connected_sockets');
+  });
+
+  it('never contains ciphertext, envelope payloads, or free-text content', async () => {
+    messagesPersistedTotal.inc({ contentType: 'text' });
+    fanoutSize.observe(3);
+    pushResultTotal.inc({ result: 'sent' });
+    presenceChurnTotal.inc({ transition: 'online' });
+    backpressureEventsTotal.inc({ action: 'shed' });
+
+    const res = await request(app).get('/metrics');
+
+    expect(res.text).not.toMatch(/ciphertext/i);
+    expect(res.text).not.toMatch(/plaintext/i);
+    expect(res.text).not.toMatch(/envelope.{0,20}[A-Za-z0-9+/]{20,}/i);
+  });
+});

apps/backend/src/__tests__/security.regression.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect } from 'vitest';
+import { readFileSync, readdirSync } from 'fs';
+import { join, extname, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { validateMessagePayload } from '../lib/validateMessagePayload.js';
+import { SendMessageSchema } from '../schemas/message.schemas.js';
+
+/**
+ * Security regression checks (#388).
+ *
+ * These tests fail CI the moment any code path starts accepting a
+ * plaintext-only message, or any schema/route grows a field that could
+ * carry a raw private key or Signal session-state blob. They are a guard
+ * against regressions, not a substitute for the crypto design itself.
+ */
+
+const FORBIDDEN_FIELD_NAMES = [
+  'plaintext',
+  'plainText',
+  'privateKey',
+  'private_key',
+  'sessionState',
+  'session_state',
+  'signalSession',
+  'identityPrivateKey',
+  'preKeyPrivate',
+];
+
+const SRC_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+
+function listSourceFiles(dir: string): string[] {
+  const entries = readdirSync(dir, { withFileTypes: true });
+  const files: string[] = [];
+  for (const entry of entries) {
+    if (entry.name === '__tests__' || entry.name === 'node_modules') continue;
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...listSourceFiles(full));
+    } else if (extname(entry.name) === '.ts') {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+describe('security regression: ciphertext-only guard', () => {
+  it('rejects a text message with plaintext content and no envelopes', () => {
+    const result = validateMessagePayload({
+      contentType: 'text',
+      // @ts-expect-error - deliberately probing for a plaintext field the type doesn't allow
+      plaintext: 'hello in the clear',
+    });
+    expect(result.ok).toBe(false);
+  });
+
+  it('rejects a text message that supplies ciphertext but no per-device envelopes', () => {
+    const result = validateMessagePayload({
+      contentType: 'text',
+      ciphertext: 'some-ciphertext',
+    });
+    expect(result.ok).toBe(false);
+  });
+
+  it('accepts a text message only when envelopes carry the encrypted key', () => {
+    const result = validateMessagePayload({
+      contentType: 'text',
+      envelopes: [{ recipientDeviceId: 'device-1', ciphertext: 'enc-key' }],
+    });
+    expect(result.ok).toBe(true);
+  });
+
+  it('REST SendMessageSchema has no plaintext field', () => {
+    const shape = SendMessageSchema.shape as Record<string, unknown>;
+    expect(Object.keys(shape)).not.toContain('plaintext');
+    expect(Object.keys(shape)).not.toContain('plainText');
+  });
+});
+
+describe('security regression: no private-key/session-state field is ever accepted', () => {
+  const sourceFiles = listSourceFiles(SRC_ROOT);
+
+  it('scanned at least one route/schema file', () => {
+    expect(sourceFiles.length).toBeGreaterThan(0);
+  });
+
+  it.each(FORBIDDEN_FIELD_NAMES)('no source file declares a "%s" field', (fieldName) => {
+    const offenders: string[] = [];
+    // Matches z.object key declarations and TS interface/type field declarations,
+    // e.g. `privateKey:` — not matched inside comments-only prose or unrelated words.
+    const pattern = new RegExp(`(^|[^A-Za-z0-9_])${fieldName}\\s*[:?]\\s*[^,]`, 'm');
+
+    for (const file of sourceFiles) {
+      const content = readFileSync(file, 'utf-8');
+      if (pattern.test(content)) {
+        offenders.push(file);
+      }
+    }
+
+    expect(offenders).toEqual([]);
+  });
+});