mux-labs · Jambox11 · Sep 4, 2026 · Aug 31, 2026
diff --git a/.env.example b/.env.example
@@ -156,6 +156,26 @@ AUTH_RATE_LIMIT_MAX=10
 # Time window in milliseconds for auth rate limiting (default: 60 seconds)
 AUTH_RATE_LIMIT_WINDOW_MS=60000
 
+# ------------------------------------------------------------
+# Rate-limit table cleanup
+# Optional: interval between cleanup runs (default: 3600000 / 1 hour)
+# Stale RateLimitRecord rows older than RATE_LIMIT_CLEANUP_OLDER_THAN_MS are
+# deleted by RateLimitCleanupWorker. Without this job the table grows unbounded.
+# ------------------------------------------------------------
+RATE_LIMIT_CLEANUP_INTERVAL_MS=3600000
+# Optional: delete records older than this (default: 3600000 / 1 hour)
+RATE_LIMIT_CLEANUP_OLDER_THAN_MS=3600000
+
+# ------------------------------------------------------------
+# DLQ ops / pager notification
+# Optional: HTTP(S) URL to POST a JSON alert when webhook DLQ thresholds breach.
+# Works with Slack incoming webhooks, PagerDuty Events API, or any HTTP sink.
+# Leave unset to disable outbound notifications (metrics-only mode).
+# ------------------------------------------------------------
+DLQ_OPS_WEBHOOK_URL=
+# Optional: timeout (ms) for outbound DLQ notification calls (default: 5000)
+DLQ_OPS_WEBHOOK_TIMEOUT_MS=5000
+
 # ------------------------------------------------------------
 # OpenTelemetry / Tracing
 # Optional: Set OTEL_ENABLED=true to activate distributed tracing.

diff --git a/README.md b/README.md
@@ -544,6 +544,89 @@ Key authentication-related environment variables (when applicable):
 
 ---
 
+## Rate-Limit Record Cleanup
+
+The `RateLimitCleanupWorker` runs on a configurable interval and prunes expired
+`RateLimitRecord` rows from the database. Without this job the table grows
+unbounded as every API key × endpoint × time-window combination adds a row.
+
+The worker is automatically registered in `RateLimitModule` — no manual
+wiring is required.
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `RATE_LIMIT_CLEANUP_INTERVAL_MS` | `3600000` | How often (ms) to run the cleanup job |
+| `RATE_LIMIT_CLEANUP_OLDER_THAN_MS` | `3600000` | Delete records with `windowStart` older than this age (ms) |
+
+---
+
+## Testnet Faucet
+
+The `TestnetFaucetService` proxies Stellar Friendbot funding requests for TESTNET wallets only.
+
+### Mainnet gate (fail-closed)
+
+When `STELLAR_NETWORK` is set to `MAINNET` or `PUBLIC` the service **refuses all funding requests** with `501 Not Implemented`, regardless of `NODE_ENV`. This is an unconditional safety gate — there is no override and no silent fallback.
+
+Set `STELLAR_NETWORK=TESTNET` (the default) to enable faucet funding.
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `STELLAR_NETWORK` | `TESTNET` | Target network. Set to `MAINNET`/`PUBLIC` to block faucet calls |
+| `TESTNET_FAUCET_URL` | `https://friendbot.stellar.org` | Faucet endpoint URL |
+| `TESTNET_FAUCET_MAX_REQUESTS` | `5` | Max faucet requests per wallet per window |
+| `TESTNET_FAUCET_WINDOW_MS` | `3600000` | Throttle window length (ms) |
+
+---
+
+## Webhook DLQ Ops Notifications
+
+`WebhookDlqAlertService` monitors the webhook dead-letter queue and can POST a
+structured JSON alert to an ops endpoint (Slack, PagerDuty, or any HTTP sink)
+whenever a threshold is breached.
+
+Notification failures are **non-fatal**: a Slack outage cannot disrupt the DLQ
+check loop or normal webhook delivery.
+
+### Payload shape
+
+```json
+{
+  "service": "mux-backend",
+  "event": "dlq.threshold_breached",
+  "text": "[mux-backend] DLQ threshold breached: ...",
+  "dlqDepth": 55,
+  "totalDeliveries": 500,
+  "dlqPercentage": 11.0,
+  "oldestDlqItemAgeMs": 7200000,
+  "alerts": [
+    { "type": "ABSOLUTE_THRESHOLD", "message": "...", "value": 55, "threshold": 50 }
+  ],
+  "checkedAt": "2026-08-31T23:00:00.000Z"
+}
+```
+
+The `text` field is Slack-compatible. For PagerDuty, wrap the payload in a
+[PagerDuty Events v2](https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api)
+adapter or use a custom HTTP sink.
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DLQ_OPS_WEBHOOK_URL` | _(unset)_ | HTTP(S) URL to POST alerts to. Leave unset for metrics-only mode |
+| `DLQ_OPS_WEBHOOK_TIMEOUT_MS` | `5000` | Timeout (ms) for outbound notification calls |
+| `DLQ_CHECK_INTERVAL_MS` | `60000` | How often (ms) to poll DLQ depth |
+| `DLQ_ABSOLUTE_THRESHOLD` | `50` | Alert when DLQ depth ≥ this value |
+| `DLQ_PERCENTAGE_THRESHOLD` | `10` | Alert when DLQ% of total deliveries ≥ this value |
+| `DLQ_AGE_THRESHOLD_MS` | `3600000` | Alert when oldest DLQ item is older than this (ms) |
+
+---
+
 ## Design Principles
 
 * **Crypto is infrastructure, not UX**

diff --git a/src/rate-limit/rate-limit-cleanup.worker.spec.ts b/src/rate-limit/rate-limit-cleanup.worker.spec.ts
@@ -0,0 +1,160 @@
+import { ConfigService } from '@nestjs/config';
+import { RateLimitCleanupWorker } from './rate-limit-cleanup.worker';
+import { RateLimitService } from './rate-limit.service';
+
+describe('RateLimitCleanupWorker', () => {
+  let worker: RateLimitCleanupWorker;
+  let mockRateLimitService: jest.Mocked<Pick<RateLimitService, 'cleanupOldRecords'>>;
+  let mockConfigService: jest.Mocked<Pick<ConfigService, 'get'>>;
+
+  beforeEach(() => {
+    jest.useFakeTimers();
+
+    mockRateLimitService = {
+      cleanupOldRecords: jest.fn().mockResolvedValue(0),
+    };
+
+    mockConfigService = {
+      get: jest.fn((key: string, defaultValue: any) => {
+        const config: Record<string, any> = {
+          RATE_LIMIT_CLEANUP_INTERVAL_MS: 3_600_000,
+          RATE_LIMIT_CLEANUP_OLDER_THAN_MS: 3_600_000,
+        };
+        return config[key] ?? defaultValue;
+      }),
+    };
+
+    worker = new RateLimitCleanupWorker(
+      mockRateLimitService as unknown as RateLimitService,
+      mockConfigService as unknown as ConfigService,
+    );
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+    jest.clearAllMocks();
+  });
+
+  // ---------------------------------------------------------------------------
+  // onModuleInit / onModuleDestroy lifecycle
+  // ---------------------------------------------------------------------------
+
+  describe('lifecycle', () => {
+    it('should start a timer on onModuleInit', () => {
+      const setIntervalSpy = jest.spyOn(global, 'setInterval');
+      worker.onModuleInit();
+      expect(setIntervalSpy).toHaveBeenCalledTimes(1);
+      worker.onModuleDestroy();
+    });
+
+    it('should clear the timer on onModuleDestroy', () => {
+      const clearIntervalSpy = jest.spyOn(global, 'clearInterval');
+      worker.onModuleInit();
+      worker.onModuleDestroy();
+      expect(clearIntervalSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('should call run() on each timer tick', async () => {
+      const runSpy = jest.spyOn(worker, 'run').mockResolvedValue(0);
+      worker.onModuleInit();
+
+      // Fast-forward one interval
+      jest.advanceTimersByTime(3_600_000);
+      // Allow pending microtasks to settle
+      await Promise.resolve();
+
+      expect(runSpy).toHaveBeenCalledTimes(1);
+      worker.onModuleDestroy();
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // run()
+  // ---------------------------------------------------------------------------
+
+  describe('run()', () => {
+    it('should call cleanupOldRecords with the configured olderThanMs', async () => {
+      mockRateLimitService.cleanupOldRecords.mockResolvedValue(5);
+
+      const deleted = await worker.run();
+
+      expect(mockRateLimitService.cleanupOldRecords).toHaveBeenCalledWith(
+        3_600_000,
+      );
+      expect(deleted).toBe(5);
+    });
+
+    it('should return 0 and not throw when cleanupOldRecords throws', async () => {
+      mockRateLimitService.cleanupOldRecords.mockRejectedValue(
+        new Error('DB connection lost'),
+      );
+
+      await expect(worker.run()).resolves.toBe(0);
+    });
+
+    it('should skip run() and return 0 when already running (re-entrancy guard)', async () => {
+      // Simulate a long-running cleanup
+      let resolve!: (v: number) => void;
+      const pending = new Promise<number>((r) => {
+        resolve = r;
+      });
+      mockRateLimitService.cleanupOldRecords.mockReturnValue(pending);
+
+      // Start first run (will hang)
+      const first = worker.run();
+      // Immediately start second run — should be skipped
+      const second = await worker.run();
+
+      expect(second).toBe(0);
+      expect(mockRateLimitService.cleanupOldRecords).toHaveBeenCalledTimes(1);
+
+      // Resolve the first run
+      resolve(3);
+      const firstResult = await first;
+      expect(firstResult).toBe(3);
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Configuration edge cases
+  // ---------------------------------------------------------------------------
+
+  describe('configuration', () => {
+    it('should use RATE_LIMIT_CLEANUP_INTERVAL_MS from config', () => {
+      mockConfigService.get.mockImplementation(
+        (key: string, defaultValue: any) => {
+          if (key === 'RATE_LIMIT_CLEANUP_INTERVAL_MS') return 1_800_000;
+          return defaultValue;
+        },
+      );
+
+      const customWorker = new RateLimitCleanupWorker(
+        mockRateLimitService as unknown as RateLimitService,
+        mockConfigService as unknown as ConfigService,
+      );
+
+      // Verify the worker was constructed without error
+      expect(customWorker).toBeDefined();
+    });
+
+    it('should use RATE_LIMIT_CLEANUP_OLDER_THAN_MS from config when running', async () => {
+      mockConfigService.get.mockImplementation(
+        (key: string, defaultValue: any) => {
+          if (key === 'RATE_LIMIT_CLEANUP_OLDER_THAN_MS') return 7_200_000;
+          return defaultValue;
+        },
+      );
+
+      const customWorker = new RateLimitCleanupWorker(
+        mockRateLimitService as unknown as RateLimitService,
+        mockConfigService as unknown as ConfigService,
+      );
+
+      await customWorker.run();
+
+      expect(mockRateLimitService.cleanupOldRecords).toHaveBeenCalledWith(
+        7_200_000,
+      );
+    });
+  });
+});
diff --git a/src/rate-limit/rate-limit-cleanup.worker.ts b/src/rate-limit/rate-limit-cleanup.worker.ts
@@ -0,0 +1,93 @@
+import {
+  Injectable,
+  Logger,
+  OnModuleInit,
+  OnModuleDestroy,
+} from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { RateLimitService } from './rate-limit.service';
+
+/**
+ * RateLimitCleanupWorker
+ *
+ * Periodically calls RateLimitService.cleanupOldRecords() to prune expired
+ * rate-limit windows from the database. Without this, the ApiKeyUsage /
+ * RateLimitRecord tables grow unbounded and degrade query performance.
+ *
+ * Configuration (environment variables):
+ *   RATE_LIMIT_CLEANUP_INTERVAL_MS  – polling interval in ms (default: 3_600_000 / 1 h)
+ *   RATE_LIMIT_CLEANUP_OLDER_THAN_MS – delete records older than this (default: 3_600_000 / 1 h)
+ */
+@Injectable()
+export class RateLimitCleanupWorker implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(RateLimitCleanupWorker.name);
+  private timer: NodeJS.Timeout | null = null;
+  private running = false;
+
+  private readonly intervalMs: number;
+  private readonly olderThanMs: number;
+
+  constructor(
+    private readonly rateLimitService: RateLimitService,
+    private readonly configService: ConfigService,
+  ) {
+    this.intervalMs = this.configService.get<number>(
+      'RATE_LIMIT_CLEANUP_INTERVAL_MS',
+      3_600_000,
+    );
+    this.olderThanMs = this.configService.get<number>(
+      'RATE_LIMIT_CLEANUP_OLDER_THAN_MS',
+      3_600_000,
+    );
+  }
+
+  onModuleInit(): void {
+    this.timer = setInterval(() => this.run(), this.intervalMs);
+    this.logger.log(
+      `Rate-limit cleanup worker started (interval: ${this.intervalMs}ms, ` +
+        `olderThan: ${this.olderThanMs}ms)`,
+    );
+  }
+
+  onModuleDestroy(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    this.logger.log('Rate-limit cleanup worker stopped');
+  }
+
+  /**
+   * Runs one cleanup cycle. Re-entrant guard prevents concurrent runs if an
+   * interval fires while the previous cycle is still in progress.
+   */
+  async run(): Promise<number> {
+    if (this.running) {
+      this.logger.warn(
+        'Rate-limit cleanup already running, skipping this tick',
+      );
+      return 0;
+    }
+
+    this.running = true;
+    try {
+      const deleted = await this.rateLimitService.cleanupOldRecords(
+        this.olderThanMs,
+      );
+      if (deleted > 0) {
+        this.logger.log(
+          `Rate-limit cleanup tick: deleted ${deleted} stale record(s)`,
+        );
+      }
+      return deleted;
+    } catch (err) {
+      this.logger.error(
+        'Rate-limit cleanup tick failed',
+        err instanceof Error ? err.message : String(err),
+      );
+      return 0;
+    } finally {
+      this.running = false;
+    }
+  }
+}
diff --git a/src/rate-limit/rate-limit.module.ts b/src/rate-limit/rate-limit.module.ts
@@ -1,11 +1,13 @@
 import { Module } from '@nestjs/common';
+import { ConfigModule } from '@nestjs/config';
 import { RateLimitService } from './rate-limit.service';
 import { RateLimitGuard } from './rate-limit.guard';
+import { RateLimitCleanupWorker } from './rate-limit-cleanup.worker';
 import { PrismaModule } from '../prisma/prisma.module';
 
 @Module({
-  imports: [PrismaModule],
-  providers: [RateLimitService, RateLimitGuard],
+  imports: [PrismaModule, ConfigModule],
+  providers: [RateLimitService, RateLimitGuard, RateLimitCleanupWorker],
   exports: [RateLimitService, RateLimitGuard],
 })
 export class RateLimitModule {}