Add Prometheus, Grafana, and Alertmanager monitoring stack

apps/api/package.json

@@ -26,10 +26,11 @@
     "@nestjs/platform-express": "^11.0.1",
     "@nestjs/terminus": "^11.1.1",
     "@nestjs/throttler": "^6.5.0",
+    "@nestjs/swagger": "^11.2.6",
     "@stellar/stellar-sdk": "^14.6.1",
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",
-    "@nestjs/swagger": "^11.2.6",
+    "prom-client": "^15.1.3",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",
     "swagger-ui-express": "^5.0.1"
@@ -77,4 +78,4 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   }
-}
+}

apps/api/src/app.module.ts

@@ -4,6 +4,7 @@ import { ThrottlerModule } from '@nestjs/throttler';
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
 import { HealthModule } from './health/health.module';
+import { MetricsModule } from './metrics/metrics.module';
 import { TreasuryModule } from './treasury/treasury.module';
 import { AuthModule } from './auth/auth.module';
 import { JwtAuthGuard } from './auth/guards/jwt-auth.guard';
@@ -12,6 +13,7 @@ import { ThrottlerRedisGuard } from './rate-limiter/guards/throttler-redis.guard
 @Module({
   imports: [
     HealthModule,
+    MetricsModule,
     TreasuryModule,
     AuthModule,
     ThrottlerModule.forRoot({

apps/api/src/metrics/http-metrics.interceptor.ts

@@ -0,0 +1,57 @@
+import {
+  Injectable,
+  NestInterceptor,
+  ExecutionContext,
+  CallHandler,
+} from '@nestjs/common';
+import type { Request, Response } from 'express';
+import { Observable } from 'rxjs';
+import { MetricsService } from './metrics.service';
+
+@Injectable()
+export class HttpMetricsInterceptor implements NestInterceptor {
+  constructor(private readonly metrics: MetricsService) {}
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
+    if (context.getType() !== 'http') {
+      return next.handle();
+    }
+
+    const req = context.switchToHttp().getRequest<Request>();
+    const res = context.switchToHttp().getResponse<Response>();
+    const start = process.hrtime.bigint();
+
+    let recorded = false;
+    const record = () => {
+      if (recorded) {
+        return;
+      }
+      recorded = true;
+      res.removeListener('finish', record);
+      res.removeListener('close', record);
+
+      const durationSec = Number(process.hrtime.bigint() - start) / 1e9;
+      const route =
+        typeof req.route === 'object' && req.route && 'path' in req.route
+          ? `${req.baseUrl ?? ''}${String(req.route.path)}`
+          : (req.path ?? 'unknown');
+      const method = req.method ?? 'unknown';
+      const status = res.statusCode ?? 0;
+      const statusClass = httpStatusClass(status);
+
+      this.metrics.observeHttpRequest(method, route, statusClass, durationSec);
+    };
+
+    res.once('finish', record);
+    res.once('close', record);
+
+    return next.handle();
+  }
+}
+
+function httpStatusClass(code: number): string {
+  if (code >= 100 && code < 600) {
+    return `${Math.floor(code / 100)}xx`;
+  }
+  return 'unknown';
+}

apps/api/src/metrics/metrics.controller.ts

@@ -0,0 +1,18 @@
+import { Controller, Get, Res } from '@nestjs/common';
+import { SkipThrottle } from '@nestjs/throttler';
+import type { Response } from 'express';
+import { Public } from '../auth/decorators/public.decorator';
+import { MetricsService } from './metrics.service';
+
+@Controller('metrics')
+@SkipThrottle()
+export class MetricsController {
+  constructor(private readonly metrics: MetricsService) {}
+
+  @Get()
+  @Public()
+  async scrape(@Res({ passthrough: true }) res: Response): Promise<string> {
+    res.setHeader('Content-Type', this.metrics.metricsContentType);
+    return this.metrics.getMetrics();
+  }
+}

apps/api/src/metrics/metrics.module.ts

@@ -0,0 +1,18 @@
+import { Module } from '@nestjs/common';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { HttpMetricsInterceptor } from './http-metrics.interceptor';
+import { MetricsController } from './metrics.controller';
+import { MetricsService } from './metrics.service';
+
+@Module({
+  controllers: [MetricsController],
+  providers: [
+    MetricsService,
+    {
+      provide: APP_INTERCEPTOR,
+      useClass: HttpMetricsInterceptor,
+    },
+  ],
+  exports: [MetricsService],
+})
+export class MetricsModule {}

apps/api/src/metrics/metrics.service.ts

@@ -0,0 +1,51 @@
+import { Injectable } from '@nestjs/common';
+import * as client from 'prom-client';
+
+@Injectable()
+export class MetricsService {
+  private readonly registry: client.Registry;
+  private readonly httpRequestDuration: client.Histogram<string>;
+  private readonly httpRequestsTotal: client.Counter<string>;
+
+  constructor() {
+    this.registry = new client.Registry();
+    client.collectDefaultMetrics({ register: this.registry });
+
+    this.httpRequestDuration = new client.Histogram({
+      name: 'http_request_duration_seconds',
+      help: 'HTTP request latency in seconds',
+      labelNames: ['method', 'route', 'status_class'],
+      buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+      registers: [this.registry],
+    });
+
+    this.httpRequestsTotal = new client.Counter({
+      name: 'http_requests_total',
+      help: 'Total HTTP requests',
+      labelNames: ['method', 'route', 'status_class'],
+      registers: [this.registry],
+    });
+  }
+
+  get metricsContentType(): string {
+    return (
+      (this.registry as client.Registry & { contentType?: string }).contentType ??
+      'text/plain; version=0.0.4; charset=utf-8'
+    );
+  }
+
+  observeHttpRequest(
+    method: string,
+    route: string,
+    statusClass: string,
+    durationSec: number,
+  ): void {
+    const labels = { method, route, status_class: statusClass };
+    this.httpRequestDuration.observe(labels, durationSec);
+    this.httpRequestsTotal.inc(labels);
+  }
+
+  async getMetrics(): Promise<string> {
+    return this.registry.metrics();
+  }
+}

monitoring/README.md

@@ -0,0 +1,63 @@
+# Monitoring (Prometheus + Grafana + Alertmanager)
+
+## What this stack does
+
+- **API**: scrapes `GET /metrics` on the NestJS service (`prom-client` histograms and counters).
+- **Web apps**: scrapes synthetic uptime/latency via the **blackbox exporter** (HTTP probes).
+- **Dashboards**: Grafana imports `grafana/dashboards/stellar-pay-overview.json` automatically.
+- **Alerts**: Prometheus evaluates rules in `prometheus/alerts/` and sends notifications to Alertmanager.
+
+## Ports (defaults in `prometheus/prometheus.yml`)
+
+| Service        | Host port | Notes |
+|----------------|-----------|--------|
+| API            | 3000      | `METRICS` at `/metrics` |
+| Frontend       | 3001      | set `PORT=3001` for `next dev` / `next start` |
+| Admin dashboard | 3002     | set `PORT=3002` |
+| Prometheus     | 9090      | |
+| Alertmanager   | 9093      | |
+| Blackbox       | 9115      | optional; used internally by Prometheus relabel |
+| Grafana        | 3003      | map container 3000 → host 3003 |
+
+Adjust targets in `prometheus/prometheus.yml` if your local ports differ.
+
+## Run the monitoring stack
+
+From the repository root (requires Docker):
+
+```bash
+docker compose -f monitoring/docker-compose.yml up -d
+```
+
+Then open **Grafana**: `http://localhost:3003` (default login `admin` / `admin`).
+
+- **Metrics**: open the **Stellar Pay — Overview** dashboard (throughput, 5xx ratio, p95 latency, blackbox probes).
+- **Alerts in Grafana**: the dashboard includes a **Firing alert rules** table (Prometheus `ALERTS` metric). You can also open the pre-provisioned **Alertmanager** datasource under **Connections → Data sources**, or use Alertmanager’s UI at `http://localhost:9093`.
+
+## Run the application services locally
+
+Examples:
+
+```bash
+# API (default PORT 3000)
+pnpm --filter api start:dev
+
+# Frontend on 3001 (POSIX)
+cd apps/frontend && PORT=3001 pnpm dev
+
+# Admin on 3002 (POSIX)
+cd apps/admin-dashboard && PORT=3002 pnpm dev
+```
+
+PowerShell:
+
+```powershell
+pnpm --filter api start:dev
+cd apps/frontend; $env:PORT = 3001; pnpm dev
+cd apps/admin-dashboard; $env:PORT = 3002; pnpm dev
+```
+
+## Test that Prometheus sees data
+
+- Prometheus targets: `http://localhost:9090/targets`
+- API metrics: `http://localhost:3000/metrics`

monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,11 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'null'
+  group_wait: 10s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: 'null'

monitoring/blackbox/blackbox.yml

@@ -0,0 +1,9 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ['HTTP/1.1', 'HTTP/2.0']
+      method: GET
+      follow_redirects: true
+      preferred_ip_protocol: ip4

monitoring/docker-compose.yml

@@ -0,0 +1,53 @@
+services:
+  prometheus:
+    image: prom/prometheus:v2.53.1
+    ports:
+      - '9090:9090'
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/alerts:/etc/prometheus/alerts:ro
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --web.enable-lifecycle
+    depends_on:
+      - alertmanager
+      - blackbox-exporter
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    ports:
+      - '9093:9093'
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.25.0
+    ports:
+      - '9115:9115'
+    volumes:
+      - ./blackbox/blackbox.yml:/config/blackbox.yml:ro
+    command:
+      - --config.file=/config/blackbox.yml
+
+  grafana:
+    image: grafana/grafana:11.3.0
+    ports:
+      - '3003:3000'
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_USERS_ALLOW_SIGN_UP: 'false'
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - prometheus
+
+networks:
+  default:
+    name: stellar-pay-monitoring