diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 00000000..9e0f560a --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,97 @@ +# Logging & Monitoring + +This document describes the structured logging format, the metrics the service +exports, the health-check endpoints, and the alerting thresholds (#367). + +## Structured logging + +Logs are emitted as structured JSON via the pino-based Nest logger +(`src/config/nest-pino-logger.ts`, `src/config/logger.ts`); request/response +logging is applied by `src/common/middleware/logging.middleware.ts` and errors +are captured by `src/common/filters/global-exception.filter.ts`. + +Every log line includes at least: + +| Field | Description | +| ----------- | ------------------------------------------------------- | +| `level` | Severity (`trace`/`debug`/`info`/`warn`/`error`/`fatal`)| +| `time` | ISO-8601 / epoch-ms timestamp | +| `requestId` | Correlation ID for the request (propagated per request) | +| `userId` | Authenticated user id when present | +| `method` | HTTP method | +| `url` | Request path | +| `statusCode`| Response status | +| `msg` | Human-readable message | + +Because the output is JSON, it can be shipped directly to Loki, ELK, or any +JSON log pipeline without a custom parser. + +## Metrics + +Prometheus metrics are exposed (unauthenticated, for in-cluster scraping) at: + +``` +GET /metrics # text/plain; version=0.0.4 +``` + +Metric definitions live in `src/config/metrics.ts` and are recorded by +`src/metrics/metrics.interceptor.ts`. Default Node/process metrics are exported +with the `stellaiverse_` prefix. + +| Metric | Type | Labels | Meaning | +| -------------------------------------------------- | --------- | ----------------------------- | ------------------------------------ | +| `stellaiverse_http_requests_total` | counter | method, route, status_code | Total HTTP requests | +| `stellaiverse_http_errors_total` | counter | method, route, status_code | Total error responses | +| `stellaiverse_http_request_duration_seconds` | histogram | method, route, status_code | Request latency (SLI) | +| `stellaiverse_http_requests_in_progress` | gauge | method, route | In-flight requests (saturation) | +| `stellaiverse_database_query_duration_seconds` | histogram | operation, table | DB query latency | +| `stellaiverse_active_connections` | gauge | type | Active connections | +| `stellaiverse_auth_attempts_total` | counter | method | Authentication attempts | +| `stellaiverse_auth_success_total` | counter | method | Successful authentications | +| `stellaiverse_auth_failures_total` | counter | method, reason | Failed authentications | + +A ready-to-import Grafana dashboard is provided at +`monitoring/grafana/stellaiverse-backend-dashboard.json`. + +## Health checks + +Terminus-backed endpoints (all public, no auth), suitable for orchestrator +probes: + +| Endpoint | Purpose | Use as | +| ------------------- | ---------- | -------------------------- | +| `GET /health` | Full check | Manual / dashboards | +| `GET /health/live` | Liveness | Kubernetes `livenessProbe` | +| `GET /health/ready` | Readiness | Kubernetes `readinessProbe`| + +- **Liveness** performs no dependency checks — it only confirms the process can + serve HTTP, so a transient downstream outage does not cause a pod restart. +- **Readiness** verifies critical dependencies; the orchestrator routes traffic + to the pod only while it returns `200`. + +Example Kubernetes probe configuration: + +```yaml +livenessProbe: + httpGet: { path: /health/live, port: 3000 } + initialDelaySeconds: 10 + periodSeconds: 10 +readinessProbe: + httpGet: { path: /health/ready, port: 3000 } + initialDelaySeconds: 5 + periodSeconds: 5 +``` + +## SLOs and alerting thresholds + +Example Prometheus rules live in `monitoring/prometheus/alerts.yml`. + +| SLO / signal | Threshold | Severity | Alert | +| ------------------- | -------------------------------------- | -------- | ---------------------------- | +| Availability | error rate > 5% for 5m | critical | `HighHttpErrorRate` | +| Target up | `up == 0` for 1m | critical | `TargetDown` | +| Latency | p95 request latency > 1s for 10m | warning | `HighRequestLatencyP95` | +| DB latency | p95 query latency > 500ms for 10m | warning | `HighDatabaseQueryLatencyP95`| +| Saturation | in-flight requests > 100 for 5m | warning | `RequestQueueBacklog` | +| Event-loop lag | lag > 200ms for 5m | warning | `EventLoopLagHigh` | +| Auth failures | > 5 failures/s for 5m | warning | `AuthFailureSpike` | diff --git a/monitoring/grafana/stellaiverse-backend-dashboard.json b/monitoring/grafana/stellaiverse-backend-dashboard.json new file mode 100644 index 00000000..2d9af927 --- /dev/null +++ b/monitoring/grafana/stellaiverse-backend-dashboard.json @@ -0,0 +1,108 @@ +{ + "title": "StellAIverse Backend — Service Overview", + "uid": "stellaiverse-backend", + "schemaVersion": 39, + "editable": true, + "tags": ["stellaiverse", "backend", "slo"], + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": {} + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Request rate (req/s)", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(stellaiverse_http_requests_total[5m])) by (status_code)", + "legendFormat": "{{status_code}}" + } + ] + }, + { + "id": 2, + "title": "Error rate (% of requests)", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "targets": [ + { + "expr": "sum(rate(stellaiverse_http_errors_total[5m])) / clamp_min(sum(rate(stellaiverse_http_requests_total[5m])), 1)", + "legendFormat": "error ratio" + } + ] + }, + { + "id": 3, + "title": "Latency p50 / p95 / p99 (s)", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(stellaiverse_http_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(stellaiverse_http_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(stellaiverse_http_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "In-flight requests (saturation)", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "sum(stellaiverse_http_requests_in_progress)", + "legendFormat": "in progress" + } + ] + }, + { + "id": 5, + "title": "DB query p95 latency (s)", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(stellaiverse_database_query_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "db p95" + } + ] + }, + { + "id": 6, + "title": "Auth failures (per second)", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "targets": [ + { + "expr": "sum(rate(stellaiverse_auth_failures_total[5m])) by (reason)", + "legendFormat": "{{reason}}" + } + ] + } + ] +} diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 00000000..bba555b1 --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,93 @@ +# Prometheus alerting rules for StellAIverse backend (#367). +# +# SLO/SLI-driven alerts over the metrics exported at GET /metrics +# (see src/config/metrics.ts). Load into Prometheus via `rule_files:` and wire +# the `severity` labels to your Alertmanager routes. +# +# Target SLOs: +# - Availability : >= 99.9% of HTTP requests are non-5xx (error budget 0.1%). +# - Latency : 95th-percentile request latency < 1s. +groups: + - name: stellaiverse-availability + rules: + - alert: TargetDown + expr: up{job="stellaiverse-backend"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "StellAIverse backend target is down" + description: "Prometheus cannot scrape {{ $labels.instance }} (up == 0) for >1m." + + - alert: HighHttpErrorRate + # SLI: fraction of responses that are errors, over a 5m window. + expr: | + sum(rate(stellaiverse_http_errors_total[5m])) + / clamp_min(sum(rate(stellaiverse_http_requests_total[5m])), 1) + > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "HTTP error rate above 5% SLO" + description: "5xx/error responses exceed the 5% error-rate SLO for 5m (current: {{ printf \"%.2f\" $value }})." + + - name: stellaiverse-latency + rules: + - alert: HighRequestLatencyP95 + expr: | + histogram_quantile( + 0.95, + sum(rate(stellaiverse_http_request_duration_seconds_bucket[5m])) by (le) + ) > 1 + for: 10m + labels: + severity: warning + annotations: + summary: "p95 request latency above 1s SLO" + description: "95th-percentile HTTP latency has exceeded 1s for 10m ({{ printf \"%.2f\" $value }}s)." + + - alert: HighDatabaseQueryLatencyP95 + expr: | + histogram_quantile( + 0.95, + sum(rate(stellaiverse_database_query_duration_seconds_bucket[5m])) by (le) + ) > 0.5 + for: 10m + labels: + severity: warning + annotations: + summary: "p95 database query latency above 500ms" + description: "95th-percentile DB query latency has exceeded 500ms for 10m ({{ printf \"%.3f\" $value }}s)." + + - name: stellaiverse-saturation + rules: + - alert: RequestQueueBacklog + # In-flight requests are a proxy for queue length / saturation. + expr: sum(stellaiverse_http_requests_in_progress) > 100 + for: 5m + labels: + severity: warning + annotations: + summary: "Large in-flight request backlog" + description: "More than 100 HTTP requests have been in progress for 5m ({{ $value }})." + + - alert: EventLoopLagHigh + expr: stellaiverse_nodejs_eventloop_lag_seconds > 0.2 + for: 5m + labels: + severity: warning + annotations: + summary: "Node.js event-loop lag is high" + description: "Event-loop lag above 200ms for 5m ({{ printf \"%.3f\" $value }}s) — the process is CPU-saturated." + + - name: stellaiverse-security + rules: + - alert: AuthFailureSpike + expr: sum(rate(stellaiverse_auth_failures_total[5m])) > 5 + for: 5m + labels: + severity: warning + annotations: + summary: "Elevated authentication failure rate" + description: "More than 5 auth failures/sec over 5m ({{ printf \"%.2f\" $value }}/s) — possible credential-stuffing." diff --git a/src/app.controller.ts b/src/app.controller.ts index ab0ddf83..ba7cd1b8 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -52,6 +52,42 @@ export class AppController { return this.health.check([() => this.riskManagementHealth.isHealthy()]); } + @Public() + @Get("health/live") + @HealthCheck() + @ApiOperation({ + summary: "Liveness Probe", + description: + "Orchestrator liveness probe. Returns 200 while the process is up and " + + "able to serve requests. It performs no dependency checks, so a transient " + + "downstream outage does not cause the pod to be restarted.", + operationId: "getLiveness", + }) + @ApiResponse({ status: 200, description: "Process is alive" }) + checkLiveness() { + // No indicators: a passing response simply means the event loop is + // responsive and the app can serve HTTP — the correct semantics for + // Kubernetes `livenessProbe`. + return this.health.check([]); + } + + @Public() + @Get("health/ready") + @HealthCheck() + @ApiOperation({ + summary: "Readiness Probe", + description: + "Orchestrator readiness probe. Returns 200 only when the service and its " + + "critical dependencies are ready to receive traffic; the orchestrator " + + "routes traffic to the pod only while this check passes.", + operationId: "getReadiness", + }) + @ApiResponse({ status: 200, description: "Service is ready for traffic" }) + @ApiResponse({ status: 503, description: "Service is not ready" }) + checkReadiness() { + return this.health.check([() => this.riskManagementHealth.isHealthy()]); + } + @Public() @Get("info") @RateLimit({ level: "standard" }) // Default standard level