diff --git a/metrics/alerts.yml b/metrics/alerts.yml new file mode 100644 index 0000000..6343925 --- /dev/null +++ b/metrics/alerts.yml @@ -0,0 +1,133 @@ +groups: + - name: stellar-router-alerts + interval: 30s + rules: + # Circuit Breaker Alerts + - alert: CircuitBreakerOpen + expr: router_middleware_circuit_open == 1 + for: 1m + labels: + severity: warning + component: middleware + annotations: + summary: "Circuit breaker open for route {{ $labels.route }}" + description: "Circuit breaker has been open on route {{ $labels.route }} for more than 1 minute. This suggests repeated failures in the routed contract." + + - alert: CircuitBreakerStuck + expr: router_middleware_circuit_open == 1 + for: 5m + labels: + severity: critical + component: middleware + annotations: + summary: "Circuit breaker stuck open on route {{ $labels.route }}" + description: "Circuit breaker has been open on route {{ $labels.route }} for more than 5 minutes. Manual intervention may be required to reset the circuit." + + # Failure Rate Alerts + - alert: HighConsecutiveFailures + expr: router_middleware_failure_count > 10 + for: 2m + labels: + severity: warning + component: middleware + annotations: + summary: "High consecutive failures on route {{ $labels.route }}" + description: "Route {{ $labels.route }} has {{ $value }} consecutive failures. Circuit breaker may trigger soon." + + - alert: CriticalFailureCount + expr: router_middleware_failure_count > 20 + for: 1m + labels: + severity: critical + component: middleware + annotations: + summary: "Critical failure count on route {{ $labels.route }}" + description: "Route {{ $labels.route }} has {{ $value }} consecutive failures. Immediate investigation required." + + # Route Pause Alerts + - alert: RoutePaused + expr: router_core_route_paused == 1 + for: 5m + labels: + severity: warning + component: core + annotations: + summary: "Route {{ $labels.route }} is paused" + description: "Route {{ $labels.route }} has been paused for more than 5 minutes. Clients cannot access this route." + + - alert: GlobalRouterPaused + expr: router_core_paused == 1 + for: 1m + labels: + severity: critical + component: core + annotations: + summary: "Global router pause activated" + description: "The stellar-router has been globally paused. All routing is blocked. This may indicate an incident or maintenance." + + # Rate Limit Alerts + - alert: RateLimitApproaching + expr: | + (rate(router_middleware_total_calls[5m]) / 60) > 0.8 * + scalar(count(router_middleware_total_calls) by (route)) + for: 2m + labels: + severity: warning + component: middleware + annotations: + summary: "Rate limit approaching for route {{ $labels.route }}" + description: "Route {{ $labels.route }} is at 80% of its configured rate limit. Requests may start being rejected soon." + + # Scraper Health Alerts + - alert: MetricsScraperFailing + expr: increase(router_scrape_errors_total[5m]) > 5 + for: 2m + labels: + severity: warning + component: exporter + annotations: + summary: "Metrics scraper failing for contract {{ $labels.contract }}" + description: "Contract {{ $labels.contract }} scrape has failed {{ $value }} times in the last 5 minutes." + + - alert: MetricsExporterDown + expr: router_up == 0 + for: 1m + labels: + severity: critical + component: exporter + annotations: + summary: "Metrics exporter is down" + description: "The stellar-router metrics exporter has failed to scrape contract state. Metrics are stale." + + # Scrape Duration Alerts + - alert: HighScrapeDuration + expr: histogram_quantile(0.95, router_scrape_duration_seconds_bucket) > 5 + for: 2m + labels: + severity: warning + component: exporter + annotations: + summary: "High p95 scrape latency for contract {{ $labels.contract }}" + description: "P95 scrape latency for {{ $labels.contract }} is {{ $value }}s. RPC endpoint may be slow or overloaded." + + # No Activity Alerts + - alert: NoRoutingActivity + expr: increase(router_core_total_routed[10m]) == 0 + for: 10m + labels: + severity: info + component: core + annotations: + summary: "No routing activity detected" + description: "No routes have been resolved in the last 10 minutes. This may indicate a network issue or lack of traffic." + + # Registry Alerts + - alert: RegistryEmpty + expr: router_registry_total_names == 0 + for: 5m + labels: + severity: warning + component: registry + annotations: + summary: "Registry contains no entries" + description: "The router-registry is empty. No routes can be registered or resolved."