diff --git a/production/loki-mixin/alerts.libsonnet b/production/loki-mixin/alerts.libsonnet index 0d34086d9b3a5..3adc04b7033e3 100644 --- a/production/loki-mixin/alerts.libsonnet +++ b/production/loki-mixin/alerts.libsonnet @@ -41,14 +41,14 @@ { alert: 'LokiRequestLatency', expr: ||| - %(group_prefix_jobs)s_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*|/schedulerpb.SchedulerForQuerier/QuerierLoop"} > 1 + %(group_prefix_jobs)s_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*|/schedulerpb.SchedulerForQuerier/QuerierLoop"} > %(loki_p99_request_latency_threshold_seconds)s ||| % $._config, 'for': '15m', labels: { severity: 'critical', }, annotations: { - summary: 'Loki request error latency is high.', + summary: 'Loki request latency is high.', description: std.strReplace(||| {{ $labels.cluster }} {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. |||, 'cluster', $._config.per_cluster_label), diff --git a/production/loki-mixin/config.libsonnet b/production/loki-mixin/config.libsonnet index 3624e55ad64ab..88cf70b5f50f6 100644 --- a/production/loki-mixin/config.libsonnet +++ b/production/loki-mixin/config.libsonnet @@ -93,5 +93,8 @@ meta_monitoring: { enabled: false, }, + + //Alert thresholds + loki_p99_request_latency_threshold_seconds: 1, }, }