[PLAT-12758] Undo 1 minute PSS retrieval + fix graph start/end times

Summary: I accidentally changed pg_stat_statement retrieval period from 10 mins to 1 min. Undid that. Also, I've made cahnges in graph start/end time calculation logic to be more reasonable for short anomalies. Test Plan: unit tested Reviewers: cdavid, rmadhavan Reviewed By: rmadhavan Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D33083
yugabyte · Mar 12, 2024 · 097bf94 · 097bf94
1 parent be35691
commit 097bf94
Show file tree

Hide file tree

Showing 5 changed files with 240 additions and 1,256 deletions.
diff --git a/...ckend/src/main/java/com/yugabyte/troubleshoot/ts/service/anomaly/AnomalyDetectorBase.java b/...ckend/src/main/java/com/yugabyte/troubleshoot/ts/service/anomaly/AnomalyDetectorBase.java
@@ -32,28 +32,33 @@ protected AnomalyDetectorBase(
   }
 
   protected Pair<Instant, Instant> calculateGraphStartEndTime(
-      Instant detectionStartTime,
-      Instant detectionEndTime,
-      Instant anomalyStartTime,
-      Instant anomalyEndTime) {
-    long startTime = detectionStartTime.getEpochSecond();
-    long endTime = detectionEndTime.getEpochSecond();
+      AnomalyDetectionContext context, Instant anomalyStartTime, Instant anomalyEndTime) {
+    long startTime = context.getStartTime().getEpochSecond();
+    long endTime = context.getEndTime().getEpochSecond();
 
     long effectiveAnomalyStartTime =
         anomalyStartTime != null ? anomalyStartTime.getEpochSecond() : startTime;
     long effectiveAnomalyEndTime =
         anomalyEndTime != null ? anomalyEndTime.getEpochSecond() : endTime;
     long anomalyMiddle = (effectiveAnomalyStartTime + effectiveAnomalyEndTime) / 2;
     long anomalySize = effectiveAnomalyEndTime - effectiveAnomalyStartTime;
-    long anomalyBasedStartTime = anomalyMiddle - anomalySize * 2;
-    long anomalyBasedEndTime = anomalyMiddle + anomalySize * 2;
+    long anomalySizeBasedStartTime = anomalyMiddle - anomalySize * 2;
+    long anomalySizeBasedEndTime = anomalyMiddle + anomalySize * 2;
 
-    if (anomalyBasedStartTime > startTime) {
-      startTime = anomalyBasedStartTime;
+    // 20 points before and after the anomaly should be good enough
+    long stepBasedStartTime = anomalyMiddle - context.getStepSeconds() * 20;
+    long stepBasedEndTime = anomalyMiddle + context.getStepSeconds() * 20;
+
+    long desiredStartTime = Math.min(anomalySizeBasedStartTime, stepBasedStartTime);
+    long desiredEndTime = Math.max(anomalySizeBasedEndTime, stepBasedEndTime);
+
+    if (desiredStartTime > startTime) {
+      startTime = desiredStartTime;
     }
-    if (anomalyBasedEndTime < endTime) {
-      endTime = anomalyBasedEndTime;
+    if (desiredEndTime < endTime) {
+      endTime = desiredEndTime;
     }
+
     return ImmutablePair.of(Instant.ofEpochSecond(startTime), Instant.ofEpochSecond(endTime));
   }
 
@@ -156,8 +161,7 @@ protected void createAnomalies(
                   ? Instant.ofEpochMilli(graphAnomaly.getEndTime())
                   : null;
           Pair<Instant, Instant> graphStartEndTime =
-              calculateGraphStartEndTime(
-                  context.getStartTime(), context.getEndTime(), anomalyStartTime, anomalyEndTime);
+              calculateGraphStartEndTime(context, anomalyStartTime, anomalyEndTime);
           Anomaly.AnomalyBuilder anomalyBuilder =
               Anomaly.builder()
                   .uuid(UUID.randomUUID())

diff --git a/troubleshoot/backend/src/main/resources/application.properties b/troubleshoot/backend/src/main/resources/application.properties
@@ -21,9 +21,9 @@ management.metrics.distribution.percentiles-histogram.http.server.requests=true
 management.endpoints.web.exposure.include=health,prometheus,info,flyway,configprops
 
 # Scheduler configuration
-task.universe_details_query.period=PT1M
+task.universe_details_query.period=PT10M
 task.universe_details_query.threads=2
-task.pg_stat_statements_query.period=PT1M
+task.pg_stat_statements_query.period=PT10M
 task.pg_stat_statements_query.threads=5
 task.pg_stat_statements_nodes_query.threads=20
 task.metric_query.threads=10

diff --git a/troubleshoot/backend/src/test/resources/anomaly/query_latency/anomalies.json b/troubleshoot/backend/src/test/resources/anomaly/query_latency/anomalies.json
@@ -230,11 +230,11 @@
     "affectedTables": null,
     "summary": "Latencies increased for query 'select * from some_table' in database 'yugabyte'",
     "detectionTime": null,
-    "startTime": "2024-01-18T15:40:48Z",
-    "endTime": "2024-01-18T16:31:12Z",
+    "startTime": "2024-01-18T15:40:00Z",
+    "endTime": "2024-01-18T16:40:00Z",
     "graphStartTime": "2024-01-18T15:00:00Z",
-    "graphEndTime": "2024-01-18T17:46:48Z",
-    "graphStepSeconds": 144
+    "graphEndTime": "2024-01-18T19:00:00Z",
+    "graphStepSeconds": 1200
   },
   {
     "uuid": null,
@@ -463,10 +463,10 @@
     "affectedTables": null,
     "summary": "Latencies increased for query 'select * from some_table' in database 'postgres'",
     "detectionTime": null,
-    "startTime": "2024-01-18T16:21:36Z",
-    "endTime": "2024-01-18T17:21:36Z",
+    "startTime": "2024-01-18T16:20:00Z",
+    "endTime": "2024-01-18T17:20:00Z",
     "graphStartTime": "2024-01-18T15:00:00Z",
-    "graphEndTime": "2024-01-18T18:51:36Z",
-    "graphStepSeconds": 144
+    "graphEndTime": "2024-01-18T19:00:00Z",
+    "graphStepSeconds": 1200
   }
 ]