fix: long timeranges for task queries truncated (#1276)

Fixes HDX-2618

Related to ClickHouse/support-escalation#6113

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: knudtty

.changeset/silver-forks-rhyme.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+fix: set a max size for alert timeranges

packages/api/src/tasks/__tests__/util.test.ts

@@ -1,4 +1,5 @@
 import {
+  calcAlertDateRange,
   escapeJsonString,
   roundDownTo,
   roundDownToXMinutes,
@@ -335,4 +336,192 @@ describe('util', () => {
       expect(escapeJsonString('foo\u0000bar')).toBe('foo\\u0000bar');
     });
   });
+
+  describe('calcAlertDateRange', () => {
+    const now = Date.now();
+    const oneMinuteMs = 60 * 1000;
+    const oneHourMs = 60 * oneMinuteMs;
+
+    it('should return unchanged dates when range is within limits', () => {
+      const startTime = now - 10 * oneMinuteMs; // 10 minutes ago
+      const endTime = now;
+      const windowSizeInMins = 5;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      expect(start.getTime()).toBe(startTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should truncate start time when too many windows (> 50)', () => {
+      const windowSizeInMins = 1;
+      const maxWindows = 50;
+      const tooManyWindowsMs =
+        (maxWindows + 10) * windowSizeInMins * oneMinuteMs; // 60 minutes
+      const startTime = now - tooManyWindowsMs;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      // Should truncate to exactly 50 windows
+      const expectedStartTime =
+        endTime - maxWindows * windowSizeInMins * oneMinuteMs;
+      expect(start.getTime()).toBe(expectedStartTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should truncate start time when time range exceeds 6 hours for short windows (< 15 mins)', () => {
+      const windowSizeInMins = 10;
+      const maxLookbackTime = 6 * oneHourMs; // 6 hours for windows < 15 minutes
+      const tooLongRangeMs = maxLookbackTime + oneHourMs; // 7 hours
+      const startTime = now - tooLongRangeMs;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      const expectedStartTime = endTime - maxLookbackTime;
+      expect(start.getTime()).toBeGreaterThan(startTime);
+      expect(start.getTime()).toBe(expectedStartTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should truncate start time when time range exceeds 24 hours for long windows (>= 15 mins)', () => {
+      const windowSizeInMins = 30;
+      const maxLookbackTime = 24 * oneHourMs; // 24 hours for windows >= 15 minutes
+      const tooLongRangeMs = maxLookbackTime + 2 * oneHourMs; // 26 hours
+      const startTime = now - tooLongRangeMs;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      const expectedStartTime = endTime - maxLookbackTime;
+      expect(start.getTime()).toBe(expectedStartTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should apply the more restrictive truncation when both limits are exceeded', () => {
+      const windowSizeInMins = 1;
+      const maxWindows = 50;
+      const maxLookbackTime = 6 * oneHourMs; // 6 hours for 1-minute windows
+
+      // Create a range that exceeds both limits
+      const excessiveRangeMs = Math.max(
+        (maxWindows + 100) * windowSizeInMins * oneMinuteMs, // 150 windows
+        maxLookbackTime + 2 * oneHourMs, // 8 hours
+      );
+      const startTime = now - excessiveRangeMs;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      // Should use the more restrictive limit (maxWindows in this case)
+      const expectedStartTime =
+        endTime - maxWindows * windowSizeInMins * oneMinuteMs;
+      expect(start.getTime()).toBe(expectedStartTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should handle very large window sizes correctly', () => {
+      const windowSizeInMins = 120; // 2 hours
+      const maxLookbackTime = 24 * oneHourMs; // 24 hours for large windows
+      const normalRange = 12 * oneHourMs; // 12 hours - within limit
+      const startTime = now - normalRange;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      // Should remain unchanged since within limits
+      expect(start.getTime()).toBe(startTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should handle exactly 50 windows without truncation', () => {
+      const windowSizeInMins = 5;
+      const maxWindows = 50;
+      const exactlyMaxWindowsMs = maxWindows * windowSizeInMins * oneMinuteMs; // 250 minutes
+      const startTime = now - exactlyMaxWindowsMs;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      // Should remain unchanged since exactly at the limit
+      expect(start.getTime()).toBe(startTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should handle fractional windows correctly', () => {
+      const windowSizeInMins = 7;
+      const partialWindowsMs = 7.5 * windowSizeInMins * oneMinuteMs; // 7.5 windows
+      const startTime = now - partialWindowsMs;
+      const endTime = now;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      // Should remain unchanged since well within limits
+      expect(start.getTime()).toBe(startTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should handle zero time range', () => {
+      const startTime = now;
+      const endTime = now;
+      const windowSizeInMins = 5;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      expect(start.getTime()).toBe(startTime);
+      expect(end.getTime()).toBe(endTime);
+    });
+
+    it('should return Date objects', () => {
+      const startTime = now - 10 * oneMinuteMs;
+      const endTime = now;
+      const windowSizeInMins = 5;
+
+      const [start, end] = calcAlertDateRange(
+        startTime,
+        endTime,
+        windowSizeInMins,
+      );
+
+      expect(start).toBeInstanceOf(Date);
+      expect(end).toBeInstanceOf(Date);
+    });
+  });
 });

packages/api/src/tasks/checkAlerts.ts

@@ -36,7 +36,11 @@ import {
   renderAlertTemplate,
 } from '@/tasks/template';
 import { CheckAlertsTaskArgs, HdxTask } from '@/tasks/types';
-import { roundDownToXMinutes, unflattenObject } from '@/tasks/util';
+import {
+  calcAlertDateRange,
+  roundDownToXMinutes,
+  unflattenObject,
+} from '@/tasks/util';
 import logger from '@/utils/logger';
 
 import { tasksTracer } from './tracer';
@@ -171,18 +175,22 @@ export const processAlert = async (
       );
       return;
     }
-    const checkStartTime = previous
-      ? previous.createdAt
-      : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins);
-    const checkEndTime = nowInMinsRoundDown;
+    const dateRange = calcAlertDateRange(
+      (previous
+        ? previous.createdAt
+        : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins)
+      ).getTime(),
+      nowInMinsRoundDown.getTime(),
+      windowSizeInMins,
+    );
 
     let chartConfig: ChartConfigWithOptDateRange | undefined;
     if (details.taskType === AlertTaskType.SAVED_SEARCH) {
       const savedSearch = details.savedSearch;
       chartConfig = {
         connection: connectionId,
         displayType: DisplayType.Line,
-        dateRange: [checkStartTime, checkEndTime],
+        dateRange,
         dateRangeStartInclusive: true,
         dateRangeEndInclusive: false,
         from: source.from,
@@ -206,7 +214,7 @@ export const processAlert = async (
       if (tile.config.displayType === DisplayType.Line) {
         chartConfig = {
           connection: connectionId,
-          dateRange: [checkStartTime, checkEndTime],
+          dateRange,
           dateRangeStartInclusive: true,
           dateRangeEndInclusive: false,
           displayType: tile.config.displayType,
@@ -252,9 +260,10 @@ export const processAlert = async (
     logger.info(
       {
         alertId: alert.id,
+        chartConfig,
         checksData,
-        checkStartTime,
-        checkEndTime,
+        checkStartTime: dateRange[0],
+        checkEndTime: dateRange[1],
       },
       `Received alert metric [${alert.source} source]`,
     );

packages/api/src/tasks/util.ts

@@ -1,5 +1,7 @@
 import { set } from 'lodash';
 
+import logger from '@/utils/logger';
+
 // transfer keys of attributes with dot into nested object
 // ex: { 'a.b': 'c', 'd.e.f': 'g' } -> { a: { b: 'c' }, d: { e: { f: 'g' } } }
 export const unflattenObject = (
@@ -38,3 +40,46 @@ export const roundDownToXMinutes = (x: number) => roundDownTo(1000 * 60 * x);
 export const escapeJsonString = (str: string) => {
   return JSON.stringify(str).slice(1, -1);
 };
+
+const MAX_NUM_WINDOWS = 50;
+const maxLookbackTime = (windowSizeInMins: number) =>
+  3600_000 * (windowSizeInMins < 15 ? 6 : 24);
+export function calcAlertDateRange(
+  _startTime: number,
+  _endTime: number,
+  windowSizeInMins: number,
+): [Date, Date] {
+  let startTime = _startTime;
+  const endTime = _endTime;
+  const numWindows = (endTime - startTime) / 60_000 / windowSizeInMins;
+  // Truncate if too many windows are present
+  if (numWindows > MAX_NUM_WINDOWS) {
+    startTime = endTime - MAX_NUM_WINDOWS * 1000 * 60 * windowSizeInMins;
+    logger.info(
+      {
+        requestedStartTime: _startTime,
+        startTime,
+        endTime,
+        windowSizeInMins,
+        numWindows,
+      },
+      'startTime truncated due to too many windows',
+    );
+  }
+  // Truncate if time range is over threshold
+  const MAX_LOOKBACK_TIME = maxLookbackTime(windowSizeInMins);
+  if (endTime - startTime > MAX_LOOKBACK_TIME) {
+    startTime = endTime - MAX_LOOKBACK_TIME;
+    logger.info(
+      {
+        requestedStartTime: _startTime,
+        startTime,
+        endTime,
+        windowSizeInMins,
+        numWindows,
+      },
+      'startTime truncated due to long lookback time',
+    );
+  }
+  return [new Date(startTime), new Date(endTime)];
+}