o11y(assisted-query): Track error outcomes and reasons for AI query analytics

static/app/components/searchQueryBuilder/askSeerCombobox/utils.ts

@@ -6,31 +6,60 @@ import type {
   NoneOfTheseItem,
   QueryTokensProps,
 } from 'sentry/components/searchQueryBuilder/askSeerCombobox/types';
+import {RequestError} from 'sentry/utils/requestError/requestError';
 import {Mode} from 'sentry/views/explore/contexts/pageParamsContext/mode';
 
+function extractErrorReason(err: Error): string {
+  if (err instanceof RequestError) {
+    const detail = err.responseJSON?.detail;
+    if (typeof detail === 'string') {
+      return detail;
+    }
+    if (detail?.message) {
+      return detail.message;
+    }
+  }
+  return err.message;
+}
+
 export function trackAiQueryOutcome({
   dataset,
   mode,
   orgSlug,
   referrer,
   resultCount,
   runId,
+  error = false,
 }: {
   dataset: 'spans' | 'errors' | 'logs' | 'tracemetrics' | 'issues';
   mode: Mode | 'samples' | 'aggregate';
   orgSlug: string;
   referrer: string;
   resultCount: number;
   runId: number;
+  error?: string | boolean | Error;
 }) {
+  const outcome = error
+    ? 'error_on_load'
+    : resultCount > 0
+      ? 'has_results'
+      : 'empty_results';
+  const errorReason =
+    typeof error === 'string'
+      ? error
+      : error instanceof Error
+        ? extractErrorReason(error)
+        : undefined;
   const attributes = {
     dataset,
     mode: mode.toString(),
     org_slug: orgSlug,
     referrer,
     run_id: runId,
-    outcome: resultCount > 0 ? 'has_results' : 'empty_results',
+    outcome,
+    error_reason: errorReason,
   };
+
   Sentry.logger.info('assisted_query.outcome', {
     ...attributes,
     result_count: resultCount,

static/app/views/discover/results.tsx

@@ -371,6 +371,9 @@ export class Results extends Component<Props, State> {
       return;
     }
 
+    const aiQueryRunId = getAiQueryRunId?.() ?? null;
+    const mode = eventView.hasAggregateField() ? 'aggregate' : 'samples';
+
     try {
       const totals = await fetchTotalCount(
         api,
@@ -379,11 +382,10 @@ export class Results extends Component<Props, State> {
       );
       this.setState({totalValues: totals});
 
-      const aiQueryRunId = getAiQueryRunId?.() ?? null;
       if (aiQueryRunId !== null) {
         trackAiQueryOutcome({
           dataset: 'errors',
-          mode: eventView.hasAggregateField() ? 'aggregate' : 'samples',
+          mode,
           referrer: 'errors',
           resultCount: totals,
           orgSlug: organization.slug,
@@ -392,6 +394,17 @@ export class Results extends Component<Props, State> {
       }
     } catch (err) {
       Sentry.captureException(err);
+      if (aiQueryRunId !== null) {
+        trackAiQueryOutcome({
+          dataset: 'errors',
+          mode,
+          referrer: 'errors',
+          resultCount: 0,
+          orgSlug: organization.slug,
+          runId: aiQueryRunId,
+          error: err instanceof Error ? err : true,
+        });
+      }
     }
   }
 

static/app/views/explore/hooks/useAnalytics.tsx

@@ -103,12 +103,13 @@ function useTrackAnalytics({
 
   const tableError =
     queryType === 'aggregate'
-      ? (aggregatesTableResult.result.error?.message ?? '')
+      ? aggregatesTableResult.result.error
       : queryType === 'traces'
-        ? (tracesTableResult?.error?.message ?? '')
-        : (spansTableResult.result.error?.message ?? '');
-  const chartError = timeseriesResult.error?.message ?? '';
-  const query_status = tableError || chartError ? 'error' : 'success';
+        ? tracesTableResult?.error
+        : spansTableResult.result.error;
+  const chartError = timeseriesResult.error;
+  const query_status = tableError?.message || chartError?.message ? 'error' : 'success';
+  const tableErrorBox = useBox(tableError);
 
   const {isLoading: isLoadingSeerSetup} = useOrganizationSeerSetup({
     enabled: !organization.hideAiFeatures,
@@ -172,6 +173,7 @@ function useTrackAnalytics({
         referrer: 'spans',
         resultCount: aggregatesTableResult.result.data?.length ?? 0,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
 
@@ -219,6 +221,7 @@ function useTrackAnalytics({
     query,
     queryType,
     query_status,
+    tableErrorBox,
     timeseriesResult.data,
     timeseriesResult.isPending,
     title,
@@ -304,6 +307,7 @@ function useTrackAnalytics({
         referrer: 'spans',
         resultCount: spansTableResult.result.data?.length ?? 0,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
   }, [
@@ -327,6 +331,7 @@ function useTrackAnalytics({
     spansTableResult.result.data?.length,
     spansTableResult.result.isPending,
     spansTableResult.result.meta?.dataScanned,
+    tableErrorBox,
     timeseriesResult.data,
     timeseriesResult.isPending,
     title,
@@ -495,6 +500,7 @@ function useTrackAnalytics({
         referrer: 'traces',
         resultCount: tracesTableResult.result.data?.json?.data?.length ?? 0,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
   }, [
@@ -513,6 +519,7 @@ function useTrackAnalytics({
     query,
     queryType,
     query_status,
+    tableErrorBox,
     timeseriesResult.data,
     timeseriesResult.isPending,
     title,
@@ -657,8 +664,9 @@ export function useLogAnalytics({
   const fields = useQueryParamsFields();
   const page_source = source;
 
-  const tableError = logsTableResult.error?.message ?? '';
-  const query_status = tableError ? 'error' : 'success';
+  const tableError = logsTableResult.error;
+  const query_status = tableError?.message ? 'error' : 'success';
+  const tableErrorBox = useBox(tableError);
   const autorefreshEnabled = useLogsAutoRefreshEnabled();
   const autorefreshBox = useBox(autorefreshEnabled); // Boxed to avoid useEffect firing analytics on changes.
   const aggregatesResultLengthBox = useBox(
@@ -756,6 +764,7 @@ export function useLogAnalytics({
         referrer: 'logs',
         resultCount: resultLengthBox.current,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
   }, [
@@ -778,6 +787,7 @@ export function useLogAnalytics({
     mode,
     resultLengthBox,
     sortBysBox,
+    tableErrorBox,
     yAxesBox,
     getRunIdForAnalytics,
   ]);
@@ -846,6 +856,7 @@ export function useLogAnalytics({
         referrer: 'logs',
         resultCount: aggregatesResultLengthBox.current,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
   }, [
@@ -868,6 +879,7 @@ export function useLogAnalytics({
     query,
     query_status,
     search,
+    tableErrorBox,
     yAxes,
     getRunIdForAnalytics,
   ]);
@@ -948,9 +960,10 @@ export function useMetricsPanelAnalytics({
 
   const tableError =
     mode === Mode.AGGREGATE
-      ? (metricAggregatesTableResult.result.error?.message ?? '')
-      : (metricSamplesTableResult.error?.message ?? '');
-  const query_status = tableError ? 'error' : 'success';
+      ? metricAggregatesTableResult.result.error
+      : metricSamplesTableResult.error;
+  const query_status = tableError?.message ? 'error' : 'success';
+  const tableErrorBox = useBox(tableError);
 
   const aggregatesResultLengthBox = useBox(
     metricAggregatesTableResult.result.data?.length || 0
@@ -1031,6 +1044,7 @@ export function useMetricsPanelAnalytics({
         referrer: 'tracemetrics',
         resultCount: resultLengthBox.current,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
   }, [
@@ -1055,6 +1069,7 @@ export function useMetricsPanelAnalytics({
     aggregateFunctionBox,
     groupBysBox,
     metricTypeBox,
+    tableErrorBox,
     getRunIdForAnalytics,
   ]);
 
@@ -1086,6 +1101,7 @@ export function useMetricsPanelAnalytics({
         referrer: 'tracemetrics',
         resultCount: aggregatesResultLengthBox.current,
         runId: aiQueryRunId,
+        error: tableErrorBox.current || false,
       });
     }
   }, [
@@ -1107,6 +1123,7 @@ export function useMetricsPanelAnalytics({
     metricNameBox,
     query,
     query_status,
+    tableErrorBox,
     getRunIdForAnalytics,
   ]);
 }