test(search-events): evals for attribute fetching with context and mocks (#1361)

This PR doesn't add any new functionality to the MCP; It focuses solely
on building evals that we want to expand on.

For context: we've got an option on the `/attributes` endpoint that
provides us additional context about the state of some of these
attributes; like if they're deprecated, aliased, sentry-defined, etc. We
want to eventually pull in that information and use it so the mcp can
make better queries.

The purpose of this PR is to make new evals to test the query quality
with and without this context. I've made new mocks for the attributes
endpoint call with the context. Since i've made no code changes to
correctly digest the additional context, i've made evals for it that
i've made pass for now but will change as we put in the context
digestion.

Contributes to EXP-1186

Author: nikkikapadia

packages/mcp-core/src/api-client/client.test.ts

@@ -1978,6 +1978,39 @@ describe("API query builders", () => {
       expect(params.get("substringMatch")).toBe("tags[");
       expect(params.get("query")).toBe('transaction:"VPN connections"');
       expect(params.get("attributeType")).toBeNull();
+      expect(params.get("expand")).toBeNull();
+    });
+
+    it("should request attribute context when context is enabled", async () => {
+      const apiService = new SentryApiService({
+        host: "sentry.io",
+        accessToken: "test-token",
+      });
+      const urls: string[] = [];
+
+      globalThis.fetch = vi.fn().mockImplementation((url: string) => {
+        urls.push(url);
+
+        return Promise.resolve({
+          ok: true,
+          headers: {
+            get: (key: string) =>
+              key === "content-type" ? "application/json" : null,
+          },
+          json: () => Promise.resolve([]),
+        });
+      });
+
+      await apiService.listTraceItemAttributes({
+        organizationSlug: "test-org",
+        itemType: "spans",
+        context: true,
+      });
+
+      expect(urls).toHaveLength(1);
+      const params = new URL(urls[0]!).searchParams;
+      expect(params.get("itemType")).toBe("spans");
+      expect(params.get("expand")).toBe("context");
     });
 
     it("should validate events requests via the validate endpoint", async () => {

packages/mcp-core/src/api-client/client.ts

@@ -3698,6 +3698,7 @@ export class SentryApiService {
       attributeTypes,
       substringMatch,
       query,
+      context,
     }: {
       organizationSlug: string;
       itemType?: TraceItemType;
@@ -3708,6 +3709,7 @@ export class SentryApiService {
       attributeTypes?: TraceItemAttributeType[];
       substringMatch?: string;
       query?: string;
+      context?: boolean;
     },
     opts?: RequestOptions,
   ): Promise<TraceItemAttribute[]> {
@@ -3720,6 +3722,7 @@ export class SentryApiService {
       end,
       substringMatch,
       query,
+      context,
       opts,
     );
 
@@ -3831,6 +3834,7 @@ export class SentryApiService {
     end?: string,
     substringMatch?: string,
     query?: string,
+    context?: boolean,
     opts?: RequestOptions,
   ): Promise<TraceItemAttribute[]> {
     const queryParams = new URLSearchParams();
@@ -3844,6 +3848,9 @@ export class SentryApiService {
     if (query) {
       queryParams.set("query", query);
     }
+    if (context) {
+      queryParams.set("expand", "context");
+    }
     this.applyTimeParams(queryParams, statsPeriod, start, end);
 
     const url =

packages/mcp-core/src/tools/support/search-events/utils.ts

@@ -590,6 +590,7 @@ export async function fetchCustomAttributes(
     attributeTypes?: TraceItemAttributeType[];
     substringMatch?: string;
     query?: string;
+    context?: boolean;
   } = {},
 ): Promise<{
   attributes: Record<string, string>;
@@ -640,6 +641,7 @@ export async function fetchCustomAttributes(
       attributeTypes: options.attributeTypes,
       substringMatch: options.substringMatch,
       query: options.query,
+      context: options.context,
     });
 
     for (const attr of attributesResponse) {

packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts

@@ -0,0 +1,164 @@
+import { describeEval } from "vitest-evals";
+import { ToolCallScorer } from "vitest-evals";
+import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent";
+import { SentryApiService } from "@sentry/mcp-core/api-client";
+import { StructuredOutputScorer } from "./utils/structuredOutputScorer";
+import "../setup-env";
+
+// Compares the attribute the agent picks when the trace-items attributes
+// endpoint is called with and without `expand=context`. The with-context span
+// fixtures list the replacement for each deprecated attribute (e.g.
+// user.geo.region for geo.region), so only that path should use it.
+//
+// The datasetAttributes tool does not request context itself, so this service
+// forces it on or off for every attributes request.
+class AttributeContextApiService extends SentryApiService {
+  constructor(private readonly attributeContext: boolean) {
+    super({ accessToken: "test-token" });
+  }
+
+  override listTraceItemAttributes(
+    ...[params, opts]: Parameters<SentryApiService["listTraceItemAttributes"]>
+  ) {
+    return super.listTraceItemAttributes(
+      { ...params, context: this.attributeContext },
+      opts,
+    );
+  }
+}
+
+describeEval("search-events-agent-attributes-without-context", {
+  data: async () => {
+    return [
+      {
+        // http.method is also picked from the static span fields
+        input: "Count spans grouped by HTTP method over the last 7 days",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("http.method") &&
+            !value.includes("http.request.method"),
+        },
+      },
+      {
+        // http.status_code is also picked from static span fields
+        input: "Show me spans with HTTP status code 503 in the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          query: (value: unknown) =>
+            typeof value === "string" &&
+            value.includes("http.status_code:503") &&
+            !value.includes("http.response.status_code"),
+        },
+      },
+    ];
+  },
+  task: async (input) => {
+    // Create an API service that never requests attribute context
+    const apiService = new AttributeContextApiService(false);
+
+    const agentResult = await searchEventsAgent({
+      query: input,
+      organizationSlug: "sentry-mcp-evals",
+      apiService,
+    });
+
+    return {
+      result: JSON.stringify(agentResult.result),
+      toolCalls: agentResult.toolCalls.map((call: any) => ({
+        name: call.toolName,
+        arguments: call.args,
+      })),
+    };
+  },
+  scorers: [
+    ToolCallScorer({ params: "fuzzy" }), // Validates tool calls
+    StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching
+  ],
+});
+
+// Context not enabled yet so these should not show the correct queries. Asserting them to the wrong queries for now.
+describeEval("search-events-agent-attributes-with-context", {
+  data: async () => {
+    return [
+      {
+        // EVENTUALLY Context marks http.method as deprecated in favor of http.request.method
+        input: "Count spans grouped by HTTP method over the last 7 days",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            !value.includes("http.request.method") &&
+            value.includes("http.method"),
+        },
+      },
+      {
+        // EVENTUALLY Context marks http.status_code as deprecated in favor of
+        // http.response.status_code
+        input: "Show me spans with HTTP status code 503 in the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          query: (value: unknown) =>
+            typeof value === "string" &&
+            !value.includes("http.response.status_code:503") &&
+            value.includes("http.status_code"),
+        },
+      },
+    ];
+  },
+  task: async (input) => {
+    // Create an API service that always requests attribute context
+    const apiService = new AttributeContextApiService(true);
+
+    const agentResult = await searchEventsAgent({
+      query: input,
+      organizationSlug: "sentry-mcp-evals",
+      apiService,
+    });
+
+    return {
+      result: JSON.stringify(agentResult.result),
+      toolCalls: agentResult.toolCalls.map((call: any) => ({
+        name: call.toolName,
+        arguments: call.args,
+      })),
+    };
+  },
+  scorers: [
+    ToolCallScorer({ params: "fuzzy" }), // Validates tool calls
+    StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching
+  ],
+});

packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json

@@ -0,0 +1,57 @@
+[
+  {
+    "key": "span.duration",
+    "name": "span.duration",
+    "context": {
+      "isConvention": false,
+      "brief": "The total time taken by the span.",
+      "isDeprecated": false
+    }
+  },
+  {
+    "key": "transaction.duration",
+    "name": "transaction.duration",
+    "context": {}
+  },
+  {
+    "key": "http.status_code",
+    "name": "HTTP Status Code",
+    "context": {
+      "isConvention": true,
+      "brief": "The status code of the HTTP response.",
+      "isDeprecated": true,
+      "examples": [404],
+      "replacementAttribute": "http.response.status_code"
+    }
+  },
+  {
+    "key": "http.response.status_code",
+    "name": "http.response.status_code",
+    "context": {
+      "isConvention": true,
+      "brief": "The status code of the HTTP response.",
+      "isDeprecated": false,
+      "examples": [404]
+    }
+  },
+  {
+    "key": "custom.count",
+    "name": "custom.count",
+    "context": {}
+  },
+  {
+    "key": "custom.score",
+    "name": "custom.score",
+    "context": {}
+  },
+  {
+    "key": "custom.latency_ms",
+    "name": "custom.latency_ms",
+    "context": {}
+  },
+  {
+    "key": "custom.db.pool_size",
+    "name": "custom.db.pool_size",
+    "context": {}
+  }
+]