From 30f9fe228b0272d225fd539bc4b5b1604b53de8b Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:45:12 +0000 Subject: [PATCH 1/2] ci(evals): run eval workflow on OpenRouter Switch the GitHub Actions eval job and eval harness defaults from OPENAI_API_KEY/gpt-4o to OPENROUTER_API_KEY so CI no longer depends on a direct OpenAI key. Co-Authored-By: Daniel Griesser --- .github/workflows/eval.yml | 3 ++- docs/testing/overview.md | 2 +- packages/mcp-server-evals/src/bin/start-mock-stdio.ts | 5 ++++- .../mcp-server-evals/src/evals/search-events.eval.ts | 2 +- .../src/evals/search-issue-events.eval.ts | 2 +- .../mcp-server-evals/src/evals/search-issues.eval.ts | 2 +- .../src/evals/utils/mcpToolCallRunner.ts | 6 ++---- .../src/evals/utils/toolPredictionScorer.ts | 10 +++++----- packages/mcp-server-mocks/src/utils.ts | 8 ++++++-- 9 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 78fca2421..578edcf7c 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -60,7 +60,8 @@ jobs: run: pnpm eval:ci evals continue-on-error: true env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + EMBEDDED_AGENT_PROVIDER: openrouter - name: Create eval status check uses: actions/github-script@v7 diff --git a/docs/testing/overview.md b/docs/testing/overview.md index 9e00ba9b9..80f9a4055 100644 --- a/docs/testing/overview.md +++ b/docs/testing/overview.md @@ -279,7 +279,7 @@ describeEval("tool-name", { ### Running Evals ```bash -# Requires OPENAI_API_KEY in .env +# Requires OPENROUTER_API_KEY in .env pnpm eval # Run specific eval diff --git a/packages/mcp-server-evals/src/bin/start-mock-stdio.ts b/packages/mcp-server-evals/src/bin/start-mock-stdio.ts index 31612ac29..367afff96 100644 --- a/packages/mcp-server-evals/src/bin/start-mock-stdio.ts +++ b/packages/mcp-server-evals/src/bin/start-mock-stdio.ts @@ -8,7 +8,10 @@ import type { ServerContext } from "@sentry/mcp-core/types"; mswServer.listen({ onUnhandledRequest: (req, print) => { - if (req.url.startsWith("https://api.openai.com/")) { + if ( + req.url.startsWith("https://api.openai.com/") || + req.url.startsWith("https://openrouter.ai/") + ) { return; } diff --git a/packages/mcp-server-evals/src/evals/search-events.eval.ts b/packages/mcp-server-evals/src/evals/search-events.eval.ts index 79f06d2dd..fcd49c712 100644 --- a/packages/mcp-server-evals/src/evals/search-events.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events.eval.ts @@ -1,7 +1,7 @@ import { describeEval } from "vitest-evals"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; -// Note: This eval requires OPENAI_API_KEY to be set in the environment +// Note: This eval requires OPENROUTER_API_KEY to be set in the environment // The search_events tool uses the AI SDK to translate natural language queries describeEval("search-events", { data: async () => { diff --git a/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts b/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts index 61f693939..0b1e118c8 100644 --- a/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts @@ -1,7 +1,7 @@ import { describeEval } from "vitest-evals"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; -// Note: This eval requires OPENAI_API_KEY to be set in the environment +// Note: This eval requires OPENROUTER_API_KEY to be set in the environment // The search_issue_events tool uses the AI SDK to translate natural language queries describeEval("search-issue-events", { data: async () => { diff --git a/packages/mcp-server-evals/src/evals/search-issues.eval.ts b/packages/mcp-server-evals/src/evals/search-issues.eval.ts index c504c165a..2de7b9491 100644 --- a/packages/mcp-server-evals/src/evals/search-issues.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issues.eval.ts @@ -1,7 +1,7 @@ import { describeEval } from "vitest-evals"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; -// Note: This eval requires OPENAI_API_KEY to be set in the environment +// Note: This eval requires OPENROUTER_API_KEY to be set in the environment // The search_issues tool uses the AI SDK to translate natural language queries describeEval("search-issues", { data: async () => { diff --git a/packages/mcp-server-evals/src/evals/utils/mcpToolCallRunner.ts b/packages/mcp-server-evals/src/evals/utils/mcpToolCallRunner.ts index d3a4939b4..4704df7fe 100644 --- a/packages/mcp-server-evals/src/evals/utils/mcpToolCallRunner.ts +++ b/packages/mcp-server-evals/src/evals/utils/mcpToolCallRunner.ts @@ -1,10 +1,8 @@ import { experimental_createMCPClient } from "@ai-sdk/mcp"; import { Experimental_StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio"; -import { openai } from "@ai-sdk/openai"; +import { getOpenRouterModel } from "@sentry/mcp-core/internal/agents/openrouter-provider"; import { generateText, stepCountIs, type LanguageModel } from "ai"; -const defaultModel = openai("gpt-4o"); - function toToolCall(call: { toolName: string; input: unknown }) { const input = call.input && typeof call.input === "object" && !Array.isArray(call.input) @@ -18,7 +16,7 @@ function toToolCall(call: { toolName: string; input: unknown }) { } export function McpToolCallTaskRunner( - model: LanguageModel = defaultModel, + model: LanguageModel = getOpenRouterModel(), maxSteps = 6, ) { return async function McpToolCallTaskRunner(input: string) { diff --git a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts b/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts index d199bebc7..cc5b03241 100644 --- a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts +++ b/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts @@ -1,8 +1,8 @@ -import { openai } from "@ai-sdk/openai"; import { generateObject, type LanguageModel } from "ai"; import { z } from "zod"; import { experimental_createMCPClient } from "@ai-sdk/mcp"; import { Experimental_StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio"; +import { getOpenRouterModel } from "@sentry/mcp-core/internal/agents/openrouter-provider"; // Cache for available tools to avoid reconnecting for each test let cachedTools: string[] | null = null; @@ -64,8 +64,6 @@ interface ToolPredictionScorerOptions { result?: any; } -const defaultModel = openai("gpt-4o"); - const predictionSchema = z.object({ score: z.number().min(0).max(1).describe("Score from 0 to 1"), rationale: z.string().describe("Explanation of the score"), @@ -129,7 +127,7 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp * A scorer that uses AI to predict what tools would be called without executing them. * This is much faster than actually running the tools and checking what was called. * - * @param model - Optional language model to use for predictions (defaults to gpt-4o) + * @param model - Optional language model to use for predictions (defaults to OpenRouter) * @returns A scorer function that compares predicted vs expected tool calls * * @example @@ -170,7 +168,9 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp * If `expectedTools` is not provided in test data, the scorer is automatically skipped * and returns `{ score: null }` to allow other scorers to run without interference. */ -export function ToolPredictionScorer(model: LanguageModel = defaultModel) { +export function ToolPredictionScorer( + model: LanguageModel = getOpenRouterModel(), +) { return async function ToolPredictionScorer( opts: ToolPredictionScorerOptions, ) { diff --git a/packages/mcp-server-mocks/src/utils.ts b/packages/mcp-server-mocks/src/utils.ts index 5a456f1d7..0a397e634 100644 --- a/packages/mcp-server-mocks/src/utils.ts +++ b/packages/mcp-server-mocks/src/utils.ts @@ -17,8 +17,12 @@ export function startMockServer(options?: { mswServer.listen({ onUnhandledRequest: (req: any, print: any) => { - // Ignore OpenAI requests if specified (default behavior for AI agent tests) - if (ignoreOpenAI && req.url.startsWith("https://api.openai.com/")) { + // Ignore LLM provider requests if specified (default behavior for AI agent tests) + if ( + ignoreOpenAI && + (req.url.startsWith("https://api.openai.com/") || + req.url.startsWith("https://openrouter.ai/")) + ) { return; } From 01adde22fa543a11d85955323743066e49a4d093 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:53:44 +0000 Subject: [PATCH 2/2] fix(evals): configure OpenRouter structured output --- .../src/evals/utils/toolPredictionScorer.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts b/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts index cc5b03241..e626b7f38 100644 --- a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts +++ b/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts @@ -205,6 +205,12 @@ export function ToolPredictionScorer( expectedDescription, ), schema: predictionSchema, + providerOptions: { + openai: { + structuredOutputs: false, + strictJsonSchema: false, + }, + }, experimental_telemetry: { isEnabled: true, functionId: "tool_prediction_scorer",