Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export function setAnthropicBaseUrl(baseUrl: string | undefined): void {
* Retrieve an Anthropic language model configured from environment variables and explicit config.
*
* Configuration:
* - ANTHROPIC_API_KEY: API key for authentication (required)
* - ANTHROPIC_MODEL: Model to use (default: "claude-sonnet-4-5") - env var OK
* - Base URL: Must be set via setAnthropicBaseUrl() - NOT from env vars (security risk)
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/**
* Integration tests for OpenAI provider
*
* These tests verify that the OpenAI provider configuration works correctly
* with the actual OpenAI API. They require OPENAI_API_KEY to be set.
*
* Unlike unit tests (which mock the AI SDK) and evals (which test tool prediction
* but don't execute handlers), these tests actually call the embedded agents
* with real OpenAI API calls while mocking Sentry API responses.
*
* This catches issues like:
* - #623: structuredOutputs causing validation errors with nullable fields
* - 405 errors from unsupported parameters (reasoningEffort, temperature)
*/
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { http, HttpResponse } from "msw";
import { setupServer } from "msw/node";
import { searchIssuesAgent } from "../../tools/search-issues/agent";
import { SentryApiService } from "../../api-client";
import { setAgentProvider } from "./provider-factory";

// Mock Sentry API server - intercepts Sentry calls but bypasses OpenAI
const mswServer = setupServer(
// Mock the issue search fields endpoint (called by issueFields tool)
http.get("*/api/0/organizations/*/issues/", () => {
return HttpResponse.json([]);
}),
// Mock the tags endpoint for field discovery
http.get("*/api/0/organizations/*/tags/", () => {
return HttpResponse.json([
{ key: "environment", name: "Environment" },
{ key: "level", name: "Level" },
{ key: "release", name: "Release" },
{ key: "browser", name: "Browser" },
]);
}),
// Mock whoami endpoint (called by whoami tool)
http.get("*/api/0/users/me/", () => {
return HttpResponse.json({
id: "12345",
email: "test@example.com",
name: "Test User",
});
}),
);

describe("OpenAI Provider Integration", () => {
const hasOpenAIKey = Boolean(process.env.OPENAI_API_KEY);

beforeAll(() => {
if (hasOpenAIKey) {
// Explicitly set OpenAI provider to ensure we test OpenAI even if
// ANTHROPIC_API_KEY is also set (auto-detect prefers Anthropic)
setAgentProvider("openai");
mswServer.listen({ onUnhandledRequest: "bypass" });
}
});

afterAll(() => {
if (hasOpenAIKey) {
setAgentProvider(undefined); // Reset for other tests
mswServer.close();
}
});
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

it.skipIf(!hasOpenAIKey)(
"searchIssuesAgent translates natural language to Sentry query",
async () => {
// This tests the actual code path that broke in #623 and with gateway 405s
// - Uses the real searchIssuesAgent (not mocked)
// - Calls real OpenAI API
// - Mocks only the Sentry API responses

const apiService = new SentryApiService({
accessToken: "test-token",
host: "sentry.io",
});

const { result } = await searchIssuesAgent({
query: "unresolved errors in production from the last week",
organizationSlug: "test-org",
apiService,
});

// Verify the agent returned a valid structured response
expect(result).toBeDefined();
expect(typeof result.query).toBe("string");
expect(typeof result.explanation).toBe("string");

// The query should contain relevant Sentry search syntax
// (exact output varies, but should be parseable)
expect(result.query.length).toBeGreaterThan(0);

// sort can be null or one of the valid values
expect(
result.sort === null ||
["date", "freq", "new", "user"].includes(result.sort),
).toBe(true);
},
{ timeout: 60000 },
);

it.skipIf(!hasOpenAIKey)(
"searchIssuesAgent handles nullable sort field correctly",
async () => {
// Specifically test that nullable fields work (the #623 issue)
const apiService = new SentryApiService({
accessToken: "test-token",
host: "sentry.io",
});

const { result } = await searchIssuesAgent({
query: "all issues", // Vague query where sort preference is unclear
organizationSlug: "test-org",
apiService,
});

// Should succeed without "required field" validation errors
expect(result).toBeDefined();
// sort being null is valid and tests the nullable field handling
expect(
result.sort === null ||
["date", "freq", "new", "user"].includes(result.sort),
).toBe(true);
},
{ timeout: 60000 },
);
});
58 changes: 8 additions & 50 deletions packages/mcp-core/src/internal/agents/openai-provider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,62 +2,23 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { getOpenAIModel, setOpenAIBaseUrl } from "./openai-provider.js";

describe("openai-provider", () => {
const originalEnv = process.env.OPENAI_REASONING_EFFORT;
const originalModel = process.env.OPENAI_MODEL;

beforeEach(() => {
setOpenAIBaseUrl(undefined);
// biome-ignore lint/performance/noDelete: Required to properly unset environment variable
delete process.env.OPENAI_MODEL;
});

afterEach(() => {
if (originalEnv === undefined) {
if (originalModel === undefined) {
// biome-ignore lint/performance/noDelete: Required to properly unset environment variable
delete process.env.OPENAI_REASONING_EFFORT;
delete process.env.OPENAI_MODEL;
} else {
process.env.OPENAI_REASONING_EFFORT = originalEnv;
process.env.OPENAI_MODEL = originalModel;
}
});

describe("reasoning effort configuration", () => {
it("uses default reasoning effort when env var is not set", () => {
// biome-ignore lint/performance/noDelete: Required to properly unset environment variable
delete process.env.OPENAI_REASONING_EFFORT;

const model = getOpenAIModel();

// The model object should be created with default reasoning effort
expect(model).toBeDefined();
expect(model.modelId).toBe("gpt-5");
});

it("disables reasoning effort when env var is empty string", () => {
process.env.OPENAI_REASONING_EFFORT = "";

const model = getOpenAIModel();

// The model object should be created without reasoning effort
expect(model).toBeDefined();
expect(model.modelId).toBe("gpt-5");
});

it("uses specified reasoning effort when env var is set", () => {
process.env.OPENAI_REASONING_EFFORT = "high";

const model = getOpenAIModel();

// The model object should be created with high reasoning effort
expect(model).toBeDefined();
expect(model.modelId).toBe("gpt-5");
});

it("throws error for invalid reasoning effort value", () => {
process.env.OPENAI_REASONING_EFFORT = "invalid";

expect(() => getOpenAIModel()).toThrow(
'Invalid OPENAI_REASONING_EFFORT value: "invalid". Must be one of: "low", "medium", "high", or "" (empty string to disable). Default is "low".',
);
});
});

describe("base URL configuration", () => {
it("uses default base URL when not configured", () => {
const model = getOpenAIModel();
Expand Down Expand Up @@ -90,14 +51,11 @@ describe("openai-provider", () => {
});

it("uses OPENAI_MODEL env var when set", () => {
process.env.OPENAI_MODEL = "gpt-4o";
process.env.OPENAI_MODEL = "gpt-4-turbo";

const model = getOpenAIModel();

expect(model.modelId).toBe("gpt-4o");

// biome-ignore lint/performance/noDelete: Required to properly unset environment variable
delete process.env.OPENAI_MODEL;
expect(model.modelId).toBe("gpt-4-turbo");
});
});
});
43 changes: 11 additions & 32 deletions packages/mcp-core/src/internal/agents/openai-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,6 @@ import { USER_AGENT } from "../../version";

// Default configuration constants
const DEFAULT_OPENAI_MODEL = "gpt-5";
const VALID_REASONING_EFFORTS = ["low", "medium", "high"] as const;
const DEFAULT_REASONING_EFFORT: (typeof VALID_REASONING_EFFORTS)[number] =
"low";

type ReasoningEffort = (typeof VALID_REASONING_EFFORTS)[number];

// Module-level state for baseURL (set only via explicit configuration, not env vars)
let configuredBaseUrl: string | undefined;
Expand All @@ -24,45 +19,29 @@ export function setOpenAIBaseUrl(baseUrl: string | undefined): void {
/**
* Retrieve an OpenAI language model configured from environment variables and explicit config.
*
* Works with:
* - OpenAI API directly (default)
* - Any OpenAI-compatible API via custom base URL (Vercel AI Gateway, Azure OpenAI, etc.)
*
* Configuration:
* - OPENAI_API_KEY: API key for authentication (required)
* - OPENAI_MODEL: Model to use (default: "gpt-5") - env var OK
* - OPENAI_REASONING_EFFORT: Reasoning effort for o1 models: "low", "medium", "high", or "" to disable (default: "low") - env var OK
* - Base URL: Must be set via setOpenAIBaseUrl() - NOT from env vars (security risk)
*
* Note: Model name format depends on your provider:
* - Direct OpenAI: "gpt-4o", "gpt-4-turbo", etc.
* - Vercel AI Gateway: "openai/gpt-4o", "anthropic/claude-sonnet-4.5", etc.
* - Other providers: Check their documentation
*/
export function getOpenAIModel(model?: string): LanguageModelV1 {
const defaultModel = process.env.OPENAI_MODEL || DEFAULT_OPENAI_MODEL;

// Handle reasoning effort: empty string explicitly disables it, undefined uses default
const envReasoningEffort = process.env.OPENAI_REASONING_EFFORT;
let reasoningEffort: ReasoningEffort | undefined;

if (envReasoningEffort === "") {
// Empty string explicitly disables reasoning effort
reasoningEffort = undefined;
} else if (envReasoningEffort === undefined) {
// Not set - use default
reasoningEffort = DEFAULT_REASONING_EFFORT;
} else if (
VALID_REASONING_EFFORTS.includes(envReasoningEffort as ReasoningEffort)
) {
// Valid value
reasoningEffort = envReasoningEffort as ReasoningEffort;
} else {
// Invalid value - provide helpful error with all valid options
const validValues = VALID_REASONING_EFFORTS.map((v) => `"${v}"`).join(", ");
throw new Error(
`Invalid OPENAI_REASONING_EFFORT value: "${envReasoningEffort}". Must be one of: ${validValues}, or "" (empty string to disable). Default is "${DEFAULT_REASONING_EFFORT}".`,
);
}

const factory = createOpenAI({
...(configuredBaseUrl && { baseURL: configuredBaseUrl }),
headers: {
"User-Agent": USER_AGENT,
},
});

return factory(model ?? defaultModel, {
...(reasoningEffort && { reasoningEffort }),
});
return factory(model ?? defaultModel);
}
2 changes: 1 addition & 1 deletion packages/mcp-core/src/internal/agents/provider-factory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ function buildProvider(type: AgentProviderType): EmbeddedAgentProvider {
strictJsonSchema: false,
},
}),
// GPT-5 only supports temperature of 1
// GPT-5 only supports temperature=1 (AI SDK defaults to 0)
getTemperature: () => 1,
};
}
Expand Down
Loading