diff --git a/apps/desktop/src/main/maker-host/__tests__/codexProxyHost.test.ts b/apps/desktop/src/main/maker-host/__tests__/codexProxyHost.test.ts index 5547ed9d5d..c51f9ca437 100644 --- a/apps/desktop/src/main/maker-host/__tests__/codexProxyHost.test.ts +++ b/apps/desktop/src/main/maker-host/__tests__/codexProxyHost.test.ts @@ -661,9 +661,9 @@ describe('chatBridgeCapabilitiesForRoute', () => { ['https://coding.dashscope.aliyuncs.com.evil.example/v1', 'qwen3.7-plus'], ['https://example.com/v1', 'qwen3.7-plus'], ['https://coding.dashscope.aliyuncs.com/v1', 'qwen3-coder-next'], - ])('keeps image input disabled for non-matching route %s / %s', async (upstream, model) => { + ])('enables image_url by default for non-matching route %s / %s (fail-open)', async (upstream, model) => { const { chatBridgeCapabilitiesForRoute } = await freshCodexProxyHost(); - expect(chatBridgeCapabilitiesForRoute(upstream, model).imageInput).toBeUndefined(); + expect(chatBridgeCapabilitiesForRoute(upstream, model).imageInput).toBe('image_url'); }); it('passes image support into the handler for a preset-derived custom Kimi route', async () => { diff --git a/apps/desktop/src/main/maker-host/codex-proxy-host.ts b/apps/desktop/src/main/maker-host/codex-proxy-host.ts index f8a0a5c876..7b41a2d64c 100644 --- a/apps/desktop/src/main/maker-host/codex-proxy-host.ts +++ b/apps/desktop/src/main/maker-host/codex-proxy-host.ts @@ -715,6 +715,7 @@ const CHAT_BRIDGE_DEFAULT_CAPABILITIES: ChatBridgeCapabilities = { maxTokensField: 'max_tokens', reasoningField: 'none', streamUsage: true, + imageInput: 'image_url', // Responses fields with direct Chat equivalents. Provider-specific unsupported fields can // be removed later when the model capability catalog becomes more granular. passthroughFields: [ @@ -788,14 +789,11 @@ function rewriteChatBridgeModel(model: string, stripPrefix: string | undefined): /** * 在模型级多模态能力元数据接入路由前,图片桥接先按已验证的上游能力显式开启。 * - * 当前覆盖: - * - Moonshot Kimi K3 - * - Volcengine Doubao Seed 系列 - * - Alibaba Cloud Bailian Coding Plan Qwen 3.7 Plus + * 默认已启用 `imageInput: 'image_url'`(fail-open)。此函数仅在已验证路由上 + * 显式覆盖,确保白名单路由的 `imageInput` 始终为 `'image_url'`。 * - * 这里认官方 DNS 边界 + 上游 model,不认 provider id(预设创建后会生成用户自定义 - * id),也不对所有 openai-chat 供应商放开。未命中继续沿用 fail-closed 默认—— - * 无图片能力的上游(如 DeepSeek)保持发送前显式报错,不静默吞图。 + * 非白名单路由同样走 fail-open:桥接层转换格式,上游不支持时返回错误, + * 客户端通过 `isUnsupportedResponsesImageErrorPayload` 检测并提示用户。 */ export function chatBridgeCapabilitiesForRoute( upstream: string, diff --git a/packages/responses-chat-bridge/src/__tests__/types.test.ts b/packages/responses-chat-bridge/src/__tests__/types.test.ts index 6a65269dbb..f33f1c622e 100644 --- a/packages/responses-chat-bridge/src/__tests__/types.test.ts +++ b/packages/responses-chat-bridge/src/__tests__/types.test.ts @@ -19,6 +19,10 @@ function codexUnexpectedResponse(messageOrBody: string): string { return `unexpected status 400 Bad Request: ${messageOrBody}, url: http://127.0.0.1/v1/responses`; } +function codexUnexpectedStatus(status: number, messageOrBody: string): string { + return `unexpected status ${status}: ${messageOrBody}, url: http://127.0.0.1/v1/responses`; +} + describe('isUnsupportedResponsesImageErrorPayload', () => { it.each([ "input content part 'input_image'", @@ -43,6 +47,144 @@ describe('isUnsupportedResponsesImageErrorPayload', () => { expect(isUnsupportedResponsesImageErrorPayload(codexUnexpectedResponse(payload))).toBe(true); }); + it('accepts upstream provider image rejection (DeepSeek-style 400)', () => { + const payload = JSON.stringify({ + error: { + code: 'invalid_request_error', + message: 'image_url content part is not supported by this model', + }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it('accepts upstream provider multimodal rejection', () => { + const payload = JSON.stringify({ + error: { + code: 'invalid_request', + message: 'This model does not support multimodal input', + }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it('accepts handler-wrapped upstream image rejection (DeepSeek via bridge)', () => { + // handler.ts wraps: responsesError(status, 'upstream_error', rawUpstreamBody) + const innerError = JSON.stringify({ + error: { code: 'invalid_request_error', message: 'image_url content part is not supported' }, + }); + const payload = JSON.stringify({ + error: { code: 'upstream_error', message: innerError }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it('accepts upstream rejection using error.type without a code field', () => { + const payload = JSON.stringify({ + error: { + type: 'invalid_request_error', + message: 'This model does not support image_url content parts', + }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it('accepts a plain-text (non-JSON) upstream 400 body wrapped by the handler', () => { + const payload = JSON.stringify({ + error: { + code: 'upstream_error', + message: 'image_url content part is not supported by this model', + }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it('accepts a Codex-extracted plain-text capability rejection', () => { + // Codex extracts error.message from the wrapped upstream_error envelope, so + // the coordinator sees the plain-text message after the unexpected-status + // prefix instead of the JSON envelope. + expect( + isUnsupportedResponsesImageErrorPayload( + codexUnexpectedResponse('image_url content part is not supported by this model'), + ), + ).toBe(true); + }); + + it('rejects Codex-extracted invalid-image-content errors (not capability rejection)', () => { + expect( + isUnsupportedResponsesImageErrorPayload( + codexUnexpectedResponse('Invalid image_url: image exceeds maximum size'), + ), + ).toBe(false); + }); + + it('accepts Codex-rendered 415/422 capability rejections (non-400 client errors)', () => { + // Some OpenAI-compatible upstreams use 415 (unsupported_media_type) or + // 422 (unprocessable_entity) to signal the model does not accept image + // content parts. The plain-text classifier must not be gated on 400. + expect( + isUnsupportedResponsesImageErrorPayload( + codexUnexpectedStatus(415, 'image_url content part is not supported by this model'), + ), + ).toBe(true); + expect( + isUnsupportedResponsesImageErrorPayload( + codexUnexpectedStatus(422, 'image input is not supported by this model'), + ), + ).toBe(true); + }); + + it('accepts Codex-rendered 422 with HTTP reason phrase', () => { + // Codex includes the HTTP reason phrase, e.g. + // 'unexpected status 422 Unprocessable Entity: ...'. + expect( + isUnsupportedResponsesImageErrorPayload( + 'unexpected status 422 Unprocessable Entity: image_url content part is not supported by this model, url: http://127.0.0.1/v1/responses', + ), + ).toBe(true); + expect( + isUnsupportedResponsesImageErrorPayload( + 'unexpected status 415 Unsupported Media Type: image input is not supported by this model, url: http://127.0.0.1/v1/responses', + ), + ).toBe(true); + }); + + it('accepts a JSON error whose error value is a plain string (Ollama-style)', () => { + const payload = JSON.stringify({ + error: { code: 'upstream_error', message: JSON.stringify({ error: 'this model does not support images' }) }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it('accepts a handler-wrapped rejection whose inner error uses error.type', () => { + const innerError = JSON.stringify({ + error: { type: 'invalid_request_error', message: 'image input is not supported' }, + }); + const payload = JSON.stringify({ + error: { code: 'upstream_error', message: innerError }, + }); + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(true); + }); + + it.each([ + JSON.stringify({ + error: { code: 'invalid_request_error', message: 'Invalid image_url: image exceeds maximum size' }, + }), + JSON.stringify({ + error: { code: 'invalid_request_error', message: 'image_url must be a valid URL' }, + }), + JSON.stringify({ + error: { code: 'invalid_request_error', message: 'image exceeds maximum size' }, + }), + JSON.stringify({ + error: { code: 'upstream_error', message: 'Invalid image_url: image exceeds maximum size' }, + }), + ])('rejects invalid-image-content errors (not capability rejection): %s', (payload) => { + // A message about the image being invalid (size, format, URL) is NOT a + // capability rejection — stripping the attachment would resend text without + // the image the user asked about. + expect(isUnsupportedResponsesImageErrorPayload(payload)).toBe(false); + }); + it.each([ unsupportedFeaturePayload("input content part 'input_file'"), unsupportedFeaturePayload("input content part 'input_image'", 'invalid_request'), diff --git a/packages/responses-chat-bridge/src/types.ts b/packages/responses-chat-bridge/src/types.ts index b474bc5259..c6aed94adc 100644 --- a/packages/responses-chat-bridge/src/types.ts +++ b/packages/responses-chat-bridge/src/types.ts @@ -290,8 +290,8 @@ export interface ChatBridgeCapabilities { */ reasoningHistoryField?: ChatReasoningHistoryField; /** - * Responses `input_image` 的上游等价形态。默认未声明 = fail closed;只由 - * 已确认支持视觉输入的运行时(当前为 upstream 白名单)开启。 + * Responses `input_image` 的上游等价形态。默认 `image_url`(fail-open:桥接层 + * 负责格式转换,上游不支持时由客户端检测并提示用户)。 */ imageInput?: ChatImageInput; /** Responses `input_file` 的上游等价形态;默认未声明 = fail closed。 */ @@ -368,7 +368,8 @@ export interface ResponsesChatBridgeHandler { const UNSUPPORTED_RESPONSES_FEATURE_MESSAGE_PREFIX = 'Responses feature is not supported by the Chat Completions bridge: '; const RESPONSES_IMAGE_CONTENT_PART_TYPES = new Set(['input_image', 'image_url', 'image']); -const CODEX_UNEXPECTED_BAD_REQUEST_PREFIX = /^unexpected status 400(?: Bad Request)?: /; +const CODEX_UNEXPECTED_BAD_REQUEST_PREFIX = + /^unexpected status (?:400(?: Bad Request)?|415(?: Unsupported Media Type)?|422(?: Unprocessable Entity)?): /; const CODEX_ERROR_METADATA_MARKERS = [ ', url: ', ', cf-ray: ', @@ -396,14 +397,106 @@ function isUnsupportedResponsesImageFeature(feature: string): boolean { || contentPartType.startsWith('input_image.'); } +// Capability-rejection keywords: an upstream message that states the *model* +// does not support image/vision input. Deliberately narrower than a generic +// "mentions image" test: messages like "Invalid image_url: image exceeds +// maximum size" or "image_url must be a valid URL" describe invalid *content*, +// not a missing capability. Treating those as capability rejection would strip +// the user's attachment and resend text without the image they asked about. +function imageRejectionKeywords(msg: string): boolean { + const m = msg.toLowerCase(); + // Capability rejection: the model/provider explicitly says it does not support + // image/vision input. This is distinct from content errors like "image format + // not supported" or "image exceeds maximum size" which describe invalid content. + const capabilityPhrases = [ + 'image input is not supported', + 'vision is not supported', + 'images are not supported', + 'image_url content part is not supported', + 'image content part is not supported', + 'multimodal input is not supported', + 'does not support image input', + 'does not support vision input', + 'does not support image_url', + 'does not support images', + 'does not support multimodal', + 'does not support image', + 'does not support vision', + 'not support image', + 'not support vision', + 'not support multimodal', + ]; + for (const phrase of capabilityPhrases) { + if (m.includes(phrase)) return true; + } + return false; +} + function isUnsupportedResponsesImageErrorObject(value: unknown): boolean { if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; const error = (value as Record).error; if (typeof error !== 'object' || error === null || Array.isArray(error)) return false; const { code, message } = error as Record; - if (code !== 'unsupported_feature' || typeof message !== 'string') return false; - const feature = unsupportedResponsesFeatureFromMessage(message); - return feature !== null && isUnsupportedResponsesImageFeature(feature); + if (typeof message !== 'string') return false; + // Bridge-generated unsupported_feature error (fail-closed path) + if (code === 'unsupported_feature') { + const feature = unsupportedResponsesFeatureFromMessage(message); + return feature !== null && isUnsupportedResponsesImageFeature(feature); + } + // Upstream provider rejection: detect image-related 400 errors from providers + // that don't support vision (e.g. DeepSeek, Ollama without vision model). + // + // The handler (handler.ts) wraps upstream errors as: + // {error: {code: "upstream_error", message: ""}} + // So we check both the outer code and also parse the wrapped inner error. + const { type } = error as Record; + // Accept both `code` (OpenAI-style) and `type` (some providers only emit + // `type: 'invalid_request_error'` with no code at all) as the + // request-invalid signal. + // Accept invalid_request (400), unsupported_media_type (415), and + // unprocessable_entity (422) as request-invalid signals. Some providers use + // 415/422 to indicate the model does not accept image content parts. + const invalidRequest = + (typeof code === 'string' && (code.includes('invalid_request') + || code.includes('unsupported_media') || code.includes('unprocessable'))) + || (typeof type === 'string' && (type.toLowerCase().includes('invalid_request') + || type.toLowerCase().includes('unsupported_media') + || type.toLowerCase().includes('unprocessable'))); + if (invalidRequest + && !message.startsWith(UNSUPPORTED_RESPONSES_FEATURE_MESSAGE_PREFIX)) { + if (imageRejectionKeywords(message)) return true; + } + if (code === 'upstream_error' && typeof message === 'string') { + const inner = parseJson(message); + if (inner && typeof inner === 'object' && inner !== null) { + const innerErr = (inner as Record).error; + if (typeof innerErr === 'string') { + // Some providers (e.g. Ollama) return a native JSON error whose + // `error` value is a plain string, e.g. {"error":"this model + // does not support images"}. Classify it directly. + if (imageRejectionKeywords(innerErr)) return true; + } else if (innerErr && typeof innerErr === 'object') { + const ic = (innerErr as Record).code; + const it = (innerErr as Record).type; + const im = (innerErr as Record).message; + const innerInvalid = + (typeof ic === 'string' && (ic.includes('invalid_request') + || ic.includes('unsupported_media') || ic.includes('unprocessable'))) + || (typeof it === 'string' && (it.toLowerCase().includes('invalid_request') + || it.toLowerCase().includes('unsupported_media') + || it.toLowerCase().includes('unprocessable'))); + if (innerInvalid && typeof im === 'string' && imageRejectionKeywords(im)) { + return true; + } + } + } else { + // Upstream returned a plain-text (non-JSON) 400 body, e.g. + // "image_url content part is not supported by this model". The handler + // wraps it as upstream_error with the raw text as message. + if (imageRejectionKeywords(message)) return true; + } + } + return false; } function parseJson(value: string): unknown { @@ -455,7 +548,12 @@ export function isUnsupportedResponsesImageErrorPayload(payload: string | null): const message = stripCodexErrorMetadata(renderedBody); const feature = unsupportedResponsesFeatureFromMessage(message); - return feature !== null && isUnsupportedResponsesImageFeature(feature); + if (feature !== null && isUnsupportedResponsesImageFeature(feature)) return true; + // Codex extracts error.message from the wrapped upstream_error envelope, so a + // plain-text capability rejection (e.g. "image_url content part is not + // supported by this model, url: ...") arrives directly after the prefix. + // Classify the stripped plain text with the same keyword set as the JSON path. + return imageRejectionKeywords(message); } export class UnsupportedResponsesFeatureError extends Error {