From 7f26e68b425a4a7e2a00e78d8b927c451705ecea Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:04:57 +0800 Subject: [PATCH 1/2] Add canonical video input support --- src/model/index.ts | 1 + src/model/protocol/canonical.ts | 12 ++- src/model/protocol/multimodal.ts | 9 ++- src/model/providers/google/request.ts | 3 +- .../providers/openai-responses/request.ts | 4 + src/model/providers/openai/request.ts | 7 ++ .../request/materializeMediaReferences.ts | 10 +++ src/router/utils/mediaRequirements.ts | 5 +- tests/model/request/videoInput.spec.ts | 77 +++++++++++++++++++ 9 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 tests/model/request/videoInput.spec.ts diff --git a/src/model/index.ts b/src/model/index.ts index 769221a29..6b5ed799a 100644 --- a/src/model/index.ts +++ b/src/model/index.ts @@ -79,6 +79,7 @@ export type { CanonicalTextBlock, CanonicalThinkingBlock, CanonicalThinkingConfig, + CanonicalVideoBlock, CanonicalToolCall, CanonicalToolCallBlock, CanonicalToolChoice, diff --git a/src/model/protocol/canonical.ts b/src/model/protocol/canonical.ts index 6b09299ef..214121af8 100644 --- a/src/model/protocol/canonical.ts +++ b/src/model/protocol/canonical.ts @@ -51,6 +51,15 @@ export type CanonicalAudioBlock = { durationSeconds?: number; }; +export type CanonicalVideoBlock = { + type: "video"; + source: "base64" | "url"; + data: string; + mimeType: string; + bytes?: number; + durationSeconds?: number; +}; + export type CanonicalToolCall = { id: string; name: string; @@ -114,7 +123,7 @@ export type CanonicalMediaReferenceBlock = { preview: string; hasMore: boolean; mimeType: string; - mediaType: "image" | "pdf" | "audio"; + mediaType: "image" | "pdf" | "audio" | "video"; pages?: number; detail?: "auto" | "low" | "high"; reason?: string; @@ -128,6 +137,7 @@ export type CanonicalContentBlock = | CanonicalImageBlock | CanonicalPdfBlock | CanonicalAudioBlock + | CanonicalVideoBlock | CanonicalToolCallBlock | CanonicalToolResultBlock | CanonicalToolResultReferenceBlock diff --git a/src/model/protocol/multimodal.ts b/src/model/protocol/multimodal.ts index 39c727b48..1ddb964c6 100644 --- a/src/model/protocol/multimodal.ts +++ b/src/model/protocol/multimodal.ts @@ -6,7 +6,7 @@ import type { } from "./canonical.js"; import { messageContent } from "./clone.js"; -export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio"] as const; +export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio", "video"] as const; export type InputModality = (typeof SUPPORTED_INPUT_MODALITIES)[number]; @@ -42,7 +42,7 @@ export function downgradeUnsupportedContent( constraints: MultimodalConstraints, ): void { const allowed = new Set(constraints.input); - if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio")) return; + if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio") && allowed.has("video")) return; for (const msg of messages) { const content = messageContent(msg); @@ -90,6 +90,9 @@ function mediaBlockToPlaceholder( if (block.type === "audio" && !allowed.has("audio")) { return `[Audio: ${block.mimeType} — omitted, model does not support audio input]`; } + if (block.type === "video" && !allowed.has("video")) { + return `[Video: ${block.mimeType} — omitted, model does not support video input]`; + } return undefined; } @@ -103,6 +106,8 @@ export function contentBlockToInputModality(block: CanonicalContentBlock): Input return "pdf"; case "audio": return "audio"; + case "video": + return "video"; case "thinking": case "tool_call": case "tool_result": diff --git a/src/model/providers/google/request.ts b/src/model/providers/google/request.ts index 193f98af8..7f10796e5 100644 --- a/src/model/providers/google/request.ts +++ b/src/model/providers/google/request.ts @@ -167,6 +167,7 @@ function toGoogleParts(block: CanonicalContentBlock, toolNamesById: Map, + block: Extract, ): Part { if (block.source === "url") { return { fileData: { fileUri: block.data, mimeType: block.mimeType } }; diff --git a/src/model/providers/openai-responses/request.ts b/src/model/providers/openai-responses/request.ts index 4efc5b598..5c5647ad4 100644 --- a/src/model/providers/openai-responses/request.ts +++ b/src/model/providers/openai-responses/request.ts @@ -195,6 +195,10 @@ function toResponsesContentPart(block: CanonicalContentBlock): Record { + const request: CanonicalModelRequest = { + provider: "compatible", + model: "video-model", + messages: [{ + role: "user", + content: [{ + type: "video", + source: "url", + data: "https://example.com/input.mp4", + mimeType: "video/mp4", + }], + }], + }; + const config = modelConfig(); + + const { provider, model } = validateModelRequest(request, config); + const required = collectRequiredInputModalities(request.messages); + const body = buildOpenAIRequest(request, model, provider); + + assert.deepEqual(required, ["video"]); + assert.equal(supportsRequiredModalities(model.multimodal, required), true); + assert.deepEqual(body.messages, [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "https://example.com/input.mp4" }, + }], + }]); +}); + +function modelConfig(): ModelConfig { + const capabilities: ModelCapabilities = { + supportsToolUse: true, + supportsStreaming: true, + supportsParallelToolCalls: true, + supportsThinking: true, + supportsJsonSchema: true, + supportsSystemPrompt: true, + supportsPromptCache: true, + maxContextTokens: 1_000_000, + maxOutputTokens: 16_384, + }; + const model: ModelDefinition = { + id: "video-model", + capabilities, + multimodal: { input: ["text", "image", "video"] }, + }; + const provider: ProviderConfig = { + id: "compatible", + protocol: "openai", + url: "https://example.invalid/v1", + apiKey: "test", + headers: {}, + models: { [model.id]: model }, + }; + return { providers: { [provider.id]: provider } }; +} From 2bea82c5848da49df47b1029d6842921ba91ad46 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:16:44 +0800 Subject: [PATCH 2/2] Serialize video inputs for compatible APIs --- src/model/providers/anthropic/request.ts | 7 ++++++ .../providers/openai-responses/request.ts | 7 +++--- tests/model/request/videoInput.spec.ts | 22 +++++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/model/providers/anthropic/request.ts b/src/model/providers/anthropic/request.ts index 59b9913ac..92e155bd3 100644 --- a/src/model/providers/anthropic/request.ts +++ b/src/model/providers/anthropic/request.ts @@ -184,6 +184,13 @@ function toAnthropicContentBlock(block: CanonicalContentBlock): unknown { type: "audio", source: { type: "base64", media_type: block.mimeType, data: block.data }, }; + case "video": + return block.source === "url" + ? { type: "video", source: { type: "url", url: block.data } } + : { + type: "video", + source: { type: "base64", media_type: block.mimeType, data: block.data }, + }; case "tool_call": return { type: "tool_use", id: block.id, name: block.name, input: block.input }; case "tool_result": diff --git a/src/model/providers/openai-responses/request.ts b/src/model/providers/openai-responses/request.ts index 5c5647ad4..20d0bb249 100644 --- a/src/model/providers/openai-responses/request.ts +++ b/src/model/providers/openai-responses/request.ts @@ -196,9 +196,10 @@ function toResponsesContentPart(block: CanonicalContentBlock): Record