diff --git a/src/model/index.ts b/src/model/index.ts index 769221a29..6b5ed799a 100644 --- a/src/model/index.ts +++ b/src/model/index.ts @@ -79,6 +79,7 @@ export type { CanonicalTextBlock, CanonicalThinkingBlock, CanonicalThinkingConfig, + CanonicalVideoBlock, CanonicalToolCall, CanonicalToolCallBlock, CanonicalToolChoice, diff --git a/src/model/protocol/canonical.ts b/src/model/protocol/canonical.ts index 6b09299ef..214121af8 100644 --- a/src/model/protocol/canonical.ts +++ b/src/model/protocol/canonical.ts @@ -51,6 +51,15 @@ export type CanonicalAudioBlock = { durationSeconds?: number; }; +export type CanonicalVideoBlock = { + type: "video"; + source: "base64" | "url"; + data: string; + mimeType: string; + bytes?: number; + durationSeconds?: number; +}; + export type CanonicalToolCall = { id: string; name: string; @@ -114,7 +123,7 @@ export type CanonicalMediaReferenceBlock = { preview: string; hasMore: boolean; mimeType: string; - mediaType: "image" | "pdf" | "audio"; + mediaType: "image" | "pdf" | "audio" | "video"; pages?: number; detail?: "auto" | "low" | "high"; reason?: string; @@ -128,6 +137,7 @@ export type CanonicalContentBlock = | CanonicalImageBlock | CanonicalPdfBlock | CanonicalAudioBlock + | CanonicalVideoBlock | CanonicalToolCallBlock | CanonicalToolResultBlock | CanonicalToolResultReferenceBlock diff --git a/src/model/protocol/multimodal.ts b/src/model/protocol/multimodal.ts index 39c727b48..1ddb964c6 100644 --- a/src/model/protocol/multimodal.ts +++ b/src/model/protocol/multimodal.ts @@ -6,7 +6,7 @@ import type { } from "./canonical.js"; import { messageContent } from "./clone.js"; -export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio"] as const; +export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio", "video"] as const; export type InputModality = (typeof SUPPORTED_INPUT_MODALITIES)[number]; @@ -42,7 +42,7 @@ export function downgradeUnsupportedContent( constraints: MultimodalConstraints, ): void { const allowed = new Set(constraints.input); - if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio")) return; + if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio") && allowed.has("video")) return; for (const msg of messages) { const content = messageContent(msg); @@ -90,6 +90,9 @@ function mediaBlockToPlaceholder( if (block.type === "audio" && !allowed.has("audio")) { return `[Audio: ${block.mimeType} — omitted, model does not support audio input]`; } + if (block.type === "video" && !allowed.has("video")) { + return `[Video: ${block.mimeType} — omitted, model does not support video input]`; + } return undefined; } @@ -103,6 +106,8 @@ export function contentBlockToInputModality(block: CanonicalContentBlock): Input return "pdf"; case "audio": return "audio"; + case "video": + return "video"; case "thinking": case "tool_call": case "tool_result": diff --git a/src/model/providers/anthropic/request.ts b/src/model/providers/anthropic/request.ts index 59b9913ac..92e155bd3 100644 --- a/src/model/providers/anthropic/request.ts +++ b/src/model/providers/anthropic/request.ts @@ -184,6 +184,13 @@ function toAnthropicContentBlock(block: CanonicalContentBlock): unknown { type: "audio", source: { type: "base64", media_type: block.mimeType, data: block.data }, }; + case "video": + return block.source === "url" + ? { type: "video", source: { type: "url", url: block.data } } + : { + type: "video", + source: { type: "base64", media_type: block.mimeType, data: block.data }, + }; case "tool_call": return { type: "tool_use", id: block.id, name: block.name, input: block.input }; case "tool_result": diff --git a/src/model/providers/google/request.ts b/src/model/providers/google/request.ts index 193f98af8..7f10796e5 100644 --- a/src/model/providers/google/request.ts +++ b/src/model/providers/google/request.ts @@ -167,6 +167,7 @@ function toGoogleParts(block: CanonicalContentBlock, toolNamesById: Map, + block: Extract, ): Part { if (block.source === "url") { return { fileData: { fileUri: block.data, mimeType: block.mimeType } }; diff --git a/src/model/providers/openai-responses/request.ts b/src/model/providers/openai-responses/request.ts index 4efc5b598..20d0bb249 100644 --- a/src/model/providers/openai-responses/request.ts +++ b/src/model/providers/openai-responses/request.ts @@ -195,6 +195,11 @@ function toResponsesContentPart(block: CanonicalContentBlock): Record { + const request: CanonicalModelRequest = { + provider: "compatible", + model: "video-model", + messages: [{ + role: "user", + content: [{ + type: "video", + source: "url", + data: "https://example.com/input.mp4", + mimeType: "video/mp4", + }], + }], + }; + const config = modelConfig(); + + const { provider, model } = validateModelRequest(request, config); + const required = collectRequiredInputModalities(request.messages); + const openAIBody = buildOpenAIRequest(request, model, provider); + const responsesBody = buildOpenAIResponsesRequest(request, model, provider); + const anthropicBody = buildAnthropicRequest(request, model); + + assert.deepEqual(required, ["video"]); + assert.equal(supportsRequiredModalities(model.multimodal, required), true); + assert.deepEqual(openAIBody.messages, [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "https://example.com/input.mp4" }, + }], + }]); + assert.deepEqual(responsesBody.input, [{ + role: "user", + content: [{ + type: "input_video", + video_url: "https://example.com/input.mp4", + }], + }]); + assert.deepEqual(anthropicBody.messages, [{ + role: "user", + content: [{ + type: "video", + source: { type: "url", url: "https://example.com/input.mp4" }, + }], + }]); +}); + +function modelConfig(): ModelConfig { + const capabilities: ModelCapabilities = { + supportsToolUse: true, + supportsStreaming: true, + supportsParallelToolCalls: true, + supportsThinking: true, + supportsJsonSchema: true, + supportsSystemPrompt: true, + supportsPromptCache: true, + maxContextTokens: 1_000_000, + maxOutputTokens: 16_384, + }; + const model: ModelDefinition = { + id: "video-model", + capabilities, + multimodal: { input: ["text", "image", "video"] }, + }; + const provider: ProviderConfig = { + id: "compatible", + protocol: "openai", + url: "https://example.invalid/v1", + apiKey: "test", + headers: {}, + models: { [model.id]: model }, + }; + return { providers: { [provider.id]: provider } }; +}