Add end-to-end video input support

src/model/index.ts

@@ -79,6 +79,7 @@ export type {
   CanonicalTextBlock,
   CanonicalThinkingBlock,
   CanonicalThinkingConfig,
+  CanonicalVideoBlock,
   CanonicalToolCall,
   CanonicalToolCallBlock,
   CanonicalToolChoice,

src/model/protocol/canonical.ts

@@ -51,6 +51,15 @@ export type CanonicalAudioBlock = {
   durationSeconds?: number;
 };
 
+export type CanonicalVideoBlock = {
+  type: "video";
+  source: "base64" | "url";
+  data: string;
+  mimeType: string;
+  bytes?: number;
+  durationSeconds?: number;
+};
+
 export type CanonicalToolCall = {
   id: string;
   name: string;
@@ -114,7 +123,7 @@ export type CanonicalMediaReferenceBlock = {
   preview: string;
   hasMore: boolean;
   mimeType: string;
-  mediaType: "image" | "pdf" | "audio";
+  mediaType: "image" | "pdf" | "audio" | "video";
   pages?: number;
   detail?: "auto" | "low" | "high";
   reason?: string;
@@ -128,6 +137,7 @@ export type CanonicalContentBlock =
   | CanonicalImageBlock
   | CanonicalPdfBlock
   | CanonicalAudioBlock
+  | CanonicalVideoBlock
   | CanonicalToolCallBlock
   | CanonicalToolResultBlock
   | CanonicalToolResultReferenceBlock

src/model/protocol/multimodal.ts

@@ -6,7 +6,7 @@ import type {
 } from "./canonical.js";
 import { messageContent } from "./clone.js";
 
-export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio"] as const;
+export const SUPPORTED_INPUT_MODALITIES = ["text", "image", "pdf", "audio", "video"] as const;
 
 export type InputModality = (typeof SUPPORTED_INPUT_MODALITIES)[number];
 
@@ -42,7 +42,7 @@ export function downgradeUnsupportedContent(
   constraints: MultimodalConstraints,
 ): void {
   const allowed = new Set<InputModality>(constraints.input);
-  if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio")) return;
+  if (allowed.has("image") && allowed.has("pdf") && allowed.has("audio") && allowed.has("video")) return;
 
   for (const msg of messages) {
     const content = messageContent(msg);
@@ -90,6 +90,9 @@ function mediaBlockToPlaceholder(
   if (block.type === "audio" && !allowed.has("audio")) {
     return `[Audio: ${block.mimeType} — omitted, model does not support audio input]`;
   }
+  if (block.type === "video" && !allowed.has("video")) {
+    return `[Video: ${block.mimeType} — omitted, model does not support video input]`;
+  }
   return undefined;
 }
 
@@ -103,6 +106,8 @@ export function contentBlockToInputModality(block: CanonicalContentBlock): Input
       return "pdf";
     case "audio":
       return "audio";
+    case "video":
+      return "video";
     case "thinking":
     case "tool_call":
     case "tool_result":

src/model/providers/anthropic/request.ts

@@ -184,6 +184,13 @@ function toAnthropicContentBlock(block: CanonicalContentBlock): unknown {
             type: "audio",
             source: { type: "base64", media_type: block.mimeType, data: block.data },
           };
+    case "video":
+      return block.source === "url"
+        ? { type: "video", source: { type: "url", url: block.data } }
+        : {
+            type: "video",
+            source: { type: "base64", media_type: block.mimeType, data: block.data },
+          };
     case "tool_call":
       return { type: "tool_use", id: block.id, name: block.name, input: block.input };
     case "tool_result":

src/model/providers/google/request.ts

@@ -167,6 +167,7 @@ function toGoogleParts(block: CanonicalContentBlock, toolNamesById: Map<string,
     case "image":
     case "pdf":
     case "audio":
+    case "video":
       return [toGoogleMediaPart(block)];
     case "tool_call":
       return [{
@@ -198,7 +199,7 @@ function toGoogleParts(block: CanonicalContentBlock, toolNamesById: Map<string,
 }
 
 function toGoogleMediaPart(
-  block: Extract<CanonicalContentBlock, { type: "image" | "pdf" | "audio" }>,
+  block: Extract<CanonicalContentBlock, { type: "image" | "pdf" | "audio" | "video" }>,
 ): Part {
   if (block.source === "url") {
     return { fileData: { fileUri: block.data, mimeType: block.mimeType } };

src/model/providers/openai-responses/request.ts

@@ -195,6 +195,11 @@ function toResponsesContentPart(block: CanonicalContentBlock): Record<string, un
       return block.source === "url"
         ? [{ type: "input_text", text: `[Audio URL: ${block.data}]` }]
         : [{ type: "input_text", text: "[Audio content omitted]" }];
+    case "video":
+      return [{
+        type: "input_video",
+        video_url: block.source === "url" ? block.data : `data:${block.mimeType};base64,${block.data}`,
+      }];
     case "media_reference":
       return [{ type: "input_text", text: block.preview }];
     case "tool_call":

src/model/providers/openai/request.ts

@@ -324,6 +324,13 @@ function toOpenAIContent(blocks: CanonicalContentBlock[]): string | unknown[] {
         return block.source === "url"
           ? { type: "input_audio", audio_url: block.data }
           : { type: "input_audio", input_audio: { data: block.data, format: block.mimeType } };
+      case "video":
+        return {
+          type: "video_url",
+          video_url: {
+            url: block.source === "url" ? block.data : `data:${block.mimeType};base64,${block.data}`,
+          },
+        };
       case "pdf":
         return {
           type: "image_url",

src/model/request/materializeMediaReferences.ts

@@ -110,6 +110,16 @@ function toMediaBlock(
     };
   }
 
+  if (block.mediaType === "video") {
+    return {
+      type: "video",
+      source: "base64",
+      data,
+      mimeType: block.mimeType,
+      bytes: block.originalBytes,
+    };
+  }
+
   return undefined;
 }
 

src/router/utils/mediaRequirements.ts

@@ -4,7 +4,7 @@ import type {
 } from "../../model/index.js";
 import type { InputModality, MultimodalConstraints } from "../../model/index.js";
 
-const MEDIA_MODALITY_ORDER: InputModality[] = ["image", "pdf", "audio"];
+const MEDIA_MODALITY_ORDER: InputModality[] = ["image", "pdf", "audio", "video"];
 
 export function collectRequiredInputModalities(
   messages: CanonicalMessage[],
@@ -52,6 +52,9 @@ function collectFromBlock(
     case "audio":
       required.add("audio");
       return;
+    case "video":
+      required.add("video");
+      return;
     case "tool_result":
       for (const item of block.content) {
         if (item.type === "image" || item.type === "pdf") {

tests/model/request/videoInput.spec.ts

@@ -0,0 +1,95 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { buildAnthropicRequest } from "../../../src/model/providers/anthropic/request.js";
+import { buildOpenAIResponsesRequest } from "../../../src/model/providers/openai-responses/request.js";
+import {
+  buildOpenAIRequest,
+} from "../../../src/model/providers/openai/request.js";
+import { validateModelRequest } from "../../../src/model/request/validateModelRequest.js";
+import type {
+  CanonicalModelRequest,
+  ModelConfig,
+  ModelDefinition,
+  ProviderConfig,
+} from "../../../src/model/protocol/canonical.js";
+import type { ModelCapabilities } from "../../../src/model/protocol/capabilities.js";
+import {
+  collectRequiredInputModalities,
+  supportsRequiredModalities,
+} from "../../../src/router/utils/mediaRequirements.js";
+
+test("video input is validated, routed, and serialized for compatible models", () => {
+  const request: CanonicalModelRequest = {
+    provider: "compatible",
+    model: "video-model",
+    messages: [{
+      role: "user",
+      content: [{
+        type: "video",
+        source: "url",
+        data: "https://example.com/input.mp4",
+        mimeType: "video/mp4",
+      }],
+    }],
+  };
+  const config = modelConfig();
+
+  const { provider, model } = validateModelRequest(request, config);
+  const required = collectRequiredInputModalities(request.messages);
+  const openAIBody = buildOpenAIRequest(request, model, provider);
+  const responsesBody = buildOpenAIResponsesRequest(request, model, provider);
+  const anthropicBody = buildAnthropicRequest(request, model);
+
+  assert.deepEqual(required, ["video"]);
+  assert.equal(supportsRequiredModalities(model.multimodal, required), true);
+  assert.deepEqual(openAIBody.messages, [{
+    role: "user",
+    content: [{
+      type: "video_url",
+      video_url: { url: "https://example.com/input.mp4" },
+    }],
+  }]);
+  assert.deepEqual(responsesBody.input, [{
+    role: "user",
+    content: [{
+      type: "input_video",
+      video_url: "https://example.com/input.mp4",
+    }],
+  }]);
+  assert.deepEqual(anthropicBody.messages, [{
+    role: "user",
+    content: [{
+      type: "video",
+      source: { type: "url", url: "https://example.com/input.mp4" },
+    }],
+  }]);
+});
+
+function modelConfig(): ModelConfig {
+  const capabilities: ModelCapabilities = {
+    supportsToolUse: true,
+    supportsStreaming: true,
+    supportsParallelToolCalls: true,
+    supportsThinking: true,
+    supportsJsonSchema: true,
+    supportsSystemPrompt: true,
+    supportsPromptCache: true,
+    maxContextTokens: 1_000_000,
+    maxOutputTokens: 16_384,
+  };
+  const model: ModelDefinition = {
+    id: "video-model",
+    capabilities,
+    multimodal: { input: ["text", "image", "video"] },
+  };
+  const provider: ProviderConfig = {
+    id: "compatible",
+    protocol: "openai",
+    url: "https://example.invalid/v1",
+    apiKey: "test",
+    headers: {},
+    models: { [model.id]: model },
+  };
+  return { providers: { [provider.id]: provider } };
+}