Implement Codex CLI streaming and image attachments

apps/codex-claw/src/routes/api/send.ts

@@ -1,7 +1,89 @@
+import { Buffer } from 'node:buffer'
 import { randomUUID } from 'node:crypto'
 import { createFileRoute } from '@tanstack/react-router'
 import { json } from '@tanstack/react-start'
-import { resolveCodexSession, sendCodexPrompt } from '../../server/codex-cli'
+import {
+  isSupportedCodexImageMimeType,
+  resolveCodexSession,
+  sendCodexPrompt,
+} from '../../server/codex-cli'
+
+type ParsedAttachment = {
+  mimeType: string
+  content: string
+  name?: string
+}
+
+type AttachmentParseResult =
+  | {
+      ok: true
+      attachments: Array<ParsedAttachment> | undefined
+    }
+  | {
+      ok: false
+      error: string
+    }
+
+function parseAttachments(rawAttachments: unknown): AttachmentParseResult {
+  if (typeof rawAttachments === 'undefined') {
+    return { ok: true, attachments: undefined }
+  }
+
+  if (!Array.isArray(rawAttachments)) {
+    return { ok: false, error: 'attachments must be an array' }
+  }
+
+  const attachments: Array<ParsedAttachment> = []
+
+  for (const rawAttachment of rawAttachments) {
+    if (!rawAttachment || typeof rawAttachment !== 'object') {
+      return { ok: false, error: 'attachment must be an object' }
+    }
+
+    const attachment = rawAttachment as Record<string, unknown>
+    const mimeType =
+      typeof attachment.mimeType === 'string' ? attachment.mimeType.trim() : ''
+    const rawContent =
+      typeof attachment.content === 'string' ? attachment.content : ''
+    const content = normalizeBase64Content(rawContent)
+
+    if (!isSupportedCodexImageMimeType(mimeType)) {
+      return {
+        ok: false,
+        error:
+          'Unsupported attachment type. Please use PNG, JPG, GIF, or WebP images.',
+      }
+    }
+
+    if (!isValidBase64Content(content)) {
+      return {
+        ok: false,
+        error: 'Attachment image data could not be decoded.',
+      }
+    }
+
+    attachments.push({
+      mimeType,
+      content,
+      name: typeof attachment.name === 'string' ? attachment.name : undefined,
+    })
+  }
+
+  return { ok: true, attachments }
+}
+
+function normalizeBase64Content(content: string) {
+  const trimmed = content.trim()
+  const dataUrlMatch = /^data:[^,]+;base64,(?<data>[\s\S]+)$/i.exec(trimmed)
+  return (dataUrlMatch?.groups?.data ?? trimmed).replace(/\s/g, '')
+}
+
+function isValidBase64Content(content: string) {
+  if (!content) return false
+  if (content.length % 4 === 1) return false
+  if (!/^[A-Za-z0-9+/]+={0,2}$/.test(content)) return false
+  return Buffer.from(content, 'base64').length > 0
+}
 
 export const Route = createFileRoute('/api/send')({
   server: {
@@ -21,16 +103,14 @@ export const Route = createFileRoute('/api/send')({
           const thinking =
             typeof body.thinking === 'string' ? body.thinking : undefined
 
-          const rawAttachments = body.attachments
-          const attachments = Array.isArray(rawAttachments)
-            ? rawAttachments.filter(
-                (a: unknown): a is { mimeType: string; content: string } =>
-                  typeof a === 'object' &&
-                  a !== null &&
-                  typeof (a as Record<string, unknown>).mimeType === 'string' &&
-                  typeof (a as Record<string, unknown>).content === 'string',
-              )
-            : undefined
+          const parsedAttachments = parseAttachments(body.attachments)
+          if (!parsedAttachments.ok) {
+            return json(
+              { ok: false, error: parsedAttachments.error },
+              { status: 400 },
+            )
+          }
+          const attachments = parsedAttachments.attachments
 
           if (!message.trim() && (!attachments || attachments.length === 0)) {
             return json(

apps/codex-claw/src/screens/chat/chat-screen.tsx

@@ -259,6 +259,7 @@ export function ChatScreen({
     const attachmentsPayload = attachments?.map((a) => ({
       mimeType: a.file.type,
       content: a.base64,
+      name: a.file.name,
     }))
 
     fetch('/api/send', {

apps/codex-claw/src/screens/chat/components/message-item.tsx

@@ -121,7 +121,7 @@ export function assistantPartRenderOrder(
       }
       continue
     }
-    if (showToolMessages) {
+    if (part.type === 'toolCall' && showToolMessages) {
       order.push('toolCall')
     }
   }

apps/codex-claw/src/screens/chat/types.ts

@@ -27,7 +27,20 @@ export type ThinkingContent = {
   thinkingSignature?: string
 }
 
-export type MessageContent = TextContent | ToolCallContent | ThinkingContent
+export type ImageContent = {
+  type: 'image'
+  source: {
+    type: 'base64'
+    media_type: string
+    data: string
+  }
+}
+
+export type MessageContent =
+  | TextContent
+  | ToolCallContent
+  | ThinkingContent
+  | ImageContent
 
 export type GatewayMessage = {
   role?: string

apps/codex-claw/src/server/codex-cli.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from 'vitest'
+
+import { mergeAssistantText, processCodexJsonLine } from './codex-cli'
+
+describe('processCodexJsonLine', function () {
+  it('parses assistant deltas before the final completed item', function () {
+    const first = processCodexJsonLine(
+      JSON.stringify({
+        type: 'item.updated',
+        item: { type: 'agent_message', text: 'Hel' },
+      }),
+    )
+    const second = processCodexJsonLine(
+      JSON.stringify({
+        type: 'item.updated',
+        item: { type: 'agent_message', text: 'Hello' },
+      }),
+    )
+    const final = processCodexJsonLine(
+      JSON.stringify({
+        type: 'item.completed',
+        item: { type: 'agent_message', text: 'Hello there' },
+      }),
+    )
+
+    expect(first).toEqual({ kind: 'assistant-delta', text: 'Hel' })
+    expect(second).toEqual({ kind: 'assistant-delta', text: 'Hello' })
+    expect(final).toEqual({ kind: 'assistant-final', text: 'Hello there' })
+  })
+
+  it('ignores non-assistant events', function () {
+    expect(
+      processCodexJsonLine(
+        JSON.stringify({
+          type: 'item.completed',
+          item: { type: 'reasoning', text: 'hidden' },
+        }),
+      ),
+    ).toBeNull()
+  })
+})
+
+describe('mergeAssistantText', function () {
+  it('keeps cumulative snapshots in order', function () {
+    const first = mergeAssistantText('', 'Hel', 'assistant-delta')
+    const second = mergeAssistantText(first, 'Hello', 'assistant-delta')
+    const final = mergeAssistantText(second, 'Hello there', 'assistant-final')
+
+    expect(first).toBe('Hel')
+    expect(second).toBe('Hello')
+    expect(final).toBe('Hello there')
+  })
+
+  it('appends token deltas without duplicating repeated chunks', function () {
+    const first = mergeAssistantText('', 'Hel', 'assistant-delta')
+    const second = mergeAssistantText(first, 'lo', 'assistant-delta')
+    const repeated = mergeAssistantText(second, 'lo', 'assistant-delta')
+
+    expect(first).toBe('Hel')
+    expect(second).toBe('Hello')
+    expect(repeated).toBe('Hello')
+  })
+})