fix: wire image data through to Claude for screenshot/photo support (RichardAtCT#168)

The bot was downloading and base64-encoding images but only passing a
text prompt to Claude, never the actual image data. This sends images
as multimodal content blocks via the SDK AsyncIterable path so Claude
can actually see uploaded screenshots and photos.

Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>

Author: 2 people

src/bot/features/image_handler.py

@@ -57,7 +57,6 @@ async def process_image(
         else:
             prompt = self._create_generic_prompt(caption)
 
-        # Convert to base64 for Claude (if supported in future)
         base64_image = base64.b64encode(image_bytes).decode("utf-8")
 
         return ProcessedImage(

src/bot/orchestrator.py

@@ -54,6 +54,13 @@ def _format_size(size_bytes: int) -> str:
         return f"{size_bytes / (1024 * 1024):.1f} MB"
 
 
+_MEDIA_TYPE_MAP = {
+    "png": "image/png",
+    "jpeg": "image/jpeg",
+    "gif": "image/gif",
+    "webp": "image/webp",
+}
+
 # Patterns that look like secrets/credentials in CLI arguments
 _SECRET_PATTERNS: List[re.Pattern[str]] = [
     # API keys / tokens (sk-ant-..., sk-..., ghp_..., gho_..., github_pat_..., xoxb-...)
@@ -1751,13 +1758,22 @@ async def agentic_photo(
             processed_image = await image_handler.process_image(
                 photo, update.message.caption
             )
+            fmt = processed_image.metadata.get("format", "png")
+            images = [
+                {
+                    "data": processed_image.base64_data,
+                    "media_type": _MEDIA_TYPE_MAP.get(fmt, "image/png"),
+                }
+            ]
+
             await self._handle_agentic_media_message(
                 update=update,
                 context=context,
                 prompt=processed_image.prompt,
                 progress_msg=progress_msg,
                 user_id=user_id,
                 chat=chat,
+                images=images,
             )
 
         except Exception as e:
@@ -1820,6 +1836,7 @@ async def _handle_agentic_media_message(
         progress_msg: Any,
         user_id: int,
         chat: Any,
+        images: Optional[List[Dict[str, str]]] = None,
     ) -> None:
         """Run a media-derived prompt through Claude and send responses."""
         assert update.message is not None
@@ -1866,6 +1883,7 @@ async def _handle_agentic_media_message(
                 session_id=session_id,
                 on_stream=on_stream,
                 force_new=force_new,
+                images=images,
             )
         finally:
             heartbeat.cancel()

src/claude/facade.py

@@ -42,6 +42,7 @@ async def run_command(
         interrupt_event: Optional["asyncio.Event"] = None,
         chat_id: Optional[int] = None,
         message_thread_id: Optional[int] = None,
+        images: Optional[List[Dict[str, str]]] = None,
     ) -> ClaudeResponse:
         """Run Claude Code command with full integration."""
         logger.info(
@@ -124,6 +125,7 @@ async def run_command(
                     continue_session=should_continue,
                     stream_callback=on_stream,
                     interrupt_event=interrupt_event,
+                    images=images,
                 )
                 execution_completed = True
             except Exception as resume_error:
@@ -156,6 +158,7 @@ async def run_command(
                         continue_session=False,
                         stream_callback=on_stream,
                         interrupt_event=interrupt_event,
+                        images=images,
                     )
                     execution_completed = True
                 else:
@@ -219,6 +222,7 @@ async def _execute(
         continue_session: bool = False,
         stream_callback: Optional[Callable] = None,
         interrupt_event: Optional[asyncio.Event] = None,
+        images: Optional[List[Dict[str, str]]] = None,
     ) -> ClaudeResponse:
         """Execute command via SDK."""
         return await self.sdk_manager.execute_command(
@@ -228,6 +232,7 @@ async def _execute(
             continue_session=continue_session,
             stream_callback=stream_callback,
             interrupt_event=interrupt_event,
+            images=images,
         )
 
     async def _find_resumable_session(

src/claude/sdk_integration.py

@@ -4,7 +4,7 @@
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, AsyncIterator, Callable, Dict, List, Optional
 
 import structlog
 from claude_agent_sdk import (
@@ -277,6 +277,7 @@ async def execute_command(
         continue_session: bool = False,
         stream_callback: Optional[Callable[[StreamUpdate], None]] = None,
         interrupt_event: Optional[asyncio.Event] = None,
+        images: Optional[List[Dict[str, str]]] = None,
     ) -> ClaudeResponse:
         """Execute Claude Code command via SDK."""
         start_time = asyncio.get_event_loop().time()
@@ -370,7 +371,37 @@ async def _run_client() -> None:
                 client = ClaudeSDKClient(options)
                 try:
                     await client.connect()
-                    await client.query(prompt)
+
+                    if images:
+                        content_blocks: List[Dict[str, Any]] = []
+                        for img in images:
+                            media_type = img.get("media_type", "image/png")
+                            content_blocks.append(
+                                {
+                                    "type": "image",
+                                    "source": {
+                                        "type": "base64",
+                                        "media_type": media_type,
+                                        "data": img["data"],
+                                    },
+                                }
+                            )
+                        content_blocks.append({"type": "text", "text": prompt})
+
+                        multimodal_msg = {
+                            "type": "user",
+                            "message": {
+                                "role": "user",
+                                "content": content_blocks,
+                            },
+                        }
+
+                        async def _multimodal_prompt() -> AsyncIterator[Dict[str, Any]]:
+                            yield multimodal_msg
+
+                        await client.query(_multimodal_prompt())
+                    else:
+                        await client.query(prompt)
 
                     assert client._query is not None
                     async for raw_data in client._query.receive_messages():