Hand the agent a file for anything that is not a picture

A video, a note or a drawing the person placed was drawn and sent as a
picture, as a board was. Now only pictures go as pictures; the rest keep
their drawing for the tile and point the agent at the file behind them,
`files/…` or `canvas.json#<id>`, in the project they were attached from.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: Jing-yilin

RELEASE-NOTES.md

@@ -19,9 +19,10 @@ The app, its skills and the toolkit carry one version.
 
 Everything below is on `main` and reaches no install until a version is cut.
 
-- **A mockup goes to the agent as its file.** Its **+** still shows a picture
-  of it in the chat, but the agent is handed the board's `.html` path rather
-  than that picture, since the file is what it reads and changes.
+- **Only pictures go to the agent as pictures.** The **+** on a board, a
+  video or a note still shows a picture of it in the chat, but the agent is
+  handed the file behind it rather than that picture, since the file is what
+  it can read and change.
 
 ## v1.7.0
 

canvas/server/agent.ts

@@ -311,13 +311,15 @@ export function createAgentServer(options: {
             )
               return send(400, "bad image data");
             if (
-              i.page !== undefined &&
-              (typeof i.page !== "object" ||
+              i.reference !== undefined &&
+              (typeof i.reference !== "object" ||
                 !["project", "community"].every(
-                  (k) => i.page[k] === undefined || typeof i.page[k] === "string",
+                  (k) =>
+                    i.reference[k] === undefined ||
+                    typeof i.reference[k] === "string",
                 ))
             )
-              return send(400, "bad image page");
+              return send(400, "bad image reference");
           }
           // The body cap above is the panel's limit in base64; a client that is not the
           // panel meets the limit itself here, in the bytes the files come out as.
@@ -507,20 +509,24 @@ export function createAgentServer(options: {
               name: string;
               type: string;
               data: string;
-              page?: { project?: string; community?: string };
+              reference?: { project?: string; community?: string };
             }) => {
-              // A mockup's picture is kept for the panel, and the agent is pointed at its
-              // file, `<slug>/<file>.html` in the canvases of the project it was attached
-              // from, which need not be the one it is sent from. A community project's is not
-              // on this machine, so it keeps the name `sp fetch` finds it by.
-              const [slug, file, ...rest] = i.name.split("/");
-              const from = i.page?.project && projects().get(i.page.project);
+              // Anything but a picture — a board, a video, a note — keeps its picture for the
+              // panel, and the agent is pointed at its file, `<slug>/<path>` in the canvases of
+              // the project it was attached from, which need not be the one it is sent from. A
+              // community project's is not on this machine, so it keeps the name `sp fetch`
+              // finds it by.
+              const [slug, ...rest] = i.name.split("/");
+              const ref = i.reference;
+              const from = ref?.project && projects().get(ref.project);
               const local =
-                i.page?.community === undefined &&
-                (i.page?.project === undefined || from) &&
-                rest.length === 0 &&
+                ref?.community === undefined &&
+                (ref?.project === undefined || from) &&
                 SAFE_NAME.test(slug) &&
-                SAFE_NAME.test(file ?? "");
+                rest.length > 0 &&
+                rest.every(
+                  (s) => s && !s.startsWith(".") && !s.includes("\\"),
+                );
               return {
                 ...i,
                 path: picture(
@@ -529,19 +535,19 @@ export function createAgentServer(options: {
                   i.type,
                   Buffer.from(i.data, "base64"),
                 ),
-                page:
-                  i.page &&
+                reference:
+                  ref &&
                   (local
                     ? path.join(
                         folderOf(
                           from ? path.join(from, CANVASES) : examplesDir,
                           examplesDir,
                           slug,
                         ),
-                        file,
+                        ...rest,
                       )
-                    : i.page.community
-                      ? `${i.name} of the community project ${i.page.community}`
+                    : ref.community
+                      ? `${i.name} of the community project ${ref.community}`
                       : i.name),
               };
             },

canvas/src/ChatPanel.tsx

@@ -87,9 +87,9 @@ const OPEN_KEY = "sp-chat-open";
 
 /**
  * What the canvas hands the chat panel when the button is pressed (canvasAttach.tsx): a picture
- * to attach to the message, or the reason none was. A board comes over as a picture too, for its
- * tile only: the agent is handed the board's file, which its name says, and the panel shows that
- * name under the tile. And the start
+ * to attach to the message, or the reason none was. A board, a video or a note comes over as a
+ * picture too, for its tile only: the agent is handed the file its name says, and the panel shows
+ * that name under the tile. And the start
  * of a message, from the strip's "+" (CanvasStrip.tsx), because a canvas is only ever the
  * agent's work, and a folder with no boards in it is not one. And a whole message, sent as it is,
  * from the new-project dialog (AppShell.tsx), which starts the agent defining the product.
@@ -105,8 +105,9 @@ const OPEN_KEY = "sp-chat-open";
 export const CANVAS_ATTACH = "sp:canvas-attach";
 
 export type CanvasAttachDetail =
-  /** `page` for a mockup, whose drawing is only the tile's: the agent is handed its file. */
-  | { kind: "board"; name: string; src: string; page?: true }
+  /** `reference` for anything but a picture, whose drawing is only the tile's: the agent is
+   *  pointed at its file instead. */
+  | { kind: "board"; name: string; src: string; reference?: true }
   | { kind: "image"; file: File }
   | { kind: "error"; message: string }
   | { kind: "draft"; text: string }
@@ -251,9 +252,9 @@ interface Attached {
   url: string;
   /** A board still being drawn, with no `url` yet, or one whose drawing failed. */
   state?: "pending" | "failed";
-  /** A mockup, and the project it was attached from, which the panel may since have left: the
-   *  agent gets its file there rather than this picture of it (agents.ts). */
-  page?: { project?: string; community?: string };
+  /** Anything but a picture, and the project it was attached from, which the panel may since
+   *  have left: the agent gets its file there rather than this picture of it (agents.ts). */
+  reference?: { project?: string; community?: string };
 }
 
 /**
@@ -718,7 +719,7 @@ export function ChatPanel(props: {
                 type: r.file.type,
                 size: r.file.size,
                 url: r.url,
-                page: t.page,
+                reference: t.reference,
               }
             : t;
         });
@@ -764,7 +765,7 @@ export function ChatPanel(props: {
    * drawing asked of the server, to land in that tile. Asked for again it keeps the tile it has —
    * one already there or on its way is only named again, and one that failed is drawn again.
    */
-  const addBoard = (name: string, src: string, page?: true) => {
+  const addBoard = (name: string, src: string, reference?: true) => {
     let tile = tray.current.find((t) => t.name === name);
     if (!tile && tray.current.length >= MAX_IMAGES)
       return setSendError(
@@ -785,7 +786,7 @@ export function ChatPanel(props: {
       size: 0,
       url: "",
       state: "pending",
-      page: page && { project, community },
+      reference: reference && { project, community },
     };
     const next = tile;
     tray.current = [...tray.current.filter((t) => t.n !== next.n), next].sort(
@@ -827,7 +828,7 @@ export function ChatPanel(props: {
       // that is still being read.
       if (detail.kind === "board") {
         adds.current = adds.current.then(() =>
-          addBoard(detail.name, detail.src, detail.page),
+          addBoard(detail.name, detail.src, detail.reference),
         );
         return;
       }
@@ -945,12 +946,12 @@ export function ChatPanel(props: {
           agent,
           model,
           effort,
-          images: attached.map(({ n, name, type, url, page }) => ({
+          images: attached.map(({ n, name, type, url, reference }) => ({
             n,
             name,
             type,
             data: url.slice(url.indexOf(",") + 1),
-            page,
+            reference,
           })),
         }),
       });

canvas/src/agents.test.ts

@@ -94,7 +94,7 @@ describe("AGENTS", () => {
       type: "image/png",
       data: "CCC",
       path: "/tmp/sp-chat-r/2.png",
-      page: "/proj/canvases/shop/01-home.html",
+      reference: "/proj/canvases/shop/01-home.html",
     };
     expect(
       JSON.parse(def("claude").stdin("tighten #2", "P", [board])).message

canvas/src/agents.ts

@@ -80,9 +80,10 @@ export interface AgentImage {
   /** Where the server wrote it, for an agent that takes files rather than bytes; gone once
    *  that agent has exited. */
   path: string;
-  /** A mockup's HTML file, when the picture is only the panel's drawing of one: the agent is
-   *  pointed at the file and not handed the picture, since the file is what it reads and edits. */
-  page?: string;
+  /** The file behind it — a board's HTML, a video, a canvas.json record — when the picture is
+   *  only the panel's drawing of it: the agent is pointed at the file, which is what it can read
+   *  and change, and not handed the picture. */
+  reference?: string;
 }
 
 /** What the composer chose, handed to `args`. An empty string means the CLI decides. */
@@ -203,8 +204,8 @@ export const AGENTS: AgentDef[] = [
     // terminal, so the number arrives as something already read rather than a local convention.
     stdin: (message, _preamble, images) => {
       const blocks = images.flatMap((i) =>
-        i.page
-          ? [{ type: "text", text: `[Image #${i.n}] ${i.page}` }]
+        i.reference
+          ? [{ type: "text", text: `[Image #${i.n}] ${i.reference}` }]
           : [
               { type: "text", text: `[Image #${i.n}] ${i.name}` },
               {
@@ -315,7 +316,7 @@ export const AGENTS: AgentDef[] = [
     stdin: (message, preamble, images) =>
       [
         preamble,
-        images.map((i) => `[Image #${i.n}] ${i.page ?? i.path}`).join("\n"),
+        images.map((i) => `[Image #${i.n}] ${i.reference ?? i.path}`).join("\n"),
         message,
       ]
         .filter(Boolean)

canvas/src/canvasAttach.tsx

@@ -54,17 +54,19 @@ async function attach(editor: Editor, target: TLShape) {
           `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`,
         window.location.href,
       ).href;
-      return dispatchAttach({ kind: "board", name, src, page: true });
+      return dispatchAttach({ kind: "board", name, src, reference: true });
     }
-    // One of the person's own: whatever it is, the agent gets a picture of it, named by where it
-    // reads the thing itself (canvasContent.ts).
+    // One of the person's own, drawn for the tile and named by where the agent reads the thing
+    // itself (canvasContent.ts). A picture goes over as one; anything else — a video, a note, a
+    // drawing — is a file the agent is pointed at, as a board is.
     const slug = personsShape(editor, target);
     if (slug) {
       const { blob } = await editor.toImage([target.id], { format: "png" });
       return dispatchAttach({
         kind: "board",
         name: personsShapeName(editor, target, slug),
         src: URL.createObjectURL(blob),
+        reference: target.type === "image" ? undefined : true,
       });
     }
     const shape = target as TLImageShape;