Hand the agent a file for anything that is not a picture

RELEASE-NOTES.md

@@ -19,9 +19,10 @@ The app, its skills and the toolkit carry one version.
 
 Everything below is on `main` and reaches no install until a version is cut.
 
-- **A mockup goes to the agent as its file.** Its **+** still shows a picture
-  of it in the chat, but the agent is handed the board's `.html` path rather
-  than that picture, since the file is what it reads and changes.
+- **Only pictures go to the agent as pictures.** The **+** on a board, a
+  video or a note still shows a picture of it in the chat, but the agent is
+  handed the file behind it rather than that picture, since the file is what
+  it can read and change.
 
 ## v1.7.0
 

canvas/server/agent.ts

@@ -17,7 +17,7 @@ import path from "node:path";
 import { CANVASES } from "./boards.ts";
 import { command, stop } from "./command.ts";
 import { AGENT_SKILLS, installSkills } from "./skills.ts";
-import { folderOf, sameOrigin } from "./sp.ts";
+import { canvasFile, folderOf, sameOrigin } from "./sp.ts";
 import { SAFE_NAME } from "../src/layoutEdit.ts";
 import {
   attach,
@@ -311,13 +311,15 @@ export function createAgentServer(options: {
             )
               return send(400, "bad image data");
             if (
-              i.page !== undefined &&
-              (typeof i.page !== "object" ||
+              i.reference !== undefined &&
+              (typeof i.reference !== "object" ||
                 !["project", "community"].every(
-                  (k) => i.page[k] === undefined || typeof i.page[k] === "string",
+                  (k) =>
+                    i.reference[k] === undefined ||
+                    typeof i.reference[k] === "string",
                 ))
             )
-              return send(400, "bad image page");
+              return send(400, "bad image reference");
           }
           // The body cap above is the panel's limit in base64; a client that is not the
           // panel meets the limit itself here, in the bytes the files come out as.
@@ -507,20 +509,23 @@ export function createAgentServer(options: {
               name: string;
               type: string;
               data: string;
-              page?: { project?: string; community?: string };
+              reference?: { project?: string; community?: string };
             }) => {
-              // A mockup's picture is kept for the panel, and the agent is pointed at its
-              // file, `<slug>/<file>.html` in the canvases of the project it was attached
-              // from, which need not be the one it is sent from. A community project's is not
-              // on this machine, so it keeps the name `sp fetch` finds it by.
-              const [slug, file, ...rest] = i.name.split("/");
-              const from = i.page?.project && projects().get(i.page.project);
-              const local =
-                i.page?.community === undefined &&
-                (i.page?.project === undefined || from) &&
-                rest.length === 0 &&
-                SAFE_NAME.test(slug) &&
-                SAFE_NAME.test(file ?? "");
+              // Anything but a picture (a board, a video, a note) keeps its picture for the
+              // panel, and the agent is pointed at its file, in the canvases of the project it
+              // was attached from, which need not be the one it is sent from. A community
+              // project's is not on this machine, so it keeps the name `sp fetch` finds it by.
+              const ref = i.reference;
+              const from = ref?.project && projects().get(ref.project);
+              const file =
+                ref?.community === undefined &&
+                (ref?.project === undefined || from)
+                  ? canvasFile(
+                      from ? path.join(from, CANVASES) : examplesDir,
+                      examplesDir,
+                      i.name,
+                    )
+                  : undefined;
               return {
                 ...i,
                 path: picture(
@@ -529,20 +534,12 @@ export function createAgentServer(options: {
                   i.type,
                   Buffer.from(i.data, "base64"),
                 ),
-                page:
-                  i.page &&
-                  (local
-                    ? path.join(
-                        folderOf(
-                          from ? path.join(from, CANVASES) : examplesDir,
-                          examplesDir,
-                          slug,
-                        ),
-                        file,
-                      )
-                    : i.page.community
-                      ? `${i.name} of the community project ${i.page.community}`
-                      : i.name),
+                reference:
+                  ref &&
+                  (file ??
+                    (ref.community
+                      ? `${i.name} of the community project ${ref.community}`
+                      : i.name)),
               };
             },
           );

canvas/server/sp.test.ts

@@ -5,7 +5,7 @@ import os from "node:os";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 import { expect, it, vi } from "vitest";
-import { createSpServer } from "./sp.ts";
+import { canvasFile, createSpServer } from "./sp.ts";
 
 // The examples directory is listed beside the project's canvases, shadowed by a folder of the
 // project's own, refused every write, and cloned into the project.
@@ -522,3 +522,34 @@ async function serve(options: Parameters<typeof createSpServer>[0]) {
   };
   return { ask, listen, close };
 }
+
+// What a chat attachment names is a file in a canvas folder, and nothing a browser sends reaches
+// outside one.
+it("resolves an attachment's name to a file in its canvas folder", () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "sp-attach-"));
+  const own = path.join(tmp, "project");
+  const examples = path.join(tmp, "examples");
+  fs.mkdirSync(path.join(examples, "demo"), { recursive: true });
+  expect(canvasFile(own, examples, "shop/01-home.html")).toBe(
+    path.join(own, "shop", "01-home.html"),
+  );
+  expect(canvasFile(own, examples, "shop/files/clip.mp4")).toBe(
+    path.join(own, "shop", "files", "clip.mp4"),
+  );
+  expect(canvasFile(own, examples, "shop/canvas.json#shape:a1")).toBe(
+    path.join(own, "shop", "canvas.json#shape:a1"),
+  );
+  expect(canvasFile(own, examples, "demo/01-a.html")).toBe(
+    path.join(examples, "demo", "01-a.html"),
+  );
+  for (const name of [
+    "shop",
+    "../etc/passwd",
+    "shop/../../etc",
+    "shop/files/..",
+    "shop//x",
+    "shop/.env",
+    "shop/a\\..\\..\\x",
+  ])
+    expect(canvasFile(own, examples, name)).toBeUndefined();
+});

canvas/server/sp.ts

@@ -80,6 +80,20 @@ export function folderOf(
   return !isCanvas && fs.existsSync(example) ? example : own;
 }
 
+/**
+ * The file a chat attachment names as `<slug>/<path>`: a board's HTML, a file under `files/`, or
+ * `canvas.json#<shape id>`. Undefined for a name that is not one, so nothing a browser sends can
+ * point outside a canvas folder: no part may be empty, start with a dot, or hold a backslash.
+ */
+export function canvasFile(canvasesDir: string, examplesDir: string, name: string) {
+  const [slug, ...rest] = name.split("/");
+  return SAFE_NAME.test(slug) &&
+    rest.length > 0 &&
+    rest.every((s) => s && !s.startsWith(".") && !s.includes("\\"))
+    ? path.join(folderOf(canvasesDir, examplesDir, slug), ...rest)
+    : undefined;
+}
+
 /**
  * The project's own settings, beside its canvases: which cover it chose, and the name it is shown
  * by when that is not its folder's. A project made without a name is an "Untitled" folder whose

canvas/src/ChatPanel.tsx

@@ -87,9 +87,9 @@ const OPEN_KEY = "sp-chat-open";
 
 /**
  * What the canvas hands the chat panel when the button is pressed (canvasAttach.tsx): a picture
- * to attach to the message, or the reason none was. A board comes over as a picture too, for its
- * tile only: the agent is handed the board's file, which its name says, and the panel shows that
- * name under the tile. And the start
+ * to attach to the message, or the reason none was. A board, a video or a note comes over as a
+ * picture too, for its tile only: the agent is handed the file its name says, and the panel shows
+ * that name under the tile. And the start
  * of a message, from the strip's "+" (CanvasStrip.tsx), because a canvas is only ever the
  * agent's work, and a folder with no boards in it is not one. And a whole message, sent as it is,
  * from the new-project dialog (AppShell.tsx), which starts the agent defining the product.
@@ -105,8 +105,9 @@ const OPEN_KEY = "sp-chat-open";
 export const CANVAS_ATTACH = "sp:canvas-attach";
 
 export type CanvasAttachDetail =
-  /** `page` for a mockup, whose drawing is only the tile's: the agent is handed its file. */
-  | { kind: "board"; name: string; src: string; page?: true }
+  /** `reference` for anything but a picture, whose drawing is only the tile's: the agent is
+   *  pointed at its file instead. */
+  | { kind: "board"; name: string; src: string; reference?: true }
   | { kind: "image"; file: File }
   | { kind: "error"; message: string }
   | { kind: "draft"; text: string }
@@ -251,9 +252,9 @@ interface Attached {
   url: string;
   /** A board still being drawn, with no `url` yet, or one whose drawing failed. */
   state?: "pending" | "failed";
-  /** A mockup, and the project it was attached from, which the panel may since have left: the
-   *  agent gets its file there rather than this picture of it (agents.ts). */
-  page?: { project?: string; community?: string };
+  /** Anything but a picture, and the project it was attached from, which the panel may since
+   *  have left: the agent gets its file there rather than this picture of it (agents.ts). */
+  reference?: { project?: string; community?: string };
 }
 
 /**
@@ -718,7 +719,7 @@ export function ChatPanel(props: {
                 type: r.file.type,
                 size: r.file.size,
                 url: r.url,
-                page: t.page,
+                reference: t.reference,
               }
             : t;
         });
@@ -764,7 +765,7 @@ export function ChatPanel(props: {
    * drawing asked of the server, to land in that tile. Asked for again it keeps the tile it has —
    * one already there or on its way is only named again, and one that failed is drawn again.
    */
-  const addBoard = (name: string, src: string, page?: true) => {
+  const addBoard = (name: string, src: string, reference?: true) => {
     let tile = tray.current.find((t) => t.name === name);
     if (!tile && tray.current.length >= MAX_IMAGES)
       return setSendError(
@@ -785,7 +786,7 @@ export function ChatPanel(props: {
       size: 0,
       url: "",
       state: "pending",
-      page: page && { project, community },
+      reference: reference && { project, community },
     };
     const next = tile;
     tray.current = [...tray.current.filter((t) => t.n !== next.n), next].sort(
@@ -827,7 +828,7 @@ export function ChatPanel(props: {
       // that is still being read.
       if (detail.kind === "board") {
         adds.current = adds.current.then(() =>
-          addBoard(detail.name, detail.src, detail.page),
+          addBoard(detail.name, detail.src, detail.reference),
         );
         return;
       }
@@ -945,12 +946,12 @@ export function ChatPanel(props: {
           agent,
           model,
           effort,
-          images: attached.map(({ n, name, type, url, page }) => ({
+          images: attached.map(({ n, name, type, url, reference }) => ({
             n,
             name,
             type,
             data: url.slice(url.indexOf(",") + 1),
-            page,
+            reference,
           })),
         }),
       });

canvas/src/agents.test.ts

@@ -94,7 +94,7 @@ describe("AGENTS", () => {
       type: "image/png",
       data: "CCC",
       path: "/tmp/sp-chat-r/2.png",
-      page: "/proj/canvases/shop/01-home.html",
+      reference: "/proj/canvases/shop/01-home.html",
     };
     expect(
       JSON.parse(def("claude").stdin("tighten #2", "P", [board])).message

canvas/src/agents.ts

@@ -80,9 +80,10 @@ export interface AgentImage {
   /** Where the server wrote it, for an agent that takes files rather than bytes; gone once
    *  that agent has exited. */
   path: string;
-  /** A mockup's HTML file, when the picture is only the panel's drawing of one: the agent is
-   *  pointed at the file and not handed the picture, since the file is what it reads and edits. */
-  page?: string;
+  /** The file behind it (a board's HTML, a video, a canvas.json record) when the picture is
+   *  only the panel's drawing of it: the agent is pointed at the file, which is what it can read
+   *  and change, and not handed the picture. */
+  reference?: string;
 }
 
 /** What the composer chose, handed to `args`. An empty string means the CLI decides. */
@@ -203,8 +204,8 @@ export const AGENTS: AgentDef[] = [
     // terminal, so the number arrives as something already read rather than a local convention.
     stdin: (message, _preamble, images) => {
       const blocks = images.flatMap((i) =>
-        i.page
-          ? [{ type: "text", text: `[Image #${i.n}] ${i.page}` }]
+        i.reference
+          ? [{ type: "text", text: `[Image #${i.n}] ${i.reference}` }]
           : [
               { type: "text", text: `[Image #${i.n}] ${i.name}` },
               {
@@ -315,7 +316,7 @@ export const AGENTS: AgentDef[] = [
     stdin: (message, preamble, images) =>
       [
         preamble,
-        images.map((i) => `[Image #${i.n}] ${i.page ?? i.path}`).join("\n"),
+        images.map((i) => `[Image #${i.n}] ${i.reference ?? i.path}`).join("\n"),
         message,
       ]
         .filter(Boolean)

canvas/src/canvasAttach.tsx

@@ -54,17 +54,19 @@ async function attach(editor: Editor, target: TLShape) {
           `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`,
         window.location.href,
       ).href;
-      return dispatchAttach({ kind: "board", name, src, page: true });
+      return dispatchAttach({ kind: "board", name, src, reference: true });
     }
-    // One of the person's own: whatever it is, the agent gets a picture of it, named by where it
-    // reads the thing itself (canvasContent.ts).
+    // One of the person's own, drawn for the tile and named by where the agent reads the thing
+    // itself (canvasContent.ts). A picture goes over as one. Anything else, such as a video, a
+    // note or a drawing, is a file the agent is pointed at, as a board is.
     const slug = personsShape(editor, target);
     if (slug) {
       const { blob } = await editor.toImage([target.id], { format: "png" });
       return dispatchAttach({
         kind: "board",
         name: personsShapeName(editor, target, slug),
         src: URL.createObjectURL(blob),
+        reference: target.type === "image" ? undefined : true,
       });
     }
     const shape = target as TLImageShape;