Merge pull request #198 from ReScienceLab/magic-pen

Magic pen: Option-drag over a board quotes that region in the chat

Author: Jing-yilin

canvas/src/ChatPanel.tsx

@@ -101,6 +101,8 @@ const OPEN_KEY = "sp-chat-open";
  * A board comes as `board`: its name and where the server draws it. Drawing takes seconds, so the
  * panel puts its tile and its number up at once and asks for the drawing itself — from here and
  * not from the canvas's frame, which a reload or a change of tab would take the answer away with.
+ * A region the magic pen marked (magicPen.tsx) comes with `crop`, the part of that drawing it is,
+ * as fractions of it: the server draws at the display's pixel ratio, which only the drawing knows.
  *
  * On `window`, because the panel is a sibling of `<Tldraw>` and the button renders inside it.
  * Here rather than beside the button, so the home page, which has the panel and no canvas, has
@@ -111,7 +113,13 @@ export const CANVAS_ATTACH = "sp:canvas-attach";
 export type CanvasAttachDetail =
   /** `reference` for anything but a picture, whose drawing is only the tile's: the agent is
    *  pointed at its file instead. */
-  | { kind: "board"; name: string; src: string; reference?: true }
+  | {
+      kind: "board";
+      name: string;
+      src: string;
+      crop?: { x: number; y: number; w: number; h: number };
+      reference?: true;
+    }
   | { kind: "image"; file: File }
   | { kind: "error"; message: string }
   | { kind: "draft"; text: string }
@@ -829,7 +837,12 @@ export function ChatPanel(props: {
    * drawing asked of the server, to land in that tile. Asked for again it keeps the tile it has —
    * one already there or on its way is only named again, and one that failed is drawn again.
    */
-  const addBoard = (name: string, src: string, reference?: true) => {
+  const addBoard = (
+    name: string,
+    src: string,
+    crop?: { x: number; y: number; w: number; h: number },
+    reference?: true,
+  ) => {
     let tile = tray.current.find((t) => t.name === name);
     if (!tile && tray.current.length >= MAX_IMAGES)
       return setSendError(
@@ -865,7 +878,21 @@ export function ChatPanel(props: {
     fetch(src)
       .then(async (shot) => {
         if (!shot.ok) throw new Error(await shot.text());
-        const png = await shot.blob();
+        let png = await shot.blob();
+        if (crop) {
+          const whole = await createImageBitmap(png);
+          const [w, h] = [whole.width, whole.height];
+          const part = await createImageBitmap(
+            whole,
+            Math.round(crop.x * w),
+            Math.round(crop.y * h),
+            Math.round(crop.w * w),
+            Math.round(crop.h * h),
+          );
+          const canvas = new OffscreenCanvas(part.width, part.height);
+          canvas.getContext("2d")!.drawImage(part, 0, 0);
+          png = await canvas.convertToBlob({ type: "image/png" });
+        }
         void addImages([new File([png], name, { type: png.type })], true);
       })
       .catch((error) => {
@@ -892,7 +919,7 @@ export function ChatPanel(props: {
       // that is still being read.
       if (detail.kind === "board") {
         adds.current = adds.current.then(() =>
-          addBoard(detail.name, detail.src, detail.reference),
+          addBoard(detail.name, detail.src, detail.crop, detail.reference),
         );
         return;
       }

canvas/src/canvasAttach.tsx

@@ -16,20 +16,35 @@ import { CANVAS_ATTACH, type CanvasAttachDetail } from "./ChatPanel";
 import { personsShape, personsShapeName, saveNow } from "./canvasContent";
 import { canvasBoardRef } from "./canvasLibrary";
 import { Plus } from "./geistIcons";
-import {
-  asCanvasTarget,
-  shapeUnderPointer,
-} from "./canvasClicks";
+import { asCanvasTarget, shapeUnderPointer } from "./canvasClicks";
 
 /** A board, a picture the library placed, or anything the person put on a project canvas: what
  *  the **+** answers for. The canvas answers only for the first two (asCanvasTarget). */
 const addable = (editor: Editor, shape: TLShape | undefined): TLShape | null =>
   asCanvasTarget(shape) ?? (personsShape(editor, shape) ? shape! : null);
 
 /** To the agent's panel, which is the window's, outside the canvas's frame (AppShell.tsx). */
-const dispatchAttach = (detail: CanvasAttachDetail) =>
+// oxlint-disable-next-line react/only-export-components
+export const dispatchAttach = (detail: CanvasAttachDetail) =>
   window.parent.dispatchEvent(new CustomEvent(CANVAS_ATTACH, { detail }));
 
+/** A board by its `<slug>/<file>.html`, and where the server draws it. */
+// oxlint-disable-next-line react/only-export-components
+export function boardShot(board: CanvasFileShape) {
+  const { w, h, path } = board.props;
+  const ref = canvasBoardRef(path);
+  if (!ref) throw new Error("that board has no file behind it");
+  const name = `${ref.slug}/${ref.file}`;
+  // At most 4000 a side, as CanvasFileShapeUtil's toSvg draws one.
+  const scale = Math.min(1, 4000 / Math.max(w, h));
+  const src = new URL(
+    `${import.meta.env.BASE_URL}__sp/shoot?path=${encodeURIComponent(name)}` +
+      `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`,
+    window.location.href,
+  ).href;
+  return { name, src };
+}
+
 /**
  * A board or a picture, handed to the chat. A board is a page in an `<iframe>`, so the server
  * draws it first (`/__sp/shoot`, server/sp.ts) and it goes over under its own
@@ -43,17 +58,7 @@ async function attach(editor: Editor, target: TLShape) {
     // A board, whoever placed it, before the check below that an agent-placed one also passes:
     // `toImage` of a board comes back blank, so the server shoots it.
     if (target.type === CANVAS_FILE_SHAPE_TYPE) {
-      const { w, h, path } = (target as CanvasFileShape).props;
-      const ref = canvasBoardRef(path);
-      if (!ref) throw new Error("that board has no file behind it");
-      const name = `${ref.slug}/${ref.file}`;
-      // At most 4000 a side, as CanvasFileShapeUtil's toSvg draws one.
-      const scale = Math.min(1, 4000 / Math.max(w, h));
-      const src = new URL(
-        `${import.meta.env.BASE_URL}__sp/shoot?path=${encodeURIComponent(name)}` +
-          `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`,
-        window.location.href,
-      ).href;
+      const { name, src } = boardShot(target as CanvasFileShape);
       return dispatchAttach({ kind: "board", name, src, reference: true });
     }
     // One of the person's own, drawn for the tile and named by where the agent reads the thing

canvas/src/canvasChrome.tsx

@@ -10,7 +10,9 @@ import {
   atom,
   type Atom,
   ConversionsMenuGroup,
+  DefaultColorStyle,
   DefaultContextMenu,
+  getColorValue,
   DefaultShapeWrapper,
   SelectAllMenuItem,
   TldrawUiButton,
@@ -42,6 +44,7 @@ import {
   CanvasSelectionAttachButton,
 } from "./canvasAttach";
 import { CommentUserDialog } from "./CommentUserDialog";
+import { MagicPen } from "./magicPen";
 import { GROUNDS, groundEditable, groundOf, setGround } from "./canvasGround";
 import {
   linkedBoard,
@@ -235,7 +238,13 @@ function setFresh(fresh: ReadonlyMap<TLShapeId, Fresh>) {
 
 /** Rings `ids` until the pointer passes over each, so the reader sees what just arrived or changed. */
 export function markFresh(ids: TLShapeId[], kind: Fresh) {
-  if (ids.length) setFresh(new Map([...freshShapes().get(), ...ids.map((id) => [id, kind] as const)]));
+  if (ids.length)
+    setFresh(
+      new Map([
+        ...freshShapes().get(),
+        ...ids.map((id) => [id, kind] as const),
+      ]),
+    );
 }
 
 export const canvasChromeComponents: TLComponents = {
@@ -258,7 +267,11 @@ export const canvasChromeComponents: TLComponents = {
       useEffect(() => {
         if (!fresh) return;
         const seen = (info: TLEventInfo) => {
-          if (info.name !== "pointer_move" || shapeUnderPointer(editor)?.id !== id) return;
+          if (
+            info.name !== "pointer_move" ||
+            shapeUnderPointer(editor)?.id !== id
+          )
+            return;
           const rest = new Map(freshShapes().get());
           rest.delete(id);
           setFresh(rest);
@@ -270,7 +283,9 @@ export const canvasChromeComponents: TLComponents = {
         <DefaultShapeWrapper
           ref={ref}
           {...props}
-          className={fresh ? `${props.className ?? ""} sp-fresh` : props.className}
+          className={
+            fresh ? `${props.className ?? ""} sp-fresh` : props.className
+          }
           data-fresh={fresh}
         />
       );
@@ -351,15 +366,22 @@ export const canvasChromeComponents: TLComponents = {
     const cover =
       canvasIndex().served && canvasIndex().project
         ? (board ?? (over && canvasImageRef(over.id)))
-      : undefined;
+        : undefined;
     const element =
       board && pointedElement.current?.path === (over as CanvasFileShape).props.path
         ? pointedElement.current.box
         : undefined;
 
     // The canvas's ground, the strip's swatch as presets. Custom opens that swatch's picker.
-    const page = chrome.activeTab.kind === "canvas" ? pageOf(chrome.activeTab) : undefined;
+    const page =
+      chrome.activeTab.kind === "canvas" ? pageOf(chrome.activeTab) : undefined;
     const ground = page ? groundOf(page) : undefined;
+    // The colour of what the right-click picked out that can take one: an arrow, a line, a note
+    // the agent or the person drew. A board or anything else the layout placed is locked.
+    const colourable = editor
+      .getSelectedShapes()
+      .filter((shape) => !shape.isLocked && "color" in shape.props);
+    const colours = editor.getCurrentTheme().colors[editor.getColorMode()];
 
     return (
       <DefaultContextMenu {...props}>
@@ -379,10 +401,14 @@ export const canvasChromeComponents: TLComponents = {
           {links.length > 0 && (
             <TldrawUiMenuItem
               id="copy-link"
-              label={links.length > 1 ? `Copy ${links.length} links` : "Copy link"}
+              label={
+                links.length > 1 ? `Copy ${links.length} links` : "Copy link"
+              }
               icon={<Copy />}
               kbd="cmd+c,ctrl+c"
-              onSelect={() => void navigator.clipboard.writeText(links.join("\n"))}
+              onSelect={() =>
+                void navigator.clipboard.writeText(links.join("\n"))
+              }
             />
           )}
           {cover && !isExample(cover.slug) && (
@@ -422,13 +448,46 @@ export const canvasChromeComponents: TLComponents = {
                 checked={!GROUNDS.some(([, color]) => color === ground)}
                 onSelect={() =>
                   document
-                    .querySelector<HTMLInputElement>(".sp-canvas-tabs-ground input")
+                    .querySelector<HTMLInputElement>(
+                      ".sp-canvas-tabs-ground input",
+                    )
                     ?.showPicker()
                 }
               />
             </TldrawUiMenuSubmenu>
           </TldrawUiMenuGroup>
         )}
+        {colourable.length > 0 && (
+          <TldrawUiMenuGroup id="colour">
+            <TldrawUiMenuSubmenu id="colour" label="Colour">
+              {DefaultColorStyle.values.map((color) => (
+                <TldrawUiMenuItem
+                  key={color}
+                  id={`colour-${color}`}
+                  label={
+                    color[0].toUpperCase() + color.slice(1).replace("-", " ")
+                  }
+                  iconLeft={
+                    <span
+                      className="sp-menu-swatch"
+                      style={{
+                        background: getColorValue(colours, color, "solid"),
+                      }}
+                    />
+                  }
+                  isSelected={colourable.every(
+                    (shape) => (shape.props as { color: string }).color === color,
+                  )}
+                  // tldraw leaves a locked shape as it is, so a board in the selection too is
+                  // no matter.
+                  onSelect={() => {
+                    editor.setStyleForSelectedShapes(DefaultColorStyle, color);
+                  }}
+                />
+              ))}
+            </TldrawUiMenuSubmenu>
+          </TldrawUiMenuGroup>
+        )}
         {/* tldraw's items one at a time, not its groups: this canvas is read, and every shape on
             it is locked and rebuilt from layout.json, so only what works on a locked shape is
             here. A group would bring Cut, Delete and Duplicate, greyed out on every shape here,
@@ -522,6 +581,7 @@ export const canvasChromeComponents: TLComponents = {
             whose boards its server draws as it draws the project's own (server/projects.ts). */}
         {local() && <CanvasAttachButtons />}
         {local() && <CanvasSelectionAttachButton />}
+        {local() && <MagicPen />}
         {/* Out of the tool as well as the bubble. Escape closes only the bubble and leaves the
             next click placing another one, which is not what an accidental comment wants. The
             draft is kept either way, so a real comment interrupted here is there next time. */}

canvas/src/index.css

@@ -1929,6 +1929,22 @@ a.sp-menu-row {
    over it (canvasAttach.tsx). The layer this is drawn on takes no pointer events, so the buttons
    ask for them back. Inside the corner, not over it: a button that crossed the shape's edge would
    leave the shape as the pointer arrived, and take itself away with the hover. */
+/* The magic pen (magicPen.tsx): over the whole canvas while Option is held, and the region it
+   marks on a board. */
+.sp-pen {
+  position: absolute;
+  inset: 0;
+  z-index: 1;
+  pointer-events: auto;
+}
+
+.sp-pen-region {
+  position: absolute;
+  border: 1.5px dashed var(--ds-focus);
+  border-radius: 4px;
+  background: color-mix(in srgb, var(--ds-focus) 12%, transparent);
+}
+
 .sp-attach {
   position: absolute;
   display: flex;
@@ -2981,3 +2997,11 @@ a.sp-menu-row {
 .sp-chat-made button[data-made="updated"] {
   --ds-focus: light-dark(#1a9338, #3dbe5c);
 }
+
+/* A colour's dot in the canvas menu's Colour row (canvasChrome.tsx). */
+.sp-menu-swatch {
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  box-shadow: inset 0 0 0 1px color-mix(in srgb, currentColor 25%, transparent);
+}