diff --git a/PRODUCT_BRIEF.md b/PRODUCT_BRIEF.md index 9c7c254..78db543 100644 --- a/PRODUCT_BRIEF.md +++ b/PRODUCT_BRIEF.md @@ -19,8 +19,10 @@ settings, and local or remote execution easier to operate without reimplementing availability and behavior do not diverge. - Quick Open searches through bounded authorized operations. Flutter never receives or chooses an absolute path it does not already own. -- A visible text preview may be staged as reviewed context for a prompt. It does not become a second - runtime authority. +- The Universal Working Set lets a user deliberately stage exact material from a file preview, + transcript message, review diff, selected terminal text, or browser accessibility snapshot. The + user can inspect and remove each item before it joins one ordinary OMP prompt; it does not become + a second runtime authority. - Light and dark themes use neutral surfaces with minimal semantic accent. - OMP identity uses the existing pi/connector mark and Pi Pink `#e83174` accent. diff --git a/apps/desktop/src/browser-dom-automation.ts b/apps/desktop/src/browser-dom-automation.ts index 912b9f8..ed58e10 100644 --- a/apps/desktop/src/browser-dom-automation.ts +++ b/apps/desktop/src/browser-dom-automation.ts @@ -22,6 +22,7 @@ interface SnapshotElement { readonly ref: string; readonly role: string; readonly name: string; + readonly visible: true; readonly text?: string; readonly value?: string; readonly bounds?: { x: number; y: number; width: number; height: number }; @@ -97,7 +98,15 @@ function accessibleName(element: Element): string { if (placeholder) return bound(placeholder); } if (element instanceof HTMLImageElement && element.alt) return bound(element.alt); - return bound((element.textContent ?? "").replace(/\s+/gu, " ").trim(), 8_192); + const tag = element.tagName.toLowerCase(); + const textNamedElement = + tag === "a" || + tag === "button" || + /^h[1-6]$/u.test(tag) || + element.children.length === 0; + return textNamedElement + ? bound((element.textContent ?? "").replace(/\s+/gu, " ").trim(), 8_192) + : ""; } function elementRef(element: Element): string { const old = elements.get(element); @@ -139,21 +148,43 @@ function boundsOf(element: Element): { x: number; y: number; width: number; heig } function isVisible(element: Element): boolean { if (!element.isConnected) return false; - const style = (element.ownerDocument.defaultView ?? window).getComputedStyle(element); + const view = element.ownerDocument.defaultView ?? window; + let current: Element | null = element; + while (current !== null) { + const style = view.getComputedStyle(current); + if ( + current.hasAttribute("hidden") || + current.getAttribute("aria-hidden") === "true" || + style.display === "none" || + style.visibility === "hidden" || + style.visibility === "collapse" || + Number(style.opacity) === 0 + ) { + return false; + } + current = current.parentElement; + } const rect = element.getBoundingClientRect(); - return style.display !== "none" && style.visibility !== "hidden" && style.visibility !== "collapse" && Number(style.opacity) !== 0 && rect.width > 0 && rect.height > 0; + return rect.width > 0 && rect.height > 0; } function isDisabled(element: Element): boolean { return (element as HTMLButtonElement | HTMLInputElement | HTMLSelectElement | HTMLTextAreaElement).disabled === true || element.getAttribute("aria-disabled") === "true" || element.closest("fieldset[disabled]") !== null; } function snapshotNode(element: Element, depth: number, budget: { count: number }): SnapshotElement { - if (budget.count >= MAX_SNAPSHOT_ELEMENTS) return { ref: elementRef(element), role: roleFor(element), name: accessibleName(element) }; + if (budget.count >= MAX_SNAPSHOT_ELEMENTS) { + return { + ref: elementRef(element), + role: roleFor(element), + name: accessibleName(element), + visible: true, + }; + } budget.count += 1; const result: SnapshotElement = { - ref: elementRef(element), role: roleFor(element), name: accessibleName(element), + ref: elementRef(element), role: roleFor(element), name: accessibleName(element), visible: true, ...(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || element instanceof HTMLSelectElement ? { value: bound(element.value) } : {}), ...(element.children.length === 0 && element.textContent?.trim() ? { text: bound(element.textContent.trim(), 8_192) } : {}), - ...(isVisible(element) ? { bounds: boundsOf(element) } : {}), + bounds: boundsOf(element), ...(isDisabled(element) ? { disabled: true } : {}), ...(element instanceof HTMLInputElement && (element.type === "checkbox" || element.type === "radio") ? { checked: element.checked } : {}), ...(element.getAttribute("aria-expanded") !== null ? { expanded: element.getAttribute("aria-expanded") === "true" } : {}), @@ -162,6 +193,7 @@ function snapshotNode(element: Element, depth: number, budget: { count: number } const children: SnapshotElement[] = []; for (const child of Array.from(element.children).slice(0, MAX_SNAPSHOT_ELEMENTS)) { if (budget.count >= MAX_SNAPSHOT_ELEMENTS) break; + if (!isVisible(child)) continue; children.push(snapshotNode(child, depth + 1, budget)); } return children.length ? { ...result, children } : result; @@ -172,14 +204,15 @@ function snapshot(): JsonValue { const flat: SnapshotElement[] = []; const visit = (element: Element): void => { if (flat.length >= MAX_SNAPSHOT_ELEMENTS) return; - flat.push(snapshotNode(element, MAX_DEPTH, { count: 0 })); + if (isVisible(element)) flat.push(snapshotNode(element, MAX_DEPTH, { count: 0 })); for (const child of Array.from(element.children)) visit(child); }; visit(body); + const tree = isVisible(body) ? snapshotNode(body, 0, { count: 0 }) : null; return json({ url: bound(doc.URL), title: bound(doc.title), readyState: doc.readyState, viewport: { x: 0, y: 0, width: Math.max(0, window.innerWidth), height: Math.max(0, window.innerHeight) }, - tree: snapshotNode(body, 0, { count: 0 }), elements: flat, capturedAt: Date.now(), truncated: flat.length >= MAX_SNAPSHOT_ELEMENTS, + tree, elements: flat, capturedAt: Date.now(), truncated: flat.length >= MAX_SNAPSHOT_ELEMENTS, }); } function postAction(params: Record): Record { diff --git a/apps/desktop/test/browser-dom-automation.test.ts b/apps/desktop/test/browser-dom-automation.test.ts index f900358..04286e6 100644 --- a/apps/desktop/test/browser-dom-automation.test.ts +++ b/apps/desktop/test/browser-dom-automation.test.ts @@ -5,19 +5,139 @@ import { resetBrowserDomAutomation, } from "../src/browser-dom-automation.ts"; -const originalDocument = Object.getOwnPropertyDescriptor(globalThis, "document"); -const originalWindow = Object.getOwnPropertyDescriptor(globalThis, "window"); +const DOM_GLOBALS = [ + "document", + "window", + "Element", + "HTMLElement", + "HTMLButtonElement", + "HTMLFrameElement", + "HTMLIFrameElement", + "HTMLImageElement", + "HTMLInputElement", + "HTMLSelectElement", + "HTMLTextAreaElement", +] as const; +const originalGlobals = new Map( + DOM_GLOBALS.map((name) => [name, Object.getOwnPropertyDescriptor(globalThis, name)]), +); -function exposeGlobal(name: "document" | "window", value: unknown): void { +function exposeGlobal(name: (typeof DOM_GLOBALS)[number], value: unknown): void { Object.defineProperty(globalThis, name, { configurable: true, value, writable: true }); } function restoreGlobals(): void { resetBrowserDomAutomation(); - if (originalDocument === undefined) Reflect.deleteProperty(globalThis, "document"); - else Object.defineProperty(globalThis, "document", originalDocument); - if (originalWindow === undefined) Reflect.deleteProperty(globalThis, "window"); - else Object.defineProperty(globalThis, "window", originalWindow); + for (const name of DOM_GLOBALS) { + const descriptor = originalGlobals.get(name); + if (descriptor === undefined) Reflect.deleteProperty(globalThis, name); + else Object.defineProperty(globalThis, name, descriptor); + } +} + +interface FakeStyle { + readonly display: string; + readonly visibility: string; + readonly opacity: string; +} + +class FakeElement { + readonly attributes = new Map(); + readonly children: FakeElement[] = []; + readonly isConnected = true; + readonly ownerDocument: FakeDocument; + readonly style: FakeStyle; + readonly tagName: string; + parentElement: FakeElement | null = null; + textContent: string; + + constructor( + tagName: string, + ownerDocument: FakeDocument, + textContent = "", + style: FakeStyle = { display: "block", visibility: "visible", opacity: "1" }, + ) { + this.tagName = tagName; + this.ownerDocument = ownerDocument; + this.textContent = textContent; + this.style = style; + } + + append(child: FakeElement): void { + child.parentElement = this; + this.children.push(child); + } + + getAttribute(name: string): string | null { + return this.attributes.get(name) ?? null; + } + + hasAttribute(name: string): boolean { + return this.attributes.has(name); + } + + closest(): null { + return null; + } + + getBoundingClientRect(): { x: number; y: number; width: number; height: number } { + return { x: 0, y: 0, width: 640, height: 24 }; + } +} + +class FakeHTMLElement extends FakeElement {} +class FakeHTMLButtonElement extends FakeHTMLElement {} +class FakeHTMLFrameElement extends FakeHTMLElement { readonly contentDocument = null; } +class FakeHTMLIFrameElement extends FakeHTMLElement { readonly contentDocument = null; } +class FakeHTMLImageElement extends FakeHTMLElement { readonly alt = ""; } +class FakeHTMLInputElement extends FakeHTMLElement { + readonly checked = false; + readonly disabled = false; + readonly id = ""; + readonly type = "text"; + readonly value: string; + constructor(tagName: string, ownerDocument: FakeDocument, value: string) { + super(tagName, ownerDocument); + this.value = value; + } +} +class FakeHTMLSelectElement extends FakeHTMLElement { readonly disabled = false; readonly value = ""; } +class FakeHTMLTextAreaElement extends FakeHTMLElement { readonly disabled = false; readonly value = ""; } + +class FakeDocument { + readonly URL = "https://example.test/dashboard"; + readonly defaultView: FakeWindow; + readonly readyState = "complete"; + readonly title = "Dashboard"; + body!: FakeHTMLElement; + + constructor() { + this.defaultView = new FakeWindow(); + } +} + +class FakeWindow { + alert = () => undefined; + confirm = () => false; + prompt = () => null; + readonly innerHeight = 720; + readonly innerWidth = 1_280; + getComputedStyle(element: FakeElement): FakeStyle { return element.style; } + getSelection(): null { return null; } +} + +function exposeSnapshotDom(document: FakeDocument): void { + exposeGlobal("document", document); + exposeGlobal("window", document.defaultView); + exposeGlobal("Element", FakeElement); + exposeGlobal("HTMLElement", FakeHTMLElement); + exposeGlobal("HTMLButtonElement", FakeHTMLButtonElement); + exposeGlobal("HTMLFrameElement", FakeHTMLFrameElement); + exposeGlobal("HTMLIFrameElement", FakeHTMLIFrameElement); + exposeGlobal("HTMLImageElement", FakeHTMLImageElement); + exposeGlobal("HTMLInputElement", FakeHTMLInputElement); + exposeGlobal("HTMLSelectElement", FakeHTMLSelectElement); + exposeGlobal("HTMLTextAreaElement", FakeHTMLTextAreaElement); } describe("browser DOM Design Mode", () => { @@ -54,3 +174,59 @@ describe("browser DOM Design Mode", () => { } }); }); + +describe("browser DOM accessibility snapshot", () => { + it("keeps visible content, excludes hidden DOM, and avoids container text duplication", async () => { + const document = new FakeDocument(); + const body = new FakeHTMLElement("BODY", document, "Visible account Hidden secret Password"); + const main = new FakeHTMLElement("MAIN", document, "Visible account Hidden secret Password"); + const heading = new FakeHTMLElement("H1", document, "Visible account"); + const hidden = new FakeHTMLElement("DIV", document, "Hidden secret", { + display: "none", + visibility: "visible", + opacity: "1", + }); + const input = new FakeHTMLInputElement("INPUT", document, "never-stage-this"); + input.attributes.set("placeholder", "Password"); + body.append(main); + main.append(heading); + main.append(hidden); + main.append(input); + document.body = body; + + try { + exposeSnapshotDom(document); + const result = await executeBrowserDomAutomation("browser.snapshot", {}); + const snapshot = (result as { readonly snapshot: { readonly elements: readonly Record[] } }).snapshot; + + expect( + snapshot.elements.some( + (element) => + element.role === "heading" && + element.name === "Visible account" && + element.visible === true, + ), + ).toBe(true); + expect( + snapshot.elements.some( + (element) => + element.role === "textbox" && + element.name === "Password" && + element.value === "never-stage-this" && + element.visible === true, + ), + ).toBe(true); + expect(snapshot.elements.some((element) => element.name === "Hidden secret")).toBe(false); + expect( + snapshot.elements.some( + (element) => + (element.role === "document" || element.role === "main") && + typeof element.name === "string" && + element.name.includes("Visible account"), + ), + ).toBe(false); + } finally { + restoreGlobals(); + } + }); +}); diff --git a/apps/desktop/test/browser-runtime.test.ts b/apps/desktop/test/browser-runtime.test.ts index 96071f2..cc14b1a 100644 --- a/apps/desktop/test/browser-runtime.test.ts +++ b/apps/desktop/test/browser-runtime.test.ts @@ -141,7 +141,7 @@ async function settleBackgroundWork(): Promise { } describe("BrowserRuntime native view lifecycle", () => { - it("routes design mode through the exact owner-scoped browser surface", async () => { + it("routes accessibility snapshots and design mode through the exact owner-scoped surface", async () => { electron.reset(); const calls: unknown[] = []; const runtime = new BrowserRuntime({ @@ -166,6 +166,15 @@ describe("BrowserRuntime native view lifecycle", () => { automationCoordinator: { call: async (call) => { calls.push(call); + if (call.method === "browser.snapshot") { + return { + snapshot: { + url: "https://example.test/", + title: "Example", + elements: [{ role: "heading", name: "Visible heading", visible: true }], + }, + }; + } return { enabled: true, selection: "Heading" }; }, dispose: () => {}, @@ -180,9 +189,19 @@ describe("BrowserRuntime native view lifecycle", () => { surfaceId: created.surface.surfaceId, enabled: true, })); + const snapshot = await runtime.call(browserCall("browser.snapshot", { + surfaceId: created.surface.surfaceId, + })); expect(result).toEqual({ enabled: true, selection: "Heading" }); - expect(calls).toHaveLength(1); + expect(snapshot).toEqual({ + snapshot: { + url: "https://example.test/", + title: "Example", + elements: [{ role: "heading", name: "Visible heading", visible: true }], + }, + }); + expect(calls).toHaveLength(2); expect(calls[0]).toMatchObject({ method: "browser.design_mode.set", ownerSessionId: OWNER_A, @@ -191,6 +210,11 @@ describe("BrowserRuntime native view lifecycle", () => { enabled: true, }, }); + expect(calls[1]).toMatchObject({ + method: "browser.snapshot", + ownerSessionId: OWNER_A, + request: { surfaceId: created.surface.surfaceId }, + }); await runtime.dispose(); }); diff --git a/apps/web/src/actions/actions.test.ts b/apps/web/src/actions/actions.test.ts index 29aa47a..f4e279f 100644 --- a/apps/web/src/actions/actions.test.ts +++ b/apps/web/src/actions/actions.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vite-plus/test"; import { createMemoryPersistence } from "../state/persistence.ts"; +import { createComposerStore } from "../features/composer/composer-store.ts"; +import { + captureReviewContext, + captureTerminalContext, + captureTranscriptContext, +} from "../features/context-packet/context-packet.ts"; import { createWorkspaceStore } from "../state/workspace-store.ts"; import { createInspectorStore, type InspectorStoreApi } from "../features/panes/inspector-store.ts"; import type { ProjectGroup } from "../lib/session-tree.ts"; @@ -106,8 +112,10 @@ function setup() { shell: null, browser: null, }; + const composer = createComposerStore(); const environment: ActionEnvironment = { workspace, + composer, platform, railOverlaid: () => false, shellData: () => shellData, @@ -117,6 +125,7 @@ function setup() { }; return { workspace, + composer, inspector, destinations, registry: createActionRegistry(CORE_ACTIONS, environment), @@ -191,6 +200,110 @@ describe("typed action registry", () => { ]); }); + it("captures, removes, and clears the working set through shared actions", () => { + const { composer, registry } = setup(); + const first = captureTranscriptContext(firstSession.id, { + id: "message-1", + role: "assistant", + text: "First response", + }); + const second = captureTranscriptContext(firstSession.id, { + id: "message-2", + role: "user", + text: "Second message", + }); + if (first === null || second === null) throw new Error("expected context items"); + + expect( + registry.execute({ + id: "context.capture", + args: { sessionId: firstSession.id, item: first }, + }).executed, + ).toBe(true); + expect(composer.getState().contextItemsBySessionId[firstSession.id]).toEqual([first]); + + expect( + registry.execute({ + id: "context.remove", + args: { sessionId: firstSession.id, itemId: first.id }, + }).executed, + ).toBe(true); + expect(composer.getState().contextItemsBySessionId[firstSession.id]).toBeUndefined(); + + registry.execute({ + id: "context.capture", + args: { sessionId: firstSession.id, item: first }, + }); + registry.execute({ + id: "context.capture", + args: { sessionId: firstSession.id, item: second }, + }); + expect( + registry.execute({ id: "context.clear", args: { sessionId: firstSession.id } }).executed, + ).toBe(true); + expect(composer.getState().contextItemsBySessionId[firstSession.id]).toBeUndefined(); + }); + + it("stages terminal selections and review patches through the shared capture action", () => { + const { composer, registry } = setup(); + const terminal = captureTerminalContext( + firstSession.id, + { terminalId: "terminal-1", title: "Tests", text: "selected failure only" }, + { id: "terminal-selection" }, + ); + const review = captureReviewContext( + firstSession.id, + { path: "src/app.ts", patch: "@@ -1 +1 @@\n-old\n+new" }, + { id: "review-patch" }, + ); + if (terminal === null || review === null) throw new Error("expected source context"); + + for (const item of [terminal, review]) { + expect( + registry.execute({ + id: "context.capture", + args: { sessionId: firstSession.id, item }, + }).executed, + ).toBe(true); + } + + expect(composer.getState().contextItemsBySessionId[firstSession.id]).toEqual([ + terminal, + review, + ]); + }); + + it("refuses working-set capture for another or inactive session", () => { + const { composer, registry } = setup(); + const item = captureTranscriptContext(firstSession.id, { + id: "message-1", + role: "assistant", + text: "Response", + }); + if (item === null) throw new Error("expected context item"); + + expect( + registry.execute({ + id: "context.capture", + args: { sessionId: secondSession.id, item }, + }).availability, + ).toEqual({ status: "disabled", reason: "This context belongs to a different session." }); + + const inactiveItem = captureTranscriptContext(secondSession.id, { + id: "message-2", + role: "assistant", + text: "Inactive response", + }); + if (inactiveItem === null) throw new Error("expected inactive context item"); + expect( + registry.execute({ + id: "context.capture", + args: { sessionId: secondSession.id, item: inactiveItem }, + }).availability, + ).toEqual({ status: "hidden" }); + expect(composer.getState().contextItemsBySessionId).toEqual({}); + }); + it("selects a transcript-linked agent and opens the shared Agents surface", () => { const { inspector, registry, workspace } = setup(); inspector.getState().ingestAgent({ diff --git a/apps/web/src/actions/core-actions.ts b/apps/web/src/actions/core-actions.ts index 797284d..8e34ad7 100644 --- a/apps/web/src/actions/core-actions.ts +++ b/apps/web/src/actions/core-actions.ts @@ -1,4 +1,5 @@ import { flattenFileIndex } from "../features/composer/file-refs.ts"; +import { contextSourceDescription } from "../features/context-packet/context-packet.ts"; import { selectSessionView } from "../state/workspace-store.ts"; import { resolveTheme } from "../theme/theme.ts"; import type { @@ -247,6 +248,65 @@ const fileOpen = defineAction({ }, }); +const contextCapture = defineAction({ + id: "context.capture", + group: "workspace", + surfaces: ["context-source"], + label: () => "Add to working set", + description: (_environment, args) => contextSourceDescription(args.item.source), + availability: (environment, args) => { + if (args.item.sessionId !== args.sessionId) { + return { status: "disabled", reason: "This context belongs to a different session." }; + } + return activeCurrentSessionAvailability(environment, args.sessionId); + }, + run: (environment, args) => { + environment.composer.getState().addContextItem(args.sessionId, args.item); + return ACTION_COMPLETED; + }, +}); + +const contextRemove = defineAction({ + id: "context.remove", + group: "workspace", + surfaces: ["context-source"], + label: () => "Remove from working set", + description: () => "Context for the next new message", + availability: (environment, args) => { + const active = activeCurrentSessionAvailability(environment, args.sessionId); + if (active.status !== "enabled") return active; + return environment.composer + .getState() + .contextItemsBySessionId[args.sessionId]?.some((item) => item.id === args.itemId) + ? ACTION_ENABLED + : { status: "disabled", reason: "This context item is no longer in the working set." }; + }, + run: (environment, args) => { + environment.composer.getState().removeContextItem(args.sessionId, args.itemId); + return ACTION_COMPLETED; + }, +}); + +const contextClear = defineAction({ + id: "context.clear", + group: "workspace", + surfaces: ["context-source"], + label: () => "Clear working set", + description: () => "Context for the next new message", + availability: (environment, args) => { + const active = activeCurrentSessionAvailability(environment, args.sessionId); + if (active.status !== "enabled") return active; + return (environment.composer.getState().contextItemsBySessionId[args.sessionId]?.length ?? 0) > + 0 + ? ACTION_ENABLED + : { status: "disabled", reason: "The working set is already empty." }; + }, + run: (environment, args) => { + environment.composer.getState().clearContextItems(args.sessionId); + return ACTION_COMPLETED; + }, +}); + const agentOpen = defineAction({ id: "agent.open", group: "workspace", @@ -363,6 +423,9 @@ export const CORE_ACTIONS: readonly AnyActionDefinition[] = Object.freeze([ sessionOpen, surfaceToggle, fileOpen, + contextCapture, + contextRemove, + contextClear, agentOpen, reviewOpen, previewOpen, diff --git a/apps/web/src/actions/types.ts b/apps/web/src/actions/types.ts index e3b5177..6d71861 100644 --- a/apps/web/src/actions/types.ts +++ b/apps/web/src/actions/types.ts @@ -1,6 +1,8 @@ import type { SessionStatus } from "@t4-code/ui"; import type { FileRefEntry } from "../features/composer/file-refs.ts"; +import type { ComposerStoreApi } from "../features/composer/composer-store.ts"; +import type { ContextPacketItem } from "../features/context-packet/context-packet.ts"; import type { InspectorStoreApi } from "../features/panes/inspector-store.ts"; import type { ProjectGroup } from "../lib/session-tree.ts"; import type { WorkspaceData } from "../lib/workspace-data.ts"; @@ -17,6 +19,9 @@ export type ActionId = | "session.open" | "surface.toggle" | "file.open" + | "context.capture" + | "context.remove" + | "context.clear" | "agent.open" | "review.open" | "preview.open" @@ -46,6 +51,9 @@ export interface ActionArguments { readonly path: string; readonly source?: "loaded" | "project-search"; }; + readonly "context.capture": { readonly sessionId: string; readonly item: ContextPacketItem }; + readonly "context.remove": { readonly sessionId: string; readonly itemId: string }; + readonly "context.clear": { readonly sessionId: string }; readonly "agent.open": { readonly sessionId: string; readonly agentId: string }; readonly "review.open": { readonly sessionId: string; readonly turnId: string }; readonly "preview.open": { readonly sessionId: string }; @@ -72,7 +80,12 @@ export const ACTION_COMPLETED = Object.freeze({ completed: true as const }); export type ActionRunResult = typeof ACTION_COMPLETED; export type ActionGroup = "workspace" | "navigate" | "app"; -export type ActionSurface = "quick-open" | "shortcut" | "workspace-menu" | "tool-link"; +export type ActionSurface = + | "quick-open" + | "shortcut" + | "workspace-menu" + | "tool-link" + | "context-source"; export type ActionIcon = "search" | "terminal" | ActionSessionSurface; export interface ActionPresentation { @@ -103,6 +116,7 @@ export type ActionDestination = */ export interface ActionEnvironment { readonly workspace: WorkspaceStoreApi; + readonly composer: ComposerStoreApi; readonly platform: RendererPlatform; readonly railOverlaid: () => boolean; readonly shellData: () => WorkspaceData; diff --git a/apps/web/src/components/AppShell.tsx b/apps/web/src/components/AppShell.tsx index 0d52bf3..d8e3456 100644 --- a/apps/web/src/components/AppShell.tsx +++ b/apps/web/src/components/AppShell.tsx @@ -15,6 +15,7 @@ import { type ActionDestination, } from "../actions/index.ts"; import { handoffTranscriptSearchQuery } from "../features/transcript-search/index.ts"; +import { composerStore } from "../features/composer/composer-store.ts"; import { TRANSCRIPT_SEARCH_ROUTE } from "../features/transcript-search/route.ts"; import { getInspectorStore } from "../features/panes/inspector-store.ts"; import { startDesktopRuntime, useDesktopRuntimeSnapshot } from "../platform/desktop-runtime.ts"; @@ -186,6 +187,7 @@ export function AppShell() { () => createActionRegistry(CORE_ACTIONS, { workspace: workspaceStore, + composer: composerStore, platform: rendererPlatform, railOverlaid: () => railOverlaid, shellData: getShellData, diff --git a/apps/web/src/features/browser/BrowserWorkspace.tsx b/apps/web/src/features/browser/BrowserWorkspace.tsx index 2456c86..ee79aa5 100644 --- a/apps/web/src/features/browser/BrowserWorkspace.tsx +++ b/apps/web/src/features/browser/BrowserWorkspace.tsx @@ -34,6 +34,7 @@ import { Download, Focus, Globe2, + Layers3, LoaderCircle, Minus, PanelRightOpen, @@ -54,9 +55,16 @@ import { type KeyboardEvent as ReactKeyboardEvent, } from "react"; +import { useActionRegistry } from "../../actions/index.ts"; import type { WorkspaceProject, WorkspaceSession } from "../../lib/workspace-data.ts"; import { rendererPlatform, useWorkspace, workspaceStore } from "../../state/store-instance.ts"; import { selectSessionView } from "../../state/workspace-store.ts"; +import { useComposer } from "../composer/composer-store.ts"; +import { + captureBrowserSnapshotContext, + type BrowserPageContextSnapshot, + type ContextPacketItem, +} from "../context-packet/context-packet.ts"; import { applyBrowserEvent, browserCall, @@ -86,6 +94,9 @@ import { const FIELD_CLASS = "min-h-11 min-w-0 rounded-md border border-input bg-popover px-3 text-base text-foreground outline-none transition-shadow duration-(--motion-duration-fast) placeholder:text-muted-foreground focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-1 focus-visible:ring-offset-background disabled:pointer-events-none disabled:opacity-64 sm:min-h-8 sm:text-sm"; +const EMPTY_CONTEXT_ITEMS = [] as const; +export const BROWSER_CONTEXT_CAPTURE_METHOD = "browser.snapshot" as const; + interface PendingTrustAction { readonly option: BrowserProfileOption; readonly surface?: BrowserSurfaceState; @@ -133,6 +144,53 @@ export function settleBrowserWorkspaceCall( .catch(() => undefined); } +function browserPageContextSnapshot(value: unknown): BrowserPageContextSnapshot | null { + if (typeof value !== "object" || value === null || !("snapshot" in value)) return null; + const snapshot = (value as { readonly snapshot?: unknown }).snapshot; + if ( + typeof snapshot !== "object" || + snapshot === null || + !("url" in snapshot) || + typeof snapshot.url !== "string" || + !("title" in snapshot) || + typeof snapshot.title !== "string" || + !("elements" in snapshot) || + !Array.isArray(snapshot.elements) + ) { + return null; + } + const elements = snapshot.elements.filter( + (element): element is BrowserPageContextSnapshot["elements"][number] => + typeof element === "object" && + element !== null && + "role" in element && + typeof element.role === "string" && + "name" in element && + typeof element.name === "string" && + (!("text" in element) || element.text === undefined || typeof element.text === "string") && + (!("visible" in element) || + element.visible === undefined || + typeof element.visible === "boolean"), + ); + return { + url: snapshot.url, + title: snapshot.title, + elements, + ...("truncated" in snapshot && snapshot.truncated === true ? { truncated: true } : {}), + }; +} + +export function captureBrowserPageResult( + sessionId: string, + surfaceId: SurfaceId, + result: unknown, +): ContextPacketItem | null { + const snapshot = browserPageContextSnapshot(result); + return snapshot === null + ? null + : captureBrowserSnapshotContext(sessionId, surfaceId, snapshot); +} + function BrowserUnsupported({ session, @@ -225,6 +283,7 @@ export function BrowserWorkspace({ readonly session: WorkspaceSession; readonly project: WorkspaceProject; }) { + const actionRegistry = useActionRegistry(); const port = rendererPlatform.browser; const callBrowser = useCallback( (method: BrowserMethod, request: Readonly>) => @@ -252,6 +311,9 @@ export function BrowserWorkspace({ const [focusMode, setFocusMode] = useState(false); const [devtoolsOpen, setDevtoolsOpen] = useState(false); const [zoomBySurface, setZoomBySurface] = useState>>({}); + const stagedContext = useComposer( + (state) => state.contextItemsBySessionId[session.id] ?? EMPTY_CONTEXT_ITEMS, + ); const modelRef = useRef(model); const lifecycleRef = useRef(0); const boundsLifecycleRef = useRef(0); @@ -647,6 +709,18 @@ export function BrowserWorkspace({ return result; }; + const capturePageContext = async () => { + if (activeSurface === null) return; + const result = await runAutomation("Capturing page context", BROWSER_CONTEXT_CAPTURE_METHOD, {}); + if (result === null) return; + const item = captureBrowserPageResult(session.id, activeSurface.surfaceId, result); + if (item === null) { + setActionError("The browser did not return readable page context."); + return; + } + actionRegistry.execute({ id: "context.capture", args: { sessionId: session.id, item } }); + }; + const setZoom = async (next: number) => { if (activeSurface === null) return; const generation = lifecycleRef.current; @@ -678,6 +752,12 @@ export function BrowserWorkspace({ [activeSurface?.surfaceId, model.runtimeErrors], ); const currentZoom = activeSurface === null ? 1 : (zoomBySurface[activeSurface.surfaceId] ?? 1); + const contextAlreadyAdded = + activeSurface !== null && + stagedContext.some( + (item) => + item.source.kind === "browser" && item.source.surfaceId === activeSurface.surfaceId, + ); if (port === null) return ; @@ -1143,6 +1223,15 @@ export function BrowserWorkspace({ + -
    - {items.map((item) => ( -
  • -