From 078ea77a1efc5aada193f660077a2e7cd72a3c9d Mon Sep 17 00:00:00 2001 From: hangerye Date: Thu, 9 Oct 2025 01:47:31 +0800 Subject: [PATCH 01/71] feat: tools optimize --- .gitignore | 6 +- app/chrome-extension/common/message-types.ts | 6 + .../background/tools/browser/computer.ts | 859 ++++++++++++++++++ .../background/tools/browser/index.ts | 2 + .../background/tools/browser/interaction.ts | 21 +- .../background/tools/browser/read-page.ts | 111 +++ .../background/tools/browser/screenshot.ts | 74 +- .../accessibility-tree-helper.js | 593 ++++++++++++ .../inject-scripts/click-helper.js | 58 +- .../inject-scripts/fill-helper.js | 125 ++- .../interactive-elements-helper.js | 49 +- .../utils/screenshot-context.ts | 53 ++ app/chrome-extension/wxt.config.ts | 9 + app/native-server/package.json | 2 + docs/TOOLS.md | 92 +- packages/shared/src/tools.ts | 195 ++-- pnpm-lock.yaml | 69 +- 17 files changed, 2154 insertions(+), 170 deletions(-) create mode 100644 app/chrome-extension/entrypoints/background/tools/browser/computer.ts create mode 100644 app/chrome-extension/entrypoints/background/tools/browser/read-page.ts create mode 100644 app/chrome-extension/inject-scripts/accessibility-tree-helper.js create mode 100644 app/chrome-extension/utils/screenshot-context.ts diff --git a/.gitignore b/.gitignore index b856834e..de5e339e 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,8 @@ dist false/ metadata-v1.3/ registry.npmmirror.com/ -registry.npmjs.com/ \ No newline at end of file +registry.npmjs.com/ + +other/ +tools_optimize.md +Agents.md \ No newline at end of file diff --git a/app/chrome-extension/common/message-types.ts b/app/chrome-extension/common/message-types.ts index 4cc03c86..673c3df1 100644 --- a/app/chrome-extension/common/message-types.ts +++ b/app/chrome-extension/common/message-types.ts @@ -41,6 +41,7 @@ export const CONTENT_MESSAGE_TYPES = { KEYBOARD_HELPER_PING: 'keyboard_helper_ping', SCREENSHOT_HELPER_PING: 'screenshot_helper_ping', INTERACTIVE_ELEMENTS_HELPER_PING: 'interactive_elements_helper_ping', + ACCESSIBILITY_TREE_HELPER_PING: 'chrome_read_page_ping', } as const; // Tool action message types (for chrome.runtime.sendMessage) @@ -64,6 +65,11 @@ export const TOOL_MESSAGE_TYPES = { // Interactive elements GET_INTERACTIVE_ELEMENTS: 'getInteractiveElements', + // Accessibility tree + GENERATE_ACCESSIBILITY_TREE: 'generateAccessibilityTree', + RESOLVE_REF: 'resolveRef', + ENSURE_REF_FOR_SELECTOR: 'ensureRefForSelector', + // Network requests NETWORK_SEND_REQUEST: 'sendPureNetworkRequest', diff --git a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts new file mode 100644 index 00000000..fcbd8972 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts @@ -0,0 +1,859 @@ +import { createErrorResponse, ToolResult } from '@/common/tool-handler'; +import { BaseBrowserToolExecutor } from '../base-browser'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { ERROR_MESSAGES, TIMEOUTS } from '@/common/constants'; +import { TOOL_MESSAGE_TYPES } from '@/common/message-types'; +import { clickTool, fillTool } from './interaction'; +import { keyboardTool } from './keyboard'; +import { screenshotTool } from './screenshot'; +import { screenshotContextManager, scaleCoordinates } from '@/utils/screenshot-context'; + +type MouseButton = 'left' | 'right' | 'middle'; + +interface Coordinates { + x: number; + y: number; +} + +interface ComputerParams { + action: + | 'left_click' + | 'right_click' + | 'double_click' + | 'triple_click' + | 'left_click_drag' + | 'scroll' + | 'type' + | 'key' + | 'hover' + | 'wait' + | 'fill' + | 'screenshot'; + // click/scroll coordinates in screenshot space (if screenshot context exists) or viewport space + coordinates?: Coordinates; // for click/scroll; for drag, this is endCoordinates + startCoordinates?: Coordinates; // for drag start + // Optional element refs (from chrome_read_page) as alternative to coordinates + ref?: string; // click target or drag end + startRef?: string; // drag start + scrollDirection?: 'up' | 'down' | 'left' | 'right'; + scrollAmount?: number; + text?: string; // for type/key + duration?: number; // seconds for wait + // For fill + selector?: string; + value?: string; +} + +// Minimal CDP helper encapsulated here to avoid scattering CDP code +class CDPHelper { + private static active = new Set(); + + static async attach(tabId: number): Promise { + // If already attached by us, skip + const targets = await chrome.debugger.getTargets(); + const existing = targets.find((t) => t.tabId === tabId && t.attached); + if (existing) { + if (existing.extensionId === chrome.runtime.id) { + this.active.add(tabId); + return; + } + throw new Error( + `Debugger is already attached to tab ${tabId} by another tool (e.g., DevTools/extension)`, + ); + } + await chrome.debugger.attach({ tabId }, '1.3'); + this.active.add(tabId); + } + + static async detach(tabId: number): Promise { + if (!this.active.has(tabId)) return; + try { + await chrome.debugger.detach({ tabId }); + } finally { + this.active.delete(tabId); + } + } + + static async send(tabId: number, method: string, params?: object): Promise { + try { + return await chrome.debugger.sendCommand({ tabId }, method, params); + } catch (e: any) { + // Try reattach once if lost + if ( + String(e?.message || e) + .toLowerCase() + .includes('not attached') + ) { + await this.attach(tabId); + return await chrome.debugger.sendCommand({ tabId }, method, params); + } + throw e; + } + } + + static async dispatchMouseEvent(tabId: number, opts: any) { + const params: any = { + type: opts.type, + x: Math.round(opts.x), + y: Math.round(opts.y), + modifiers: opts.modifiers || 0, + }; + if ( + opts.type === 'mousePressed' || + opts.type === 'mouseReleased' || + opts.type === 'mouseMoved' + ) { + params.button = opts.button || 'none'; + if (opts.type === 'mousePressed' || opts.type === 'mouseReleased') { + params.clickCount = opts.clickCount || 1; + } + // Per CDP: buttons is ignored for mouseWheel + params.buttons = opts.buttons !== undefined ? opts.buttons : 0; + } + if (opts.type === 'mouseWheel') { + params.deltaX = opts.deltaX || 0; + params.deltaY = opts.deltaY || 0; + } + await this.send(tabId, 'Input.dispatchMouseEvent', params); + } + + static async insertText(tabId: number, text: string) { + await this.send(tabId, 'Input.insertText', { text }); + } + + static modifierMask(mods: string[]): number { + const map: Record = { + alt: 1, + ctrl: 2, + control: 2, + meta: 4, + cmd: 4, + command: 4, + win: 4, + windows: 4, + shift: 8, + }; + let mask = 0; + for (const m of mods) mask |= map[m] || 0; + return mask; + } + + // Enhanced key mapping for common non-character keys + private static KEY_ALIASES: Record = { + enter: { key: 'Enter', code: 'Enter' }, + return: { key: 'Enter', code: 'Enter' }, + backspace: { key: 'Backspace', code: 'Backspace' }, + delete: { key: 'Delete', code: 'Delete' }, + tab: { key: 'Tab', code: 'Tab' }, + escape: { key: 'Escape', code: 'Escape' }, + esc: { key: 'Escape', code: 'Escape' }, + space: { key: ' ', code: 'Space', text: ' ' }, + pageup: { key: 'PageUp', code: 'PageUp' }, + pagedown: { key: 'PageDown', code: 'PageDown' }, + home: { key: 'Home', code: 'Home' }, + end: { key: 'End', code: 'End' }, + arrowup: { key: 'ArrowUp', code: 'ArrowUp' }, + arrowdown: { key: 'ArrowDown', code: 'ArrowDown' }, + arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft' }, + arrowright: { key: 'ArrowRight', code: 'ArrowRight' }, + }; + + private static resolveKeyDef(token: string): { key: string; code?: string; text?: string } { + const t = (token || '').toLowerCase(); + if (this.KEY_ALIASES[t]) return this.KEY_ALIASES[t]; + if (/^f([1-9]|1[0-2])$/.test(t)) { + return { key: t.toUpperCase(), code: t.toUpperCase() }; + } + if (t.length === 1) { + const upper = t.toUpperCase(); + return { key: upper, code: `Key${upper}`, text: t }; + } + return { key: token }; + } + + static async dispatchSimpleKey(tabId: number, token: string) { + const def = this.resolveKeyDef(token); + if (def.text && def.text.length === 1) { + await this.insertText(tabId, def.text); + return; + } + await this.send(tabId, 'Input.dispatchKeyEvent', { + type: 'rawKeyDown', + key: def.key, + code: def.code, + }); + await this.send(tabId, 'Input.dispatchKeyEvent', { + type: 'keyUp', + key: def.key, + code: def.code, + }); + } + + static async dispatchKeyChord(tabId: number, chord: string) { + const parts = chord.split('+'); + const modifiers: string[] = []; + let keyToken = ''; + for (const pRaw of parts) { + const p = pRaw.trim().toLowerCase(); + if ( + ['ctrl', 'control', 'alt', 'shift', 'cmd', 'meta', 'command', 'win', 'windows'].includes(p) + ) + modifiers.push(p); + else keyToken = pRaw.trim(); + } + const mask = this.modifierMask(modifiers); + const def = this.resolveKeyDef(keyToken); + await this.send(tabId, 'Input.dispatchKeyEvent', { + type: 'rawKeyDown', + key: def.key, + code: def.code, + text: def.text, + modifiers: mask, + }); + await this.send(tabId, 'Input.dispatchKeyEvent', { + type: 'keyUp', + key: def.key, + code: def.code, + modifiers: mask, + }); + } +} + +class ComputerTool extends BaseBrowserToolExecutor { + name = TOOL_NAMES.BROWSER.COMPUTER; + + async execute(args: ComputerParams): Promise { + const params = args || ({} as ComputerParams); + if (!params.action) return createErrorResponse('Action parameter is required'); + + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (!tabs[0]) return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND); + const tab = tabs[0]; + if (!tab.id) + return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID'); + + // Helper to project coordinates using screenshot context when available + const project = (c?: Coordinates): Coordinates | undefined => { + if (!c) return undefined; + const ctx = screenshotContextManager.getContext(tab.id!); + if (!ctx) return c; + const scaled = scaleCoordinates(c.x, c.y, ctx); + return { x: scaled.x, y: scaled.y }; + }; + + switch (params.action) { + case 'hover': { + // Resolve target point from ref | selector | coordinates + let coord: Coordinates | undefined = undefined; + let resolvedBy: 'ref' | 'selector' | 'coordinates' | undefined; + + try { + if (params.ref) { + await this.injectContentScript(tab.id, [ + 'inject-scripts/accessibility-tree-helper.js', + ]); + const resolved = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: params.ref, + }); + if (resolved && resolved.success) { + coord = project({ x: resolved.center.x, y: resolved.center.y }); + resolvedBy = 'ref'; + } + } else if (params.selector) { + await this.injectContentScript(tab.id, [ + 'inject-scripts/accessibility-tree-helper.js', + ]); + const ensured = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + selector: params.selector, + }); + if (ensured && ensured.success) { + coord = project({ x: ensured.center.x, y: ensured.center.y }); + resolvedBy = 'selector'; + } + } else if (params.coordinates) { + coord = project(params.coordinates); + resolvedBy = 'coordinates'; + } + } catch (e) { + // fall through to error handling below + } + + if (!coord) + return createErrorResponse( + 'Provide ref or selector or coordinates for hover, or failed to resolve target', + ); + { + const stale = ((): any => { + if (!params.coordinates) return null; + const getHostname = (url: string): string => { + try { + return new URL(url).hostname; + } catch { + return ''; + } + }; + const currentHostname = getHostname(tab.url || ''); + const ctx = screenshotContextManager.getContext(tab.id!); + const contextHostname = (ctx as any)?.hostname as string | undefined; + if (contextHostname && contextHostname !== currentHostname) { + return createErrorResponse( + `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during hover. Capture a new screenshot or use ref/selector.`, + ); + } + return null; + })(); + if (stale) return stale; + } + + try { + await CDPHelper.attach(tab.id); + // Move pointer to target. We can dispatch a single mouseMoved; browsers will generate mouseover/mouseenter as needed. + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseMoved', + x: coord.x, + y: coord.y, + button: 'none', + buttons: 0, + }); + await CDPHelper.detach(tab.id); + + // Optional hold to allow UI (menus/tooltips) to appear + const holdMs = Math.max( + 0, + Math.min(params.duration ? params.duration * 1000 : 400, 5000), + ); + if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs)); + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'hover', + coordinates: coord, + resolvedBy, + }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + return createErrorResponse( + `Hover failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + case 'left_click': + case 'right_click': { + if (params.ref) { + // Prefer DOM click via ref + const domResult = await clickTool.execute({ + ref: params.ref, + waitForNavigation: false, + timeout: TIMEOUTS.DEFAULT_WAIT * 5, + }); + return domResult; + } + if (!params.coordinates) + return createErrorResponse('Coordinate parameter is required for click action'); + { + const stale = ((): any => { + const getHostname = (url: string): string => { + try { + return new URL(url).hostname; + } catch { + return ''; + } + }; + const currentHostname = getHostname(tab.url || ''); + const ctx = screenshotContextManager.getContext(tab.id!); + const contextHostname = (ctx as any)?.hostname as string | undefined; + if (contextHostname && contextHostname !== currentHostname) { + return createErrorResponse( + `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during ${params.action}. Capture a new screenshot or use ref/selector.`, + ); + } + return null; + })(); + if (stale) return stale; + } + const coord = project(params.coordinates)!; + // Prefer DOM path via existing click tool + const domResult = await clickTool.execute({ + coordinates: coord, + waitForNavigation: false, + timeout: TIMEOUTS.DEFAULT_WAIT * 5, + }); + if (!domResult.isError) { + return domResult; // Standardized response from click tool + } + // Fallback to CDP if DOM failed + try { + await CDPHelper.attach(tab.id); + const button: MouseButton = params.action === 'right_click' ? 'right' : 'left'; + const clickCount = 1; + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseMoved', + x: coord.x, + y: coord.y, + button: 'none', + buttons: 0, + }); + for (let i = 1; i <= clickCount; i++) { + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mousePressed', + x: coord.x, + y: coord.y, + button, + buttons: button === 'left' ? 1 : 2, + clickCount: i, + }); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseReleased', + x: coord.x, + y: coord.y, + button, + buttons: 0, + clickCount: i, + }); + } + await CDPHelper.detach(tab.id); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: params.action, + coordinates: coord, + }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + return createErrorResponse( + `CDP click failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + case 'double_click': + case 'triple_click': { + if (!params.coordinates && !params.ref) + return createErrorResponse('Provide ref or coordinates for double/triple click'); + let coord = params.coordinates ? project(params.coordinates)! : (undefined as any); + // If ref is provided, resolve center via accessibility helper + if (params.ref) { + try { + await this.injectContentScript(tab.id, [ + 'inject-scripts/accessibility-tree-helper.js', + ]); + const resolved = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: params.ref, + }); + if (resolved && resolved.success) { + coord = project({ x: resolved.center.x, y: resolved.center.y })!; + } + } catch (e) { + // ignore and use provided coordinates + } + } + if (!coord) return createErrorResponse('Failed to resolve coordinates from ref'); + { + const stale = ((): any => { + if (!params.coordinates) return null; + const getHostname = (url: string): string => { + try { + return new URL(url).hostname; + } catch { + return ''; + } + }; + const currentHostname = getHostname(tab.url || ''); + const ctx = screenshotContextManager.getContext(tab.id!); + const contextHostname = (ctx as any)?.hostname as string | undefined; + if (contextHostname && contextHostname !== currentHostname) { + return createErrorResponse( + `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during ${params.action}. Capture a new screenshot or use ref/selector.`, + ); + } + return null; + })(); + if (stale) return stale; + } + try { + await CDPHelper.attach(tab.id); + const button: MouseButton = 'left'; + const clickCount = params.action === 'double_click' ? 2 : 3; + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseMoved', + x: coord.x, + y: coord.y, + button: 'none', + buttons: 0, + }); + for (let i = 1; i <= clickCount; i++) { + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mousePressed', + x: coord.x, + y: coord.y, + button, + buttons: 1, + clickCount: i, + }); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseReleased', + x: coord.x, + y: coord.y, + button, + buttons: 0, + clickCount: i, + }); + } + await CDPHelper.detach(tab.id); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: params.action, + coordinates: coord, + }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + return createErrorResponse( + `CDP ${params.action} failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + case 'left_click_drag': { + if (!params.startCoordinates && !params.startRef) + return createErrorResponse('Provide startRef or startCoordinates for drag'); + if (!params.coordinates && !params.ref) + return createErrorResponse('Provide ref or end coordinates for drag'); + let start = params.startCoordinates + ? project(params.startCoordinates)! + : (undefined as any); + let end = params.coordinates ? project(params.coordinates)! : (undefined as any); + { + const stale = ((): any => { + if (!params.startCoordinates && !params.coordinates) return null; + const getHostname = (url: string): string => { + try { + return new URL(url).hostname; + } catch { + return ''; + } + }; + const currentHostname = getHostname(tab.url || ''); + const ctx = screenshotContextManager.getContext(tab.id!); + const contextHostname = (ctx as any)?.hostname as string | undefined; + if (contextHostname && contextHostname !== currentHostname) { + return createErrorResponse( + `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during left_click_drag. Capture a new screenshot or use ref/selector.`, + ); + } + return null; + })(); + if (stale) return stale; + } + if (params.startRef || params.ref) { + await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']); + } + if (params.startRef) { + try { + const resolved = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: params.startRef, + }); + if (resolved && resolved.success) + start = project({ x: resolved.center.x, y: resolved.center.y })!; + } catch { + // ignore + } + } + if (params.ref) { + try { + const resolved = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: params.ref, + }); + if (resolved && resolved.success) + end = project({ x: resolved.center.x, y: resolved.center.y })!; + } catch { + // ignore + } + } + if (!start || !end) return createErrorResponse('Failed to resolve drag coordinates'); + try { + await CDPHelper.attach(tab.id); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseMoved', + x: start.x, + y: start.y, + button: 'none', + buttons: 0, + }); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mousePressed', + x: start.x, + y: start.y, + button: 'left', + buttons: 1, + clickCount: 1, + }); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseMoved', + x: end.x, + y: end.y, + button: 'left', + buttons: 1, + }); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseReleased', + x: end.x, + y: end.y, + button: 'left', + buttons: 0, + clickCount: 1, + }); + await CDPHelper.detach(tab.id); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'left_click_drag', start, end }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + return createErrorResponse( + `Drag failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + case 'scroll': { + if (!params.coordinates && !params.ref) + return createErrorResponse('Provide ref or coordinates for scroll'); + let coord = params.coordinates ? project(params.coordinates)! : (undefined as any); + if (params.ref) { + try { + await this.injectContentScript(tab.id, [ + 'inject-scripts/accessibility-tree-helper.js', + ]); + const resolved = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: params.ref, + }); + if (resolved && resolved.success) + coord = project({ x: resolved.center.x, y: resolved.center.y })!; + } catch { + // ignore + } + } + if (!coord) return createErrorResponse('Failed to resolve scroll coordinates'); + { + const stale = ((): any => { + if (!params.coordinates) return null; + const getHostname = (url: string): string => { + try { + return new URL(url).hostname; + } catch { + return ''; + } + }; + const currentHostname = getHostname(tab.url || ''); + const ctx = screenshotContextManager.getContext(tab.id!); + const contextHostname = (ctx as any)?.hostname as string | undefined; + if (contextHostname && contextHostname !== currentHostname) { + return createErrorResponse( + `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during scroll. Capture a new screenshot or use ref/selector.`, + ); + } + return null; + })(); + if (stale) return stale; + } + const direction = params.scrollDirection || 'down'; + const amount = Math.max(1, Math.min(params.scrollAmount || 3, 10)); + // Convert to deltas (~100px per tick) + const unit = 100; + let deltaX = 0, + deltaY = 0; + if (direction === 'up') deltaY = -amount * unit; + if (direction === 'down') deltaY = amount * unit; + if (direction === 'left') deltaX = -amount * unit; + if (direction === 'right') deltaX = amount * unit; + try { + await CDPHelper.attach(tab.id); + await CDPHelper.dispatchMouseEvent(tab.id, { + type: 'mouseWheel', + x: coord.x, + y: coord.y, + deltaX, + deltaY, + }); + await CDPHelper.detach(tab.id); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'scroll', + coordinates: coord, + deltaX, + deltaY, + }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + return createErrorResponse( + `Scroll failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + case 'type': { + if (!params.text) + return createErrorResponse('Text parameter is required for type action'); + try { + // Optional focus via ref before typing + if (params.ref) { + await clickTool.execute({ + ref: params.ref, + waitForNavigation: false, + timeout: TIMEOUTS.DEFAULT_WAIT * 5, + }); + } + await CDPHelper.attach(tab.id); + // Use CDP insertText to avoid complex KeyboardEvent emulation for long text + await CDPHelper.insertText(tab.id, params.text); + await CDPHelper.detach(tab.id); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'type', + length: params.text.length, + }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + // Fallback to DOM-based keyboard tool + const res = await keyboardTool.execute({ + keys: params.text.split('').join(','), + delay: 0, + selector: undefined, + }); + return res; + } + } + case 'fill': { + if (!params.ref && !params.selector) { + return createErrorResponse('Provide ref or selector and a value for fill'); + } + // Reuse existing fill tool to leverage robust DOM event behavior + const res = await fillTool.execute({ + selector: params.selector as any, + ref: params.ref as any, + value: params.value as any, + } as any); + return res; + } + case 'key': { + if (!params.text) + return createErrorResponse( + 'text is required for key action (e.g., "Backspace Backspace Enter" or "cmd+a")', + ); + const tokens = params.text.trim().split(/\s+/).filter(Boolean); + try { + // Optional focus via ref before key events + if (params.ref) { + await clickTool.execute({ + ref: params.ref, + waitForNavigation: false, + timeout: TIMEOUTS.DEFAULT_WAIT * 5, + }); + } + await CDPHelper.attach(tab.id); + for (const t of tokens) { + if (t.includes('+')) await CDPHelper.dispatchKeyChord(tab.id, t); + else await CDPHelper.dispatchSimpleKey(tab.id, t); + } + await CDPHelper.detach(tab.id); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'key', keys: tokens }), + }, + ], + isError: false, + }; + } catch (e) { + await CDPHelper.detach(tab.id); + // Fallback to DOM keyboard simulation (comma-separated combinations) + const keysStr = tokens.join(','); + const res = await keyboardTool.execute({ keys: keysStr }); + return res; + } + } + case 'wait': { + const seconds = Math.max(0, Math.min(params.duration || 0, 30)); + if (!seconds) + return createErrorResponse('Duration parameter is required and must be > 0'); + await new Promise((r) => setTimeout(r, seconds * 1000)); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'wait', duration: seconds }), + }, + ], + isError: false, + }; + } + case 'screenshot': { + // Reuse existing screenshot tool; it already supports base64 save option + const result = await screenshotTool.execute({ + name: 'computer', + storeBase64: true, + fullPage: false, + }); + return result; + } + default: + return createErrorResponse(`Unsupported action: ${params.action}`); + } + } catch (error) { + console.error('Error in computer tool:', error); + return createErrorResponse( + `Failed to execute action: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} + +export const computerTool = new ComputerTool(); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/index.ts b/app/chrome-extension/entrypoints/background/tools/browser/index.ts index ccc655ee..f4bcc2ba 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/index.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/index.ts @@ -13,3 +13,5 @@ export { bookmarkSearchTool, bookmarkAddTool, bookmarkDeleteTool } from './bookm export { injectScriptTool, sendCommandToInjectScriptTool } from './inject-script'; export { consoleTool } from './console'; export { fileUploadTool } from './file-upload'; +export { readPageTool } from './read-page'; +export { computerTool } from './computer'; diff --git a/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts b/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts index 12def52b..00863497 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts @@ -11,6 +11,7 @@ interface Coordinates { interface ClickToolParams { selector?: string; // CSS selector for the element to click + ref?: string; // Element ref from accessibility tree (window.__claudeElementMap) coordinates?: Coordinates; // Coordinates to click at (x, y relative to viewport) waitForNavigation?: boolean; // Whether to wait for navigation to complete after click timeout?: number; // Timeout in milliseconds for waiting for the element or navigation @@ -35,9 +36,9 @@ class ClickTool extends BaseBrowserToolExecutor { console.log(`Starting click operation with options:`, args); - if (!selector && !coordinates) { + if (!selector && !coordinates && !args.ref) { return createErrorResponse( - ERROR_MESSAGES.INVALID_PARAMETERS + ': Either selector or coordinates must be provided', + ERROR_MESSAGES.INVALID_PARAMETERS + ': Provide ref or selector or coordinates', ); } @@ -60,6 +61,7 @@ class ClickTool extends BaseBrowserToolExecutor { action: TOOL_MESSAGE_TYPES.CLICK_ELEMENT, selector, coordinates, + ref: args.ref, waitForNavigation, timeout, }); @@ -91,8 +93,10 @@ class ClickTool extends BaseBrowserToolExecutor { export const clickTool = new ClickTool(); interface FillToolParams { - selector: string; - value: string; + selector?: string; + ref?: string; // Element ref from accessibility tree + // Accept string | number | boolean for broader form input coverage + value: string | number | boolean; } /** @@ -105,12 +109,12 @@ class FillTool extends BaseBrowserToolExecutor { * Execute fill operation */ async execute(args: FillToolParams): Promise { - const { selector, value } = args; + const { selector, ref, value } = args; console.log(`Starting fill operation with options:`, args); - if (!selector) { - return createErrorResponse(ERROR_MESSAGES.INVALID_PARAMETERS + ': Selector must be provided'); + if (!selector && !ref) { + return createErrorResponse(ERROR_MESSAGES.INVALID_PARAMETERS + ': Provide ref or selector'); } if (value === undefined || value === null) { @@ -135,10 +139,11 @@ class FillTool extends BaseBrowserToolExecutor { const result = await this.sendMessageToTab(tab.id, { action: TOOL_MESSAGE_TYPES.FILL_ELEMENT, selector, + ref, value, }); - if (result.error) { + if (result && result.error) { return createErrorResponse(result.error); } diff --git a/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts b/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts new file mode 100644 index 00000000..425e25eb --- /dev/null +++ b/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts @@ -0,0 +1,111 @@ +import { createErrorResponse, ToolResult } from '@/common/tool-handler'; +import { BaseBrowserToolExecutor } from '../base-browser'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { TOOL_MESSAGE_TYPES } from '@/common/message-types'; +import { ERROR_MESSAGES } from '@/common/constants'; + +interface ReadPageParams { + filter?: 'interactive'; // when omitted, return all visible elements +} + +class ReadPageTool extends BaseBrowserToolExecutor { + name = TOOL_NAMES.BROWSER.READ_PAGE; + + // Execute read page + async execute(args: ReadPageParams): Promise { + const { filter } = args || {}; + + try { + // Tip text returned to callers to guide next action + const standardTips = + "If the specific element you need is missing from the returned data, use the 'screenshot' tool to capture the current viewport and confirm the element's on-screen coordinates."; + + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (!tabs[0]) return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND); + const tab = tabs[0]; + if (!tab.id) + return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID'); + + // Inject helper in ISOLATED world to enable chrome.runtime messaging + await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']); + + // Ask content script to generate accessibility tree + const resp = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.GENERATE_ACCESSIBILITY_TREE, + filter: filter || null, + }); + + // Evaluate tree result and decide whether to fallback + const treeOk = resp && resp.success === true; + const pageContent: string = + resp && typeof resp.pageContent === 'string' ? resp.pageContent : ''; + const lines = pageContent + ? pageContent.split('\n').filter((l: string) => l.trim().length > 0).length + : 0; + const refCount = Array.isArray(resp?.refMap) ? resp.refMap.length : 0; + const isSparse = lines < 10 && refCount < 3; // heuristic threshold for sparse trees + + if (treeOk && !isSparse) { + // Normal path: return tree + const resultPayload = { + success: true, + filter: filter || 'all', + pageContent: resp.pageContent, + tips: standardTips, + viewport: resp.viewport, + refMapCount: refCount, + sparse: false, + }; + + return { + content: [{ type: 'text', text: JSON.stringify(resultPayload) }], + isError: false, + }; + } + + // Fallback path: try get_interactive_elements once + try { + await this.injectContentScript(tab.id, ['inject-scripts/interactive-elements-helper.js']); + const fallback = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.GET_INTERACTIVE_ELEMENTS, + includeCoordinates: true, + }); + + if (fallback && fallback.success && Array.isArray(fallback.elements)) { + const limited = fallback.elements.slice(0, 150); + const fallbackPayload = { + success: true, + fallbackUsed: true, + fallbackSource: 'get_interactive_elements', + reason: treeOk ? 'sparse_tree' : resp?.error || 'tree_failed', + treeStats: { lines, refCount }, + elements: limited, + count: fallback.elements.length, + tips: standardTips, + }; + + return { + content: [{ type: 'text', text: JSON.stringify(fallbackPayload) }], + isError: false, + }; + } + } catch (fallbackErr) { + console.warn('read_page fallback failed:', fallbackErr); + } + + // If we reach here, both tree (usable) and fallback failed + return createErrorResponse( + treeOk + ? 'Accessibility tree is too sparse and fallback failed' + : resp?.error || 'Failed to generate accessibility tree and fallback failed', + ); + } catch (error) { + console.error('Error in read page tool:', error); + return createErrorResponse( + `Error generating accessibility tree: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} + +export const readPageTool = new ReadPageTool(); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/screenshot.ts b/app/chrome-extension/entrypoints/background/tools/browser/screenshot.ts index e989246d..ff50c203 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/screenshot.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/screenshot.ts @@ -10,6 +10,7 @@ import { stitchImages, compressImage, } from '../../../../utils/image-utils'; +import { screenshotContextManager } from '@/utils/screenshot-context'; // Screenshot-specific constants const SCREENSHOT_CONSTANTS = { @@ -28,7 +29,20 @@ const SCREENSHOT_CONSTANTS = { readonly SCRIPT_INIT_DELAY: number; }; -SCREENSHOT_CONSTANTS["CAPTURE_STITCH_DELAY_MS"] = Math.max(1000 / chrome.tabs.MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND - SCREENSHOT_CONSTANTS.SCROLL_DELAY_MS, SCREENSHOT_CONSTANTS.CAPTURE_STITCH_DELAY_MS) +// Adjust CAPTURE_STITCH_DELAY_MS to respect Chrome's capture rate if available in runtime +// Some TS typings don't expose MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND; use a safe cast with a sane fallback. +const __MAX_CAP_RATE: number | undefined = (chrome.tabs as any) + ?.MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND; +if (typeof __MAX_CAP_RATE === 'number' && __MAX_CAP_RATE > 0) { + // Minimum interval between consecutive captureVisibleTab calls (ms) + const minIntervalMs = Math.ceil(1000 / __MAX_CAP_RATE); + // Our capture loop already waits SCROLL_DELAY_MS between scroll and capture; add any extra delay needed + const requiredExtraDelay = Math.max(0, minIntervalMs - SCREENSHOT_CONSTANTS.SCROLL_DELAY_MS); + SCREENSHOT_CONSTANTS.CAPTURE_STITCH_DELAY_MS = Math.max( + requiredExtraDelay, + SCREENSHOT_CONSTANTS.CAPTURE_STITCH_DELAY_MS, + ); +} interface ScreenshotToolParams { name: string; @@ -81,6 +95,8 @@ class ScreenshotTool extends BaseBrowserToolExecutor { } let finalImageDataUrl: string | undefined; + let finalImageWidthCss: number | undefined; + let finalImageHeightCss: number | undefined; const results: any = { base64: null, fileSaved: false }; let originalScroll = { x: 0, y: 0 }; @@ -102,14 +118,45 @@ class ScreenshotTool extends BaseBrowserToolExecutor { if (fullPage) { this.logInfo('Capturing full page...'); - finalImageDataUrl = await this._captureFullPage(tab.id!, args, pageDetails); + const dataUrl = await this._captureFullPage(tab.id!, args, pageDetails); + finalImageDataUrl = dataUrl; + // For full page, compute final CSS size from provided width/height fallback to pageDetails + if (args.width && args.height) { + finalImageWidthCss = args.width; + finalImageHeightCss = args.height; + } else if (args.width && !args.height) { + finalImageWidthCss = args.width; + // height will be scaled by aspect ratio; approximate with page ratio + const ratio = pageDetails.totalHeight / pageDetails.totalWidth; + finalImageHeightCss = Math.round(args.width * ratio); + } else if (!args.width && args.height) { + finalImageHeightCss = args.height; + const ratio = pageDetails.totalWidth / pageDetails.totalHeight; + finalImageWidthCss = Math.round(args.height * ratio); + } else { + finalImageWidthCss = pageDetails.totalWidth; + finalImageHeightCss = pageDetails.totalHeight; + } } else if (selector) { this.logInfo(`Capturing element: ${selector}`); - finalImageDataUrl = await this._captureElement(tab.id!, args, pageDetails.devicePixelRatio); + const cropped = await this._captureElement(tab.id!, args, pageDetails.devicePixelRatio); + finalImageDataUrl = cropped; + // For element capture, if target width/height provided, respect them; otherwise use element rect size + if (args.width && args.height) { + finalImageWidthCss = args.width; + finalImageHeightCss = args.height; + } else { + // Fallback to visible element rect (already used for crop) + // We do not have easy access to rect here; use viewport as conservative default + finalImageWidthCss = pageDetails.viewportWidth; + finalImageHeightCss = pageDetails.viewportHeight; + } } else { // Visible area only this.logInfo('Capturing visible area...'); finalImageDataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { format: 'png' }); + finalImageWidthCss = pageDetails.viewportWidth; + finalImageHeightCss = pageDetails.viewportHeight; } if (!finalImageDataUrl) { @@ -117,6 +164,27 @@ class ScreenshotTool extends BaseBrowserToolExecutor { } // 2. Process output + // Update screenshot context for coordinate scaling by tools like chrome_computer + try { + if (typeof finalImageWidthCss === 'number' && typeof finalImageHeightCss === 'number') { + let hostname = ''; + try { + hostname = tab.url ? new URL(tab.url).hostname : ''; + } catch { + // ignore + } + screenshotContextManager.setContext(tab.id!, { + screenshotWidth: finalImageWidthCss, + screenshotHeight: finalImageHeightCss, + viewportWidth: pageDetails.viewportWidth, + viewportHeight: pageDetails.viewportHeight, + devicePixelRatio: pageDetails.devicePixelRatio, + hostname, + }); + } + } catch (e) { + console.warn('Failed to set screenshot context:', e); + } if (storeBase64 === true) { // Compress image for base64 output to reduce size const compressed = await compressImage(finalImageDataUrl, { diff --git a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js new file mode 100644 index 00000000..5ee2454a --- /dev/null +++ b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js @@ -0,0 +1,593 @@ +/* eslint-disable */ +// accessibility-tree-helper.js +// Injected script to generate an accessibility-like tree of the visible page +// Elements receive stable refs (ref_*) via WeakRef mapping for later reference. + +(function () { + if (window.__ACCESSIBILITY_TREE_HELPER_INITIALIZED__) return; + window.__ACCESSIBILITY_TREE_HELPER_INITIALIZED__ = true; + + // Traversal and output limits to ensure stability on very large/complex pages + const MAX_DEPTH = 30; // maximum DOM depth to traverse + const MAX_NODES = 4000; // hard limit to avoid long blocking on huge DOMs + const MAX_LINE_LABEL = 100; // max characters for a single label in output + const REF_MAP_LIMIT = 1000; // limit size of the ref map to keep payload small + + // Keep a weak map from ref id to elements + if (!window.__claudeElementMap) window.__claudeElementMap = {}; + if (!window.__claudeRefCounter) window.__claudeRefCounter = 0; + + /** + * Infer ARIA-like role from element + * @param {Element} el + * @returns {string} + */ + function inferRole(el) { + const role = el.getAttribute('role'); + if (role) return role; + const tag = el.tagName.toLowerCase(); + const type = el.getAttribute('type') || ''; + const map = { + a: 'link', + button: 'button', + input: + type === 'submit' || type === 'button' + ? 'button' + : type === 'checkbox' + ? 'checkbox' + : type === 'radio' + ? 'radio' + : type === 'file' + ? 'button' + : 'textbox', + select: 'combobox', + textarea: 'textbox', + h1: 'heading', + h2: 'heading', + h3: 'heading', + h4: 'heading', + h5: 'heading', + h6: 'heading', + img: 'image', + nav: 'navigation', + main: 'main', + header: 'banner', + footer: 'contentinfo', + section: 'region', + article: 'article', + aside: 'complementary', + form: 'form', + table: 'table', + ul: 'list', + ol: 'list', + li: 'listitem', + label: 'label', + }; + return map[tag] || 'generic'; + } + + /** + * Derive readable label for element + * @param {Element} el + * @returns {string} + */ + function inferLabel(el) { + const tag = el.tagName.toLowerCase(); + if (tag === 'select') { + const sel = /** @type {HTMLSelectElement} */ (el); + const opt = sel.querySelector('option[selected]') || sel.options[sel.selectedIndex]; + if (opt && opt.textContent) return opt.textContent.trim(); + } + const aria = el.getAttribute('aria-label'); + if (aria && aria.trim()) return aria.trim(); + const placeholder = el.getAttribute('placeholder'); + if (placeholder && placeholder.trim()) return placeholder.trim(); + const title = el.getAttribute('title'); + if (title && title.trim()) return title.trim(); + const alt = el.getAttribute('alt'); + if (alt && alt.trim()) return alt.trim(); + if (/** @type {HTMLElement} */ (el).id) { + const lab = document.querySelector(`label[for="${/** @type {HTMLElement} */ (el).id}"]`); + if (lab && lab.textContent && lab.textContent.trim()) return lab.textContent.trim(); + } + if (tag === 'input') { + const input = /** @type {HTMLInputElement} */ (el); + const type = input.getAttribute('type') || ''; + const val = input.getAttribute('value'); + if (type === 'submit' && val && val.trim()) return val.trim(); + if (input.value && input.value.length < 50 && input.value.trim()) return input.value.trim(); + } + if (['button', 'a', 'summary'].includes(tag)) { + let text = ''; + for (let i = 0; i < el.childNodes.length; i++) { + const n = el.childNodes[i]; + if (n.nodeType === Node.TEXT_NODE) text += n.textContent || ''; + } + if (text.trim()) return text.trim(); + } + if (/^h[1-6]$/.test(tag)) { + const t = el.textContent; + if (t && t.trim()) return t.trim().substring(0, MAX_LINE_LABEL); + } + if (tag === 'img') { + const src = el.getAttribute('src'); + if (src) { + const file = src.split('/').pop()?.split('?')[0]; + return `Image: ${file}`; + } + } + let agg = ''; + for (let i = 0; i < el.childNodes.length; i++) { + const n = el.childNodes[i]; + if (n.nodeType === Node.TEXT_NODE) agg += n.textContent || ''; + } + if (agg && agg.trim() && agg.trim().length >= 3) { + const v = agg.trim(); + return v.length > 50 ? v.substring(0, 50) + '...' : v; + } + return ''; + } + + /** + * Check if element is visible in DOM + * @param {Element} el + */ + function isVisible(el) { + const cs = window.getComputedStyle(/** @type {HTMLElement} */ (el)); + if (cs.display === 'none' || cs.visibility === 'hidden' || cs.opacity === '0') return false; + const he = /** @type {HTMLElement} */ (el); + return he.offsetWidth > 0 && he.offsetHeight > 0; + } + + /** + * Whether the element is interactive + * @param {Element} el + */ + function isInteractive(el) { + // Native interactive tags + const tag = el.tagName.toLowerCase(); + if (['a', 'button', 'input', 'select', 'textarea', 'details', 'summary'].includes(tag)) + return true; + + // Generic interactive hints + if (el.getAttribute('onclick') != null) return true; + if ( + el.getAttribute('tabindex') != null && + String(el.getAttribute('tabindex')).trim() !== '' && + !String(el.getAttribute('tabindex')).trim().startsWith('-') + ) + return true; + if (el.getAttribute('contenteditable') === 'true') return true; + + // ARIA roles commonly used by custom elements + const role = (el.getAttribute && el.getAttribute('role')) || ''; + const interactiveRoles = new Set([ + 'button', + 'link', + 'checkbox', + 'radio', + 'switch', + 'slider', + 'option', + 'menuitem', + 'textbox', + 'searchbox', + 'combobox', + 'spinbutton', + 'tab', + 'treeitem', + ]); + if (role && interactiveRoles.has(role.toLowerCase())) return true; + + // Shadow host case: treat host as interactive if its open shadow root contains + // an interactive control (textarea/input/select/button/a or contenteditable). + try { + const anyEl = /** @type {any} */ (el); + const sr = anyEl && anyEl.shadowRoot ? anyEl.shadowRoot : null; + if (sr) { + const inner = sr.querySelector( + 'input, textarea, select, button, a[href], [contenteditable="true"], [role="button"], [role="link"], [role="textbox"], [role="combobox"], [role="searchbox"], [role="menuitem"], [role="option"], [role="switch"], [role="radio"], [role="checkbox"], [role="tab"], [role="slider"]', + ); + if (inner) return true; + } + } catch (_) { + /* ignore */ + } + return false; + } + + /** + * Structural containers useful to include + * @param {Element} el + */ + function isStructural(el) { + const tag = el.tagName.toLowerCase(); + if ( + [ + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'nav', + 'main', + 'header', + 'footer', + 'section', + 'article', + 'aside', + ].includes(tag) + ) + return true; + return el.getAttribute('role') != null; + } + + /** + * Form-ish containers to keep + * @param {Element} el + */ + function isFormishContainer(el) { + const tag = el.tagName.toLowerCase(); + const role = (el.getAttribute && el.getAttribute('role')) || ''; + const id = /** @type {HTMLElement} */ (el).id || ''; + // Normalize className for HTML/SVG elements + let cls = ''; + try { + const attr = el.getAttribute && el.getAttribute('class'); + if (typeof attr === 'string') cls = attr; + else { + const cn = /** @type {any} */ (el).className; + if (typeof cn === 'string') cls = cn; + else if (cn && typeof cn.baseVal === 'string') cls = cn.baseVal; + } + } catch (e) { + /* ignore */ + } + return ( + role === 'search' || + role === 'form' || + role === 'group' || + role === 'toolbar' || + role === 'navigation' || + tag === 'form' || + tag === 'fieldset' || + tag === 'nav' || + tag === 'legend' || + id.includes('search') || + cls.includes('search') || + id.includes('form') || + cls.includes('form') || + id.includes('menu') || + cls.includes('menu') || + id.includes('nav') || + cls.includes('nav') + ); + } + + /** + * Whether to include element in tree under config + * @param {Element} el + * @param {{filter?: 'all'|'interactive'}} cfg + */ + function shouldInclude(el, cfg) { + const tag = el.tagName.toLowerCase(); + if (['script', 'style', 'meta', 'link', 'title', 'noscript'].includes(tag)) return false; + if (el.getAttribute('aria-hidden') === 'true') return false; + if (!isVisible(el)) return false; + if (cfg.filter !== 'all') { + const r = /** @type {HTMLElement} */ (el).getBoundingClientRect(); + if ( + !(r.top < window.innerHeight && r.bottom > 0 && r.left < window.innerWidth && r.right > 0) + ) + return false; + } + if (cfg.filter === 'interactive') return isInteractive(el); + if (isInteractive(el)) return true; + if (isStructural(el)) return true; + if (inferLabel(el).length > 0) return true; + return isFormishContainer(el); + } + + /** + * Generate a fairly stable CSS selector + * @param {Element} el + * @returns {string} + */ + function generateSelector(el) { + if (!(el instanceof Element)) return ''; + if (/** @type {HTMLElement} */ (el).id) { + const idSel = `#${CSS.escape(/** @type {HTMLElement} */ (el).id)}`; + if (document.querySelectorAll(idSel).length === 1) return idSel; + } + for (const attr of ['data-testid', 'data-cy', 'name']) { + const attrValue = el.getAttribute(attr); + if (attrValue) { + const s = `[${attr}="${CSS.escape(attrValue)}"]`; + if (document.querySelectorAll(s).length === 1) return s; + } + } + let path = ''; + let current = el; + while (current && current.nodeType === Node.ELEMENT_NODE && current.tagName !== 'BODY') { + let selector = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = Array.from(parent.children).filter( + (child) => child.tagName === current.tagName, + ); + if (siblings.length > 1) { + const index = siblings.indexOf(current) + 1; + selector += `:nth-of-type(${index})`; + } + } + path = path ? `${selector} > ${path}` : selector; + current = parent; + } + return path ? `body > ${path}` : 'body'; + } + + /** + * Traverse DOM and build pageContent lines; collect ref map for interactive nodes. + * @param {Element} el + * @param {number} depth + * @param {{filter?: 'all'|'interactive'}} cfg + * @param {string[]} out + * @param {Array<{ref:string, selector:string, rect:{x:number,y:number,width:number,height:number}}>} refMap + */ + function traverse(el, depth, cfg, out, refMap, state) { + if (depth > MAX_DEPTH || !el || !el.tagName) return; + if (state.processed >= MAX_NODES) return; + if (state.visited.has(el)) return; + state.visited.add(el); + const include = shouldInclude(el, cfg) || depth === 0; + if (include) { + const role = inferRole(el); + let label = inferLabel(el); + let refId = null; + for (const k in window.__claudeElementMap) { + if (window.__claudeElementMap[k].deref && window.__claudeElementMap[k].deref() === el) { + refId = k; + break; + } + } + if (!refId) { + refId = `ref_${++window.__claudeRefCounter}`; + window.__claudeElementMap[refId] = new WeakRef(el); + } + const rect = /** @type {HTMLElement} */ (el).getBoundingClientRect(); + const cx = Math.round(rect.left + rect.width / 2); + const cy = Math.round(rect.top + rect.height / 2); + let line = `${' '.repeat(depth)}- ${role}`; + if (label) { + label = label.replace(/\s+/g, ' ').substring(0, MAX_LINE_LABEL); + line += ` "${label.replace(/"/g, '\\"')}"`; + } + line += ` [ref=${refId}] (x=${cx},y=${cy})`; + if (/** @type {HTMLElement} */ (el).id) line += ` id="${/** @type {HTMLElement} */ (el).id}"`; + const href = el.getAttribute('href'); + if (href) line += ` href="${href}"`; + const type = el.getAttribute('type'); + if (type) line += ` type="${type}"`; + const placeholder = el.getAttribute('placeholder'); + if (placeholder) line += ` placeholder="${placeholder}"`; + // Surface disabled/pointer-events for better agent judgement + try { + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true'; + if (disabled) line += ` disabled`; + const cs = window.getComputedStyle(/** @type {HTMLElement} */ (el)); + if (cs && cs.pointerEvents === 'none') line += ` pe=none`; + } catch (_) { + /* ignore style issues */ + } + out.push(line); + state.included++; + state.processed++; + + // Only collect ref mapping for interactive elements to limit cost + if (isInteractive(el) && refMap.length < REF_MAP_LIMIT) { + refMap.push({ + ref: /** @type {string} */ (refId), + selector: generateSelector(el), + rect: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }, + }); + } + } + if (state.processed >= MAX_NODES) return; + // Traverse light DOM children + if (/** @type {HTMLElement} */ (el).children && depth < MAX_DEPTH) { + const children = /** @type {HTMLElement} */ (el).children; + for (let i = 0; i < children.length; i++) { + if (state.processed >= MAX_NODES) break; + traverse(children[i], include ? depth + 1 : depth, cfg, out, refMap, state); + } + } + // Traverse shadow DOM roots (limited by MAX_DEPTH and MAX_NODES) + try { + const anyEl = /** @type {any} */ (el); + if (anyEl && anyEl.shadowRoot && depth < MAX_DEPTH) { + const srChildren = anyEl.shadowRoot.children || []; + for (let i = 0; i < srChildren.length; i++) { + if (state.processed >= MAX_NODES) break; + traverse(srChildren[i], include ? depth + 1 : depth, cfg, out, refMap, state); + } + } + } catch (_) { + /* ignore shadow errors */ + } + } + + /** + * Generate tree and return + * @param {'all'|'interactive'|null} filter + */ + function __generateAccessibilityTree(filter) { + try { + const start = performance && performance.now ? performance.now() : Date.now(); + const out = []; + const cfg = { filter: filter || undefined }; + const refMap = []; + const state = { processed: 0, included: 0, visited: new WeakSet() }; + if (document.body) traverse(document.body, 0, cfg, out, refMap, state); + for (const k in window.__claudeElementMap) { + if (!window.__claudeElementMap[k].deref || !window.__claudeElementMap[k].deref()) + delete window.__claudeElementMap[k]; + } + const pageContent = out + .filter((line) => !/^\s*- generic \[ref=ref_\d+\]$/.test(line)) + .join('\n'); + const end = performance && performance.now ? performance.now() : Date.now(); + return { + pageContent, + viewport: { + width: window.innerWidth, + height: window.innerHeight, + dpr: window.devicePixelRatio || 1, + }, + stats: { + processed: state.processed, + included: state.included, + durationMs: Math.round(end - start), + }, + refMap, + }; + } catch (err) { + throw new Error( + 'Error generating accessibility tree: ' + + (err && err.message ? err.message : 'Unknown error'), + ); + } + } + + // Expose API on window + window.__generateAccessibilityTree = __generateAccessibilityTree; + + // Chrome message bridge for ping and tree generation + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { + try { + if (request && request.action === 'chrome_read_page_ping') { + sendResponse({ status: 'pong' }); + return false; + } + if (request && request.action === 'generateAccessibilityTree') { + const result = __generateAccessibilityTree(request.filter || null); + sendResponse({ success: true, ...result }); + return true; + } + if (request && request.action === 'ensureRefForSelector') { + try { + const sel = String(request.selector || '').trim(); + if (!sel) { + sendResponse({ success: false, error: 'selector is required' }); + return true; + } + const el = document.querySelector(sel); + if (!el) { + sendResponse({ success: false, error: `selector not found: ${sel}` }); + return true; + } + let refId = null; + for (const k in window.__claudeElementMap) { + if (window.__claudeElementMap[k].deref && window.__claudeElementMap[k].deref() === el) { + refId = k; + break; + } + } + if (!refId) { + refId = `ref_${++window.__claudeRefCounter}`; + window.__claudeElementMap[refId] = new WeakRef(el); + } + const rect = /** @type {HTMLElement} */ (el).getBoundingClientRect(); + sendResponse({ + success: true, + ref: refId, + center: { + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2), + }, + }); + return true; + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + } + if (request && request.action === 'resolveRef') { + const ref = request.ref; + try { + const map = window.__claudeElementMap; + const weak = map && map[ref]; + const el = weak && typeof weak.deref === 'function' ? weak.deref() : null; + if (!el || !(el instanceof Element)) { + sendResponse({ success: false, error: `ref "${ref}" not found or expired` }); + return true; + } + const rect = /** @type {HTMLElement} */ (el).getBoundingClientRect(); + sendResponse({ + success: true, + rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, + center: { + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2), + }, + selector: (function () { + // Simple selector generation inline to avoid duplication + const generateSelector = function (node) { + if (!(node instanceof Element)) return ''; + if (node.id) { + const idSel = `#${CSS.escape(node.id)}`; + if (document.querySelectorAll(idSel).length === 1) return idSel; + } + for (const attr of ['data-testid', 'data-cy', 'name']) { + const val = node.getAttribute(attr); + if (val) { + const s = `[${attr}="${CSS.escape(val)}"]`; + if (document.querySelectorAll(s).length === 1) return s; + } + } + let path = ''; + let current = node; + while ( + current && + current.nodeType === Node.ELEMENT_NODE && + current.tagName !== 'BODY' + ) { + let sel = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => c.tagName === current.tagName, + ); + if (siblings.length > 1) { + const idx = siblings.indexOf(current) + 1; + sel += `:nth-of-type(${idx})`; + } + } + path = path ? `${sel} > ${path}` : sel; + current = parent; + } + return path ? `body > ${path}` : 'body'; + }; + return generateSelector(el); + })(), + }); + return true; + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + } + } catch (e) { + sendResponse({ success: false, error: e && e.message ? e.message : String(e) }); + return true; + } + return false; + }); + + console.log('Accessibility tree helper script loaded'); +})(); diff --git a/app/chrome-extension/inject-scripts/click-helper.js b/app/chrome-extension/inject-scripts/click-helper.js index f5ee600e..bad7f839 100644 --- a/app/chrome-extension/inject-scripts/click-helper.js +++ b/app/chrome-extension/inject-scripts/click-helper.js @@ -21,13 +21,63 @@ if (window.__CLICK_HELPER_INITIALIZED__) { waitForNavigation = false, timeout = 5000, coordinates = null, + ref = null, ) { try { let element = null; let elementInfo = null; let clickX, clickY; - if (coordinates && typeof coordinates.x === 'number' && typeof coordinates.y === 'number') { + if (ref && typeof ref === 'string') { + // Resolve element from weak map + let target = null; + try { + const map = window.__claudeElementMap; + const weak = map && map[ref]; + target = weak && typeof weak.deref === 'function' ? weak.deref() : null; + } catch (e) { + // ignore + } + + if (!target || !(target instanceof Element)) { + return { + error: `Element ref "${ref}" not found. Please call chrome_read_page first and ensure the ref is still valid.`, + }; + } + + element = target; + element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'center' }); + await new Promise((resolve) => setTimeout(resolve, 80)); + + const rect = element.getBoundingClientRect(); + clickX = rect.left + rect.width / 2; + clickY = rect.top + rect.height / 2; + elementInfo = { + tagName: element.tagName, + id: element.id, + className: element.className, + text: element.textContent?.trim().substring(0, 100) || '', + href: element.href || null, + type: element.type || null, + isVisible: true, + rect: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + top: rect.top, + right: rect.right, + bottom: rect.bottom, + left: rect.left, + }, + clickMethod: 'ref', + ref, + }; + } else if ( + coordinates && + typeof coordinates.x === 'number' && + typeof coordinates.y === 'number' + ) { clickX = coordinates.x; clickY = coordinates.y; @@ -125,7 +175,10 @@ if (window.__CLICK_HELPER_INITIALIZED__) { }); } - if (element && elementInfo.clickMethod === 'selector') { + if ( + element && + (elementInfo.clickMethod === 'selector' || elementInfo.clickMethod === 'ref') + ) { element.click(); } else { simulateClick(clickX, clickY); @@ -217,6 +270,7 @@ if (window.__CLICK_HELPER_INITIALIZED__) { request.waitForNavigation, request.timeout, request.coordinates, + request.ref, ) .then(sendResponse) .catch((error) => { diff --git a/app/chrome-extension/inject-scripts/fill-helper.js b/app/chrome-extension/inject-scripts/fill-helper.js index 7c2ed85e..413273cd 100644 --- a/app/chrome-extension/inject-scripts/fill-helper.js +++ b/app/chrome-extension/inject-scripts/fill-helper.js @@ -12,13 +12,31 @@ if (window.__FILL_HELPER_INITIALIZED__) { * @param {string} value - Value to fill into the element * @returns {Promise} - Result of the fill operation */ - async function fillElement(selector, value) { + async function fillElement(selector, value, ref = null) { try { // Find the element - const element = document.querySelector(selector); + let element = null; + if (ref && typeof ref === 'string') { + try { + const map = window.__claudeElementMap; + const weak = map && map[ref]; + element = weak && typeof weak.deref === 'function' ? weak.deref() : null; + } catch (e) { + // ignore + } + if (!element || !(element instanceof Element)) { + return { + error: `Element ref "${ref}" not found. Please call chrome_read_page first and ensure the ref is still valid.`, + }; + } + } else { + element = document.querySelector(selector); + } if (!element) { return { - error: `Element with selector "${selector}" not found`, + error: selector + ? `Element with selector "${selector}" not found` + : `Element for ref not found`, }; } @@ -52,6 +70,7 @@ if (window.__FILL_HELPER_INITIALIZED__) { // Check if element is an input, textarea, or select const validTags = ['INPUT', 'TEXTAREA', 'SELECT']; + // Keep a permissive list to allow type-specific branches below to handle behavior const validInputTypes = [ 'text', 'email', @@ -66,6 +85,9 @@ if (window.__FILL_HELPER_INITIALIZED__) { 'time', 'week', 'color', + 'checkbox', + 'radio', + 'range', ]; if (!validTags.includes(element.tagName)) { @@ -75,7 +97,7 @@ if (window.__FILL_HELPER_INITIALIZED__) { }; } - // For input elements, check if the type is valid + // For input elements, check if the type is valid (allow type-specific branches below) if ( element.tagName === 'INPUT' && !validInputTypes.includes(element.type) && @@ -94,6 +116,92 @@ if (window.__FILL_HELPER_INITIALIZED__) { // Focus the element element.focus(); + // Type-specific handling for tricky inputs first + if (element.tagName === 'INPUT' && element.type === 'checkbox') { + // Accept boolean or string-like boolean + let checkedVal; + if (typeof value === 'boolean') { + checkedVal = value; + } else if (typeof value === 'string') { + const v = value.trim().toLowerCase(); + if (['true', '1', 'yes', 'on'].includes(v)) checkedVal = true; + else if (['false', '0', 'no', 'off'].includes(v)) checkedVal = false; + } + if (typeof checkedVal !== 'boolean') { + return { + error: + 'Checkbox requires a boolean (true/false) or a boolean-like string ("true"/"false"/"on"/"off").', + elementInfo, + }; + } + const previous = element.checked; + element.checked = checkedVal; + element.focus(); + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + element.blur(); + return { + success: true, + message: `Checkbox set to ${element.checked}`, + elementInfo: { ...elementInfo, checked: element.checked, previousChecked: previous }, + }; + } + + if (element.tagName === 'INPUT' && element.type === 'radio') { + // For radios, the selector/ref should target the specific input to select + const previous = element.checked; + element.checked = true; + element.focus(); + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + element.blur(); + return { + success: true, + message: 'Radio selected', + elementInfo: { + ...elementInfo, + checked: element.checked, + previousChecked: previous, + name: element.name || null, + }, + }; + } + + if (element.tagName === 'INPUT' && element.type === 'range') { + const numericValue = typeof value === 'number' ? value : Number(value); + if (Number.isNaN(numericValue)) { + return { error: 'Range input requires a numeric value', elementInfo }; + } + const previous = element.value; + element.value = String(numericValue); + element.focus(); + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + element.blur(); + return { + success: true, + message: `Set range to ${element.value} (min: ${element.min}, max: ${element.max})`, + elementInfo: { ...elementInfo, value: element.value }, + }; + } + + if (element.tagName === 'INPUT' && element.type === 'number') { + if (value !== '' && value !== null && value !== undefined && Number.isNaN(Number(value))) { + return { error: 'Number input requires a numeric value', elementInfo }; + } + const previous = element.value; + element.value = String(value ?? ''); + element.focus(); + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + element.blur(); + return { + success: true, + message: `Set number input to ${element.value} (previous: ${previous})`, + elementInfo: { ...elementInfo, value: element.value }, + }; + } + // Fill the element based on its type if (element.tagName === 'SELECT') { // For select elements, find the option with matching value or text @@ -117,15 +225,12 @@ if (window.__FILL_HELPER_INITIALIZED__) { element.dispatchEvent(new Event('change', { bubbles: true })); } else { // For input and textarea elements - - // Clear the current value + // Clear the current value then set new value element.value = ''; element.dispatchEvent(new Event('input', { bubbles: true })); - // Set the new value - element.value = value; + element.value = String(value); - // Trigger input and change events element.dispatchEvent(new Event('input', { bubbles: true })); element.dispatchEvent(new Event('change', { bubbles: true })); } @@ -189,7 +294,7 @@ if (window.__FILL_HELPER_INITIALIZED__) { // Listen for messages from the extension chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { if (request.action === 'fillElement') { - fillElement(request.selector, request.value) + fillElement(request.selector, request.value, request.ref) .then(sendResponse) .catch((error) => { sendResponse({ diff --git a/app/chrome-extension/inject-scripts/interactive-elements-helper.js b/app/chrome-extension/inject-scripts/interactive-elements-helper.js index cbb4e3cf..681b057a 100644 --- a/app/chrome-extension/inject-scripts/interactive-elements-helper.js +++ b/app/chrome-extension/inject-scripts/interactive-elements-helper.js @@ -34,17 +34,56 @@ 'input:not([type="button"]):not([type="submit"]):not([type="checkbox"]):not([type="radio"])', checkbox: 'input[type="checkbox"], [role="checkbox"]', radio: 'input[type="radio"], [role="radio"]', - textarea: 'textarea', - select: 'select', + textarea: 'textarea, [role="textbox"], [role="searchbox"]', + select: 'select, [role="combobox"]', tab: '[role="tab"]', // Generic interactive elements: combines tabindex, common roles, and explicit handlers. // This is the key to finding custom-built interactive components. - interactive: `[onclick], [tabindex]:not([tabindex^="-"]), [role="menuitem"], [role="slider"], [role="option"], [role="treeitem"]`, + interactive: `[onclick], [tabindex]:not([tabindex^="-"]), [role="menuitem"], [role="slider"], [role="option"], [role="treeitem"], [role="switch"]`, }; // A combined selector for ANY interactive element, used in the fallback logic. const ANY_INTERACTIVE_SELECTOR = Object.values(ELEMENT_CONFIG).join(', '); + // Query helpers that pierce open shadow roots. These are used only in fallback paths or + // when a selector is explicitly provided, to keep costs bounded. + function* walkAllNodesDeep(root) { + const stack = [root]; + const MAX = 12000; // safety bound + let count = 0; + while (stack.length) { + const node = stack.pop(); + if (!node) continue; + if (++count > MAX) break; + yield node; + const anyNode = /** @type {any} */ (node); + try { + const children = node.children ? Array.from(node.children) : []; + for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]); + const sr = anyNode && anyNode.shadowRoot ? anyNode.shadowRoot : null; + if (sr && sr.children) { + const srChildren = Array.from(sr.children); + for (let i = srChildren.length - 1; i >= 0; i--) stack.push(srChildren[i]); + } + } catch (_) { + /* ignore */ + } + } + } + + function querySelectorAllDeep(selector, root = document) { + const results = []; + for (const node of walkAllNodesDeep(root)) { + if (!(node instanceof Element)) continue; + try { + if (node.matches && node.matches(selector)) results.push(node); + } catch (_) { + /* ignore invalid selectors for given node */ + } + } + return results; + } + // --- Core Helper Functions --- /** @@ -221,7 +260,7 @@ .join(', '); if (!selectorsToFind) return []; - const targetElements = Array.from(document.querySelectorAll(selectorsToFind)); + const targetElements = querySelectorAllDeep(selectorsToFind); const uniqueElements = new Set(targetElements); const results = []; @@ -325,7 +364,7 @@ let elements; if (request.selector) { // If a selector is provided, bypass the text-based logic and use a direct query. - const foundEls = Array.from(document.querySelectorAll(request.selector)); + const foundEls = querySelectorAllDeep(request.selector); elements = foundEls.map((el) => createElementInfo( el, diff --git a/app/chrome-extension/utils/screenshot-context.ts b/app/chrome-extension/utils/screenshot-context.ts new file mode 100644 index 00000000..16ecc632 --- /dev/null +++ b/app/chrome-extension/utils/screenshot-context.ts @@ -0,0 +1,53 @@ +// Simple in-memory screenshot context manager per tab +// Used to scale coordinates from screenshot space to viewport space + +export interface ScreenshotContext { + // Final screenshot dimensions (in CSS pixels after any scaling) + screenshotWidth: number; + screenshotHeight: number; + // Viewport dimensions (CSS pixels) + viewportWidth: number; + viewportHeight: number; + // Device pixel ratio at capture time (optional, for reference) + devicePixelRatio?: number; + // Hostname of the page when the screenshot was taken (used for domain safety checks) + hostname?: string; + // Timestamp + timestamp: number; +} + +const TTL_MS = 5 * 60 * 1000; // 5 minutes + +const contexts = new Map(); + +export const screenshotContextManager = { + setContext(tabId: number, ctx: Omit) { + contexts.set(tabId, { ...ctx, timestamp: Date.now() }); + }, + getContext(tabId: number): ScreenshotContext | undefined { + const ctx = contexts.get(tabId); + if (!ctx) return undefined; + if (Date.now() - ctx.timestamp > TTL_MS) { + contexts.delete(tabId); + return undefined; + } + return ctx; + }, + clear(tabId: number) { + contexts.delete(tabId); + }, +}; + +// Scale screenshot-space coordinates (x,y) to viewport CSS pixels +export function scaleCoordinates( + x: number, + y: number, + ctx: ScreenshotContext, +): { x: number; y: number } { + if (!ctx.screenshotWidth || !ctx.screenshotHeight || !ctx.viewportWidth || !ctx.viewportHeight) { + return { x, y }; + } + const sx = (x / ctx.screenshotWidth) * ctx.viewportWidth; + const sy = (y / ctx.screenshotHeight) * ctx.viewportHeight; + return { x: Math.round(sx), y: Math.round(sy) }; +} diff --git a/app/chrome-extension/wxt.config.ts b/app/chrome-extension/wxt.config.ts index e18a89a6..dd0de345 100644 --- a/app/chrome-extension/wxt.config.ts +++ b/app/chrome-extension/wxt.config.ts @@ -66,6 +66,8 @@ export default defineConfig({ }, vite: (env) => ({ plugins: [ + // Ensure static assets are available as early as possible to avoid race conditions in dev + // Copy workers/_locales/inject-scripts into the build output before other steps viteStaticCopy({ targets: [ { @@ -81,6 +83,13 @@ export default defineConfig({ dest: '_locales', }, ], + // Copy at buildStart to avoid ENOENT when extension first boots in dev + hook: 'buildStart', + // Enable watch so changes to these files are reflected during dev + watch: { + // Use default patterns inferred from targets; explicit true enables watching + // Vite plugin will watch src patterns and re-copy on change + } as any, }) as any, ], build: { diff --git a/app/native-server/package.json b/app/native-server/package.json index 954dc512..6453f3c3 100644 --- a/app/native-server/package.json +++ b/app/native-server/package.json @@ -35,11 +35,13 @@ "dependencies": { "@fastify/cors": "^11.0.1", "@modelcontextprotocol/sdk": "^1.11.0", + "@types/node-fetch": "2", "chalk": "^5.4.1", "chrome-mcp-shared": "workspace:*", "commander": "^13.1.0", "fastify": "^5.3.2", "is-admin": "^4.0.0", + "node-fetch": "2", "pino": "^9.6.0", "uuid": "^11.1.0" }, diff --git a/docs/TOOLS.md b/docs/TOOLS.md index a3b4f63e..be045831 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -247,6 +247,24 @@ Send custom HTTP requests. ## 🔍 Content Analysis +### `chrome_read_page` + +Build an accessibility-like tree of the current page (visible viewport by default) with stable `ref_*` identifiers and viewport info. Useful for semantic element discovery or agent planning. + +Parameters: + +- `filter` (string, optional): `interactive` to only include interactive elements; default includes structural and labeled nodes. + +Example: + +```json +{ + "filter": "interactive" +} +``` + +Response contains `pageContent` (text tree), `viewport`, and a `refMapCount` summary. Use `chrome_get_interactive_elements` or your own logic to act on returned refs. + ### `search_tabs_content` AI-powered semantic search across browser tabs. @@ -308,46 +326,70 @@ Extract HTML or text content from web pages. } ``` -### `chrome_get_interactive_elements` +### `chrome_get_interactive_elements` (deprecated) -Find clickable and interactive elements on the page. +Replaced by `chrome_read_page` as the primary discovery tool. The `read_page` implementation will automatically fallback to the interactive-elements logic when the accessibility tree is unavailable or too sparse. This tool is no longer listed via ListTools and is kept only for backward compatibility. -**Parameters**: +## 🎯 Interaction -- `tabId` (number, optional): Specific tab ID (default: active tab) +### `chrome_computer` -**Response**: +Unified advanced interaction tool that prioritizes high-level DOM actions with CDP fallback. Supports hover, click, drag, scroll, typing, key chords, fill, wait and screenshot. If a recent screenshot was taken via `chrome_screenshot`, coordinates are auto-scaled from screenshot space to viewport space. + +Parameters: + +- `action` (string, required): `left_click` | `right_click` | `double_click` | `triple_click` | `left_click_drag` | `scroll` | `type` | `key` | `fill` | `hover` | `wait` | `screenshot` +- `ref` (string, optional): element ref from `chrome_read_page` (preferred). Used for click/scroll/type/key and as drag end when provided +- `coordinates` (object, optional): `{ "x": 100, "y": 200 }` for click/scroll or drag end +- `startRef` (string, optional): element ref for drag start +- `startCoordinates` (object, optional): for `left_click_drag` when no `startRef` +- `scrollDirection` (string, optional): `up` | `down` | `left` | `right` +- `scrollAmount` (number, optional): ticks 1–10 (default 3) +- `text` (string, optional): for `type` (raw text) or `key` (space-separated chords/keys like `"cmd+a Enter"`) +- `duration` (number, optional): seconds for `wait` (max 30) +- `selector` (string, optional): for `fill` when no `ref` +- `value` (string, optional): for `fill` value + +Examples: ```json -{ - "elements": [ - { - "selector": "#submit-button", - "type": "button", - "text": "Submit", - "visible": true, - "clickable": true - } - ] -} +{ "action": "left_click", "coordinates": { "x": 420, "y": 260 } } ``` -## 🎯 Interaction +```json +{ "action": "key", "text": "cmd+a Backspace" } +``` + +````json +{ "action": "fill", "ref": "ref_7", "value": "user@example.com" } + +```json +{ "action": "hover", "ref": "ref_12", "duration": 0.6 } +```` + +```` + +```json +{ "action": "left_click_drag", "startRef": "ref_10", "ref": "ref_15" } +```` ### `chrome_click_element` -Click elements using CSS selectors. +Click elements using a ref, selector, or coordinates. **Parameters**: -- `selector` (string, required): CSS selector for target element -- `tabId` (number, optional): Specific tab ID (default: active tab) +- `ref` (string, optional): Element ref from `chrome_read_page` (preferred when available) +- `selector` (string, optional): CSS selector for target element +- `coordinates` (object, optional): `{ "x": 120, "y": 240 }` viewport coordinates + +At least one of `ref`, `selector`, or `coordinates` must be provided. **Example**: ```json { - "selector": "#submit-button" + "ref": "ref_42" } ``` @@ -357,15 +399,17 @@ Fill form fields or select options. **Parameters**: -- `selector` (string, required): CSS selector for target element +- `ref` (string, optional): Element ref from `chrome_read_page` +- `selector` (string, optional): CSS selector for target element - `value` (string, required): Value to fill or select -- `tabId` (number, optional): Specific tab ID (default: active tab) + +Provide `ref` or `selector` to identify the element. **Example**: ```json { - "selector": "#email-input", + "ref": "ref_7", "value": "user@example.com" } ``` diff --git a/packages/shared/src/tools.ts b/packages/shared/src/tools.ts index 9180aa02..068e15b9 100644 --- a/packages/shared/src/tools.ts +++ b/packages/shared/src/tools.ts @@ -27,6 +27,8 @@ export const TOOL_NAMES = { SEND_COMMAND_TO_INJECT_SCRIPT: 'chrome_send_command_to_inject_script', CONSOLE: 'chrome_console', FILE_UPLOAD: 'chrome_upload_file', + READ_PAGE: 'chrome_read_page', + COMPUTER: 'chrome_computer', }, }; @@ -40,6 +42,90 @@ export const TOOL_SCHEMAS: Tool[] = [ required: [], }, }, + { + name: TOOL_NAMES.BROWSER.READ_PAGE, + description: + 'Get an accessibility tree representation of visible elements on the page. Only returns elements that are visible in the viewport. Optionally filter for only interactive elements.\nTip: If the returned elements do not include the specific element you need, use the computer tool\'s screenshot (action="screenshot") to capture the element\'s on-screen coordinates, then operate by coordinates.', + inputSchema: { + type: 'object', + properties: { + filter: { + type: 'string', + description: + 'Filter elements: "interactive" for such as buttons/links/inputs only (default: all visible elements)', + }, + }, + required: [], + }, + }, + { + name: TOOL_NAMES.BROWSER.COMPUTER, + description: + "Use a mouse and keyboard to interact with a web browser, and take screenshots.\n* Whenever you intend to click on an element like an icon, you should consult a read_page to determine the ref of the element before moving the cursor.\n* If you tried clicking on a program or link but it failed to load, even after waiting, try screenshot and then adjusting your click location so that the tip of the cursor visually falls on the element that you want to click.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.", + inputSchema: { + type: 'object', + properties: { + action: { + type: 'string', + description: + 'Action to perform: left_click | right_click | double_click | triple_click | left_click_drag | scroll | type | key | fill | hover | wait | screenshot', + }, + ref: { + type: 'string', + description: + 'Element ref from chrome_read_page. For click/scroll/key/type and drag end when provided; takes precedence over coordinates.', + }, + coordinates: { + type: 'object', + properties: { + x: { type: 'number', description: 'X coordinate' }, + y: { type: 'number', description: 'Y coordinate' }, + }, + description: + 'Coordinates for actions (in screenshot space if a recent screenshot was taken, otherwise viewport). Required for click/scroll and as end point for drag.', + }, + startCoordinates: { + type: 'object', + properties: { + x: { type: 'number' }, + y: { type: 'number' }, + }, + description: 'Starting coordinates for drag action', + }, + startRef: { + type: 'string', + description: 'Drag start ref from chrome_read_page (alternative to startCoordinates).', + }, + scrollDirection: { + type: 'string', + description: 'Scroll direction: up | down | left | right', + }, + scrollAmount: { + type: 'number', + description: 'Scroll ticks (1-10), default 3', + }, + text: { + type: 'string', + description: + 'Text to type (for action=type) or keys/chords separated by space (for action=key, e.g. "Backspace Enter" or "cmd+a")', + }, + // For action=fill + selector: { + type: 'string', + description: 'CSS selector for fill (alternative to ref).', + }, + value: { + oneOf: [{ type: 'string' }, { type: 'boolean' }, { type: 'number' }], + description: 'Value to set for action=fill (string | boolean | number)', + }, + duration: { + type: 'number', + description: 'Seconds to wait for action=wait (max 30s)', + }, + }, + required: ['action'], + }, + }, { name: TOOL_NAMES.BROWSER.NAVIGATE, description: 'Navigate to a URL or refresh the current tab', @@ -65,7 +151,7 @@ export const TOOL_SCHEMAS: Tool[] = [ { name: TOOL_NAMES.BROWSER.SCREENSHOT, description: - 'Take a screenshot of the current page or a specific element(if you want to see the page, recommend to use chrome_get_web_content first)', + '[Prefer read_page over taking a screenshot and Prefer chrome_computer] Take a screenshot of the current page or a specific element. For new usage, use chrome_computer with action="screenshot". Use this tool if you need advanced options.', inputSchema: { type: 'object', properties: { @@ -172,87 +258,6 @@ export const TOOL_SCHEMAS: Tool[] = [ required: [], }, }, - { - name: TOOL_NAMES.BROWSER.CLICK, - description: 'Click on an element in the current page or at specific coordinates', - inputSchema: { - type: 'object', - properties: { - selector: { - type: 'string', - description: - 'CSS selector for the element to click. Either selector or coordinates must be provided. if coordinates are not provided, the selector must be provided.', - }, - coordinates: { - type: 'object', - description: - 'Coordinates to click at (relative to viewport). If provided, takes precedence over selector.', - properties: { - x: { - type: 'number', - description: 'X coordinate relative to the viewport', - }, - y: { - type: 'number', - description: 'Y coordinate relative to the viewport', - }, - }, - required: ['x', 'y'], - }, - waitForNavigation: { - type: 'boolean', - description: 'Wait for page navigation to complete after click (default: false)', - }, - timeout: { - type: 'number', - description: - 'Timeout in milliseconds for waiting for the element or navigation (default: 5000)', - }, - }, - required: [], - }, - }, - { - name: TOOL_NAMES.BROWSER.FILL, - description: 'Fill a form element or select an option with the specified value', - inputSchema: { - type: 'object', - properties: { - selector: { - type: 'string', - description: 'CSS selector for the input element to fill or select', - }, - value: { - type: 'string', - description: 'Value to fill or select into the element', - }, - }, - required: ['selector', 'value'], - }, - }, - { - name: TOOL_NAMES.BROWSER.GET_INTERACTIVE_ELEMENTS, - description: 'Get interactive elements from the current page', - inputSchema: { - type: 'object', - properties: { - textQuery: { - type: 'string', - description: 'Text to search for within interactive elements (fuzzy search)', - }, - selector: { - type: 'string', - description: - 'CSS selector to filter interactive elements. Takes precedence over textQuery if both are provided.', - }, - includeCoordinates: { - type: 'boolean', - description: 'Include element coordinates in the response (default: true)', - }, - }, - required: [], - }, - }, { name: TOOL_NAMES.BROWSER.NETWORK_REQUEST, description: 'Send a network request from the browser with cookies and other browser context', @@ -335,29 +340,6 @@ export const TOOL_SCHEMAS: Tool[] = [ required: [], }, }, - { - name: TOOL_NAMES.BROWSER.KEYBOARD, - description: 'Simulate keyboard events in the browser', - inputSchema: { - type: 'object', - properties: { - keys: { - type: 'string', - description: 'Keys to simulate (e.g., "Enter", "Ctrl+C", "A,B,C" for sequence)', - }, - selector: { - type: 'string', - description: - 'CSS selector for the element to send keyboard events to (optional, defaults to active element)', - }, - delay: { - type: 'number', - description: 'Delay between key sequences in milliseconds (optional, default: 0)', - }, - }, - required: ['keys'], - }, - }, { name: TOOL_NAMES.BROWSER.HISTORY, description: 'Retrieve and search browsing history from Chrome', @@ -556,7 +538,8 @@ export const TOOL_SCHEMAS: Tool[] = [ }, { name: TOOL_NAMES.BROWSER.FILE_UPLOAD, - description: 'Upload files to web forms with file input elements using Chrome DevTools Protocol', + description: + 'Upload files to web forms with file input elements using Chrome DevTools Protocol', inputSchema: { type: 'object', properties: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c4a1ceb..5ef98e35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -105,6 +105,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.11.0 version: 1.12.1 + '@types/node-fetch': + specifier: '2' + version: 2.6.13 chalk: specifier: ^5.4.1 version: 5.4.1 @@ -120,6 +123,9 @@ importers: is-admin: specifier: ^4.0.0 version: 4.0.0 + node-fetch: + specifier: '2' + version: 2.7.0 pino: specifier: ^9.6.0 version: 9.7.0 @@ -906,67 +912,56 @@ packages: resolution: {integrity: sha512-UsQD5fyLWm2Fe5CDM7VPYAo+UC7+2Px4Y+N3AcPh/LdZu23YcuGPegQly++XEVaC8XUTFVPscl5y5Cl1twEI4A==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.42.0': resolution: {integrity: sha512-/i8NIrlgc/+4n1lnoWl1zgH7Uo0XK5xK3EDqVTf38KvyYgCU/Rm04+o1VvvzJZnVS5/cWSd07owkzcVasgfIkQ==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.42.0': resolution: {integrity: sha512-eoujJFOvoIBjZEi9hJnXAbWg+Vo1Ov8n/0IKZZcPZ7JhBzxh2A+2NFyeMZIRkY9iwBvSjloKgcvnjTbGKHE44Q==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.42.0': resolution: {integrity: sha512-/3NrcOWFSR7RQUQIuZQChLND36aTU9IYE4j+TB40VU78S+RA0IiqHR30oSh6P1S9f9/wVOenHQnacs/Byb824g==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.42.0': resolution: {integrity: sha512-O8AplvIeavK5ABmZlKBq9/STdZlnQo7Sle0LLhVA7QT+CiGpNVe197/t8Aph9bhJqbDVGCHpY2i7QyfEDDStDg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.42.0': resolution: {integrity: sha512-6Qb66tbKVN7VyQrekhEzbHRxXXFFD8QKiFAwX5v9Xt6FiJ3BnCVBuyBxa2fkFGqxOCSGGYNejxd8ht+q5SnmtA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.42.0': resolution: {integrity: sha512-KQETDSEBamQFvg/d8jajtRwLNBlGc3aKpaGiP/LvEbnmVUKlFta1vqJqTrvPtsYsfbE/DLg5CC9zyXRX3fnBiA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.42.0': resolution: {integrity: sha512-qMvnyjcU37sCo/tuC+JqeDKSuukGAd+pVlRl/oyDbkvPJ3awk6G6ua7tyum02O3lI+fio+eM5wsVd66X0jQtxw==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.42.0': resolution: {integrity: sha512-I2Y1ZUgTgU2RLddUHXTIgyrdOwljjkmcZ/VilvaEumtS3Fkuhbw4p4hgHc39Ypwvo2o7sBFNl2MquNvGCa55Iw==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.42.0': resolution: {integrity: sha512-Gfm6cV6mj3hCUY8TqWa63DB8Mx3NADoFwiJrMpoZ1uESbK8FQV3LXkhfry+8bOniq9pqY1OdsjFWNsSbfjPugw==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.42.0': resolution: {integrity: sha512-g86PF8YZ9GRqkdi0VoGlcDUb4rYtQKyTD1IVtxxN4Hpe7YqLBShA7oHMKU6oKTCi3uxwW4VkIGnOaH/El8de3w==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.42.0': resolution: {integrity: sha512-+axkdyDGSp6hjyzQ5m1pgcvQScfHnMCcsXkx8pTgy/6qBmWVhtRVlgxjWwDp67wEXXUr0x+vD6tp5W4x6V7u1A==} @@ -1067,6 +1062,9 @@ packages: '@types/minimatch@3.0.5': resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==} + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + '@types/node@18.19.111': resolution: {integrity: sha512-90sGdgA+QLJr1F9X79tQuEut0gEYIfkX9pydI4XGRgvFo9g2JWswefI+WUSUHPYVBHYSEfTEqBxA5hQvAZB3Mw==} @@ -2302,6 +2300,10 @@ packages: resolution: {integrity: sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==} engines: {node: '>= 6'} + form-data@4.0.4: + resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} + engines: {node: '>= 6'} + formdata-node@6.0.3: resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==} engines: {node: '>= 18'} @@ -3259,6 +3261,15 @@ packages: node-fetch-native@1.6.6: resolution: {integrity: sha512-8Mc2HhqPdlIfedsuZoc3yioPuzp6b+L5jRCRY1QzuWZh2EGJVQrGppC6V6cF0bLdbW0+O2YpqCA25aF/1lvipQ==} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-forge@1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} @@ -3927,6 +3938,7 @@ packages: source-map@0.8.0-beta.0: resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==} engines: {node: '>= 8'} + deprecated: The work that was done in this beta branch won't be included in future versions spawn-sync@1.0.15: resolution: {integrity: sha512-9DWBgrgYZzNghseho0JOuh+5fg9u6QWhAWa51QC7+U5rCheZ/j1DrEZnyE0RBBRqZ9uEXGPgSSM0nky6burpVw==} @@ -4135,6 +4147,9 @@ packages: resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==} hasBin: true + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tr46@1.0.1: resolution: {integrity: sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==} @@ -4419,12 +4434,18 @@ packages: resolution: {integrity: sha512-u/IiZaZ7dHFqTM1MLF27rBy8mS9fEEsqoOKL0u+kQdOLmEioA/0Szp67ADd3WAJZLd8/hO8cFST1IC/YMXKIjQ==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + webidl-conversions@4.0.2: resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==} webpack-virtual-modules@0.6.2: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + whatwg-url@7.1.0: resolution: {integrity: sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==} @@ -5491,6 +5512,11 @@ snapshots: '@types/minimatch@3.0.5': {} + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 22.15.30 + form-data: 4.0.4 + '@types/node@18.19.111': dependencies: undici-types: 5.26.5 @@ -6913,6 +6939,14 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + form-data@4.0.4: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + formdata-node@6.0.3: {} formidable@3.5.4: @@ -7960,6 +7994,10 @@ snapshots: node-fetch-native@1.6.6: {} + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-forge@1.3.1: {} node-int64@0.4.0: {} @@ -8965,6 +9003,8 @@ snapshots: touch@3.1.1: {} + tr46@0.0.3: {} + tr46@1.0.1: dependencies: punycode: 2.3.1 @@ -9282,10 +9322,17 @@ snapshots: - supports-color - utf-8-validate + webidl-conversions@3.0.1: {} + webidl-conversions@4.0.2: {} webpack-virtual-modules@0.6.2: {} + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + whatwg-url@7.1.0: dependencies: lodash.sortby: 4.7.0 From d4e2fb3c8de7784f53a44ffe5125f08782a40d1f Mon Sep 17 00:00:00 2001 From: hangerye Date: Thu, 9 Oct 2025 10:00:36 +0800 Subject: [PATCH 02/71] feat: commit for store --- app/chrome-extension/common/message-types.ts | 4 + .../background/tools/base-browser.ts | 3 +- .../background/tools/browser/computer.ts | 179 ++++++++++++++++-- .../background/tools/browser/dialog.ts | 77 ++++++++ .../background/tools/browser/index.ts | 1 + .../inject-scripts/wait-helper.js | 171 +++++++++++++++++ packages/shared/src/tools.ts | 40 ++++ 7 files changed, 461 insertions(+), 14 deletions(-) create mode 100644 app/chrome-extension/entrypoints/background/tools/browser/dialog.ts create mode 100644 app/chrome-extension/inject-scripts/wait-helper.js diff --git a/app/chrome-extension/common/message-types.ts b/app/chrome-extension/common/message-types.ts index 673c3df1..38d3097d 100644 --- a/app/chrome-extension/common/message-types.ts +++ b/app/chrome-extension/common/message-types.ts @@ -42,6 +42,7 @@ export const CONTENT_MESSAGE_TYPES = { SCREENSHOT_HELPER_PING: 'screenshot_helper_ping', INTERACTIVE_ELEMENTS_HELPER_PING: 'interactive_elements_helper_ping', ACCESSIBILITY_TREE_HELPER_PING: 'chrome_read_page_ping', + WAIT_HELPER_PING: 'wait_helper_ping', } as const; // Tool action message types (for chrome.runtime.sendMessage) @@ -73,6 +74,9 @@ export const TOOL_MESSAGE_TYPES = { // Network requests NETWORK_SEND_REQUEST: 'sendPureNetworkRequest', + // Wait helper + WAIT_FOR_TEXT: 'waitForText', + // Semantic similarity engine SIMILARITY_ENGINE_INIT: 'similarityEngineInit', SIMILARITY_ENGINE_COMPUTE_BATCH: 'similarityEngineComputeBatch', diff --git a/app/chrome-extension/entrypoints/background/tools/base-browser.ts b/app/chrome-extension/entrypoints/background/tools/base-browser.ts index bb77b97b..b17bd6ec 100644 --- a/app/chrome-extension/entrypoints/background/tools/base-browser.ts +++ b/app/chrome-extension/entrypoints/background/tools/base-browser.ts @@ -19,6 +19,7 @@ export abstract class BaseBrowserToolExecutor implements ToolExecutor { files: string[], injectImmediately = false, world: 'MAIN' | 'ISOLATED' = 'ISOLATED', + allFrames: boolean = false, ): Promise { console.log(`Injecting ${files.join(', ')} into tab ${tabId}`); @@ -50,7 +51,7 @@ export abstract class BaseBrowserToolExecutor implements ToolExecutor { try { await chrome.scripting.executeScript({ - target: { tabId }, + target: { tabId, allFrames }, files, injectImmediately, world, diff --git a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts index fcbd8972..17d2c66f 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts @@ -28,6 +28,8 @@ interface ComputerParams { | 'hover' | 'wait' | 'fill' + | 'fill_form' + | 'resize_page' | 'screenshot'; // click/scroll coordinates in screenshot space (if screenshot context exists) or viewport space coordinates?: Coordinates; // for click/scroll; for drag, this is endCoordinates @@ -243,6 +245,58 @@ class ComputerTool extends BaseBrowserToolExecutor { }; switch (params.action) { + case 'resize_page': { + const width = Number((params as any).coordinates?.x || (params as any).text); + const height = Number((params as any).coordinates?.y || (params as any).value); + const w = Number((params as any).width ?? width); + const h = Number((params as any).height ?? height); + if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) { + return createErrorResponse( + 'Provide width and height for resize_page (positive numbers)', + ); + } + try { + // Prefer precise CDP emulation + await CDPHelper.attach(tab.id); + try { + await chrome.debugger.sendCommand( + { tabId: tab.id }, + 'Emulation.setDeviceMetricsOverride', + { + width: Math.round(w), + height: Math.round(h), + deviceScaleFactor: 0, + mobile: false, + screenWidth: Math.round(w), + screenHeight: Math.round(h), + }, + ); + } finally { + await CDPHelper.detach(tab.id); + } + } catch (e) { + // Fallback: window resize + if (tab.windowId !== undefined) { + await chrome.windows.update(tab.windowId, { + width: Math.round(w), + height: Math.round(h), + }); + } else { + return createErrorResponse( + `Failed to resize via CDP and cannot determine windowId: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'resize_page', width: w, height: h }), + }, + ], + isError: false, + }; + } case 'hover': { // Resolve target point from ref | selector | coordinates let coord: Coordinates | undefined = undefined; @@ -782,6 +836,52 @@ class ComputerTool extends BaseBrowserToolExecutor { } as any); return res; } + case 'fill_form': { + const elements = (params as any).elements as Array<{ + ref: string; + value: string | number | boolean; + }>; + if (!Array.isArray(elements) || elements.length === 0) { + return createErrorResponse('elements must be a non-empty array for fill_form'); + } + const results: Array<{ ref: string; ok: boolean; error?: string }> = []; + for (const item of elements) { + if (!item || !item.ref) { + results.push({ ref: String(item?.ref || ''), ok: false, error: 'missing ref' }); + continue; + } + try { + const r = await fillTool.execute({ + ref: item.ref as any, + value: item.value as any, + } as any); + const ok = !r.isError; + results.push({ ref: item.ref, ok, error: ok ? undefined : 'failed' }); + } catch (e) { + results.push({ + ref: item.ref, + ok: false, + error: String(e instanceof Error ? e.message : e), + }); + } + } + const successCount = results.filter((r) => r.ok).length; + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'fill_form', + filled: successCount, + total: results.length, + results, + }), + }, + ], + isError: false, + }; + } case 'key': { if (!params.text) return createErrorResponse( @@ -821,19 +921,72 @@ class ComputerTool extends BaseBrowserToolExecutor { } } case 'wait': { - const seconds = Math.max(0, Math.min(params.duration || 0, 30)); - if (!seconds) - return createErrorResponse('Duration parameter is required and must be > 0'); - await new Promise((r) => setTimeout(r, seconds * 1000)); - return { - content: [ - { - type: 'text', - text: JSON.stringify({ success: true, action: 'wait', duration: seconds }), - }, - ], - isError: false, - }; + const hasTextCondition = + typeof (params as any).text === 'string' && (params as any).text.trim().length > 0; + if (hasTextCondition) { + try { + // Conditional wait for text appearance/disappearance using content script + await this.injectContentScript( + tab.id, + ['inject-scripts/wait-helper.js'], + false, + 'ISOLATED', + true, + ); + const appear = (params as any).appear !== false; // default to true + const timeoutMs = Math.max( + 0, + Math.min(((params as any).timeout as number) || 10000, 120000), + ); + const resp = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.WAIT_FOR_TEXT, + text: (params as any).text, + appear, + timeout: timeoutMs, + }); + if (!resp || resp.success !== true) { + return createErrorResponse( + resp && resp.reason === 'timeout' + ? `wait_for timed out after ${timeoutMs}ms for text: ${(params as any).text}` + : `wait_for failed: ${resp && resp.error ? resp.error : 'unknown error'}`, + ); + } + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'wait_for', + appear, + text: (params as any).text, + matched: resp.matched || null, + tookMs: resp.tookMs, + }), + }, + ], + isError: false, + }; + } catch (e) { + return createErrorResponse( + `wait_for failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } else { + const seconds = Math.max(0, Math.min((params as any).duration || 0, 30)); + if (!seconds) + return createErrorResponse('Duration parameter is required and must be > 0'); + await new Promise((r) => setTimeout(r, seconds * 1000)); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'wait', duration: seconds }), + }, + ], + isError: false, + }; + } } case 'screenshot': { // Reuse existing screenshot tool; it already supports base64 save option diff --git a/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts b/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts new file mode 100644 index 00000000..274562aa --- /dev/null +++ b/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts @@ -0,0 +1,77 @@ +import { createErrorResponse, ToolResult } from '@/common/tool-handler'; +import { BaseBrowserToolExecutor } from '../base-browser'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; + +interface HandleDialogParams { + action: 'accept' | 'dismiss'; + promptText?: string; +} + +/** + * Handle JavaScript dialogs (alert/confirm/prompt) via CDP Page.handleJavaScriptDialog + */ +class HandleDialogTool extends BaseBrowserToolExecutor { + name = TOOL_NAMES.BROWSER.HANDLE_DIALOG; + + async execute(args: HandleDialogParams): Promise { + const { action, promptText } = args || ({} as HandleDialogParams); + if (!action || (action !== 'accept' && action !== 'dismiss')) { + return createErrorResponse('action must be "accept" or "dismiss"'); + } + + try { + const [activeTab] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (!activeTab?.id) return createErrorResponse('No active tab found'); + + // Attach debugger and try handling the dialog + try { + await chrome.debugger.attach({ tabId: activeTab.id }, '1.3'); + } catch (e: any) { + if (String(e?.message || '').includes('attached')) { + // If already attached by us, proceed; otherwise fail with clear message + const targets = await chrome.debugger.getTargets(); + const existing = targets.find((t) => t.tabId === activeTab.id && t.attached); + if (!existing || existing.extensionId !== chrome.runtime.id) { + return createErrorResponse( + `Debugger already attached to tab ${activeTab.id} by another client (e.g., DevTools). Close it and retry.`, + ); + } + } else { + throw e; + } + } + + try { + // Enable Page domain to be safe + await chrome.debugger.sendCommand({ tabId: activeTab.id }, 'Page.enable'); + await chrome.debugger.sendCommand({ tabId: activeTab.id }, 'Page.handleJavaScriptDialog', { + accept: action === 'accept', + promptText: action === 'accept' ? promptText : undefined, + }); + } finally { + // Best-effort detach if we were the owners + try { + await chrome.debugger.detach({ tabId: activeTab.id }); + } catch { + // ignore + } + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action, promptText: promptText || null }), + }, + ], + isError: false, + }; + } catch (error) { + return createErrorResponse( + `Failed to handle dialog: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} + +export const handleDialogTool = new HandleDialogTool(); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/index.ts b/app/chrome-extension/entrypoints/background/tools/browser/index.ts index f4bcc2ba..d473e08c 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/index.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/index.ts @@ -15,3 +15,4 @@ export { consoleTool } from './console'; export { fileUploadTool } from './file-upload'; export { readPageTool } from './read-page'; export { computerTool } from './computer'; +export { handleDialogTool } from './dialog'; diff --git a/app/chrome-extension/inject-scripts/wait-helper.js b/app/chrome-extension/inject-scripts/wait-helper.js new file mode 100644 index 00000000..d97dde51 --- /dev/null +++ b/app/chrome-extension/inject-scripts/wait-helper.js @@ -0,0 +1,171 @@ +/* eslint-disable */ +// wait-helper.js +// Listen for text appearance/disappearance in the current document using MutationObserver. +// Returns a stable ref (compatible with accessibility-tree-helper) for the first matching element. + +(function () { + if (window.__WAIT_HELPER_INITIALIZED__) return; + window.__WAIT_HELPER_INITIALIZED__ = true; + + // Ensure ref mapping infra exists (compatible with accessibility-tree-helper.js) + if (!window.__claudeElementMap) window.__claudeElementMap = {}; + if (!window.__claudeRefCounter) window.__claudeRefCounter = 0; + + function isVisible(el) { + try { + if (!(el instanceof Element)) return false; + const style = getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') + return false; + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return false; + return true; + } catch { + return false; + } + } + + function normalize(str) { + return String(str || '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + } + + function matchesText(el, needle) { + const t = normalize(needle); + if (!t) return false; + try { + if (!isVisible(el)) return false; + const aria = el.getAttribute('aria-label'); + if (aria && normalize(aria).includes(t)) return true; + const title = el.getAttribute('title'); + if (title && normalize(title).includes(t)) return true; + const alt = el.getAttribute('alt'); + if (alt && normalize(alt).includes(t)) return true; + const placeholder = el.getAttribute('placeholder'); + if (placeholder && normalize(placeholder).includes(t)) return true; + // input/textarea value + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + const value = el.value || el.getAttribute('value'); + if (value && normalize(value).includes(t)) return true; + } + const text = el.innerText || el.textContent || ''; + if (normalize(text).includes(t)) return true; + } catch {} + return false; + } + + function findElementByText(text) { + // Fast path: query common interactive elements first + const prioritized = Array.from( + document.querySelectorAll('a,button,input,textarea,select,label,summary,[role]'), + ); + for (const el of prioritized) if (matchesText(el, text)) return el; + + // Fallback: broader scan with cap to avoid blocking on huge pages + const walker = document.createTreeWalker( + document.body || document.documentElement, + NodeFilter.SHOW_ELEMENT, + ); + let count = 0; + while (walker.nextNode()) { + const el = /** @type {Element} */ (walker.currentNode); + if (matchesText(el, text)) return el; + if (++count > 5000) break; // Hard cap to avoid long scans + } + return null; + } + + function ensureRefForElement(el) { + // Try to reuse an existing ref + for (const k in window.__claudeElementMap) { + const weak = window.__claudeElementMap[k]; + if (weak && typeof weak.deref === 'function' && weak.deref() === el) return k; + } + const refId = `ref_${++window.__claudeRefCounter}`; + window.__claudeElementMap[refId] = new WeakRef(el); + return refId; + } + + function centerOf(el) { + const r = el.getBoundingClientRect(); + return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) }; + } + + function waitFor({ text, appear = true, timeout = 5000 }) { + return new Promise((resolve) => { + const start = Date.now(); + let resolved = false; + + const check = () => { + try { + const match = findElementByText(text); + if (appear) { + if (match) { + const ref = ensureRefForElement(match); + const center = centerOf(match); + done({ success: true, matched: { ref, center }, tookMs: Date.now() - start }); + } + } else { + // wait for disappearance + if (!match) { + done({ success: true, matched: null, tookMs: Date.now() - start }); + } + } + } catch {} + }; + + const done = (result) => { + if (resolved) return; + resolved = true; + obs && obs.disconnect(); + clearTimeout(timer); + resolve(result); + }; + + const obs = new MutationObserver(() => check()); + try { + obs.observe(document.documentElement || document.body, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + }); + } catch {} + + // Initial check + check(); + const timer = setTimeout( + () => { + done({ success: false, reason: 'timeout', tookMs: Date.now() - start }); + }, + Math.max(0, timeout), + ); + }); + } + + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { + try { + if (request && request.action === 'wait_helper_ping') { + sendResponse({ status: 'pong' }); + return false; + } + if (request && request.action === 'waitForText') { + const text = String(request.text || '').trim(); + const appear = request.appear !== false; // default true + const timeout = Number(request.timeout || 5000); + if (!text) { + sendResponse({ success: false, error: 'text is required' }); + return true; + } + waitFor({ text, appear, timeout }).then((res) => sendResponse(res)); + return true; // async + } + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + return false; + }); +})(); diff --git a/packages/shared/src/tools.ts b/packages/shared/src/tools.ts index 068e15b9..7bd1b421 100644 --- a/packages/shared/src/tools.ts +++ b/packages/shared/src/tools.ts @@ -29,6 +29,7 @@ export const TOOL_NAMES = { FILE_UPLOAD: 'chrome_upload_file', READ_PAGE: 'chrome_read_page', COMPUTER: 'chrome_computer', + HANDLE_DIALOG: 'chrome_handle_dialog', }, }; @@ -118,6 +119,30 @@ export const TOOL_SCHEMAS: Tool[] = [ oneOf: [{ type: 'string' }, { type: 'boolean' }, { type: 'number' }], description: 'Value to set for action=fill (string | boolean | number)', }, + elements: { + type: 'array', + description: 'For action=fill_form: list of elements to fill (ref + value)', + items: { + type: 'object', + properties: { + ref: { type: 'string', description: 'Element ref from chrome_read_page' }, + value: { type: 'string', description: 'Value to set (stringified if non-string)' }, + }, + required: ['ref', 'value'], + }, + }, + width: { type: 'number', description: 'For action=resize_page: viewport width' }, + height: { type: 'number', description: 'For action=resize_page: viewport height' }, + appear: { + type: 'boolean', + description: + 'For action=wait with text: whether to wait for the text to appear (true, default) or disappear (false)', + }, + timeout: { + type: 'number', + description: + 'For action=wait with text: timeout in milliseconds (default 10000, max 120000)', + }, duration: { type: 'number', description: 'Seconds to wait for action=wait (max 30s)', @@ -571,4 +596,19 @@ export const TOOL_SCHEMAS: Tool[] = [ required: ['selector'], }, }, + { + name: TOOL_NAMES.BROWSER.HANDLE_DIALOG, + description: 'Handle JavaScript dialogs (alert/confirm/prompt) via CDP', + inputSchema: { + type: 'object', + properties: { + action: { type: 'string', description: 'accept | dismiss' }, + promptText: { + type: 'string', + description: 'Optional prompt text when accepting a prompt', + }, + }, + required: ['action'], + }, + }, ]; From 9fca59a6745632cbc9b469c998976ea3bd562ab2 Mon Sep 17 00:00:00 2001 From: hangwin Date: Thu, 9 Oct 2025 04:11:50 +0000 Subject: [PATCH 03/71] feat(extension): shared CDP session manager; refactor dialog/computer/network tools - Add utils/cdp-session-manager with refcount + owner tracking - Migrate dialog.ts to use session manager - Migrate computer.ts to use session manager - Migrate network-capture-debugger.ts to use session manager - Lint + vue-tsc clean; wxt build passes --- .../background/tools/browser/computer.ts | 60 ++-------- .../background/tools/browser/dialog.ts | 37 ++---- .../tools/browser/network-capture-debugger.ts | 91 +++++---------- .../utils/cdp-session-manager.ts | 109 ++++++++++++++++++ 4 files changed, 155 insertions(+), 142 deletions(-) create mode 100644 app/chrome-extension/utils/cdp-session-manager.ts diff --git a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts index 17d2c66f..cc3d6447 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts @@ -7,6 +7,7 @@ import { clickTool, fillTool } from './interaction'; import { keyboardTool } from './keyboard'; import { screenshotTool } from './screenshot'; import { screenshotContextManager, scaleCoordinates } from '@/utils/screenshot-context'; +import { cdpSessionManager } from '@/utils/cdp-session-manager'; type MouseButton = 'left' | 'right' | 'middle'; @@ -48,49 +49,16 @@ interface ComputerParams { // Minimal CDP helper encapsulated here to avoid scattering CDP code class CDPHelper { - private static active = new Set(); - static async attach(tabId: number): Promise { - // If already attached by us, skip - const targets = await chrome.debugger.getTargets(); - const existing = targets.find((t) => t.tabId === tabId && t.attached); - if (existing) { - if (existing.extensionId === chrome.runtime.id) { - this.active.add(tabId); - return; - } - throw new Error( - `Debugger is already attached to tab ${tabId} by another tool (e.g., DevTools/extension)`, - ); - } - await chrome.debugger.attach({ tabId }, '1.3'); - this.active.add(tabId); + await cdpSessionManager.attach(tabId, 'computer'); } static async detach(tabId: number): Promise { - if (!this.active.has(tabId)) return; - try { - await chrome.debugger.detach({ tabId }); - } finally { - this.active.delete(tabId); - } + await cdpSessionManager.detach(tabId, 'computer'); } static async send(tabId: number, method: string, params?: object): Promise { - try { - return await chrome.debugger.sendCommand({ tabId }, method, params); - } catch (e: any) { - // Try reattach once if lost - if ( - String(e?.message || e) - .toLowerCase() - .includes('not attached') - ) { - await this.attach(tabId); - return await chrome.debugger.sendCommand({ tabId }, method, params); - } - throw e; - } + return await cdpSessionManager.sendCommand(tabId, method, params); } static async dispatchMouseEvent(tabId: number, opts: any) { @@ -259,18 +227,14 @@ class ComputerTool extends BaseBrowserToolExecutor { // Prefer precise CDP emulation await CDPHelper.attach(tab.id); try { - await chrome.debugger.sendCommand( - { tabId: tab.id }, - 'Emulation.setDeviceMetricsOverride', - { - width: Math.round(w), - height: Math.round(h), - deviceScaleFactor: 0, - mobile: false, - screenWidth: Math.round(w), - screenHeight: Math.round(h), - }, - ); + await CDPHelper.send(tab.id, 'Emulation.setDeviceMetricsOverride', { + width: Math.round(w), + height: Math.round(h), + deviceScaleFactor: 0, + mobile: false, + screenWidth: Math.round(w), + screenHeight: Math.round(h), + }); } finally { await CDPHelper.detach(tab.id); } diff --git a/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts b/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts index 274562aa..b30fca36 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts @@ -1,6 +1,7 @@ import { createErrorResponse, ToolResult } from '@/common/tool-handler'; import { BaseBrowserToolExecutor } from '../base-browser'; import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { cdpSessionManager } from '@/utils/cdp-session-manager'; interface HandleDialogParams { action: 'accept' | 'dismiss'; @@ -22,40 +23,16 @@ class HandleDialogTool extends BaseBrowserToolExecutor { try { const [activeTab] = await chrome.tabs.query({ active: true, currentWindow: true }); if (!activeTab?.id) return createErrorResponse('No active tab found'); + const tabId = activeTab.id!; - // Attach debugger and try handling the dialog - try { - await chrome.debugger.attach({ tabId: activeTab.id }, '1.3'); - } catch (e: any) { - if (String(e?.message || '').includes('attached')) { - // If already attached by us, proceed; otherwise fail with clear message - const targets = await chrome.debugger.getTargets(); - const existing = targets.find((t) => t.tabId === activeTab.id && t.attached); - if (!existing || existing.extensionId !== chrome.runtime.id) { - return createErrorResponse( - `Debugger already attached to tab ${activeTab.id} by another client (e.g., DevTools). Close it and retry.`, - ); - } - } else { - throw e; - } - } - - try { - // Enable Page domain to be safe - await chrome.debugger.sendCommand({ tabId: activeTab.id }, 'Page.enable'); - await chrome.debugger.sendCommand({ tabId: activeTab.id }, 'Page.handleJavaScriptDialog', { + // Use shared CDP session manager for safe attach/detach with refcount + await cdpSessionManager.withSession(tabId, 'dialog', async () => { + await cdpSessionManager.sendCommand(tabId, 'Page.enable'); + await cdpSessionManager.sendCommand(tabId, 'Page.handleJavaScriptDialog', { accept: action === 'accept', promptText: action === 'accept' ? promptText : undefined, }); - } finally { - // Best-effort detach if we were the owners - try { - await chrome.debugger.detach({ tabId: activeTab.id }); - } catch { - // ignore - } - } + }); return { content: [ diff --git a/app/chrome-extension/entrypoints/background/tools/browser/network-capture-debugger.ts b/app/chrome-extension/entrypoints/background/tools/browser/network-capture-debugger.ts index c6adc540..907c4ee5 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/network-capture-debugger.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/network-capture-debugger.ts @@ -1,6 +1,7 @@ import { createErrorResponse, ToolResult } from '@/common/tool-handler'; import { BaseBrowserToolExecutor } from '../base-browser'; import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { cdpSessionManager } from '@/utils/cdp-session-manager'; interface NetworkDebuggerStartToolParams { url?: string; // URL to navigate to or focus. If not provided, uses active tab. @@ -218,35 +219,15 @@ class NetworkDebuggerStartTool extends BaseBrowserToolExecutor { // Get tab information const tab = await chrome.tabs.get(tabId); - // Check if debugger is already attached - const targets = await chrome.debugger.getTargets(); - const existingTarget = targets.find( - (t) => t.tabId === tabId && t.attached && t.type === 'page', - ); - if (existingTarget && !existingTarget.extensionId) { - throw new Error( - `Debugger is already attached to tab ${tabId} by another tool (e.g., DevTools).`, - ); - } - - // Attach debugger - try { - await chrome.debugger.attach({ tabId }, DEBUGGER_PROTOCOL_VERSION); - } catch (error: any) { - if (error.message?.includes('Cannot attach to the target with an attached client')) { - throw new Error( - `Debugger is already attached to tab ${tabId}. This might be DevTools or another extension.`, - ); - } - throw error; - } + // Attach via shared manager (handles conflicts and refcount) + await cdpSessionManager.attach(tabId, 'network-capture'); // Enable network tracking try { - await chrome.debugger.sendCommand({ tabId }, 'Network.enable'); + await cdpSessionManager.sendCommand(tabId, 'Network.enable'); } catch (error: any) { - await chrome.debugger - .detach({ tabId }) + await cdpSessionManager + .detach(tabId, 'network-capture') .catch((e) => console.warn('Error detaching after failed enable:', e)); throw error; } @@ -290,8 +271,8 @@ class NetworkDebuggerStartTool extends BaseBrowserToolExecutor { // Clean up resources if (this.captureData.has(tabId)) { - await chrome.debugger - .detach({ tabId }) + await cdpSessionManager + .detach(tabId, 'network-capture') .catch((e) => console.warn('Cleanup detach error:', e)); this.cleanupCapture(tabId); } @@ -673,14 +654,8 @@ class NetworkDebuggerStartTool extends BaseBrowserToolExecutor { const responseBodyPromise = (async () => { try { - // Check if debugger is still attached to this tabId - const attachedTabs = await chrome.debugger.getTargets(); - if (!attachedTabs.some((target) => target.tabId === tabId && target.attached)) { - // console.warn(`NetworkDebuggerStartTool: Debugger not attached to tab ${tabId} when trying to get response body for ${requestId}.`); - throw new Error(`Debugger not attached to tab ${tabId}`); - } - - const result = (await chrome.debugger.sendCommand({ tabId }, 'Network.getResponseBody', { + // Will attach temporarily if needed + const result = (await cdpSessionManager.sendCommand(tabId, 'Network.getResponseBody', { requestId, })) as { body: string; base64Encoded: boolean }; return result; @@ -734,33 +709,21 @@ class NetworkDebuggerStartTool extends BaseBrowserToolExecutor { ); try { - // Detach debugger first to prevent further events. - // Check if debugger is attached before trying to send commands or detach - const attachedTargets = await chrome.debugger.getTargets(); - const isAttached = attachedTargets.some( - (target) => target.tabId === tabId && target.attached, - ); - - if (isAttached) { - try { - await chrome.debugger.sendCommand({ tabId }, 'Network.disable'); - } catch (e) { - console.warn( - `NetworkDebuggerStartTool: Error disabling network for tab ${tabId} (possibly already detached):`, - e, - ); - } - try { - await chrome.debugger.detach({ tabId }); - } catch (e) { - console.warn( - `NetworkDebuggerStartTool: Error detaching debugger for tab ${tabId} (possibly already detached):`, - e, - ); - } - } else { - console.log( - `NetworkDebuggerStartTool: Debugger was not attached to tab ${tabId} at stopCapture.`, + // Attempt to disable network and detach via manager; it will no-op if others own the session + try { + await cdpSessionManager.sendCommand(tabId, 'Network.disable'); + } catch (e) { + console.warn( + `NetworkDebuggerStartTool: Error disabling network for tab ${tabId} (possibly already detached):`, + e, + ); + } + try { + await cdpSessionManager.detach(tabId, 'network-capture'); + } catch (e) { + console.warn( + `NetworkDebuggerStartTool: Error detaching debugger for tab ${tabId} (possibly already detached):`, + e, ); } } catch (error: any) { @@ -1003,8 +966,8 @@ class NetworkDebuggerStartTool extends BaseBrowserToolExecutor { // If a tabId was involved and debugger might be attached, try to clean up. const tabIdToClean = tabToOperateOn?.id; if (tabIdToClean && this.captureData.has(tabIdToClean)) { - await chrome.debugger - .detach({ tabId: tabIdToClean }) + await cdpSessionManager + .detach(tabIdToClean, 'network-capture') .catch((e) => console.warn('Cleanup detach error:', e)); this.cleanupCapture(tabIdToClean); } diff --git a/app/chrome-extension/utils/cdp-session-manager.ts b/app/chrome-extension/utils/cdp-session-manager.ts new file mode 100644 index 00000000..8f04c233 --- /dev/null +++ b/app/chrome-extension/utils/cdp-session-manager.ts @@ -0,0 +1,109 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; + +type OwnerTag = string; + +interface TabSessionState { + refCount: number; + owners: Set; + attachedByUs: boolean; +} + +const DEBUGGER_PROTOCOL_VERSION = '1.3'; + +class CDPSessionManager { + private sessions = new Map(); + + private getState(tabId: number): TabSessionState | undefined { + return this.sessions.get(tabId); + } + + private setState(tabId: number, state: TabSessionState) { + this.sessions.set(tabId, state); + } + + async attach(tabId: number, owner: OwnerTag = 'unknown'): Promise { + const state = this.getState(tabId); + if (state && state.attachedByUs) { + state.refCount += 1; + state.owners.add(owner); + return; + } + + // Check existing attachments + const targets = await chrome.debugger.getTargets(); + const existing = targets.find((t) => t.tabId === tabId && t.attached); + if (existing) { + if (existing.extensionId === chrome.runtime.id) { + // Already attached by us (e.g., previous tool). Adopt and refcount. + this.setState(tabId, { + refCount: state ? state.refCount + 1 : 1, + owners: new Set([...(state?.owners || []), owner]), + attachedByUs: true, + }); + return; + } + // Another client (DevTools/other extension) is attached + throw new Error( + `Debugger is already attached to tab ${tabId} by another client (e.g., DevTools/extension)`, + ); + } + + // Attach freshly + await chrome.debugger.attach({ tabId }, DEBUGGER_PROTOCOL_VERSION); + this.setState(tabId, { refCount: 1, owners: new Set([owner]), attachedByUs: true }); + } + + async detach(tabId: number, owner: OwnerTag = 'unknown'): Promise { + const state = this.getState(tabId); + if (!state) return; // Nothing to do + + // Update ownership/refcount + if (state.owners.has(owner)) state.owners.delete(owner); + state.refCount = Math.max(0, state.refCount - 1); + + if (state.refCount > 0) { + // Still in use by other owners + return; + } + + // We are the last owner + try { + if (state.attachedByUs) { + await chrome.debugger.detach({ tabId }); + } + } catch (e) { + // Best-effort detach; ignore + } finally { + this.sessions.delete(tabId); + } + } + + /** + * Convenience wrapper: ensures attach before fn, and balanced detach after. + */ + async withSession(tabId: number, owner: OwnerTag, fn: () => Promise): Promise { + await this.attach(tabId, owner); + try { + return await fn(); + } finally { + await this.detach(tabId, owner); + } + } + + /** + * Send a CDP command. Requires that this manager has attached to the tab. + * If not attached by us, will attempt a one-shot attach around the call. + */ + async sendCommand(tabId: number, method: string, params?: object): Promise { + const state = this.getState(tabId); + if (state && state.attachedByUs) { + return (await chrome.debugger.sendCommand({ tabId }, method, params)) as T; + } + // Fallback: temporary session + return await this.withSession(tabId, `send:${method}`, async () => { + return (await chrome.debugger.sendCommand({ tabId }, method, params)) as T; + }); + } +} + +export const cdpSessionManager = new CDPSessionManager(); From f6cb4c0cefdbd167f197e29173047f77f5e22d00 Mon Sep 17 00:00:00 2001 From: hangwin Date: Thu, 9 Oct 2025 07:05:07 +0000 Subject: [PATCH 04/71] refactor(extension): migrate file-upload & console to shared CDP manager --- .../background/tools/browser/console.ts | 117 +++++++-- .../background/tools/browser/file-upload.ts | 233 ++++++------------ 2 files changed, 169 insertions(+), 181 deletions(-) diff --git a/app/chrome-extension/entrypoints/background/tools/browser/console.ts b/app/chrome-extension/entrypoints/background/tools/browser/console.ts index 8af45d0b..b65d03fb 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/console.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/console.ts @@ -1,6 +1,7 @@ import { createErrorResponse, ToolResult } from '@/common/tool-handler'; import { BaseBrowserToolExecutor } from '../base-browser'; import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { cdpSessionManager } from '@/utils/cdp-session-manager'; const DEBUGGER_PROTOCOL_VERSION = '1.3'; const DEFAULT_MAX_MESSAGES = 100; @@ -16,6 +17,7 @@ interface ConsoleMessage { level: string; text: string; args?: any[]; + argsSerialized?: any[]; source?: string; url?: string; lineNumber?: number; @@ -177,28 +179,8 @@ class ConsoleTool extends BaseBrowserToolExecutor { // Get tab information const tab = await chrome.tabs.get(tabId); - // Check if debugger is already attached - const targets = await chrome.debugger.getTargets(); - const existingTarget = targets.find( - (t) => t.tabId === tabId && t.attached && t.type === 'page', - ); - if (existingTarget && !existingTarget.extensionId) { - throw new Error( - `Debugger is already attached to tab ${tabId} by another tool (e.g., DevTools).`, - ); - } - - // Attach debugger - try { - await chrome.debugger.attach({ tabId }, DEBUGGER_PROTOCOL_VERSION); - } catch (error: any) { - if (error.message?.includes('Cannot attach to the target with an attached client')) { - throw new Error( - `Debugger is already attached to tab ${tabId}. This might be DevTools or another extension.`, - ); - } - throw error; - } + // Attach via shared manager + await cdpSessionManager.attach(tabId, 'console'); // Set up event listener to collect messages const collectedMessages: any[] = []; @@ -235,15 +217,90 @@ class ConsoleTool extends BaseBrowserToolExecutor { try { // Enable Runtime domain first to capture console API calls and exceptions - await chrome.debugger.sendCommand({ tabId }, 'Runtime.enable'); + await cdpSessionManager.sendCommand(tabId, 'Runtime.enable'); // Also enable Log domain to capture other log entries - await chrome.debugger.sendCommand({ tabId }, 'Log.enable'); + await cdpSessionManager.sendCommand(tabId, 'Log.enable'); // Wait for all messages to be flushed await new Promise((resolve) => setTimeout(resolve, 2000)); // Process collected messages + // Helper to deeply serialize console arguments when possible + const serializeArg = async (arg: any): Promise => { + try { + if (!arg) return arg; + if (Object.prototype.hasOwnProperty.call(arg, 'unserializableValue')) { + return arg.unserializableValue; + } + if (Object.prototype.hasOwnProperty.call(arg, 'value')) { + return arg.value; + } + if (arg.objectId) { + const resp = (await cdpSessionManager.sendCommand(tabId, 'Runtime.callFunctionOn', { + objectId: arg.objectId, + functionDeclaration: + 'function(maxDepth, maxProps){\n' + + ' const seen=new WeakSet();\n' + + ' function S(v,d){\n' + + ' try{\n' + + ' if(d<0) return "[MaxDepth]";\n' + + ' if(v===null) return null;\n' + + ' const t=typeof v;\n' + + ' if(t!=="object"){\n' + + ' if(t==="bigint") return v.toString()+"n";\n' + + ' return v;\n' + + ' }\n' + + ' if(seen.has(v)) return "[Circular]";\n' + + ' seen.add(v);\n' + + ' if(Array.isArray(v)){\n' + + ' const out=[];\n' + + ' for(let i=0;i=maxProps){ out.push("[...truncated]"); break; }\n' + + ' out.push(S(v[i], d-1));\n' + + ' }\n' + + ' return out;\n' + + ' }\n' + + ' if(v instanceof Date) return {__type:"Date", value:v.toISOString()};\n' + + ' if(v instanceof RegExp) return {__type:"RegExp", value:String(v)};\n' + + ' if(v instanceof Map){\n' + + ' const out={__type:"Map", entries:[]}; let c=0;\n' + + ' for(const [k,val] of v.entries()){\n' + + ' if(c++>=maxProps){ out.entries.push(["[...truncated]","[...truncated]"]); break; }\n' + + ' out.entries.push([S(k,d-1), S(val,d-1)]);\n' + + ' }\n' + + ' return out;\n' + + ' }\n' + + ' if(v instanceof Set){\n' + + ' const out={__type:"Set", values:[]}; let c=0;\n' + + ' for(const val of v.values()){\n' + + ' if(c++>=maxProps){ out.values.push("[...truncated]"); break; }\n' + + ' out.values.push(S(val,d-1));\n' + + ' }\n' + + ' return out;\n' + + ' }\n' + + ' const out={}; let c=0;\n' + + ' for(const key in v){\n' + + ' if(c++>=maxProps){ out.__truncated__=true; break; }\n' + + ' try{ out[key]=S(v[key], d-1); }catch(e){ out[key]="[Thrown]"; }\n' + + ' }\n' + + ' return out;\n' + + ' }catch(e){ return "[Unserializable]" }\n' + + ' }\n' + + ' return S(this, maxDepth);\n' + + '}', + arguments: [{ value: 3 }, { value: 100 }], + silent: true, + returnByValue: true, + })) as any; + return resp?.result?.value ?? '[Unavailable]'; + } + return '[Unknown]'; + } catch (e) { + return '[SerializeError]'; + } + }; + for (const entry of collectedMessages) { if (messages.length >= maxMessages) { limitReached = true; @@ -265,6 +322,12 @@ class ConsoleTool extends BaseBrowserToolExecutor { if (entry.args && Array.isArray(entry.args)) { message.args = entry.args; + // Attempt deep serialization for better fidelity + const serialized: any[] = []; + for (const a of entry.args) { + serialized.push(await serializeArg(a)); + } + message.argsSerialized = serialized; } messages.push(message); @@ -294,19 +357,19 @@ class ConsoleTool extends BaseBrowserToolExecutor { chrome.debugger.onEvent.removeListener(eventListener); try { - await chrome.debugger.sendCommand({ tabId }, 'Runtime.disable'); + await cdpSessionManager.sendCommand(tabId, 'Runtime.disable'); } catch (e) { console.warn(`ConsoleTool: Error disabling Runtime for tab ${tabId}:`, e); } try { - await chrome.debugger.sendCommand({ tabId }, 'Log.disable'); + await cdpSessionManager.sendCommand(tabId, 'Log.disable'); } catch (e) { console.warn(`ConsoleTool: Error disabling Log for tab ${tabId}:`, e); } try { - await chrome.debugger.detach({ tabId }); + await cdpSessionManager.detach(tabId, 'console'); } catch (e) { console.warn(`ConsoleTool: Error detaching debugger for tab ${tabId}:`, e); } diff --git a/app/chrome-extension/entrypoints/background/tools/browser/file-upload.ts b/app/chrome-extension/entrypoints/background/tools/browser/file-upload.ts index 7e3caf12..c7efe3f9 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/file-upload.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/file-upload.ts @@ -1,6 +1,7 @@ import { createErrorResponse, ToolResult } from '@/common/tool-handler'; import { BaseBrowserToolExecutor } from '../base-browser'; import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { cdpSessionManager } from '@/utils/cdp-session-manager'; interface FileUploadToolParams { selector: string; // CSS selector for the file input element @@ -17,16 +18,8 @@ interface FileUploadToolParams { */ class FileUploadTool extends BaseBrowserToolExecutor { name = TOOL_NAMES.BROWSER.FILE_UPLOAD; - private activeDebuggers: Map = new Map(); - constructor() { super(); - // Clean up debuggers on tab removal - chrome.tabs.onRemoved.addListener((tabId) => { - if (this.activeDebuggers.has(tabId)) { - this.cleanupDebugger(tabId); - } - }); } /** @@ -43,12 +36,10 @@ class FileUploadTool extends BaseBrowserToolExecutor { } if (!filePath && !fileUrl && !base64Data) { - return createErrorResponse( - 'One of filePath, fileUrl, or base64Data must be provided', - ); + return createErrorResponse('One of filePath, fileUrl, or base64Data must be provided'); } - let tabId: number | undefined; + let tabId: number; try { // Get current tab @@ -56,7 +47,7 @@ class FileUploadTool extends BaseBrowserToolExecutor { if (!tabs[0]?.id) { return createErrorResponse('No active tab found'); } - tabId = tabs[0].id; + tabId = tabs[0].id!; // Prepare file paths let files: string[] = []; @@ -78,75 +69,59 @@ class FileUploadTool extends BaseBrowserToolExecutor { files = [tempFilePath]; } - // Attach debugger to the tab - await this.attachDebugger(tabId); - - // Enable necessary CDP domains - await chrome.debugger.sendCommand({ tabId }, 'DOM.enable', {}); - await chrome.debugger.sendCommand({ tabId }, 'Runtime.enable', {}); + // Use shared CDP session manager to attach/do work/detach safely + await cdpSessionManager.withSession(tabId, 'file-upload', async () => { + // Enable necessary CDP domains + await cdpSessionManager.sendCommand(tabId, 'DOM.enable', {}); + await cdpSessionManager.sendCommand(tabId, 'Runtime.enable', {}); - // Get the document - const { root } = await chrome.debugger.sendCommand( - { tabId }, - 'DOM.getDocument', - { depth: -1, pierce: true }, - ) as { root: { nodeId: number } }; + // Get the document + const { root } = (await cdpSessionManager.sendCommand(tabId, 'DOM.getDocument', { + depth: -1, + pierce: true, + })) as { root: { nodeId: number } }; - // Find the file input element using the selector - const { nodeId } = await chrome.debugger.sendCommand( - { tabId }, - 'DOM.querySelector', - { + // Find the file input element using the selector + const { nodeId } = (await cdpSessionManager.sendCommand(tabId, 'DOM.querySelector', { nodeId: root.nodeId, selector: selector, - }, - ) as { nodeId: number }; + })) as { nodeId: number }; - if (!nodeId || nodeId === 0) { - throw new Error(`Element with selector "${selector}" not found`); - } + if (!nodeId || nodeId === 0) { + throw new Error(`Element with selector "${selector}" not found`); + } - // Verify it's actually a file input - const { node } = await chrome.debugger.sendCommand( - { tabId }, - 'DOM.describeNode', - { nodeId }, - ) as { node: { nodeName: string; attributes?: string[] } }; + // Verify it's actually a file input + const { node } = (await cdpSessionManager.sendCommand(tabId, 'DOM.describeNode', { + nodeId, + })) as { node: { nodeName: string; attributes?: string[] } }; - if (node.nodeName !== 'INPUT') { - throw new Error(`Element with selector "${selector}" is not an input element`); - } + if (node.nodeName !== 'INPUT') { + throw new Error(`Element with selector "${selector}" is not an input element`); + } - // Check if it's a file input by looking for type="file" in attributes - const attributes = node.attributes || []; - let isFileInput = false; - for (let i = 0; i < attributes.length; i += 2) { - if (attributes[i] === 'type' && attributes[i + 1] === 'file') { - isFileInput = true; - break; + // Check if it's a file input by looking for type="file" in attributes + const attributes = node.attributes || []; + let isFileInput = false; + for (let i = 0; i < attributes.length; i += 2) { + if (attributes[i] === 'type' && attributes[i + 1] === 'file') { + isFileInput = true; + break; + } } - } - if (!isFileInput) { - throw new Error(`Element with selector "${selector}" is not a file input (type="file")`); - } + if (!isFileInput) { + throw new Error(`Element with selector "${selector}" is not a file input (type="file")`); + } - // Set the files on the input element - // This is the key CDP command that Playwright and Puppeteer use - await chrome.debugger.sendCommand( - { tabId }, - 'DOM.setFileInputFiles', - { - nodeId: nodeId, - files: files, - }, - ); + // Set the files on the input element + await cdpSessionManager.sendCommand(tabId, 'DOM.setFileInputFiles', { + nodeId, + files, + }); - // Trigger change event to ensure the page reacts to the file upload - await chrome.debugger.sendCommand( - { tabId }, - 'Runtime.evaluate', - { + // Trigger change event to ensure the page reacts to the file upload + await cdpSessionManager.sendCommand(tabId, 'Runtime.evaluate', { expression: ` (function() { const element = document.querySelector('${selector.replace(/'/g, "\\'")}'); @@ -158,11 +133,8 @@ class FileUploadTool extends BaseBrowserToolExecutor { return false; })() `, - }, - ); - - // Clean up debugger - await this.detachDebugger(tabId); + }); + }); return { content: [ @@ -181,11 +153,8 @@ class FileUploadTool extends BaseBrowserToolExecutor { }; } catch (error) { console.error('Error in file upload operation:', error); - - // Clean up debugger if attached - if (tabId !== undefined && this.activeDebuggers.has(tabId)) { - await this.detachDebugger(tabId); - } + + // Session manager handles detach; nothing extra needed here return createErrorResponse( `Error uploading file: ${error instanceof Error ? error.message : String(error)}`, @@ -193,58 +162,7 @@ class FileUploadTool extends BaseBrowserToolExecutor { } } - /** - * Attach debugger to a tab - */ - private async attachDebugger(tabId: number): Promise { - // Check if debugger is already attached - const targets = await chrome.debugger.getTargets(); - const existingTarget = targets.find( - (t) => t.tabId === tabId && t.attached, - ); - - if (existingTarget) { - if (existingTarget.extensionId === chrome.runtime.id) { - // Our extension already attached - console.log('Debugger already attached by this extension'); - return; - } else { - throw new Error( - 'Debugger is already attached to this tab by another extension or DevTools', - ); - } - } - - // Attach debugger - await chrome.debugger.attach({ tabId }, '1.3'); - this.activeDebuggers.set(tabId, true); - console.log(`Debugger attached to tab ${tabId}`); - } - - /** - * Detach debugger from a tab - */ - private async detachDebugger(tabId: number): Promise { - if (!this.activeDebuggers.has(tabId)) { - return; - } - - try { - await chrome.debugger.detach({ tabId }); - console.log(`Debugger detached from tab ${tabId}`); - } catch (error) { - console.warn(`Error detaching debugger from tab ${tabId}:`, error); - } finally { - this.activeDebuggers.delete(tabId); - } - } - - /** - * Clean up debugger connection - */ - private cleanupDebugger(tabId: number): void { - this.activeDebuggers.delete(tabId); - } + // All debugger attach/detach is centrally managed by cdpSessionManager /** * Prepare file from URL or base64 data using native messaging host @@ -265,15 +183,20 @@ class FileUploadTool extends BaseBrowserToolExecutor { // Create listener for the response const handleMessage = (message: any) => { - if (message.type === 'file_operation_response' && - message.responseToRequestId === requestId) { + if ( + message.type === 'file_operation_response' && + message.responseToRequestId === requestId + ) { clearTimeout(timeout); chrome.runtime.onMessage.removeListener(handleMessage); - + if (message.payload?.success && message.payload?.filePath) { resolve(message.payload.filePath); } else { - console.error('Native host failed to prepare file:', message.error || message.payload?.error); + console.error( + 'Native host failed to prepare file:', + message.error || message.payload?.error, + ); resolve(null); } } @@ -283,26 +206,28 @@ class FileUploadTool extends BaseBrowserToolExecutor { chrome.runtime.onMessage.addListener(handleMessage); // Send message to background script to forward to native host - chrome.runtime.sendMessage({ - type: 'forward_to_native', - message: { - type: 'file_operation', - requestId: requestId, - payload: { - action: 'prepareFile', - fileUrl, - base64Data, - fileName, + chrome.runtime + .sendMessage({ + type: 'forward_to_native', + message: { + type: 'file_operation', + requestId: requestId, + payload: { + action: 'prepareFile', + fileUrl, + base64Data, + fileName, + }, }, - }, - }).catch((error) => { - console.error('Error sending message to background:', error); - clearTimeout(timeout); - chrome.runtime.onMessage.removeListener(handleMessage); - resolve(null); - }); + }) + .catch((error) => { + console.error('Error sending message to background:', error); + clearTimeout(timeout); + chrome.runtime.onMessage.removeListener(handleMessage); + resolve(null); + }); }); } } -export const fileUploadTool = new FileUploadTool(); \ No newline at end of file +export const fileUploadTool = new FileUploadTool(); From 25f416635b6b4e8ae18630104860b6fc016dc5ec Mon Sep 17 00:00:00 2001 From: hangwin Date: Thu, 9 Oct 2025 09:18:31 +0000 Subject: [PATCH 05/71] feat(extension): add unified chrome_userscript tool - Implement UserscriptTool with create/list/get/enable/disable/update/remove/send_command/export - Auto strategy selection (insertCSS, persistent ISOLATED/MAIN) with CSP-aware fallback - Persistence via chrome.storage; tab re-injection on navigation - Register tool and schema; add USERSCRIPTS storage key - Add Vue SFC typings (env.d.ts); fix lint/typecheck --- app/chrome-extension/common/constants.ts | 1 + .../background/tools/browser/index.ts | 1 + .../background/tools/browser/userscript.ts | 620 ++++++++++++++++++ app/chrome-extension/env.d.ts | 7 + packages/shared/src/tools.ts | 22 + 5 files changed, 651 insertions(+) create mode 100644 app/chrome-extension/entrypoints/background/tools/browser/userscript.ts create mode 100644 app/chrome-extension/env.d.ts diff --git a/app/chrome-extension/common/constants.ts b/app/chrome-extension/common/constants.ts index 6cd5cc4b..b8656d23 100644 --- a/app/chrome-extension/common/constants.ts +++ b/app/chrome-extension/common/constants.ts @@ -102,6 +102,7 @@ export const STORAGE_KEYS = { SEMANTIC_MODEL: 'selectedModel', USER_PREFERENCES: 'userPreferences', VECTOR_INDEX: 'vectorIndex', + USERSCRIPTS: 'userscripts', } as const; // Notification Configuration diff --git a/app/chrome-extension/entrypoints/background/tools/browser/index.ts b/app/chrome-extension/entrypoints/background/tools/browser/index.ts index d473e08c..05f86bcb 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/index.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/index.ts @@ -16,3 +16,4 @@ export { fileUploadTool } from './file-upload'; export { readPageTool } from './read-page'; export { computerTool } from './computer'; export { handleDialogTool } from './dialog'; +export { userscriptTool } from './userscript'; diff --git a/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts b/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts new file mode 100644 index 00000000..d3d3fbb0 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts @@ -0,0 +1,620 @@ +import { createErrorResponse, ToolResult } from '@/common/tool-handler'; +import { BaseBrowserToolExecutor } from '../base-browser'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { ExecutionWorld, STORAGE_KEYS } from '@/common/constants'; + +type UserscriptAction = + | 'create' + | 'list' + | 'get' + | 'enable' + | 'disable' + | 'update' + | 'remove' + | 'send_command' + | 'export'; + +interface UserscriptArgsBase { + action: UserscriptAction; + args?: any; +} + +interface CreateArgs { + script: string; + name?: string; + description?: string; + matches?: string[]; + excludes?: string[]; + persist?: boolean; // default true + runAt?: 'document_start' | 'document_end' | 'document_idle' | 'auto'; // default auto(document_idle) + world?: 'auto' | 'ISOLATED' | 'MAIN'; // default auto(ISOLATED) + allFrames?: boolean; // default true + mode?: 'auto' | 'css' | 'persistent' | 'once'; // default auto + dnrFallback?: boolean; // default true + tags?: string[]; +} + +type UpdateArgs = Partial> & { id: string; script?: string }; + +interface UserscriptRecord { + id: string; + name?: string; + description?: string; + script: string; + sourceType: 'JS' | 'CSS' | 'TM'; + matches: string[]; + excludes: string[]; + runAt: 'document_start' | 'document_end' | 'document_idle'; + world: 'ISOLATED' | 'MAIN'; + allFrames: boolean; + persist: boolean; + dnrFallback: boolean; + tags?: string[]; + enabled: boolean; + createdAt: number; + updatedAt: number; + installedBy?: string; + lastError?: string; + applyCount?: number; +} + +// In-memory tracking of active injections per tab +type ActiveInjection = { kind: 'css' | 'js'; world?: 'ISOLATED' | 'MAIN' }; +const activeInjections: Map> = new Map(); + +async function loadAllRecords(): Promise> { + const res = await chrome.storage.local.get([STORAGE_KEYS.USERSCRIPTS]); + return (res[STORAGE_KEYS.USERSCRIPTS] as Record) || {}; +} + +async function saveAllRecords(records: Record): Promise { + await chrome.storage.local.set({ [STORAGE_KEYS.USERSCRIPTS]: records }); +} + +// Simple FNV-1a hash for deterministic IDs +function fnv1a(str: string): string { + let h = 0x811c9dc5; + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h += (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24); + } + // Force to unsigned and hex + return (h >>> 0).toString(16); +} + +function now(): number { + return Date.now(); +} + +// Basic TM header parser (subset) +function parseUserscriptMeta(source: string): { + meta: Record; + isTM: boolean; +} { + const meta: Record = {}; + const start = source.indexOf('==UserScript=='); + const end = source.indexOf('==/UserScript=='); + if (start !== -1 && end !== -1 && end > start) { + const block = source.slice(start, end).split(/\r?\n/); + for (const line of block) { + const m = line.match(/@([\w-]+)\s+(.+)/); + if (m) { + const k = m[1].trim(); + const v = m[2].trim(); + if (!meta[k]) meta[k] = []; + meta[k].push(v); + } + } + return { meta, isTM: true }; + } + return { meta: {}, isTM: false }; +} + +function pick(arr: T[] | undefined): T | undefined { + return arr && arr.length > 0 ? arr[0] : undefined; +} + +function deriveName(meta: Record, fallback?: string): string | undefined { + return pick(meta['name']) || fallback; +} + +function toBoolean(val: any, d: boolean): boolean { + return typeof val === 'boolean' ? val : d; +} + +// Very light CSS heuristic +function isLikelyCSS(source: string): boolean { + const trimmed = source.trim(); + if (trimmed.startsWith('/*') && trimmed.includes('==UserStyle')) return true; + if (/^[.#\w\-\s*,:>+~\n\r{}();'"%!@/]+$/.test(trimmed)) { + // no obvious JS keywords + if ( + !/(function|=>|var\s|let\s|const\s|document\.|window\.|\beval\b|new\s+Function)/.test(trimmed) + ) { + // has CSS braces and colons + const colon = (trimmed.match(/:/g) || []).length; + const brace = (trimmed.match(/[{}]/g) || []).length; + return colon > 0 && brace >= 2; + } + } + return false; +} + +function normalizeMatches(matches?: string[], currentUrl?: string): string[] { + if (matches && matches.length > 0) return matches; + if (!currentUrl) return ['']; + try { + const u = new URL(currentUrl); + const host = u.hostname; + const base = host.startsWith('www.') ? host.slice(4) : host; + return [`${u.protocol}//*.${base}/*`, `${u.protocol}//${host}/*`]; + } catch { + return ['']; + } +} + +// Simple URL match using chrome match patterns subset +function matchUrl(patterns: string[], url?: string): boolean { + if (!url) return false; + try { + const u = new URL(url); + for (const p of patterns) { + if (p === '') return true; + const m = p.match(/^(\*|https?:)\/\/([^/]+)\/(.*)$/); + if (!m) continue; + const proto = m[1]; + const host = m[2]; + const path = m[3]; + if (proto !== '*' && proto !== u.protocol.replace(':', '')) continue; + // host wildcard + const hostRegex = new RegExp( + '^' + + host + .split('.') + .map((h) => (h === '*' ? '[^.]+' : h.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'))) + .join('\\.') + + '$', + ); + if (!hostRegex.test(u.hostname)) continue; + // path wildcard + const pathRegex = new RegExp( + '^' + path.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$', + ); + const testPath = (u.pathname + (u.search || '') + (u.hash || '')).replace(/^\//, ''); + if (pathRegex.test(testPath)) return true; + } + } catch { + return false; + } + return false; +} + +async function getActiveTab(): Promise { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + return tabs[0] || null; +} + +async function insertCssToTab(tabId: number, css: string, allFrames: boolean) { + await chrome.scripting.insertCSS({ target: { tabId, allFrames }, css }); +} + +async function removeCssFromTab(tabId: number, css: string, allFrames: boolean) { + try { + await chrome.scripting.removeCSS({ target: { tabId, allFrames }, css }); + } catch (e) { + // ignore if not present + } +} + +async function injectJsPersistent( + tabId: number, + code: string, + world: 'ISOLATED' | 'MAIN', + allFrames: boolean, +) { + if (world === ExecutionWorld.MAIN) { + // Ensure bridge is present in ISOLATED + await chrome.scripting.executeScript({ + target: { tabId, allFrames }, + files: ['inject-scripts/inject-bridge.js'], + world: ExecutionWorld.ISOLATED, + }); + // MAIN world code with command handler wrapper + const wrapped = `(() => { + try { + // Optional command API: window.__userscript_onCommand(action, payload) + window.addEventListener('chrome-mcp:execute', (ev) => { + const { action, payload, requestId } = ev.detail || {}; + try { + let result; + const handler = (window as any).__userscript_onCommand; + if (typeof handler === 'function') { + result = handler(action, payload); + } + window.dispatchEvent(new CustomEvent('chrome-mcp:response', { detail: { requestId, data: result } })); + } catch (err) { + window.dispatchEvent(new CustomEvent('chrome-mcp:response', { detail: { requestId, error: String(err && (err as any).message || err) } })); + } + }); + (new Function(${JSON.stringify(code)}))(); + } catch (e) { + console.warn('Userscript MAIN injection error:', e); + } + })();`; + await chrome.scripting.executeScript({ + target: { tabId, allFrames }, + func: (src) => { + try { + // Using Function constructor intentionally to evaluate user-provided script + new Function(src)(); + } catch (e) { + console.warn('Userscript MAIN wrapper execution error:', e); + } + }, + args: [wrapped], + world: ExecutionWorld.MAIN, + }); + } else { + // ISOLATED world code with message handler + await chrome.scripting.executeScript({ + target: { tabId, allFrames }, + func: (userCode) => { + try { + const handlerName = '__userscript_onCommand__'; + (chrome.runtime.onMessage as any).addListener( + (req: any, _sender: any, sendResponse: any) => { + if (!req || req.type !== 'userscript:command') return; + const { action, payload, scriptId } = req; + try { + const handler = (globalThis as any)[handlerName]; + let result; + if (typeof handler === 'function') { + result = handler(action, payload, scriptId); + } + sendResponse({ data: result }); + } catch (err) { + sendResponse({ error: String((err && (err as any).message) || err) }); + } + return true; + }, + ); + // Using Function constructor intentionally to evaluate user-provided script + new Function(userCode)(); + } catch (e) { + console.warn('Userscript ISOLATED injection error:', e); + } + }, + args: [code], + world: ExecutionWorld.ISOLATED, + }); + } +} + +function setActiveInjection(tabId: number, id: string, inj: ActiveInjection) { + let m = activeInjections.get(tabId); + if (!m) { + m = new Map(); + activeInjections.set(tabId, m); + } + m.set(id, inj); +} + +function clearActiveInjection(tabId: number, id: string) { + const m = activeInjections.get(tabId); + if (m) m.delete(id); +} + +async function reinjectForTab(tabId: number, url?: string) { + const all = await loadAllRecords(); + for (const rec of Object.values(all)) { + if (!rec.enabled || !rec.persist) continue; + if (!matchUrl(rec.matches, url)) continue; + try { + if (rec.sourceType === 'CSS') { + await insertCssToTab(tabId, rec.script, rec.allFrames); + setActiveInjection(tabId, rec.id, { kind: 'css' }); + } else { + await injectJsPersistent(tabId, rec.script, rec.world, rec.allFrames); + setActiveInjection(tabId, rec.id, { kind: 'js', world: rec.world }); + } + } catch (e) { + console.warn('Reinject failed for tab', tabId, rec.id, e); + } + } +} + +// Tab update listener: re-apply enabled persistent scripts +chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.status === 'complete') { + reinjectForTab(tabId, tab.url).catch(() => {}); + } +}); + +class UserscriptTool extends BaseBrowserToolExecutor { + name = TOOL_NAMES.BROWSER.USERSCRIPT; + + async execute(params: UserscriptArgsBase): Promise { + try { + const { action } = params; + const args = params.args || {}; + + switch (action) { + case 'create': + return await this.create(args as CreateArgs); + case 'list': + return await this.list(args); + case 'get': + return await this.get(args); + case 'enable': + return await this.enable(args, true); + case 'disable': + return await this.enable(args, false); + case 'update': + return await this.update(args as UpdateArgs); + case 'remove': + return await this.remove(args); + case 'send_command': + return await this.sendCommand(args); + case 'export': + return await this.exportAll(); + default: + return createErrorResponse(`Unknown action: ${String(action)}`); + } + } catch (error) { + console.error('Userscript tool error:', error); + return createErrorResponse( + `Userscript error: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + private async create(args: CreateArgs): Promise { + const active = await getActiveTab(); + if (!active || !active.id) return createErrorResponse('No active tab found'); + const currentUrl = active.url; + + const { meta, isTM } = parseUserscriptMeta(args.script); + const name = args.name || deriveName(meta, undefined); + const description = args.description || pick(meta['description']); + const matches = normalizeMatches(args.matches || meta['match'] || meta['include'], currentUrl); + const excludes = args.excludes || meta['exclude'] || []; + + const runAt: UserscriptRecord['runAt'] = + (args.runAt && args.runAt !== 'auto' ? args.runAt : (pick(meta['run-at']) as any)) || + 'document_idle'; + const requestedWorld = + (args.world && args.world !== 'auto' ? args.world : (pick(meta['inject-into']) as any)) || + 'ISOLATED'; + const allFrames = toBoolean(args.allFrames, true); + const persist = toBoolean(args.persist, true); + const dnrFallback = toBoolean(args.dnrFallback, true); + const mode = args.mode || 'auto'; + + const sourceType: UserscriptRecord['sourceType'] = isTM + ? 'TM' + : mode === 'css' || isLikelyCSS(args.script) + ? 'CSS' + : 'JS'; + + const id = `us_${fnv1a((name || '') + '|' + args.script)}`; + + const record: UserscriptRecord = { + id, + name, + description, + script: args.script, + sourceType, + matches, + excludes, + runAt, + world: requestedWorld === 'MAIN' ? 'MAIN' : 'ISOLATED', + allFrames, + persist, + dnrFallback, + tags: args.tags, + enabled: true, + createdAt: now(), + updatedAt: now(), + applyCount: 0, + }; + + const all = await loadAllRecords(); + all[id] = record; + await saveAllRecords(all); + + // Apply to current tab immediately if matches + let applied = false; + const fallbacks: string[] = []; + try { + if (sourceType === 'CSS') { + await insertCssToTab(active.id!, record.script, record.allFrames); + setActiveInjection(active.id!, id, { kind: 'css' }); + applied = true; + } else { + try { + await injectJsPersistent(active.id!, record.script, record.world, record.allFrames); + setActiveInjection(active.id!, id, { kind: 'js', world: record.world }); + applied = true; + } catch (e) { + // Try fallback to ISOLATED if MAIN blocked by CSP + if (record.world === 'MAIN') { + fallbacks.push('MAIN->ISOLATED'); + await injectJsPersistent(active.id!, record.script, 'ISOLATED', record.allFrames); + setActiveInjection(active.id!, id, { kind: 'js', world: 'ISOLATED' }); + applied = true; + } else { + throw e; + } + } + } + } catch (e) { + all[id].lastError = e instanceof Error ? e.message : String(e); + await saveAllRecords(all); + } + + const result = { + id, + status: all[id].lastError ? 'queued' : applied ? 'applied' : 'queued', + strategy: { + kind: sourceType === 'CSS' ? 'insertCSS' : `persistent_${all[id].world.toLowerCase()}`, + runAt: all[id].runAt, + world: all[id].world, + allFrames: all[id].allFrames, + fallbacksTried: fallbacks, + }, + warnings: [], + }; + + return { + content: [{ type: 'text', text: JSON.stringify(result) }], + isError: false, + }; + } + + private async list(_args: any): Promise { + const all = await loadAllRecords(); + const items = Object.values(all).map((r) => ({ + id: r.id, + name: r.name, + status: r.enabled ? 'enabled' : 'disabled', + sourceType: r.sourceType, + matches: r.matches, + world: r.world, + runAt: r.runAt, + tags: r.tags || [], + lastError: r.lastError, + updatedAt: r.updatedAt, + })); + return { + content: [{ type: 'text', text: JSON.stringify({ ok: true, items }) }], + isError: false, + }; + } + + private async get(args: any): Promise { + const { id } = args || {}; + if (!id) return createErrorResponse('id is required'); + const all = await loadAllRecords(); + const rec = all[id]; + if (!rec) return createErrorResponse('userscript not found'); + return { + content: [{ type: 'text', text: JSON.stringify({ ok: true, record: rec }) }], + isError: false, + }; + } + + private async enable(args: any, enabled: boolean): Promise { + const { id } = args || {}; + if (!id) return createErrorResponse('id is required'); + const all = await loadAllRecords(); + const rec = all[id]; + if (!rec) return createErrorResponse('userscript not found'); + rec.enabled = enabled; + rec.updatedAt = now(); + await saveAllRecords(all); + return { content: [{ type: 'text', text: JSON.stringify({ ok: true }) }], isError: false }; + } + + private async update(args: UpdateArgs): Promise { + const { id, ...rest } = args; + if (!id) return createErrorResponse('id is required'); + const all = await loadAllRecords(); + const rec = all[id]; + if (!rec) return createErrorResponse('userscript not found'); + + if (rest.name !== undefined) rec.name = rest.name; + if (rest.description !== undefined) rec.description = rest.description; + if (rest.matches) rec.matches = rest.matches; + if (rest.excludes) rec.excludes = rest.excludes; + if (rest.runAt && rest.runAt !== 'auto') rec.runAt = rest.runAt; + if (rest.world && rest.world !== 'auto') rec.world = rest.world as any; + if (typeof rest.allFrames === 'boolean') rec.allFrames = rest.allFrames; + if (typeof rest.persist === 'boolean') rec.persist = rest.persist; + if (typeof rest.dnrFallback === 'boolean') rec.dnrFallback = rest.dnrFallback; + if (rest.tags) rec.tags = rest.tags; + if (typeof rest.script === 'string') rec.script = rest.script; + rec.updatedAt = now(); + await saveAllRecords(all); + return { content: [{ type: 'text', text: JSON.stringify({ ok: true }) }], isError: false }; + } + + private async remove(args: any): Promise { + const { id } = args || {}; + if (!id) return createErrorResponse('id is required'); + const all = await loadAllRecords(); + const rec = all[id]; + if (!rec) return createErrorResponse('userscript not found'); + delete all[id]; + await saveAllRecords(all); + + // Attempt cleanup on active tab + const active = await getActiveTab(); + if (active && active.id) { + try { + if (rec.sourceType === 'CSS') { + await removeCssFromTab(active.id, rec.script, rec.allFrames); + } else { + // Send cleanup signal via bridge (MAIN) or ignore if isolated + chrome.tabs.sendMessage(active.id, { type: 'chrome-mcp:cleanup' }).catch(() => {}); + } + clearActiveInjection(active.id, rec.id); + } catch (err) { + console.warn('Userscript cleanup failed:', err); + } + } + + return { content: [{ type: 'text', text: JSON.stringify({ ok: true }) }], isError: false }; + } + + private async sendCommand(args: any): Promise { + const { id, payload, tabId } = args || {}; + if (!id) return createErrorResponse('id is required'); + const tab = tabId ? await chrome.tabs.get(tabId).catch(() => null) : await getActiveTab(); + if (!tab || !tab.id) return createErrorResponse('No active tab found'); + + const all = await loadAllRecords(); + const rec = all[id]; + if (!rec) return createErrorResponse('userscript not found'); + + try { + if (rec.world === 'MAIN') { + // Use bridge + const result = await chrome.tabs.sendMessage(tab.id, { + action: 'userscript:command', + payload, + targetWorld: 'MAIN', + }); + return { + content: [{ type: 'text', text: JSON.stringify({ ok: true, result }) }], + isError: false, + }; + } else { + // ISOLATED handler + const result = await chrome.tabs.sendMessage(tab.id, { + type: 'userscript:command', + action: 'userscript:command', + payload, + scriptId: id, + }); + return { + content: [{ type: 'text', text: JSON.stringify({ ok: true, result }) }], + isError: false, + }; + } + } catch (e) { + return createErrorResponse( + `send_command failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + + private async exportAll(): Promise { + const all = await loadAllRecords(); + return { + content: [{ type: 'text', text: JSON.stringify({ ok: true, data: all }) }], + isError: false, + }; + } +} + +export const userscriptTool = new UserscriptTool(); diff --git a/app/chrome-extension/env.d.ts b/app/chrome-extension/env.d.ts new file mode 100644 index 00000000..7ad1e39b --- /dev/null +++ b/app/chrome-extension/env.d.ts @@ -0,0 +1,7 @@ +declare module '*.vue' { + import type { DefineComponent } from 'vue'; + type Props = Record; + type RawBindings = Record; + const component: DefineComponent; + export default component; +} diff --git a/packages/shared/src/tools.ts b/packages/shared/src/tools.ts index 7bd1b421..111d282b 100644 --- a/packages/shared/src/tools.ts +++ b/packages/shared/src/tools.ts @@ -30,6 +30,7 @@ export const TOOL_NAMES = { READ_PAGE: 'chrome_read_page', COMPUTER: 'chrome_computer', HANDLE_DIALOG: 'chrome_handle_dialog', + USERSCRIPT: 'chrome_userscript', }, }; @@ -151,6 +152,27 @@ export const TOOL_SCHEMAS: Tool[] = [ required: ['action'], }, }, + { + name: TOOL_NAMES.BROWSER.USERSCRIPT, + description: + 'Unified userscript tool (create/list/get/enable/disable/update/remove/send_command/export). Paste JS/CSS/Tampermonkey script and the system will auto-select the best strategy (insertCSS / persistent script in ISOLATED or MAIN world / once by CDP) with CSP-aware fallbacks.', + inputSchema: { + type: 'object', + properties: { + action: { + type: 'string', + description: + 'Operation to perform: create | list | get | enable | disable | update | remove | send_command | export', + }, + args: { + type: 'object', + description: + 'Arguments for the specified action. For create: { script, name?, description?, matches?, excludes?, persist?, runAt?, world?, allFrames?, mode?, dnrFallback?, tags? }', + }, + }, + required: ['action'], + }, + }, { name: TOOL_NAMES.BROWSER.NAVIGATE, description: 'Navigate to a URL or refresh the current tab', From 7070f25e622fabfe2f9b81db37409dbdbb3de78c Mon Sep 17 00:00:00 2001 From: hangwin Date: Thu, 9 Oct 2025 09:56:05 +0000 Subject: [PATCH 06/71] feat(extension/userscript): enterprise upgrades - Add emergency global switch (USERSCRIPTS_DISABLED) - Implement CSP probe and fallback (MAIN -> ISOLATED) - Support mode=once via CDP Runtime.evaluate - Add SHA-256 hashing, metrics (injectMs), and status flags - Add webNavigation + declarativeNetRequest permissions - Enhance list filtering (query/status/domain) --- app/chrome-extension/common/constants.ts | 1 + .../background/tools/browser/userscript.ts | 204 +++++++++++++++--- app/chrome-extension/wxt.config.ts | 2 + 3 files changed, 174 insertions(+), 33 deletions(-) diff --git a/app/chrome-extension/common/constants.ts b/app/chrome-extension/common/constants.ts index b8656d23..cb0cca15 100644 --- a/app/chrome-extension/common/constants.ts +++ b/app/chrome-extension/common/constants.ts @@ -103,6 +103,7 @@ export const STORAGE_KEYS = { USER_PREFERENCES: 'userPreferences', VECTOR_INDEX: 'vectorIndex', USERSCRIPTS: 'userscripts', + USERSCRIPTS_DISABLED: 'userscripts_disabled', } as const; // Notification Configuration diff --git a/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts b/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts index d3d3fbb0..dc89d553 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/userscript.ts @@ -2,6 +2,7 @@ import { createErrorResponse, ToolResult } from '@/common/tool-handler'; import { BaseBrowserToolExecutor } from '../base-browser'; import { TOOL_NAMES } from 'chrome-mcp-shared'; import { ExecutionWorld, STORAGE_KEYS } from '@/common/constants'; +import { cdpSessionManager } from '@/utils/cdp-session-manager'; type UserscriptAction = | 'create' @@ -56,6 +57,9 @@ interface UserscriptRecord { installedBy?: string; lastError?: string; applyCount?: number; + lastAppliedAt?: number; + sha256?: string; + cspBlocked?: boolean; } // In-memory tracking of active injections per tab @@ -86,6 +90,33 @@ function now(): number { return Date.now(); } +async function computeSHA256(input: string): Promise { + const enc = new TextEncoder().encode(input); + const digest = await crypto.subtle.digest('SHA-256', enc); + const bytes = Array.from(new Uint8Array(digest)); + return bytes.map((b) => b.toString(16).padStart(2, '0')).join(''); +} + +async function probeUnsafeEvalInMain(tabId: number): Promise { + try { + const res = await chrome.scripting.executeScript({ + target: { tabId, allFrames: false }, + world: ExecutionWorld.MAIN, + func: () => { + try { + // If page CSP blocks unsafe-eval, this will throw + return !!new Function('return 1')(); + } catch { + return false; + } + }, + }); + return Array.isArray(res) && res[0] && (res[0] as any).result === true; + } catch { + return false; + } +} + // Basic TM header parser (subset) function parseUserscriptMeta(source: string): { meta: Record; @@ -305,6 +336,11 @@ function clearActiveInjection(tabId: number, id: string) { } async function reinjectForTab(tabId: number, url?: string) { + // Emergency global switch + const flag = (await chrome.storage.local.get([STORAGE_KEYS.USERSCRIPTS_DISABLED]))[ + STORAGE_KEYS.USERSCRIPTS_DISABLED + ]; + if (flag) return; const all = await loadAllRecords(); for (const rec of Object.values(all)) { if (!rec.enabled || !rec.persist) continue; @@ -314,6 +350,16 @@ async function reinjectForTab(tabId: number, url?: string) { await insertCssToTab(tabId, rec.script, rec.allFrames); setActiveInjection(tabId, rec.id, { kind: 'css' }); } else { + // Probe CSP when targeting MAIN + if (rec.world === 'MAIN') { + const ok = await probeUnsafeEvalInMain(tabId); + if (!ok) { + rec.cspBlocked = true; + await injectJsPersistent(tabId, rec.script, 'ISOLATED', rec.allFrames); + setActiveInjection(tabId, rec.id, { kind: 'js', world: 'ISOLATED' }); + continue; + } + } await injectJsPersistent(tabId, rec.script, rec.world, rec.allFrames); setActiveInjection(tabId, rec.id, { kind: 'js', world: rec.world }); } @@ -330,6 +376,49 @@ chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { } }); +// webNavigation based runAt mapping +chrome.webNavigation.onCommitted.addListener(async (details) => { + if (details.frameId !== 0) return; + const tab = await chrome.tabs.get(details.tabId).catch(() => null); + if (!tab) return; + const disabled = (await chrome.storage.local.get([STORAGE_KEYS.USERSCRIPTS_DISABLED]))[ + STORAGE_KEYS.USERSCRIPTS_DISABLED + ]; + if (disabled) return; + const all = await loadAllRecords(); + for (const rec of Object.values(all)) { + if (!rec.enabled || !rec.persist || rec.runAt !== 'document_start') continue; + if (!matchUrl(rec.matches, tab.url)) continue; + try { + if (rec.sourceType === 'CSS') await insertCssToTab(details.tabId, rec.script, rec.allFrames); + else await injectJsPersistent(details.tabId, rec.script, rec.world, rec.allFrames); + } catch { + // noop + } + } +}); + +chrome.webNavigation.onDOMContentLoaded.addListener(async (details) => { + if (details.frameId !== 0) return; + const tab = await chrome.tabs.get(details.tabId).catch(() => null); + if (!tab) return; + const disabled = (await chrome.storage.local.get([STORAGE_KEYS.USERSCRIPTS_DISABLED]))[ + STORAGE_KEYS.USERSCRIPTS_DISABLED + ]; + if (disabled) return; + const all = await loadAllRecords(); + for (const rec of Object.values(all)) { + if (!rec.enabled || !rec.persist || rec.runAt !== 'document_end') continue; + if (!matchUrl(rec.matches, tab.url)) continue; + try { + if (rec.sourceType === 'CSS') await insertCssToTab(details.tabId, rec.script, rec.allFrames); + else await injectJsPersistent(details.tabId, rec.script, rec.world, rec.allFrames); + } catch { + // noop + } + } +}); + class UserscriptTool extends BaseBrowserToolExecutor { name = TOOL_NAMES.BROWSER.USERSCRIPT; @@ -373,6 +462,10 @@ class UserscriptTool extends BaseBrowserToolExecutor { if (!active || !active.id) return createErrorResponse('No active tab found'); const currentUrl = active.url; + const emergency = (await chrome.storage.local.get([STORAGE_KEYS.USERSCRIPTS_DISABLED]))[ + STORAGE_KEYS.USERSCRIPTS_DISABLED + ]; + const { meta, isTM } = parseUserscriptMeta(args.script); const name = args.name || deriveName(meta, undefined); const description = args.description || pick(meta['description']); @@ -396,6 +489,7 @@ class UserscriptTool extends BaseBrowserToolExecutor { ? 'CSS' : 'JS'; + const sha256 = await computeSHA256(args.script).catch(() => undefined); const id = `us_${fnv1a((name || '') + '|' + args.script)}`; const record: UserscriptRecord = { @@ -416,53 +510,83 @@ class UserscriptTool extends BaseBrowserToolExecutor { createdAt: now(), updatedAt: now(), applyCount: 0, + sha256, }; const all = await loadAllRecords(); - all[id] = record; - await saveAllRecords(all); + if (record.persist) { + all[id] = record; + await saveAllRecords(all); + } // Apply to current tab immediately if matches let applied = false; const fallbacks: string[] = []; + let cspBlocked = false; + const t0 = performance.now(); try { - if (sourceType === 'CSS') { + if (mode === 'once') { + // Once: CDP evaluate in page + await cdpSessionManager.withSession(active.id!, 'userscript_once', async () => { + const expression = `(function(){try{return (function(){${record.script}\n})()}catch(e){return {__error:String(e&&e.message||e)}}})()`; + const result: any = await cdpSessionManager.sendCommand(active.id!, 'Runtime.evaluate', { + expression, + returnByValue: true, + awaitPromise: true, + }); + if (result?.result?.value?.__error) { + throw new Error(result.result.value.__error); + } + }); + applied = true; + } else if (sourceType === 'CSS') { await insertCssToTab(active.id!, record.script, record.allFrames); setActiveInjection(active.id!, id, { kind: 'css' }); applied = true; } else { - try { - await injectJsPersistent(active.id!, record.script, record.world, record.allFrames); - setActiveInjection(active.id!, id, { kind: 'js', world: record.world }); - applied = true; - } catch (e) { - // Try fallback to ISOLATED if MAIN blocked by CSP - if (record.world === 'MAIN') { + // Probe CSP preflight when target MAIN + if (record.world === 'MAIN') { + const ok = await probeUnsafeEvalInMain(active.id!); + if (!ok) { + cspBlocked = true; fallbacks.push('MAIN->ISOLATED'); await injectJsPersistent(active.id!, record.script, 'ISOLATED', record.allFrames); setActiveInjection(active.id!, id, { kind: 'js', world: 'ISOLATED' }); applied = true; - } else { - throw e; } } + if (!applied) { + await injectJsPersistent(active.id!, record.script, record.world, record.allFrames); + setActiveInjection(active.id!, id, { kind: 'js', world: record.world }); + applied = true; + } } } catch (e) { - all[id].lastError = e instanceof Error ? e.message : String(e); - await saveAllRecords(all); + if (record.persist) { + all[id].lastError = e instanceof Error ? e.message : String(e); + all[id].cspBlocked = cspBlocked; + await saveAllRecords(all); + } } const result = { id, - status: all[id].lastError ? 'queued' : applied ? 'applied' : 'queued', + status: record.persist && all[id]?.lastError ? 'queued' : applied ? 'applied' : 'queued', strategy: { - kind: sourceType === 'CSS' ? 'insertCSS' : `persistent_${all[id].world.toLowerCase()}`, - runAt: all[id].runAt, - world: all[id].world, - allFrames: all[id].allFrames, + kind: + mode === 'once' + ? 'once_cdp' + : sourceType === 'CSS' + ? 'insertCSS' + : `persistent_${(record.persist ? all[id]?.world || record.world : record.world).toLowerCase()}`, + runAt: record.persist ? all[id]?.runAt || record.runAt : record.runAt, + world: record.persist ? all[id]?.world || record.world : record.world, + allFrames: record.persist ? (all[id]?.allFrames ?? record.allFrames) : record.allFrames, fallbacksTried: fallbacks, + cspBlocked, }, - warnings: [], + warnings: emergency ? ['USERSCRIPTS_DISABLED is ON, injection skipped'] : [], + metrics: { injectMs: Math.round(performance.now() - t0) }, }; return { @@ -471,20 +595,34 @@ class UserscriptTool extends BaseBrowserToolExecutor { }; } - private async list(_args: any): Promise { + private async list(args: any): Promise { const all = await loadAllRecords(); - const items = Object.values(all).map((r) => ({ - id: r.id, - name: r.name, - status: r.enabled ? 'enabled' : 'disabled', - sourceType: r.sourceType, - matches: r.matches, - world: r.world, - runAt: r.runAt, - tags: r.tags || [], - lastError: r.lastError, - updatedAt: r.updatedAt, - })); + const q = (args && args.query ? String(args.query).toLowerCase() : '').trim(); + const status = args && args.status ? String(args.status) : ''; + const domain = args && args.domain ? String(args.domain) : ''; + const items = Object.values(all) + .filter((r) => (status ? (status === 'enabled' ? r.enabled : !r.enabled) : true)) + .filter((r) => (domain ? matchUrl(r.matches, `https://${domain}/`) : true)) + .filter((r) => + q + ? (r.name || '').toLowerCase().includes(q) || + (r.description || '').toLowerCase().includes(q) + : true, + ) + .map((r) => ({ + id: r.id, + name: r.name, + status: r.enabled ? 'enabled' : 'disabled', + sourceType: r.sourceType, + matches: r.matches, + world: r.world, + runAt: r.runAt, + tags: r.tags || [], + lastError: r.lastError, + updatedAt: r.updatedAt, + applyCount: r.applyCount || 0, + lastAppliedAt: r.lastAppliedAt || null, + })); return { content: [{ type: 'text', text: JSON.stringify({ ok: true, items }) }], isError: false, diff --git a/app/chrome-extension/wxt.config.ts b/app/chrome-extension/wxt.config.ts index dd0de345..dfcda6cf 100644 --- a/app/chrome-extension/wxt.config.ts +++ b/app/chrome-extension/wxt.config.ts @@ -38,11 +38,13 @@ export default defineConfig({ 'scripting', 'downloads', 'webRequest', + 'webNavigation', 'debugger', 'history', 'bookmarks', 'offscreen', 'storage', + 'declarativeNetRequest', ], host_permissions: [''], web_accessible_resources: [ From 09e07f276be2258a9fc4447cd7de1e5022b3d7ad Mon Sep 17 00:00:00 2001 From: hangwin Date: Thu, 9 Oct 2025 11:06:23 +0000 Subject: [PATCH 07/71] feat(extension/options): add Userscripts Manager UI and tool-call bridge Register options page via manifest (wxt.config.ts). Add options page (index.html, main.ts, App.vue). Support create/list/toggle/delete/export userscripts. Bridge UI to tools: handle 'call_tool' in background. Lint/typecheck/build passed. --- .../entrypoints/background/native-host.ts | 10 + .../entrypoints/options/App.vue | 387 ++++++++++++++++++ .../entrypoints/options/index.html | 12 + .../entrypoints/options/main.ts | 4 + app/chrome-extension/wxt.config.ts | 4 + 5 files changed, 417 insertions(+) create mode 100644 app/chrome-extension/entrypoints/options/App.vue create mode 100644 app/chrome-extension/entrypoints/options/index.html create mode 100644 app/chrome-extension/entrypoints/options/main.ts diff --git a/app/chrome-extension/entrypoints/background/native-host.ts b/app/chrome-extension/entrypoints/background/native-host.ts index 72a9df39..41696ff6 100644 --- a/app/chrome-extension/entrypoints/background/native-host.ts +++ b/app/chrome-extension/entrypoints/background/native-host.ts @@ -180,6 +180,16 @@ export const initNativeHostListener = () => { chrome.runtime.onStartup.addListener(connectNativeHost); chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + // Allow UI to call tools directly + if (message && message.type === 'call_tool' && message.name) { + handleCallTool({ name: message.name, args: message.args }) + .then((res) => sendResponse({ success: true, result: res })) + .catch((err) => + sendResponse({ success: false, error: err instanceof Error ? err.message : String(err) }), + ); + return true; + } + if ( message === NativeMessageType.CONNECT_NATIVE || message.type === NativeMessageType.CONNECT_NATIVE diff --git a/app/chrome-extension/entrypoints/options/App.vue b/app/chrome-extension/entrypoints/options/App.vue new file mode 100644 index 00000000..0a2f0f18 --- /dev/null +++ b/app/chrome-extension/entrypoints/options/App.vue @@ -0,0 +1,387 @@ + @@ -310,6 +334,9 @@ import { getMessage } from '@/utils/i18n'; import ConfirmDialog from './components/ConfirmDialog.vue'; import ProgressIndicator from './components/ProgressIndicator.vue'; import ModelCacheManagement from './components/ModelCacheManagement.vue'; +import FlowEditor from './components/FlowEditor.vue'; +import BuilderEditor from './components/BuilderEditor.vue'; +import ScheduleDialog from './components/ScheduleDialog.vue'; import { DocumentIcon, DatabaseIcon, @@ -329,6 +356,15 @@ const filteredRrFlows = computed(() => rrOnlyBound.value ? rrFlows.value.filter(isFlowBoundToCurrent) : rrFlows.value, ); +// Flow editor state +const showFlowEditor = ref(false); +const editingFlow = ref(null); +const showBuilderEditor = ref(false); +const editingFlowBuilder = ref(null); +const showSchedule = ref(false); +const schedulingFlowId = ref(null); +const schedules = ref([]); + const loadFlows = async () => { try { const res = await chrome.runtime.sendMessage({ type: BACKGROUND_MESSAGE_TYPES.RR_LIST_FLOWS }); @@ -420,6 +456,93 @@ const deleteFlow = async (flowId: string) => { } }; +function editFlow(flow: any) { + editingFlow.value = flow; + showFlowEditor.value = true; +} + +function openBuilder(flow: any) { + editingFlowBuilder.value = flow; + showBuilderEditor.value = true; +} + +async function saveEditedFlow(f: any) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_SAVE_FLOW, + flow: f, + }); + if (res && res.success) { + showFlowEditor.value = false; + editingFlow.value = null; + await loadFlows(); + } + } catch (e) { + console.error('保存失败:', e); + } +} + +async function saveEditedFlowFromBuilder(f: any) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_SAVE_FLOW, + flow: f, + }); + if (res && res.success) { + showBuilderEditor.value = false; + editingFlowBuilder.value = null; + await loadFlows(); + } + } catch (e) { + console.error('保存失败:', e); + } +} + +async function openSchedule(flowId: string) { + schedulingFlowId.value = flowId; + await loadSchedules(); + showSchedule.value = true; +} + +async function loadSchedules() { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_LIST_SCHEDULES, + }); + if (res && res.success) schedules.value = res.schedules || []; + } catch (e) { + console.error('加载定时失败:', e); + } +} + +async function saveSchedule(schedule: any) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_SCHEDULE_FLOW, + schedule, + }); + if (res && res.success) { + await loadSchedules(); + showSchedule.value = false; + schedulingFlowId.value = null; + } + } catch (e) { + console.error('保存定时失败:', e); + } +} + +async function removeSchedule(id: string) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_UNSCHEDULE_FLOW, + scheduleId: id, + }); + if (res && res.success) await loadSchedules(); + } catch (e) { + console.error('删除计划失败:', e); + } +} + const nativeConnectionStatus = ref<'unknown' | 'connected' | 'disconnected'>('unknown'); const isConnecting = ref(false); const nativeServerPort = ref(12306); diff --git a/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue b/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue new file mode 100644 index 00000000..c8b76f94 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue @@ -0,0 +1,416 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue b/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue new file mode 100644 index 00000000..0a78b672 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue @@ -0,0 +1,363 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue b/app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue new file mode 100644 index 00000000..0d4a0fa7 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue @@ -0,0 +1,233 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue new file mode 100644 index 00000000..78e683e9 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue @@ -0,0 +1,202 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue new file mode 100644 index 00000000..d3b3982e --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue @@ -0,0 +1,80 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue new file mode 100644 index 00000000..14d2b3b9 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue @@ -0,0 +1,442 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue new file mode 100644 index 00000000..499bb885 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue @@ -0,0 +1,70 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts b/app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts new file mode 100644 index 00000000..8cbb6209 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts @@ -0,0 +1,182 @@ +import type { + Flow as FlowV2, + NodeBase, + Edge as EdgeV2, +} from '@/entrypoints/background/record-replay/types'; + +export function newId(prefix: string) { + return `${prefix}_${Math.random().toString(36).slice(2, 8)}`; +} + +export type NodeType = NodeBase['type']; + +export function defaultConfigFor(t: NodeType): any { + if (t === 'click' || t === 'fill') + return { target: { candidates: [] }, value: t === 'fill' ? '' : undefined }; + if (t === 'navigate') return { url: '' }; + if (t === 'wait') return { condition: { text: '', appear: true } }; + if (t === 'assert') return { assert: { exists: '' } }; + if (t === 'key') return { keys: '' }; + if (t === 'delay') return { ms: 1000 }; + if (t === 'http') return { method: 'GET', url: '', headers: {}, body: null, saveAs: '' }; + if (t === 'extract') return { selector: '', attr: 'text', js: '', saveAs: '' }; + if (t === 'openTab') return { url: '', newWindow: false }; + if (t === 'switchTab') return { tabId: null, urlContains: '', titleContains: '' }; + if (t === 'closeTab') return { tabIds: [], url: '' }; + if (t === 'script') return { world: 'ISOLATED', code: '', saveAs: '', assign: {} }; + return {}; +} + +export function stepsToNodes(steps: any[]): NodeBase[] { + const arr: NodeBase[] = []; + steps.forEach((s, i) => { + const id = s.id || newId(String(s.type || 'step')); + const node: NodeBase = { + id, + type: (s.type || 'script') as NodeType, + name: '', + disabled: false, + ui: { x: 200, y: 120 + i * 120 }, + config: mapStepToConfig(s), + }; + arr.push(node); + }); + return arr; +} + +export function mapStepToConfig(s: any) { + const t = s.type; + if (t === 'click' || t === 'dblclick') + return { target: s.target || { candidates: [] }, after: s.after, before: s.before }; + if (t === 'fill') return { target: s.target || { candidates: [] }, value: s.value || '' }; + if (t === 'wait') return { condition: s.condition || { text: '', appear: true } }; + if (t === 'assert') return { assert: s.assert || { exists: '' }, failStrategy: s.failStrategy }; + if (t === 'navigate') return { url: s.url || '' }; + if (t === 'script') return { world: s.world || 'ISOLATED', code: s.code || '' }; + return { ...s }; +} + +export function mapConfigToStep(n: NodeBase) { + const base = { id: n.id, type: n.type } as any; + const c = n.config || {}; + if (n.type === 'click' || n.type === 'dblclick') + return { ...base, target: c.target || { candidates: [] }, after: c.after, before: c.before }; + if (n.type === 'fill') + return { ...base, target: c.target || { candidates: [] }, value: c.value || '' }; + if (n.type === 'key') return { ...base, keys: c.keys || '' }; + if (n.type === 'wait') return { ...base, condition: c.condition || { text: '', appear: true } }; + if (n.type === 'assert') + return { ...base, assert: c.assert || { exists: '' }, failStrategy: c.failStrategy }; + if (n.type === 'navigate') return { ...base, url: c.url || '' }; + if (n.type === 'delay') + return { + ...base, + type: 'wait', + timeoutMs: Math.max(0, Number(c.ms ?? 1000)), + condition: { navigation: true }, + }; + if (n.type === 'http') + return { + ...base, + type: 'http', + method: c.method || 'GET', + url: c.url || '', + headers: c.headers || {}, + body: c.body, + saveAs: c.saveAs || '', + } as any; + if (n.type === 'extract') + return { + ...base, + type: 'extract', + selector: c.selector || '', + attr: c.attr || 'text', + js: c.js || '', + saveAs: c.saveAs || '', + } as any; + if (n.type === 'openTab') + return { ...base, type: 'openTab', url: c.url || '', newWindow: !!c.newWindow } as any; + if (n.type === 'switchTab') + return { + ...base, + type: 'switchTab', + tabId: c.tabId || undefined, + urlContains: c.urlContains || '', + titleContains: c.titleContains || '', + } as any; + if (n.type === 'closeTab') + return { + ...base, + type: 'closeTab', + tabIds: Array.isArray(c.tabIds) ? c.tabIds : undefined, + url: c.url || '', + } as any; + if (n.type === 'script') + return { + ...base, + world: c.world || 'ISOLATED', + code: c.code || '', + when: c.when, + saveAs: c.saveAs || '', + assign: c.assign || {}, + } as any; + return { ...base }; +} + +export function topoOrder(nodes: NodeBase[], edges: EdgeV2[]): NodeBase[] { + const id2n = new Map(nodes.map((n) => [n.id, n] as const)); + const indeg = new Map(nodes.map((n) => [n.id, 0] as const)); + for (const e of edges) + if (!e.label || e.label === 'default') indeg.set(e.to, (indeg.get(e.to) || 0) + 1); + const q: string[] = nodes.filter((n) => (indeg.get(n.id) || 0) === 0).map((n) => n.id); + const out: NodeBase[] = []; + const nexts = new Map(nodes.map((n) => [n.id, [] as string[]] as const)); + for (const e of edges) if (!e.label || e.label === 'default') nexts.get(e.from)!.push(e.to); + while (q.length) { + const id = q.shift()!; + const n = id2n.get(id); + if (!n) continue; + out.push(n); + for (const v of nexts.get(id)!) { + indeg.set(v, (indeg.get(v) || 0) - 1); + if ((indeg.get(v) || 0) === 0) q.push(v); + } + } + if (out.length === nodes.length) return out; + return nodes.slice(); +} + +export function nodesToSteps(nodes: NodeBase[], edges: EdgeV2[]): any[] { + const order = edges.length ? topoOrder(nodes, edges) : nodes.slice(); + return order.map((n) => mapConfigToStep(n)); +} + +export function autoChainEdges(nodes: NodeBase[]): EdgeV2[] { + const arr: EdgeV2[] = []; + for (let i = 0; i < nodes.length - 1; i++) + arr.push({ id: newId('e'), from: nodes[i].id, to: nodes[i + 1].id, label: 'default' }); + return arr; +} + +export function summarizeNode(n?: NodeBase | null): string { + if (!n) return ''; + if (n.type === 'click' || n.type === 'fill') + return n.config?.target?.candidates?.[0]?.value || '未配置选择器'; + if (n.type === 'navigate') return n.config?.url || ''; + if (n.type === 'key') return n.config?.keys || ''; + if (n.type === 'delay') return `${Number(n.config?.ms || 0)}ms`; + if (n.type === 'http') return `${n.config?.method || 'GET'} ${n.config?.url || ''}`; + if (n.type === 'extract') return `${n.config?.selector || ''} -> ${n.config?.saveAs || ''}`; + if (n.type === 'openTab') return `open ${n.config?.url || ''}`; + if (n.type === 'switchTab') + return `switch ${n.config?.tabId || n.config?.urlContains || n.config?.titleContains || ''}`; + if (n.type === 'closeTab') return `close ${n.config?.url || ''}`; + if (n.type === 'wait') return JSON.stringify(n.config?.condition || {}); + if (n.type === 'assert') return JSON.stringify(n.config?.assert || {}); + if (n.type === 'script') return (n.config?.code || '').slice(0, 30); + return ''; +} + +export function cloneFlow(flow: FlowV2): FlowV2 { + return JSON.parse(JSON.stringify(flow)); +} diff --git a/app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts b/app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts new file mode 100644 index 00000000..cb24f373 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts @@ -0,0 +1,86 @@ +import type { NodeBase } from '@/entrypoints/background/record-replay/types'; + +export function validateNode(n: NodeBase): string[] { + const errs: string[] = []; + if (n.disabled) return errs; // 忽略禁用节点 + const c: any = n.config || {}; + + switch (n.type) { + case 'click': + case 'dblclick': + case 'fill': { + const hasCandidate = !!c?.target?.candidates?.length; + if (!hasCandidate) errs.push('缺少目标选择器候选'); + if (n.type === 'fill' && (!('value' in c) || c.value === undefined)) errs.push('缺少输入值'); + break; + } + case 'wait': { + if (!c?.condition) errs.push('缺少等待条件'); + break; + } + case 'assert': { + if (!c?.assert) errs.push('缺少断言条件'); + break; + } + case 'navigate': { + if (!c?.url) errs.push('缺少 URL'); + break; + } + case 'http': { + if (!c?.url) errs.push('HTTP: 缺少 URL'); + if (c?.assign && typeof c.assign === 'object') { + const pathRe = /^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+|\[\d+\])*$/; + for (const v of Object.values(c.assign)) { + const s = String(v); + if (!pathRe.test(s)) errs.push(`Assign: 路径非法 ${s}`); + } + } + break; + } + case 'extract': { + if (!c?.saveAs) errs.push('Extract: 需填写保存变量名'); + if (!c?.selector && !c?.js) errs.push('Extract: 需提供 selector 或 js'); + break; + } + case 'switchTab': { + if (!c?.tabId && !c?.urlContains && !c?.titleContains) + errs.push('SwitchTab: 需提供 tabId 或 URL/标题包含'); + break; + } + case 'closeTab': { + // 允许空（关闭当前标签页），不强制 + break; + } + case 'script': { + // 若配置了 saveAs/assign，应提供 code + const hasAssign = c?.assign && Object.keys(c.assign).length > 0; + if ((c?.saveAs || hasAssign) && !String(c?.code || '').trim()) + errs.push('Script: 配置了保存/映射但缺少代码'); + if (hasAssign) { + const pathRe = /^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+|\[\d+\])*$/; + for (const v of Object.values(c.assign || {})) { + const s = String(v); + if (!pathRe.test(s)) errs.push(`Assign: 路径非法 ${s}`); + } + } + break; + } + } + return errs; +} + +export function validateFlow(nodes: NodeBase[]): { + totalErrors: number; + nodeErrors: Record; +} { + const nodeErrors: Record = {}; + let totalErrors = 0; + for (const n of nodes) { + const e = validateNode(n); + if (e.length) { + nodeErrors[n.id] = e; + totalErrors += e.length; + } + } + return { totalErrors, nodeErrors }; +} diff --git a/app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts b/app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts new file mode 100644 index 00000000..87f6b71b --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts @@ -0,0 +1,251 @@ +import { reactive, ref } from 'vue'; +import type { + Flow as FlowV2, + NodeBase, + Edge as EdgeV2, +} from '@/entrypoints/background/record-replay/types'; +import { + autoChainEdges, + cloneFlow, + defaultConfigFor, + newId, + nodesToSteps, + stepsToNodes, + summarizeNode, + topoOrder, +} from '../model/transforms'; + +export function useBuilderStore(initial?: FlowV2 | null) { + const flowLocal = reactive({ id: '', name: '', version: 1, steps: [], variables: [] }); + const nodes = reactive([]); + const edges = reactive([]); + const activeNodeId = ref(null); + const pendingFrom = ref(null); + const paletteTypes = [ + 'click', + 'fill', + 'key', + 'wait', + 'assert', + 'navigate', + 'script', + 'delay', + 'http', + 'extract', + 'openTab', + 'switchTab', + 'closeTab', + ] as NodeBase['type'][]; + + // --- history (undo/redo) --- + type Snapshot = { + flow: Pick; + nodes: NodeBase[]; + edges: EdgeV2[]; + }; + const HISTORY_MAX = 50; + const past: Snapshot[] = []; + const future: Snapshot[] = []; + function takeSnapshot(): Snapshot { + return { + flow: { name: flowLocal.name, description: flowLocal.description } as any, + nodes: JSON.parse(JSON.stringify(nodes)), + edges: JSON.parse(JSON.stringify(edges)), + }; + } + function applySnapshot(s: Snapshot) { + flowLocal.name = (s.flow as any).name || ''; + (flowLocal as any).description = (s.flow as any).description || ''; + nodes.splice(0, nodes.length, ...JSON.parse(JSON.stringify(s.nodes))); + edges.splice(0, edges.length, ...JSON.parse(JSON.stringify(s.edges))); + } + function recordChange() { + past.push(takeSnapshot()); + // clear redo stack on new change + future.length = 0; + if (past.length > HISTORY_MAX) past.splice(0, past.length - HISTORY_MAX); + } + function undo() { + if (past.length === 0) return; + const current = takeSnapshot(); + const prev = past.pop()!; + future.push(current); + applySnapshot(prev); + } + function redo() { + if (future.length === 0) return; + const current = takeSnapshot(); + const next = future.pop()!; + past.push(current); + applySnapshot(next); + } + + function layoutIfNeeded() { + const startX = 120, + startY = 80, + gapY = 120; + nodes.forEach((n, i) => { + if (!n.ui || isNaN(n.ui.x) || isNaN(n.ui.y)) n.ui = { x: startX, y: startY + i * gapY }; + }); + } + + function initFromFlow(flow: FlowV2) { + const deep = cloneFlow(flow); + Object.assign(flowLocal, deep); + nodes.splice( + 0, + nodes.length, + ...(Array.isArray(deep.nodes) && deep.nodes.length + ? deep.nodes + : stepsToNodes(deep.steps || [])), + ); + edges.splice( + 0, + edges.length, + ...(Array.isArray(deep.edges) && deep.edges.length ? deep.edges : autoChainEdges(nodes)), + ); + layoutIfNeeded(); + activeNodeId.value = nodes[0]?.id || null; + // reset history + past.length = 0; + future.length = 0; + past.push(takeSnapshot()); + } + + function selectNode(id: string) { + if (pendingFrom.value && pendingFrom.value !== id) { + onConnect(pendingFrom.value, id); + pendingFrom.value = null; + } + activeNodeId.value = id; + } + + function addNode(t: NodeBase['type']) { + const id = newId(t); + const n: NodeBase = { + id, + type: t, + name: '', + disabled: false, + config: defaultConfigFor(t), + ui: { x: 200 + nodes.length * 24, y: 120 + nodes.length * 96 }, + }; + nodes.push(n); + if (nodes.length > 1) { + const prev = nodes[nodes.length - 2]; + edges.push({ id: newId('e'), from: prev.id, to: id, label: 'default' }); + } + activeNodeId.value = id; + recordChange(); + } + + function duplicateNode(id: string) { + const src = nodes.find((n) => n.id === id); + if (!src) return; + const cp: NodeBase = JSON.parse(JSON.stringify(src)); + cp.id = newId(src.type); + cp.name = src.name ? `${src.name} Copy` : ''; + const baseX = cp.ui && typeof cp.ui.x === 'number' ? cp.ui.x : 200; + const baseY = cp.ui && typeof cp.ui.y === 'number' ? cp.ui.y : 120; + cp.ui = { x: baseX + 40, y: baseY + 40 }; + nodes.push(cp); + activeNodeId.value = cp.id; + recordChange(); + } + + function removeNode(id: string) { + const idx = nodes.findIndex((n) => n.id === id); + if (idx < 0) return; + nodes.splice(idx, 1); + for (let i = edges.length - 1; i >= 0; i--) { + const e = edges[i]; + if (e.from === id || e.to === id) edges.splice(i, 1); + } + activeNodeId.value = nodes[Math.min(idx, nodes.length - 1)]?.id || null; + recordChange(); + } + + function setNodePosition(id: string, x: number, y: number) { + const n = nodes.find((n) => n.id === id); + if (!n) return; + n.ui = { x: Math.round(x), y: Math.round(y) }; + // 不计入历史栈，避免频繁记录；由用户触发操作（连接/新增/删除等）记录。 + } + + function connectFrom(id: string) { + pendingFrom.value = id; + } + + function onConnect(sourceId: string, targetId: string) { + // 单一默认出边：删除同源 default 出边 + for (let i = edges.length - 1; i >= 0; i--) { + const e = edges[i]; + if (e.from === sourceId && (!e.label || e.label === 'default')) edges.splice(i, 1); + } + edges.push({ id: newId('e'), from: sourceId, to: targetId, label: 'default' }); + recordChange(); + } + + function importFromSteps() { + const arr = stepsToNodes(flowLocal.steps || []); + nodes.splice(0, nodes.length, ...arr); + edges.splice(0, edges.length, ...autoChainEdges(arr)); + layoutIfNeeded(); + recordChange(); + } + + function exportSteps() { + return nodesToSteps(nodes, edges); + } + + function summarize(id?: string) { + const n = nodes.find((x) => x.id === id); + return summarizeNode(n || null); + } + + // 自动排版：根据拓扑顺序纵向排列，列宽 300、行高 120；若存在分叉，简单按顺序换行 + function layoutAuto() { + const order = topoOrder(nodes, edges); + const startX = 120, + startY = 80, + stepY = 120, + stepX = 300, + maxPerCol = Math.max(6, Math.ceil(order.length / 3)); + let col = 0, + row = 0; + for (const n of order) { + n.ui = { x: startX + col * stepX, y: startY + row * stepY } as any; + row++; + if (row >= maxPerCol) { + row = 0; + col++; + } + } + recordChange(); + } + + if (initial) initFromFlow(initial); + + return { + flowLocal, + nodes, + edges, + activeNodeId, + pendingFrom, + paletteTypes, + undo, + redo, + initFromFlow, + selectNode, + addNode, + duplicateNode, + removeNode, + setNodePosition, + connectFrom, + onConnect, + importFromSteps, + exportSteps, + summarize, + layoutAuto, + }; +} diff --git a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js index ae7eee57..7fa4fa0c 100644 --- a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js +++ b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js @@ -537,23 +537,61 @@ const sel = String(request.selector || '').trim(); let el = null; if (useText && textQuery) { - const all = Array.from(document.querySelectorAll('body *')); - for (const node of all) { + const normalize = (s) => + String(s || '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + const query = normalize(textQuery); + const bigrams = (s) => { + const arr = []; + for (let i = 0; i < s.length - 1; i++) arr.push(s.slice(i, i + 2)); + return arr; + }; + const dice = (a, b) => { + if (!a || !b) return 0; + const A = bigrams(a); + const B = bigrams(b); + if (A.length === 0 || B.length === 0) return 0; + let inter = 0; + const map = new Map(); + for (const t of A) map.set(t, (map.get(t) || 0) + 1); + for (const t of B) { + const c = map.get(t) || 0; + if (c > 0) { + inter++; + map.set(t, c - 1); + } + } + return (2 * inter) / (A.length + B.length); + }; + let best = { el: null, score: 0 }; + const walker = document.createTreeWalker( + document.body || document.documentElement, + NodeFilter.SHOW_ELEMENT, + ); + let visited = 0; + while (walker.nextNode()) { + const node = /** @type {Element} */ (walker.currentNode); try { const cs = window.getComputedStyle(node); if (cs.display === 'none' || cs.visibility === 'hidden' || cs.opacity === '0') continue; const rect = /** @type {HTMLElement} */ (node).getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) continue; - const txt = (node.textContent || '').trim(); - if (txt && txt.includes(textQuery)) { + const txt = normalize(node.textContent || ''); + if (!txt) continue; + // quick path: substring contains + if (txt.includes(query)) { el = node; break; } - } catch (_) { - /* ignore */ - } + const sc = dice(txt, query); + if (sc > best.score) best = { el: node, score: sc }; + } catch {} + if (++visited > 5000) break; } + if (!el && best.el && best.score >= 0.6) el = best.el; } else { if (!sel) { sendResponse({ success: false, error: 'selector is required' }); @@ -625,21 +663,136 @@ } if (request && request.action === 'collectVariables') { try { - const vars = Array.isArray(request.variables) ? request.variables : []; + let vars = Array.isArray(request.variables) ? request.variables : []; + if ((!vars || vars.length === 0) && request.payload) { + try { + const p = JSON.parse(String(request.payload || '{}')); + if (Array.isArray(p.variables)) vars = p.variables; + } catch {} + } + const useOverlay = request.useOverlay !== false; // default true const values = {}; + if (!useOverlay) { + for (const v of vars) { + const key = String(v && v.key ? v.key : ''); + if (!key) continue; + const label = v.label || key; + const def = v.default || ''; + const promptText = `请输入参数 ${label} (${key})`; + let val = window.prompt(promptText, def); + if (typeof val !== 'string') val = def; + values[key] = val; + } + sendResponse({ success: true, values }); + return true; + } + // Build overlay form + const hostId = '__rr_var_overlay__'; + let host = document.getElementById(hostId); + if (host) host.remove(); + host = document.createElement('div'); + host.id = hostId; + Object.assign(host.style, { + position: 'fixed', + inset: '0', + background: 'rgba(0,0,0,0.35)', + zIndex: 2147483646, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + }); + const panel = document.createElement('div'); + Object.assign(panel.style, { + background: '#fff', + borderRadius: '8px', + width: 'min(520px, 96vw)', + maxHeight: '80vh', + overflow: 'auto', + boxShadow: '0 8px 24px rgba(0,0,0,0.2)', + padding: '16px', + fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif', + }); + const title = document.createElement('div'); + title.textContent = '请输入回放参数'; + Object.assign(title.style, { fontSize: '16px', fontWeight: '600', marginBottom: '12px' }); + const form = document.createElement('form'); for (const v of vars) { - const key = String(v && v.key ? v.key : ''); - if (!key) continue; - const label = v.label || key; - const def = v.default || ''; - const promptText = `请输入参数 ${label} (${key})`; - // Note: prompt in page context; in some sites may be blocked by CSP - let val = window.prompt(promptText, def); - if (typeof val !== 'string') val = def; - values[key] = val; + const row = document.createElement('div'); + Object.assign(row.style, { marginBottom: '10px' }); + const label = document.createElement('label'); + label.textContent = `${v.label || v.key}${v.sensitive ? ' (敏感)' : ''}`; + Object.assign(label.style, { + display: 'block', + marginBottom: '6px', + fontWeight: '500', + }); + const input = document.createElement('input'); + input.type = v.sensitive ? 'password' : 'text'; + input.name = String(v.key); + input.value = String(v.default || ''); + Object.assign(input.style, { + width: '100%', + boxSizing: 'border-box', + padding: '8px 10px', + border: '1px solid #d0d7de', + borderRadius: '6px', + outline: 'none', + }); + row.appendChild(label); + row.appendChild(input); + form.appendChild(row); } - sendResponse({ success: true, values }); - return true; + const actions = document.createElement('div'); + Object.assign(actions.style, { display: 'flex', gap: '8px', marginTop: '12px' }); + const ok = document.createElement('button'); + ok.type = 'submit'; + ok.textContent = '确定'; + Object.assign(ok.style, { + background: '#0969da', + color: '#fff', + border: 'none', + padding: '8px 16px', + borderRadius: '6px', + cursor: 'pointer', + }); + const cancel = document.createElement('button'); + cancel.type = 'button'; + cancel.textContent = '取消'; + Object.assign(cancel.style, { + background: '#f3f4f6', + color: '#111', + border: '1px solid #d0d7de', + padding: '8px 16px', + borderRadius: '6px', + cursor: 'pointer', + }); + actions.appendChild(ok); + actions.appendChild(cancel); + panel.appendChild(title); + panel.appendChild(form); + panel.appendChild(actions); + host.appendChild(panel); + document.documentElement.appendChild(host); + + const cleanup = () => { + try { + host.remove(); + } catch {} + }; + cancel.onclick = () => { + cleanup(); + sendResponse({ success: false, cancelled: true }); + }; + form.onsubmit = (e) => { + e.preventDefault(); + for (const v of vars) { + const el = form.querySelector(`input[name="${CSS.escape(String(v.key))}"]`); + if (el) values[v.key] = /** @type {HTMLInputElement} */ (el).value; + } + cleanup(); + sendResponse({ success: true, values }); + }; + return true; // async } catch (e) { sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); return true; diff --git a/app/chrome-extension/inject-scripts/recorder.js b/app/chrome-extension/inject-scripts/recorder.js index e0a10f95..89cbb0bd 100644 --- a/app/chrome-extension/inject-scripts/recorder.js +++ b/app/chrome-extension/inject-scripts/recorder.js @@ -11,6 +11,9 @@ const sampledDrag = []; let isRecording = false; + let isPaused = false; + let hideInputValues = false; + let highlightBox = null; let pendingFlow = { id: `flow_${Date.now()}`, name: '未命名录制', @@ -104,7 +107,7 @@ } function onClick(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; const el = e.target instanceof Element ? e.target : null; if (!el) return; const target = buildTarget(el); @@ -112,14 +115,15 @@ } function onInput(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; const el = e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement ? e.target : null; if (!el) return; const target = buildTarget(el); - const isSensitive = SENSITIVE_INPUT_TYPES.has((el.getAttribute('type') || '').toLowerCase()); + const isSensitive = + hideInputValues || SENSITIVE_INPUT_TYPES.has((el.getAttribute('type') || '').toLowerCase()); let value = el.value || ''; if (isSensitive) { const varKey = el.name ? el.name : `var_${Math.random().toString(36).slice(2, 6)}`; @@ -130,7 +134,7 @@ } function onKeydown(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; // modifier+key or Enter/Backspace etc const mods = []; if (e.ctrlKey) mods.push('ctrl'); @@ -144,22 +148,11 @@ pushStep({ type: 'key', keys, screenshotOnFail: false }); } - function onKeyup(e) { - if (!isRecording) return; - const mods = []; - if (e.ctrlKey) mods.push('ctrl'); - if (e.metaKey) mods.push('cmd'); - if (e.altKey) mods.push('alt'); - if (e.shiftKey) mods.push('shift'); - let keyToken = e.key || ''; - keyToken = keyToken.length === 1 ? keyToken.toLowerCase() : keyToken.toLowerCase(); - const keys = mods.length ? `${mods.join('+')}+${keyToken}` : keyToken; - pushStep({ type: 'key', keys, screenshotOnFail: false }); - } + // keyup 不再记录，避免重复噪声 // Composition IME events (record markers for analysis; playback is no-op via script step) function onCompositionStart() { - if (!isRecording) return; + if (!isRecording || isPaused) return; pushStep({ type: 'script', world: 'ISOLATED', @@ -169,7 +162,7 @@ }); } function onCompositionEnd() { - if (!isRecording) return; + if (!isRecording || isPaused) return; pushStep({ type: 'script', world: 'ISOLATED', @@ -181,7 +174,7 @@ let lastScrollAt = 0; function onScroll(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; const nowTs = now(); if (nowTs - lastScrollAt < THROTTLE_SCROLL_MS) return; lastScrollAt = nowTs; @@ -227,7 +220,7 @@ document.addEventListener('change', onInput, true); document.addEventListener('input', onInput, true); document.addEventListener('keydown', onKeydown, true); - document.addEventListener('keyup', onKeyup, true); + // document.addEventListener('keyup', onKeyup, true); document.addEventListener('compositionstart', onCompositionStart, true); document.addEventListener('compositionend', onCompositionEnd, true); window.addEventListener('scroll', onScroll, { passive: true }); @@ -241,7 +234,7 @@ document.removeEventListener('change', onInput, true); document.removeEventListener('input', onInput, true); document.removeEventListener('keydown', onKeydown, true); - document.removeEventListener('keyup', onKeyup, true); + // document.removeEventListener('keyup', onKeyup, true); document.removeEventListener('compositionstart', onCompositionStart, true); document.removeEventListener('compositionend', onCompositionEnd, true); window.removeEventListener('scroll', onScroll, { passive: true }); @@ -269,7 +262,9 @@ function start(flowMeta) { reset(flowMeta || {}); isRecording = true; + isPaused = false; attach(); + ensureOverlay(); chrome.runtime.sendMessage({ type: 'rr_recorder_event', payload: { kind: 'start', flow: pendingFlow }, @@ -279,6 +274,7 @@ function stop() { isRecording = false; detach(); + removeOverlay(); chrome.runtime.sendMessage({ type: 'rr_recorder_event', payload: { kind: 'stop', flow: pendingFlow }, @@ -286,6 +282,101 @@ return pendingFlow; } + function pause() { + isPaused = true; + updateOverlayStatus(); + } + + function resume() { + isRecording = true; + isPaused = false; + attach(); + ensureOverlay(); + updateOverlayStatus(); + } + + function ensureOverlay() { + let root = document.getElementById('__rr_rec_overlay'); + if (root) return; + root = document.createElement('div'); + root.id = '__rr_rec_overlay'; + Object.assign(root.style, { + position: 'fixed', + top: '10px', + right: '10px', + zIndex: 2147483646, + fontFamily: 'system-ui,-apple-system,Segoe UI,Roboto,Arial', + }); + root.innerHTML = ` +
+ 录制中 + + + +
+ `; + document.documentElement.appendChild(root); + const btnPause = root.querySelector('#__rr_pause'); + const btnStop = root.querySelector('#__rr_stop'); + const hideChk = root.querySelector('#__rr_hide_values'); + hideChk.checked = hideInputValues; + hideChk.addEventListener('change', () => (hideInputValues = hideChk.checked)); + btnPause.addEventListener('click', () => { + if (!isPaused) pause(); + else resume(); + }); + btnStop.addEventListener('click', () => { + stop(); + }); + updateOverlayStatus(); + // element highlight box + highlightBox = document.createElement('div'); + Object.assign(highlightBox.style, { + position: 'fixed', + border: '2px solid rgba(59,130,246,0.9)', + borderRadius: '4px', + background: 'rgba(59,130,246,0.15)', + pointerEvents: 'none', + zIndex: 2147483645, + }); + document.documentElement.appendChild(highlightBox); + document.addEventListener('mousemove', onHoverMove, true); + } + + function removeOverlay() { + try { + const root = document.getElementById('__rr_rec_overlay'); + if (root) root.remove(); + if (highlightBox) highlightBox.remove(); + document.removeEventListener('mousemove', onHoverMove, true); + } catch {} + } + + function updateOverlayStatus() { + const badge = document.getElementById('__rr_badge'); + const pauseBtn = document.getElementById('__rr_pause'); + if (badge) badge.textContent = isPaused ? '已暂停' : '录制中'; + if (pauseBtn) pauseBtn.textContent = isPaused ? '继续' : '暂停'; + } + + function onHoverMove(e) { + if (!highlightBox || !isRecording || isPaused) return; + const el = e.target instanceof Element ? e.target : null; + if (!el) return; + try { + const r = el.getBoundingClientRect(); + Object.assign(highlightBox.style, { + left: `${Math.round(r.left)}px`, + top: `${Math.round(r.top)}px`, + width: `${Math.round(Math.max(0, r.width))}px`, + height: `${Math.round(Math.max(0, r.height))}px`, + display: r.width > 0 && r.height > 0 ? 'block' : 'none', + }); + } catch {} + } + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { try { if (request && request.action === 'rr_recorder_control') { @@ -294,10 +385,12 @@ start(request.meta || {}); sendResponse({ success: true }); return true; + } else if (cmd === 'pause') { + pause(); + sendResponse({ success: true }); + return true; } else if (cmd === 'resume') { - // Attach without resetting flow or sending start event - isRecording = true; - attach(); + resume(); sendResponse({ success: true }); return true; } else if (cmd === 'stop') { diff --git a/app/chrome-extension/inject-scripts/wait-helper.js b/app/chrome-extension/inject-scripts/wait-helper.js index d97dde51..77ffb110 100644 --- a/app/chrome-extension/inject-scripts/wait-helper.js +++ b/app/chrome-extension/inject-scripts/wait-helper.js @@ -145,6 +145,58 @@ }); } + function waitForSelector({ selector, visible = true, timeout = 5000 }) { + return new Promise((resolve) => { + const start = Date.now(); + let resolved = false; + + const isMatch = () => { + try { + const el = document.querySelector(selector); + if (!el) return null; + if (!visible) return el; + return isVisible(el) ? el : null; + } catch { + return null; + } + }; + + const done = (result) => { + if (resolved) return; + resolved = true; + obs && obs.disconnect(); + clearTimeout(timer); + resolve(result); + }; + + const check = () => { + const el = isMatch(); + if (el) { + const ref = ensureRefForElement(el); + const center = centerOf(el); + done({ success: true, matched: { ref, center }, tookMs: Date.now() - start }); + } + }; + + const obs = new MutationObserver(check); + try { + obs.observe(document.documentElement || document.body, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + }); + } catch {} + + // initial check + check(); + const timer = setTimeout( + () => done({ success: false, reason: 'timeout', tookMs: Date.now() - start }), + Math.max(0, timeout), + ); + }); + } + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { try { if (request && request.action === 'wait_helper_ping') { @@ -162,6 +214,17 @@ waitFor({ text, appear, timeout }).then((res) => sendResponse(res)); return true; // async } + if (request && request.action === 'waitForSelector') { + const selector = String(request.selector || '').trim(); + const visible = request.visible !== false; // default true + const timeout = Number(request.timeout || 5000); + if (!selector) { + sendResponse({ success: false, error: 'selector is required' }); + return true; + } + waitForSelector({ selector, visible, timeout }).then((res) => sendResponse(res)); + return true; // async + } } catch (e) { sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); return true; diff --git a/app/chrome-extension/package.json b/app/chrome-extension/package.json index bbb9f26f..fb862ce9 100644 --- a/app/chrome-extension/package.json +++ b/app/chrome-extension/package.json @@ -21,6 +21,10 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.11.0", + "@vue-flow/background": "^1.3.2", + "@vue-flow/controls": "^1.1.3", + "@vue-flow/core": "^1.47.0", + "@vue-flow/minimap": "^1.5.4", "@xenova/transformers": "^2.17.2", "chrome-mcp-shared": "workspace:*", "date-fns": "^4.1.0", diff --git a/builder.design.md b/builder.design.md new file mode 100644 index 00000000..2c0cc4f3 --- /dev/null +++ b/builder.design.md @@ -0,0 +1,199 @@ +# 模块 C — 编排画布（Builder）技术设计（builder.design.md） + +版本：v1.0（基于 builder.prd.md） +目标读者：前端架构/扩展工程/MCP 工具维护者 + +## 1. 架构概览 + +- 编辑端（Popup 内） + - 画布 UI：Vue + 画布库（建议 VueFlow），包含节点库、画布、迷你地图、属性面板、搜索/对齐、撤销/重做、自动保存。 + - 状态管理：本地 store（可用组合式 API/Pinia），维护 `nodes/edges/variables/meta` 与 UI 状态。 + - 序列化：与 `FlowV2` 对齐，保存至 `chrome.storage.local`，并确保与 `Flow` 线性模式互转。 +- 执行端（Background） + - Runner 增强：在存在 `nodes/edges` 时按 DAG 模式执行；缺省时沿用 steps[] 线性执行。 + - Node Registry：节点注册表（type→validate/run 映射），各节点内部复用 MCP 工具（chrome\_\*）。 +- 消息/发布 + - 导入/导出/发布工具：沿用现有 message types 与 native host 动态工具注册流程。 + +## 2. 数据模型 + +### 2.1 Flow V2 接口（与 design.md 保持一致） + +```ts +export type NodeType = + | 'click' + | 'fill' + | 'key' + | 'wait' + | 'assert' + | 'script' + | 'navigate' + | 'openTab' + | 'switchTab' + | 'closeTab' + | 'http' + | 'extract' + | 'delay'; + +export interface NodeBase { + id: string; + type: NodeType; + name?: string; + disabled?: boolean; + config?: any; // 每个节点的专有配置（见 2.2） + ui?: { x: number; y: number }; +} +export interface Edge { + id: string; + from: string; + to: string; + label?: 'default' | 'true' | 'false' | 'onError'; // M1 仅 default +} +export interface FlowV2 extends Flow { + // 兼容 V1 + nodes?: NodeBase[]; + edges?: Edge[]; + subflows?: Record; +} +``` + +### 2.2 节点配置（config）约定 + +- 通用：`timeoutMs?`、`retry? { count; intervalMs; backoff? }`、`screenshotOnFail?: boolean`、`saveAs?: string`（extract/http/script 可用）。 +- click/fill：`target: TargetLocator`（含 ref/candidates），fill 另有 `value: string`（支持 `{var}`）。 +- key：`keys: string`（e.g. `Backspace Enter` / `cmd+a`）。 +- wait：`condition: { text | selector | navigation | networkIdle }`。 +- assert：`assert: { exists | visible | textPresent | attribute{ selector; name; equals? | matches? } }`。 +- script：`world?: 'MAIN'|'ISOLATED'`，`code: string`，`assign?: Record`（把返回对象字段映射到 vars）。 +- http：`method; url; headers?; body?; assign?: Record`（JSONPath → vars）。 +- extract：`selector/attr/text/regex/js`（至少一种），`saveAs: string`。 +- navigate/openTab/switchTab/closeTab：与现有浏览器工具参数一致（tabTarget/startUrl/refresh 保持在运行选项层）。 + +## 3. 画布 UI 设计 + +- 组件构成 + - 左侧：节点库（基础动作分组）、搜索。 + - 中间：画布区（VueFlow），支持缩放/平移/网格吸附/多选/框选/快捷键（Del/⌘C/⌘V/⌘Z/⌘Shift+Z）。 + - 右侧：属性面板（分组：基本、目标/选择器、等待/断言、变量/映射、重试/超时、备注）。 + - 底部：日志区（运行时使用，可隐藏）。 +- 交互细节 + - 新建：拖入节点自动放置，连接时高亮可落点；禁止自环；断开连线回收端点。 + - 校验：必填项红框/提示；不合法连线阻止；保存前校验。 + - 兼容：从 steps[] 打开时自动串联节点；保存时可选择“覆盖 steps[]”（线性模式）或“仅保存 nodes/edges”。 +- 性能与体验 + - 大图优化：虚拟化/分层渲染；节点模板缓存；平移/缩放节流；100~300 节点流畅。 + +## 4. Runner（DAG 模式） + +### 4.1 执行语义（M1） + +- 当 `nodes/edges` 存在且不为空：按拓扑排序获得可执行序列；仅处理 label=default 的边。 +- 逐节点执行： + 1. 变量展开：把 config 中字符串字段里的 `{var}` 替换为 ctx.vars 的值； + 2. validate：节点注册表校验 config； + 3. run：映射到 MCP 工具（见 5），拿到结果； + 4. saveAs/assign：把产出写入 ctx.vars（或 ctx.outputs[节点ID]）。 + 5. 日志：推送节点级 RunLogEntry；失败按 `failStrategy`/`retry` 处理。 +- 退出条件：到达尾部或遇到不可恢复错误。 + +### 4.2 伪代码 + +```ts +async function runDag(flow: FlowV2, options): Promise { + const ctx = { vars: resolveVars(flow.variables, options.args), outputs: {}, runId }; + const order = topoSort(flow.nodes, flow.edges); // label=default only + for (const nodeId of order) { + const node = getNode(nodeId); + if (node.disabled) continue; + const runtime = registry[node.type]; + const conf = expandTemplates(node.config, ctx.vars); + try { + runtime.validate(conf); + const out = await runtime.run({ tabId, ctx, logger }, conf); + if (conf.saveAs) ctx.vars[conf.saveAs] = out?.value ?? out; + if (conf.assign) applyAssign(ctx.vars, out, conf.assign); // JSONPath 支持留到 M2 + logSuccess(nodeId); + } catch (e) { + const retryOk = await maybeRetry(runtime, conf, e); + if (!retryOk) handleFail(nodeId, e); // stop/continue + } + } + return summarize(ctx); +} +``` + +## 5. 节点注册表与工具映射 + +- 注册表结构 + +```ts +export interface NodeRuntime { + validate(config: T): { ok: boolean; errors?: string[] }; + run(ctx: NodeContext, config: T): Promise; +} +export interface NodeContext { + tabId: number; + vars: Record; + outputs: Record; + runId: string; + logger: (e: RunLogEntry) => void; +} +``` + +- 工具映射（与现有一致）： + - click → `chrome_click_element`（双击用 `chrome_computer.double_click`）。 + - fill → `chrome_fill_or_select`；key → `chrome_keyboard`。 + - wait/assert → `wait-helper` + `chrome_read_page`；navigate/open/switch/close → `chrome_navigate`/窗口工具。 + - script → `chrome_inject_script`（MAIN/ISOLATED）。 + - http → `chrome_network_request`（已有则复用）。 + - extract → `chrome_read_page` + 内容脚本聚合。 + +## 6. 存储/导入/导出/发布 + +- 存储：沿用 `chrome.storage.local`，key 不变（rr_flows）；在 Flow 实体上新增 `nodes/edges` 字段。 +- 导入导出：与线性一致，JSON 中可同时含 steps/nodes/edges；导入时做版本迁移。 +- 发布为 MCP 动态工具：工具名 `flow.`；inputSchema 基于 variables + 运行选项生成；调用时走通用 `record_replay_flow_run`。 + +## 7. 校验与错误处理 + +- 编辑期校验：必填/格式/选择器空值；运行期校验：validate + 容错（默认 stop）。 +- 失败截图：统一由 runner 在 catch 路径触发 screenshot 工具；日志中标出节点 ID 与错误原因。 +- 选择器回退：沿用 selector-engine 策略；fallback 用信息写入日志，并在编辑器弹出更新提示。 + +## 8. 安全与隐私 + +- 敏感变量：不落盘，不导出；运行时仅从 args 注入；Overlay 表单/提示敏感字段。 +- 注入世界：ISOLATED 为默认；MAIN 仅在用户明确选择时使用。 +- 权限最小化：不新增超出现有扩展范围的权限。 + +## 9. 性能策略 + +- 编辑端：虚拟化/节流；避免频繁全图重绘；自动保存去抖（≥500ms）。 +- 执行端：节点超时/重试/退避；M2 后引入并发与限流（全局/域级）。 + +## 10. 迁移与兼容 + +- 打开旧 Flow（仅 steps[]）→ 自动生成链式 DAG（nodes/edges）并允许切换“线性/画布”视图； +- 只要 nodes/edges 存在，优先 DAG 执行；否则走 steps[]；导出时兼容两者。 + +## 11. 开发分期（落地建议） + +- M1（2~3 周）： + - 选型并接入 VueFlow；完成画布基础、节点库、连线、属性面板、撤销/重做、自动保存； + - 支持基础节点与 DAG 串行执行；线性兼容；导入导出/发布； + - 日志/失败截图在节点上联动高亮（可先在列表展示）。 +- M2： + - If/Else/While/ForEach；OnError 分支；从节点开始调试；open/switch/close 完善；分组/折叠。 +- M3： + - 并发/限流；表达式/JSONPath 映射器；数据集/凭据；高级调试。 + +## 12. 开放问题 + +- JSONPath 与表达式语言的边界与安全沙箱如何定义？ +- 节点产出统一结构与 assign/saveAs 的歧义如何避免？ +- 画布超大规模（500+ 节点）时的严重退化处理策略？ +- 团队协作与冲突解决是否需要引入（ID 锁/合并策略）？ + +--- + +说明：本设计围绕“高度复用现有 MCP 工具、最小化新增复杂度”的原则，画布仅作为编排与可视化层；执行统一收口到背景 Runner 与工具层，便于稳定落地与运维。 diff --git a/builder.prd.md b/builder.prd.md new file mode 100644 index 00000000..489f3a27 --- /dev/null +++ b/builder.prd.md @@ -0,0 +1,146 @@ +# 模块 C — 编排画布（Builder）PRD v1.0 + +版本：v1.0（参考 Automa，兼容现有 Record & Replay） +状态：Ready for Design +负责人：产品/前端架构 + +## 0. 现状与定位（与录制回放的关系） + +- 现状：已具备“录制 → 线性步骤（steps[]）→ 回放/发布”的闭环；回放统一复用 MCP 工具（chrome\_\*），并支持变量、失败截图、网络片段等。 +- 定位：本编排画布是“录制回放模块的可视化编辑层”。 + - 录制完成的工作流可“导入画布”进行二次编排与参数化； + - 也支持在画布里“从零拖拽节点”新建工作流； + - 保存后仍与已有回放/发布/定时/导入导出机制完全打通（同一存储与执行通道）。 +- 目标用户路径（高频）：录制草稿 → 打开画布调整/补空 → 保存 → 回放验证 → 发布为 MCP 动态工具/配置定时 → 运营。 + +## 1. 背景与目标 + +- 背景：现有线性 steps[] 能覆盖大多数串行场景，但难以表达条件、循环、并发、多标签切换等复杂流程；编辑体验也不利于可视化理解与协作。 +- 目标：提供“节点 + 连线”的可视化编排画布（DAG），在保持与线性模式完全兼容的前提下，一步到位地支撑复杂业务流程的创建、调试与运行。 +- 价值： + - 降低复杂自动化的心智负担（所见即所得）。 + - 增强流程表达力（条件/循环/分支/子流程/并发）。 + - 与 MCP 工具层打通，形成可沉淀、可分享、可重放的企业级资产。 + +## 2. 范围与非范围 + +- 范围（MVP/M1）： + - 画布编辑：节点拖拽、连线、选择、移动、缩放、对齐、撤销/重做、自动保存。 + - 节点类型（基础动作）：click/fill/key/wait/assert/navigate/script/openTab/switchTab/closeTab/delay/extract/http。 + - 属性面板：每个节点的配置编辑（选择器候选、变量/占位符、等待/断言、超时/重试、保存为变量等）。 + - 变量系统：全局 variables 与节点产出保存（saveAs/assign），字符串字段支持占位符 `{var}`。 + - 执行：无条件边时按拓扑串行执行；步骤失败截图与日志；OnError 可选（stop/continue/retry）。 + - 兼容：线性 steps[] 自动映射为链式 DAG；DAG 缺省时沿用线性模式。 + - 导入/导出/发布为 MCP 动态工具（沿用现有机制，Schema 由变量推导）。 + - 来源与入口： + - 来源：① 录制得到的 steps 一键转化为 DAG（链式）；② 画布新建（拖拽节点）；③ JSON 导入； + - 入口：Popup 的“录制与回放”列表进入“编辑”；录制完成弹出“前往画布编辑”。 +- 非范围（M1 之外）： + - 高级控制流：If/Else、While/Until、ForEach（并发度控制）、OnError 分支（M2）。 + - 数据集（Dataset）/可视化 JSON 映射器/凭据库/表达式引擎（M2/M3）。 + - 团队协作、多用户并发编辑与权限（后续版本）。 + +## 3. 用户与关键场景 + +- 用户：运营/测试/开发/数据标注人员。 +- 关键场景： + 1. 登录并进入后台 → 根据条件决定分支 → 下载报告（click/fill/wait/assert/navigate/http）。 + 2. 批量表单填充 → 提交失败重试 → 记录成功项（foreach/delay/assert/saveAs）。 + 3. 爬取分页数据 → 提取字段到变量/数据集 → 导出 JSON/CSV（extract/http/脚本处理）。 + +## 4. 端到端流程（闭环） + +- 从录制到编排： + + 1. 开始录制（Popup）→ 页面内浮层反馈与事件捕获 → 停止录制； + 2. 生成 Flow 草稿（steps[]）→ 提示“前往画布编辑”； + 3. 画布自动将 steps 链式映射为 nodes/edges → 在属性面板完善：选择器候选优先级、等待/断言、变量占位、saveAs/assign； + 4. 保存 Flow（同一存储键），可导出 JSON。 + +- 从零拖拽到回放： + + 1. 画布中新建 → 从节点库拖入基础节点，连线构建链路； + 2. 在属性面板配置选择器/变量/等待/断言/脚本等 → 保存； + 3. 回放验证（可选择 “当前/新标签”、“起始 URL”）→ 查看日志与失败截图 → 调整后再次保存。 + +- 发布与定时： + 1. Flow 发布为 MCP 动态工具（flow.），输入 Schema 由 variables + 运行选项生成； + 2. 可配置定时执行（interval/daily/once），执行结果写入运行记录； + 3. 运行失败时截图与错误节点高亮，支持导出/导入迁移。 + +## 5. 功能需求（FR） + +- 画布交互（FR-BLD-001~010） + + - FR-BLD-001 画布基础：缩放/平移、对齐网格、吸附对齐、迷你地图。 + - FR-BLD-002 节点库：从侧边栏拖入节点、复制/粘贴、批量选择、删除。 + - FR-BLD-003 连线：连接/断开、自动修复、避免自环；边标签（默认/true/false/onError 预留）。 + - FR-BLD-004 属性面板：点选节点后右侧编辑配置；校验并提示错误。 + - FR-BLD-005 撤销/重做/自动保存；版本与恢复点（简版）。 + - FR-BLD-006 搜索与定位：按节点名/类型/变量引用查找并高亮。 + - FR-BLD-007 从节点开始调试（M2）；单步/断点（M3）。 + - FR-BLD-008 兼容线性：steps↔DAG 双向转换（线性自动生成链式图）。 + - FR-BLD-009 运行入口：整流回放/从选中开始回放；日志与失败截图联动高亮。 + - FR-BLD-010 导入导出：JSON（带 nodes/edges/variables/meta）。 + +- 节点与属性（FR-BLD-011~030） + + - FR-BLD-011 click/fill/key/wait/assert/navigate/script/delay：与线性动作参数一致。 + - FR-BLD-012 openTab/switchTab/closeTab：多标签编排基础动作。 + - FR-BLD-013 http（GET/POST/...）：可保存响应字段到变量（M1 可选若已有工具）。 + - FR-BLD-014 extract：按 selector/属性/文本/正则/JS 抽取，saveAs 到变量。 + - FR-BLD-015 重试策略：count/intervalMs/backoff。 + - FR-BLD-016 超时/失败策略：stop/continue/retry；失败截图开关。 + - FR-BLD-017 选择器候选与优先级：ref/css/attr/aria/text/xpath；回退记录与提示。 + - FR-BLD-018 变量系统：全局 variables；字符串字段支持 `{var}`。脚本/HTTP 支持 assign。 + - FR-BLD-019 运行选项：tabTarget/startUrl/refresh/captureNetwork/returnLogs/timeoutMs。 + - FR-BLD-020 节点命名/备注/标签；节点分组与折叠（M2）。 + +- 执行与日志（FR-BLD-031~040） + - FR-BLD-031 DAG 执行：无条件边时顺序拓扑执行；失败按策略处理。 + - FR-BLD-032 日志：节点级耗时/状态/错误信息；失败截图；网络片段（可选）。 + - FR-BLD-033 上下文：vars/outputs；saveAs/assign 写入；回放后返回 summary/outputs/logs。 + - FR-BLD-034 绑定校验：不匹配时拒绝或需 startUrl；与当前绑定规则一致。 + +## 6. 数据模型（导出/存储） + +- 兼容 Flow V1：`steps[]` 不变。 +- 新增 Flow V2： + - `nodes: NodeBase[]`、`edges: Edge[]`、`subflows?: Record`。 + - NodeBase：`{ id,type,name?,disabled?,config?,ui? }`；Edge：`{ id,from,to,label? }`（label 预留 default/true/false/onError）。 + - 变量：与现有 `variables[]` 统一；字符串字段支持 `{var}`；脚本/HTTP 支持 assign 映射到 vars。 + +## 7. 验收标准 + +- 线性流程自动转 DAG，回放结果与线性一致（10 次成功率 ≥ 95%）。 +- 画布可稳定编辑 200+ 节点；撤销/重做可靠；保存/导出/导入无损。 +- 失败报告包含失败截图与错误节点高亮；用户 1 分钟内定位问题。 +- 节点回退/选择器提示清晰，编辑器可直接调整优先级并保存。 + +## 8. 性能与非功能需求（NFR） + +- 画布交互：常见 200 节点保持流畅（60fps 优先级次之，确保不卡顿）。 +- 执行性能：单节点默认等待 ≤ 10s；并发/循环在 M2 引入；全局并发上限与限流可配置。 +- 稳定性：10 步基准用例回放成功率 ≥ 95%。 +- 安全与隐私：敏感变量不落盘；导出不含敏感值；CSP 兼容；权限最小化。 + +## 9. 里程碑 + +- M1（画布基础 + 串行 DAG） + - 画布/节点库/连线/属性面板/撤销重做/自动保存；基础节点（click/fill/key/wait/assert/navigate/script/delay/extract/http\*）； + - DAG 执行（无条件边，串行拓扑）；线性兼容；日志/截图/变量；导入导出/发布。 +- M2（控制流与多标签） + - If/Else/While/ForEach；OnError 分支；openTab/switchTab/closeTab 完善；从节点开始调试；节点分组/折叠；数据集/凭据（可选）。 +- M3（并发与限流） + - foreach 并发、全局并发上限/域级限流；表达式/JSONPath 映射器；高级调试与监控。 + +--- + +注：节点与工具的映射严格复用现有 MCP 工具（chrome\_\*），减少新增复杂度并保证稳定性。 + +## 10. 成功指标（KPI） + +- 20 分钟内：从录制到画布编排再到首次成功回放（≥ 80% 用户）。 +- 稳定性：基准 10 步流程回放成功率 ≥ 95%，并在页面小改动（文案/结构微调）下仍 ≥ 90%。 +- 工具化：发布为 MCP 工具后，十次调用成功率 ≥ 95%，参数缺失可被 Schema 友好提示。 +- 编辑效率：200+ 节点画布交互不卡顿（≥ 30 fps）；撤销/重做成功率 100%。 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed89a4a3..d165c7d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,18 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.11.0 version: 1.12.1 + '@vue-flow/background': + specifier: ^1.3.2 + version: 1.3.2(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3)) + '@vue-flow/controls': + specifier: ^1.1.3 + version: 1.1.3(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3)) + '@vue-flow/core': + specifier: ^1.47.0 + version: 1.47.0(vue@3.5.16(typescript@5.8.3)) + '@vue-flow/minimap': + specifier: ^1.5.4 + version: 1.5.4(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3)) '@xenova/transformers': specifier: ^2.17.2 version: 2.17.2 @@ -1083,6 +1095,9 @@ packages: '@types/supertest@6.0.3': resolution: {integrity: sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w==} + '@types/web-bluetooth@0.0.20': + resolution: {integrity: sha512-g9gZnnXVq7gM7v3tJCWV/qw7w+KeOlSHAhgF9RytFyifW6AF61hdT2ucrYhPq9hLs5JIryeupHV3qGk95dH9ow==} + '@types/yargs-parser@21.0.3': resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -1167,6 +1182,29 @@ packages: '@volar/typescript@2.4.14': resolution: {integrity: sha512-p8Z6f/bZM3/HyCdRNFZOEEzts51uV8WHeN8Tnfnm2EBv6FDB2TQLzfVx7aJvnl8ofKAOnS64B2O8bImBFaauRw==} + '@vue-flow/background@1.3.2': + resolution: {integrity: sha512-eJPhDcLj1wEo45bBoqTXw1uhl0yK2RaQGnEINqvvBsAFKh/camHJd5NPmOdS1w+M9lggc9igUewxaEd3iCQX2w==} + peerDependencies: + '@vue-flow/core': ^1.23.0 + vue: ^3.3.0 + + '@vue-flow/controls@1.1.3': + resolution: {integrity: sha512-XCf+G+jCvaWURdFlZmOjifZGw3XMhN5hHlfMGkWh9xot+9nH9gdTZtn+ldIJKtarg3B21iyHU8JjKDhYcB6JMw==} + peerDependencies: + '@vue-flow/core': ^1.23.0 + vue: ^3.3.0 + + '@vue-flow/core@1.47.0': + resolution: {integrity: sha512-w+qrm/xjQP5NUeKUOMIbQvpOeivTbGZtY2lGffK5kHiN3ZLyEazhESc8OeIV9NZkK2T5DIeyX/nhHxCC45HLiw==} + peerDependencies: + vue: ^3.3.0 + + '@vue-flow/minimap@1.5.4': + resolution: {integrity: sha512-l4C+XTAXnRxsRpUdN7cAVFBennC1sVRzq4bDSpVK+ag7tdMczAnhFYGgbLkUw3v3sY6gokyWwMl8CDonp8eB2g==} + peerDependencies: + '@vue-flow/core': ^1.23.0 + vue: ^3.3.0 + '@vue/compiler-core@3.5.16': resolution: {integrity: sha512-AOQS2eaQOaaZQoL1u+2rCJIKDruNXVBZSiUD3chnUrsoX5ZTQMaCvXlWNIfxBJuU15r1o7+mpo5223KVtIhAgQ==} @@ -1207,6 +1245,15 @@ packages: '@vue/shared@3.5.16': resolution: {integrity: sha512-c/0fWy3Jw6Z8L9FmTyYfkpM5zklnqqa9+a6dz3DvONRKW2NEbh46BP0FHuLFSWi2TnQEtp91Z6zOWNrU6QiyPg==} + '@vueuse/core@10.11.1': + resolution: {integrity: sha512-guoy26JQktXPcz+0n3GukWIy/JDNKti9v6VEMu6kV2sYBsWuGiTU8OWdg+ADfUbHg3/3DlqySDe7JmdHrktiww==} + + '@vueuse/metadata@10.11.1': + resolution: {integrity: sha512-IGa5FXd003Ug1qAZmyE8wF3sJ81xGLSqTqtQ6jaVfkeZ4i5kS2mwQF61yhVqojRnenVew5PldLyRgvdl4YYuSw==} + + '@vueuse/shared@10.11.1': + resolution: {integrity: sha512-LHpC8711VFZlDaYUXEBbFBCQ7GS3dVU9mjOhhMhXP6txTV4EhYQg/KGnQuvt/sPAtoUKq7VVUnL6mVtFoL42sA==} + '@webext-core/fake-browser@1.3.2': resolution: {integrity: sha512-jFyPWWz+VkHAC9DRIiIPOyu6X/KlC8dYqSKweHz6tsDb86QawtVgZSpYcM+GOQBlZc5DHFo92jJ7cIq4uBnU0A==} @@ -1796,6 +1843,44 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + dargs@8.1.0: resolution: {integrity: sha512-wAV9QHOsNbwnWdNW2FYvE1P56wtgSbM+3SZcdGiWQILwVjACCXDCI3Ai8QlCjMDB8YK5zySiXZYBiwGmNY3lnw==} engines: {node: '>=12'} @@ -4062,6 +4147,7 @@ packages: supertest@7.1.1: resolution: {integrity: sha512-aI59HBTlG9e2wTjxGJV+DygfNLgnWbGdZxiA/sgrnNNikIW8lbDvCtF6RnhZoJ82nU7qv7ZLjrvWqCEm52fAmw==} engines: {node: '>=14.18.0'} + deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} @@ -4406,6 +4492,17 @@ packages: vscode-uri@3.1.0: resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} + vue-demi@0.14.10: + resolution: {integrity: sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==} + engines: {node: '>=12'} + hasBin: true + peerDependencies: + '@vue/composition-api': ^1.0.0-rc.1 + vue: ^3.0.0-0 || ^2.6.0 + peerDependenciesMeta: + '@vue/composition-api': + optional: true + vue-eslint-parser@10.1.3: resolution: {integrity: sha512-dbCBnd2e02dYWsXoqX5yKUZlOt+ExIpq7hmHKPb5ZqKcjf++Eo0hMseFTZMLKThrUk61m+Uv6A2YSBve6ZvuDQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -5545,6 +5642,8 @@ snapshots: '@types/methods': 1.1.4 '@types/superagent': 8.1.9 + '@types/web-bluetooth@0.0.20': {} + '@types/yargs-parser@21.0.3': {} '@types/yargs@17.0.33': @@ -5665,6 +5764,34 @@ snapshots: path-browserify: 1.0.1 vscode-uri: 3.1.0 + '@vue-flow/background@1.3.2(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vue-flow/core': 1.47.0(vue@3.5.16(typescript@5.8.3)) + vue: 3.5.16(typescript@5.8.3) + + '@vue-flow/controls@1.1.3(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vue-flow/core': 1.47.0(vue@3.5.16(typescript@5.8.3)) + vue: 3.5.16(typescript@5.8.3) + + '@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vueuse/core': 10.11.1(vue@3.5.16(typescript@5.8.3)) + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + vue: 3.5.16(typescript@5.8.3) + transitivePeerDependencies: + - '@vue/composition-api' + + '@vue-flow/minimap@1.5.4(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vue-flow/core': 1.47.0(vue@3.5.16(typescript@5.8.3)) + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + vue: 3.5.16(typescript@5.8.3) + '@vue/compiler-core@3.5.16': dependencies: '@babel/parser': 7.27.5 @@ -5737,6 +5864,25 @@ snapshots: '@vue/shared@3.5.16': {} + '@vueuse/core@10.11.1(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@types/web-bluetooth': 0.0.20 + '@vueuse/metadata': 10.11.1 + '@vueuse/shared': 10.11.1(vue@3.5.16(typescript@5.8.3)) + vue-demi: 0.14.10(vue@3.5.16(typescript@5.8.3)) + transitivePeerDependencies: + - '@vue/composition-api' + - vue + + '@vueuse/metadata@10.11.1': {} + + '@vueuse/shared@10.11.1(vue@3.5.16(typescript@5.8.3))': + dependencies: + vue-demi: 0.14.10(vue@3.5.16(typescript@5.8.3)) + transitivePeerDependencies: + - '@vue/composition-api' + - vue + '@webext-core/fake-browser@1.3.2': dependencies: lodash.merge: 4.6.2 @@ -6372,6 +6518,42 @@ snapshots: csstype@3.1.3: {} + d3-color@3.1.0: {} + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-selection@3.0.0: {} + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + dargs@8.1.0: {} date-fns@4.1.0: {} @@ -9260,6 +9442,10 @@ snapshots: vscode-uri@3.1.0: {} + vue-demi@0.14.10(vue@3.5.16(typescript@5.8.3)): + dependencies: + vue: 3.5.16(typescript@5.8.3) + vue-eslint-parser@10.1.3(eslint@9.28.0(jiti@2.4.2)): dependencies: debug: 4.4.1(supports-color@5.5.0) diff --git a/task.md b/task.md new file mode 100644 index 00000000..278b400a --- /dev/null +++ b/task.md @@ -0,0 +1,67 @@ +## 录制回放 · 编排画布（Builder）落地进度 + +更新时间：2025-10-10 + +### 已完成（本次） + +- 数据模型：在 `record-replay/types.ts` 扩展 `Flow`，新增可选 `nodes`/`edges`（Flow V2 结构），兼容线性 `steps[]`。 +- 画布编辑器（M1 骨架）：新增 `popup/components/BuilderEditor.vue`，使用 VueFlow 渲染 DAG。 + - 节点库：click/fill/key/wait/assert/navigate/script/delay。 + - 画布：缩放/平移（VueFlow 内置）、网格吸附、拖拽、连线（默认单一出边）。 + - 属性面板：按节点类型编辑（选择器候选、fill 值、wait/assert/navigate/script）。 + - 互转：支持 steps→nodes（链式）与 nodes→steps（按 default 边拓扑），保存时同步覆盖 `steps[]` 以保证可立即回放。 +- 集成入口：在 Popup 的“录制与回放”列表加入“画布编辑”按钮；保存沿用 `RR_SAVE_FLOW`。 +- Runner 增强：`flow-runner.ts` 在检测到 `nodes/edges` 时，运行期进行 DAG→steps 线性化（按 default 边拓扑），复用现有线性执行与日志/截图机制。 +- Builder 小增强： + - 节点类型补充 key/delay 的默认配置、属性面板与摘要展示。 + - 快捷键：Delete/Backspace 删除选中；Cmd/Ctrl+D 复制；Cmd/Ctrl+S 保存。 + - 自动保存：节点/连线/名称变化 800ms 去抖自动保存，状态提示（保存中/已保存）。 + - 搜索定位：顶栏输入命中节点名或首个选择器，回车自动聚焦到节点并选中。 + - 新增节点类型与执行： + - http（method/url/headers/body/saveAs），Runner 调用 NETWORK_REQUEST 并可保存 JSON 响应到变量。 + - extract（selector/attr/js/saveAs），Runner 在页面执行提取并保存变量。 + - openTab/switchTab/closeTab，Runner 分别创建新标签/切换标签/关闭标签（支持按 url/title 匹配）。 + - script 支持 saveAs/assign：执行返回值支持保存到变量；assign 支持点路径（a.b[0].c）映射多个变量。 + - 校验与提示： + - 节点级校验（http/extract/switchTab/script 等必填/组合约束）与 UI 提示（字段红框+错误列表）。 + - 顶栏显示错误计数，便于定位问题。 + - 映射编辑器：KeyValueEditor 组件，用于 script/http 的 assign 键值映射编辑。 + - 从选中节点回放：Builder 顶栏支持从当前选中节点启动回放（传入 startNodeId）。 + - 错误列表面板：可展开全局错误列表，点击条目定位并聚焦到对应节点。 + - 快捷键补充：⌘/Ctrl+Z 撤销；⌘/Ctrl+Shift+Z 重做。 + - 自动排版与视图：一键自动排版（简单拓扑布局），自适应视图（fit view）。 + - 导出：Builder 顶栏直接导出 Flow JSON（保持与后台导出一致）。 + - 历史栈优化：限制最多 50 个快照，避免内存无限增长。 + - 字段级高亮：点击错误项可高亮并滚动到对应属性字段（PropertyPanel）。 + +### 目录结构与模块拆分 + +- 遵循组件聚合原则，将编辑器完整聚合在一个目录下：`popup/components/builder/` + - `model/transforms.ts`：DAG/steps 互转、ID 生成、默认配置、拓扑排序、摘要。 + - `store/useBuilderStore.ts`：编辑器状态与操作（选择/新增/删除/连线/布局/导入导出）。 + - `components/{Canvas,Sidebar,PropertyPanel}.vue`：画布/节点库/属性面板子组件。 + +### 待安装依赖 + +- 画布基于 VueFlow：需要安装 + - `@vue-flow/core` + - `@vue-flow/controls` + - `@vue-flow/minimap` + +### 下一步规划（短期） + +1. DAG 执行路径进行更细粒度控制（后续支持 true/false/onError 边），并补齐 http/extract/openTab/switchTab/closeTab 映射。 +2. 画布体验升级：撤销/重做历史上限与压缩策略、MiniMap/Controls 配置、节点模板与样式。 +3. 属性面板补齐更多节点（http/extract/openTab/switchTab/closeTab），并完善字段校验/提示。 +4. 自动保存（去抖 ≥500ms）与版本快照；画布搜索与定位。 + +### 里程碑对齐（builder.prd.md） + +- M1：画布基础 + 串行 DAG（当前已具备可用骨架，待 Runner DAG） +- M2：控制流（If/Else/While/ForEach）、OnError 分支、多标签完善（后续迭代） +- M3：并发与限流（后续迭代） + +### 影响范围与兼容性 + +- 不破坏原有线性流程与回放；保存时同步 `steps[]`，可立即回放。 +- 类型扩展仅新增可选字段，旧数据不受影响；导入导出将兼容两种结构。 From f037536ba851c542439a7e1267a7d3596536e01f Mon Sep 17 00:00:00 2001 From: hangerye Date: Mon, 13 Oct 2025 10:07:14 +0800 Subject: [PATCH 19/71] feat: editor temp store --- app/chrome-extension/common/constants.ts | 1 + app/chrome-extension/common/message-types.ts | 10 + .../background/record-replay/flow-runner.ts | 1206 +---------------- .../background/record-replay/flow-store.ts | 20 +- .../background/record-replay/index.ts | 266 +++- .../background/record-replay/node-registry.ts | 1067 +++++++++++++++ .../background/record-replay/rr-utils.ts | 226 +++ .../background/record-replay/runner.ts | 597 ++++++++ .../record-replay/selector-engine.ts | 65 +- .../background/record-replay/trigger-store.ts | 61 + .../background/record-replay/types.ts | 116 +- .../background/tools/browser/download.ts | 123 ++ .../background/tools/browser/index.ts | 1 + .../tools/browser/network-request.ts | 5 + .../background/tools/browser/read-page.ts | 9 +- .../entrypoints/builder/App.vue | 827 +++++++++++ .../entrypoints/builder/index.html | 13 + .../entrypoints/builder/main.ts | 7 + .../entrypoints/options/App.vue | 4 +- .../options/components/FlowEditor.vue | 762 ----------- .../entrypoints/popup/App.vue | 265 ++-- .../popup/components/BuilderEditor.vue | 775 +++++++++-- .../popup/components/FlowEditor.vue | 363 ----- .../components/builder/components/Canvas.vue | 457 +++++-- .../builder/components/PropertyPanel.vue | 1184 +++++++++++++--- .../components/builder/components/Sidebar.vue | 724 +++++++++- .../builder/components/nodes/NodeCard.vue | 57 + .../builder/components/nodes/NodeIf.vue | 97 ++ .../builder/components/nodes/node-util.ts | 90 ++ .../components/builder/model/transforms.ts | 158 +-- .../components/builder/model/validation.ts | 41 + .../builder/store/useBuilderStore.ts | 123 +- .../entrypoints/sidepanel/App.vue | 405 ++++++ .../entrypoints/sidepanel/index.html | 13 + .../entrypoints/sidepanel/main.ts | 7 + .../entrypoints/styles/tailwind.css | 151 +++ app/chrome-extension/env.d.ts | 1 + .../accessibility-tree-helper.js | 263 ++++ .../inject-scripts/dom-observer.js | 87 ++ .../inject-scripts/network-helper.js | 145 +- .../inject-scripts/recorder.js | 48 + app/chrome-extension/package.json | 5 + app/chrome-extension/tailwind.config.ts | 24 + app/chrome-extension/types/icons.d.ts | 8 + app/chrome-extension/wxt.config.ts | 73 +- app/native-server/src/file-handler.ts | 34 + app/native-server/src/mcp/register-tools.ts | 19 +- automa.md | 221 +++ background_op.md | 181 +++ design.md | 11 +- docs/ISSUE.md | 1190 ++++++++++++++++ docs/record-replay-ui-ux.md | 35 + eslint.config.js | 8 + packages/shared/src/index.ts | 1 + packages/shared/src/rr-graph.ts | 312 +++++ packages/shared/src/tools.ts | 19 + pnpm-lock.yaml | 674 ++++++++- record_replay.prd.md | 19 +- register.md | 240 ++++ 59 files changed, 10672 insertions(+), 3242 deletions(-) create mode 100644 app/chrome-extension/entrypoints/background/record-replay/node-registry.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/runner.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/trigger-store.ts create mode 100644 app/chrome-extension/entrypoints/background/tools/browser/download.ts create mode 100644 app/chrome-extension/entrypoints/builder/App.vue create mode 100644 app/chrome-extension/entrypoints/builder/index.html create mode 100644 app/chrome-extension/entrypoints/builder/main.ts delete mode 100644 app/chrome-extension/entrypoints/options/components/FlowEditor.vue delete mode 100644 app/chrome-extension/entrypoints/popup/components/FlowEditor.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/nodes/NodeCard.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/nodes/NodeIf.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/nodes/node-util.ts create mode 100644 app/chrome-extension/entrypoints/sidepanel/App.vue create mode 100644 app/chrome-extension/entrypoints/sidepanel/index.html create mode 100644 app/chrome-extension/entrypoints/sidepanel/main.ts create mode 100644 app/chrome-extension/entrypoints/styles/tailwind.css create mode 100644 app/chrome-extension/inject-scripts/dom-observer.js create mode 100644 app/chrome-extension/tailwind.config.ts create mode 100644 app/chrome-extension/types/icons.d.ts create mode 100644 automa.md create mode 100644 background_op.md create mode 100644 docs/ISSUE.md create mode 100644 docs/record-replay-ui-ux.md create mode 100644 packages/shared/src/rr-graph.ts create mode 100644 register.md diff --git a/app/chrome-extension/common/constants.ts b/app/chrome-extension/common/constants.ts index c2f39e11..f128aff8 100644 --- a/app/chrome-extension/common/constants.ts +++ b/app/chrome-extension/common/constants.ts @@ -109,6 +109,7 @@ export const STORAGE_KEYS = { RR_RUNS: 'rr_runs', RR_PUBLISHED: 'rr_published_flows', RR_SCHEDULES: 'rr_schedules', + RR_TRIGGERS: 'rr_triggers', } as const; // Notification Configuration diff --git a/app/chrome-extension/common/message-types.ts b/app/chrome-extension/common/message-types.ts index 03ac06fe..cf93fcec 100644 --- a/app/chrome-extension/common/message-types.ts +++ b/app/chrome-extension/common/message-types.ts @@ -34,6 +34,12 @@ export const BACKGROUND_MESSAGE_TYPES = { RR_EXPORT_FLOW: 'rr_export_flow', RR_EXPORT_ALL: 'rr_export_all', RR_IMPORT_FLOW: 'rr_import_flow', + RR_LIST_RUNS: 'rr_list_runs', + // Triggers + RR_LIST_TRIGGERS: 'rr_list_triggers', + RR_SAVE_TRIGGER: 'rr_save_trigger', + RR_DELETE_TRIGGER: 'rr_delete_trigger', + RR_REFRESH_TRIGGERS: 'rr_refresh_triggers', // Scheduling RR_SCHEDULE_FLOW: 'rr_schedule_flow', RR_UNSCHEDULE_FLOW: 'rr_unschedule_flow', @@ -60,6 +66,7 @@ export const CONTENT_MESSAGE_TYPES = { INTERACTIVE_ELEMENTS_HELPER_PING: 'interactive_elements_helper_ping', ACCESSIBILITY_TREE_HELPER_PING: 'chrome_read_page_ping', WAIT_HELPER_PING: 'wait_helper_ping', + DOM_OBSERVER_PING: 'dom_observer_ping', } as const; // Tool action message types (for chrome.runtime.sendMessage) @@ -100,6 +107,9 @@ export const TOOL_MESSAGE_TYPES = { // Record & Replay content script bridge RR_RECORDER_CONTROL: 'rr_recorder_control', RR_RECORDER_EVENT: 'rr_recorder_event', + // DOM observer trigger bridge + SET_DOM_TRIGGERS: 'set_dom_triggers', + DOM_TRIGGER_FIRED: 'dom_trigger_fired', } as const; // Type unions for type safety diff --git a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts index 01bb425a..75ec6182 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts @@ -1,1203 +1,3 @@ -import { TOOL_NAMES } from 'chrome-mcp-shared'; -import { handleCallTool } from '../tools'; -import type { - Flow, - RunLogEntry, - RunRecord, - RunResult, - Step, - StepAssert, - StepFill, - StepKey, - StepScroll, - StepDrag, - StepWait, - StepScript, - NodeBase as DagNode, - Edge as DagEdge, -} from './types'; -import { appendRun } from './flow-store'; -import { locateElement } from './selector-engine'; - -// design note: linear flow executor using existing tools; keeps logs and failure screenshot - -export interface RunOptions { - tabTarget?: 'current' | 'new'; - refresh?: boolean; - captureNetwork?: boolean; - returnLogs?: boolean; - timeoutMs?: number; - startUrl?: string; - args?: Record; - startNodeId?: string; // start executing from this node/step id if present -} - -export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { - const runId = `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - const startAt = Date.now(); - const logs: RunLogEntry[] = []; - const vars: Record = Object.create(null); - for (const v of flow.variables || []) { - if (v.default !== undefined) vars[v.key] = v.default; - } - if (options.args) Object.assign(vars, options.args); - - // Helper: ensure target tab according to tabTarget/startUrl, and optionally refresh - const ensureTab = async () => { - const target = options.tabTarget || 'current'; - const startUrl = options.startUrl; - const [active] = await chrome.tabs.query({ active: true, currentWindow: true }); - if (target === 'new') { - let urlToOpen = startUrl; - if (!urlToOpen) { - // duplicate current active tab's URL when startUrl not provided - urlToOpen = active?.url || 'about:blank'; - } - const created = await chrome.tabs.create({ url: urlToOpen, active: true }); - // Best-effort wait for loading to begin and settle a bit - await new Promise((r) => setTimeout(r, 500)); - } else { - // current tab target - if (startUrl) { - await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url: startUrl } }); - } else if (options.refresh) { - await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { refresh: true } }); - } - } - }; - await ensureTab(); - - // helper to apply assign mapping: { varName: 'a.b[0].c' } - function applyAssign(target: Record, source: any, assign: Record) { - const getByPath = (obj: any, path: string) => { - try { - const parts = path - .replace(/\[(\d+)\]/g, '.$1') - .split('.') - .filter(Boolean); - let cur = obj; - for (const p of parts) { - if (cur == null) return undefined; - cur = cur[p as any]; - } - return cur; - } catch { - return undefined; - } - }; - for (const [k, v] of Object.entries(assign || {})) { - target[k] = getByPath(source, String(v)); - } - } - - // Ensure helper scripts are present for overlay/collectVariables - try { - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - } catch { - /* ignore */ - } - - // Collect missing variables via lightweight prompt overlay - try { - const needed = (flow.variables || []).filter( - (v) => - (options.args?.[v.key] == null || options.args?.[v.key] === '') && - (v.rules?.required || (v.default ?? '') === ''), - ); - if (needed.length > 0) { - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.SEND_COMMAND_TO_INJECT_SCRIPT, - args: { - eventName: 'collectVariables', - payload: JSON.stringify({ variables: needed, useOverlay: true }), - }, - }); - // Fallback: if direct collectVariables without payload not supported, call with explicit variables - let values: Record | null = null; - try { - const t = (res?.content || []).find((c: any) => c.type === 'text')?.text; - const j = t ? JSON.parse(t) : null; - if (j && j.success && j.values) values = j.values; - } catch { - /* ignore */ - } - if (!values) { - const res2 = await chrome.tabs - .query({ active: true, currentWindow: true }) - .then(async (tabs) => { - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') return null; - return await chrome.tabs.sendMessage(tabId, { - action: 'collectVariables', - variables: needed, - useOverlay: true, - } as any); - }); - if (res2 && res2.success && res2.values) values = res2.values; - } - if (values) Object.assign(vars, values); - } - } catch { - // ignore prompt failures - } - - // Init simple overlay for real-time logs - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) { - await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'init' } as any); - } - } catch { - /* ignore */ - } - - // Binding enforcement: if bindings exist and no startUrl, verify current tab URL matches - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const currentUrl = tabs?.[0]?.url || ''; - const bindings = flow.meta?.bindings || []; - if (!options.startUrl && bindings.length > 0) { - const ok = bindings.some((b) => { - try { - if (b.type === 'domain') return new URL(currentUrl).hostname.includes(b.value); - if (b.type === 'path') return new URL(currentUrl).pathname.startsWith(b.value); - if (b.type === 'url') return currentUrl.startsWith(b.value); - } catch { - // ignore - } - return false; - }); - if (!ok) { - return { - runId: `run_${Date.now()}`, - success: false, - summary: { total: 0, success: 0, failed: 0, tookMs: 0 }, - url: currentUrl, - outputs: null, - logs: [ - { - stepId: 'binding-check', - status: 'failed', - message: - 'Flow binding mismatch. Provide startUrl or open a page matching flow.meta.bindings.', - }, - ], - screenshots: { onFailure: null }, - }; - } - } - } catch { - // ignore binding errors and continue - } - - // Optional: capture network for the whole run using Debugger-based tool (independent of webRequest) - let failed = 0; - let networkCaptureStarted = false; - const stopAndSummarizeNetwork = async () => { - try { - const stopRes = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_STOP, - args: {}, - }); - const text = (stopRes?.content || []).find((c: any) => c.type === 'text')?.text; - if (!text) return; - const data = JSON.parse(text); - const requests: any[] = Array.isArray(data?.requests) ? data.requests : []; - // Summarize top XHR/Fetch calls (method, url, status, duration) - const snippets = requests - .filter((r) => ['XHR', 'Fetch'].includes(String(r.type))) - .slice(0, 10) - .map((r) => ({ - method: String(r.method || 'GET'), - url: String(r.url || ''), - status: r.statusCode || r.status, - ms: Math.max(0, (r.responseTime || 0) - (r.requestTime || 0)), - })); - logs.push({ - stepId: 'network-capture', - status: 'success', - message: `Captured ${Number(data?.requestCount || 0)} requests` as any, - networkSnippets: snippets, - } as any); - } catch { - // ignore - } - }; - - // Helper: wait for network idle using webRequest-based capture loop - const waitForNetworkIdle = async (totalTimeoutMs: number, idleThresholdMs: number) => { - const deadline = Date.now() + Math.max(500, totalTimeoutMs); - const threshold = Math.max(200, idleThresholdMs); - while (Date.now() < deadline) { - // Start ephemeral capture with inactivity window - await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_CAPTURE_START, - args: { - includeStatic: false, - maxCaptureTime: Math.min(60_000, Math.max(threshold + 500, 2_000)), - inactivityTimeout: threshold, - }, - }); - // Give time for inactivity window to elapse if present - await new Promise((r) => setTimeout(r, threshold + 200)); - const stopRes = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_CAPTURE_STOP, - args: {}, - }); - const text = (stopRes?.content || []).find((c: any) => c.type === 'text')?.text; - try { - const json = text ? JSON.parse(text) : null; - const captureEnd = Number(json?.captureEndTime) || Date.now(); - const reqs: any[] = Array.isArray(json?.requests) ? json.requests : []; - const lastActivity = reqs.reduce( - (acc, r) => { - const t = Number(r.responseTime || r.requestTime || 0); - return t > acc ? t : acc; - }, - Number(json?.captureStartTime || 0), - ); - if (captureEnd - lastActivity >= threshold) { - return; // idle window achieved - } - } catch { - // ignore parse errors, try again until deadline - } - // Small backoff before next attempt - await new Promise((r) => setTimeout(r, Math.min(500, threshold))); - } - throw new Error('wait for network idle timed out'); - }; - - // Start long-running network capture if requested - if (options.captureNetwork) { - try { - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_START, - args: { includeStatic: false, maxCaptureTime: 3 * 60_000, inactivityTimeout: 0 }, - }); - if (!(res as any)?.isError) networkCaptureStarted = true; - } catch { - // ignore capture start failure - } - } - - // If DAG present, linearize to steps for M1 (default edges, topo order) - const stepsToRun: Step[] = (() => { - try { - if (Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0) { - const nodes = ((flow as any).nodes || []) as DagNode[]; - const edges = (((flow as any).edges || []) as DagEdge[]).filter( - (e) => !e.label || e.label === 'default', - ); - const order = topoOrder(nodes, edges); - return order.map((n) => mapDagNodeToStep(n)); - } - } catch { - // ignore and fallback - } - return flow.steps || []; - })(); - - // If a startNodeId is provided, slice the plan to start from that node/step id - const startIdx = options.startNodeId - ? stepsToRun.findIndex((s) => s?.id === options.startNodeId) - : -1; - const steps = startIdx >= 0 ? stepsToRun.slice(startIdx) : stepsToRun.slice(); - - try { - const pendingAfterScripts: StepScript[] = []; - for (const step of steps) { - const t0 = Date.now(); - const maxRetries = Math.max(0, step.retry?.count ?? 0); - const baseInterval = Math.max(0, step.retry?.intervalMs ?? 0); - let attempt = 0; - const doDelay = async (i: number) => { - const delay = - baseInterval > 0 - ? step.retry?.backoff === 'exp' - ? baseInterval * Math.pow(2, i) - : baseInterval - : 0; - if (delay > 0) await new Promise((r) => setTimeout(r, delay)); - }; - // Execution with retry - while (true) { - try { - // resolve string templates {var} - const resolveTemplate = (val?: string): string | undefined => - (val || '').replace(/\{([^}]+)\}/g, (_m, k) => (vars[k] ?? '').toString()); - - // Defer 'script' steps marked as after to run after next non-script step - if (step.type === 'script' && (step as any).when === 'after') { - pendingAfterScripts.push(step as any); - // Do not execute now; will run after the next non-script step (or at the end) - logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); - break; - } - - let stepLogged = false; - // Helper get current active tab URL and status - const getActiveTabInfo = async () => { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tab = tabs[0]; - return { url: tab?.url || '', status: (tab as any)?.status || '' }; - }; - // Wait for navigation completion or readiness - const waitForNavigation = async (prevUrl: string, timeoutMs: number) => { - const deadline = Date.now() + Math.max(1000, Math.min(timeoutMs || 15000, 30000)); - let sawLoading = false; - while (Date.now() < deadline) { - const { url, status } = await getActiveTabInfo(); - if (url && url !== prevUrl) return true; - if (status === 'loading') sawLoading = true; - if (sawLoading && status === 'complete') return true; - await new Promise((r) => setTimeout(r, 200)); - } - // as a last attempt, try a brief network idle wait - try { - await waitForNetworkIdle(2000, 800); - return true; - } catch (e) { - // noop - void 0; - } - throw new Error('navigation timeout'); - }; - - switch (step.type) { - case 'http': { - const s = step as any; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_REQUEST, - args: { - url: s.url, - method: s.method || 'GET', - headers: s.headers || {}, - body: s.body, - }, - }); - const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; - try { - const payload = text ? JSON.parse(text) : null; - if (s.saveAs && payload !== undefined) vars[s.saveAs] = payload; - if (s.assign && payload !== undefined) applyAssign(vars, payload, s.assign); - } catch { - // ignore parse error - } - break; - } - case 'extract': { - const s = step as any; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - let value: any = null; - if (s.js && String(s.js).trim()) { - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (code: string) => { - try { - return (0, eval)(code); - } catch (e) { - return null; - } - }, - args: [String(s.js)], - } as any); - value = result; - } else if (s.selector) { - const attr = String(s.attr || 'text'); - const sel = String(s.selector); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (selector: string, attr: string) => { - try { - const el = document.querySelector(selector) as any; - if (!el) return null; - if (attr === 'text' || attr === 'textContent') - return (el.textContent || '').trim(); - return el.getAttribute ? el.getAttribute(attr) : null; - } catch { - return null; - } - }, - args: [sel, attr], - } as any); - value = result; - } - if (s.saveAs) vars[s.saveAs] = value; - break; - } - case 'openTab': { - const s = step as any; - if (s.newWindow) { - await chrome.windows.create({ url: s.url || undefined, focused: true }); - } else { - await chrome.tabs.create({ url: s.url || undefined, active: true }); - } - break; - } - case 'switchTab': { - const s = step as any; - let targetTabId: number | undefined = s.tabId; - if (!targetTabId) { - const tabs = await chrome.tabs.query({}); - const hit = tabs.find( - (t) => - (s.urlContains && (t.url || '').includes(String(s.urlContains))) || - (s.titleContains && (t.title || '').includes(String(s.titleContains))), - ); - targetTabId = (hit && hit.id) as number | undefined; - } - if (targetTabId) { - await handleCallTool({ - name: TOOL_NAMES.BROWSER.SWITCH_TAB, - args: { tabId: targetTabId }, - }); - } else { - throw new Error('switchTab: no matching tab'); - } - break; - } - case 'closeTab': { - const s = step as any; - const args: any = {}; - if (Array.isArray(s.tabIds) && s.tabIds.length) args.tabIds = s.tabIds; - if (s.url) args.url = s.url; - const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.CLOSE_TABS, args }); - if ((res as any).isError) throw new Error('closeTab failed'); - break; - } - case 'scroll': { - const s = step as StepScroll; - const top = s.offset?.y ?? undefined; - const left = s.offset?.x ?? undefined; - const selectorFromTarget = (s.target?.candidates || []).find( - (c) => c.type === 'css' || c.type === 'attr', - )?.value; - - let code = ''; - if (s.mode === 'offset' && !s.target) { - const t = top != null ? Number(top) : 'undefined'; - const l = left != null ? Number(left) : 'undefined'; - code = `try { window.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {}`; - } else if (s.mode === 'element' && selectorFromTarget) { - code = `(() => { try { const el = document.querySelector(${JSON.stringify( - selectorFromTarget, - )}); if (el) el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }); } catch (e) {} })();`; - } else if (s.mode === 'container' && selectorFromTarget) { - const t = top != null ? Number(top) : 'undefined'; - const l = left != null ? Number(left) : 'undefined'; - code = `(() => { try { const el = document.querySelector(${JSON.stringify( - selectorFromTarget, - )}); if (el && typeof el.scrollTo === 'function') el.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {} })();`; - } else { - const direction = top != null && Number(top) < 0 ? 'up' : 'down'; - const amount = 3; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'scroll', scrollDirection: direction, scrollAmount: amount }, - }); - if ((res as any).isError) throw new Error('scroll failed'); - } - - if (code) { - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { type: 'MAIN', jsScript: code }, - }); - if ((res as any).isError) throw new Error('scroll failed'); - } - break; - } - case 'drag': { - const s = step as StepDrag; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - let startRef: string | undefined; - let endRef: string | undefined; - try { - if (typeof tabId === 'number') { - const locatedStart = await locateElement(tabId, s.start); - const locatedEnd = await locateElement(tabId, s.end); - startRef = locatedStart?.ref || s.start.ref; - endRef = locatedEnd?.ref || s.end.ref; - } - } catch { - // ignore - } - - let startCoordinates: { x: number; y: number } | undefined; - let endCoordinates: { x: number; y: number } | undefined; - if ((!startRef || !endRef) && Array.isArray(s.path) && s.path.length >= 2) { - startCoordinates = { x: Number(s.path[0].x), y: Number(s.path[0].y) }; - const last = s.path[s.path.length - 1]; - endCoordinates = { x: Number(last.x), y: Number(last.y) }; - } - - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { - action: 'left_click_drag', - startRef, - ref: endRef, - startCoordinates, - coordinates: endCoordinates, - }, - }); - if ((res as any).isError) throw new Error('drag failed'); - break; - } - case 'click': - case 'dblclick': { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - // Ensure helper script is loaded by leveraging existing read_page tooling - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const located = await locateElement(tabId, (step as any).target); - const first = (step as any).target?.candidates?.[0]?.type; - const resolvedBy = located?.resolvedBy || (located?.ref ? 'ref' : ''); - const fallbackUsed = - resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; - // minimal visibility check via resolveRef rect - if (located?.ref) { - const resolved = await chrome.tabs.sendMessage(tabId, { - action: 'resolveRef', - ref: located.ref, - } as any); - const rect = resolved?.rect; - if (!rect || rect.width <= 0 || rect.height <= 0) { - throw new Error('element not visible'); - } - } - // auto scroll into view if possible (unified) - try { - const sel = !located?.ref - ? (step as any).target?.candidates?.find( - (c: any) => c.type === 'css' || c.type === 'attr', - )?.value - : undefined; - if (sel) { - await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { - type: 'MAIN', - jsScript: `try{var el=document.querySelector(${JSON.stringify(sel)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, - }, - }); - } - } catch { - /* ignore */ - } - const prevInfo = await getActiveTabInfo(); - let res: any; - if (step.type === 'dblclick') { - // Use precise CDP-based double click for robustness - res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'double_click', ref: located?.ref || (step as any).target?.ref }, - }); - } else { - res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.CLICK, - args: { - ref: located?.ref || (step as any).target?.ref, - selector: !located?.ref - ? (step as any).target?.candidates?.find( - (c: any) => c.type === 'css' || c.type === 'attr', - )?.value - : undefined, - waitForNavigation: false, // we handle navigation explicitly below - timeout: Math.max(1000, Math.min(step.timeoutMs || 10000, 30000)), - }, - }); - } - if ((res as any).isError) throw new Error('click failed'); - // If navigation requested, wait explicitly with retries handled by outer loop - if ((step as any).after?.waitForNavigation) { - await waitForNavigation(prevInfo.url, Math.max(step.timeoutMs || 15000, 3000)); - } - if (fallbackUsed) { - logs.push({ - stepId: step.id, - status: 'success', - message: `Selector fallback used (${first} -> ${resolvedBy})`, - fallbackUsed: true, - fallbackFrom: String(first), - fallbackTo: String(resolvedBy), - tookMs: Date.now() - t0, - } as any); - continue; - } - break; - } - case 'fill': { - const s = step as StepFill; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const located = await locateElement(tabId, s.target); - const first = s.target?.candidates?.[0]?.type; - const resolvedBy = located?.resolvedBy || (located?.ref ? 'ref' : ''); - const fallbackUsed = - resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; - const value = resolveTemplate(s.value) ?? ''; - // minimal visibility check via resolveRef rect - if (located?.ref) { - const resolved = await chrome.tabs.sendMessage(tabId, { - action: 'resolveRef', - ref: located.ref, - } as any); - const rect = resolved?.rect; - if (!rect || rect.width <= 0 || rect.height <= 0) { - throw new Error('element not visible'); - } - } - // auto scroll into view if possible before fill - try { - const sel = !located?.ref - ? s.target.candidates?.find((c) => c.type === 'css' || c.type === 'attr')?.value - : undefined; - if (sel) { - await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { - type: 'MAIN', - jsScript: `try{var el=document.querySelector(${JSON.stringify(sel)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, - }, - }); - } - } catch { - /* ignore */ - } - // ensure focus before typing - try { - if (located?.ref) { - await chrome.tabs.sendMessage(tabId, { - action: 'focusByRef', - ref: located.ref, - } as any); - } else { - const sel = s.target.candidates?.find( - (c) => c.type === 'css' || c.type === 'attr', - )?.value; - if (sel) { - await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { - type: 'MAIN', - jsScript: `try{var el=document.querySelector(${JSON.stringify(sel)});if(el&&el.focus){el.focus();}}catch(e){}`, - }, - }); - } - } - } catch { - /* ignore */ - } - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.FILL, - args: { - ref: located?.ref || s.target.ref, - selector: !located?.ref - ? s.target.candidates?.find((c) => c.type === 'css' || c.type === 'attr')?.value - : undefined, - value, - }, - }); - if ((res as any).isError) throw new Error('fill failed'); - if (fallbackUsed) { - logs.push({ - stepId: step.id, - status: 'success', - message: `Selector fallback used (${first} -> ${resolvedBy})`, - fallbackUsed: true, - fallbackFrom: String(first), - fallbackTo: String(resolvedBy), - tookMs: Date.now() - t0, - } as any); - stepLogged = true; - break; - } - break; - } - case 'key': { - const s = step as StepKey; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.KEYBOARD, - args: { keys: s.keys }, - }); - if ((res as any).isError) throw new Error('key failed'); - break; - } - case 'wait': { - const s = step as StepWait; - if ('text' in s.condition) { - // Use wait-helper for text appearance/disappearance for more robustness - try { - await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { type: 'ISOLATED', jsScript: '' }, - }); - } catch (e) { - // noop - void 0; - } - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - // Ensure wait-helper is present - await chrome.scripting.executeScript({ - target: { tabId }, - files: ['inject-scripts/wait-helper.js'], - world: 'ISOLATED', - } as any); - const resp = await chrome.tabs.sendMessage(tabId, { - action: 'waitForText', - text: s.condition.text, - appear: s.condition.appear !== false, - timeout: Math.max(0, Math.min(step.timeoutMs || 10000, 120000)), - } as any); - if (!resp || resp.success !== true) throw new Error('wait text failed'); - } else if ('networkIdle' in s.condition) { - const total = Math.min(Math.max(1000, step.timeoutMs || 5000), 120000); - const idle = Math.min(1500, Math.max(500, Math.floor(total / 3))); - await waitForNetworkIdle(total, idle); - } else if ('navigation' in s.condition) { - // best-effort: wait a fixed time - const delay = Math.min(step.timeoutMs || 5000, 20000); - await new Promise((r) => setTimeout(r, delay)); - } else if ('selector' in s.condition) { - // Use wait-helper to wait for selector visibility - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - await chrome.scripting.executeScript({ - target: { tabId }, - files: ['inject-scripts/wait-helper.js'], - world: 'ISOLATED', - } as any); - const resp = await chrome.tabs.sendMessage(tabId, { - action: 'waitForSelector', - selector: (s.condition as any).selector, - visible: (s.condition as any).visible !== false, - timeout: Math.max(0, Math.min(step.timeoutMs || 10000, 120000)), - } as any); - if (!resp || resp.success !== true) throw new Error('wait selector failed'); - } - break; - } - case 'assert': { - const s = step as StepAssert; - // resolve using read_page to ensure element/text - if ('textPresent' in s.assert) { - const text = s.assert.textPresent; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'wait', text, appear: true, timeout: step.timeoutMs || 5000 }, - }); - if ((res as any).isError) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: 'assert text failed', - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error('assert text failed'); - } - } else if ('exists' in s.assert || 'visible' in s.assert) { - const selector = (s.assert as any).exists || (s.assert as any).visible; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const ensured = await chrome.tabs.sendMessage(tabId, { - action: 'ensureRefForSelector', - selector, - } as any); - if (!ensured || !ensured.success) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: 'assert selector not found', - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error('assert selector not found'); - } - if ('visible' in s.assert) { - const rect = ensured && ensured.center ? ensured.center : null; - // Minimal visibility check based on existence and center - if (!rect) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: 'assert visible failed', - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error('assert visible failed'); - } - } - } else if ('attribute' in s.assert) { - const { selector, name, equals, matches } = (s.assert as any).attribute || {}; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const resp = await chrome.tabs.sendMessage(tabId, { - action: 'getAttributeForSelector', - selector, - name, - } as any); - if (!resp || !resp.success) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: 'assert attribute: element not found', - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error('assert attribute: element not found'); - } - const actual: string | null = resp.value ?? null; - if (equals !== undefined && equals !== null) { - const expected = resolveTemplate(String(equals)) ?? ''; - if (String(actual) !== String(expected)) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: `assert attribute equals failed: ${name} actual=${String(actual)} expected=${String( - expected, - )}`, - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error( - `assert attribute equals failed: ${name} actual=${String(actual)} expected=${String( - expected, - )}`, - ); - } - } else if (matches !== undefined && matches !== null) { - try { - const re = new RegExp(String(matches)); - if (!re.test(String(actual))) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: `assert attribute matches failed: ${name} actual=${String(actual)} regex=${String( - matches, - )}`, - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error( - `assert attribute matches failed: ${name} actual=${String(actual)} regex=${String( - matches, - )}`, - ); - } - } catch (e) { - throw new Error(`invalid regex for attribute matches: ${String(matches)}`); - } - } else { - // Only check existence if no comparator provided - if (actual == null) { - if (s.failStrategy === 'warn') { - logs.push({ - stepId: step.id, - status: 'warning', - message: `assert attribute failed: ${name} missing`, - tookMs: Date.now() - t0, - }); - stepLogged = true; - break; - } - throw new Error(`assert attribute failed: ${name} missing`); - } - } - } - break; - } - case 'script': { - const s = step as any; - const world = s.world || 'ISOLATED'; - const code = String(s.code || ''); - if (!code.trim()) break; - // Prefer executeScript to capture return value for saveAs/assign - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (userCode: string) => { - try { - return (0, eval)(userCode); - } catch (e) { - return null; - } - }, - args: [code], - world: world as any, - } as any); - if (s.saveAs) vars[s.saveAs] = result; - if (s.assign && typeof s.assign === 'object') applyAssign(vars, result, s.assign); - break; - } - case 'navigate': { - const url = (step as any).url; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NAVIGATE, - args: { url }, - }); - if ((res as any).isError) throw new Error('navigate failed'); - break; - } - default: { - // not implemented types in M1 - await new Promise((r) => setTimeout(r, 0)); - } - } - if (!stepLogged) { - logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); - } - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) { - await chrome.tabs.sendMessage(tabs[0].id, { - action: 'rr_overlay', - cmd: 'append', - text: stepLogged ? `! ${step.type} (${step.id})` : `✔ ${step.type} (${step.id})`, - } as any); - } - } catch { - /* ignore */ - } - // Run any deferred after-scripts now that a non-script step completed - if (pendingAfterScripts.length > 0) { - while (pendingAfterScripts.length) { - const s = pendingAfterScripts.shift()!; - const tScript = Date.now(); - const world = (s as any).world || 'ISOLATED'; - const code = String((s as any).code || ''); - if (code.trim()) { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (userCode: string) => { - try { - return (0, eval)(userCode); - } catch { - return null; - } - }, - args: [code], - world: world as any, - } as any); - if ((s as any).saveAs) vars[(s as any).saveAs] = result; - if ((s as any).assign && typeof (s as any).assign === 'object') - applyAssign(vars, result, (s as any).assign); - } - logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); - } - } - break; // success, exit retry loop - } catch (e: any) { - if (attempt < maxRetries) { - logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); - await doDelay(attempt); - attempt += 1; - continue; - } - failed++; - logs.push({ - stepId: step.id, - status: 'failed', - message: e?.message || String(e), - tookMs: Date.now() - t0, - }); - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) { - await chrome.tabs.sendMessage(tabs[0].id, { - action: 'rr_overlay', - cmd: 'append', - text: `✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`, - } as any); - } - } catch { - /* ignore */ - } - if (step.screenshotOnFail !== false) { - try { - const shot = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'screenshot' }, - }); - const img = (shot?.content?.find((c: any) => c.type === 'image') as any) - ?.data as string; - if (img) logs[logs.length - 1].screenshotBase64 = img; - } catch { - // ignore - } - } - // stop on first failure after retries - throw e; - } - } - } - // Flush any trailing after-scripts if present - if (pendingAfterScripts.length > 0) { - while (pendingAfterScripts.length) { - const s = pendingAfterScripts.shift()!; - const tScript = Date.now(); - const world = (s as any).world || 'ISOLATED'; - const code = String((s as any).code || ''); - if (code.trim()) { - const wrapped = `(() => { try { ${code} } catch (e) { console.error('flow script error:', e); } })();`; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { type: world, jsScript: wrapped }, - }); - if ((res as any).isError) throw new Error('script(after) execution failed'); - } - logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); - } - } - } finally { - if (networkCaptureStarted) { - await stopAndSummarizeNetwork(); - } - } - - const tookMs = Date.now() - startAt; - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) { - await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'done' } as any); - } - } catch { - /* ignore */ - } - const record: RunRecord = { - id: runId, - flowId: flow.id, - startedAt: new Date(startAt).toISOString(), - finishedAt: new Date().toISOString(), - success: failed === 0, - entries: logs, - }; - await appendRun(record); - - return { - runId, - success: failed === 0, - summary: { - total: steps.length, - success: steps.length - failed, - failed, - tookMs, - }, - url: null, - outputs: null, - logs: options.returnLogs ? logs : undefined, - screenshots: { onFailure: logs.find((l) => l.status === 'failed')?.screenshotBase64 }, - }; -} - -// --- DAG helpers (M1: default-edge serial) --- -function topoOrder(nodes: DagNode[], edges: DagEdge[]): DagNode[] { - const id2n = new Map(nodes.map((n) => [n.id, n] as const)); - const indeg = new Map(nodes.map((n) => [n.id, 0] as const)); - for (const e of edges) indeg.set(e.to, (indeg.get(e.to) || 0) + 1); - const nexts = new Map(nodes.map((n) => [n.id, [] as string[]] as const)); - for (const e of edges) nexts.get(e.from)!.push(e.to); - const q: string[] = nodes.filter((n) => (indeg.get(n.id) || 0) === 0).map((n) => n.id); - const out: DagNode[] = []; - while (q.length) { - const id = q.shift()!; - const n = id2n.get(id); - if (!n) continue; - out.push(n); - for (const v of nexts.get(id)!) { - indeg.set(v, (indeg.get(v) || 0) - 1); - if ((indeg.get(v) || 0) === 0) q.push(v); - } - } - return out.length === nodes.length ? out : nodes.slice(); -} - -function mapDagNodeToStep(n: DagNode): Step { - const c: any = n.config || {}; - const base = { id: n.id } as any; - if (n.type === 'click' || n.type === 'dblclick') - return { - ...base, - type: n.type, - target: c.target || { candidates: [] }, - before: c.before, - after: c.after, - } as any; - if (n.type === 'fill') - return { - ...base, - type: 'fill', - target: c.target || { candidates: [] }, - value: c.value || '', - } as any; - if (n.type === 'key') return { ...base, type: 'key', keys: c.keys || '' } as any; - if (n.type === 'wait') - return { ...base, type: 'wait', condition: c.condition || { text: '', appear: true } } as any; - if (n.type === 'assert') - return { - ...base, - type: 'assert', - assert: c.assert || { exists: '' }, - failStrategy: c.failStrategy, - } as any; - if (n.type === 'navigate') return { ...base, type: 'navigate', url: c.url || '' } as any; - if (n.type === 'script') - return { - ...base, - type: 'script', - world: c.world || 'ISOLATED', - code: c.code || '', - when: c.when, - } as any; - if (n.type === 'delay') - return { - ...base, - type: 'wait', - timeoutMs: Math.max(0, Number(c.ms ?? 1000)), - condition: { navigation: true }, - } as any; - // Fallback: no-op script - return { ...base, type: 'script', world: 'ISOLATED', code: '' } as any; -} +// thin re-export for backward compatibility +export { runFlow } from './runner'; +export type { RunOptions } from './runner'; diff --git a/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts b/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts index 48964973..36f1e4f2 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts @@ -43,7 +43,25 @@ export async function listRuns(): Promise { export async function appendRun(record: RunRecord): Promise { const runs = await listRuns(); runs.push(record); - await chrome.storage.local.set({ [STORAGE_KEYS.RR_RUNS]: runs }); + // Trim to keep last 10 runs per flowId to avoid unbounded growth + try { + const byFlow = new Map(); + for (const r of runs) { + const list = byFlow.get(r.flowId) || []; + list.push(r); + byFlow.set(r.flowId, list); + } + const merged: RunRecord[] = []; + for (const [fid, arr] of byFlow.entries()) { + // keep last 10 by startedAt chronological order + arr.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime()); + const last = arr.slice(Math.max(0, arr.length - 10)); + merged.push(...last); + } + await chrome.storage.local.set({ [STORAGE_KEYS.RR_RUNS]: merged }); + } catch { + await chrome.storage.local.set({ [STORAGE_KEYS.RR_RUNS]: runs }); + } } export async function listPublished(): Promise { diff --git a/app/chrome-extension/entrypoints/background/record-replay/index.ts b/app/chrome-extension/entrypoints/background/record-replay/index.ts index dff91642..ab9032c8 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/index.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/index.ts @@ -15,6 +15,9 @@ import { removeSchedule, type FlowSchedule, } from './flow-store'; +import { listRuns } from './flow-store'; +import { STORAGE_KEYS } from '@/common/constants'; +import { listTriggers, saveTrigger, deleteTrigger, type FlowTrigger } from './trigger-store'; import { runFlow } from './flow-runner'; // design note: background listener for record & replay; manages start/stop and storage @@ -59,14 +62,14 @@ async function rescheduleAlarms() { } async function ensureRecorderInjected(tabId: number): Promise { - // Inject helper and recorder scripts + // Inject helper and recorder scripts into all frames to aggregate same-origin iframes await chrome.scripting.executeScript({ - target: { tabId }, + target: { tabId, allFrames: true }, files: ['inject-scripts/accessibility-tree-helper.js'], world: 'ISOLATED', } as any); await chrome.scripting.executeScript({ - target: { tabId }, + target: { tabId, allFrames: true }, files: ['inject-scripts/recorder.js'], world: 'ISOLATED', } as any); @@ -78,15 +81,22 @@ async function startRecording(meta?: Partial): Promise<{ success: boolean; try { await ensureRecorderInjected(tab.id); currentRecording = { tabId: tab.id }; - await chrome.tabs.sendMessage(tab.id, { - action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, - cmd: 'start', - meta: { - id: meta?.id, - name: meta?.name, - description: meta?.description, - }, - }); + // Broadcast to all frames + const frames = await chrome.webNavigation.getAllFrames({ tabId: tab.id }); + const targets = Array.isArray(frames) && frames.length ? frames : [{ frameId: 0 } as any]; + await Promise.all( + targets.map((f) => + chrome.tabs.sendMessage( + tab.id!, + { + action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, + cmd: 'start', + meta: { id: meta?.id, name: meta?.name, description: meta?.description }, + } as any, + { frameId: f.frameId } as any, + ), + ), + ); return { success: true }; } catch (e: any) { return { success: false, error: e?.message || String(e) }; @@ -96,10 +106,12 @@ async function startRecording(meta?: Partial): Promise<{ success: boolean; async function stopRecording(): Promise<{ success: boolean; flow?: Flow; error?: string }> { if (!currentRecording) return { success: false, error: 'No active recording' }; try { - const flowRes: any = await chrome.tabs.sendMessage(currentRecording.tabId, { - action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, - cmd: 'stop', - }); + // Ask top frame to stop and return flow (child frames carry step batches too) + const flowRes: any = await chrome.tabs.sendMessage( + currentRecording.tabId, + { action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, cmd: 'stop' } as any, + { frameId: 0 } as any, + ); const flowFromTab = (flowRes && flowRes.flow) as Flow | undefined; // 合并后台聚合的步骤（跨标签页）与内容脚本返回的步骤 const aggregated = currentRecording.flow; @@ -135,6 +147,8 @@ async function stopRecording(): Promise<{ success: boolean; flow?: Flow; error?: export function initRecordReplayListeners() { // On startup, re-schedule alarms rescheduleAlarms().catch(() => {}); + // Initialize trigger engine (contextMenus/commands/url/dom) + initTriggerEngine().catch(() => {}); chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { try { @@ -267,7 +281,53 @@ export function initRecordReplayListeners() { } case BACKGROUND_MESSAGE_TYPES.RR_IMPORT_FLOW: { importFlowFromJson(message.json) - .then((flows) => sendResponse({ success: true, imported: flows.length })) + .then((flows) => sendResponse({ success: true, imported: flows.length, flows })) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_LIST_RUNS: { + listRuns() + .then((runs) => sendResponse({ success: true, runs })) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_LIST_TRIGGERS: { + listTriggers() + .then((triggers) => sendResponse({ success: true, triggers })) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_SAVE_TRIGGER: { + const t = message.trigger as FlowTrigger; + if (!t || !t.id || !t.type || !t.flowId) { + sendResponse({ success: false, error: 'invalid trigger' }); + return true; + } + saveTrigger(t) + .then(async () => { + await refreshTriggers(); + sendResponse({ success: true }); + }) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_DELETE_TRIGGER: { + const id = String(message.id || ''); + if (!id) { + sendResponse({ success: false, error: 'invalid id' }); + return true; + } + deleteTrigger(id) + .then(async () => { + await refreshTriggers(); + sendResponse({ success: true }); + }) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_REFRESH_TRIGGERS: { + refreshTriggers() + .then(() => sendResponse({ success: true })) .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); return true; } @@ -345,10 +405,17 @@ export function initRecordReplayListeners() { // 确保该标签页注入并继续录制（不重置流） try { await ensureRecorderInjected(activeInfo.tabId); - await chrome.tabs.sendMessage(activeInfo.tabId, { - action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, - cmd: 'resume', - } as any); + const frames = await chrome.webNavigation.getAllFrames({ tabId: activeInfo.tabId }); + const targets = Array.isArray(frames) && frames.length ? frames : [{ frameId: 0 } as any]; + await Promise.all( + targets.map((f) => + chrome.tabs.sendMessage( + activeInfo.tabId!, + { action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, cmd: 'resume' } as any, + { frameId: f.frameId } as any, + ), + ), + ); } catch { /* ignore */ } @@ -383,10 +450,17 @@ export function initRecordReplayListeners() { // 同时确保继续录制 try { await ensureRecorderInjected(tabId); - await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, - cmd: 'resume', - } as any); + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + const targets = Array.isArray(frames) && frames.length ? frames : [{ frameId: 0 } as any]; + await Promise.all( + targets.map((f) => + chrome.tabs.sendMessage( + tabId!, + { action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, cmd: 'resume' } as any, + { frameId: f.frameId } as any, + ), + ), + ); } catch { /* ignore */ } @@ -403,10 +477,17 @@ export function initRecordReplayListeners() { // 仍确保脚本在该页活跃，用于持续录制 try { await ensureRecorderInjected(tabId); - await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, - cmd: 'resume', - } as any); + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + const targets = Array.isArray(frames) && frames.length ? frames : [{ frameId: 0 } as any]; + await Promise.all( + targets.map((f) => + chrome.tabs.sendMessage( + tabId!, + { action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, cmd: 'resume' } as any, + { frameId: f.frameId } as any, + ), + ), + ); } catch { /* ignore */ } @@ -438,6 +519,133 @@ export function initRecordReplayListeners() { // ignore } }); + + // Trigger engine: contextMenus/commands/url/dom + if ((chrome as any).contextMenus?.onClicked?.addListener) { + chrome.contextMenus.onClicked.addListener(async (info) => { + try { + const triggers = await listTriggers(); + const t = triggers.find( + (x) => x.type === 'contextMenu' && (x as any).menuId === info.menuItemId, + ); + if (!t || t.enabled === false) return; + const flow = await getFlow(t.flowId); + if (!flow) return; + await runFlow(flow, { args: t.args || {}, returnLogs: false }); + } catch {} + }); + } + chrome.commands.onCommand.addListener(async (command) => { + try { + const triggers = await listTriggers(); + const t = triggers.find((x) => x.type === 'command' && (x as any).commandKey === command); + if (!t || t.enabled === false) return; + const flow = await getFlow(t.flowId); + if (!flow) return; + await runFlow(flow, { args: t.args || {}, returnLogs: false }); + } catch {} + }); + chrome.webNavigation.onCommitted.addListener(async (details) => { + try { + if (details.frameId !== 0) return; + const url = details.url || ''; + const triggers = await listTriggers(); + const list = triggers.filter((x) => x.type === 'url' && x.enabled !== false) as any[]; + for (const t of list) { + if (matchUrl(url, (t as any).match || [])) { + const flow = await getFlow(t.flowId); + if (!flow) continue; + await runFlow(flow, { args: t.args || {}, returnLogs: false }); + } + } + } catch {} + }); + chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + try { + if (message && message.action === 'dom_trigger_fired') { + const id = message.triggerId; + listTriggers().then(async (arr) => { + const t = arr.find((x) => x.id === id && x.type === 'dom'); + if (!t || t.enabled === false) return; + const flow = await getFlow(t.flowId); + if (!flow) return; + await runFlow(flow, { args: t.args || {}, returnLogs: false }); + }); + sendResponse({ ok: true }); + return true; + } + } catch {} + return false; + }); +} + +function matchUrl( + u: string, + rules: Array<{ kind: 'url' | 'domain' | 'path'; value: string }>, +): boolean { + try { + const url = new URL(u); + for (const r of rules || []) { + const v = String(r.value || ''); + if (r.kind === 'url' && u.startsWith(v)) return true; + if (r.kind === 'domain' && url.hostname.includes(v)) return true; + if (r.kind === 'path' && url.pathname.startsWith(v)) return true; + } + } catch {} + return false; +} + +async function refreshTriggers() { + try { + const triggers = await listTriggers(); + // Guard: contextMenus permission may be missing in some builds + if ((chrome as any).contextMenus?.removeAll && (chrome as any).contextMenus?.create) { + try { + await chrome.contextMenus.removeAll(); + } catch {} + for (const t of triggers) { + if (t.type === 'contextMenu' && t.enabled !== false) { + const id = `rr_menu_${t.id}`; + (t as any).menuId = id; + await chrome.contextMenus.create({ + id, + title: (t as any).title || '运行工作流', + contexts: (t as any).contexts || ['all'], + }); + } + } + } + await chrome.storage.local.set({ [STORAGE_KEYS.RR_TRIGGERS]: triggers }); + const domTriggers = triggers + .filter((x) => x.type === 'dom' && x.enabled !== false) + .map((x: any) => ({ + id: x.id, + selector: x.selector, + appear: x.appear !== false, + once: x.once !== false, + debounceMs: x.debounceMs ?? 800, + })); + const tabs = await chrome.tabs.query({}); + for (const t of tabs) { + if (!t.id) continue; + try { + await chrome.scripting.executeScript({ + target: { tabId: t.id, allFrames: true }, + files: ['inject-scripts/dom-observer.js'], + world: 'ISOLATED', + } as any); + await chrome.tabs.sendMessage(t.id, { + action: 'set_dom_triggers', + triggers: domTriggers, + } as any); + } catch {} + } + } catch {} +} + +// Backward-compatible init function; initialize all trigger-related hooks/state +async function initTriggerEngine() { + await refreshTriggers(); } // Alarm listener executes scheduled flows diff --git a/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts b/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts new file mode 100644 index 00000000..18e201ce --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts @@ -0,0 +1,1067 @@ +// node-registry.ts — execute a single step +// Note: keep side-effects minimal; use provided helpers and ctx.logger for logs + +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '../tools'; +import type { + RunLogEntry, + Step, + StepAssert, + StepDrag, + StepFill, + StepKey, + StepScroll, + StepScript, + StepWait, +} from './types'; +import { locateElement } from './selector-engine'; +import { + applyAssign, + expandTemplatesDeep, + waitForNetworkIdle, + waitForNavigation, +} from './rr-utils'; + +export interface ExecCtx { + vars: Record; + logger: (e: RunLogEntry) => void; + // Current frame context for same-origin iframe operations; undefined means top frame + frameId?: number; +} + +export interface ExecResult { + alreadyLogged?: boolean; + deferAfterScript?: StepScript | null; + // next edge label to follow; supports 'true'/'false' (legacy) and + // arbitrary labels like 'case:' / 'case:else' for conditional branches + nextLabel?: string; + control?: + | { kind: 'foreach'; listVar: string; itemVar: string; subflowId: string } + | { kind: 'while'; condition: any; subflowId: string; maxIterations: number }; +} + +// NodeRuntime registry scaffolding (incremental adoption) +export interface NodeRuntime { + validate?: (step: S) => { ok: boolean; errors?: string[] }; + run: (ctx: ExecCtx, step: S) => Promise; +} + +const registry: Partial>> = { + http: { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_REQUEST, + args: { + url: s.url, + method: s.method || 'GET', + headers: s.headers || {}, + body: s.body, + formData: s.formData, + }, + }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload !== undefined) ctx.vars[s.saveAs] = payload; + if (s.assign && payload !== undefined) applyAssign(ctx.vars, payload, s.assign); + } catch {} + }, + }, + extract: { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + let value: any = null; + if (s.js && String(s.js).trim()) { + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (code: string) => { + try { + return (0, eval)(code); + } catch (e) { + return null; + } + }, + args: [String(s.js)], + } as any); + value = result; + } else if (s.selector) { + const attr = String(s.attr || 'text'); + const sel = String(s.selector); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (selector: string, attr: string) => { + try { + const el = document.querySelector(selector) as any; + if (!el) return null; + if (attr === 'text' || attr === 'textContent') return (el.textContent || '').trim(); + return el.getAttribute ? el.getAttribute(attr) : null; + } catch { + return null; + } + }, + args: [sel, attr], + } as any); + value = result; + } + if (s.saveAs) ctx.vars[s.saveAs] = value; + }, + }, + script: { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + if (s.when === 'after') return { deferAfterScript: s }; + const world = s.world || 'ISOLATED'; + const code = String(s.code || ''); + if (!code.trim()) return {}; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if (s.saveAs) ctx.vars[s.saveAs] = result; + if (s.assign && typeof s.assign === 'object') applyAssign(ctx.vars, result, s.assign); + return {}; + }, + }, + openTab: { + run: async (_ctx, step) => { + const s: any = expandTemplatesDeep(step as any, {}); + if (s.newWindow) await chrome.windows.create({ url: s.url || undefined, focused: true }); + else await chrome.tabs.create({ url: s.url || undefined, active: true }); + }, + }, + switchTab: { + run: async (_ctx, step) => { + const s: any = expandTemplatesDeep(step as any, {}); + let targetTabId: number | undefined = s.tabId; + if (!targetTabId) { + const tabs = await chrome.tabs.query({}); + const hit = tabs.find( + (t) => + (s.urlContains && (t.url || '').includes(String(s.urlContains))) || + (s.titleContains && (t.title || '').includes(String(s.titleContains))), + ); + targetTabId = (hit && hit.id) as number | undefined; + } + if (!targetTabId) throw new Error('switchTab: no matching tab'); + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.SWITCH_TAB, + args: { tabId: targetTabId }, + }); + if ((res as any).isError) throw new Error('switchTab failed'); + }, + }, + closeTab: { + run: async (_ctx, step) => { + const s: any = expandTemplatesDeep(step as any, {}); + const args: any = {}; + if (Array.isArray(s.tabIds) && s.tabIds.length) args.tabIds = s.tabIds; + if (s.url) args.url = s.url; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.CLOSE_TABS, args }); + if ((res as any).isError) throw new Error('closeTab failed'); + }, + }, + scroll: { + run: async (_ctx, step: StepScroll) => { + const s = step as StepScroll; + const top = s.offset?.y ?? undefined; + const left = s.offset?.x ?? undefined; + const selectorFromTarget = (s.target?.candidates || []).find( + (c) => c.type === 'css' || c.type === 'attr', + )?.value; + let code = ''; + if (s.mode === 'offset' && !s.target) { + const t = top != null ? Number(top) : 'undefined'; + const l = left != null ? Number(left) : 'undefined'; + code = `try { window.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {}`; + } else if (s.mode === 'element' && selectorFromTarget) { + code = `(() => { try { const el = document.querySelector(${JSON.stringify(selectorFromTarget)}); if (el) el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }); } catch (e) {} })();`; + } else if (s.mode === 'container' && selectorFromTarget) { + const t = top != null ? Number(top) : 'undefined'; + const l = left != null ? Number(left) : 'undefined'; + code = `(() => { try { const el = document.querySelector(${JSON.stringify(selectorFromTarget)}); if (el && typeof el.scrollTo === 'function') el.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {} })();`; + } else { + const direction = top != null && Number(top) < 0 ? 'up' : 'down'; + const amount = 3; + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'scroll', scrollDirection: direction, scrollAmount: amount }, + }); + if ((res as any).isError) throw new Error('scroll failed'); + return {}; + } + if (code) { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { type: 'MAIN', jsScript: code }, + }); + if ((res as any).isError) throw new Error('scroll failed'); + } + return {}; + }, + }, + drag: { + run: async (_ctx, step: StepDrag) => { + const s = step as StepDrag; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + let startRef: string | undefined; + let endRef: string | undefined; + try { + if (typeof tabId === 'number') { + const locatedStart = await locateElement(tabId, s.start); + const locatedEnd = await locateElement(tabId, s.end); + startRef = (locatedStart as any)?.ref || s.start.ref; + endRef = (locatedEnd as any)?.ref || s.end.ref; + } + } catch {} + let startCoordinates: { x: number; y: number } | undefined; + let endCoordinates: { x: number; y: number } | undefined; + if ((!startRef || !endRef) && Array.isArray(s.path) && s.path.length >= 2) { + startCoordinates = { x: Number(s.path[0].x), y: Number(s.path[0].y) }; + const last = s.path[s.path.length - 1]; + endCoordinates = { x: Number(last.x), y: Number(last.y) }; + } + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { + action: 'left_click_drag', + startRef, + ref: endRef, + startCoordinates, + coordinates: endCoordinates, + }, + }); + if ((res as any).isError) throw new Error('drag failed'); + }, + }, + click: { + validate: (step) => { + const ok = !!(step as any).target?.candidates?.length; + return ok ? { ok } : { ok, errors: ['缺少目标选择器候选'] }; + }, + run: async (ctx, step) => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const located = await locateElement(tabId, s.target, ctx.frameId); + const first = s.target?.candidates?.[0]?.type; + const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); + const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; + if ((located as any)?.ref) { + const resolved = await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + ); + const rect = resolved?.rect; + if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); + } + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.CLICK, + args: { + ref: (located as any)?.ref || s.target?.ref, + selector: !(located as any)?.ref + ? s.target?.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined, + waitForNavigation: false, + timeout: Math.max(1000, Math.min(s.timeoutMs || 10000, 30000)), + }, + }); + if ((res as any).isError) throw new Error('click failed'); + if (fallbackUsed) + ctx.logger({ + stepId: step.id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + }, + }, + dblclick: { + validate: (step) => { + const ok = !!(step as any).target?.candidates?.length; + return ok ? { ok } : { ok, errors: ['缺少目标选择器候选'] }; + }, + run: async (ctx, step) => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const located = await locateElement(tabId, s.target, ctx.frameId); + const first = s.target?.candidates?.[0]?.type; + const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); + const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; + if ((located as any)?.ref) { + const resolved = await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + ); + const rect = resolved?.rect; + if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); + } + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'double_click', ref: (located as any)?.ref || (step as any).target?.ref }, + }); + if ((res as any).isError) throw new Error('dblclick failed'); + if (fallbackUsed) + ctx.logger({ + stepId: step.id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + }, + }, + fill: { + validate: (step) => { + const ok = !!(step as any).target?.candidates?.length && 'value' in (step as any); + return ok ? { ok } : { ok, errors: ['缺少目标选择器候选或输入值'] }; + }, + run: async (ctx, step: StepFill) => { + const s = step as StepFill; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const located = await locateElement(tabId, s.target, ctx.frameId); + const first = s.target?.candidates?.[0]?.type; + const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); + const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; + const value = (s.value || '').replace(/\{([^}]+)\}/g, (_m, k) => + (ctx.vars[k] ?? '').toString(), + ); + if ((located as any)?.ref) { + const resolved = await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + ); + const rect = resolved?.rect; + if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); + } + // Special-case: file inputs must use CDP setInputFiles (file upload tool) instead of value assignment + // Prefer CSS/attr selector when available + const cssSelector = !(located as any)?.ref + ? s.target.candidates?.find((c) => c.type === 'css' || c.type === 'attr')?.value + : undefined; + if (cssSelector) { + try { + const attr = await chrome.tabs.sendMessage( + tabId, + { + action: 'getAttributeForSelector', + selector: cssSelector, + name: 'type', + } as any, + { frameId: ctx.frameId } as any, + ); + const typeName = (attr && attr.value ? String(attr.value) : '').toLowerCase(); + if (typeName === 'file') { + const uploadRes = await handleCallTool({ + name: TOOL_NAMES.BROWSER.FILE_UPLOAD, + args: { selector: cssSelector, filePath: value }, + }); + if ((uploadRes as any).isError) throw new Error('file upload failed'); + if (fallbackUsed) + ctx.logger({ + stepId: step.id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + return {} as any; + } + } catch { + // continue to normal fill on errors + } + } + try { + // Scroll into view then focus before filling + if (cssSelector) + await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { + type: 'MAIN', + jsScript: `try{var el=document.querySelector(${JSON.stringify(cssSelector)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, + }, + }); + } catch {} + try { + if ((located as any)?.ref) + await chrome.tabs.sendMessage( + tabId, + { action: 'focusByRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + ); + else if (cssSelector) + await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { + type: 'MAIN', + jsScript: `try{var el=document.querySelector(${JSON.stringify(cssSelector)});if(el&&el.focus){el.focus();}}catch(e){}`, + }, + }); + } catch {} + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.FILL, + args: { + ref: (located as any)?.ref || s.target?.ref, + selector: cssSelector, + value, + }, + }); + if ((res as any).isError) throw new Error('fill failed'); + if (fallbackUsed) + ctx.logger({ + stepId: step.id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + }, + }, + key: { + run: async (_ctx, step: StepKey) => { + const s = expandTemplatesDeep(step as StepKey, {}); + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.KEYBOARD, + args: { keys: (s as StepKey).keys }, + }); + if ((res as any).isError) throw new Error('key failed'); + }, + }, + wait: { + validate: (step) => { + const ok = !!(step as any).condition; + return ok ? { ok } : { ok, errors: ['缺少等待条件'] }; + }, + run: async (_ctx, step: StepWait) => { + const s = expandTemplatesDeep(step as StepWait, {}); + const cond = (s as StepWait).condition as + | { selector: string; visible?: boolean } + | { text: string; appear?: boolean } + | { navigation: true } + | { networkIdle: true } + | { sleep: number }; + if ('text' in cond) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await chrome.scripting.executeScript({ + target: { tabId }, + files: ['inject-scripts/wait-helper.js'], + world: 'ISOLATED', + } as any); + const resp = await chrome.tabs.sendMessage( + tabId, + { + action: 'waitForText', + text: cond.text, + appear: cond.appear !== false, + timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), + } as any, + { frameId: ctx.frameId } as any, + ); + if (!resp || resp.success !== true) throw new Error('wait text failed'); + } else if ('networkIdle' in cond) { + const total = Math.min(Math.max(1000, (s as any).timeoutMs || 5000), 120000); + const idle = Math.min(1500, Math.max(500, Math.floor(total / 3))); + await waitForNetworkIdle(total, idle); + } else if ('navigation' in cond) { + await waitForNavigation((s as any).timeoutMs); + } else if ('sleep' in cond) { + const ms = Math.max(0, Number(cond.sleep ?? 0)); + await new Promise((r) => setTimeout(r, ms)); + } else if ('selector' in cond) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await chrome.scripting.executeScript({ + target: { tabId }, + files: ['inject-scripts/wait-helper.js'], + world: 'ISOLATED', + } as any); + const resp = await chrome.tabs.sendMessage(tabId, { + action: 'waitForSelector', + selector: cond.selector, + visible: cond.visible !== false, + timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), + } as any); + if (!resp || resp.success !== true) throw new Error('wait selector failed'); + } + }, + }, + assert: { + validate: (step) => { + const s = step as any; + const ok = !!s.assert; + // basic shape checks for attribute + if (ok && s.assert && 'attribute' in s.assert) { + const a = s.assert.attribute || {}; + if (!a.selector || !a.name) + return { ok: false, errors: ['assert.attribute: 需提供 selector 与 name'] }; + } + return ok ? { ok } : { ok, errors: ['缺少断言条件'] }; + }, + run: async (ctx, step: StepAssert) => { + const s = expandTemplatesDeep(step as StepAssert, {}) as StepAssert; + const failStrategy = (s as any).failStrategy || 'stop'; + const fail = (msg: string) => { + if (failStrategy === 'warn') { + ctx.logger({ stepId: step.id, status: 'warning', message: msg }); + return { alreadyLogged: true } as any; + } + // retry/stop -> throw to let runner decide by step.retry + throw new Error(msg); + }; + if ('textPresent' in s.assert) { + const text = (s.assert as any).textPresent; + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'wait', text, appear: true, timeout: (step as any).timeoutMs || 5000 }, + }); + if ((res as any).isError) return fail('assert text failed'); + } else if ('exists' in s.assert || 'visible' in s.assert) { + const selector = (s.assert as any).exists || (s.assert as any).visible; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) return fail('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const ensured = await chrome.tabs.sendMessage(tabId, { + action: 'ensureRefForSelector', + selector, + } as any); + if (!ensured || !ensured.success) return fail('assert selector not found'); + if ('visible' in s.assert) { + const rect = ensured && ensured.center ? ensured.center : null; + if (!rect) return fail('assert visible failed'); + } + } else if ('attribute' in s.assert) { + const { selector, name, equals, matches } = (s.assert as any).attribute || {}; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) return fail('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const resp = await chrome.tabs.sendMessage( + tabId, + { action: 'getAttributeForSelector', selector, name } as any, + { frameId: ctx.frameId } as any, + ); + if (!resp || !resp.success) return fail('assert attribute: element not found'); + const actual: string | null = resp.value ?? null; + if (equals !== undefined && equals !== null) { + const expected = String(equals); + if (String(actual) !== String(expected)) + return fail( + `assert attribute equals failed: ${name} actual=${String(actual)} expected=${String(expected)}`, + ); + } else if (matches !== undefined && matches !== null) { + try { + const re = new RegExp(String(matches)); + if (!re.test(String(actual))) + return fail( + `assert attribute matches failed: ${name} actual=${String(actual)} regex=${String(matches)}`, + ); + } catch { + return fail(`invalid regex for attribute matches: ${String(matches)}`); + } + } else { + if (actual == null) return fail(`assert attribute failed: ${name} missing`); + } + } + return {} as any; + }, + }, + navigate: { + validate: (step) => { + const ok = !!(step as any).url; + return ok ? { ok } : { ok, errors: ['缺少 URL'] }; + }, + run: async (_ctx, step) => { + const url = (step as any).url; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url } }); + if ((res as any).isError) throw new Error('navigate failed'); + }, + }, + if: { + validate: (step) => { + const s = step as any; + const hasBranches = Array.isArray(s.branches) && s.branches.length > 0; + const ok = hasBranches || !!s.condition; + return ok ? { ok } : { ok, errors: ['缺少条件或分支'] }; + }, + run: async (ctx, step) => { + const s: any = step; + // Branch-first evaluation when branches[] provided; fallback to legacy condition + if (Array.isArray(s.branches) && s.branches.length > 0) { + // evaluate in order; first matched wins + const evalExpr = (expr: string): boolean => { + const code = String(expr || '').trim(); + if (!code) return false; + try { + // Note: basic eval with limited scope. Support both `vars` and legacy `workflow` identifiers. + // This preserves backward compatibility with older expressions written as `workflow.xxx`. + const fn = new Function( + 'vars', + 'workflow', + `try { return !!(${code}); } catch (e) { return false; }`, + ); + return !!fn(ctx.vars, ctx.vars); + } catch { + return false; + } + }; + for (const b of s.branches) { + if (evalExpr(b.expr)) return { nextLabel: `case:${String(b.id)}` } as ExecResult; + } + // none matched + return { nextLabel: s.else === false ? 'default' : 'case:else' } as ExecResult; + } else { + // legacy single condition -> true/false + const cond = s.condition || {}; + let result = false; + try { + if (typeof cond.expression === 'string' && cond.expression.trim()) { + const fn = new Function( + 'vars', + `try { return !!(${cond.expression}); } catch (e) { return false; }`, + ); + result = !!fn(ctx.vars); + } else if (typeof cond.var === 'string') { + const v = ctx.vars[cond.var]; + if ('equals' in cond) result = String(v) === String(cond.equals); + else result = !!v; + } + } catch { + result = false; + } + return { nextLabel: result ? 'true' : 'false' } as ExecResult; + } + }, + }, + foreach: { + validate: (step) => { + const s = step as any; + const ok = + typeof s.listVar === 'string' && + s.listVar && + typeof s.subflowId === 'string' && + s.subflowId; + return ok ? { ok } : { ok, errors: ['foreach: 需提供 listVar 与 subflowId'] }; + }, + run: async (_ctx, step) => { + const s: any = step; + const itemVar = typeof s.itemVar === 'string' && s.itemVar ? s.itemVar : 'item'; + return { + control: { + kind: 'foreach', + listVar: String(s.listVar), + itemVar, + subflowId: String(s.subflowId), + }, + } as ExecResult; + }, + }, + while: { + validate: (step) => { + const s = step as any; + const ok = !!s.condition && typeof s.subflowId === 'string' && s.subflowId; + return ok ? { ok } : { ok, errors: ['while: 需提供 condition 与 subflowId'] }; + }, + run: async (_ctx, step) => { + const s: any = step; + const max = Math.max(1, Math.min(10000, Number(s.maxIterations ?? 100))); + return { + control: { + kind: 'while', + condition: s.condition, + subflowId: String(s.subflowId), + maxIterations: max, + }, + } as ExecResult; + }, + }, + executeFlow: { + validate: (step) => { + const s: any = step; + const ok = typeof s.flowId === 'string' && !!s.flowId; + return ok ? { ok } : { ok, errors: ['需提供 flowId'] }; + }, + run: async (ctx, step) => { + const s: any = step; + const { getFlow } = await import('./flow-store'); + const flow = await getFlow(String(s.flowId)); + if (!flow) throw new Error('referenced flow not found'); + const inline = s.inline !== false; // default inline + if (!inline) { + const { runFlow } = await import('./flow-runner'); + await runFlow(flow, { args: s.args || {}, returnLogs: false }); + return; + } + // Inline: execute referenced flow's steps with current ctx/vars + const { + defaultEdgesOnly, + topoOrder, + mapDagNodeToStep, + waitForNetworkIdle, + waitForNavigation, + } = await import('./rr-utils'); + const vars = ctx.vars; + if (s.args && typeof s.args === 'object') Object.assign(vars, s.args); + const hasDag = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; + const nodes = hasDag ? (((flow as any).nodes || []) as any[]) : []; + const edges = hasDag ? (((flow as any).edges || []) as any[]) : []; + const defaultEdges = hasDag ? defaultEdgesOnly(edges as any) : []; + const order = hasDag ? topoOrder(nodes as any, defaultEdges as any) : []; + const stepsToRun: any[] = hasDag + ? order.map((n) => mapDagNodeToStep(n as any)) + : ((flow.steps || []) as any[]); + for (const st of stepsToRun) { + const t0 = Date.now(); + const maxRetries = Math.max(0, (st as any).retry?.count ?? 0); + const baseInterval = Math.max(0, (st as any).retry?.intervalMs ?? 0); + let attempt = 0; + const doDelay = async (i: number) => { + const delay = + baseInterval > 0 + ? (st as any).retry?.backoff === 'exp' + ? baseInterval * Math.pow(2, i) + : baseInterval + : 0; + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + }; + while (true) { + try { + const beforeInfo = await (async () => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tab = tabs[0]; + return { url: tab?.url || '', status: (tab as any)?.status || '' }; + })(); + const result = await executeStep(ctx, st as any); + if ((st.type === 'click' || st.type === 'dblclick') && (st as any).after) { + const after = (st as any).after as any; + if (after.waitForNavigation) + await waitForNavigation((st as any).timeoutMs, beforeInfo.url); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((st as any).timeoutMs || 5000, 120000), 1200); + } + if (!result?.alreadyLogged) + ctx.logger({ stepId: st.id, status: 'success', tookMs: Date.now() - t0 } as any); + break; + } catch (e: any) { + if (attempt < maxRetries) { + ctx.logger({ + stepId: st.id, + status: 'retrying', + message: e?.message || String(e), + } as any); + await doDelay(attempt); + attempt += 1; + continue; + } + ctx.logger({ + stepId: st.id, + status: 'failed', + message: e?.message || String(e), + tookMs: Date.now() - t0, + } as any); + throw e; + } + } + } + }, + }, + handleDownload: { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const args: any = { + filenameContains: s.filenameContains || undefined, + timeoutMs: Math.max(1000, Math.min(Number(s.timeoutMs ?? 60000), 300000)), + waitForComplete: s.waitForComplete !== false, + }; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.HANDLE_DOWNLOAD, args }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload && payload.download) ctx.vars[s.saveAs] = payload.download; + } catch {} + }, + }, + // P0: screenshot node (wrapper around screenshot tool) + screenshot: { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const args: any = { name: 'workflow', storeBase64: true }; + if (s.fullPage) args.fullPage = true; + if (s.selector && typeof s.selector === 'string' && s.selector.trim()) + args.selector = s.selector; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.SCREENSHOT, args }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload && payload.base64Data) ctx.vars[s.saveAs] = payload.base64Data; + } catch {} + }, + }, + // P0: trigger custom DOM event + triggerEvent: { + validate: (step) => { + const s: any = step; + const ok = !!s?.target?.candidates?.length && typeof s?.event === 'string' && s.event; + return ok ? { ok } : { ok, errors: ['缺少目标选择器或事件类型'] }; + }, + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const located = await locateElement(tabId, s.target, ctx.frameId); + const cssSelector = !(located as any)?.ref + ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined; + let sel = cssSelector as string | undefined; + if (!sel && (located as any)?.ref) { + try { + const resolved = await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + ); + sel = resolved?.selector; + } catch {} + } + if (!sel) throw new Error('triggerEvent: selector not resolved'); + const world: any = 'MAIN'; + const ev = String(s.event || '').trim(); + const bubbles = s.bubbles !== false; + const cancelable = s.cancelable === true; + await chrome.scripting.executeScript({ + target: { + tabId, + frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, + } as any, + world, + func: (selector: string, type: string, bubbles: boolean, cancelable: boolean) => { + try { + const el = document.querySelector(selector); + if (!el) return false; + const e = new Event(type, { bubbles, cancelable }); + el.dispatchEvent(e); + return true; + } catch (e) { + return false; + } + }, + args: [sel, ev, !!bubbles, !!cancelable], + } as any); + }, + }, + // P0: set attribute node + setAttribute: { + validate: (step) => { + const s: any = step; + const ok = !!s?.target?.candidates?.length && typeof s?.name === 'string' && s.name; + return ok ? { ok } : { ok, errors: ['需提供目标选择器与属性名'] }; + }, + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const located = await locateElement(tabId, s.target, ctx.frameId); + const cssSelector = !(located as any)?.ref + ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined; + let sel = cssSelector as string | undefined; + if (!sel && (located as any)?.ref) { + try { + const resolved = await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + ); + sel = resolved?.selector; + } catch {} + } + if (!sel) throw new Error('setAttribute: selector not resolved'); + const world: any = 'MAIN'; + const name = String(s.name || ''); + const value = s.value != null ? String(s.value) : null; + const remove = s.remove === true || value == null; + await chrome.scripting.executeScript({ + target: { + tabId, + frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, + } as any, + world, + func: (selector: string, name: string, value: string | null, remove: boolean) => { + try { + const el = document.querySelector(selector) as any; + if (!el) return false; + if (remove) { + el.removeAttribute(name); + return true; + } + el.setAttribute(name, String(value ?? '')); + return true; + } catch { + return false; + } + }, + args: [sel, name, value, remove], + } as any); + }, + }, + // P0: switch to a same-origin iframe by index or url substring + switchFrame: { + run: async (ctx, step) => { + const s: any = step; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + // discover frames via webNavigation API + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + if (!Array.isArray(frames) || frames.length === 0) { + ctx.frameId = undefined; + return; + } + // choose by index (excluding 0 which is top frame) + let target: any | undefined; + const idx = Number(s?.frame?.index ?? NaN); + if (Number.isFinite(idx)) { + const list = frames.filter((f) => f.frameId !== 0); + target = list[Math.max(0, Math.min(list.length - 1, idx))]; + } + // choose by urlContains if provided + const urlContains = String(s?.frame?.urlContains || '').trim(); + if (!target && urlContains) + target = frames.find((f) => typeof f.url === 'string' && f.url.includes(urlContains)); + // fallback to top (clear) + if (!target) ctx.frameId = undefined; + else ctx.frameId = target.frameId; + // ensure helper injected into all frames for subsequent operations + try { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } catch {} + ctx.logger({ + stepId: step.id, + status: 'success', + message: `frameId=${String(ctx.frameId ?? 'top')}`, + } as any); + }, + }, + // P0: loop over elements matching a selector and branch into subflow + loopElements: { + validate: (step) => { + const s: any = step; + const ok = + typeof s?.selector === 'string' && + s.selector && + typeof s?.subflowId === 'string' && + s.subflowId; + return ok ? { ok } : { ok, errors: ['需提供 selector 与 subflowId'] }; + }, + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const world: any = 'MAIN'; + const selector = String(s.selector || ''); + const res = await chrome.scripting.executeScript({ + target: { + tabId, + frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, + } as any, + world, + func: (sel: string) => { + try { + const list = Array.from(document.querySelectorAll(sel)); + const toCss = (node: Element) => { + try { + if ((node as HTMLElement).id) { + const idSel = `#${CSS.escape((node as HTMLElement).id)}`; + if (document.querySelectorAll(idSel).length === 1) return idSel; + } + } catch {} + let path = ''; + let current: Element | null = node; + while (current && current.tagName !== 'BODY') { + let part = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => (c as any).tagName === current!.tagName, + ); + if (siblings.length > 1) { + const idx = siblings.indexOf(current) + 1; + part += `:nth-of-type(${idx})`; + } + } + path = path ? `${part} > ${path}` : part; + current = parent; + } + return path ? `body > ${path}` : 'body'; + }; + return list.map(toCss); + } catch (e) { + return []; + } + }, + args: [selector], + } as any); + const arr: string[] = (res && Array.isArray(res[0]?.result) ? res[0].result : []) as any; + const listVar = String(s.saveAs || 'elements'); + const itemVar = String(s.itemVar || 'item'); + ctx.vars[listVar] = arr; + return { + control: { kind: 'foreach', listVar, itemVar, subflowId: String(s.subflowId) }, + } as any; + }, + }, +}; + +// New unified executeStep using registry only +export async function executeStep(ctx: ExecCtx, step: Step): Promise { + const runtime = (registry as any)[step.type] as NodeRuntime | undefined; + if (!runtime) throw new Error(`unsupported step type: ${String(step.type)}`); + const v = runtime.validate ? runtime.validate(step as any) : { ok: true }; + if (!(v as any).ok) throw new Error(((v as any).errors || []).join(', ') || 'validation failed'); + const out = await runtime.run(ctx, step as any); + return (out || {}) as ExecResult; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts b/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts new file mode 100644 index 00000000..44692174 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts @@ -0,0 +1,226 @@ +// rr-utils.ts — shared helpers for record-replay runner +// Note: comments in English + +import { + TOOL_NAMES, + topoOrder as sharedTopoOrder, + mapNodeToStep as sharedMapNodeToStep, +} from 'chrome-mcp-shared'; +import type { Edge as DagEdge, NodeBase as DagNode } from './types'; +import { handleCallTool } from '../tools'; + +export function applyAssign( + target: Record, + source: any, + assign: Record, +) { + const getByPath = (obj: any, path: string) => { + try { + const parts = path + .replace(/\[(\d+)\]/g, '.$1') + .split('.') + .filter(Boolean); + let cur = obj; + for (const p of parts) { + if (cur == null) return undefined; + cur = (cur as any)[p as any]; + } + return cur; + } catch { + return undefined; + } + }; + for (const [k, v] of Object.entries(assign || {})) { + target[k] = getByPath(source, String(v)); + } +} + +export function expandTemplatesDeep(value: T, scope: Record): T { + const replaceOne = (s: string) => + s.replace(/\{([^}]+)\}/g, (_m, k) => (scope[k] ?? '').toString()); + const walk = (v: any): any => { + if (v == null) return v; + if (typeof v === 'string') return replaceOne(v); + if (Array.isArray(v)) return v.map((x) => walk(x)); + if (typeof v === 'object') { + const out: any = {}; + for (const [k, val] of Object.entries(v)) out[k] = walk(val); + return out; + } + return v; + }; + return walk(value); +} + +export async function ensureTab(options: { + tabTarget?: 'current' | 'new'; + startUrl?: string; + refresh?: boolean; +}) { + const target = options.tabTarget || 'current'; + const startUrl = options.startUrl; + const [active] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (target === 'new') { + let urlToOpen = startUrl; + if (!urlToOpen) urlToOpen = active?.url || 'about:blank'; + await chrome.tabs.create({ url: urlToOpen, active: true }); + await new Promise((r) => setTimeout(r, 500)); + } else { + if (startUrl) + await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url: startUrl } }); + else if (options.refresh) + await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { refresh: true } }); + } +} + +export async function waitForNetworkIdle(totalTimeoutMs: number, idleThresholdMs: number) { + const deadline = Date.now() + Math.max(500, totalTimeoutMs); + const threshold = Math.max(200, idleThresholdMs); + while (Date.now() < deadline) { + await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_CAPTURE_START, + args: { + includeStatic: false, + maxCaptureTime: Math.min(60_000, Math.max(threshold + 500, 2_000)), + inactivityTimeout: threshold, + }, + }); + await new Promise((r) => setTimeout(r, threshold + 200)); + const stopRes = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_CAPTURE_STOP, + args: {}, + }); + const text = (stopRes as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const json = text ? JSON.parse(text) : null; + const captureEnd = Number(json?.captureEndTime) || Date.now(); + const reqs: any[] = Array.isArray(json?.requests) ? json.requests : []; + const lastActivity = reqs.reduce( + (acc, r) => { + const t = Number(r.responseTime || r.requestTime || 0); + return t > acc ? t : acc; + }, + Number(json?.captureStartTime || 0), + ); + if (captureEnd - lastActivity >= threshold) return; // idle reached + } catch { + // ignore parse errors + } + await new Promise((r) => setTimeout(r, Math.min(500, threshold))); + } + throw new Error('wait for network idle timed out'); +} + +// Event-driven navigation wait helper +// Waits for top-frame navigation completion or SPA history updates on active tab. +// Falls back to short network idle on timeout. +export async function waitForNavigation(timeoutMs?: number, prevUrl?: string): Promise { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const timeout = Math.max(1000, Math.min(timeoutMs || 15000, 30000)); + const startedAt = Date.now(); + + await new Promise((resolve, reject) => { + let done = false; + let timer: any = null; + const cleanup = () => { + try { + chrome.webNavigation.onCommitted.removeListener(onCommitted); + } catch {} + try { + chrome.webNavigation.onCompleted.removeListener(onCompleted); + } catch {} + try { + (chrome.webNavigation as any).onHistoryStateUpdated?.removeListener?.( + onHistoryStateUpdated, + ); + } catch {} + try { + chrome.tabs.onUpdated.removeListener(onTabUpdated); + } catch {} + if (timer) { + try { + clearTimeout(timer); + } catch {} + } + }; + const finish = () => { + if (done) return; + done = true; + cleanup(); + resolve(); + }; + const onCommitted = (details: any) => { + if ( + details && + details.tabId === tabId && + details.frameId === 0 && + details.timeStamp >= startedAt + ) { + // committed observed; we'll wait for completion or SPA fallback + } + }; + const onCompleted = (details: any) => { + if ( + details && + details.tabId === tabId && + details.frameId === 0 && + details.timeStamp >= startedAt + ) + finish(); + }; + const onHistoryStateUpdated = (details: any) => { + if ( + details && + details.tabId === tabId && + details.frameId === 0 && + details.timeStamp >= startedAt + ) + finish(); + }; + const onTabUpdated = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => { + if (updatedTabId !== tabId) return; + if (changeInfo.status === 'complete') finish(); + if (typeof changeInfo.url === 'string' && (!prevUrl || changeInfo.url !== prevUrl)) finish(); + }; + const onTimeout = async () => { + cleanup(); + try { + await waitForNetworkIdle(2000, 800); + resolve(); + } catch { + reject(new Error('navigation timeout')); + } + }; + + chrome.webNavigation.onCommitted.addListener(onCommitted); + chrome.webNavigation.onCompleted.addListener(onCompleted); + try { + (chrome.webNavigation as any).onHistoryStateUpdated?.addListener?.(onHistoryStateUpdated); + } catch {} + chrome.tabs.onUpdated.addListener(onTabUpdated); + timer = setTimeout(onTimeout, timeout); + }); +} + +export function topoOrder(nodes: DagNode[], edges: DagEdge[]): DagNode[] { + return sharedTopoOrder(nodes as any, edges as any) as any; +} + +// Helper: filter only default edges (no label or label === 'default') +export function defaultEdgesOnly(edges: DagEdge[] = []): DagEdge[] { + return (edges || []).filter((e) => !e.label || e.label === 'default'); +} + +export function mapDagNodeToStep(n: DagNode): any { + const s: any = sharedMapNodeToStep(n as any); + if ((n as any)?.type === 'if') { + // forward extended conditional config for DAG mode + const cfg: any = (n as any).config || {}; + if (Array.isArray(cfg.branches)) s.branches = cfg.branches; + if ('else' in cfg) s.else = cfg.else; + if (cfg.condition && !s.condition) s.condition = cfg.condition; // backward-compat + } + return s; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/runner.ts b/app/chrome-extension/entrypoints/background/record-replay/runner.ts new file mode 100644 index 00000000..38b7b925 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/runner.ts @@ -0,0 +1,597 @@ +// runner.ts — orchestrates record-replay flow execution using registry + utils +// Note: comments in English + +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '../tools'; +import type { Flow, RunLogEntry, RunRecord, RunResult, Step, StepScript } from './types'; +import { appendRun } from './flow-store'; +import { + mapDagNodeToStep, + topoOrder, + ensureTab, + expandTemplatesDeep, + waitForNetworkIdle, + applyAssign, + defaultEdgesOnly, + waitForNavigation, +} from './rr-utils'; +import { executeStep } from './node-registry'; + +export interface RunOptions { + tabTarget?: 'current' | 'new'; + refresh?: boolean; + captureNetwork?: boolean; + returnLogs?: boolean; + timeoutMs?: number; + startUrl?: string; + args?: Record; + startNodeId?: string; +} + +export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { + const runId = `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const startAt = Date.now(); + const logs: RunLogEntry[] = []; + // Global deadline for run (optional) + const globalTimeout = Math.max(0, Number(options.timeoutMs || 0)); + const deadline = globalTimeout > 0 ? startAt + globalTimeout : 0; + const ensureWithinDeadline = () => { + if (deadline > 0 && Date.now() > deadline) { + const err = new Error('Global timeout reached'); + // mark a synthetic log entry for visibility + logs.push({ stepId: 'global-timeout', status: 'failed', message: 'Global timeout reached' }); + throw err; + } + }; + + // prepare variables + const vars: Record = Object.create(null); + for (const v of flow.variables || []) if (v.default !== undefined) vars[v.key] = v.default; + if (options.args) Object.assign(vars, options.args); + + // ensure tab per options + await ensureTab({ + tabTarget: options.tabTarget, + startUrl: options.startUrl, + refresh: options.refresh, + }); + + // pre-load read_page to init bridges + try { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } catch {} + + // collect required variables via overlay prompt + try { + const needed = (flow.variables || []).filter( + (v) => + (options.args?.[v.key] == null || options.args?.[v.key] === '') && + (v.rules?.required || (v.default ?? '') === ''), + ); + if (needed.length) { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.SEND_COMMAND_TO_INJECT_SCRIPT, + args: { + eventName: 'collectVariables', + payload: JSON.stringify({ variables: needed, useOverlay: true }), + }, + }); + let values: Record | null = null; + try { + const t = (res?.content || []).find((c: any) => c.type === 'text')?.text; + const j = t ? JSON.parse(t) : null; + if (j && j.success && j.values) values = j.values; + } catch {} + if (!values) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId === 'number') { + const res2 = await chrome.tabs.sendMessage(tabId, { + action: 'collectVariables', + variables: needed, + useOverlay: true, + } as any); + if (res2 && res2.success && res2.values) values = res2.values; + } + } + if (values) Object.assign(vars, values); + } + } catch {} + + // init overlay for on-screen log + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) + await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'init' } as any); + } catch {} + + // binding enforcement + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const currentUrl = tabs?.[0]?.url || ''; + const bindings = flow.meta?.bindings || []; + if (!options.startUrl && bindings.length > 0) { + const ok = bindings.some((b) => { + try { + if (b.type === 'domain') return new URL(currentUrl).hostname.includes(b.value); + if (b.type === 'path') return new URL(currentUrl).pathname.startsWith(b.value); + if (b.type === 'url') return currentUrl.startsWith(b.value); + } catch {} + return false; + }); + if (!ok) { + return { + runId, + success: false, + summary: { total: 0, success: 0, failed: 0, tookMs: 0 }, + url: currentUrl, + outputs: null, + logs: [ + { + stepId: 'binding-check', + status: 'failed', + message: + 'Flow binding mismatch. Provide startUrl or open a page matching flow.meta.bindings.', + }, + ], + screenshots: { onFailure: null }, + } as RunResult; + } + } + } catch {} + + // long-running network capture (debugger) if requested + let networkCaptureStarted = false; + const stopAndSummarizeNetwork = async () => { + try { + const stopRes = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_STOP, + args: {}, + }); + const text = (stopRes?.content || []).find((c: any) => c.type === 'text')?.text; + if (!text) return; + const data = JSON.parse(text); + const requests: any[] = Array.isArray(data?.requests) ? data.requests : []; + const snippets = requests + .filter((r) => ['XHR', 'Fetch'].includes(String(r.type))) + .slice(0, 10) + .map((r) => ({ + method: String(r.method || 'GET'), + url: String(r.url || ''), + status: r.statusCode || r.status, + ms: Math.max(0, (r.responseTime || 0) - (r.requestTime || 0)), + })); + logs.push({ + stepId: 'network-capture', + status: 'success', + message: `Captured ${Number(data?.requestCount || 0)} requests` as any, + networkSnippets: snippets, + } as any); + } catch {} + }; + if (options.captureNetwork) { + try { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_START, + args: { includeStatic: false, maxCaptureTime: 3 * 60_000, inactivityTimeout: 0 }, + }); + if (!(res as any)?.isError) networkCaptureStarted = true; + } catch {} + } + + const hasDag = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; + const nodes = hasDag ? (((flow as any).nodes || []) as any[]) : []; + const edges = hasDag ? (((flow as any).edges || []) as any[]) : []; + const defaultEdges = hasDag ? defaultEdgesOnly(edges as any) : []; + const order = hasDag ? topoOrder(nodes as any, defaultEdges as any) : []; + const stepsToRun: Step[] = hasDag + ? order.map((n) => mapDagNodeToStep(n as any)) + : ((flow.steps || []) as Step[]); + const startIdx = + !hasDag && options.startNodeId + ? stepsToRun.findIndex((s) => s?.id === options.startNodeId) + : -1; + const steps = !hasDag + ? startIdx >= 0 + ? stepsToRun.slice(startIdx) + : stepsToRun.slice() + : stepsToRun; + + let failed = 0; + const logger = (e: RunLogEntry) => logs.push(e); + const ctx = { vars, logger }; + + // deferred after-scripts + const pendingAfterScripts: StepScript[] = []; + + // small helpers + const appendOverlay = async (text: string) => { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) + await chrome.tabs.sendMessage(tabs[0].id, { + action: 'rr_overlay', + cmd: 'append', + text, + } as any); + } catch {} + }; + const evalCondition = (cond: any): boolean => { + try { + if (cond && typeof cond.expression === 'string' && cond.expression.trim()) { + const fn = new Function( + 'vars', + `try { return !!(${cond.expression}); } catch (e) { return false; }`, + ); + return !!fn(vars); + } + if (cond && typeof cond.var === 'string') { + const v = vars[cond.var]; + if ('equals' in cond) return String(v) === String(cond.equals); + return !!v; + } + } catch {} + return false; + }; + + // execute a subflow by id (default edges only) + const runSubflowById = async (subflowId: string) => { + const sub = (flow.subflows || {})[subflowId]; + if (!sub || !Array.isArray(sub.nodes) || sub.nodes.length === 0) return; + const sNodes: any[] = sub.nodes; + const sEdges: any[] = defaultEdgesOnly((sub.edges || []) as any) as any[]; + const sOrder = topoOrder(sNodes as any, sEdges as any); + const sSteps: Step[] = sOrder.map((n) => mapDagNodeToStep(n as any)) as any; + for (const step of sSteps) { + const t0 = Date.now(); + const maxRetries = Math.max(0, (step as any).retry?.count ?? 0); + const baseInterval = Math.max(0, (step as any).retry?.intervalMs ?? 0); + let attempt = 0; + const doDelay = async (i: number) => { + const delay = + baseInterval > 0 + ? (step as any).retry?.backoff === 'exp' + ? baseInterval * Math.pow(2, i) + : baseInterval + : 0; + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + }; + while (true) { + try { + const beforeInfo = await getActiveTabInfo(); + const result = await executeStep(ctx, step); + if ((step.type === 'click' || step.type === 'dblclick') && (step as any).after) { + const after = (step as any).after as any; + if (after.waitForNavigation) + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); + } + if (!result?.alreadyLogged) + logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); + await appendOverlay(`✔ ${step.type} (${step.id})`); + break; + } catch (e: any) { + if (attempt < maxRetries) { + logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); + await doDelay(attempt); + attempt += 1; + continue; + } + logs.push({ + stepId: step.id, + status: 'failed', + message: e?.message || String(e), + tookMs: Date.now() - t0, + }); + await appendOverlay(`✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`); + if ((step as any).screenshotOnFail !== false) { + try { + const shot = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'screenshot' }, + }); + const img = (shot?.content?.find((c: any) => c.type === 'image') as any) + ?.data as string; + if (img) logs[logs.length - 1].screenshotBase64 = img; + } catch {} + } + throw e; + } + } + } + }; + const getActiveTabInfo = async () => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tab = tabs[0]; + return { url: tab?.url || '', status: (tab as any)?.status || '' }; + }; + const waitForNavigationDone = async (prevUrl: string, timeoutMs?: number) => { + await waitForNavigation(timeoutMs, prevUrl); + }; + + try { + if (!hasDag) { + // Linear execution (legacy steps) + for (const step of steps) { + const t0 = Date.now(); + ensureWithinDeadline(); + const maxRetries = Math.max(0, (step as any).retry?.count ?? 0); + const baseInterval = Math.max(0, (step as any).retry?.intervalMs ?? 0); + let attempt = 0; + const doDelay = async (i: number) => { + const delay = + baseInterval > 0 + ? (step as any).retry?.backoff === 'exp' + ? baseInterval * Math.pow(2, i) + : baseInterval + : 0; + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + }; + while (true) { + try { + const beforeInfo = await getActiveTabInfo(); + // special handling for script when=after: defer + if (step.type === 'script' && (step as any).when === 'after') { + pendingAfterScripts.push(step as any); + logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); + break; + } + const result = await executeStep(ctx, step); + // handle click/dblclick after.waitForNavigation / waitForNetworkIdle + if ((step.type === 'click' || step.type === 'dblclick') && (step as any).after) { + const after = (step as any).after as any; + if (after.waitForNavigation) + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); + } + if (!result?.alreadyLogged) { + logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); + } + await appendOverlay(`✔ ${step.type} (${step.id})`); + // control flows + if (result?.control) { + if (result.control.kind === 'foreach') { + const list = Array.isArray(vars[result.control.listVar]) + ? (vars[result.control.listVar] as any[]) + : []; + for (const it of list) { + vars[result.control.itemVar] = it; + await runSubflowById(result.control.subflowId); + } + } else if (result.control.kind === 'while') { + let i = 0; + while ( + i < result.control.maxIterations && + evalCondition(result.control.condition) + ) { + await runSubflowById(result.control.subflowId); + i++; + } + } + } + if (result?.deferAfterScript) pendingAfterScripts.push(result.deferAfterScript); + // run any deferred after-scripts + if (pendingAfterScripts.length > 0) { + while (pendingAfterScripts.length) { + const s = pendingAfterScripts.shift()!; + const tScript = Date.now(); + const world = (s as any).world || 'ISOLATED'; + const code = String((s as any).code || ''); + if (code.trim()) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if ((s as any).saveAs) vars[(s as any).saveAs] = result; + if ((s as any).assign && typeof (s as any).assign === 'object') + applyAssign(vars, result, (s as any).assign); + } + logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); + } + } + break; // success + } catch (e: any) { + if (attempt < maxRetries) { + logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); + await doDelay(attempt); + attempt += 1; + continue; + } + failed++; + logs.push({ + stepId: step.id, + status: 'failed', + message: e?.message || String(e), + tookMs: Date.now() - t0, + }); + await appendOverlay(`✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`); + if ((step as any).screenshotOnFail !== false) { + try { + const shot = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'screenshot' }, + }); + const img = (shot?.content?.find((c: any) => c.type === 'image') as any) + ?.data as string; + if (img) logs[logs.length - 1].screenshotBase64 = img; + } catch {} + } + throw e; + } + } + } + } else { + // Graph traversal execution (DAG) + const id2node = new Map(nodes.map((n: any) => [n.id, n] as const)); + const outEdges = new Map>(); + for (const e of edges) { + if (!outEdges.has(e.from)) outEdges.set(e.from, []); + outEdges.get(e.from)!.push(e); + } + const indeg = new Map(nodes.map((n: any) => [n.id, 0] as const)); + for (const e of edges) indeg.set(e.to, (indeg.get(e.to) || 0) + 1); + let currentId = + options.startNodeId && id2node.has(options.startNodeId) + ? options.startNodeId + : nodes.find((n: any) => (indeg.get(n.id) || 0) === 0)?.id || nodes[0]?.id; + let guard = 0; + while (currentId && guard++ < 10000) { + ensureWithinDeadline(); + const node = id2node.get(currentId); + if (!node) break; + const step: any = mapDagNodeToStep(node as any); + const t0 = Date.now(); + const maxRetries = Math.max(0, (step as any).retry?.count ?? 0); + const baseInterval = Math.max(0, (step as any).retry?.intervalMs ?? 0); + let attempt = 0; + const doDelay = async (i: number) => { + const delay = + baseInterval > 0 + ? (step as any).retry?.backoff === 'exp' + ? baseInterval * Math.pow(2, i) + : baseInterval + : 0; + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + }; + const beforeInfo = await getActiveTabInfo(); + let jumpedOnError = false; + try { + const result = await executeStep(ctx, step); + if ((step.type === 'click' || step.type === 'dblclick') && (step as any).after) { + const after = (step as any).after as any; + if (after.waitForNavigation) + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); + } + if (!result?.alreadyLogged) + logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); + await appendOverlay(`✔ ${step.type} (${step.id})`); + // choose next by label + let nextLabel: string = 'default'; + if (result?.nextLabel) nextLabel = String(result.nextLabel); + const oes = (outEdges.get(currentId) || []) as any[]; + const edge = + oes.find((e) => String(e.label || 'default') === nextLabel) || + oes.find((e) => !e.label || e.label === 'default'); + currentId = edge ? edge.to : undefined; + } catch (e: any) { + if (attempt < maxRetries) { + logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); + await doDelay(attempt); + attempt += 1; + continue; + } + logs.push({ + stepId: step.id, + status: 'failed', + message: e?.message || String(e), + tookMs: Date.now() - t0, + }); + await appendOverlay(`✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`); + if ((step as any).screenshotOnFail !== false) { + try { + const shot = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'screenshot' }, + }); + const img = (shot?.content?.find((c: any) => c.type === 'image') as any) + ?.data as string; + if (img) logs[logs.length - 1].screenshotBase64 = img; + } catch {} + } + // onError jump + const oes = (outEdges.get(currentId) || []) as any[]; + const errEdge = oes.find((e) => e.label === 'onError'); + if (errEdge) { + currentId = errEdge.to; + jumpedOnError = true; + } else { + throw e; + } + } + if (!jumpedOnError) { + // flush deferred after-scripts + if (pendingAfterScripts.length > 0) { + while (pendingAfterScripts.length) { + const s = pendingAfterScripts.shift()!; + const tScript = Date.now(); + const world = (s as any).world || 'ISOLATED'; + const code = String((s as any).code || ''); + if (code.trim()) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if ((s as any).saveAs) vars[(s as any).saveAs] = result; + if ((s as any).assign && typeof (s as any).assign === 'object') + applyAssign(vars, result, (s as any).assign); + } + logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); + } + } + } + } + } + } finally { + if (networkCaptureStarted) await stopAndSummarizeNetwork(); + } + + const tookMs = Date.now() - startAt; + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) + await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'done' } as any); + } catch {} + + const record: RunRecord = { + id: runId, + flowId: flow.id, + startedAt: new Date(startAt).toISOString(), + finishedAt: new Date().toISOString(), + success: failed === 0, + entries: logs, + }; + await appendRun(record); + + // outputs: filter sensitive variables + const sensitiveKeys = new Set( + (flow.variables || []).filter((v) => v.sensitive).map((v) => v.key), + ); + const outputs: Record = {}; + for (const [k, v] of Object.entries(vars)) if (!sensitiveKeys.has(k)) outputs[k] = v; + + return { + runId, + success: failed === 0, + summary: { total: steps.length, success: steps.length - failed, failed, tookMs }, + url: null, + outputs, + logs: options.returnLogs ? logs : undefined, + screenshots: { onFailure: logs.find((l) => l.status === 'failed')?.screenshotBase64 }, + }; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts b/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts index 43339903..92cc7668 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts @@ -15,14 +15,19 @@ export interface LocatedElement { export async function locateElement( tabId: number, target: TargetLocator, + frameId?: number, ): Promise { // Try ref first if (target.ref) { try { - const res = await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.RESOLVE_REF, - ref: target.ref, - }); + const res = await chrome.tabs.sendMessage( + tabId, + { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: target.ref, + } as any, + { frameId } as any, + ); if (res && res.success && res.center) { return { ref: target.ref, center: res.center, resolvedBy: 'ref' }; } @@ -34,20 +39,28 @@ export async function locateElement( for (const c of target.candidates || []) { try { if (c.type === 'css' || c.type === 'attr') { - const ensured = await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - selector: c.value, - }); + const ensured = await chrome.tabs.sendMessage( + tabId, + { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + selector: c.value, + } as any, + { frameId } as any, + ); if (ensured && ensured.success && ensured.ref && ensured.center) { return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; } } else if (c.type === 'text') { // Search by visible innerText contains value - const ensured = await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - useText: true, - text: c.value, - } as any); + const ensured = await chrome.tabs.sendMessage( + tabId, + { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + useText: true, + text: c.value, + } as any, + { frameId } as any, + ); if (ensured && ensured.success && ensured.ref && ensured.center) { return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; } @@ -82,21 +95,29 @@ export async function locateElement( ); } for (const sel of ariaSelectors) { - const ensured = await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - selector: sel, - }); + const ensured = await chrome.tabs.sendMessage( + tabId, + { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + selector: sel, + } as any, + { frameId } as any, + ); if (ensured && ensured.success && ensured.ref && ensured.center) { return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; } } } else if (c.type === 'xpath') { // Minimal xpath support via document.evaluate through injected helper - const ensured = await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - selector: c.value, - isXPath: true, - } as any); + const ensured = await chrome.tabs.sendMessage( + tabId, + { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + selector: c.value, + isXPath: true, + } as any, + { frameId } as any, + ); if (ensured && ensured.success && ensured.ref && ensured.center) { return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; } diff --git a/app/chrome-extension/entrypoints/background/record-replay/trigger-store.ts b/app/chrome-extension/entrypoints/background/record-replay/trigger-store.ts new file mode 100644 index 00000000..eb8342c0 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/trigger-store.ts @@ -0,0 +1,61 @@ +import { STORAGE_KEYS } from '@/common/constants'; + +export type TriggerType = 'url' | 'contextMenu' | 'command' | 'dom'; + +export interface BaseTrigger { + id: string; + type: TriggerType; + enabled: boolean; + flowId: string; + args?: Record; +} + +export interface UrlTrigger extends BaseTrigger { + type: 'url'; + match: Array<{ kind: 'url' | 'domain' | 'path'; value: string }>; +} + +export interface ContextMenuTrigger extends BaseTrigger { + type: 'contextMenu'; + title: string; + contexts?: chrome.contextMenus.ContextType[]; +} + +export interface CommandTrigger extends BaseTrigger { + type: 'command'; + commandKey: string; // e.g., run_quick_trigger_1 +} + +export interface DomTrigger extends BaseTrigger { + type: 'dom'; + selector: string; + appear?: boolean; // default true + once?: boolean; // default true + debounceMs?: number; // default 800 +} + +export type FlowTrigger = UrlTrigger | ContextMenuTrigger | CommandTrigger | DomTrigger; + +export async function listTriggers(): Promise { + const res = await chrome.storage.local.get([STORAGE_KEYS.RR_TRIGGERS]); + const arr = (res[STORAGE_KEYS.RR_TRIGGERS] as FlowTrigger[]) || []; + return arr; +} + +export async function saveTrigger(t: FlowTrigger): Promise { + const arr = await listTriggers(); + const idx = arr.findIndex((x) => x.id === t.id); + if (idx >= 0) arr[idx] = t; + else arr.push(t); + await chrome.storage.local.set({ [STORAGE_KEYS.RR_TRIGGERS]: arr }); +} + +export async function deleteTrigger(id: string): Promise { + const arr = await listTriggers(); + const filtered = arr.filter((x) => x.id !== id); + await chrome.storage.local.set({ [STORAGE_KEYS.RR_TRIGGERS]: filtered }); +} + +export function toId(prefix = 'trg') { + return `${prefix}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/types.ts b/app/chrome-extension/entrypoints/background/record-replay/types.ts index 7aaad0c0..2a8f5d08 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/types.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/types.ts @@ -17,18 +17,28 @@ export type StepType = | 'click' | 'dblclick' | 'fill' + | 'triggerEvent' + | 'setAttribute' + | 'screenshot' + | 'switchFrame' + | 'loopElements' | 'key' | 'scroll' | 'drag' | 'wait' | 'assert' | 'script' + | 'if' + | 'foreach' + | 'while' | 'navigate' | 'http' | 'extract' | 'openTab' | 'switchTab' - | 'closeTab'; + | 'closeTab' + | 'handleDownload' + | 'executeFlow'; export interface StepBase { id: string; @@ -51,6 +61,42 @@ export interface StepFill extends StepBase { value: string; // may contain {var} } +export interface StepTriggerEvent extends StepBase { + type: 'triggerEvent'; + target: TargetLocator; + event: string; // e.g. 'input', 'change', 'mouseover' + bubbles?: boolean; + cancelable?: boolean; +} + +export interface StepSetAttribute extends StepBase { + type: 'setAttribute'; + target: TargetLocator; + name: string; + value?: string; // when omitted and remove=true, remove attribute + remove?: boolean; +} + +export interface StepScreenshot extends StepBase { + type: 'screenshot'; + selector?: string; + fullPage?: boolean; + saveAs?: string; // variable name to store base64 +} + +export interface StepSwitchFrame extends StepBase { + type: 'switchFrame'; + frame?: { index?: number; urlContains?: string }; +} + +export interface StepLoopElements extends StepBase { + type: 'loopElements'; + selector: string; + saveAs?: string; // list var name + itemVar?: string; // default 'item' + subflowId: string; +} + export interface StepKey extends StepBase { type: 'key'; keys: string; // e.g. "Backspace Enter" or "cmd+a" @@ -77,7 +123,8 @@ export interface StepWait extends StepBase { | { selector: string; visible?: boolean } | { text: string; appear?: boolean } | { navigation: true } - | { networkIdle: true }; + | { networkIdle: true } + | { sleep: number }; } export interface StepAssert extends StepBase { @@ -98,21 +145,61 @@ export interface StepScript extends StepBase { when?: 'before' | 'after'; } +export interface StepIf extends StepBase { + type: 'if'; + // condition supports: { var: string; equals?: any } | { expression: string } + condition: any; +} + +export interface StepForeach extends StepBase { + type: 'foreach'; + listVar: string; + itemVar?: string; + subflowId: string; +} + +export interface StepWhile extends StepBase { + type: 'while'; + condition: any; + subflowId: string; + maxIterations?: number; +} + export type Step = | StepClick | StepFill + | StepTriggerEvent + | StepSetAttribute + | StepScreenshot + | StepSwitchFrame + | StepLoopElements | StepKey | StepScroll | StepDrag | StepWait | StepAssert | StepScript + | StepIf + | StepForeach + | StepWhile | (StepBase & { type: 'navigate'; url: string }) | StepHttp | StepExtract | StepOpenTab | StepSwitchTab - | StepCloseTab; + | StepCloseTab + | (StepBase & { + type: 'handleDownload'; + filenameContains?: string; + saveAs?: string; + waitForComplete?: boolean; + }) + | (StepBase & { + type: 'executeFlow'; + flowId: string; + inline?: boolean; + args?: Record; + }); export interface StepHttp extends StepBase { type: 'http'; @@ -120,6 +207,7 @@ export interface StepHttp extends StepBase { url: string; headers?: Record; body?: any; + formData?: any; saveAs?: string; assign?: Record; } @@ -151,26 +239,40 @@ export interface StepCloseTab extends StepBase { url?: string; } +export type VariableType = 'string' | 'number' | 'boolean' | 'enum' | 'array'; + export interface VariableDef { key: string; label?: string; sensitive?: boolean; - default?: string; - rules?: { required?: boolean; pattern?: string }; + // default value can be string/number/boolean/array depending on type + default?: any; // keep broad for backward compatibility + type?: VariableType; // default to 'string' when omitted + rules?: { required?: boolean; pattern?: string; enum?: string[] }; } export type NodeType = | 'click' | 'dblclick' | 'fill' + | 'triggerEvent' + | 'setAttribute' + | 'screenshot' + | 'switchFrame' + | 'loopElements' | 'key' | 'wait' | 'assert' | 'script' + | 'if' + | 'foreach' + | 'while' | 'navigate' | 'openTab' | 'switchTab' | 'closeTab' + | 'handleDownload' + | 'executeFlow' | 'http' | 'extract' | 'delay'; @@ -188,7 +290,9 @@ export interface Edge { id: string; from: string; to: string; - label?: 'default' | 'true' | 'false' | 'onError'; + // label identifies the logical branch. Keep 'default' for linear/main path. + // For conditionals, use arbitrary strings like 'case:' or 'else'. + label?: string; } export interface Flow { diff --git a/app/chrome-extension/entrypoints/background/tools/browser/download.ts b/app/chrome-extension/entrypoints/background/tools/browser/download.ts new file mode 100644 index 00000000..54779539 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/tools/browser/download.ts @@ -0,0 +1,123 @@ +import { createErrorResponse, ToolResult } from '@/common/tool-handler'; +import { BaseBrowserToolExecutor } from '../base-browser'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; + +interface HandleDownloadParams { + filenameContains?: string; + timeoutMs?: number; // default 60000 + waitForComplete?: boolean; // default true +} + +/** + * Tool: wait for a download and return info + */ +class HandleDownloadTool extends BaseBrowserToolExecutor { + name = TOOL_NAMES.BROWSER.HANDLE_DOWNLOAD as any; + + async execute(args: HandleDownloadParams): Promise { + const filenameContains = String(args?.filenameContains || '').trim(); + const waitForComplete = args?.waitForComplete !== false; + const timeoutMs = Math.max(1000, Math.min(Number(args?.timeoutMs ?? 60000), 300000)); + + try { + const result = await waitForDownload({ filenameContains, waitForComplete, timeoutMs }); + return { + content: [{ type: 'text', text: JSON.stringify({ success: true, download: result }) }], + isError: false, + }; + } catch (e: any) { + return createErrorResponse(`Handle download failed: ${e?.message || String(e)}`); + } + } +} + +async function waitForDownload(opts: { + filenameContains?: string; + waitForComplete: boolean; + timeoutMs: number; +}) { + const { filenameContains, waitForComplete, timeoutMs } = opts; + return new Promise((resolve, reject) => { + let timer: any = null; + const onError = (err: any) => { + cleanup(); + reject(err instanceof Error ? err : new Error(String(err))); + }; + const cleanup = () => { + try { + if (timer) clearTimeout(timer); + } catch {} + try { + chrome.downloads.onCreated.removeListener(onCreated); + } catch {} + try { + chrome.downloads.onChanged.removeListener(onChanged); + } catch {} + }; + const matches = (item: chrome.downloads.DownloadItem) => { + if (!filenameContains) return true; + const name = (item.filename || '').split(/[/\\]/).pop() || ''; + return name.includes(filenameContains) || (item.url || '').includes(filenameContains); + }; + const fulfill = async (item: chrome.downloads.DownloadItem) => { + // try to fill more details via downloads.search + try { + const [found] = await chrome.downloads.search({ id: item.id }); + const out = found || item; + cleanup(); + resolve({ + id: out.id, + filename: out.filename, + url: out.url, + mime: (out as any).mime || undefined, + fileSize: out.fileSize ?? out.totalBytes ?? undefined, + state: out.state, + danger: out.danger, + startTime: out.startTime, + endTime: (out as any).endTime || undefined, + exists: (out as any).exists, + }); + return; + } catch { + cleanup(); + resolve({ id: item.id, filename: item.filename, url: item.url, state: item.state }); + } + }; + const onCreated = (item: chrome.downloads.DownloadItem) => { + try { + if (!matches(item)) return; + if (!waitForComplete) { + fulfill(item); + } + } catch {} + }; + const onChanged = (delta: chrome.downloads.DownloadDelta) => { + try { + if (!delta || typeof delta.id !== 'number') return; + // pull item and check + chrome.downloads + .search({ id: delta.id }) + .then((arr) => { + const item = arr && arr[0]; + if (!item) return; + if (!matches(item)) return; + if (waitForComplete && item.state === 'complete') fulfill(item); + }) + .catch(() => {}); + } catch {} + }; + chrome.downloads.onCreated.addListener(onCreated); + chrome.downloads.onChanged.addListener(onChanged); + timer = setTimeout(() => onError(new Error('Download wait timed out')), timeoutMs); + // Try to find an already-running matching download + chrome.downloads + .search({ state: waitForComplete ? 'in_progress' : undefined }) + .then((arr) => { + const hit = (arr || []).find((d) => matches(d)); + if (hit && !waitForComplete) fulfill(hit); + }) + .catch(() => {}); + }); +} + +export const handleDownloadTool = new HandleDownloadTool(); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/index.ts b/app/chrome-extension/entrypoints/background/tools/browser/index.ts index 29a2e8aa..0639d8b3 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/index.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/index.ts @@ -16,6 +16,7 @@ export { fileUploadTool } from './file-upload'; export { readPageTool } from './read-page'; export { computerTool } from './computer'; export { handleDialogTool } from './dialog'; +export { handleDownloadTool } from './download'; export { userscriptTool } from './userscript'; export { performanceStartTraceTool, diff --git a/app/chrome-extension/entrypoints/background/tools/browser/network-request.ts b/app/chrome-extension/entrypoints/background/tools/browser/network-request.ts index 96ca1967..93b58cdb 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/network-request.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/network-request.ts @@ -11,6 +11,10 @@ interface NetworkRequestToolParams { headers?: Record; // User-provided headers body?: any; // User-provided body timeout?: number; // Timeout for the network request itself + // Optional multipart/form-data descriptor. When provided, overrides body and lets the helper build FormData. + // Shape: { fields?: Record, files?: Array<{ name: string, fileUrl?: string, filePath?: string, base64Data?: string, filename?: string, contentType?: string }> } + // Or a compact array: [ [name, fileSpec, filename?], ... ] where fileSpec can be 'url:...', 'file:/abs/path', 'base64:...' + formData?: any; } /** @@ -54,6 +58,7 @@ class NetworkRequestTool extends BaseBrowserToolExecutor { method: method, headers: headers, body: body, + formData: args.formData || null, timeout: timeout, }); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts b/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts index 425e25eb..c8accfa3 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/read-page.ts @@ -27,7 +27,14 @@ class ReadPageTool extends BaseBrowserToolExecutor { return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID'); // Inject helper in ISOLATED world to enable chrome.runtime messaging - await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']); + // Inject into all frames to support same-origin iframe operations + await this.injectContentScript( + tab.id, + ['inject-scripts/accessibility-tree-helper.js'], + false, + 'ISOLATED', + true, + ); // Ask content script to generate accessibility tree const resp = await this.sendMessageToTab(tab.id, { diff --git a/app/chrome-extension/entrypoints/builder/App.vue b/app/chrome-extension/entrypoints/builder/App.vue new file mode 100644 index 00000000..e671ce25 --- /dev/null +++ b/app/chrome-extension/entrypoints/builder/App.vue @@ -0,0 +1,827 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/builder/index.html b/app/chrome-extension/entrypoints/builder/index.html new file mode 100644 index 00000000..7afd432a --- /dev/null +++ b/app/chrome-extension/entrypoints/builder/index.html @@ -0,0 +1,13 @@ + + + + + + 工作流编辑器 + + + +
+ + + diff --git a/app/chrome-extension/entrypoints/builder/main.ts b/app/chrome-extension/entrypoints/builder/main.ts new file mode 100644 index 00000000..486851ff --- /dev/null +++ b/app/chrome-extension/entrypoints/builder/main.ts @@ -0,0 +1,7 @@ +import { createApp } from 'vue'; +import App from './App.vue'; + +// Tailwind first, then custom tokens +import '../styles/tailwind.css'; + +createApp(App).mount('#app'); diff --git a/app/chrome-extension/entrypoints/options/App.vue b/app/chrome-extension/entrypoints/options/App.vue index d87faecf..694f9cf1 100644 --- a/app/chrome-extension/entrypoints/options/App.vue +++ b/app/chrome-extension/entrypoints/options/App.vue @@ -137,8 +137,7 @@ - - + @@ -146,7 +145,6 @@ import { ref, onMounted } from 'vue'; import { TOOL_NAMES } from 'chrome-mcp-shared'; import { STORAGE_KEYS } from '@/common/constants'; -import FlowEditor from './components/FlowEditor.vue'; type ListItem = { id: string; diff --git a/app/chrome-extension/entrypoints/options/components/FlowEditor.vue b/app/chrome-extension/entrypoints/options/components/FlowEditor.vue deleted file mode 100644 index d91e6502..00000000 --- a/app/chrome-extension/entrypoints/options/components/FlowEditor.vue +++ /dev/null @@ -1,762 +0,0 @@ - - - - - diff --git a/app/chrome-extension/entrypoints/popup/App.vue b/app/chrome-extension/entrypoints/popup/App.vue index 7a63e8f4..05e6ed74 100644 --- a/app/chrome-extension/entrypoints/popup/App.vue +++ b/app/chrome-extension/entrypoints/popup/App.vue @@ -227,7 +227,7 @@

录制与回放

-
+
- -
-
-
- -
-
暂无录制流
-
-
-
{{ f.name }}
-
{{ f.description || '' }}
-
-
- - - - - - -
-
+
@@ -295,26 +279,7 @@ @cancel="hideClearDataConfirmation" /> - - - +
@@ -334,9 +299,6 @@ import { getMessage } from '@/utils/i18n'; import ConfirmDialog from './components/ConfirmDialog.vue'; import ProgressIndicator from './components/ProgressIndicator.vue'; import ModelCacheManagement from './components/ModelCacheManagement.vue'; -import FlowEditor from './components/FlowEditor.vue'; -import BuilderEditor from './components/BuilderEditor.vue'; -import ScheduleDialog from './components/ScheduleDialog.vue'; import { DocumentIcon, DatabaseIcon, @@ -349,28 +311,32 @@ import { // Record & Replay state const rrRecording = ref(false); -const rrFlows = ref>([]); +const rrFlows = ref< + Array<{ id: string; name: string; description?: string; meta?: any; variables?: any[] }> +>([]); const rrOnlyBound = ref(false); +const rrSearch = ref(''); const currentTabUrl = ref(''); -const filteredRrFlows = computed(() => - rrOnlyBound.value ? rrFlows.value.filter(isFlowBoundToCurrent) : rrFlows.value, -); +const filteredRrFlows = computed(() => { + const base = rrOnlyBound.value ? rrFlows.value.filter(isFlowBoundToCurrent) : rrFlows.value; + const q = rrSearch.value.trim().toLowerCase(); + if (!q) return base; + return base.filter((f: any) => { + const name = String(f.name || '').toLowerCase(); + const domain = String(f?.meta?.domain || '').toLowerCase(); + const tags = ((f?.meta?.tags || []) as any[]).join(',').toLowerCase(); + return name.includes(q) || domain.includes(q) || tags.includes(q); + }); +}); -// Flow editor state -const showFlowEditor = ref(false); -const editingFlow = ref(null); -const showBuilderEditor = ref(false); -const editingFlowBuilder = ref(null); -const showSchedule = ref(false); -const schedulingFlowId = ref(null); -const schedules = ref([]); +// Flow editor在独立窗口中打开；在popup不再展示繁杂列表 const loadFlows = async () => { try { const res = await chrome.runtime.sendMessage({ type: BACKGROUND_MESSAGE_TYPES.RR_LIST_FLOWS }); if (res && res.success) rrFlows.value = res.flows || []; } catch (e) { - console.error('加载录制流失败:', e); + /* ignore */ } }; @@ -391,6 +357,8 @@ function isFlowBoundToCurrent(flow: any) { } } +// 运行记录与覆盖项在侧边栏页面查看 + const startRecording = async () => { if (rrRecording.value) return; try { @@ -421,127 +389,50 @@ const stopRecording = async () => { const runFlow = async (flowId: string) => { try { + // load flow to get runOptions + let flow: any = null; + try { + const getRes = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_GET_FLOW, + flowId, + }); + if (getRes && getRes.success) flow = getRes.flow; + } catch {} + const runOptions = (flow && flow.meta && flow.meta.runOptions) || {}; + // No per-run overrides in popup; sidepanel/editor manage advanced options + const ov: any = {}; const res = await chrome.runtime.sendMessage({ type: BACKGROUND_MESSAGE_TYPES.RR_RUN_FLOW, flowId, - options: { returnLogs: true }, + options: { ...runOptions, ...ov, returnLogs: true }, }); - if (!(res && res.success)) console.warn('回放失败'); + if (!(res && res.success)) { + console.warn('回放失败'); + return; + } + // If failed, open builder and focus the failed node + try { + const result = res.result; + if (result && result.success === false) { + const logs = result.logs || []; + const failed = logs.find((l: any) => l.status === 'failed'); + if (failed && failed.stepId) { + // 打开独立编辑窗口并定位失败节点 + if (flow) openBuilderWindow(flow.id, String(failed.stepId)); + } + } else if (result && result.success === true) { + // If run succeeded but selector fallback was used, suggest updating priorities + const logs = result.logs || []; + const fb = logs.find((l: any) => l.fallbackUsed && l.fallbackTo); + if (fb && flow) openBuilderWindow(flow.id, String(fb.stepId || '')); + } + } catch {} } catch (e) { console.error('回放失败:', e); } }; -const publishFlow = async (flowId: string) => { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_PUBLISH_FLOW, - flowId, - }); - if (!(res && res.success)) console.warn('发布失败'); - } catch (e) { - console.error('发布失败:', e); - } -}; - -const deleteFlow = async (flowId: string) => { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_DELETE_FLOW, - flowId, - }); - if (res && res.success) await loadFlows(); - } catch (e) { - console.error('删除失败:', e); - } -}; - -function editFlow(flow: any) { - editingFlow.value = flow; - showFlowEditor.value = true; -} - -function openBuilder(flow: any) { - editingFlowBuilder.value = flow; - showBuilderEditor.value = true; -} - -async function saveEditedFlow(f: any) { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_SAVE_FLOW, - flow: f, - }); - if (res && res.success) { - showFlowEditor.value = false; - editingFlow.value = null; - await loadFlows(); - } - } catch (e) { - console.error('保存失败:', e); - } -} - -async function saveEditedFlowFromBuilder(f: any) { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_SAVE_FLOW, - flow: f, - }); - if (res && res.success) { - showBuilderEditor.value = false; - editingFlowBuilder.value = null; - await loadFlows(); - } - } catch (e) { - console.error('保存失败:', e); - } -} - -async function openSchedule(flowId: string) { - schedulingFlowId.value = flowId; - await loadSchedules(); - showSchedule.value = true; -} - -async function loadSchedules() { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_LIST_SCHEDULES, - }); - if (res && res.success) schedules.value = res.schedules || []; - } catch (e) { - console.error('加载定时失败:', e); - } -} - -async function saveSchedule(schedule: any) { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_SCHEDULE_FLOW, - schedule, - }); - if (res && res.success) { - await loadSchedules(); - showSchedule.value = false; - schedulingFlowId.value = null; - } - } catch (e) { - console.error('保存定时失败:', e); - } -} - -async function removeSchedule(id: string) { - try { - const res = await chrome.runtime.sendMessage({ - type: BACKGROUND_MESSAGE_TYPES.RR_UNSCHEDULE_FLOW, - scheduleId: id, - }); - if (res && res.success) await loadSchedules(); - } catch (e) { - console.error('删除计划失败:', e); - } -} +// 旧的“克隆/发布/定时/覆盖项”在侧边栏或编辑器中处理 const nativeConnectionStatus = ref<'unknown' | 'connected' | 'disconnected'>('unknown'); const isConnecting = ref(false); @@ -643,6 +534,29 @@ const getStatusClass = () => { } }; +// Open sidepanel from popup for workflow management +async function openWorkflowSidepanel() { + try { + const current = await chrome.windows.getCurrent(); + // Ensure the side panel uses our page + if ((chrome.sidePanel as any)?.setOptions) { + await (chrome.sidePanel as any).setOptions({ path: 'sidepanel.html', enabled: true }); + } + if (chrome.sidePanel && (chrome.sidePanel as any).open) { + await (chrome.sidePanel as any).open({ windowId: current.id! }); + } + } catch (e) { + console.warn('打开侧边栏失败:', e); + } +} + +function openBuilderWindow(flowId?: string, focusNodeId?: string) { + const url = new URL(chrome.runtime.getURL('builder.html')); + if (flowId) url.searchParams.set('flowId', flowId); + if (focusNodeId) url.searchParams.set('focus', focusNodeId); + chrome.windows.create({ url: url.toString(), type: 'popup', width: 1280, height: 800 }); +} + const getStatusText = () => { if (nativeConnectionStatus.value === 'connected') { if (serverStatus.value.isRunning) { @@ -2176,6 +2090,13 @@ onUnmounted(() => { border: 1px solid #eee; border-radius: 6px; } + .rr-runoverrides { + margin-top: 6px; + border: 1px dashed #e5e7eb; + border-radius: 8px; + padding: 8px; + background: #f9fafb; + } .rr-meta { display: flex; flex-direction: column; diff --git a/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue b/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue index c8b76f94..22761927 100644 --- a/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue +++ b/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue @@ -1,55 +1,210 @@ @@ -94,7 +268,12 @@ import Canvas from './builder/components/Canvas.vue'; import Sidebar from './builder/components/Sidebar.vue'; import PropertyPanel from './builder/components/PropertyPanel.vue'; -const props = defineProps<{ visible: boolean; flow: FlowV2 | null }>(); +const props = defineProps<{ + visible: boolean; + flow: FlowV2 | null; + initialFocusNodeId?: string | null; + fallbackHint?: { nodeId: string; toType: string } | null; +}>(); const emit = defineEmits(['close', 'save']); const store = useBuilderStore(); @@ -125,11 +304,27 @@ const fitSeq = ref(0); function focusSearch() { const q = search.value.trim().toLowerCase(); if (!q) return; - const hit = store.nodes.find( - (n) => - (n.name || '').toLowerCase().includes(q) || - (n.config?.target?.candidates?.[0]?.value || '').toLowerCase().includes(q), - ); + const matches = (n: any): boolean => { + if ((n.name || '').toLowerCase().includes(q)) return true; + if ((n.type || '').toLowerCase().includes(q)) return true; + try { + // search first selector + if ((n.config?.target?.candidates?.[0]?.value || '').toLowerCase().includes(q)) return true; + // deep search string fields including variable placeholders {var} + const walk = (v: any): boolean => { + if (v == null) return false; + if (typeof v === 'string') + return v.toLowerCase().includes(q) || v.toLowerCase().includes(`{${q}}`); + if (Array.isArray(v)) return v.some(walk); + if (typeof v === 'object') return Object.values(v).some(walk); + return false; + }; + return walk(n.config); + } catch { + return false; + } + }; + const hit = store.nodes.find((n) => matches(n)); if (hit) { store.selectNode(hit.id); focusNodeId.value = hit.id; @@ -144,13 +339,52 @@ function exportToSteps() { store.flowLocal.steps = nodesToSteps(store.nodes, store.edges); } function save() { - store.flowLocal.steps = nodesToSteps(store.nodes, store.edges); + // Only map steps when editing main graph + if (store.isEditingMain()) store.flowLocal.steps = nodesToSteps(store.nodes, store.edges); const result = JSON.parse( JSON.stringify({ ...store.flowLocal, nodes: store.nodes, edges: store.edges }), ); emit('save', result); } +// Fallback suggestion notice + apply/undo helpers +const fallbackNotice = ref<{ nodeId: string; type: string; prevIndex: number } | null>(null); +function applyFallbackPromotion(nodeId: string, toType: string) { + const node = store.nodes.find((n) => n.id === nodeId); + if (!node || (node.type !== 'click' && node.type !== 'fill')) return; + const cands = (node as any).config?.target?.candidates as Array<{ type: string; value: string }>; + if (!Array.isArray(cands) || !cands.length) return; + const idx = cands.findIndex((c) => c.type === String(toType)); + if (idx > 0) { + const cand = cands.splice(idx, 1)[0]; + cands.unshift(cand); + fallbackNotice.value = { nodeId, type: String(toType), prevIndex: idx }; + focusNode(nodeId); + highlightField.value = 'target.candidates'; + setTimeout(() => (highlightField.value = null), 1500); + } +} +function undoFallbackPromotion() { + const n = fallbackNotice.value; + if (!n) return; + const node = store.nodes.find((x) => x.id === n.nodeId); + if (!node || (node.type !== 'click' && node.type !== 'fill')) { + fallbackNotice.value = null; + return; + } + const cands = (node as any).config?.target?.candidates as Array<{ type: string; value: string }>; + if (!Array.isArray(cands) || cands.length === 0) { + fallbackNotice.value = null; + return; + } + const currentIdx = cands.findIndex((c) => c.type === n.type); + if (currentIdx >= 0 && n.prevIndex >= 0 && n.prevIndex < cands.length) { + const cand = cands.splice(currentIdx, 1)[0]; + cands.splice(n.prevIndex, 0, cand); + } + fallbackNotice.value = null; +} + async function runFromSelected() { if (!selectedId.value || !store.flowLocal?.id) return; try { @@ -158,20 +392,87 @@ async function runFromSelected() { const res = await chrome.runtime.sendMessage({ type: BACKGROUND_MESSAGE_TYPES.RR_RUN_FLOW, flowId: store.flowLocal.id, - options: { returnLogs: true, startNodeId: selectedId.value }, + options: { + ...(((store.flowLocal as any).meta && (store.flowLocal as any).meta.runOptions) || {}), + returnLogs: true, + startNodeId: selectedId.value, + }, }); - if (!(res && res.success)) console.warn('从选中节点回放失败'); + if (!(res && res.success)) { + console.warn('从选中节点回放失败'); + return; + } + // Focus first failed step/node if any + try { + const logs = res.result?.logs || []; + const failed = logs.find((l: any) => l.status === 'failed'); + if (failed && failed.stepId) { + focusNode(String(failed.stepId)); + } + // If selector fallback was used for this step, promote the matched type + const thisStepLogs = logs.filter((l: any) => l.stepId === selectedId.value); + const fb = thisStepLogs.find((l: any) => l.fallbackUsed && l.fallbackTo); + if (fb && fb.fallbackTo) applyFallbackPromotion(selectedId.value, String(fb.fallbackTo)); + } catch {} } catch (e) { console.error('从选中节点回放失败:', e); } } +async function runAll() { + if (!store.flowLocal?.id) return; + try { + await save(); + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_RUN_FLOW, + flowId: store.flowLocal.id, + options: { + ...(((store.flowLocal as any).meta && (store.flowLocal as any).meta.runOptions) || {}), + returnLogs: true, + }, + }); + if (!(res && res.success)) return; + try { + const logs = res.result?.logs || []; + const failed = logs.find((l: any) => l.status === 'failed'); + if (failed && failed.stepId) focusNode(String(failed.stepId)); + } catch {} + } catch (e) { + console.error('整流回放失败:', e); + } +} + +function onAddNodeAt(type: string, x: number, y: number) { + try { + store.addNodeAt(type as any, x, y); + } catch {} +} + function focusNode(id: string) { store.selectNode(id); focusNodeId.value = id; setTimeout(() => (focusNodeId.value = null), 300); } +// Auto focus when parent passes a node id (e.g., failed step) +watch( + () => props.initialFocusNodeId, + (nid) => { + if (nid) focusNode(nid); + }, + { immediate: true }, +); + +// Apply fallback hint from parent (promote matched candidate type) +watch( + () => props.fallbackHint, + (hint) => { + if (!hint) return; + applyFallbackPromotion(hint.nodeId, String(hint.toType)); + }, + { immediate: true }, +); + function focusError(nid: string, msg: string) { const node = store.nodes.find((n) => n.id === nid); if (!node) return focusNode(nid); @@ -195,6 +496,19 @@ function fitAll() { fitSeq.value++; } +// Canvas controls via template ref +const canvasRef = ref(null); +function zoomIn() { + try { + canvasRef.value?.zoomIn?.(); + } catch {} +} +function zoomOut() { + try { + canvasRef.value?.zoomOut?.(); + } catch {} +} + async function exportFlow() { try { await save(); @@ -251,6 +565,15 @@ function onKey(e: KeyboardEvent) { e.preventDefault(); store.duplicateNode(id); } + } else if (isMeta && e.key.toLowerCase?.() === 'c') { + // copy (single selection) + e.preventDefault(); + if (id) (window as any).__builder_clipboard = id; + } else if (isMeta && e.key.toLowerCase?.() === 'v') { + // paste (duplicate selected or copied id) + e.preventDefault(); + const srcId = id || (window as any).__builder_clipboard; + if (srcId) store.duplicateNode(srcId); } else if (isMeta && e.key.toLowerCase?.() === 'z') { e.preventDefault(); if (e.shiftKey) store.redo(); @@ -296,121 +619,367 @@ watch( .builder-modal { position: fixed; inset: 0; - background: rgba(0, 0, 0, 0.35); + background: rgba(0, 0, 0, 0.4); z-index: 2147483646; display: flex; align-items: center; justify-content: center; + backdrop-filter: blur(4px); } .builder { width: 96vw; - height: 90vh; - background: #fff; - border-radius: 10px; + height: 92vh; + background: var(--rr-bg); + border-radius: 16px; display: flex; flex-direction: column; overflow: hidden; + box-shadow: 0 20px 60px rgba(0, 0, 0, 0.25); +} +.builder.rr-theme { + --rr-bg: #0a0a0a; + --rr-topbar: #1a1a1a; + --rr-card: #1a1a1a; + --rr-elevated: #262626; + --rr-border: #2a2a2a; + --rr-border-light: #333333; + --rr-subtle: #1f1f1f; + --rr-text: #e5e5e5; + --rr-text-secondary: #a3a3a3; + --rr-text-weak: #737373; + --rr-muted: #525252; + --rr-brand: #f59e0b; + --rr-brand-strong: #d97706; + --rr-accent: #3b82f6; + --rr-success: #22c55e; + --rr-warn: #f59e0b; + --rr-danger: #ef4444; + --rr-hover: #262626; } + +/* 顶部工具栏 */ .topbar { - height: 48px; + height: 64px; display: flex; align-items: center; justify-content: space-between; - padding: 0 12px; - border-bottom: 1px solid #e5e7eb; + padding: 0 20px; + border-bottom: 1px solid var(--rr-border); + background: #ededed; + gap: 20px; } -.topbar .left { + +.topbar-left { display: flex; - gap: 8px; align-items: center; + gap: 12px; + flex: 1; + min-width: 0; } -.topbar .tip { - color: #6b7280; - font-size: 12px; + +.btn-back { + width: 36px; + height: 36px; + display: flex; + align-items: center; + justify-content: center; + border: none; + background: transparent; + border-radius: 8px; + cursor: pointer; + color: var(--rr-text-secondary); + transition: all 0.15s; +} + +.btn-back:hover { + background: var(--rr-hover); + color: var(--rr-text); +} + +.workflow-title { + border: none; + background: transparent; + font-size: 16px; + font-weight: 600; + color: var(--rr-text); + padding: 8px 12px; + border-radius: 6px; + outline: none; + max-width: 400px; } -.topbar .right { + +.workflow-title:hover { + background: var(--rr-hover); +} + +.workflow-title:focus { + background: var(--rr-hover); +} + +.draft-badge { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 12px; + background: var(--rr-subtle); + color: var(--rr-text-secondary); + font-size: 13px; + font-weight: 500; + border-radius: 8px; +} + +.topbar-right { display: flex; + align-items: center; gap: 8px; +} + +/* 工具栏按钮组 */ +.toolbar-group { + display: flex; align-items: center; + gap: 4px; + padding: 4px; + background: var(--rr-subtle); + border-radius: 8px; } -.btn { - border: 1px solid #d1d5db; - background: #fff; + +.toolbar-btn { + width: 36px; + height: 36px; + display: flex; + align-items: center; + justify-content: center; + border: none; + background: transparent; border-radius: 6px; - padding: 6px 10px; cursor: pointer; + color: var(--rr-text-secondary); + transition: all 0.15s; +} + +.toolbar-btn:hover:not(:disabled) { + background: var(--rr-card); + color: var(--rr-text); +} + +.toolbar-btn:disabled { + opacity: 0.4; + cursor: not-allowed; +} + +.toolbar-divider { + width: 1px; + height: 24px; + background: var(--rr-border); + margin: 0 4px; +} + +/* 主按钮 */ +.btn-primary { + display: flex; + align-items: center; + gap: 6px; + padding: 8px 16px; + border: none; + background: var(--rr-text); + color: var(--rr-card); + border-radius: 8px; + font-size: 14px; + font-weight: 600; + cursor: pointer; + transition: all 0.15s; } -.btn.primary { - background: #111; + +.btn-primary:hover { + background: var(--rr-text-secondary); + transform: translateY(-1px); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); +} + +.btn-publish { + display: flex; + align-items: center; + gap: 6px; + padding: 8px 16px; + border: none; + background: var(--rr-brand); + color: #fff; + border-radius: 8px; + font-size: 14px; + font-weight: 600; + cursor: pointer; + transition: all 0.15s; +} + +.btn-publish:hover { + background: var(--rr-brand-strong); + transform: translateY(-1px); + box-shadow: 0 4px 12px rgba(245, 158, 11, 0.3); +} + +/* 图标按钮 */ +.toolbar-btn-icon { + position: relative; + width: 36px; + height: 36px; + display: flex; + align-items: center; + justify-content: center; + border: 1px solid var(--rr-border); + background: var(--rr-card); + border-radius: 8px; + cursor: pointer; + color: var(--rr-text-secondary); + transition: all 0.15s; +} + +.toolbar-btn-icon:hover { + background: var(--rr-hover); + border-color: var(--rr-text-weak); + color: var(--rr-text); +} + +.error-badge-count { + position: absolute; + top: -4px; + right: -4px; + min-width: 18px; + height: 18px; + display: flex; + align-items: center; + justify-content: center; + padding: 0 5px; + background: var(--rr-danger); color: #fff; - border-color: #111; + font-size: 11px; + font-weight: 600; + border-radius: 9px; + border: 2px solid var(--rr-topbar); } .main { flex: 1; display: grid; - grid-template-columns: 280px 1fr 360px; + grid-template-columns: 180px 1fr 420px; + gap: 0; + padding: 0; + overflow: hidden; + background: var(--rr-bg); } -.topbar .search { - border: 1px solid #d1d5db; - border-radius: 6px; - padding: 6px 10px; - margin-right: 8px; - min-width: 240px; + +/* 底部工具栏 */ +.bottombar { + height: 40px; + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 12px; + border-top: 1px solid var(--rr-border); + background: #f0f0f0; } -.topbar .status { - color: #6b7280; +.bottombar .status { + color: var(--rr-muted); font-size: 12px; - margin-right: 8px; - min-width: 48px; - display: inline-block; } +.zoom-group { + display: flex; + align-items: center; + gap: 8px; +} +.zoom-btn { + min-width: 40px; + height: 28px; + padding: 0 10px; + border: 1px solid var(--rr-border); + background: var(--rr-card); + color: var(--rr-text); + border-radius: 6px; + font-size: 13px; + cursor: pointer; +} +.zoom-btn:hover { + background: var(--rr-hover); +} + +/* 错误面板 */ .error-panel { position: absolute; - right: 12px; - top: 56px; - width: 420px; - max-height: 50vh; - background: #fff; - border: 1px solid #e5e7eb; - border-radius: 8px; - box-shadow: 0 4px 16px rgba(0, 0, 0, 0.08); - padding: 10px; + right: 360px; + top: 80px; + width: 380px; + max-height: 60vh; + background: var(--rr-card); + border: 1px solid var(--rr-border); + border-radius: 12px; + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.12); + padding: 16px; overflow: auto; + z-index: 50; } .err-title { font-weight: 600; - margin-bottom: 6px; + font-size: 14px; + margin-bottom: 12px; + color: var(--rr-text); +} +.err-list { + display: flex; + flex-direction: column; + gap: 8px; } .err-item { display: grid; - grid-template-columns: 120px 1fr; - gap: 6px; - padding: 6px; - border: 1px solid #f3f4f6; - border-radius: 6px; + grid-template-columns: 100px 1fr; + gap: 10px; + padding: 10px; + border: 1px solid var(--rr-border-light); + border-radius: 8px; cursor: pointer; - margin-bottom: 6px; + transition: all 0.15s; } .err-item:hover { - background: #f9fafb; + background: var(--rr-subtle); + border-color: var(--rr-border); } .err-item .nid { font-size: 12px; - color: #374151; + font-weight: 600; + color: var(--rr-text-secondary); +} +.err-item .elist { + display: flex; + flex-direction: column; + gap: 4px; } .err-item .e { font-size: 12px; - color: #ef4444; + color: var(--rr-danger); + line-height: 1.4; } -.btn.import { - position: relative; - overflow: hidden; -} -.btn.import input { - position: absolute; - inset: 0; - opacity: 0; + +/* 通知栏 */ +.notice-top { + background: linear-gradient(135deg, #3b82f6 0%, #2563eb 100%); + color: #fff; + padding: 10px 24px; + display: flex; + align-items: center; + gap: 12px; + font-size: 14px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); +} +.notice-top .mini { + padding: 4px 12px; + background: rgba(255, 255, 255, 0.2); + border: 1px solid rgba(255, 255, 255, 0.3); + color: #fff; + border-radius: 6px; + font-size: 13px; cursor: pointer; + transition: all 0.15s; +} +.notice-top .mini:hover { + background: rgba(255, 255, 255, 0.3); } diff --git a/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue b/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue deleted file mode 100644 index 0a78b672..00000000 --- a/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue +++ /dev/null @@ -1,363 +0,0 @@ - - - - - diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue index 78e683e9..cafc18aa 100644 --- a/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue @@ -1,65 +1,43 @@ diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue index 14d2b3b9..6c9d78a8 100644 --- a/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue @@ -1,227 +1,475 @@