🐛 fix(desktop): 使用真实 CDP 输入并修复搜索框定位

apps/desktop/scripts/browser-runtime.e2e.mjs

@@ -122,7 +122,19 @@ const server = createServer((request, response) => {
     <a id="download" href="/download" download="fixture.txt">Download</a>
     <output id="result"></output>
     <iframe id="cross-origin-frame" src="\${frameOrigin}/"></iframe>
+    <form id="search-form" style="display:none;position:fixed;right:8px;bottom:8px;width:260px;height:70px">
+      <div style="position:relative;width:260px;height:40px">
+        <input id="kw" name="wd" title="搜索" placeholder="请输入搜索内容" style="box-sizing:border-box;width:260px;height:40px">
+        <label for="kw" style="position:absolute;inset:0;color:transparent">搜索</label>
+      </div>
+      <button id="su" type="submit">百度一下</button>
+    </form>
+    <output id="search-result"></output>
     <script>
+      document.querySelector('#search-form').addEventListener('submit', event => {
+        event.preventDefault();
+        document.querySelector('#search-result').textContent = document.querySelector('#kw').value;
+      });
       document.modelContext = {
         getTools: () => [{
           name: 'set_result',
@@ -265,7 +277,8 @@ app.whenReady().then(async () => {
     const navigatedUrl = await call('url', { tabId: 'fixture-tab' })
     check(navigatedUrl.startsWith(origin), 'fixture navigation failed: ' + JSON.stringify(navigatedUrl))
     setStage('popup-policy')
-    await call('click', { tabId: 'fixture-tab', locator: { version: 1, steps: [{ kind: 'css', selector: '#open-popup' }] } })
+    const popupAction = await call('click', { tabId: 'fixture-tab', locator: { version: 1, steps: [{ kind: 'css', selector: '#open-popup' }] } })
+    check(popupAction.effect?.kind === 'new_tab_requested', 'Agent action did not report its popup effect')
     await waitUntil(() => events.some(event => event.method === 'browser:popup-request' && event.params?.tabId === 'fixture-tab'))
     check(BrowserWindow.getAllWindows().length === 1, 'Agent-triggered popup bypassed confirmation')
     const popupCreated = new Promise(resolvePopup => view.once('did-create-window', resolvePopup))
@@ -276,6 +289,13 @@ app.whenReady().then(async () => {
     await popupWindow.webContents.executeJavaScript(\`new Promise(resolve => document.readyState === 'complete' ? resolve() : addEventListener('load', resolve, { once: true }))\`)
     check(await popupWindow.webContents.executeJavaScript(\`document.querySelector('#opener').textContent\`) === 'present', 'user popup lost its opener relationship')
     popupWindow.close()
+    const takenOver = await userCall('get', { tabId: 'fixture-tab' })
+    check(takenOver.agentControlState === 'paused_by_user', 'real user input did not pause Agent browser control')
+    await checkRejects(() => call('url', { tabId: 'fixture-tab' }), 'user_takeover_required', 'Agent kept controlling a user-taken-over tab')
+    await checkRejects(() => call('ensure', { tabId: 'takeover-side-tab' }), 'user_takeover_required', 'Agent opened a side tab to bypass a user takeover')
+    await checkRejects(() => call('close', { tabId: 'fixture-tab' }), 'user_takeover_required', 'Agent closed a tab the user had taken over')
+    const resumedControl = await userCall('agentControl:resume', { tabId: 'fixture-tab' })
+    check(resumedControl.agentControlState === 'active' && resumedControl.generation > takenOver.generation, 'explicit user resume did not invalidate old Agent page state')
     if (process.env.LUME_BROWSER_POPUP_ONLY === '1') {
       writeFileSync(resultPath, JSON.stringify({ ok: true, assertions }))
       return
@@ -309,6 +329,18 @@ app.whenReady().then(async () => {
     const geolocationPermission = await view.executeJavaScript("new Promise(resolve => navigator.geolocation.getCurrentPosition(() => resolve('allowed'), error => resolve(error.code === 1 ? 'denied' : 'other'), { timeout: 1000 }))")
     check(geolocationPermission === 'denied', 'agent browser session did not deny site permissions')
     const locator = selector => ({ version: 1, steps: [{ kind: 'css', selector }] })
+    await view.executeJavaScript("document.querySelector('#search-form').style.display = 'block'")
+    const searchByRole = { version: 1, steps: [{ kind: 'role', role: 'textbox', name: '搜索', exact: true }] }
+    await call('fill', { tabId: 'fixture-tab', locator: searchByRole, text: 'agent loop' })
+    check(await call('locator:inputValue', { tabId: 'fixture-tab', locator: locator('#kw') }) === 'agent loop', 'role textbox fill did not resolve the associated search label')
+    await call('press', { tabId: 'fixture-tab', locator: searchByRole, key: 'Enter' })
+    check(await call('locator:innerText', { tabId: 'fixture-tab', locator: locator('#search-result') }) === 'agent loop', 'search textbox Enter did not submit the form')
+    check(view.getURL() === origin + '/', 'search textbox Enter unexpectedly navigated: ' + view.getURL())
+    await call('fill', { tabId: 'fixture-tab', locator: locator('#kw'), text: 'Lume browser' })
+    await call('click', { tabId: 'fixture-tab', locator: locator('#su') })
+    check(await call('locator:innerText', { tabId: 'fixture-tab', locator: locator('#search-result') }) === 'Lume browser', 'CSS search textbox fill did not submit the expected value')
+    check(view.getURL() === origin + '/', 'search button unexpectedly navigated: ' + view.getURL())
+    await view.executeJavaScript("document.querySelector('#search-form').style.display = 'none'")
     await call('fill', { tabId: 'fixture-tab', locator: locator('#name'), text: 'Lume Agent' })
     const inputValue = await view.executeJavaScript("document.querySelector('#name').value")
     check(inputValue === 'Lume Agent', 'locator fill did not update the input: ' + JSON.stringify(inputValue))
@@ -317,7 +349,7 @@ app.whenReady().then(async () => {
     check(await call('locator:evaluate', { tabId: 'fixture-tab', locator: locator('#name'), expression: '(element, suffix) => element.value + suffix', arg: '!' }) === 'Lume Agent!', 'locator evaluate did not receive the strict element')
     await checkRejects(() => call('locator:evaluate', { tabId: 'fixture-tab', locator: locator('#name'), expression: '(element) => { element.value = \"mutated\"; return element.value }' }), 'action_denied', 'locator evaluate allowed a side effect')
     check(await call('locator:inputValue', { tabId: 'fixture-tab', locator: locator('#name') }) === 'Lume Agent', 'rejected locator evaluate changed the page')
-    check(await call('locator:count', { tabId: 'fixture-tab', locator: locator('button') }) === 3, 'locator count was incorrect')
+    check(await call('locator:count', { tabId: 'fixture-tab', locator: locator('button') }) === 4, 'locator count was incorrect')
     await call('locator:waitFor', { tabId: 'fixture-tab', locator: locator('#submit'), state: 'visible', timeoutMs: 1000 })
     await call('wait:url', { tabId: 'fixture-tab', url: origin + '/*', timeoutMs: 1000 })
     await call('click', { tabId: 'fixture-tab', locator: locator('#submit') })
@@ -365,6 +397,12 @@ app.whenReady().then(async () => {
       staleTargetObserved = actionResult.status === 'rejected'
     }
     check(staleTargetObserved, 'navigation race allowed an action against a stale target')
+    await view.loadURL(origin + '/')
+    const queuedNavigation = call('navigate', { tabId: 'fixture-tab', url: origin + '/queued-navigation' })
+    const queuedOldPageAction = call('fill', { tabId: 'fixture-tab', locator: locator('#name'), text: 'must-not-run' })
+    const [queuedNavigationResult, queuedActionResult] = await Promise.allSettled([queuedNavigation, queuedOldPageAction])
+    check(queuedNavigationResult.status === 'fulfilled', 'queued navigation did not complete')
+    check(queuedActionResult.status === 'rejected' && String(queuedActionResult.reason?.code ?? queuedActionResult.reason?.message).includes('stale_target'), 'an old-page action ran after queued navigation')
     const beforeCrash = await call('get', { tabId: 'fixture-tab' })
     setStage('crash-recovery')
     view.forcefullyCrashRenderer()

apps/desktop/src/browser-action-effect.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, test } from "bun:test"
+import { detectBrowserActionEffect, type BrowserActionEffectSnapshot } from "./browser-action-effect"
+
+describe("detectBrowserActionEffect", () => {
+  test("prioritizes blocking and navigation effects over DOM noise", () => {
+    const before = snapshot()
+    expect(detectBrowserActionEffect(before, { ...before, dialogId: "dialog-1", domRevision: 2 })).toEqual({ kind: "dialog_opened" })
+    expect(detectBrowserActionEffect(before, { ...before, generation: 2, url: "https://example.test/next", domRevision: 2 })).toEqual({ kind: "navigation", url: "https://example.test/next" })
+  })
+
+  test("reports new tabs, downloads, popup requests and DOM changes", () => {
+    const before = snapshot()
+    expect(detectBrowserActionEffect(before, { ...before, tabIds: ["tab-1", "tab-2"] })).toEqual({ kind: "new_tab_opened", new_tab_ids: ["tab-2"] })
+    expect(detectBrowserActionEffect(before, { ...before, popupCount: 1 })).toEqual({ kind: "new_tab_requested" })
+    expect(detectBrowserActionEffect(before, { ...before, downloadIds: ["download-1"] })).toEqual({ kind: "download_started" })
+    expect(detectBrowserActionEffect(before, { ...before, domRevision: 2 })).toEqual({ kind: "dom_changed" })
+    expect(detectBrowserActionEffect(before, before)).toBeUndefined()
+  })
+})
+
+function snapshot(): BrowserActionEffectSnapshot {
+  return {
+    domRevision: 1,
+    downloadIds: [],
+    generation: 1,
+    lifecycle: "active",
+    popupCount: 0,
+    tabIds: ["tab-1"],
+    url: "https://example.test/",
+  }
+}

apps/desktop/src/browser-action-effect.ts

@@ -0,0 +1,38 @@
+export type BrowserActionEffectKind =
+  | "crashed"
+  | "dialog_opened"
+  | "new_tab_opened"
+  | "new_tab_requested"
+  | "download_started"
+  | "navigation"
+  | "dom_changed"
+  | "no_detectable_change"
+
+export interface BrowserActionEffectSnapshot {
+  dialogId?: string
+  domRevision: number
+  downloadIds: string[]
+  generation: number
+  lifecycle?: string
+  popupCount: number
+  tabIds: string[]
+  url: string
+}
+
+export interface BrowserActionEffect {
+  kind: BrowserActionEffectKind
+  new_tab_ids?: string[]
+  url?: string
+}
+
+export function detectBrowserActionEffect(before: BrowserActionEffectSnapshot, after: BrowserActionEffectSnapshot): BrowserActionEffect | undefined {
+  if (after.lifecycle === "crashed" && before.lifecycle !== "crashed") return { kind: "crashed" }
+  if (after.dialogId && after.dialogId !== before.dialogId) return { kind: "dialog_opened" }
+  const newTabIds = after.tabIds.filter((tabId) => !before.tabIds.includes(tabId))
+  if (newTabIds.length) return { kind: "new_tab_opened", new_tab_ids: newTabIds }
+  if (after.popupCount > before.popupCount) return { kind: "new_tab_requested" }
+  if (after.downloadIds.some((downloadId) => !before.downloadIds.includes(downloadId))) return { kind: "download_started" }
+  if (after.generation !== before.generation || after.url !== before.url) return { kind: "navigation", url: after.url }
+  if (after.domRevision > before.domRevision) return { kind: "dom_changed" }
+  return undefined
+}

apps/desktop/src/browser-action-queue.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from "bun:test"
+import { BrowserActionQueue } from "./browser-action-queue"
+
+describe("BrowserActionQueue", () => {
+  test("serializes mutations that share a Browser session", async () => {
+    const queue = new BrowserActionQueue()
+    const order: string[] = []
+    let releaseFirst!: () => void
+    const firstGate = new Promise<void>((resolve) => { releaseFirst = resolve })
+
+    const first = queue.run("session-1", async () => {
+      order.push("first:start")
+      await firstGate
+      order.push("first:end")
+      return 1
+    })
+    const second = queue.run("session-1", async () => {
+      order.push("second")
+      return 2
+    })
+
+    await Promise.resolve()
+    expect(order).toEqual(["first:start"])
+    releaseFirst()
+    await expect(Promise.all([first, second])).resolves.toEqual([1, 2])
+    expect(order).toEqual(["first:start", "first:end", "second"])
+  })
+
+  test("continues the queue after an action fails", async () => {
+    const queue = new BrowserActionQueue()
+    const first = queue.run("session-1", async () => { throw new Error("failed") })
+    const second = queue.run("session-1", async () => "recovered")
+
+    await expect(first).rejects.toThrow("failed")
+    await expect(second).resolves.toBe("recovered")
+  })
+
+  test("cancels queued actions without replaying them after user takeover", async () => {
+    const queue = new BrowserActionQueue()
+    let releaseFirst!: () => void
+    const firstGate = new Promise<void>((resolve) => { releaseFirst = resolve })
+    const first = queue.run("session-1", () => firstGate)
+    const second = queue.run("session-1", async () => "must-not-run")
+
+    await Promise.resolve()
+    queue.cancel("session-1")
+    const cancelled = second.then(() => undefined, (error: Error) => error)
+    releaseFirst()
+
+    await expect(first).resolves.toBeUndefined()
+    expect((await cancelled)?.message).toBe("user_takeover_required")
+  })
+})

apps/desktop/src/browser-action-queue.ts

@@ -0,0 +1,24 @@
+export class BrowserActionQueue {
+  private readonly epochs = new Map<string, number>()
+  private readonly tails = new Map<string, Promise<void>>()
+
+  async run<T>(key: string, action: () => Promise<T>): Promise<T> {
+    const epoch = this.epochs.get(key) ?? 0
+    const previous = this.tails.get(key) ?? Promise.resolve()
+    const execution = previous.then(() => {
+      if ((this.epochs.get(key) ?? 0) !== epoch) throw Object.assign(new Error("user_takeover_required"), { code: "user_takeover_required" })
+      return action()
+    })
+    const tail = execution.then(() => undefined, () => undefined)
+    this.tails.set(key, tail)
+    try {
+      return await execution
+    } finally {
+      if (this.tails.get(key) === tail) this.tails.delete(key)
+    }
+  }
+
+  cancel(key: string): void {
+    this.epochs.set(key, (this.epochs.get(key) ?? 0) + 1)
+  }
+}

apps/desktop/src/browser-cdp-input.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, test } from "bun:test"
+import {
+  dispatchBrowserClick,
+  dispatchBrowserKey,
+  dispatchBrowserText,
+  focusBrowserPoint,
+  type BrowserCdpCommandSender,
+} from "./browser-cdp-input"
+
+describe("browser CDP input", () => {
+  test("dispatches a complete real mouse click sequence", async () => {
+    const sender = recorder()
+
+    await dispatchBrowserClick(sender, { x: 12, y: 34 })
+
+    expect(sender.calls.map((call) => [call.method, call.params.type, call.params.clickCount])).toEqual([
+      ["Input.dispatchMouseEvent", "mouseMoved", undefined],
+      ["Input.dispatchMouseEvent", "mousePressed", 1],
+      ["Input.dispatchMouseEvent", "mouseReleased", 1],
+    ])
+  })
+
+  test("replaces text with select-all, backspace and insertText", async () => {
+    const sender = recorder()
+
+    await dispatchBrowserText(sender, "Lume", { platform: "win32", replace: true })
+
+    expect(sender.calls.filter((call) => call.method === "Input.dispatchKeyEvent").map((call) => [call.params.type, call.params.key])).toEqual([
+      ["rawKeyDown", "Control"],
+      ["rawKeyDown", "a"],
+      ["keyUp", "a"],
+      ["keyUp", "Control"],
+      ["rawKeyDown", "Backspace"],
+      ["keyUp", "Backspace"],
+    ])
+    expect(sender.calls.at(-1)).toEqual({ method: "Input.insertText", params: { text: "Lume" } })
+  })
+
+  test("dispatches modifier key lifecycles around the main key", async () => {
+    const sender = recorder()
+
+    await dispatchBrowserKey(sender, "Control+Shift+P")
+
+    expect(sender.calls.map((call) => [call.params.type, call.params.key, call.params.modifiers])).toEqual([
+      ["rawKeyDown", "Control", 2],
+      ["rawKeyDown", "Shift", 10],
+      ["rawKeyDown", "P", 10],
+      ["keyUp", "P", 10],
+      ["keyUp", "Shift", 2],
+      ["keyUp", "Control", 0],
+    ])
+  })
+
+  test("dispatches Enter as a native text-producing key event", async () => {
+    const sender = recorder()
+
+    await dispatchBrowserKey(sender, "Enter")
+
+    expect(sender.calls.map((call) => [call.params.type, call.params.key, call.params.text])).toEqual([
+      ["keyDown", "Enter", "\r"],
+      ["keyUp", "Enter", undefined],
+    ])
+  })
+
+  test("focuses the deepest DOM node at a page coordinate", async () => {
+    const sender = recorder({ backendNodeId: 42 })
+
+    await focusBrowserPoint(sender, { x: 10.2, y: 20.7 })
+
+    expect(sender.calls.map((call) => call.method)).toEqual([
+      "DOM.enable",
+      "DOM.getNodeForLocation",
+      "DOM.focus",
+      "DOM.resolveNode",
+      "Runtime.callFunctionOn",
+      "Runtime.releaseObject",
+    ])
+    expect(sender.calls[1]?.params).toEqual({ x: 10, y: 21, includeUserAgentShadowDOM: false })
+    expect(sender.calls[2]?.params).toEqual({ backendNodeId: 42 })
+    expect(sender.calls[5]?.params).toEqual({ objectId: "object-1" })
+  })
+
+  test("focuses the associated control when the physical hit node is a label", async () => {
+    const sender = recorder({ backendNodeId: 42 }, true)
+
+    await focusBrowserPoint(sender, { x: 10, y: 20 })
+
+    expect(sender.calls.map((call) => call.method)).toEqual([
+      "DOM.enable",
+      "DOM.getNodeForLocation",
+      "DOM.focus",
+      "DOM.resolveNode",
+      "Runtime.callFunctionOn",
+      "Runtime.releaseObject",
+    ])
+    expect(sender.calls.find((call) => call.method === "Runtime.callFunctionOn")?.params.functionDeclaration).toContain("label?.control")
+  })
+})
+
+function recorder(result: unknown = {}, failFocus = false): BrowserCdpCommandSender & { calls: Array<{ method: string; params: Record<string, unknown> }> } {
+  const calls: Array<{ method: string; params: Record<string, unknown> }> = []
+  return {
+    calls,
+    async sendCommand(method, params = {}) {
+      calls.push({ method, params })
+      if (method === "DOM.getNodeForLocation") return result
+      if (method === "DOM.focus" && failFocus) throw new Error("Element is not focusable")
+      if (method === "DOM.resolveNode") return { object: { objectId: "object-1" } }
+      if (method === "Runtime.callFunctionOn") return { result: { value: true } }
+      return {}
+    },
+  }
+}