diff --git a/coworker/agent.py b/coworker/agent.py index d24f5c59..c73ab550 100644 --- a/coworker/agent.py +++ b/coworker/agent.py @@ -100,7 +100,7 @@ def _enabled_connector_tools(secrets: SecretStore) -> tuple[set[str], set[str]]: def _skill_dirs(workspace: Optional[Path]) -> list[Path]: - dirs = [state_dir() / "skills"] + dirs = [Path(__file__).parent / "skills" / "builtin", state_dir() / "skills"] if workspace is not None: dirs.append(workspace / ".coworker" / "skills") return dirs diff --git a/coworker/audit.py b/coworker/audit.py index 349d20b1..68009909 100644 --- a/coworker/audit.py +++ b/coworker/audit.py @@ -128,8 +128,8 @@ def _sanitize_args(tool: str, args: dict[str, Any]) -> dict[str, Any]: lk = str(key).lower() if any(s in lk for s in _SECRET_KEYS): out[key] = "[redacted]" - elif tool == "browser_type" and lk == "text": - out[key] = "[redacted input]" + elif tool == "browser_exec" and lk == "code": + out[key] = "[redacted browser code]" elif any(b == lk or lk.endswith("_" + b) for b in _BODY_KEYS): out[key] = "[redacted body]" else: diff --git a/coworker/config.py b/coworker/config.py index 336d4af8..ff8756d9 100644 --- a/coworker/config.py +++ b/coworker/config.py @@ -39,6 +39,12 @@ class Config: port: int = 8765 # Web search provider: "duckduckgo" (keyless default) | "tavily" | "brave" (need a key). web_search_provider: str = "duckduckgo" + # Where the browser tools run. "local" drives the user's own Chrome; "cloud" routes + # every call to a Browser Use Cloud browser (needs `browser-use auth login`). + # Cloud is off for now + browser_backend: str = "local" + # Daemon name for the cloud browser + browser_cloud_name: str = "coworker" # OpenWorker Cloud (sign-in + managed connectors). Config, never constants: # dev/staging/BYO-VPC deployments point these at their own instances. cloud_base_url: str = "https://api.openworker.com" @@ -67,6 +73,8 @@ class Config: "host", "port", "web_search_provider", + "browser_backend", + "browser_cloud_name", "cloud_base_url", "cloud_auth_domain", "cloud_client_id", @@ -77,7 +85,8 @@ class Config: # These fields change what consequential actions can run without a prompt, so the normal # workspace override pass never applies them. `allowed_commands` is added separately only # for a canonically trusted workspace; `auto_allow` remains user-global only. -_GLOBAL_ONLY_FIELDS = {"allowed_commands", "auto_allow"} + +_GLOBAL_ONLY_FIELDS = {"allowed_commands", "auto_allow", "browser_backend", "browser_cloud_name"} _WORKSPACE_FIELDS = _FIELDS - _GLOBAL_ONLY_FIELDS diff --git a/coworker/connectors/browser_automation.py b/coworker/connectors/browser_automation.py index 357cb005..3fe00621 100644 --- a/coworker/connectors/browser_automation.py +++ b/coworker/connectors/browser_automation.py @@ -1,585 +1,321 @@ -"""Playwright-backed browser automation tools for Cowork. - -The dependency is optional. If Playwright or its browser binaries are not installed, the -tools return a clear setup error instead of breaking engine construction. -""" +"""The browser tool: Browser Use CLI""" from __future__ import annotations +import base64 +import json import re -import tempfile -import threading import time -import base64 -from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Callable, Optional import aisuite as ai -from ..web.guard import check_url +from .browser_cli import available, call, emit, ensure_cloud_browser, run_code +_TAB_MARKER = "\U0001f434 " -def _meta( - name: str, *, approval: bool = False, capabilities: Optional[list[str]] = None -): - return ai.ToolMetadata( - name=name, - category="connector", - risk_level="medium" if approval else "low", - capabilities=capabilities or ["browser"], - requires_approval=approval, - ) +# Screenshots the model can actually see; capped so a screenshot-happy session cannot +# flood the context +_MAX_IMAGES_PER_CALL = 2 +_MAX_IMAGE_BYTES = 4 * 1024 * 1024 +_IMAGE_PATH_RE = re.compile(r"(? dict[str, Any]: - return { - "type": "function", - "function": { - "name": name, - "description": description, - "parameters": { - "type": "object", - "properties": properties, - "required": required, - }, - }, - } - - -def _attach(fn: Callable[..., Any], schema: dict[str, Any], *, approval: bool = True): - from .tool_defs import approval_for_tool - - name = schema["function"]["name"] - # §36: the tool registry's read/write kind wins for registered tools — reads never gate. - approval = approval_for_tool(name, default=approval) - fn.__coworker_schema__ = schema - fn.__aisuite_tool_metadata__ = _meta(name, approval=approval) - fn.__doc__ = schema["function"]["description"] - return fn - - -class _BrowserController: - def __init__(self) -> None: - self._lock = threading.RLock() - self._playwright = None - self._browser = None - self._context = None - self._page = None - self._error: Optional[str] = None - self._executor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="coworker-browser" - ) - self._state: dict[str, Any] = { - "open": False, - "url": "", - "title": "", - "status": "closed", - "last_action": "", - "last_result": "", - "last_error": "", - "screenshot_data_url": "", - "updated_at": None, - "controls": [], - } - - def _touch(self, **changes: Any) -> None: - self._state.update(changes) - self._state["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - - def _refresh_page_state(self) -> None: - if self._page is None: - self._touch(open=False, status="closed", url="", title="", controls=[]) - return - try: - snap = _snapshot(self._page, 2000) - self._touch( - open=True, - status="open", - url=self._page.url, - title=self._page.title(), - controls=snap.get("controls", [])[:30], - ) - except Exception as exc: - self._touch(open=True, status="error", last_error=str(exc)) - - def _setup_error(self, exc: Exception) -> dict[str, str]: - return { - "error": ( - "Interactive browser automation requires Playwright. Install it with " - "`pip install playwright` and `python -m playwright install chromium`." - ), - "details": str(exc), - } - - def page(self): - with self._lock: - if self._error: - return None, {"error": self._error} - if self._page is not None: - return self._page, None - try: - from playwright.sync_api import sync_playwright - - self._playwright = sync_playwright().start() - self._browser = self._playwright.chromium.launch(headless=False) - self._context = self._browser.new_context( - viewport={"width": 1280, "height": 900} - ) - self._page = self._context.new_page() - self._touch( - open=True, status="open", last_action="open browser", last_error="" - ) - return self._page, None - except Exception as exc: - self._touch(open=False, status="error", last_error=str(exc)) - return None, self._setup_error(exc) - - def _submit(self, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]: - return self._executor.submit(fn).result() - - def close(self) -> dict[str, Any]: - return self._submit(self._close_locked) - - def _close_locked(self) -> dict[str, Any]: - with self._lock: - try: - if self._context is not None: - self._context.close() - if self._browser is not None: - self._browser.close() - if self._playwright is not None: - self._playwright.stop() - except Exception as exc: - return {"error": str(exc)} - finally: - self._playwright = None - self._browser = None - self._context = None - self._page = None - self._touch(open=False, status="closed", url="", title="", controls=[]) - return {"ok": True} - - def state(self) -> dict[str, Any]: - return self._submit(self._state_locked) - - def _state_locked(self) -> dict[str, Any]: - with self._lock: - self._refresh_page_state() - return dict(self._state) - - def screenshot(self) -> dict[str, Any]: - return self._submit(self._screenshot_locked) - - def _screenshot_locked(self) -> dict[str, Any]: - with self._lock: - page, err = self.page() - if err: - return err - try: - png = page.screenshot(full_page=False) - data_url = "data:image/png;base64," + base64.b64encode(png).decode( - "ascii" - ) - self._touch( - screenshot_data_url=data_url, - last_action="screenshot", - last_result="ok", - last_error="", - ) - self._refresh_page_state() - return {"ok": True, **dict(self._state)} - except Exception as exc: - self._touch( - last_action="screenshot", last_result="error", last_error=str(exc) - ) - return {"error": str(exc)} - - def call(self, action: str, fn: Callable[[Any], dict[str, Any]]) -> dict[str, Any]: - def run() -> dict[str, Any]: - with self._lock: - page, err = self.page() - if err: - return err - self._touch(last_action=action, last_result="running", last_error="") - try: - out = fn(page) - except Exception as exc: - out = {"error": str(exc)} - if "error" in out: - self._touch( - last_action=action, - last_result="error", - last_error=str(out["error"]), - ) - else: - self._refresh_page_state() - self._touch(last_action=action, last_result="ok", last_error="") - return out - - return self._submit(run) - - -_BROWSER = _BrowserController() +STEP_LABEL_LIMIT = 80 -def browser_state() -> dict[str, Any]: - return _BROWSER.state() +def _title(value: Any) -> str: + text = str(value or "") + return text[len(_TAB_MARKER):] if text.startswith(_TAB_MARKER) else text -def browser_take_screenshot() -> dict[str, Any]: - return _BROWSER.screenshot() +def _clip(text: str, limit: int = STEP_LABEL_LIMIT) -> str: + text = " ".join(str(text).split()) + return text if len(text) <= limit else text[:limit] + " …" -def browser_close_session() -> dict[str, Any]: - return _BROWSER.close() - - -def _cap(value: int, default: int = 20000, upper: int = 100000) -> int: - try: - return max(1, min(int(value or default), upper)) - except Exception: - return default - - -def _target_locator(page, target: str): - target = target.strip() - if target.startswith("text="): - return page.get_by_text(target[5:], exact=False).first - if target.startswith("role="): - role_name = target[5:] - role, _, name = role_name.partition(":") - return page.get_by_role(role.strip(), name=name.strip() or None).first - try: - return page.locator(target).first - except Exception: - return page.get_by_text(target, exact=False).first - - -def _safe_call(fn: Callable[[], Any]) -> dict[str, Any]: - try: - return fn() - except Exception as exc: - return {"error": str(exc)} - - -def _browser_call(action: str, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]: - return _BROWSER.call(action, lambda _page: fn()) - - -_SNAPSHOT_JS = """ -() => { - const visible = (el) => { - const style = window.getComputedStyle(el); - const rect = el.getBoundingClientRect(); - return style && style.visibility !== 'hidden' && style.display !== 'none' && rect.width > 0 && rect.height > 0; - }; - const labelFor = (el) => { - if (el.labels && el.labels.length) return Array.from(el.labels).map(l => l.innerText.trim()).filter(Boolean).join(' '); - const id = el.getAttribute('id'); - if (id) { - const label = document.querySelector(`label[for="${CSS.escape(id)}"]`); - if (label) return label.innerText.trim(); - } - return ''; - }; - const describe = (el, i) => ({ - index: i, - tag: el.tagName.toLowerCase(), - type: el.getAttribute('type') || '', - id: el.getAttribute('id') || '', - name: el.getAttribute('name') || '', - role: el.getAttribute('role') || '', - aria: el.getAttribute('aria-label') || '', - label: labelFor(el), - placeholder: el.getAttribute('placeholder') || '', - text: (el.innerText || el.value || '').trim().slice(0, 200), - href: el.getAttribute('href') || '', - selectorHint: el.getAttribute('id') ? `#${CSS.escape(el.getAttribute('id'))}` : (el.getAttribute('name') ? `[name="${el.getAttribute('name')}"]` : '') - }); - const controls = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[contenteditable="true"]')) - .filter(visible) - .slice(0, 120) - .map(describe); - return { - title: document.title, - url: location.href, - text: document.body ? document.body.innerText : '', - controls - }; -} -""" - - -def _snapshot(page, max_chars: int) -> dict[str, Any]: - data = page.evaluate(_SNAPSHOT_JS) - text = re.sub(r"\n{3,}", "\n\n", str(data.get("text") or "")) - cap = _cap(max_chars) - return { - "title": data.get("title"), - "url": data.get("url"), - "text": text[:cap], - "truncated": len(text) > cap, - "controls": data.get("controls") or [], - } - - -def make_browser_automation_tools() -> list[Callable[..., Any]]: - tools: list[Callable[..., Any]] = [] - - def browser_open_url( - url: str, wait_until: str = "domcontentloaded" - ) -> dict[str, Any]: - if not url.lower().startswith(("http://", "https://")): - return {"error": "url must start with http:// or https://"} - # Same address guard as web_fetch. This is approval gated, so it is defense in - # depth, not the primary control. It checks the initial model supplied URL only; - # redirects that the browser follows internally are not hop checked here. - blocked = check_url(url) - if blocked: - return {"error": blocked} - return _BROWSER.call( - "open_url", - lambda page: ( - page.goto(url, wait_until=wait_until, timeout=30000), - {"ok": True, "url": page.url}, - )[1], - ) +def step_label(code: str) -> str: + """The leading `#` comment the model wrote, else its first line of code.""" + for line in code.splitlines(): + stripped = line.strip() + if not stripped: + continue + if stripped.startswith("#"): + label = stripped.lstrip("#").strip() + return _clip(label) if label else "Browser step" + return _clip(stripped) + return "Browser step" - browser_open_url.__name__ = "browser_open_url" - tools.append( - _attach( - browser_open_url, - _schema( - "browser_open_url", - "Open a URL in the local Playwright browser session.", - {"url": {"type": "string"}, "wait_until": {"type": "string"}}, - ["url"], - ), - approval=True, - ) - ) - def browser_snapshot(max_chars: int = 20000) -> dict[str, Any]: - return _BROWSER.call("snapshot", lambda page: _snapshot(page, max_chars)) - - browser_snapshot.__name__ = "browser_snapshot" - tools.append( - _attach( - browser_snapshot, - _schema( - "browser_snapshot", - "Return the current page text plus visible controls and selector hints.", - {"max_chars": {"type": "integer"}}, - [], - ), - approval=True, - ) - ) +def _collect_files(output: str, *, newer_than: float) -> list[dict[str, Any]]: + import mimetypes - def browser_get_text(max_chars: int = 20000) -> dict[str, Any]: - def run(page): - text = re.sub( - r"\n{3,}", "\n\n", page.locator("body").inner_text(timeout=5000) - ) - cap = _cap(max_chars) - return { - "url": page.url, - "title": page.title(), - "text": text[:cap], - "truncated": len(text) > cap, + found: list[dict[str, Any]] = [] + for match in dict.fromkeys(_FILE_RE.findall(output)): + path = Path(match) + try: + stat = path.stat() + except OSError: + continue + if stat.st_mtime < newer_than - 1 or not path.is_file(): + continue + found.append( + { + "path": str(path), + "name": path.name, + "media_type": mimetypes.guess_type(path.name)[0] or "application/octet-stream", + "bytes": stat.st_size, } - - return _BROWSER.call("get_text", run) - - browser_get_text.__name__ = "browser_get_text" - tools.append( - _attach( - browser_get_text, - _schema( - "browser_get_text", - "Read visible text from the current browser page.", - {"max_chars": {"type": "integer"}}, - [], - ), - approval=True, ) - ) + if len(found) == _MAX_FILES: + break + return found - def browser_click(target: str) -> dict[str, Any]: - return _BROWSER.call( - "click", - lambda page: ( - _target_locator(page, target).click(timeout=10000), - {"ok": True, "url": page.url}, - )[1], - ) - browser_click.__name__ = "browser_click" - tools.append( - _attach( - browser_click, - _schema( - "browser_click", - "Click a visible page element by CSS selector, text=label, role=button:Name, or text fallback. Requires approval.", - {"target": {"type": "string"}}, - ["target"], - ), - approval=True, - ) - ) +def _image_sidecar(paths: list[str], *, newer_than: float) -> list[str]: + """Fresh, small-enough images as data URLs for the `_images` sidecar.""" + urls: list[str] = [] + for raw in dict.fromkeys(paths): + path = Path(raw) + try: + stat = path.stat() + except OSError: + continue + if stat.st_mtime < newer_than - 1 or stat.st_size > _MAX_IMAGE_BYTES: + continue + media = _MEDIA_TYPES.get(path.suffix.lower()) + if not media: + continue + urls.append(f"data:{media};base64," + base64.b64encode(path.read_bytes()).decode("ascii")) + if len(urls) == _MAX_IMAGES_PER_CALL: + break + return urls + + +# -- GUI/server state (not tools) ------------------------------------------------------ + +_STATE: dict[str, Any] = { + "open": False, + "url": "", + "title": "", + "status": "closed", + "last_action": "", + "last_result": "", + "last_error": "", + "screenshot_data_url": "", + "updated_at": None, + "controls": [], + # No local window on cloud, so live_url is the only view. + "backend": "local", + "live_url": "", + "session_id": "", +} - def browser_type(target: str, text: str, clear: bool = True) -> dict[str, Any]: - def run(page): - loc = _target_locator(page, target) - if clear: - loc.fill(text, timeout=10000) - else: - loc.type(text, timeout=10000) - return {"ok": True, "url": page.url} - - return _BROWSER.call("type", run) - - browser_type.__name__ = "browser_type" - tools.append( - _attach( - browser_type, - _schema( - "browser_type", - "Fill or type into an input, textarea, or editable element. Requires approval.", - { - "target": {"type": "string"}, - "text": {"type": "string"}, - "clear": {"type": "boolean"}, - }, - ["target", "text"], - ), - approval=True, - ) - ) +_backend_ready = False - def browser_select(target: str, value: str) -> dict[str, Any]: - return _BROWSER.call( - "select", - lambda page: ( - _target_locator(page, target).select_option(value, timeout=10000), - {"ok": True, "url": page.url}, - )[1], - ) - browser_select.__name__ = "browser_select" - tools.append( - _attach( - browser_select, - _schema( - "browser_select", - "Select an option in a dropdown by selector and option value/label. Requires approval.", - {"target": {"type": "string"}, "value": {"type": "string"}}, - ["target", "value"], - ), - approval=True, - ) - ) +def _touch(**changes: Any) -> None: + _STATE.update(changes) + _STATE["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - def browser_upload_file(target: str, path: str) -> dict[str, Any]: - file_path = Path(path).expanduser().resolve() - if not file_path.exists(): - return {"error": f"file not found: {file_path}"} - return _BROWSER.call( - "upload_file", - lambda page: ( - _target_locator(page, target).set_input_files( - str(file_path), timeout=10000 - ), - {"ok": True, "path": str(file_path)}, - )[1], - ) - browser_upload_file.__name__ = "browser_upload_file" - tools.append( - _attach( - browser_upload_file, - _schema( - "browser_upload_file", - "Upload a local file through a file input. Requires approval.", - {"target": {"type": "string"}, "path": {"type": "string"}}, - ["target", "path"], - ), - approval=True, +def _note_install_if_needed() -> None: + """First use on a machine without the CLI: the rail should say what the wait is.""" + if available() is None: + _touch( + status="installing", + last_action="Installing the Browser Use CLI (one-time, ~30s)", + last_result="running", ) - ) - def browser_wait(milliseconds: int = 1000, target: str = "") -> dict[str, Any]: - def run(page): - if target: - _target_locator(page, target).wait_for( - timeout=max(1, int(milliseconds or 1000)) - ) - else: - page.wait_for_timeout(max(1, min(int(milliseconds or 1000), 30000))) - return {"ok": True, "url": page.url} - - return _BROWSER.call("wait", run) - - browser_wait.__name__ = "browser_wait" - tools.append( - _attach( - browser_wait, - _schema( - "browser_wait", - "Wait for a duration or for a target element to appear.", - {"milliseconds": {"type": "integer"}, "target": {"type": "string"}}, - [], - ), - approval=True, - ) - ) - def browser_screenshot(path: str = "") -> dict[str, Any]: - def run(page): - out = ( - Path(path).expanduser() - if path - else Path(tempfile.gettempdir()) / "coworker-browser-screenshot.png" - ) - out = out.resolve() - out.parent.mkdir(parents=True, exist_ok=True) - page.screenshot(path=str(out), full_page=True) - return {"ok": True, "path": str(out), "url": page.url} - - return _BROWSER.call("screenshot", run) - - browser_screenshot.__name__ = "browser_screenshot" - tools.append( - _attach( - browser_screenshot, - _schema( - "browser_screenshot", - "Save a full-page screenshot of the current browser page and return the local path.", - {"path": {"type": "string"}}, - [], - ), - approval=True, - ) +def _ensure_backend() -> Optional[str]: + """Start the configured cloud browser once per process; None on success.""" + global _backend_ready + if _backend_ready: + return None + info = ensure_cloud_browser() + _backend_ready = True + if info.get("error"): + return str(info["error"]) + _touch( + backend=info.get("backend", "local"), + live_url=info.get("liveUrl", ""), + session_id=info.get("id", ""), ) + return None - def browser_close() -> dict[str, Any]: - return browser_close_session() - - browser_close.__name__ = "browser_close" - tools.append( - _attach( - browser_close, - _schema( - "browser_close", - "Close the local Playwright browser session.", - {}, - [], - ), - approval=True, - ) + +def browser_state() -> dict[str, Any]: + """Live page state for the GUI. Only once the agent has actually used the browser — + the CLI drives the user's own Chrome, and the GUI polls this on a timer.""" + if not _STATE["last_action"]: + return dict(_STATE) + body = "_p = page_info()\n_out = {'url': _p.get('url', ''), 'title': _p.get('title', '')}\n" + emit("_out") + page = call(body, timeout=45.0) + if "error" in page: + return dict(_STATE) + _touch(open=True, status="open", url=page.get("url", ""), title=_title(page.get("title"))) + return dict(_STATE) + + +def browser_take_screenshot() -> dict[str, Any]: + import tempfile + + out = Path(tempfile.gettempdir()) / "coworker-browser-state.png" + result = call( + "_path = capture_screenshot(path=" + json.dumps(str(out)) + ")\n_out = {'path': _path}\n" + emit("_out"), + timeout=90.0, ) + if "error" in result: + _touch(last_action="screenshot", last_result="error", last_error=str(result["error"])) + return result + png = Path(result["path"]).read_bytes() + _touch( + screenshot_data_url="data:image/png;base64," + base64.b64encode(png).decode("ascii"), + last_action="screenshot", + last_result="ok", + last_error="", + ) + browser_state() + return {"ok": True, **dict(_STATE)} + + +def browser_close_session() -> dict[str, Any]: + """Close the tab. The CLI's daemon (and any cloud browser) outlives us by design.""" + result = call("close_tab()\n_out = {'ok': True}\n" + emit("_out"), timeout=45.0) + _touch(open=False, status="closed", url="", title="", controls=[]) + return result + + +# -- the tool -------------------------------------------------------------------------- + +_EXEC_SCHEMA = { + "type": "function", + "function": { + "name": "browser_exec", + "description": ( + "Run Python code in a real web browser via the Browser Use CLI. Browser helpers " + "are pre-imported: page_info(), new_tab(url), goto_url(url), js(expression), " + "cdp(method, **params), click_at_xy(x, y), fill_input(selector, text), " + "type_text(text), press_key(key), scroll(x, y, dy), wait_for_load(), " + "wait_for_element(selector, timeout), wait_for_network_idle(), " + "capture_screenshot(path, full), list_tabs(), switch_tab(id), close_tab(), " + "upload_file(selector, path), http_get(url). Use print(...) for any data you " + "need back — the tool returns what the code prints, and any screenshot whose " + "path the code prints comes back as an image you can see. Your calls run in " + "one persistent Python session: variables you assign survive to your next " + "call, so batch a whole sub-procedure (navigate, wait, extract) per call and " + "build on earlier results. If a call times out the session restarts (the " + "browser survives); re-derive what you need from the page. Start `code` with " + "a one-line `#` comment describing the step in plain, non-technical language " + "(under 60 characters); it is shown as the step's label while the call runs." + ), + "parameters": { + "type": "object", + "properties": { + "code": {"type": "string"}, + "timeout_seconds": {"type": "integer"}, + }, + "required": ["code"], + }, + }, +} - return tools + +def make_browser_automation_tools(roots: Optional[list[Any]] = None) -> list[Callable[..., Any]]: + def files_dir() -> Optional[Path]: + """This chat's browser-files home: /browser (email_tools convention). + + Living in the session's scratch dir buys the whole lifecycle for free: the + Artifacts rail lists it, and deleting the chat deletes it. + """ + primary = roots[0] if roots else None + if primary is None or not getattr(primary, "writable", False): + return None + directory = Path(getattr(primary, "path")) / "browser" + directory.mkdir(parents=True, exist_ok=True) + return directory + + def persist(files: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Copy files from outside the session's directories into files_dir — /tmp gets + reaped, and the chat's artifacts must live and die with the chat.""" + import shutil as _shutil + + home = files_dir() + if home is None: + return files + bases = [Path(getattr(r, "path")).resolve() for r in (roots or [])] + out: list[dict[str, Any]] = [] + for entry in files: + path = Path(entry["path"]).resolve() + if any(base in path.parents or path == base for base in bases): + out.append(entry) + continue + target = home / path.name + counter = 1 + while target.exists() and target.resolve() != path: + target = home / f"{path.stem}-{counter}{path.suffix}" + counter += 1 + try: + _shutil.copy2(path, target) + except OSError: + out.append(entry) + continue + out.append({**entry, "path": str(target), "name": target.name}) + return out + + def browser_exec(code: str, timeout_seconds: int = 120) -> dict[str, Any]: + started = time.time() + _note_install_if_needed() + backend_error = _ensure_backend() + if backend_error: + return {"error": backend_error} + result = run_code(code, timeout=float(max(1, min(int(timeout_seconds or 120), 600)))) + if "returncode" not in result: + out = dict(result) + else: + out = { + "ok": result.get("returncode") == 0, + "stdout": (result.get("stdout") or "")[:20000], + "stderr": (result.get("stderr") or "")[:4000], + } + if "error" in result: + out["error"] = result["error"] + label = step_label(code) + if out.get("ok"): + _touch(open=True, status="open", last_action=label, last_result="ok", last_error="") + else: + _touch(last_action=label, last_result="error", last_error=str(out.get("error", ""))[:500]) + images = _image_sidecar(_IMAGE_PATH_RE.findall(out.get("stdout") or ""), newer_than=started) + if images: + out["_images"] = images + files = persist(_collect_files(out.get("stdout") or "", newer_than=started)) + if files: + out["files"] = files + display: dict[str, Any] = {"label": _clip(label), "connector": "browser"} + if files: + display["files"] = files + out["_display"] = display + return out + + browser_exec.__name__ = "browser_exec" + browser_exec.__doc__ = _EXEC_SCHEMA["function"]["description"] + browser_exec.__coworker_schema__ = _EXEC_SCHEMA + from .tool_defs import approval_for_tool, kind_for_tool + + browser_exec.__aisuite_tool_metadata__ = ai.ToolMetadata( + name="browser_exec", + category="connector", + # An ungated write (auto_approve): the risk classification stays honest. + risk_level="medium" if kind_for_tool("browser_exec") == "write" else "low", + capabilities=["browser"], + requires_approval=approval_for_tool("browser_exec", default=False), + ) + return [browser_exec] diff --git a/coworker/connectors/browser_cli.py b/coworker/connectors/browser_cli.py new file mode 100644 index 00000000..4efa6003 --- /dev/null +++ b/coworker/connectors/browser_cli.py @@ -0,0 +1,375 @@ +"""Run browser actions through the Browser Use CLI. + +Takes Python on stdin, helpers pre-imported. Its daemon owns the browser, so there's +nothing to hold open here. +""" + +from __future__ import annotations + +import atexit +import json +import os +import shutil +import signal +import subprocess +import threading +import time +from pathlib import Path +from typing import Any, Optional + +# read per call so the pytest guard and runtime overrides both work +def _cli_name() -> str: + return os.environ.get("COWORKER_BROWSER_USE_CLI", "browser-use") +DEFAULT_TIMEOUT_S = 120.0 +# cloud cold-starts a container +PROVISION_TIMEOUT_S = 180.0 +# tag it, the CLI and the page print too +RESULT_PREFIX = "__COWORKER__" + + +class BrowserCLIError(RuntimeError): + pass + + +def _tool_bin_dir() -> Path: + return Path.home() / ".local" / "bin" + + +def available() -> Optional[str]: + """Path to the CLI, or None if it isn't installed.""" + # sidecar often runs without ~/.local/bin on PATH + path = shutil.which(_cli_name()) + if path: + return path + candidate = _tool_bin_dir() / _cli_name() + if candidate.is_file() and os.access(candidate, os.X_OK): + return str(candidate) + return None + + +# bump on purpose, 0.13.3 swapped the CLI's guts +CLI_PIN = "browser-use==0.13.7" + +_INSTALL_LOCK = threading.Lock() +_INSTALL_ATTEMPTED = False + + +def ensure_cli() -> Optional[str]: + """CLI path, installing on first use if missing. One attempt per process.""" + global _INSTALL_ATTEMPTED + path = available() + # overridden name = caller owns the binary + if path is not None or _cli_name() != "browser-use": + return path + with _INSTALL_LOCK: + path = available() + if path is not None or _INSTALL_ATTEMPTED: + return path + _INSTALL_ATTEMPTED = True + for command in (["uv", "tool", "install", CLI_PIN], ["pipx", "install", CLI_PIN]): + installer = shutil.which(command[0]) + if installer is None: + continue + try: + done = subprocess.run( # noqa: S603 - fixed installer commands + [installer, *command[1:]], capture_output=True, text=True, timeout=300 + ) + except Exception: # noqa: BLE001 - fall through to the next installer + continue + if done.returncode == 0: + path = available() + if path is not None: + return path + return None + + +def _setup_error(detail: str) -> dict[str, str]: + return { + "error": ( + "Browser automation requires the Browser Use CLI. OpenWorker tried to install " + "it automatically but could not (that needs `uv` or `pipx` and network). " + "Install it with `uv tool install browser-use` and make sure " + f"`{_cli_name()}` is on PATH." + ), + "details": detail, + } + + +def session_name() -> str: + """The daemon this session talks to; empty means local Chrome. + + Config gates cloud, not env -- a stray BU_NAME shouldn't bill anyone. + """ + from ..config import load_config + + cfg = load_config() + if str(getattr(cfg, "browser_backend", "local") or "local").strip().lower() != "cloud": + return "" + name = (os.environ.get("BU_NAME") or "").strip() + return name or str(getattr(cfg, "browser_cloud_name", "") or "coworker").strip() + + +def _env() -> dict[str, str]: + name = session_name() + return {**os.environ, **({"BU_NAME": name} if name else {})} + + +# the CLI runs one program then exits, so feed it a loop -- keeps vars between calls +# also raises helpers._send off its 5s ipc timeout. no retry, a timed-out cdp call may have landed +_SESSION_BOOT = """import io, json, traceback +from contextlib import redirect_stdout, redirect_stderr +try: + import browser_harness.helpers as _bh_helpers + from browser_harness import _ipc as _bh_ipc + + def _bu_patched_send(req): + c, token = _bh_ipc.connect(_bh_helpers.NAME, timeout=30.0) + try: + r = _bh_ipc.request(c, token, req) + finally: + c.close() + if "error" in r: + raise RuntimeError(r["error"]) + return r + + _bh_helpers._send = _bu_patched_send +except Exception: + pass # tests run plain python3 as the CLI +_bu_ns = dict(globals()) +while True: + with open({inp!r}) as _bu_f: + _bu_code = _bu_f.read() + if not _bu_code.strip(): + break + _bu_buf = io.StringIO() + try: + with redirect_stdout(_bu_buf), redirect_stderr(_bu_buf): + exec(_bu_code, _bu_ns) + _bu_err = 0 + except BaseException: + traceback.print_exc(file=_bu_buf) + _bu_err = 1 + with open({outp!r}, "w") as _bu_f: + _bu_f.write(json.dumps({{"exit": _bu_err, "out": _bu_buf.getvalue()}})) +""" + + +class _Session: + """One persistent CLI interpreter; killed and respawned on timeout.""" + + def __init__(self) -> None: + self._proc: Optional[subprocess.Popen[bytes]] = None + self._dir: Optional[str] = None + self._lock = threading.Lock() + + @property + def alive(self) -> bool: + return self._proc is not None and self._proc.poll() is None + + def _start(self) -> None: + import tempfile + + path = ensure_cli() + if path is None: + raise FileNotFoundError(f"{_cli_name()} not found on PATH and auto-install failed") + self._dir = tempfile.mkdtemp(prefix="coworker-browser-session-") + inp, outp = os.path.join(self._dir, "in"), os.path.join(self._dir, "out") + os.mkfifo(inp) + os.mkfifo(outp) + boot = _SESSION_BOOT.format(inp=inp, outp=outp) + self._proc = subprocess.Popen( # noqa: S603 - fixed executable, code is ours + [path], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + env=_env(), + ) + assert self._proc.stdin is not None + self._proc.stdin.write(boot.encode()) + self._proc.stdin.close() + + def _write(self, fifo: str, text: str, deadline: float) -> None: + while True: + try: + fd = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK) + except OSError: # no reader yet + if time.monotonic() >= deadline or not self.alive: + raise TimeoutError("browser session did not accept the code") + time.sleep(0.05) + continue + with os.fdopen(fd, "w") as handle: + handle.write(text) + return + + def _read(self, fifo: str, deadline: float) -> str: + fd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) + chunks: list[bytes] = [] + try: + while True: + try: + chunk: Optional[bytes] = os.read(fd, 65536) + except BlockingIOError: + chunk = None + if chunk: + chunks.append(chunk) + elif chunk is not None and chunks: + return b"".join(chunks).decode("utf-8", errors="replace") + if time.monotonic() >= deadline: + raise TimeoutError("browser session did not answer in time") + if not chunk: + time.sleep(0.02) + finally: + os.close(fd) + + def run(self, code: str, timeout: float) -> dict[str, Any]: + with self._lock: + if not self.alive: + self.close() + self._start() + assert self._dir is not None + deadline = time.monotonic() + timeout + inp, outp = os.path.join(self._dir, "in"), os.path.join(self._dir, "out") + try: + self._write(inp, code, deadline) + payload = json.loads(self._read(outp, deadline)) + except TimeoutError: + # wedged mid-exec. respawn keeps the browser, loses the namespace + self.close() + return { + "error": ( + f"browser-use timed out after {timeout:.0f}s; the session was " + "restarted (the browser survives, session variables were lost)" + ) + } + out = { + "returncode": int(payload.get("exit", 1)), + "stdout": str(payload.get("out", "")).strip(), + "stderr": "", + } + if out["returncode"] != 0: + out["error"] = out["stdout"] or "browser-use code raised" + return out + + def close(self) -> None: + proc, self._proc = self._proc, None + directory, self._dir = self._dir, None + if proc is not None: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except OSError: + pass + try: + proc.wait(timeout=5) + except Exception: # noqa: BLE001 - teardown only + pass + if directory is not None: + shutil.rmtree(directory, ignore_errors=True) + + +_SESSION = _Session() +atexit.register(_SESSION.close) + + +def run_code(code: str, *, timeout: float = DEFAULT_TIMEOUT_S) -> dict[str, Any]: + """Run `code` in the persistent session; return its output and exit flag.""" + if ensure_cli() is None: + return _setup_error(f"{_cli_name()} not found on PATH and auto-install failed") + if not hasattr(os, "mkfifo"): # windows: same contract, no persistence + return _run_code_oneshot(code, timeout=timeout) + try: + return _SESSION.run(code, timeout) + except FileNotFoundError: + return _setup_error(f"{_cli_name()} not found on PATH") + except Exception as exc: # noqa: BLE001 - surfaced to the agent, never raised + _SESSION.close() + return {"error": f"browser session failed: {exc}"} + + +def _run_code_oneshot(code: str, *, timeout: float) -> dict[str, Any]: + path = ensure_cli() + try: + completed = subprocess.run( # noqa: S603 - fixed executable, code is ours + [path], + input=code, + capture_output=True, + text=True, + timeout=timeout, + env=_env(), + ) + except subprocess.TimeoutExpired: + return {"error": f"browser-use timed out after {timeout:.0f}s"} + except Exception as exc: # noqa: BLE001 - surfaced to the agent, never raised + return _setup_error(str(exc)) + out = { + "returncode": completed.returncode, + "stdout": (completed.stdout or "").strip(), + "stderr": (completed.stderr or "").strip(), + } + if completed.returncode != 0: + out["error"] = out["stderr"] or f"browser-use exited with code {completed.returncode}" + return out + + +def reset_session() -> None: + """Kill the interpreter. For tests, and after BU_NAME changes.""" + _SESSION.close() + + +def call(body: str, *, timeout: float = DEFAULT_TIMEOUT_S) -> dict[str, Any]: + """Run a snippet ending in `emit(...)` and return the result it printed.""" + code = f"import json\n{body}\n" + out = run_code(code, timeout=timeout) + if "error" in out and out.get("returncode") not in (0, None): + return {"error": out["error"], "stderr": out.get("stderr", "")[:2000]} + if "error" in out and "returncode" not in out: + return out + for line in reversed((out.get("stdout") or "").splitlines()): + if line.startswith(RESULT_PREFIX): + try: + return json.loads(line[len(RESULT_PREFIX):]) + except json.JSONDecodeError: + break + return { + "error": "browser-use produced no result", + "stdout": (out.get("stdout") or "")[:2000], + "stderr": (out.get("stderr") or "")[:2000], + } + + +def emit(expression: str) -> str: + """Snippet tail that prints `expression` as the tagged JSON result.""" + return f"print({RESULT_PREFIX!r} + json.dumps({expression}, default=str))" + + +def ensure_cloud_browser() -> dict[str, Any]: + """Point this session's daemon at a cloud browser. + + ensure_daemon() runs first and spawns a local daemon under BU_NAME, so + start_remote_daemon refuses until we stop that one. + """ + name = session_name() + if not name: + return {"ok": True, "backend": "local"} + body = ( + "import json\n" + "from browser_harness.admin import daemon_browser_kind, restart_daemon, start_remote_daemon\n" + "_name = " + json.dumps(name) + "\n" + "if daemon_browser_kind(_name) == 'cloud':\n" + " _out = {'reused': True}\n" + "else:\n" + " restart_daemon(_name)\n" + " _b = start_remote_daemon(_name)\n" + " _out = {'id': (_b or {}).get('id', ''), 'liveUrl': (_b or {}).get('liveUrl', '')}\n" + + emit("_out") + ) + out = run_code(body, timeout=PROVISION_TIMEOUT_S) + if out.get("returncode") != 0: + return {"error": out.get("error") or "could not start the cloud browser", "backend": "cloud"} + for line in reversed((out.get("stdout") or "").splitlines()): + if line.startswith(RESULT_PREFIX): + try: + return {"ok": True, "backend": "cloud", "name": name, **json.loads(line[len(RESULT_PREFIX):])} + except json.JSONDecodeError: + break + return {"ok": True, "backend": "cloud", "name": name} diff --git a/coworker/connectors/descriptors.py b/coworker/connectors/descriptors.py index 9a6a34ab..b9e3658a 100644 --- a/coworker/connectors/descriptors.py +++ b/coworker/connectors/descriptors.py @@ -601,7 +601,10 @@ def _validate_outlook(creds: dict) -> ValidationResult: logo="browser", fields=[], instructions=[ - "No setup required. Browser tools are available to Cowork sessions." + "Uses the Browser Use CLI. Installed automatically on first browser " + "use (via uv or pipx); or install it yourself: `uv tool install browser-use`.", + "Drives your own Chrome. Set BU_NAME (and BROWSER_USE_API_KEY) to use a " + "Browser Use Cloud browser instead — see `browser-use auth login`.", ], available=True, ), diff --git a/coworker/connectors/integration_tools.py b/coworker/connectors/integration_tools.py index f0f18d0b..f6f1fb08 100644 --- a/coworker/connectors/integration_tools.py +++ b/coworker/connectors/integration_tools.py @@ -550,7 +550,8 @@ def make_integration_tools( enabled_tools: Optional[set[str]] = None, roots: Optional[list[Any]] = None, ) -> list[Callable[..., Any]]: - tools: list[Callable[..., Any]] = make_browser_automation_tools() + # Browser files land in the primary scratch + tools: list[Callable[..., Any]] = make_browser_automation_tools(roots=roots) # Email needs the session roots: attachment downloads land in the primary scratch # and outgoing attachments must resolve inside a granted directory. tools.extend(make_email_tools(secrets, roots=roots)) diff --git a/coworker/connectors/tool_defs.py b/coworker/connectors/tool_defs.py index d8b365eb..6ad6518b 100644 --- a/coworker/connectors/tool_defs.py +++ b/coworker/connectors/tool_defs.py @@ -22,6 +22,8 @@ class ConnectorToolDef: # single-argument targets are declarable in v1 (no wildcards, no composite targets), and # only write tools should declare one — reads never gate, so a rule would be meaningless. target_arg: Optional[str] = None + # Skip the interactive approval gate while keeping the write classification + auto_approve: bool = False TOOL_DEFS: tuple[ConnectorToolDef, ...] = ( @@ -34,69 +36,11 @@ class ConnectorToolDef: ), ConnectorToolDef( "browser", - "browser_open_url", - "Open URL", - "read", - "Open a URL in the Playwright browser.", - ), - ConnectorToolDef( - "browser", - "browser_snapshot", - "Snapshot page", - "read", - "Read page text and visible controls.", - ), - ConnectorToolDef( - "browser", - "browser_get_text", - "Read page text", - "read", - "Read visible text from the current browser page.", - ), - ConnectorToolDef( - "browser", - "browser_click", - "Click page", - "write", - "Click a visible browser element.", - ), - ConnectorToolDef( - "browser", - "browser_type", - "Fill field", - "write", - "Type into or fill a browser field.", - ), - ConnectorToolDef( - "browser", - "browser_select", - "Select option", - "write", - "Select a dropdown option.", - ), - ConnectorToolDef( - "browser", - "browser_upload_file", - "Upload file", - "write", - "Upload a local file through a file input.", - ), - ConnectorToolDef( - "browser", "browser_wait", "Wait", "read", "Wait for time or an element." - ), - ConnectorToolDef( - "browser", - "browser_screenshot", - "Screenshot", - "read", - "Capture a browser screenshot.", - ), - ConnectorToolDef( - "browser", - "browser_close", - "Close browser", + "browser_exec", + "Run browser code", "write", - "Close the browser session.", + "Run Python against the browser through the Browser Use CLI.", + auto_approve=True, ), ConnectorToolDef( "github", @@ -1092,6 +1036,7 @@ class ConnectorToolDef: ) _KIND_BY_NAME = {d.name: d.kind for d in TOOL_DEFS} +_AUTO_APPROVE = {d.name for d in TOOL_DEFS if d.auto_approve} # §36: the registry's read/write kind is the SINGLE source of truth for whether a @@ -1100,12 +1045,18 @@ class ConnectorToolDef: # without a registry entry keep their call-site default (MCP/experimental stay # conservative). def approval_for_tool(name: str, default: bool = True) -> bool: + if name in _AUTO_APPROVE: + return False kind = _KIND_BY_NAME.get(name) if kind is None: return default return kind != "read" +def kind_for_tool(name: str, default: str = "") -> str: + return _KIND_BY_NAME.get(name, default) + + TOOL_TO_CONNECTOR = {d.name: d.connector for d in TOOL_DEFS} TOOLS_BY_CONNECTOR: dict[str, list[ConnectorToolDef]] = {} for _def in TOOL_DEFS: @@ -1197,7 +1148,9 @@ def tool_dicts(secrets: SecretStore, connector: str) -> list[dict[str, Any]]: "kind": tool.kind, "description": tool.description, "enabled": bool(overrides.get(tool.name, tool.default_enabled)), - "requires_approval": True, + # The real gate, not a blanket True: reads and auto_approve writes run + # without asking, and the settings UI should say so. + "requires_approval": approval_for_tool(tool.name), } ) return out diff --git a/coworker/connectors/tools.py b/coworker/connectors/tools.py index 8b2f4ce4..de74f671 100644 --- a/coworker/connectors/tools.py +++ b/coworker/connectors/tools.py @@ -247,8 +247,7 @@ def _resolve_within(path: str, bases: list[Path]) -> Optional[Path]: def _render_html_png(path: Path) -> bytes: - """Headless render of a local HTML artifact → viewport PNG (1280×800). Uses the - Playwright chromium we already ship for the browser connector.""" + """Headless render of a local HTML artifact → viewport PNG (1280×800).""" from playwright.sync_api import sync_playwright with sync_playwright() as pw: diff --git a/coworker/engine.py b/coworker/engine.py index ae34d4a0..267254aa 100644 --- a/coworker/engine.py +++ b/coworker/engine.py @@ -785,10 +785,30 @@ def _record_result(self, tool_call: ToolCall, result: Any, status: str) -> Event if isinstance(result, dict) and "_display" in result: display = result.get("_display") or None result = {k: v for k, v in result.items() if k != "_display"} + # `_images` on a tool result = pictures the model should see + images: list[str] = [] + if isinstance(result, dict) and "_images" in result: + images = [u for u in (result.get("_images") or []) if isinstance(u, str)] + result = {k: v for k, v in result.items() if k != "_images"} message = _tool_result_message(tool_call, result) if display: message["_display"] = display self.messages.append(message) + if images: + self.messages.append( + { + "role": "user", + "content": [ + { + "type": "text", + "text": f"[{len(images)} screenshot{'s' if len(images) > 1 else ''} from {tool_call.name}]", + }, + *({"type": "image_url", "image_url": {"url": url}} for url in images), + ], + "ts": time.time(), + "_tool_images": True, + } + ) hidden = int((display or {}).get("hidden_by_filters") or 0) stripped = int((display or {}).get("hidden_fields") or 0) if hidden or stripped: diff --git a/coworker/skills/builtin/browser-use/SKILL.md b/coworker/skills/builtin/browser-use/SKILL.md new file mode 100644 index 00000000..7e7a856c --- /dev/null +++ b/coworker/skills/builtin/browser-use/SKILL.md @@ -0,0 +1,106 @@ +--- +name: browser-use +description: Driving a real browser well — find elements via the accessibility tree, click by coordinates, handle SPAs, iframes, dialogs, downloads, and login walls. Read this before non-trivial browser work. +--- + +# Driving the browser + +Your browser tool is `browser_exec`: it runs Python against a real Chrome through the +Browser Use CLI, with the CLI's helpers already in scope. There are no other browser +tools — every interaction is code through this one. + +Adapted from Browser Use's own skill. The upstream version invokes `browser-use` from a +shell; here the same helpers are reached through `browser_exec`. + +## When not to use the browser + +A plain fetch of public information needs no browser. If `web_fetch` or `web_search` can +read it — a public page, an API, docs — use those. Reach for the browser when the task +needs interaction (click, type, navigate), the user's logged-in session, JS rendering, or +a page that blocks plain fetches. If a fetch returns a shell page, then escalate. + +## Finding and clicking things + +Prefer the accessibility tree over screenshots. Every element's role, name and +`backendDOMNodeId` is in it, and it is far smaller than the DOM: + +```python +# Find the search box on the results page +nodes = cdp("Accessibility.getFullAXTree")["nodes"] +hits = [n for n in nodes if n.get("role", {}).get("value") == "textbox"] +print([n.get("name", {}).get("value") for n in hits][:20]) +``` + +Filter in Python before printing — the full tree is thousands of nodes. + +Node → coordinates → click: + +```python +q = cdp("DOM.getBoxModel", backendNodeId=nid)["model"]["content"] +x, y = sum(q[0::2]) / 4, sum(q[1::2]) / 4 # viewport px +click_at_xy(x, y) +``` + +Negative or oversized coordinates mean the element is off-screen — scroll first. After any +click that navigates, call `wait_for_load()`, then verify with a targeted `js(...)` or +`page_info()` check rather than assuming it worked. + +Fall back to raw HTML through `js(...)` when the AX tree lacks the element (canvas, exotic +widgets). Take a screenshot when layout or imagery is what actually matters. + +## Pages that render late + +`wait_for_load()` misses single-page apps: the document is "complete" before the framework +paints. After route changes and data fetches use `wait_for_element(selector, timeout=10)`, +or `wait_for_network_idle()`. If the current tab is stale or internal, call +`ensure_real_tab()`. + +## Typing + +`fill_input(selector, text)` focuses, clears, types with real key events, then fires the +`input`/`change` events frameworks listen for. Plain `type_text` bypasses those listeners +and can leave a submit button disabled, so prefer `fill_input` on React/Vue/Ember forms. + +## Batching and the persistent session + +`browser_exec` calls run in one persistent Python session: variables you assign survive to +the next call, so parse once, keep the result, and build on it. Batching a whole +sub-procedure — navigate, wait, extract, print — into one call is much faster than one +call per action, and the printed output is what you get back. Print the path of any +screenshot you save and it comes back as an image you can see. If a call times out the +session restarts (the browser survives); re-derive what you need from the page. Start the +code with a one-line `#` comment describing the step; it becomes the label the user sees. + +## When a click does nothing + +Before clicking, check the target is real: `el.disabled`, and +`document.elementFromPoint(x, y)` at your click point — if it returns an overlay instead +of your element, the click will land on the overlay. After clicking, verify something +changed (`page_info()`, a targeted `js(...)` check) instead of assuming. Never retry an +identical click that changed nothing — dismiss the overlay, pick another element, or use +`fill_input`/`press_key` for form controls. + +## Login walls + +Stop and ask. The exception is SSO where Chrome is already signed in — use it. Always stop +for passwords, MFA, consent screens, or an ambiguous account choice. + +## Cloud browsers + +A cloud browser is a fresh, isolated Chrome hosted by Browser Use, with clean managed IPs. +Prefer one when the work is bot-sensitive (scraping, repeated automated visits), when the +user's own browser and IP should stay out of it, or when parallel tasks would otherwise +fight over one local Chrome. + +Cloud browsers are currently disabled: every session runs against the user's local Chrome. +If a task would clearly be better on a clean isolated browser, say so and let the user +enable it (`browser_backend = "cloud"` in config) rather than trying to arrange one. + +Do not start or stop remote daemons by hand inside `browser_exec` — a daemon started that +way is not the one the other tools talk to, so the work would land somewhere invisible. + +## Escape hatches + +Raw CDP is always available with `cdp("Domain.method", **params)`. Downloads, +cross-origin iframes, drag-and-drop and dialogs each have their own mechanics; if you get +stuck on one, say what you tried rather than retrying the same call. diff --git a/pyproject.toml b/pyproject.toml index 687dad70..09fa4907 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,9 @@ dev = ["pytest>=8", "pytest-asyncio", "httpx"] # drives the real handler) — declare it so CI installs it, not just transitively. messaging = ["python-telegram-bot>=21", "slack-bolt>=1.18", "aiohttp>=3.9"] # Interactive Cowork browser automation. -browser = ["playwright>=1.44"] +# the browser connector shells out to the `browser-use` command +# playwright is only for send_file's html -> png render now +render = ["playwright>=1.44"] # AWS Bedrock provider (lazy-imported; desktop builds bundle it, pip users opt in). bedrock = ["boto3>=1.34"] @@ -57,7 +59,7 @@ where = ["."] include = ["coworker*"] [tool.setuptools.package-data] -coworker = ["personas/builtin/*.md"] +coworker = ["personas/builtin/*.md", "skills/builtin/*/SKILL.md"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/surfaces/gui/src/App.tsx b/surfaces/gui/src/App.tsx index 57f1ba8c..b99957cf 100644 --- a/surfaces/gui/src/App.tsx +++ b/surfaces/gui/src/App.tsx @@ -702,6 +702,8 @@ export function App() { d.result_preview || d.reason, d.display?.hidden_by_filters, d.standing_rule, + d.display?.label, + d.display?.files, ), ); // Refresh the right rail when something it shows may have changed: browser state, or a @@ -1722,6 +1724,8 @@ function updateLastTool( preview?: string, hidden?: number, standingRule?: string, + label?: string, + files?: any[], ): Item[] { const copy = [...items]; for (let i = copy.length - 1; i >= 0; i--) { @@ -1733,6 +1737,8 @@ function updateLastTool( preview, ...(hidden ? { hidden } : {}), ...(standingRule ? { standingRule } : {}), + ...(label ? { label } : {}), + ...(files && files.length ? { files } : {}), }; break; } diff --git a/surfaces/gui/src/components/ApprovalCard.tsx b/surfaces/gui/src/components/ApprovalCard.tsx index b3a3ba77..0b8dccf8 100644 --- a/surfaces/gui/src/components/ApprovalCard.tsx +++ b/surfaces/gui/src/components/ApprovalCard.tsx @@ -274,6 +274,7 @@ export function ApprovalCard({ {!FILE_WRITES.has(item.name) && !["run_shell", "send_message", "send_file"].includes(item.name) && !grants.length && + item.name !== "browser_exec" && shortArgs(item.args) &&
{shortArgs(item.args)}
} {reason &&
{reason}
} diff --git a/surfaces/gui/src/components/Transcript.tsx b/surfaces/gui/src/components/Transcript.tsx index 70e3e883..c58d3799 100644 --- a/surfaces/gui/src/components/Transcript.tsx +++ b/surfaces/gui/src/components/Transcript.tsx @@ -177,6 +177,13 @@ function LineText({ line }: { line: HumanLine }) { ); } +function fmtBytes(n: number): string { + if (!Number.isFinite(n)) return ""; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${(n / 1024 / 1024).toFixed(1)} MB`; +} + function StepRow({ tool, approval }: { tool: ToolItem; approval?: ApprovalItem }) { const [raw, setRaw] = useState(false); const running = tool.status === "…"; @@ -187,8 +194,21 @@ function StepRow({ tool, approval }: { tool: ToolItem; approval?: ApprovalItem } {running ? : "●"} - + {approval && approvalChip(approval.resolved)} + {!!tool.files?.length && ( + + {tool.files.map((f, i) => ( + + 📄 {f.name} + + ))} + + )} {!!tool.standingRule && ( = {}; + const labels: Record = {}; + const files: Record = {}; for (const m of messages || []) { if (m.role === "tool" && m.tool_call_id) { results[m.tool_call_id] = typeof m.content === "string" ? m.content : JSON.stringify(m.content); const hidden = Number(m._display?.hidden_by_filters || 0); if (hidden > 0) hiddenCounts[m.tool_call_id] = hidden; + const label = String(m._display?.label || ""); + if (label) labels[m.tool_call_id] = label; + if (Array.isArray(m._display?.files) && m._display.files.length) files[m.tool_call_id] = m._display.files; } } for (const m of messages || []) { if (m.role === "user") { + // Tool-screenshot carrier: the model sees it, but no human sent it — the pictures + // already surface in the browser rail, so the transcript skips it + if (m._tool_images) continue; // Connector message → structured card; the framed `content` stays for the model, but display // renders from the source sidecar. if (m.source?.connector) { @@ -53,6 +61,8 @@ export function itemsFromMessages(messages: ConversationMessage[]): Item[] { } const preview = results[tc.id]; const hidden = hiddenCounts[tc.id]; + const label = labels[tc.id]; + const produced = files[tc.id]; items.push({ kind: "tool", id: tc.id, @@ -61,6 +71,8 @@ export function itemsFromMessages(messages: ConversationMessage[]): Item[] { status: "ok", preview, ...(hidden ? { hidden } : {}), + ...(label ? { label } : {}), + ...(produced ? { files: produced } : {}), }); } } else if (m.role === "notice") { diff --git a/surfaces/gui/src/types.ts b/surfaces/gui/src/types.ts index 28fca293..687998e4 100644 --- a/surfaces/gui/src/types.ts +++ b/surfaces/gui/src/types.ts @@ -105,7 +105,8 @@ export type Item = // `hidden` = results the user's privacy filters removed before the agent saw them // (from the tool message's `_display` sidecar; the agent-visible content has no trace). // `standingRule` = the task-scoped rule that auto-allowed this call ("tool → target"). - | { kind: "tool"; id: string; name: string; args: any; status: string; preview?: string; hidden?: number; standingRule?: string } + // `browser_exec` uses the model's own leading `#` comment) or file name produced +| { kind: "tool"; id: string; name: string; args: any; status: string; preview?: string; hidden?: number; standingRule?: string; label?: string; files?: { name: string; path: string; bytes: number; media_type?: string }[] } | { kind: "approval"; name: string;