fix(tv): auto-describe after tv-remote, and on Android TV and Vega

packages/argent-mcp/src/auto-capture.ts

@@ -19,6 +19,7 @@ export const AUTO_SCREENSHOT_TOOLS = new Set([
   "paste",
   "rotate",
   "fold",
+  "tv-remote",
   "launch-app",
   "restart-app",
   "open-url",
@@ -44,6 +45,7 @@ export const AUTO_SCREENSHOT_DELAY_MS_BY_TOOL: Record<string, number> = {
   "gesture-rotate": 1500,
   "run-sequence": 15000,
   "button": 1500,
+  "tv-remote": 1500,
   "rotate": 1000,
   // The fold tool already waits for the panel hand-over; the cap covers the
   // repaint after it.
@@ -80,6 +82,7 @@ export const AUTO_DESCRIBE_TOOLS = new Set([
   "paste",
   "rotate",
   "fold",
+  "tv-remote",
   "launch-app",
   "restart-app",
   "open-url",

packages/argent-mcp/test/auto-capture.test.ts

@@ -173,6 +173,14 @@ describe("autoDescribeEnabled", () => {
   });
 });
 
+describe("auto-capture after tv-remote", () => {
+  it("captures the screen and the focus tree after a TV remote press", () => {
+    expect(shouldAutoScreenshot("mcp__argent__tv-remote")).toBe(true);
+    expect(shouldAutoDescribe("mcp__argent__tv-remote")).toBe(true);
+    expect(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["tv-remote"]).toBe(1500);
+  });
+});
+
 describe("shouldAutoDescribe", () => {
   it("is true for every listed tool, with or without a client prefix", () => {
     for (const tool of AUTO_DESCRIBE_TOOLS) {

packages/docs/docs/reference/configuration.mdx

@@ -206,12 +206,12 @@ An environment variable changes the behavior of the process that reads it. Expor
 
 ### MCP server
 
-| Variable                                  | Default                    | Description                                                                                                                                                            |
-| ----------------------------------------- | -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ARGENT_TOOLS_URL`                        | unset                      | URL of the tool-server that the MCP server and the CLI use. Takes precedence over `argent link`                                                                        |
-| `ARGENT_MCP_LOG`                          | `~/.argent/mcp-calls.log`  | Path of the log of tool calls that the MCP server writes                                                                                                               |
-| `ARGENT_AUTO_SCREENSHOT_DELAY_MS`         | per tool, `1400` otherwise | Minimum milliseconds between an interaction tool and its automatic screenshot. Raises the per-tool default; it cannot lower it. See the `disable-auto-screenshot` flag |
-| `ARGENT_TOOL_SERVER_SHUTDOWN_ON_MCP_EXIT` | unset                      | Set `1` to stop the tool-server when the editor closes the MCP server                                                                                                  |
+| Variable                                  | Default                    | Description                                                                                                                                                                                                     |
+| ----------------------------------------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ARGENT_TOOLS_URL`                        | unset                      | URL of the tool-server that the MCP server and the CLI use. Takes precedence over `argent link`                                                                                                                 |
+| `ARGENT_MCP_LOG`                          | `~/.argent/mcp-calls.log`  | Path of the log of tool calls that the MCP server writes                                                                                                                                                        |
+| `ARGENT_AUTO_SCREENSHOT_DELAY_MS`         | per tool, `1400` otherwise | Longest wait, in milliseconds, for the screen to settle before an interaction tool's automatic screenshot. The variable raises the per-tool default but cannot lower it. See the `disable-auto-screenshot` flag |
+| `ARGENT_TOOL_SERVER_SHUTDOWN_ON_MCP_EXIT` | unset                      | Set `1` to stop the tool-server when the editor closes the MCP server                                                                                                                                           |
 
 ### Telemetry
 

packages/skills/skills/argent-tv-interact/SKILL.md

@@ -8,18 +8,18 @@ description: Control and inspect TV apps via argent — Apple TV (tvOS), Android
 ## Critical
 
 - A TV is **focus-driven, not touch-driven.** Drive every interaction with `describe` + `tv-remote` + `keyboard`; never use `gesture-*` / coordinate taps — they don't apply on any TV platform.
-- **Always `describe` before navigating** to find the live cursor and your target — never guess focus from a screenshot. The cursor is the focused element; on **Vega** the toolkit often leaves `focused` false and marks the highlighted item `[selected]`, so treat `[selected]` as the cursor when nothing reports `[focused]`.
+- **Always read the focus view before navigating** (`describe`, or the tree the last action returned) to find the live cursor and your target — never guess focus from a screenshot. The cursor is the focused element; on **Vega** the toolkit often leaves `focused` false and marks the highlighted item `[selected]`, so treat `[selected]` as the cursor when nothing reports `[focused]`.
 - Pass the `udid` from `list-devices` — an Apple TV simulator UDID or an Android TV / Vega `serial`. Dispatch is automatic from the id; the same tools drive all three.
 
 ## The navigation loop
 
-1. `describe` — find the cursor and your target (returns the focused element + all focusable ones, not a tap tree).
-2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the frames to build the path.
-3. `describe` again to confirm. On a miss, repeat.
+1. `describe` — find the cursor and your target (see `describe` under Tools for what it returns on each platform).
+2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; on Vega, count rows/columns from the frames to build the path. The Apple TV / Android TV focus view is a flat list with no positions, so read the grid's shape from the screenshot.
+3. Read where focus landed from the focus view returned under `--- Elements after action (describe) ---`; call `describe` only when that section is missing. On a miss, repeat.
 
 ## Tools
 
-- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels and normalized frames. Call before and after navigating. Empty tree → see the per-platform notes.
+- `describe {udid}` — on Apple TV / Android TV, the focus view: the focused element + focusable elements with labels. On Vega, the full element tree with normalized frames, where `[focused]` / `[selected]` mark the cursor. Call it before the first press on a screen; `tv-remote`, `keyboard`, `launch-app`, `restart-app` and `run-sequence` return it after the action. Empty tree → see the per-platform notes.
 - `tv-remote {udid, button}` — D-pad / remote. `button` is one key **or a whole path** (run in one call). Keys: `up`/`down`/`left`/`right`, `select`, `back`, `menu`, `home`, `playPause`, plus media keys `rewind`/`fastForward`/`next`/`previous`/`volumeUp`/`volumeDown`/`mute`. Single: `{button:"down"}`; repeat: `{button:"down", repeat:3}`; path: `{button:["up","right","select"]}`.
 - `keyboard {udid, text}` — type into the focused field (focus it with `tv-remote` first). One call carries `text` or `key`, never both — to type and then press a key, send two `keyboard` steps in one `run-sequence`. Named `key` presses (e.g. `{key:"enter"}`) work on Vega; on Apple TV / Android TV move focus with `tv-remote` instead.
 - `launch-app` / `restart-app` / `reinstall-app {udid, bundleId}` — `bundleId` from the app manifest. Vega `reinstall-app` takes `appPath` = a `.vpkg`.

packages/tool-server/src/blueprints/android-tv-control.ts

@@ -8,8 +8,10 @@ import {
 import { adbExecOutBinary, adbShell, getAndroidRuntimeKind } from "../utils/adb";
 import { assertTypeableAndroidText, injectAndroidText } from "../utils/android-input";
 import { UnsupportedOperationError } from "../utils/capability";
+import { withUiautomatorLock } from "../utils/uiautomator-lock";
 import {
   parseUiAutomatorXml,
+  parseUiAutomatorBounds,
   attrIsTrue,
   labelOf,
 } from "../tools/describe/platforms/android/uiautomator-parser";
@@ -60,7 +62,6 @@ const KEYEVENTS: Record<TvDirection, number> = {
 interface TvNode {
   label: string;
   value: string;
-  focused: boolean;
   selected: boolean;
   disabled: boolean;
   isButton: boolean;
@@ -77,13 +78,13 @@ function valueOf(attrs: Record<string, string>): string {
 }
 
 /**
- * Collect the focused node and every focusable node. `parseUiAutomatorDump`
+ * Collect the focused nodes and every focusable node. `parseUiAutomatorDump`
  * drops the `focused` attribute, so the focus walk needs its own pass.
  */
-function collectTvNodes(xml: string): { focused: TvNode | null; focusable: TvNode[] } {
+function collectTvNodes(xml: string): { focused: TvNode[]; focusable: TvNode[] } {
   const root = parseUiAutomatorXml(xml);
   const focusable: TvNode[] = [];
-  let focused: TvNode | null = null;
+  const focused: TvNode[] = [];
   if (!root) return { focused, focusable };
 
   const stack = [root];
@@ -96,6 +97,10 @@ function collectTvNodes(xml: string): { focused: TvNode | null; focusable: TvNod
     const isFocusable = attrIsTrue(attrs, "focusable");
     const isFocused = attrIsTrue(attrs, "focused");
     if (!isFocusable && !isFocused) continue;
+    // A view outside the display gets its far edge clamped to the screen, so
+    // its bounds come back empty or inverted; `uiautomator dump` omits it.
+    const bounds = parseUiAutomatorBounds(attrs.bounds ?? "");
+    if (bounds && (bounds.w === 0 || bounds.h === 0)) continue;
 
     const label = labelOf(attrs);
     // An unlabelled focusable is a layout focus-trap: kept out of `focusable`,
@@ -104,14 +109,13 @@ function collectTvNodes(xml: string): { focused: TvNode | null; focusable: TvNod
     const tvNode: TvNode = {
       label,
       value: valueOf(attrs),
-      focused: isFocused,
       selected: attrIsTrue(attrs, "selected"),
       disabled: attrs.enabled === "false",
       isButton: /Button/.test(className),
       isEditable: /EditText/.test(className),
       pkg: attrs.package ?? "",
     };
-    if (isFocused && !focused) focused = tvNode;
+    if (isFocused) focused.push(tvNode);
     if (isFocusable && label) focusable.push(tvNode);
   }
   return { focused, focusable };
@@ -126,15 +130,42 @@ function traitsOf(n: TvNode): string[] {
   return traits;
 }
 
-function toTvElement(n: TvNode): TvElement {
+function toTvElement(n: TvNode, focused: TvNode | undefined): TvElement {
   return {
     label: n.label || undefined,
     traits: traitsOf(n),
     value: n.value || undefined,
-    isFocused: n.focused,
+    isFocused: n === focused,
   };
 }
 
+/**
+ * Focus view of a uiautomator-schema hierarchy (a `uiautomator dump` or the
+ * android-devtools helper). The helper emits one root per window, topmost
+ * first; with the on-screen keyboard up, both the key under the D-pad cursor
+ * and the app's text field are `focused`. The first is where the D-pad acts;
+ * `imePackage` keeps the keyboard out of `bundleId`.
+ */
+export function tvFocusViewFromXml(xml: string, imePackage?: string): TvDescribeResponse {
+  const { focused, focusable } = collectTvNodes(xml);
+  const top = focused[0];
+  const pkg = [...focused, ...focusable].find((n) => n.pkg && n.pkg !== imePackage)?.pkg;
+  return {
+    bundleId: pkg || top?.pkg || undefined,
+    focused: top ? toTvElement(top, top) : null,
+    focusable: focusable.map((n) => toTvElement(n, top)),
+  };
+}
+
+/** Package of the device's current input method, or undefined when unset. */
+export async function readImePackage(serial: string): Promise<string | undefined> {
+  const out = (
+    await adbShell(serial, "settings get secure default_input_method", { timeoutMs: 5_000 })
+  ).trim();
+  const pkg = out.split("/")[0];
+  return pkg && pkg !== "null" ? pkg : undefined;
+}
+
 export const androidTvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInfo> = {
   namespace: ANDROID_TV_CONTROL_NAMESPACE,
 
@@ -187,10 +218,12 @@ export const androidTvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInf
       const suffix = `${Date.now().toString(36)}-${Math.floor(Math.random() * 1e9).toString(36)}`;
       const dumpPath = `/data/local/tmp/argent-tv-dump-${suffix}.xml`;
       const raw = (
-        await adbExecOutBinary(
-          serial,
-          `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`,
-          { timeoutMs: 20_000 }
+        await withUiautomatorLock(serial, () =>
+          adbExecOutBinary(
+            serial,
+            `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`,
+            { timeoutMs: 20_000 }
+          )
         )
       ).toString("utf-8");
       if (!raw.includes("<hierarchy")) {
@@ -202,23 +235,13 @@ export const androidTvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInf
       return raw;
     }
 
-    async function read(): Promise<{ focused: TvNode | null; focusable: TvNode[] }> {
-      return collectTvNodes(await dumpHierarchy());
-    }
-
     async function pressKey(direction: TvDirection): Promise<void> {
       await adbShell(serial, `input keyevent ${KEYEVENTS[direction]}`, { timeoutMs: 10_000 });
     }
 
     const api: TvControlApi = {
       async describe(): Promise<TvDescribeResponse> {
-        const { focused, focusable } = await read();
-        const pkg = focused?.pkg || focusable.find((n) => n.pkg)?.pkg;
-        return {
-          bundleId: pkg || undefined,
-          focused: focused ? toTvElement(focused) : null,
-          focusable: focusable.map(toTvElement),
-        };
+        return tvFocusViewFromXml(await dumpHierarchy());
       },
 
       async navigate(direction: TvDirection): Promise<void> {

packages/tool-server/src/blueprints/tv-control.ts

@@ -40,12 +40,14 @@ export function tvControlRef(device: DeviceInfo): {
   };
 }
 
+// Per tool-server process, so neither a reap nor a dispose touches another
+// server's daemon, and a daemon left by an earlier process can't unlink ours.
 function axSocketPath(udid: string): string {
-  return `/tmp/argent-tv-ax-${udid.slice(0, 8)}.sock`;
+  return `/tmp/argent-tv-ax-${udid.slice(0, 8)}-${process.pid}.sock`;
 }
 
 function hidSocketPath(udid: string): string {
-  return `/tmp/argent-tv-hid-${udid.slice(0, 8)}.sock`;
+  return `/tmp/argent-tv-hid-${udid.slice(0, 8)}-${process.pid}.sock`;
 }
 
 /**
@@ -129,6 +131,16 @@ function spawnAxDaemon(udid: string, socketPath: string): ChildProcess {
   return proc;
 }
 
+// Kills every in-sim reader bound to `socketPath`. SIGKILLing `simctl spawn`
+// does not reach its reader, and a reader left running unlinks the path when
+// its idle `--timeout` fires (or on SIGTERM), even after a newer reader has
+// bound it. SIGKILL, so the reader never runs that unlink.
+function reapAxReaders(socketPath: string): Promise<void> {
+  const pattern = `tvos-ax-service --socket ${socketPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`;
+  // pkill exits 1 when nothing matched, the common case.
+  return new Promise((resolve) => execFile("pkill", ["-KILL", "-f", pattern], () => resolve()));
+}
+
 // Run the HID injector on the host. It loads SimulatorKit and holds one
 // SimDeviceLegacyClient open against the UDID for its lifetime.
 function spawnHidDaemon(udid: string, socketPath: string): ChildProcess {
@@ -302,11 +314,11 @@ export const tvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInfo> = {
       axProc.removeListener("error", onAxError);
       hidProc.removeListener("exit", onHidExit);
       hidProc.removeListener("error", onHidError);
-      // SIGKILL the ax daemon: under `simctl spawn` SIGTERM doesn't reliably
-      // reach the in-sim process, which would orphan it. The hid daemon is a
-      // direct host process, so SIGTERM reaps it.
+      // The ax reader is reaped by `reapAxReaders`. The hid daemon is a direct
+      // host process, so SIGTERM reaps it.
       if (!axProc.killed) axProc.kill("SIGKILL");
       if (!hidProc.killed) hidProc.kill("SIGTERM");
+      await reapAxReaders(axSock);
       // Drop any socket a daemon already bound, so a stale file can't make the
       // next factory's accept-probe read as "ready" against a dead socket.
       for (const p of [axSock, hidSock]) {
@@ -334,6 +346,7 @@ export const tvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInfo> = {
       } catch {
         /* no stale socket to remove */
       }
+      await reapAxReaders(axSock);
       axProc = spawnAxDaemon(udid, axSock);
       axProc.on("exit", onAxExit);
       axProc.on("error", onAxError);
@@ -465,11 +478,11 @@ export const tvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInfo> = {
             /* respawn failed — nothing extra to kill beyond the current axProc */
           }
         }
-        // SIGKILL for the same reason as spawnFreshAx: under `simctl spawn`
-        // SIGTERM doesn't reliably reach the in-sim process, which would leave
-        // it orphaned. The hid daemon is a direct host process, so SIGTERM does.
+        // The ax reader is reaped by `reapAxReaders`. The hid daemon is a
+        // direct host process, so SIGTERM reaps it.
         if (!axProc.killed) axProc.kill("SIGKILL");
         if (!hidProc.killed) hidProc.kill("SIGTERM");
+        await reapAxReaders(axSock);
         for (const p of [axSock, hidSock]) {
           try {
             fs.unlinkSync(p);

packages/tool-server/src/tools/await-screen-idle/index.ts

@@ -19,6 +19,7 @@ import { describeIos, iosRequires } from "../describe/platforms/ios";
 import { describeIosDevice } from "../describe/platforms/ios-device";
 import { describeAndroid, androidRequires } from "../describe/platforms/android";
 import { describeChromium } from "../describe/platforms/chromium";
+import { describeVega, vegaRequires } from "../describe/platforms/vega";
 
 const AWAIT_SCREEN_IDLE_TOOL_ID = "await-screen-idle";
 
@@ -30,7 +31,9 @@ const zodSchema = z.object({
   udid: z
     .string()
     .min(1)
-    .describe("Target device id from `list-devices` (iOS UDID, Android serial, or Chromium id)."),
+    .describe(
+      "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, or Chromium id)."
+    ),
   timeoutMs: z
     .number()
     .int()
@@ -72,6 +75,7 @@ const capability: ToolCapability = {
   appleRemote: { simulator: true },
   android: { emulator: true, device: true, unknown: true },
   chromium: { app: true },
+  vega: { vvd: true },
 };
 
 // Frames are normalized 0..1, so rounding to 0.01 tolerates sub-pixel jitter
@@ -112,6 +116,9 @@ export function createAwaitScreenIdleTool(registry: Registry): ToolDefinition<Pa
     if (device.platform === "android") {
       return describeAndroid(registry, device.id, undefined, androidIsTv);
     }
+    if (device.platform === "vega") {
+      return describeVega(device.id);
+    }
     return describeChromium(services.chromium as ChromiumCdpApi);
   }
 
@@ -148,6 +155,7 @@ still before the timeout. Use after a launch/navigation to wait for the UI to re
       if (device.platform === "ios") await ensureDeps(iosRequires);
       else if (device.platform === "ios-remote") await ensureDeps(["sim-remote"]);
       else if (device.platform === "android") await ensureDeps(androidRequires);
+      else if (device.platform === "vega") await ensureDeps(vegaRequires);
 
       // Resolve tvOS / Android-TV once. Physical devices skip the tvOS probe. They are never tvOS simulators.
       const isTvOs =