fix(tv): auto-describe after tv-remote, and on Android TV and Vega

packages/argent-mcp/src/auto-capture.ts

@@ -19,6 +19,7 @@ export const AUTO_SCREENSHOT_TOOLS = new Set([
   "paste",
   "rotate",
   "fold",
+  "tv-remote",
   "launch-app",
   "restart-app",
   "open-url",
@@ -44,6 +45,7 @@ export const AUTO_SCREENSHOT_DELAY_MS_BY_TOOL: Record<string, number> = {
   "gesture-rotate": 1500,
   "run-sequence": 15000,
   "button": 1500,
+  "tv-remote": 1500,
   "rotate": 1000,
   // The fold tool already waits for the panel hand-over; the cap covers the
   // repaint after it.
@@ -80,6 +82,7 @@ export const AUTO_DESCRIBE_TOOLS = new Set([
   "paste",
   "rotate",
   "fold",
+  "tv-remote",
   "launch-app",
   "restart-app",
   "open-url",

packages/argent-mcp/test/auto-capture.test.ts

@@ -173,6 +173,14 @@ describe("autoDescribeEnabled", () => {
   });
 });
 
+describe("auto-capture after tv-remote", () => {
+  it("captures the screen and the focus tree after a TV remote press", () => {
+    expect(shouldAutoScreenshot("mcp__argent__tv-remote")).toBe(true);
+    expect(shouldAutoDescribe("mcp__argent__tv-remote")).toBe(true);
+    expect(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["tv-remote"]).toBe(1500);
+  });
+});
+
 describe("shouldAutoDescribe", () => {
   it("is true for every listed tool, with or without a client prefix", () => {
     for (const tool of AUTO_DESCRIBE_TOOLS) {

packages/skills/skills/argent-tv-interact/SKILL.md

@@ -8,18 +8,18 @@ description: Control and inspect TV apps via argent — Apple TV (tvOS), Android
 ## Critical
 
 - A TV is **focus-driven, not touch-driven.** Drive every interaction with `describe` + `tv-remote` + `keyboard`; never use `gesture-*` / coordinate taps — they don't apply on any TV platform.
-- **Always `describe` before navigating** to find the live cursor and your target — never guess focus from a screenshot. The cursor is the focused element; on **Vega** the toolkit often leaves `focused` false and marks the highlighted item `[selected]`, so treat `[selected]` as the cursor when nothing reports `[focused]`.
+- **Always read the focus view before navigating** (`describe`, or the tree the last action returned) to find the live cursor and your target — never guess focus from a screenshot. The cursor is the focused element; on **Vega** the toolkit often leaves `focused` false and marks the highlighted item `[selected]`, so treat `[selected]` as the cursor when nothing reports `[focused]`.
 - Pass the `udid` from `list-devices` — an Apple TV simulator UDID or an Android TV / Vega `serial`. Dispatch is automatic from the id; the same tools drive all three.
 
 ## The navigation loop
 
 1. `describe` — find the cursor and your target (returns the focused element + all focusable ones, not a tap tree).
-2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the frames to build the path.
-3. `describe` again to confirm. On a miss, repeat.
+2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the list order (and, on Vega, the frames) to build the path.
+3. Read where focus landed from the focus view returned under `--- Elements after action (describe) ---`; call `describe` only when that section is missing. On a miss, repeat.
 
 ## Tools
 
-- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels and normalized frames. Call before and after navigating. Empty tree → see the per-platform notes.
+- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels (plus normalized frames on Vega). Call it before the first press on a screen; `tv-remote`, `keyboard` and `launch-app` return it after the action. Empty tree → see the per-platform notes.
 - `tv-remote {udid, button}` — D-pad / remote. `button` is one key **or a whole path** (run in one call). Keys: `up`/`down`/`left`/`right`, `select`, `back`, `menu`, `home`, `playPause`, plus media keys `rewind`/`fastForward`/`next`/`previous`/`volumeUp`/`volumeDown`/`mute`. Single: `{button:"down"}`; repeat: `{button:"down", repeat:3}`; path: `{button:["up","right","select"]}`.
 - `keyboard {udid, text}` — type into the focused field (focus it with `tv-remote` first). One call carries `text` or `key`, never both — to type and then press a key, send two `keyboard` steps in one `run-sequence`. Named `key` presses (e.g. `{key:"enter"}`) work on Vega; on Apple TV / Android TV move focus with `tv-remote` instead.
 - `launch-app` / `restart-app` / `reinstall-app {udid, bundleId}` — `bundleId` from the app manifest. Vega `reinstall-app` takes `appPath` = a `.vpkg`.

packages/tool-server/src/blueprints/android-tv-control.ts

@@ -8,6 +8,7 @@ import {
 import { adbExecOutBinary, adbShell, getAndroidRuntimeKind } from "../utils/adb";
 import { assertTypeableAndroidText, injectAndroidText } from "../utils/android-input";
 import { UnsupportedOperationError } from "../utils/capability";
+import { withUiautomatorLock } from "../utils/uiautomator-lock";
 import {
   parseUiAutomatorXml,
   attrIsTrue,
@@ -135,6 +136,17 @@ function toTvElement(n: TvNode): TvElement {
   };
 }
 
+/** Focus view of a uiautomator-schema hierarchy (a `uiautomator dump` or the android-devtools helper). */
+export function tvFocusViewFromXml(xml: string): TvDescribeResponse {
+  const { focused, focusable } = collectTvNodes(xml);
+  const pkg = focused?.pkg || focusable.find((n) => n.pkg)?.pkg;
+  return {
+    bundleId: pkg || undefined,
+    focused: focused ? toTvElement(focused) : null,
+    focusable: focusable.map(toTvElement),
+  };
+}
+
 export const androidTvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInfo> = {
   namespace: ANDROID_TV_CONTROL_NAMESPACE,
 
@@ -187,10 +199,12 @@ export const androidTvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInf
       const suffix = `${Date.now().toString(36)}-${Math.floor(Math.random() * 1e9).toString(36)}`;
       const dumpPath = `/data/local/tmp/argent-tv-dump-${suffix}.xml`;
       const raw = (
-        await adbExecOutBinary(
-          serial,
-          `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`,
-          { timeoutMs: 20_000 }
+        await withUiautomatorLock(serial, () =>
+          adbExecOutBinary(
+            serial,
+            `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`,
+            { timeoutMs: 20_000 }
+          )
         )
       ).toString("utf-8");
       if (!raw.includes("<hierarchy")) {
@@ -202,23 +216,13 @@ export const androidTvControlBlueprint: ServiceBlueprint<TvControlApi, DeviceInf
       return raw;
     }
 
-    async function read(): Promise<{ focused: TvNode | null; focusable: TvNode[] }> {
-      return collectTvNodes(await dumpHierarchy());
-    }
-
     async function pressKey(direction: TvDirection): Promise<void> {
       await adbShell(serial, `input keyevent ${KEYEVENTS[direction]}`, { timeoutMs: 10_000 });
     }
 
     const api: TvControlApi = {
       async describe(): Promise<TvDescribeResponse> {
-        const { focused, focusable } = await read();
-        const pkg = focused?.pkg || focusable.find((n) => n.pkg)?.pkg;
-        return {
-          bundleId: pkg || undefined,
-          focused: focused ? toTvElement(focused) : null,
-          focusable: focusable.map(toTvElement),
-        };
+        return tvFocusViewFromXml(await dumpHierarchy());
       },
 
       async navigate(direction: TvDirection): Promise<void> {

packages/tool-server/src/tools/await-screen-idle/index.ts

@@ -19,6 +19,7 @@ import { describeIos, iosRequires } from "../describe/platforms/ios";
 import { describeIosDevice } from "../describe/platforms/ios-device";
 import { describeAndroid, androidRequires } from "../describe/platforms/android";
 import { describeChromium } from "../describe/platforms/chromium";
+import { describeVega, vegaRequires } from "../describe/platforms/vega";
 
 const AWAIT_SCREEN_IDLE_TOOL_ID = "await-screen-idle";
 
@@ -30,7 +31,9 @@ const zodSchema = z.object({
   udid: z
     .string()
     .min(1)
-    .describe("Target device id from `list-devices` (iOS UDID, Android serial, or Chromium id)."),
+    .describe(
+      "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, or Chromium id)."
+    ),
   timeoutMs: z
     .number()
     .int()
@@ -72,6 +75,7 @@ const capability: ToolCapability = {
   appleRemote: { simulator: true },
   android: { emulator: true, device: true, unknown: true },
   chromium: { app: true },
+  vega: { vvd: true },
 };
 
 // Frames are normalized 0..1, so rounding to 0.01 tolerates sub-pixel jitter
@@ -112,6 +116,9 @@ export function createAwaitScreenIdleTool(registry: Registry): ToolDefinition<Pa
     if (device.platform === "android") {
       return describeAndroid(registry, device.id, undefined, androidIsTv);
     }
+    if (device.platform === "vega") {
+      return describeVega(device.id);
+    }
     return describeChromium(services.chromium as ChromiumCdpApi);
   }
 
@@ -148,6 +155,7 @@ still before the timeout. Use after a launch/navigation to wait for the UI to re
       if (device.platform === "ios") await ensureDeps(iosRequires);
       else if (device.platform === "ios-remote") await ensureDeps(["sim-remote"]);
       else if (device.platform === "android") await ensureDeps(androidRequires);
+      else if (device.platform === "vega") await ensureDeps(vegaRequires);
 
       // Resolve tvOS / Android-TV once. Physical devices skip the tvOS probe. They are never tvOS simulators.
       const isTvOs =

packages/tool-server/src/tools/await-ui-element/index.ts

@@ -97,7 +97,9 @@ const zodSchema = z
     udid: z
       .string()
       .min(1)
-      .describe("Target device id from `list-devices` (iOS UDID, Android serial, or Chromium id)."),
+      .describe(
+        "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, or Chromium id)."
+      ),
     condition: z
       .enum(["exists", "visible", "hidden", "text"])
       .describe(

packages/tool-server/src/tools/describe/index.ts

@@ -182,7 +182,7 @@ For React Native apps, debugger-component-tree returns React component names wit
 On a TV target (Apple TV / Android TV — a \`list-devices\` entry with runtimeKind 'tv') this returns
 the focus-driven view instead: the currently FOCUSED element and the list of FOCUSABLE elements,
 since a TV UI has no tap coordinates. Move the highlight with \`tv-remote\` (up/down/left/right/select/
-back/menu/home), then call describe again to confirm where focus landed.`,
+back/menu/home); over MCP its result carries the focus view after the press, elsewhere call describe again.`,
     alwaysLoad: true,
     searchHint:
       "accessibility element tree ui hierarchy tap coordinates ios android chromium vega dom tv tvos apple tv android tv fire tv focus focusable remote dpad",

packages/tool-server/src/tools/describe/platforms/android/index.ts

@@ -2,6 +2,7 @@ import { FAILURE_CODES, FailureError } from "@argent/registry";
 import type { Registry, ToolDependency } from "@argent/registry";
 import type { DescribeNode, DescribeTreeData } from "../../contract";
 import { adbExecOutBinary, isAndroidTv } from "../../../../utils/adb";
+import { withUiautomatorLock } from "../../../../utils/uiautomator-lock";
 import { resolveDevice } from "../../../../utils/device-info";
 import {
   getAndroidScreenSize,
@@ -108,10 +109,12 @@ async function uiautomatorDump(serial: string): Promise<string> {
   // wrappers, RN SVG sub-paths, bounds-less Compose containers) while keeping the
   // text, content-desc, clickable and resource-id the agent contract uses.
   // `;` rather than `&&` before `rm -f` so cleanup fires even when dump/cat fails.
-  const rawBuf = await adbExecOutBinary(
-    serial,
-    `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`,
-    { timeoutMs: 20_000 }
+  const rawBuf = await withUiautomatorLock(serial, () =>
+    adbExecOutBinary(
+      serial,
+      `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`,
+      { timeoutMs: 20_000 }
+    )
   );
   const raw = rawBuf.toString("utf-8");
   const trimmed = raw.trim();

packages/tool-server/src/tools/describe/platforms/tv.ts

@@ -3,6 +3,8 @@ import type { DescribeResult } from "../contract";
 import { formatDescribeTree } from "../format-tree";
 import { resolveTvApi } from "../../tv/tv-service";
 import { describeAndroid } from "./android";
+import { tvFocusViewFromXml } from "../../../blueprints/android-tv-control";
+import { androidDevtoolsRef, type AndroidDevtoolsApi } from "../../../blueprints/android-devtools";
 import type {
   TvControlApi,
   TvDescribeResponse,
@@ -27,7 +29,7 @@ const EMPTY_HINT =
   "exposes focus once its JS bundle has loaded. If it stays empty, take a screenshot to confirm " +
   "what's actually on screen.";
 
-// Android TV reads focus from the OS accessibility tree (uiautomator), which
+// Android TV reads focus from the OS accessibility tree, which
 // does not expose focus driven by react-native-tvos's own focus engine — so the
 // focus view can be empty on a screen that visibly has selectable tiles.
 const ANDROID_FOCUS_EMPTY_HINT =
@@ -37,6 +39,28 @@ const ANDROID_FOCUS_EMPTY_HINT =
   "these screens even though the labels aren't enumerable, so you can drive blind + screenshot " +
   "to confirm.";
 
+// A running android-devtools helper holds the device's only UiAutomation
+// connection, so a `uiautomator dump` beside it dies `Killed`. Read through the
+// helper; the dump is the fallback when the helper read fails.
+async function readAndroidTvFocus(
+  registry: Registry,
+  device: DeviceInfo,
+  api: TvControlApi
+): Promise<TvDescribeResponse> {
+  try {
+    const ref = androidDevtoolsRef(device);
+    const devtools = await registry.resolveService<AndroidDevtoolsApi>(ref.urn, ref.options);
+    return tvFocusViewFromXml((await devtools.getHierarchy()).xml);
+  } catch (err) {
+    console.debug(
+      `[describe.tv] devtools helper failed, falling back to uiautomator dump: ${
+        err instanceof Error ? err.message : String(err)
+      }`
+    );
+    return api.describe();
+  }
+}
+
 function isEmpty(res: TvDescribeResponse): boolean {
   return res.focusable.length === 0 && !res.focused;
 }
@@ -91,9 +115,12 @@ export async function describeTv(registry: Registry, device: DeviceInfo): Promis
 
   // Ride out a brief post-launch transition window (see EMPTY_RETRY_*). Apple TV
   // only: on Android TV an empty focus set is steady state for react-native-tvos
-  // screens, not a transition, so retrying would just burn uiautomator dumps
+  // screens, not a transition, so retrying would just burn reads
   // before the empty-focus fallback below.
-  let res = await api.describe();
+  let res =
+    device.platform === "android"
+      ? await readAndroidTvFocus(registry, device, api)
+      : await api.describe();
   if (device.platform !== "android") {
     for (let attempt = 1; attempt < EMPTY_RETRY_ATTEMPTS && isEmpty(res); attempt++) {
       await sleep(EMPTY_RETRY_DELAY_MS);
@@ -111,7 +138,7 @@ export async function describeTv(registry: Registry, device: DeviceInfo): Promis
   }
 
   // Android TV with a still-empty focus engine: fall back to the full
-  // uiautomator tree so describe stays useful on RN-focus-engine screens.
+  // UI tree so describe stays useful on RN-focus-engine screens.
   if (isEmpty(res) && device.platform === "android") {
     // The dispatcher routed us here via isAndroidTv, so pass isTv through to
     // skip a redundant probe. Let a capture failure propagate: describeAndroid

packages/tool-server/src/tools/tv-remote/index.ts

@@ -88,7 +88,7 @@ export function createTvRemoteTool(registry: Registry): ToolDefinition<Params, T
 A TV is navigated with a directional remote, not touch — use this instead of gesture-tap/swipe (which do not apply on a TV). Move focus with up/down/left/right, confirm with select, go back with back/menu, exit with home, and use playPause/rewind/fastForward/next/previous/volumeUp/volumeDown/mute for the corresponding remote keys. (On the Apple TV simulator the media-transport and volume keys are rejected — its HID stack ignores them; they work on Android TV and Vega.)
 Single press: { button: "down" }. Repeat the same button: { button: "down", repeat: 3 }.
 Multi-step navigation: pass a path as { button: ["up","right","right","select"] } — it runs in one tool call, far cheaper than separate presses.
-Read the screen with \`describe\` before and after to see where focus landed.
+Call \`describe\` before the first press to find the cursor. Over MCP the result carries the focus view after the press; otherwise call \`describe\` again to see where focus landed.
 Returns { pressed, count }.`,
     alwaysLoad: true,
     // A path (≤64 buttons) × repeat (≤50) flattens to thousands of presses, sent

packages/tool-server/src/utils/uiautomator-lock.ts

@@ -0,0 +1,9 @@
+import { withKeyedLock } from "./keyed-lock";
+
+// Keyed by adb serial. Overlapping `uiautomator dump`s on one device get one of
+// them `Killed`, and a timed-out wait tool leaves its dump running.
+const uiautomatorDumpLocks = new Map<string, Promise<unknown>>();
+
+export function withUiautomatorLock<T>(serial: string, fn: () => Promise<T>): Promise<T> {
+  return withKeyedLock(uiautomatorDumpLocks, serial, fn);
+}

packages/tool-server/test/await-screen-idle-vega.test.ts

@@ -0,0 +1,48 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const fetchVegaPageSource = vi.fn();
+vi.mock("../src/utils/vega-inspect", () => ({
+  fetchVegaPageSource: (...a: unknown[]) => fetchVegaPageSource(...a),
+}));
+
+import { createAwaitScreenIdleTool } from "../src/tools/await-screen-idle";
+import { __primeDepCacheForTests, __resetDepCacheForTests } from "../src/utils/check-deps";
+
+const PAGE_SOURCE = readFileSync(join(__dirname, "fixtures", "vega-page-source.xml"), "utf8");
+const VEGA_SERIAL = "amazon-4e311aa3932a35be";
+
+describe("await-screen-idle on Vega", () => {
+  beforeEach(() => {
+    fetchVegaPageSource.mockReset();
+    __resetDepCacheForTests();
+    __primeDepCacheForTests(["adb"]);
+  });
+
+  it("settles by polling the Vega automation toolkit", async () => {
+    fetchVegaPageSource.mockResolvedValue(PAGE_SOURCE);
+    const tool = createAwaitScreenIdleTool({} as any);
+
+    const result = await tool.execute(
+      {},
+      { udid: VEGA_SERIAL, timeoutMs: 2000, pollIntervalMs: 10, minStableMs: 20 }
+    );
+
+    expect(result.settled).toBe(true);
+    expect(result.polls).toBeGreaterThan(1);
+    expect(fetchVegaPageSource).toHaveBeenCalledTimes(result.polls);
+  });
+
+  it("does not settle while the toolkit is unreachable", async () => {
+    fetchVegaPageSource.mockRejectedValue(new Error("ECONNREFUSED"));
+    const tool = createAwaitScreenIdleTool({} as any);
+
+    const result = await tool.execute(
+      {},
+      { udid: VEGA_SERIAL, timeoutMs: 60, pollIntervalMs: 10, minStableMs: 20 }
+    );
+
+    expect(result.settled).toBe(false);
+  });
+});