diff --git a/packages/argent-mcp/src/auto-capture.ts b/packages/argent-mcp/src/auto-capture.ts index a1852db213..6ee9aae31a 100644 --- a/packages/argent-mcp/src/auto-capture.ts +++ b/packages/argent-mcp/src/auto-capture.ts @@ -19,6 +19,7 @@ export const AUTO_SCREENSHOT_TOOLS = new Set([ "paste", "rotate", "fold", + "tv-remote", "launch-app", "restart-app", "open-url", @@ -46,6 +47,7 @@ export const AUTO_SCREENSHOT_DELAY_MS_BY_TOOL: Record = { "gesture-rotate": 1500, "run-sequence": 15000, "button": 1500, + "tv-remote": 1500, "rotate": 1000, // The fold tool already waits for the panel hand-over; the cap covers the // repaint after it. @@ -85,6 +87,7 @@ export const AUTO_DESCRIBE_TOOLS = new Set([ "paste", "rotate", "fold", + "tv-remote", "launch-app", "restart-app", "open-url", diff --git a/packages/argent-mcp/test/auto-capture.test.ts b/packages/argent-mcp/test/auto-capture.test.ts index 7ebdd91db4..2afcccc3c1 100644 --- a/packages/argent-mcp/test/auto-capture.test.ts +++ b/packages/argent-mcp/test/auto-capture.test.ts @@ -181,6 +181,14 @@ describe("autoDescribeEnabled", () => { }); }); +describe("auto-capture after tv-remote", () => { + it("captures the screen and the focus tree after a TV remote press", () => { + expect(shouldAutoScreenshot("mcp__argent__tv-remote")).toBe(true); + expect(shouldAutoDescribe("mcp__argent__tv-remote")).toBe(true); + expect(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["tv-remote"]).toBe(1500); + }); +}); + describe("shouldAutoDescribe", () => { it("is true for every listed tool, with or without a client prefix", () => { for (const tool of AUTO_DESCRIBE_TOOLS) { diff --git a/packages/docs/docs/reference/configuration.mdx b/packages/docs/docs/reference/configuration.mdx index 20b49591b6..d2051d3d39 100644 --- a/packages/docs/docs/reference/configuration.mdx +++ b/packages/docs/docs/reference/configuration.mdx @@ -212,7 +212,7 @@ An environment variable changes the behavior of the process that reads it. Expor | `ARGENT_TOOLS_URL` | unset | URL of the tool-server that the MCP server and the CLI use. Takes precedence over `argent link` | | `ARGENT_AUTH_TOKEN` | unset | The token that the MCP server and the CLI send to the tool-server that `ARGENT_TOOLS_URL` names. `argent server start` prints the token in the `argent link` string. A tool-server that `argent server start --no-auth` starts needs no token only when the shell of that command does not export this variable. Unset the variable before you run `argent server start --no-auth` | | `ARGENT_MCP_LOG` | `~/.argent/mcp-calls.log` | Path of the log of tool calls that the MCP server writes | -| `ARGENT_AUTO_SCREENSHOT_DELAY_MS` | per tool, `1400` otherwise | Minimum milliseconds between an interaction tool and its automatic screenshot. Raises the per-tool default; it cannot lower it. See the `disable-auto-screenshot` flag | +| `ARGENT_AUTO_SCREENSHOT_DELAY_MS` | per tool, `1400` otherwise | Longest wait, in milliseconds, for the screen to settle before an interaction tool's automatic screenshot and element tree. If the tool-server cannot check the screen, the MCP server waits this full time. The variable raises the per-tool default but cannot lower it. See the `disable-auto-screenshot` and `disable-auto-describe` flags | | `ARGENT_TOOL_SERVER_SHUTDOWN_ON_MCP_EXIT` | unset | Set `1` to stop the tool-server when the editor closes the MCP server | | `ARGENT_E2E_FILES_LOG` | unset | Set `1` to make the MCP server and the CLI print one stderr line for each `run:` fragment, nested e2e test, snapshot baseline and `tool:` file argument that they send over a link, before the call. This includes the files of a step that the agent records. The MCP server or the CLI also prints one line for each baseline that it writes after the call. Each line holds the path and the result: `inline` or `upload` with the size in bytes, `listed`, `missing`, `written`, or `refused` with the reason. It never holds the content. `ARGENT_FLOW_FILES_LOG` is the old name and still works | diff --git a/packages/skills/skills/argent-tv-interact/SKILL.md b/packages/skills/skills/argent-tv-interact/SKILL.md index 6644174114..d7439a128b 100644 --- a/packages/skills/skills/argent-tv-interact/SKILL.md +++ b/packages/skills/skills/argent-tv-interact/SKILL.md @@ -8,18 +8,18 @@ description: Control and inspect TV apps via argent, Apple TV (tvOS), Android TV ## Critical - A TV is **focus-driven, not touch-driven.** Drive every interaction with `describe` + `tv-remote` + `keyboard`; never use `gesture-*` / coordinate taps; they don't apply on any TV platform. -- **Always `describe` before navigating** to find the live cursor and your target; never guess focus from a screenshot. The cursor is the focused element; on **Vega** the toolkit often leaves `focused` false and marks the highlighted item `[selected]`, so treat `[selected]` as the cursor when nothing reports `[focused]`. +- **Always read the focus view before navigating** (`describe`, or the tree the last action returned) to find the live cursor and your target; never guess focus from a screenshot. The cursor is the focused element; on **Vega** the toolkit often leaves `focused` false and marks the highlighted item `[selected]`, so treat `[selected]` as the cursor when nothing reports `[focused]`. - Pass the `udid` from `list-devices`: an Apple TV simulator UDID or an Android TV / Vega `serial`. Dispatch is automatic from the id; the same tools drive all three. ## The navigation loop -1. `describe`: find the cursor and your target (returns the focused element + all focusable ones, not a tap tree). -2. `tv-remote`: move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the frames to build the path. -3. `describe` again to confirm. On a miss, repeat. +1. `describe`: find the cursor and your target (see `describe` under Tools for what it returns on each platform). +2. `tv-remote`: move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; on Vega, count rows/columns from the frames to build the path. The Apple TV / Android TV focus view is a flat list with no positions, so read the grid's shape from the screenshot. +3. Read where focus landed from the focus view returned under `--- Elements after action (describe) ---`; call `describe` only when that section is missing. On a miss, repeat. ## Tools -- `describe {udid}`: focus view: the focused / `[selected]` element + focusable elements with labels and normalized frames. Call before and after navigating. Empty tree → see the per-platform notes. +- `describe {udid}`: on Apple TV / Android TV, the focus view: the focused element + focusable elements with labels. On Vega, the full element tree with normalized frames, where `[focused]` / `[selected]` mark the cursor. Call it before the first press on a screen; `tv-remote`, `keyboard`, `launch-app`, `restart-app` and `run-sequence` return it after the action. Empty tree → see the per-platform notes. - `tv-remote {udid, button}`: D-pad / remote. `button` is one key **or a whole path** (run in one call). Keys: `up`/`down`/`left`/`right`, `select`, `back`, `menu`, `home`, `playPause`, plus media keys `rewind`/`fastForward`/`next`/`previous`/`volumeUp`/`volumeDown`/`mute`. Single: `{button:"down"}`; repeat: `{button:"down", repeat:3}`; path: `{button:["up","right","select"]}`. - `keyboard {udid, text}`: type into the focused field (focus it with `tv-remote` first). One call carries `text` or `key`, never both; to type and then press a key, send two `keyboard` steps in one `run-sequence`. Named `key` presses (e.g. `{key:"enter"}`) work on Vega; on Apple TV / Android TV move focus with `tv-remote` instead. - `launch-app` / `restart-app` / `reinstall-app {udid, bundleId}`: `bundleId` from the app manifest. Vega `reinstall-app` takes `appPath` = a `.vpkg`. diff --git a/packages/tool-server/src/blueprints/android-tv-control.ts b/packages/tool-server/src/blueprints/android-tv-control.ts index 32afad46b6..5bfc45ae5a 100644 --- a/packages/tool-server/src/blueprints/android-tv-control.ts +++ b/packages/tool-server/src/blueprints/android-tv-control.ts @@ -9,6 +9,7 @@ import { import { adbExecOutBinary, adbShell, getAndroidRuntimeKind } from "../utils/adb"; import { assertTypeableAndroidText, injectAndroidText } from "../utils/android-input"; import { UnsupportedOperationError } from "../utils/capability"; +import { withUiautomatorLock } from "../utils/uiautomator-lock"; import { parseUiAutomatorXml, attrIsTrue, @@ -237,10 +238,12 @@ export const androidTvControlBlueprint: ServiceBlueprint/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`, - { timeoutMs: 20_000 } + await withUiautomatorLock(serial, () => + adbExecOutBinary( + serial, + `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`, + { timeoutMs: 20_000 } + ) ) ).toString("utf-8"); if (!raw.includes(" { + const pattern = `tvos-ax-service --socket ${socketPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`; + // pkill exits 1 when nothing matched, the common case. + return new Promise((resolve) => execFile("pkill", ["-KILL", "-f", pattern], () => resolve())); +} + // Run the HID injector on the host. It loads SimulatorKit and holds one // SimDeviceLegacyClient open against the UDID for its lifetime. function spawnHidDaemon(udid: string, socketPath: string): ChildProcess { @@ -302,11 +314,11 @@ export const tvControlBlueprint: ServiceBlueprint = { axProc.removeListener("error", onAxError); hidProc.removeListener("exit", onHidExit); hidProc.removeListener("error", onHidError); - // SIGKILL the ax daemon: under `simctl spawn` SIGTERM doesn't reliably - // reach the in-sim process, which would orphan it. The hid daemon is a - // direct host process, so SIGTERM reaps it. + // The ax reader is reaped by `reapAxReaders`. The hid daemon is a direct + // host process, so SIGTERM reaps it. if (!axProc.killed) axProc.kill("SIGKILL"); if (!hidProc.killed) hidProc.kill("SIGTERM"); + await reapAxReaders(axSock); // Drop any socket a daemon already bound, so a stale file can't make the // next factory's accept-probe read as "ready" against a dead socket. for (const p of [axSock, hidSock]) { @@ -334,6 +346,7 @@ export const tvControlBlueprint: ServiceBlueprint = { } catch { /* no stale socket to remove */ } + await reapAxReaders(axSock); axProc = spawnAxDaemon(udid, axSock); axProc.on("exit", onAxExit); axProc.on("error", onAxError); @@ -465,11 +478,11 @@ export const tvControlBlueprint: ServiceBlueprint = { /* respawn failed — nothing extra to kill beyond the current axProc */ } } - // SIGKILL for the same reason as spawnFreshAx: under `simctl spawn` - // SIGTERM doesn't reliably reach the in-sim process, which would leave - // it orphaned. The hid daemon is a direct host process, so SIGTERM does. + // The ax reader is reaped by `reapAxReaders`. The hid daemon is a + // direct host process, so SIGTERM reaps it. if (!axProc.killed) axProc.kill("SIGKILL"); if (!hidProc.killed) hidProc.kill("SIGTERM"); + await reapAxReaders(axSock); for (const p of [axSock, hidSock]) { try { fs.unlinkSync(p); diff --git a/packages/tool-server/src/tools/await-screen-idle/index.ts b/packages/tool-server/src/tools/await-screen-idle/index.ts index f5589c5115..adcfd26cd2 100644 --- a/packages/tool-server/src/tools/await-screen-idle/index.ts +++ b/packages/tool-server/src/tools/await-screen-idle/index.ts @@ -19,6 +19,7 @@ import { describeIos, iosRequires } from "../describe/platforms/ios"; import { describeIosDevice } from "../describe/platforms/ios-device"; import { describeAndroid, androidRequires } from "../describe/platforms/android"; import { describeChromium } from "../describe/platforms/chromium"; +import { describeVega, vegaRequires } from "../describe/platforms/vega"; import { pollHarmonyTree, harmonyRequires } from "../describe/platforms/harmony"; const AWAIT_SCREEN_IDLE_TOOL_ID = "await-screen-idle"; @@ -33,7 +34,7 @@ const zodSchema = z .string() .min(1) .describe( - "Target device id from `list-devices` (iOS UDID, Android serial, HarmonyOS id, or Chromium id)." + "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, HarmonyOS id, or Chromium id)." ), timeoutMs: z .number() @@ -79,18 +80,20 @@ const capability: ToolCapability = { appleRemote: { simulator: true }, android: { emulator: true, device: true, unknown: true }, chromium: { app: true }, + vega: { vvd: true }, harmony: { device: true }, }; // Frames are normalized 0..1, so rounding to 0.01 tolerates sub-pixel jitter -// while still catching real motion (a slide/fade animation). +// while still catching real motion (a slide/fade animation). Focus and +// selection count too: a TV remote press may move only the cursor. function treeSignature(root: DescribeNode): string { const round = (n: number) => Math.round(n * 100) / 100; const parts: string[] = []; const walk = (node: DescribeNode): void => { const f = node.frame; parts.push( - `${node.role}|${node.label ?? ""}|${node.value ?? ""}|${round(f.x)},${round(f.y)},${round(f.width)},${round(f.height)}` + `${node.role}|${node.label ?? ""}|${node.value ?? ""}|${round(f.x)},${round(f.y)},${round(f.width)},${round(f.height)}|${node.focused ? "F" : ""}${node.selected ? "S" : ""}` ); for (const child of node.children) walk(child); }; @@ -121,6 +124,9 @@ export function createAwaitScreenIdleTool(registry: Registry): ToolDefinitionHarmonyOS id, or Chromium id)." + "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, HarmonyOS id, or Chromium id)." ), condition: z .enum(["exists", "visible", "hidden", "text"]) diff --git a/packages/tool-server/src/tools/describe/index.ts b/packages/tool-server/src/tools/describe/index.ts index b545fe8715..e0f8b3cdc7 100644 --- a/packages/tool-server/src/tools/describe/index.ts +++ b/packages/tool-server/src/tools/describe/index.ts @@ -159,7 +159,7 @@ export function createDescribeTool(registry: Registry): ToolDefinitionharmony harmonyos dom tv tvos apple tv android tv fire tv focus focusable remote dpad", diff --git a/packages/tool-server/src/tools/describe/platforms/android/index.ts b/packages/tool-server/src/tools/describe/platforms/android/index.ts index 29293050ad..c7417fdafe 100644 --- a/packages/tool-server/src/tools/describe/platforms/android/index.ts +++ b/packages/tool-server/src/tools/describe/platforms/android/index.ts @@ -2,6 +2,7 @@ import { FAILURE_CODES, FailureError } from "@argent/registry"; import type { Registry, ToolDependency } from "@argent/registry"; import type { DescribeNode, DescribeTreeData } from "../../contract"; import { adbExecOutBinary, isAndroidTv } from "../../../../utils/adb"; +import { withUiautomatorLock } from "../../../../utils/uiautomator-lock"; import { resolveDevice } from "../../../../utils/device-info"; import { getAndroidScreenSize, @@ -108,10 +109,12 @@ async function uiautomatorDump(serial: string): Promise { // wrappers, RN SVG sub-paths, bounds-less Compose containers) while keeping the // text, content-desc, clickable and resource-id the agent contract uses. // `;` rather than `&&` before `rm -f` so cleanup fires even when dump/cat fails. - const rawBuf = await adbExecOutBinary( - serial, - `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`, - { timeoutMs: 20_000 } + const rawBuf = await withUiautomatorLock(serial, () => + adbExecOutBinary( + serial, + `uiautomator dump --compressed ${dumpPath} >/dev/null && cat ${dumpPath}; rm -f ${dumpPath}`, + { timeoutMs: 20_000 } + ) ); const raw = rawBuf.toString("utf-8"); const trimmed = raw.trim(); diff --git a/packages/tool-server/src/tools/describe/platforms/tv.ts b/packages/tool-server/src/tools/describe/platforms/tv.ts index 59b7e24fdb..3d63f964b3 100644 --- a/packages/tool-server/src/tools/describe/platforms/tv.ts +++ b/packages/tool-server/src/tools/describe/platforms/tv.ts @@ -21,13 +21,13 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const EMPTY_RETRY_ATTEMPTS = 3; const EMPTY_RETRY_DELAY_MS = 600; const EMPTY_HINT = - "No focusable elements after retrying and recycling the read path. The app is most likely " + - "still launching (splash / loading screen) or mid-transition — this is normal right after " + - "launch-app / restart-app. Wait ~2-3s and call describe again; a React Native app only " + - "exposes focus once its JS bundle has loaded. If it stays empty, take a screenshot to confirm " + - "what's actually on screen."; + "No focusable elements after retrying and recycling the read path. Either the app is still " + + "loading or mid-transition (after launch-app / restart-app, or a press that opened an app), " + + "or the screen has nothing focusable (sleep, screensaver, video playback). Wait ~2-3s and " + + "call describe again; a React Native app only exposes focus once its JS bundle has loaded. " + + "If it stays empty, take a screenshot to confirm what's actually on screen."; -// Android TV reads focus from the OS accessibility tree (uiautomator), which +// Android TV reads focus from the OS accessibility tree, which // does not expose focus driven by react-native-tvos's own focus engine — so the // focus view can be empty on a screen that visibly has selectable tiles. const ANDROID_FOCUS_EMPTY_HINT = @@ -83,15 +83,14 @@ function renderFocusView(res: TvDescribeResponse): string { /** * `describe` for a TV target (Apple TV simulator or Android TV / leanback * device): the focus-driven view instead of the touch element tree, since a TV - * UI has no tap coordinates — the agent moves the highlight with `tv-remote` - * and re-reads with `describe`. + * UI has no tap coordinates — the agent moves the highlight with `tv-remote`. */ export async function describeTv(registry: Registry, device: DeviceInfo): Promise { const api: TvControlApi = await resolveTvApi(registry, device.id); // Ride out a brief post-launch transition window (see EMPTY_RETRY_*). Apple TV // only: on Android TV an empty focus set is steady state for react-native-tvos - // screens, not a transition, so retrying would just burn uiautomator dumps + // screens, not a transition, so retrying would just burn reads // before the empty-focus fallback below. let res = await api.describe(); if (device.platform !== "android") { @@ -104,14 +103,14 @@ export async function describeTv(registry: Registry, device: DeviceInfo): Promis // Still empty: on Apple TV the daemon may hold a stale primaryApp cache from a // killed app, and a fresh daemon rebinds to the current foreground app. // Skipped on Android TV, where `recycleAx` is a no-op and the re-probe would - // only repeat the dump the retry loop already found empty. + // only repeat the read that just came back empty. if (isEmpty(res) && device.platform !== "android") { await api.recycleAx(); res = await api.describe(); } // Android TV with a still-empty focus engine: fall back to the full - // uiautomator tree so describe stays useful on RN-focus-engine screens. + // UI tree so describe stays useful on RN-focus-engine screens. if (isEmpty(res) && device.platform === "android") { // The dispatcher routed us here via isAndroidTv, so pass isTv through to // skip a redundant probe. Let a capture failure propagate: describeAndroid diff --git a/packages/tool-server/src/tools/tv-remote/index.ts b/packages/tool-server/src/tools/tv-remote/index.ts index bbebb03922..1bbbcb6f02 100644 --- a/packages/tool-server/src/tools/tv-remote/index.ts +++ b/packages/tool-server/src/tools/tv-remote/index.ts @@ -88,7 +88,7 @@ export function createTvRemoteTool(registry: Registry): ToolDefinition>(); + +export function withUiautomatorLock(serial: string, fn: () => Promise): Promise { + return withKeyedLock(uiautomatorDumpLocks, serial, fn); +} diff --git a/packages/tool-server/src/utils/vega-inspect.ts b/packages/tool-server/src/utils/vega-inspect.ts index d7c423d862..0f93232df0 100644 --- a/packages/tool-server/src/utils/vega-inspect.ts +++ b/packages/tool-server/src/utils/vega-inspect.ts @@ -7,6 +7,7 @@ */ import { request } from "node:http"; import { runAdb } from "./adb"; +import { withKeyedLock } from "./keyed-lock"; import { emulatorSerial } from "./vega-automation"; const TOOLKIT_DEVICE_PORT = 8383; @@ -14,6 +15,11 @@ const TOOLKIT_DEVICE_PORT = 8383; // `adb forward` instead of leaking ports. const HOST_PORT_OFFSET = 10_000; +// Keyed by adb serial. Overlapping `getPageSource` requests fail ("socket hang +// up") and can wedge the toolkit until the VVD reboots; a timed-out wait tool +// leaves its read running into the next describe. +const pageSourceLocks = new Map>(); + /** * Raw page-source XML from the on-device automation toolkit; describe parses it * and handles the empty/unavailable case. Throws if the VVD can't be discovered, @@ -21,8 +27,16 @@ const HOST_PORT_OFFSET = 10_000; */ export async function fetchVegaPageSource(timeoutMs = 15_000): Promise { const { serial, consolePort } = await emulatorSerial(); - const hostPort = consolePort + HOST_PORT_OFFSET; + return withKeyedLock(pageSourceLocks, serial, () => + fetchPageSource(serial, consolePort + HOST_PORT_OFFSET, timeoutMs) + ); +} +async function fetchPageSource( + serial: string, + hostPort: number, + timeoutMs: number +): Promise { await runAdb(["-s", serial, "forward", `tcp:${hostPort}`, `tcp:${TOOLKIT_DEVICE_PORT}`], { timeoutMs: 10_000, }); diff --git a/packages/tool-server/test/await-screen-idle-vega.test.ts b/packages/tool-server/test/await-screen-idle-vega.test.ts new file mode 100644 index 0000000000..55cda7c1a4 --- /dev/null +++ b/packages/tool-server/test/await-screen-idle-vega.test.ts @@ -0,0 +1,122 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const fetchVegaPageSource = vi.fn(); +vi.mock("../src/utils/vega-inspect", () => ({ + fetchVegaPageSource: (...a: unknown[]) => fetchVegaPageSource(...a), +})); + +let adbInstalled = true; +vi.mock("../src/utils/android-binary", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveAndroidBinary: async () => (adbInstalled ? "/usr/bin/adb" : null), + }; +}); + +import { createAwaitScreenIdleTool } from "../src/tools/await-screen-idle"; +import { + DependencyMissingError, + __primeDepCacheForTests, + __resetDepCacheForTests, +} from "../src/utils/check-deps"; + +const PAGE_SOURCE = readFileSync(join(__dirname, "fixtures", "vega-page-source.xml"), "utf8"); +const VEGA_SERIAL = "amazon-4e311aa3932a35be"; + +describe("await-screen-idle on Vega", () => { + beforeEach(() => { + fetchVegaPageSource.mockReset(); + adbInstalled = true; + __resetDepCacheForTests(); + __primeDepCacheForTests(["adb"]); + }); + + it("settles by polling the Vega automation toolkit", async () => { + fetchVegaPageSource.mockResolvedValue(PAGE_SOURCE); + const tool = createAwaitScreenIdleTool({} as any); + + const result = await tool.execute( + {}, + { udid: VEGA_SERIAL, timeoutMs: 2000, pollIntervalMs: 10, minStableMs: 20 } + ); + + expect(result.settled).toBe(true); + expect(result.polls).toBeGreaterThan(1); + expect(fetchVegaPageSource).toHaveBeenCalledTimes(result.polls); + }); + + it("does not settle while the page source keeps changing", async () => { + const moved = PAGE_SOURCE.replace("Libraries", "Loading"); + expect(moved).not.toBe(PAGE_SOURCE); + let call = 0; + fetchVegaPageSource.mockImplementation(async () => (call++ % 2 ? moved : PAGE_SOURCE)); + const tool = createAwaitScreenIdleTool({} as any); + + const result = await tool.execute( + {}, + { udid: VEGA_SERIAL, timeoutMs: 150, pollIntervalMs: 10, minStableMs: 20 } + ); + + expect(result.settled).toBe(false); + expect(result.polls).toBeGreaterThan(2); + }); + + it.each([ + [ + "selected", + PAGE_SOURCE.replace( + 'id="163208757260" selectable="true" selected="true"', + 'id="163208757260" selectable="true" selected="false"' + ).replace( + 'id="163208757263" selectable="true" selected="false"', + 'id="163208757263" selectable="true" selected="true"' + ), + ], + [ + "focused", + PAGE_SOURCE.replace( + 'id="163208757263" selectable="true" selected="false" focusable="true" focused="false"', + 'id="163208757263" selectable="true" selected="false" focusable="true" focused="true"' + ), + ], + ])("does not settle while the D-pad cursor is still moving (%s)", async (_attr, moved) => { + expect(moved).not.toBe(PAGE_SOURCE); + let call = 0; + fetchVegaPageSource.mockImplementation(async () => (call++ % 2 ? moved : PAGE_SOURCE)); + const tool = createAwaitScreenIdleTool({} as any); + + const result = await tool.execute( + {}, + { udid: VEGA_SERIAL, timeoutMs: 150, pollIntervalMs: 10, minStableMs: 20 } + ); + + expect(result.settled).toBe(false); + expect(result.polls).toBeGreaterThan(2); + }); + + it("does not settle while the toolkit is unreachable", async () => { + fetchVegaPageSource.mockRejectedValue(new Error("ECONNREFUSED")); + const tool = createAwaitScreenIdleTool({} as any); + + const result = await tool.execute( + {}, + { udid: VEGA_SERIAL, timeoutMs: 60, pollIntervalMs: 10, minStableMs: 20 } + ); + + expect(result.settled).toBe(false); + }); + + it("fails fast with the adb install hint when adb is missing", async () => { + __resetDepCacheForTests(); + adbInstalled = false; + const tool = createAwaitScreenIdleTool({} as any); + + await expect( + tool.execute({}, { udid: VEGA_SERIAL, timeoutMs: 2000, pollIntervalMs: 10, minStableMs: 20 }) + ).rejects.toBeInstanceOf(DependencyMissingError); + expect(fetchVegaPageSource).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/tool-server/test/describe-tv.test.ts b/packages/tool-server/test/describe-tv.test.ts index 12587fa53f..8a7ac2cb1a 100644 --- a/packages/tool-server/test/describe-tv.test.ts +++ b/packages/tool-server/test/describe-tv.test.ts @@ -108,6 +108,8 @@ describe("describe (TV) — empty-state resilience", () => { expect(recycleAx).toHaveBeenCalledTimes(1); expect(res.description).toContain("Focusable: (none reported)"); expect(res.hint).toMatch(/still launching|loading|transition|recycl/i); + // A press can also land on a screen with no focus at all. + expect(res.hint).toMatch(/nothing focusable \(sleep, screensaver, video playback\)/); expect(res.description).toContain("Note:"); }); }); diff --git a/packages/tool-server/test/tv-control-blueprint.test.ts b/packages/tool-server/test/tv-control-blueprint.test.ts index 4063813f34..e722a5ee5c 100644 --- a/packages/tool-server/test/tv-control-blueprint.test.ts +++ b/packages/tool-server/test/tv-control-blueprint.test.ts @@ -19,6 +19,10 @@ const h = vi.hoisted(() => { const execFileCalls: Array<{ cmd: string; args: string[] }> = []; const killedProcs: Array<{ tag: string; signal: string | undefined }> = []; const unlinked: string[] = []; + // The in-sim readers `simctl spawn` starts. Killing the spawn process does not + // kill its reader; only `pkill` does. A reader that idles out unlinks its + // socket path, as the real one does on `--timeout`. + const readers: Array<{ sock: string; alive: boolean }> = []; // When set, the host HID daemon spawn fails to ever bind its socket and exits, // so the factory's waitForSocket(hidSock) rejects and the error/cleanup path // runs. Lets a test exercise the half-up-daemon teardown without a 15s wait. @@ -39,7 +43,7 @@ const h = vi.hoisted(() => { return i >= 0 ? args[i + 1] : undefined; } - return { liveSockets, execFileCalls, killedProcs, unlinked, socketArg, state }; + return { liveSockets, execFileCalls, killedProcs, unlinked, readers, socketArg, state }; }); class FakeProc extends EventEmitter { @@ -59,8 +63,25 @@ vi.mock("node:child_process", async () => { const actual = await vi.importActual("node:child_process"); return { ...actual, - execFile: (cmd: string, args: string[]) => { + execFile: (cmd: string, args: string[], cb?: (err: Error | null) => void) => { h.execFileCalls.push({ cmd, args }); + if (cmd === "pkill") { + // Lands after the call returns, like the real pkill, so a caller that + // doesn't await it would kill the reader it spawns next on that path. + const pattern = new RegExp(args[args.length - 1]!); + setImmediate(() => { + for (const r of h.readers) { + if ( + r.alive && + pattern.test(`/fake/tvos-ax-service --socket ${r.sock} --timeout 3600`) + ) { + r.alive = false; + } + } + cb?.(null); + }); + return new FakeProc(); + } const proc = new FakeProc(); // The HID daemon runs its own host binary; the ax daemon runs via `xcrun // simctl spawn`. Tag by cmd so kill-signal assertions can tell them apart. @@ -82,6 +103,7 @@ vi.mock("node:child_process", async () => { // respawn but the new daemon has not yet rebound it. const sock = h.socketArg(args); if (sock) setTimeout(() => h.liveSockets.add(sock), 0); + if (sock && !isHid) h.readers.push({ sock, alive: true }); return proc; }, }; @@ -177,6 +199,7 @@ beforeEach(() => { h.execFileCalls.length = 0; h.killedProcs.length = 0; h.unlinked.length = 0; + h.readers.length = 0; h.state.failHidSpawn = false; h.state.hidSpawnError = false; h.state.axConnectHook = null; @@ -257,6 +280,49 @@ describe("tvControlBlueprint — ax respawn coalescing", () => { await instance.dispose(); }); + it("reaps the replaced in-sim reader on recycle, so its idle exit can't unlink the new socket", async () => { + const instance = await buildService(); + await instance.api.recycleAx(); + + // A replaced reader still alive idles out and unlinks the shared path. + for (const r of h.readers.slice(0, -1)) { + if (r.alive) h.liveSockets.delete(r.sock); + } + + const res = await instance.api.describe(); + expect(res.bundleId).toBe("com.example.tvapp"); + expect(h.readers.filter((r) => r.alive)).toEqual([h.readers[h.readers.length - 1]]); + // SIGKILL: on SIGTERM the reader runs its own unlink of the path. + const pkills = h.execFileCalls.filter((c) => c.cmd === "pkill"); + expect(pkills.length).toBeGreaterThan(0); + expect(pkills.every((c) => c.args[0] === "-KILL")).toBe(true); + await instance.dispose(); + expect(h.readers.filter((r) => r.alive)).toHaveLength(0); + }); + + it("leaves another tool-server's daemons for the same Apple TV alive and reachable", async () => { + const realPid = process.pid; + const setPid = (value: number) => Object.defineProperty(process, "pid", { value }); + try { + setPid(1001); + const a = await buildService(); + setPid(1002); + const b = await buildService(); + + await a.api.recycleAx(); + await a.dispose(); + + expect(h.readers.filter((r) => r.alive).map((r) => r.sock)).toEqual([ + "/tmp/argent-tv-ax-DDDDDDDD-1002.sock", + ]); + expect((await b.api.describe()).bundleId).toBe("com.example.tvapp"); + await expect(b.api.navigate("down")).resolves.toBeUndefined(); + await b.dispose(); + } finally { + setPid(realPid); + } + }); + it("dispose() kills both daemons and removes their sockets", async () => { const instance = await buildService(); await instance.dispose(); @@ -264,9 +330,8 @@ describe("tvControlBlueprint — ax respawn coalescing", () => { expect(h.killedProcs.length).toBeGreaterThanOrEqual(2); expect(h.unlinked.some((p) => p.includes("ax"))).toBe(true); expect(h.unlinked.some((p) => p.includes("hid"))).toBe(true); - // The in-sim ax daemon (simctl spawn) must be SIGKILLed — SIGTERM doesn't - // propagate through `simctl spawn` and would orphan it; the host hid daemon - // takes SIGTERM. + // `simctl spawn` is SIGKILLed: a forwarded SIGTERM would make the reader + // unlink the socket path. The host hid daemon takes SIGTERM. expect(h.killedProcs.find((p) => p.tag === "ax")?.signal).toBe("SIGKILL"); expect(h.killedProcs.find((p) => p.tag === "hid")?.signal).toBe("SIGTERM"); }); @@ -274,9 +339,8 @@ describe("tvControlBlueprint — ax respawn coalescing", () => { it("SIGKILLs the in-sim ax daemon and clears sockets when the factory fails mid-startup", async () => { // The hid daemon never binds its socket, so waitForSocket(hidSock) rejects // and the factory throws. The ax daemon may already be up — its in-sim - // process must be SIGKILLed (not SIGTERMed, which `simctl spawn` swallows, - // orphaning it) and any bound socket unlinked so the next attempt can't see - // a stale false-ready file. + // reader must be reaped and any bound socket unlinked so the next attempt + // can't see a stale false-ready file. h.state.failHidSpawn = true; await expect(buildService()).rejects.toThrow(); @@ -284,6 +348,7 @@ describe("tvControlBlueprint — ax respawn coalescing", () => { const hid = h.killedProcs.find((p) => p.tag === "hid"); expect(ax?.signal).toBe("SIGKILL"); expect(hid?.signal).toBe("SIGTERM"); + expect(h.readers.filter((r) => r.alive)).toHaveLength(0); // No socket survives the failed factory. expect([...h.liveSockets]).toHaveLength(0); expect(h.unlinked.some((p) => p.includes("ax"))).toBe(true); @@ -319,6 +384,7 @@ describe("tvControlBlueprint — ax respawn coalescing", () => { expect(h.killedProcs.length).toBe(axSpawnCount() + 1); // And the ax socket is gone, not left re-bound by a surviving daemon. expect([...h.liveSockets].some((p) => p.includes("ax"))).toBe(false); + expect(h.readers.filter((r) => r.alive)).toHaveLength(0); }); }); diff --git a/packages/tool-server/test/uiautomator-lock.test.ts b/packages/tool-server/test/uiautomator-lock.test.ts new file mode 100644 index 0000000000..45200e87a1 --- /dev/null +++ b/packages/tool-server/test/uiautomator-lock.test.ts @@ -0,0 +1,66 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Both uiautomator dump sites shell out through adbExecOutBinary; record how +// many run at once per serial. +const inFlight = new Map(); +const peak = new Map(); +let totalInFlight = 0; +let totalPeak = 0; +vi.mock("../src/utils/adb", async () => { + const actual = await vi.importActual("../src/utils/adb"); + return { + ...actual, + adbShell: vi.fn(async () => "Physical size: 1920x1080"), + getAndroidRuntimeKind: vi.fn(async () => "tv" as const), + adbExecOutBinary: vi.fn(async (serial: string) => { + const n = (inFlight.get(serial) ?? 0) + 1; + inFlight.set(serial, n); + peak.set(serial, Math.max(peak.get(serial) ?? 0, n)); + totalPeak = Math.max(totalPeak, ++totalInFlight); + await new Promise((r) => setTimeout(r, 20)); + totalInFlight -= 1; + inFlight.set(serial, (inFlight.get(serial) ?? 1) - 1); + return Buffer.from( + `` + ); + }), + }; +}); + +import { describeAndroid } from "../src/tools/describe/platforms/android"; +import { androidTvControlBlueprint } from "../src/blueprints/android-tv-control"; + +async function tvApi(serial: string) { + const device = { id: serial, platform: "android" as const, kind: "emulator" as const }; + return (await androidTvControlBlueprint.factory({}, device, { device })).api; +} + +describe("uiautomator dumps on one device", () => { + beforeEach(() => { + inFlight.clear(); + peak.clear(); + totalInFlight = 0; + totalPeak = 0; + }); + + it("never overlap across the describe and Android TV dump sites", async () => { + const api = await tvApi("emulator-5554"); + + await Promise.all([ + api.describe(), + describeAndroid(undefined, "emulator-5554", undefined, true), + api.describe(), + describeAndroid(undefined, "emulator-5554", undefined, true), + ]); + + expect(peak.get("emulator-5554")).toBe(1); + }); + + it("still run in parallel on different devices", async () => { + const [a, b] = await Promise.all([tvApi("emulator-5554"), tvApi("emulator-5556")]); + + await Promise.all([a.describe(), b.describe()]); + + expect(totalPeak).toBe(2); + }); +}); diff --git a/packages/tool-server/test/vega-inspect-serialize.test.ts b/packages/tool-server/test/vega-inspect-serialize.test.ts new file mode 100644 index 0000000000..fe2895063f --- /dev/null +++ b/packages/tool-server/test/vega-inspect-serialize.test.ts @@ -0,0 +1,88 @@ +/** + * On a VVD, overlapping toolkit `getPageSource` requests fail with "socket hang + * up". A wait tool that hits its deadline leaves its read running, so the + * describe that follows must queue behind it rather than overlap it. + */ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { once } from "node:events"; + +const runAdb = vi.fn(async (..._a: unknown[]) => ({ stdout: "", stderr: "", code: 0 })); +const emulatorSerial = vi.fn(); +vi.mock("../src/utils/adb", () => ({ runAdb: (...a: unknown[]) => runAdb(...a) })); +vi.mock("../src/utils/vega-automation", () => ({ + emulatorSerial: (...a: unknown[]) => emulatorSerial(...a), +})); + +import { fetchVegaPageSource } from "../src/utils/vega-inspect"; + +// Must match HOST_PORT_OFFSET in vega-inspect.ts. +const HOST_PORT_OFFSET = 10_000; +const PAGE = ''; + +let server: Server | undefined; + +afterEach(async () => { + runAdb.mockClear(); + if (server) { + server.close(); + await once(server, "close").catch(() => {}); + server = undefined; + } +}); + +/** Fake toolkit that, like the real one, fails every request that overlaps another. */ +async function singleFlightToolkit(): Promise<{ maxInFlight: () => number }> { + let inFlight = 0; + let max = 0; + server = createServer((req, res) => { + inFlight += 1; + max = Math.max(max, inFlight); + req.resume(); + setTimeout(() => { + const overlapped = max > 1; + inFlight -= 1; + if (overlapped) { + res.destroy(); + return; + } + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ jsonrpc: "2.0", id: 1, result: PAGE })); + }, 30); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const hostPort = (server.address() as AddressInfo).port; + emulatorSerial.mockResolvedValue({ + serial: "emulator-5554", + consolePort: hostPort - HOST_PORT_OFFSET, + }); + return { maxInFlight: () => max }; +} + +describe("fetchVegaPageSource serializes reads of one VVD", () => { + it("never sends a second getPageSource while one is in flight", async () => { + const toolkit = await singleFlightToolkit(); + + const results = await Promise.allSettled([ + fetchVegaPageSource(2000), + fetchVegaPageSource(2000), + fetchVegaPageSource(2000), + ]); + + expect(toolkit.maxInFlight()).toBe(1); + expect(results.map((r) => r.status)).toEqual(["fulfilled", "fulfilled", "fulfilled"]); + }); + + it("removes each read's forward before the next read adds it", async () => { + await singleFlightToolkit(); + + await Promise.allSettled([fetchVegaPageSource(2000), fetchVegaPageSource(2000)]); + + const verbs = runAdb.mock.calls.map((c) => + (c[0] as string[])[3] === "--remove" ? "rm" : "add" + ); + expect(verbs).toEqual(["add", "rm", "add", "rm"]); + }); +});