Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ mobilecli io button --device <device-id> POWER
# Send text
mobilecli io text --device <device-id> 'hello world'

# Replace the focused field's text in one step ('' clears it)
mobilecli io fill --device <device-id> 'hello world'

# Read device clipboard
mobilecli io clipboard get --device <device-id>

Expand Down
3 changes: 3 additions & 0 deletions agents/android/java/DeviceServer.java
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,9 @@ private static Object dispatchMethod(UiAutomation automation, String method, JSO
return Input.pressKey(automation, JsonRpcSocketServer.requireParam(params, "button"), null);
case "device.io.keys":
return pressKeys(automation, p.optJSONArray("keys"));
case "device.io.fill":
// "" is a valid fill: it clears the field
return Input.fill(automation, p.optString("text", ""));
case "device.io.text":
return Input.typeText(automation, JsonRpcSocketServer.requireParam(params, "text"));
case "device.clipboard.get":
Expand Down
53 changes: 53 additions & 0 deletions agents/android/java/Input.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.mobilenext.mobilecli;

import android.app.UiAutomation;
import android.os.Bundle;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityWindowInfo;
import android.os.SystemClock;
import android.view.InputDevice;
import android.view.KeyCharacterMap;
Expand Down Expand Up @@ -45,6 +48,56 @@ private static JSONArray oneFinger(int x1, int y1, double holdSeconds, int x2, i
return actions;
}

/**
* Replaces the focused text field's text through accessibility (ACTION_SET_TEXT);
* "" clears it. Keys race the field: a Compose TextField moves its cursor to the tap
* a moment after focusing, which collapses a ctrl+a selection, and a backspace then
* deletes one character. Setting the text also takes any unicode without the clipboard.
*/
@SuppressWarnings("deprecation") // recycle() is a no-op on API 33+, still frees pools below
static JSONObject fill(UiAutomation automation, String text) throws Exception {
// findFocus(FOCUS_INPUT) answers with the hosting view for Compose, which can't
// take text, so look for the focused editable node itself
AccessibilityNodeInfo field = null;
for (AccessibilityWindowInfo window : automation.getWindows()) {
AccessibilityNodeInfo root = window.getRoot();
window.recycle();
if (root == null) continue;
field = findFocusedEditable(root);
if (field != root) root.recycle();
if (field != null) break;
}
if (field == null) {
throw new RpcException(RpcException.INTERNAL_ERROR, "no text field has focus");
}
try {
Bundle args = new Bundle();
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
if (!field.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)) {
throw new RpcException(RpcException.INTERNAL_ERROR, "the focused text field can't be filled");
}
} finally {
field.recycle();
}
return ok();
}

// depth-first; the caller owns the node it gets back, every other node is recycled
@SuppressWarnings("deprecation")
private static AccessibilityNodeInfo findFocusedEditable(AccessibilityNodeInfo node) {
if (node.isFocused() && node.isEditable()) {
return node;
}
for (int i = 0; i < node.getChildCount(); i++) {
AccessibilityNodeInfo child = node.getChild(i);
if (child == null) continue;
AccessibilityNodeInfo found = findFocusedEditable(child);
if (found != child) child.recycle();
if (found != null) return found;
}
return null;
}

/** Presses one key with zero or more modifiers held: modifiers down, key down/up, modifiers up. */
static JSONObject pressKey(UiAutomation automation, String keycode, JSONArray modifiers) throws Exception {
int code = resolveKeycode(keycode);
Expand Down
17 changes: 17 additions & 0 deletions cli/io.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,21 @@ Keys name physical keys and are case-insensitive: "cmd+A" is the same as "cmd+a"
},
}

var ioFillCmd = &cobra.Command{
Use: "fill [text]",
Short: "Replace the text of the focused text field on a device",
Long: `Replaces the text of the field that has focus on the specified device; "" clears it. Tap the field first to focus it. Unlike "io text", which types key by key, this sets the text in one step.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
req := commands.FillRequest{
DeviceID: deviceId,
Text: args[0],
}

return runViaDaemon("cli.io.fill", req)
},
}

var ioSwipeCmd = &cobra.Command{
Use: "swipe [x1,y1,x2,y2]",
Short: "Swipe on a device screen from one point to another",
Expand Down Expand Up @@ -232,6 +247,7 @@ func init() {
ioCmd.AddCommand(ioButtonCmd)
ioCmd.AddCommand(ioTextCmd)
ioCmd.AddCommand(ioKeysCmd)
ioCmd.AddCommand(ioFillCmd)
ioCmd.AddCommand(ioSwipeCmd)
ioCmd.AddCommand(ioPinchCmd)

Expand All @@ -243,6 +259,7 @@ func init() {
ioButtonCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to press button on")
ioTextCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to send keys to")
ioKeysCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to press keys on")
ioFillCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to fill the focused text field on")
ioSwipeCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to swipe on")
ioPinchCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to pinch on")
ioPinchCmd.Flags().StringVar(&pinchDirection, "direction", "", "\"in\" to zoom out or \"out\" to zoom in (required)")
Expand Down
24 changes: 24 additions & 0 deletions commands/fill.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package commands

import "fmt"

type FillRequest struct {
DeviceID string `json:"deviceId"`
Text string `json:"text"`
}

// FillCommand replaces the focused text field's text on the specified device; "" clears it.
func FillCommand(req FillRequest) *CommandResponse {
targetDevice, err := FindDeviceWithAgent(req.DeviceID)
if err != nil {
return NewErrorResponse(err)
}

if err := targetDevice.Fill(req.Text); err != nil {
return NewErrorResponse(fmt.Errorf("failed to fill text on device %s: %v", targetDevice.ID(), err))
}

return NewSuccessResponse(MessageResult{
Message: fmt.Sprintf("Filled the focused text field on device %s", targetDevice.ID()),
})
}
1 change: 1 addition & 0 deletions commands/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func CLIRegistry() map[string]CLIHandler {
"cli.io.button": adapt(ButtonCommand),
"cli.io.text": adapt(TextCommand),
"cli.io.keys": adapt(KeysCommand),
"cli.io.fill": adapt(FillCommand),
"cli.io.swipe": adapt(SwipeCommand),
"cli.io.pinch": adapt(PinchCommand),
"cli.io.gesture": adapt(GestureCommand),
Expand Down
10 changes: 10 additions & 0 deletions devices/android.go
Original file line number Diff line number Diff line change
Expand Up @@ -977,6 +977,16 @@ func (d *AndroidDevice) PressKeys(combos []KeyCombo) error {
return nil
}

// Fill sets the focused field's text through accessibility, not keys: a Compose
// TextField moves its cursor to the tap a moment after focusing, which collapses
// a ctrl+a selection.
func (d *AndroidDevice) Fill(text string) error {
if _, err := d.serverRequest("device.io.fill", textParams{Text: text}); err != nil {
return fmt.Errorf("AndroidDevice: failed to fill text: %w", err)
}
return nil
}

// isAscii checks if text contains only ASCII characters
func isAscii(text string) bool {
for _, char := range text {
Expand Down
2 changes: 2 additions & 0 deletions devices/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ type ControllableDevice interface {
GetClipboard() (string, error)
SetClipboard(text string) error
PressKeys(combos []KeyCombo) error
// Fill replaces the focused text field's text; "" clears it
Fill(text string) error
PressButton(key string) error
LaunchApp(bundleID string, opts LaunchOptions) error
TerminateApp(bundleID string) error
Expand Down
11 changes: 11 additions & 0 deletions devices/ios.go
Original file line number Diff line number Diff line change
Expand Up @@ -883,6 +883,17 @@ func (d *IOSDevice) PressKeys(combos []KeyCombo) error {
return d.deviceKitClient.PressKeys(toWdaKeyCombos(combos))
}

// Fill presses the HID clear key, which empties the focused field in one event, then types
func (d *IOSDevice) Fill(text string) error {
if err := d.PressKeys([]KeyCombo{{Key: "clear"}}); err != nil {
return err
}
if text == "" {
return nil
}
return d.SendKeys(text)
}

func (d *IOSDevice) OpenURL(url string) error {
return d.deviceKitClient.OpenURL(url)
}
Expand Down
4 changes: 4 additions & 0 deletions devices/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,10 @@ func (r *RemoteDevice) PressKeys(combos []KeyCombo) error {
return r.fireRPC("device.io.keys", params{"keys": keys})
}

func (r *RemoteDevice) Fill(text string) error {
return r.fireRPC("device.io.fill", params{"text": text})
}

func (r *RemoteDevice) PressButton(key string) error {
return r.fireRPC("device.io.button", params{"button": key})
}
Expand Down
11 changes: 11 additions & 0 deletions devices/simulator.go
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,17 @@ func (s SimulatorDevice) PressKeys(combos []KeyCombo) error {
return s.deviceKitClient.PressKeys(toWdaKeyCombos(combos))
}

// Fill presses the HID clear key, which empties the focused field in one event, then types
func (s SimulatorDevice) Fill(text string) error {
if err := s.PressKeys([]KeyCombo{{Key: "clear"}}); err != nil {
return err
}
if text == "" {
return nil
}
return s.SendKeys(text)
}

func (s SimulatorDevice) Tap(x, y int) error {
return s.deviceKitClient.Tap(x, y)
}
Expand Down
30 changes: 30 additions & 0 deletions docs/openrpc.json
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,36 @@
}
}
},
{
"name": "device.io.fill",
"summary": "Fill text field",
"description": "Replaces the text of the focused text field; an empty string clears it. Unlike device.io.text, which types key by key, this sets the text in one step: on Android through accessibility, on iOS by clearing the field and typing",
"params": [
{
"name": "deviceId",
"description": "ID of the target device",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "text",
"description": "Text to put in the field, or an empty string to clear it",
"required": true,
"schema": {
"type": "string"
}
}
],
"result": {
"name": "success",
"description": "Operation result",
"schema": {
"$ref": "#/components/schemas/SuccessResult"
}
}
},
{
"name": "device.clipboard.get",
"summary": "Read clipboard",
Expand Down
35 changes: 35 additions & 0 deletions docs/openrpc.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ JSON-RPC API for mobile device automation and control
- [device.fs.rm](#devicefsrm)
- [device.info](#deviceinfo)
- [device.io.button](#deviceiobutton)
- [device.io.fill](#deviceiofill)
- [device.io.gesture](#deviceiogesture)
- [device.io.keys](#deviceiokeys)
- [device.io.longpress](#deviceiolongpress)
Expand Down Expand Up @@ -827,6 +828,40 @@ Operation result
```


### device.io.fill

**Fill text field**

Replaces the text of the focused text field; an empty string clears it. Unlike device.io.text, which types key by key, this sets the text in one step: on Android through accessibility, on iOS by clearing the field and typing

#### Parameters

| Name | Type | Required | Description |
|------|------|----------|-------------|
| `deviceId` | `string` | ✓ | ID of the target device |
| `text` | `string` | ✓ | Text to put in the field, or an empty string to clear it |

#### Response

**Type:** [`SuccessResult`](#successresult)

Operation result

#### Example Request

```json
{
"jsonrpc": "2.0",
"method": "device.io.fill",
"params": {
"deviceId": "string",
"text": "string"
},
"id": 1
}
```


### device.io.gesture

**Perform custom gesture**
Expand Down
1 change: 1 addition & 0 deletions server/dispatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ func GetMethodRegistry() map[string]HandlerFunc {
"device.io.longpress": handleIoLongPress,
"device.io.text": handleIoText,
"device.io.keys": handleIoKeys,
"device.io.fill": handleIoFill,
"device.io.button": handleIoButton,
"device.io.swipe": handleIoSwipe,
"device.io.pinch": handleIoPinch,
Expand Down
24 changes: 24 additions & 0 deletions server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,30 @@ func handleIoKeys(params json.RawMessage) (any, error) {
return okResult(response)
}

type IoFillParams struct {
DeviceID string `json:"deviceId"`
Text string `json:"text"`
}

func handleIoFill(params json.RawMessage) (any, error) {
if len(params) == 0 {
return nil, fmt.Errorf("'params' is required with fields: deviceId, text")
}

var ioFillParams IoFillParams
if err := json.Unmarshal(params, &ioFillParams); err != nil {
return nil, fmt.Errorf("invalid parameters: %w. Expected fields: deviceId, text", err)
}

req := commands.FillRequest{
DeviceID: ioFillParams.DeviceID,
Text: ioFillParams.Text,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

response := commands.FillCommand(req)
return okResult(response)
}

type IoButtonParams struct {
DeviceID string `json:"deviceId"`
Button string `json:"button"`
Expand Down
Loading