From 1b9346d8abd81ac6906bebafff48b9a10fa7f868 Mon Sep 17 00:00:00 2001 From: Patrick Loeber Date: Thu, 23 Jul 2026 18:58:41 +0200 Subject: [PATCH 1/5] add newer models and arguments --- README.md | 38 ++++++++++++++++---- agent.py | 101 +++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 116 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c6e1b57..571258e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Gemini Android Computer Use Demo -This repository contains a reference implementation for controlling an Android emulator using the **Gemini 3.5 Flash** Computer Use API (`mobile` environment) via the Google GenAI SDK. +This repository contains a reference implementation for controlling an Android emulator using the **Gemini 3.6 Flash** Computer Use API (`mobile` environment) via the Google GenAI SDK. ## Overview The agent operates in a continuous loop: 1. It captures a screenshot of the virtual device using ADB. -2. It sends the screenshot along with the user's task to Gemini 3.5 Flash. -3. The model returns structured tool commands (such as `click`, `type`, `long_press`, `drag_and_drop`, `press_key`, `go_back`, gwait`, `list_apps`, `open_app`, `take_screenshot`). +2. It sends the screenshot along with the user's task to Gemini 3.6 Flash. +3. The model returns structured tool commands (such as `click`, `type`, `long_press`, `drag_and_drop`, `press_key`, `go_back`, `wait`, `list_apps`, `open_app`, `take_screenshot`). 4. The client executing script maps the normalized coordinates (0-999) to the actual physical resolution of the screen and executes the action via ADB. 5. The loop repeats until the task is complete. @@ -52,17 +52,41 @@ source .venv/bin/activate uv pip install -r requirements.txt ``` -Then run the agent: +Then run the agent with a default or custom task: ```bash python agent.py "Find the latest blog post from philipp schmid and summarize it." ``` -Alternatively, you can run the agent with a custom task: +### Command-Line Options + +`agent.py` supports several CLI arguments: + +| Option | Short | Default | Description | +| --- | --- | --- | --- | +| `--model` | `-m` | `gemini-3.6-flash` | Gemini model ID to use | +| `--thinking-level` | `-t` | `medium` | Thinking level configuration (`minimal`, `low`, `medium`, `high`) | +| `task` | N/A | *(default prompt)* | Task description for the agent | + +Examples: ```bash -python agent.py "Open Settings and enable dark mode" -``` +# Run with a custom task +python agent.py "Find the latest blog post from philipp schmid and summarize it." + +# Specify a different model ID +python agent.py --model gemini-3.5-flas-lite "Open Settings and enable dark mode" + +# Adjust thinking level +python agent.py --thinking-level high "Open Clock and set an alarm for 7 AM" + +### Supported Models + +The following Gemini models are supported for computer use: + +* `gemini-3.6-flash` (default) +* `gemini-3.5-flash-lite` +* `gemini-3.5-flash` ## Licensing & Disclaimer diff --git a/agent.py b/agent.py index 6026959..e6188dc 100644 --- a/agent.py +++ b/agent.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import base64 import json import os @@ -176,6 +177,7 @@ def screenshot(self) -> bytes: return result.stdout + SYSTEM_PROMPT = """You are operating an Android phone. * Use the provided tools to complete the task. * Scroll down to inspect the full screen before assuming an element is missing. @@ -185,12 +187,19 @@ def screenshot(self) -> bytes: """ -def run_agent(task: str, device_id: str = None, max_turns: int = 100): +def run_agent( + task: str, + model: str = "gemini-3.6-flash", + thinking_level: str = "medium", + max_turns: int = 100, +): start_emulator() client = genai.Client() - bridge = ADBBridge(device_id) + bridge = ADBBridge() print(f"\nTask: {task}") + print(f"Model: {model}") + print(f"Thinking Level: {thinking_level}") print("-" * 40) screenshot_bytes = bridge.screenshot() @@ -202,20 +211,54 @@ def run_agent(task: str, device_id: str = None, max_turns: int = 100): "mime_type": "image/png", }, ] - + previous_interaction_id = None turn = 0 - + while turn < max_turns: turn += 1 - - interaction = client.interactions.create( - model="gemini-3.5-flash", - system_instruction=SYSTEM_PROMPT, - input=user_input, - tools=[{"type": "computer_use", "environment": "mobile"}], - previous_interaction_id=previous_interaction_id, - ) + print(f"\n==================== TURN {turn} ====================") + print(f"[Turn {turn}] Sending interaction request (previous_id: {previous_interaction_id})") + + try: + interaction = client.interactions.create( + model=model, + system_instruction=SYSTEM_PROMPT, + input=user_input, + tools=[{"type": "computer_use", "environment": "mobile"}], + generation_config={"thinking_config": {"thinking_level": thinking_level}}, + previous_interaction_id=previous_interaction_id, + ) + except Exception as e: + print(f"\n" + "!" * 60) + print(f"[INTERACTION API ERROR] Exception Type: {type(e).__module__}.{type(e).__name__}") + print(f"[Error Message]: {e}") + + # Extract raw response or metadata from SDK exception + for attr in ["http_res_text", "response_data", "message", "body", "args"]: + val = getattr(e, attr, None) + if val: + print(f"\n--- Exception.{attr} ---") + print(val) + + # Inspect underlying cause if wrapped + cause = getattr(e, "__cause__", None) + if cause: + print(f"\n--- Underlying Cause: {type(cause).__name__} ---") + print(f"Cause message: {cause}") + for attr in ["http_res_text", "response_data", "body"]: + val = getattr(cause, attr, None) + if val: + print(f"--- Cause.{attr} ---") + print(val) + print("!" * 60 + "\n") + raise + + print(f"[Turn {turn} Response] interaction_id: {interaction.id}, status: {interaction.status}") + print(f"[Turn {turn} Steps]: {len(interaction.steps)} step(s)") + for idx, step in enumerate(interaction.steps, 1): + step_type = getattr(step, "type", type(step).__name__) + print(f" Step {idx} [{step_type}]: {step}") has_function_calls = any( step.type == "function_call" @@ -281,7 +324,33 @@ def run_agent(task: str, device_id: str = None, max_turns: int = 100): if __name__ == "__main__": - task_desc = "Find the latest blog post from philipp schmid and summarize it." - if len(sys.argv) > 1: - task_desc = " ".join(sys.argv[1:]) - run_agent(task_desc) + parser = argparse.ArgumentParser(description="Run Android Computer Use AI Agent") + parser.add_argument( + "--model", + "-m", + default="gemini-3.6-flash", + help="Gemini model ID to use (default: gemini-3.6-flash)", + ) + parser.add_argument( + "--thinking-level", + "-t", + default="medium", + help="Thinking level for generation config (default: medium)", + ) + parser.add_argument( + "task", + nargs="*", + help="Task description for the agent", + ) + + args = parser.parse_args() + task_desc = ( + " ".join(args.task) + if args.task + else "Find the latest blog post from philipp schmid and summarize it." + ) + run_agent( + task=task_desc, + model=args.model, + thinking_level=args.thinking_level, + ) From 5209853c5d3553ff79b07fd7d0a7549abeece7d1 Mon Sep 17 00:00:00 2001 From: Patrick Loeber Date: Thu, 23 Jul 2026 19:01:55 +0200 Subject: [PATCH 2/5] update --- agent.py | 61 +++++++++++++++++--------------------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/agent.py b/agent.py index e6188dc..cf953f2 100644 --- a/agent.py +++ b/agent.py @@ -189,13 +189,14 @@ def screenshot(self) -> bytes: def run_agent( task: str, + device_id: str = None, model: str = "gemini-3.6-flash", thinking_level: str = "medium", max_turns: int = 100, ): start_emulator() client = genai.Client() - bridge = ADBBridge() + bridge = ADBBridge(device_id) print(f"\nTask: {task}") print(f"Model: {model}") @@ -217,49 +218,16 @@ def run_agent( while turn < max_turns: turn += 1 - print(f"\n==================== TURN {turn} ====================") - print(f"[Turn {turn}] Sending interaction request (previous_id: {previous_interaction_id})") - try: - interaction = client.interactions.create( - model=model, - system_instruction=SYSTEM_PROMPT, - input=user_input, - tools=[{"type": "computer_use", "environment": "mobile"}], - generation_config={"thinking_config": {"thinking_level": thinking_level}}, - previous_interaction_id=previous_interaction_id, - ) - except Exception as e: - print(f"\n" + "!" * 60) - print(f"[INTERACTION API ERROR] Exception Type: {type(e).__module__}.{type(e).__name__}") - print(f"[Error Message]: {e}") - - # Extract raw response or metadata from SDK exception - for attr in ["http_res_text", "response_data", "message", "body", "args"]: - val = getattr(e, attr, None) - if val: - print(f"\n--- Exception.{attr} ---") - print(val) - - # Inspect underlying cause if wrapped - cause = getattr(e, "__cause__", None) - if cause: - print(f"\n--- Underlying Cause: {type(cause).__name__} ---") - print(f"Cause message: {cause}") - for attr in ["http_res_text", "response_data", "body"]: - val = getattr(cause, attr, None) - if val: - print(f"--- Cause.{attr} ---") - print(val) - print("!" * 60 + "\n") - raise - - print(f"[Turn {turn} Response] interaction_id: {interaction.id}, status: {interaction.status}") - print(f"[Turn {turn} Steps]: {len(interaction.steps)} step(s)") - for idx, step in enumerate(interaction.steps, 1): - step_type = getattr(step, "type", type(step).__name__) - print(f" Step {idx} [{step_type}]: {step}") - + interaction = client.interactions.create( + model=model, + system_instruction=SYSTEM_PROMPT, + input=user_input, + tools=[{"type": "computer_use", "environment": "mobile"}], + generation_config={"thinking_config": {"thinking_level": thinking_level}}, + previous_interaction_id=previous_interaction_id, + ) + has_function_calls = any( step.type == "function_call" for step in interaction.steps @@ -337,6 +305,12 @@ def run_agent( default="medium", help="Thinking level for generation config (default: medium)", ) + parser.add_argument( + "--device-id", + "-d", + default=None, + help="ADB device ID (optional)", + ) parser.add_argument( "task", nargs="*", @@ -351,6 +325,7 @@ def run_agent( ) run_agent( task=task_desc, + device_id=args.device_id, model=args.model, thinking_level=args.thinking_level, ) From 104e60b1599df04811f07f807ba9bd71755093f4 Mon Sep 17 00:00:00 2001 From: Patrick Loeber Date: Thu, 23 Jul 2026 19:03:30 +0200 Subject: [PATCH 3/5] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 571258e..36aaa14 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ Examples: python agent.py "Find the latest blog post from philipp schmid and summarize it." # Specify a different model ID -python agent.py --model gemini-3.5-flas-lite "Open Settings and enable dark mode" +python agent.py --model gemini-3.5-flash-lite "Open Settings and enable dark mode" # Adjust thinking level python agent.py --thinking-level high "Open Clock and set an alarm for 7 AM" From a6a8723a5de8c669d8e5383d432a856875441dcf Mon Sep 17 00:00:00 2001 From: Patrick Loeber Date: Thu, 23 Jul 2026 19:05:21 +0200 Subject: [PATCH 4/5] simplify README --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 36aaa14..026d426 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ source .venv/bin/activate uv pip install -r requirements.txt ``` -Then run the agent with a default or custom task: +Then run the agent: ```bash python agent.py "Find the latest blog post from philipp schmid and summarize it." @@ -71,9 +71,6 @@ python agent.py "Find the latest blog post from philipp schmid and summarize it. Examples: ```bash -# Run with a custom task -python agent.py "Find the latest blog post from philipp schmid and summarize it." - # Specify a different model ID python agent.py --model gemini-3.5-flash-lite "Open Settings and enable dark mode" From b20896ab92fa321efc77b588d2b45c7880a1a8bc Mon Sep 17 00:00:00 2001 From: Patrick Loeber Date: Thu, 23 Jul 2026 19:07:23 +0200 Subject: [PATCH 5/5] fix README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 026d426..994a9cb 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ python agent.py --model gemini-3.5-flash-lite "Open Settings and enable dark mod # Adjust thinking level python agent.py --thinking-level high "Open Clock and set an alarm for 7 AM" +``` ### Supported Models