app.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install -q llama-index\n",
    "%pip install -q openai\n",
    "%pip install -q transformers\n",
    "%pip install -q accelerate\n",
    "%pip install -q python-dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.llms.openai import OpenAI\n",
    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
    "from IPython.display import Markdown, display\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "load_dotenv()  # take environment variables from .env."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"OPENAI_API_KEY\"] = os.getenv('OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Ignoring wrong pointing object 8 0 (offset 0)\n",
      "Ignoring wrong pointing object 23 0 (offset 0)\n",
      "Ignoring wrong pointing object 25 0 (offset 0)\n",
      "Ignoring wrong pointing object 28 0 (offset 0)\n",
      "Ignoring wrong pointing object 30 0 (offset 0)\n",
      "Ignoring wrong pointing object 38 0 (offset 0)\n",
      "Ignoring wrong pointing object 40 0 (offset 0)\n",
      "Ignoring wrong pointing object 42 0 (offset 0)\n",
      "Ignoring wrong pointing object 54 0 (offset 0)\n",
      "Ignoring wrong pointing object 66 0 (offset 0)\n",
      "Ignoring wrong pointing object 69 0 (offset 0)\n",
      "Ignoring wrong pointing object 71 0 (offset 0)\n",
      "Ignoring wrong pointing object 77 0 (offset 0)\n",
      "Ignoring wrong pointing object 79 0 (offset 0)\n",
      "Ignoring wrong pointing object 81 0 (offset 0)\n",
      "Ignoring wrong pointing object 83 0 (offset 0)\n",
      "Ignoring wrong pointing object 85 0 (offset 0)\n"
     ]
    }
   ],
   "source": [
    "documents = SimpleDirectoryReader(\"data\").load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(id_='29b1dfdb-5914-4d83-a38e-6838d63c446c', embedding=None, metadata={'page_label': '1', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='CRAB: Cross-environment Agent Benchmark forMultimodal Language Model AgentsTianqi Xu1,2òLinyao Chen3òDai-Jie Wu1òYanjun Chen4òZecheng ZhangXiang Yao2Zhiqiang Xie5Yongchao Chen6Shilong Liu7Bochen Qian8Philip Torr9Bernard Ghanem1†Guohao Li2,9†òEqual Contribution†Corresponding Author1KAUST2Eigent.AI3UTokyo4CMU5Stanford6Harvard7Tsinghua8SUSTech9OxfordAbstractThe development of autonomous agents increasingly relies on Multimodal Lan-guage Models (MLMs) to perform tasks described in natural language with GUIenvironments, such as websites, desktop computers, or mobile phones. Existingbenchmarks for MLM agents in interactive environments are limited by their focuson a single environment, lack of detailed and generalized evaluation methods,and the complexities of constructing tasks and evaluators. To overcome theselimitations, we introduceCrab, the ﬁrst agent benchmark framework designed tosupport cross-environment tasks, incorporating a graph-based ﬁne-grained evalua-tion method and an efﬁcient mechanism for task and evaluator construction. Ourframework supports multiple devices and can be easily extended to any environ-ment with a Python interface. LeveragingCrab, we developed a cross-platformCrab Benchmark-v0comprising 100 tasks in computer desktop and mobile phoneenvironments. We evaluated four advanced MLMs using different single andmulti-agent system conﬁgurations on this benchmark. The experimental resultsdemonstrate that the single agent with GPT-4o achieves the best completion ratio of35.26%. All framework code, agent code, and task datasets are publicly availableathttps://github.com/camel-ai/crab.1 IntroductionThe development of autonomous agents for human-centric interactive systems—such as desktopOS [54], websites [59, 19], smartphones [55, 50], and games [42, 43]—has long been an impor-tant goal of AI research, aiming to convert natural language instructions into concrete operations.Traditionally, these challenges have been addressed using reinforcement learning [31]. Recently,Large Language Models (LLMs) have demonstrated remarkable proﬁciency in natural languageunderstanding and commonsense reasoning, making them vital tools for developing autonomousagents. This utility is further enhanced by Multimodal Language Models (MLMs), which improvethe ability to interpret visual information from GUIs [5].To effectively develop MLM-based autonomous agents for real-world applications, it is essential tocreate suitable benchmarks for standardized performance evaluation. However, existing benchmarksstill have limitations in terms of interaction methods, platform diversity, evaluation metrics, statictask dataset that prevent them from closely mirroring complex real-world applications. First, existingbenchmarks that interact with the environments through pre-collected observation data from systemenvironments [40, 30, 6] fail to capture the dynamic nature of real-world scenarios without interactiveexploration where data and conditions can change unpredictably. Second, existing benchmarks arePreprint. Under review.arXiv:2407.01511v1  [cs.AI]  1 Jul 2024', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='aa38044c-2de6-483a-b81e-33686b64774c', embedding=None, metadata={'page_label': '2', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Crab FrameworkEnvironment\\nAgent SystemGraph Evaluator\\nBenchmarkDatasetUbuntu TaskInstructionEvaluatorAndroid TaskInstructionEvaluatorEnvironment\\nCross-platform TaskInstructionEvaluator\\nAndroidAgent\\nUbuntuAgent\\nMain Agent\\nInitialize Task①Observe②Plan & Instruct ③Take Action \\n④Update Evaluator CompletedActiveInactive⑤Output Metrics \\nFigure 1:Architecture of the Crab Framework demonstrating a benchmarking workﬂow for amulti-agent system.A task is initialized by assigning instructions to the main agent and a graphevaluator inside the benchmark system. The workﬂow progresses through a cycle where the mainagent observes, plans, and instructs the sub-agents, who then execute actions within their respectiveenvironments. The graph evaluator monitors the status of tasks within the environments, continuouslyupdating and outputting the task completion metrics throughout the workﬂow.typically evaluated on a single platform, either Web, Android, or Desktop OS [38, 50, 49]. However,the practical applications usually involve tasks that span multiple platforms. For example, using asmartphone to take a photo and sending it to a desktop for editing with a graphics editor is a commonreal-world task across multiple platforms. Third, existing evaluation methods are generally eithergoal-based or trajectory-based [38, 50]. Goal-based methods typically employ a coarse-grainedbinary reward, solely evaluating whether the ﬁnal system state aligns with the task’s objectives. Incontrast, trajectory-based methods can offer more nuanced metrics by assessing the agent’s actionsagainst a gold trajectory yet ignore the possibility of multiple valid pathways to complete a task,making the evaluation results less fair. Lastly, task creation within these complex systems are notstatic and extensible with ﬁxed templates [40, 49], which limits the diversity and scope of tasks.We propose a benchmark that closely mirrors real-world situations and an evaluation method thatmore accurately reﬂects an agent’s performance on complex tasks. To this end, we introduceCrab, a novelCRoss-environmentAgentBenchmark framework.Crabprovides a comprehensiveframework for evaluating cross-environment tasks in interactive environments, where the agentneeds to operate simultaneously across various devices and platforms, adapting to varied systemconditions to complete tasks efﬁciently. To the best of our knowledge,Crabis the ﬁrst autonomousagent benchmark framework that incorporates thecross-environment tasks. Moreover, we proposea novel evaluation method calledgraph evaluator. Unlike traditional goal-based and trajectory-based evaluation, our graph evaluator checks the intermediate procedures of completing a task bydecomposing the task into multiple sub-goals. Each sub-goal is assigned a judge function to verify itscompleteness, and each is considered a node in the graph evaluator. The graph structure describes thesequential and parallel relationships between the sub-goals. Therefore, it offers ﬁne-grained metricssimilar to trajectory-based evaluations while accommodating multiple valid pathways to a solution,making it more suitable for evaluating tasks that involve various correct approaches. To solve theincreasing complexity in cross-environment task construction. We also propose a highly extensiblegraph-based task construction method calledsub-task composition. Combining multiple sub-tasksin a graph with task targets allows for efﬁcient construction of various cross-environment tasks withcorresponding graph evaluators. Table 1 comparesCrabwith existing frameworks.Based onCrabframework, we develop a benchmarkCrab Benchmark-v0with two cooperatedenvironments that include an Android emulator and an Ubuntu desktop virtual machine. We have de-2', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='019f1bcc-019d-4941-9ca5-3790a6b83b64', embedding=None, metadata={'page_label': '3', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 1:Comparison of existing agent benchmark frameworks.The columns details key featuresof each framework:Interactive Environmentindicates the presence of either interactive environmentsor static datasets;Multimodal Observationspeciﬁes the availability of vision-based observations(e.g. screenshots);Cross-platformdenotes support for multiple operating systems or platforms;Evaluationdescribes the evaluation metrics, categorized asGoal-based(checking environment stateaccording solely on the ﬁnal goal),Trajectory-based(comparing agent action trajectory with agold actions sequence),Multiple(varied across tasks), orGraph-based(a DAG with each nodeas an intermediate checkpoint);Task Constructionshows the task construction method, includingHandmade(handcrafted by human),LLM-inspired(using LLM to generate task drafts but still veriﬁedand annotated by human),Template(generated by ﬁlling in the blanks in task templates), orSub-taskComposition(composing multiple sub-tasks to construct tasks and evaluators).InteractiveEnvironmentMultimodalObservationCross-platformEvaluationTaskConstructionMINIWOB++ [38] Web37Goal-based HandmadeMETAGUI [40]777Trajectory-based HandmadeGAIA [30]777Goal-based HandmadeMIND2WEB[6]777Goal-based LLM-inspiredAGENTBENCH[27] Multi-isolated77Multiple HandmadeINTERCODE[52] Code77Goal-based HandmadeWEBARENA[59] Web37Goal-based TemplateWEBSHOP[53] Web37Goal-based TemplateOMNIACT[16]777Trajectory-based HandmadeVWEBARENA[19] Web37Goal-based TemplateANDROIDARENA[50] Android37Trajectory-based LLM-inspiredOSWORLD[49] Desktop OS37Goal-based TemplateCRABDesktop OS & Android33Graph-based Sub-task Compositionveloped a total of 100 real-world tasks, encompassing both cross-environment and single-environmenttasks across multiple levels of difﬁculty. These tasks address a wide array of common real-worldapplications and tools, including but not limited to calendars, email, maps, web browsers, and ter-minals, and facilitate common interactions between smartphones and desktops. Considerable timehas been invested in verifying the accuracy and comprehensiveness of the instructions for subtasks,as well as the generalization and correctness of their evaluators. Most tasks are constructed using acareful composition of sub-tasks, while some tasks are crafted manually to accommodate speciﬁccollaborative scenarios. We test 4 popular MLMs, including GPT-4 Turbo, GPT-4o, Claude 3 Proand Gemini 1.5 Pro, across different structures of single-agent and multi-agent systems, totaling 9different agent settings in our benchmarks. The experimental results show that the single agent withGPT-4o model achieves the best completion ratio of 35.26%, underscoring the necessity for ongoingdevelopment of more effective autonomous agents. Our proposed metrics successfully distinguishbetween different methods better than previous metrics. We further analyze the different terminationreasons that reﬂect the problems inherent in the communication within the multi-agent system.2 Related Work2.1 Autonomous AgentsRecently, with the great advancement of Large Language Models (LLMs), a great number of LLM-based agents [45, 13, 48] have secured signiﬁcant accomplishments across an expanding array ofcomplicated tasks. Some works [43, 39, 4] apply LLMs to the planning of embodied agents undercomplicated environments. Some researches [34, 24] simulate the human behaviors and socialcommunications by leveraging the stunning human-like understanding and generation capability ofLLMs. LLMs are also widely used in other domains, including web navigation [28], game playing[20], ofﬁce assistant [22], and code generation [57]. Following are some related categories of agentsto our system.Multi-AgentThese researches utilize interactions between multiple agents to better problem solv-ing. Camel [21] ﬁrstly models the complex task-solving as collaboration among different agentsplaying different roles. Metagpt [11] encodes the Standardized Operating Procedures (SOPs) into astreamlined collaboration workﬂow, leveraging multiple agents as different roles in each sections.3', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='be79bf93-7081-4c11-9913-5eb05ed525c5', embedding=None, metadata={'page_label': '4', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='AutoGen[46] introduces the customization of different agents, with the correlated conversationprogramming as a new paradigm to tackle different challenges automatically.Multimodal AgentRecent researches [12, 19] achieve impressive capacity in processing image-text inputs in complicated environments. LLaV A-Plus [26] introduces different tools to obtainbetter understanding in visual input. WebGUM [8] tackles the web navigation task considering theweb pages as image input, achieving great results via corresponding ﬁne-tuning. VisualWebArena[19] receives HTML screenshots and additional ﬁgures as the visual input, applying Set-of-Markannotation to improve the performance of the agent.Operating System Control AgentWith the large models demonstrating profound knowledge ofoperating systems and formidable planning capabilities, operating system control agent [47, 44] hasemerged as a new focal point. Synapse [58] collects computer control trajectories as the exemplarsfor in-context learning of LLMs, addressing the challenge of limited context length. SceenAgent [32]broadens the computer system control tasks into a wider range over the web-related tasks. Cradle[41] gives attention on a more laborious multi-modal environment, the digital game environment,validating the potential of VLMs in solving complex tasks. MobileAgent [44] pays attention tocontrol android system via LLMs and VLMs and tests on a wider range of android apps. AndroidArena [50] underlines the collaboration among android applications and expands simple androidtasks into cross-App and constrained tasks, which veriﬁes the potential of LLM-based complicatedandroid system control. these researches are limited to speciﬁc domains and lack the capability forcross-platform control.2.2 Benchmark for Language Model AgentsVarious benchmarks are developed to validate the performance of autonomous agents based onthe reproducible environments. Miniwob++ [38] analyzes the open-domain web tasks, buildscorresponding web environment, and produces high-quality datasets considering extensive websiteand operation categories. GAIA [30] proposes a benchmark which considers the challenges ofemergency cases. Mind2Web [6] proposes a benchmark for the real-world websites which aregenuine and unpredictable, with a high coverage of domains, websites, tasks, and user-interactions.WebArena [59] provides a realistic and reproducible web environment to simulate sufﬁciently complexweb tasks. Several works [19, 10] further broaden the web tasks, considering the visual tasks to buildthe benchmark for multi-modal autonomous agents. SWEBench [15] builds a benchmark based onthe Github, focusing on the coding capacity of understanding and solving github issues. AgentBench[27], signiﬁcantly expands the scope of agent applications within the domain of computer interactiontasks. This expansion is particularly noteworthy as it encompasses the examination of these tasksacross a diverse array of complex and challenging environments. OMNIACT [16] incorporatesthe visual information of OS screen UI via segmentation and corresponding tagging, which createscorresponding tasks upon the basic elements. OSWorld [49] pays attention to the simulations acrossdiverse computer systems, taking XML and screenshots as both inputs and meticulously delineating astandardized format for both the environment and the evaluation process.Contemporary studies not only focus on tasks related to control within web and computer systemsbut also extend their scope to encompass control tasks within mobile systems. MetaGUI [40] dividesthe mobile system control tasks into dialogues and GUI operation traces, collecting GUI traces basedon the collected dialogues. AITW [36] produces a large dataset upon a large dataset of real-worldscenarios, and builds challenging multi-steps tasks based on the annotated single-step tasks as atwo-stage manner. MobileAgent [44] proposes tasks based on Ant Intelligent Assistant(AIA) system,which integrates Standing Operating Procedure(SOP) information for the creation of subtasks. AITZ[56] constructs datasets with Chain-of-Thought (COT) considerations, adding semantic annotationsaccording to visual models at each step, and developing the operational procedure for selected tasks.3 Deﬁnitions3.1 Problem FormulationConsider autonomous agents performing a task on a digital device (i.e. desktop computer). Such adevice typically has input devices (i.e. mouse and keyboard) for human interaction and output devices(i.e. screen) to allow human observation of its state. InCrab, we represent this type of device as an4', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='d5d18289-2f32-4b60-8db2-b030232a0876', embedding=None, metadata={'page_label': '5', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='environment. Formally, this environment is deﬁned as a reward-free Partially Observable MarkovDecision Process (POMDP), denoted by the tupleM⇥=(S,A,T,O), whereSrepresents the statespace,Athe action space,T⇥S✓A\\x00Sthe transition function, andOthe observation space.Considering the collaborative nature of multiple devices in real-world scenarios, we can combinemultiple environments into a setM=M1,M2,. . . ,Mn, wherenis the number of environments andeach environmentMj=(Sj,Aj,Tj,Oj). We deﬁne a task that requires operations across multipleenvironments as across-environment task. This task is formalized as a tuple(M,I,R), in whichMis the environment set,Iis the task objective in the form of natural language instructions, andRis the reward function of the task. Anagent system, designed to complete a task representedby an instructionI, can be modeled as a policy⇡((m, a)∂(I,H,o1,. . . ,on)), which deﬁnesthe probability of taking actionain environmentmwhen receiving observation(o1,. . . ,on)fromenvironment(M1,. . . ,Mn)with a history of actionsH. Anagentwithin the agent system operateswith a ﬁxed back-end MLM, a predeﬁned system prompt, and retains its chat history. An agentsystem is composed of either a single agent responsible for all planning, reasoning, and action-takingor multiple agents connected through a communication workﬂow to collaborate.3.2 Graph of Task DecompositionDecomposing a complex task into several simpler sub-tasks has been proved to be an effectiveprompting method for LLMs [17]. Some studies represent sub-tasks in a graph structure. For instance,PLaG [23] uses a graph-based structure to enhance plan reasoning within LLMs, while DyVal [60]employs directed acyclic graphs (DAGs) to facilitate dynamic evaluation of LLMs. By introducingthis concept into the realm of benchmarks, naturally, decomposing a complex task into sub-tasksthat have both sequential and parallel connections forms a DAG. Therefore, we introduce theGraphof Decomposed Tasks(GDT), which provides a new task decomposition method representingdecomposed sub-tasks within a DAG structure. In GDT, each node is a sub-task, formalized as atuple(m,i,r), wheremspeciﬁes the environment in which the sub-task is performed,iprovidesthe natural language instruction, andrrepresents the reward function. This function evaluates thestate ofmand outputs a boolean value to determine if the sub-task is completed. The edges withinGDT represent the sequential relationship between sub-tasks. An example GDT is shown in Fig. 2.4 The Crab Framework4.1 Cross-environment Agent InteractionCompared to single-environment tasks, cross-environment tasks offer three main advantages forbenchmarking agents. First, cross-environment tasks reﬂect real-world scenarios where humansuse multiple devices simultaneously to accomplish tasks. Second, these tasks require sophisticatedmessage processing and information transfer between environments. Such tasks demand that the agentplan actions, construct outputs for each environment, and remember what needs to be transferred,showcasing a high-level understanding of real-world and ability to solving complex tasks. Lastly,role-playing multi-agent systems have proven to be effective in executing complex tasks [21, 11].The underlying principle of their effectiveness is the division of responsibilities. Cross-environmenttasks are suited to multi-agent, as they can be divided by distinct observation spaces, action spaces,and specialized knowledge in each environment, as shown in Fig. 1.Crabuses a uniﬁed interface foragents to operate in all environments. We deﬁne an action by its name, the environment it belongsto, a concrete description of its functionality, and the parameters with descriptions. The agent mustprovide the action name, parameters, and the target environment in each turn.Crabtranslates theaction into its corresponding function and routes it to the physical or virtual device through thenetwork. Implementation details are in the Appendix A.1.4.2 Graph EvaluatorTo assess the capabilities of MLM agents, most benchmarks [38, 6, 19, 59] evaluate agents based onsolely the ﬁnal states of the environment after agent operations. Typically, they only judge whether theﬁnal goal is success or fail. However, this approach does not capture incremental progress made bythe agents. For instance, consider two agents tasked with installing a new application on a computer:agentasuccessfully downloads the installer but fails during the installation process, whereas agentbdoes not even try to ﬁnd the installer. Despite Agentamaking more progress, both are deemed5', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='00552266-bdf3-4621-8bf1-9cec611ef871', embedding=None, metadata={'page_label': '6', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Download the html file of the 1st item.Put all files in the same folder.Open aweb browser.Open an online shopping website. Search for T-shirts. Download html files for the top 10 items. Write a Python script to extract the relevant information in a CSV file.DecomposeCompose\\nWrite a python script thatparses html files and saves the data in a CSV file.Run the script.Enter an online shopping website.Download the html file of the 10th item.…GDT\\nFigure 2:An example of Graph of Decomposed Tasks (GDT).The bottom graph represents a GDTthat decomposes the task described in the top gray box. Each node in the GDT corresponds to a sub-task, with text boxes of matching colors providing descriptions. The overall task involves analyzingT-shirt information from an online shopping website. This task is decomposed into smaller sub-tasks,which are intermediate goals connected to form parallel or sequential relationships. Conversely, thesesub-tasks can also be composed together to form a larger, uniﬁed task.failures under the binary evaluation system, resulting in an unfair assessment of their performance.An alternative method,Trajectory-based Matching[50, 16], abandons state-based evaluation andinstead compares the agent’s actions against a predeﬁned gold action sequence for each task, givingnuanced metrics. Nevertheless, this method faces challenges in real-world systems where tasks mayhave multiple valid execution paths. For example, copying a ﬁle can be accomplished using either aﬁle manager or the command line. Inspired by the \"decomposing\" idea from GDT (Sec. 3.2), wepropose a novel integrated approach, theGraph Evaluator, which provides ﬁne-grained metrics andsupports multiple valid paths.To build a graph evaluator for a given task, we begin by decomposing the task into a GDT, whereeach sub-task is associated with an intermediate environment state critical to completing the overalltask. Nodes in the graph evaluator activate when they either have no incoming edges or after alltheir preceding tasks are completed, ensuring a sequential order of tasks. After an agent takes anaction, the system checks these active nodes to verify if the target state of each node is reached. Anode completion triggers successor nodes to activate and verify the state. This cycle repeats untilno new nodes activate, showing that the system’s task sequence aligns with the current state of theenvironment. Unlike trajectory-based methods, which compare sequences of agent actions, the GraphEvaluator does not rely on the speciﬁc actions taken by the agent, allowing it the freedom to chooseany path. Instead, it concentrates on the key intermediate states of the environment necessary forreaching the ﬁnal goal.4.3 MetricsGiven a Graph Evaluator synchronized with the environment state, it becomes possible to trackagent progress through the current status of sub-task completions. Beyond the traditionalSuccessRate (SR), which marks a task assuccessonly when all sub-tasks are completed, we introducethree metrics aiming at assessing both performance and efﬁciency of agents, leveraging the detailed6', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='0053c485-7f43-403d-9d51-fc9dcbfdbaa6', embedding=None, metadata={'page_label': '7', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='sub-task status provided by the graph evaluator. Speciﬁcally, theCompletion Ratio (CR)measuresthe proportion of completed sub-task nodes relative to the total nodes in the graph, calculated asC/N, whereCis the number of completed nodes andNis the total number of nodes. Thismetric offers a straightforward measure of an agent’s progress on a given task. TheExecutionEfﬁciency (EE), calculated asCR/A, whereAdenotes the count of executed actions. It evaluateshow efﬁciently actions are executed relative to the completion of nodes, reﬂecting the agent’s taskexecution efﬁciency. Lastly, theCost Efﬁciency (CE), calculated asCR/T, whereTis the totalnumber of model tokens used, evaluates the efﬁciency of resource consuming by the agent.4.4 Task and Evaluator ConstructionDespite the graph evaluator offers detailed evaluations, one challenge is the complexity in creatingeach evaluator. Creating a graph evaluator requires: (1) adequately decomposing a task into multiplesub-tasks, each with a well-deﬁned graph structure; and (2) engaging an expert who is well-acquaintedwith the target platform to carefully craft an evaluator for each sub-task. To streamline the creation oftasks and the development of evaluators, we consider to build GDTs by sub-tasks.There are two primary challenges in constructing GDT: (1) Sub-tasks still require manual creation,necessitating a method to quickly generate them on a large scale; (2) Properly modeling the sequentialand parallel relationships between sub-tasks, ensuring that the edges connecting sub-task nodes aresemantically meaningful and systematically applicable. A template-based approach is commonlyused to address the ﬁrst issue by generating a large number of tasks efﬁciently. To tackle the secondchallenge, we employ the message transferring concept (Sec. 4.1). Speciﬁcally, if a sub-task↵produces an output message that serves as an input for another sub-task\\x00, then↵can be considereda legitimate prerequisite of\\x00, allowing us to connect↵and\\x00with an directed edge in the GDT. Tofurther reﬁne our approach, we introduce asub-task templatestructure. Each sub-task is describedusing a natural language instruction template that includes several replaceable input attributes. Thetypes of each input attribute and the task output should be deﬁned carefully. To generate a GDT,input attributes can be ﬁlled with either a hand-crafted value corresponding to their type or linked toa task with the same output type as the input type. From the evaluator’s perspective, each sub-tasktemplate is linked to an evaluator generator that uses the input attribute value to generate evaluatorsubgraphs. Once a GDT is constructed, the graph evaluator is created by interlinking each subgraph.The description for the composed task is initially generated by GPT-4 using the sub-task descriptionsas prompts and subsequently reﬁned and polished by human reviewers.5 The Crab BenchmarkWe build an agent benchmarkCrab Benchmark-v0featuring with cross-environment, graph evaluator,and task generation throughCrabframework. The environments consists of an Android smart-phone emulator and a Ubuntu Linux desktop virtual machine. We establish both environments ina reproducible and standalone manner: the Ubuntu environment is launched on a QEMU/KVM [3,18] Virtual Machine, and the Android environment employs the Google Android Emulator1. Bothenvironments utilize snapshots to ensure a consistent state across all sessions. This allows eachexperiment to start from an identical state, providing a controlled setup for all test agents.Interaction with the Ubuntu environment is facilitated using PyAutoGUI2and MSS3, which providehigh-level commands for mouse and keyboard control and screen capture, respectively. For theAndroid environment, we use the Android Debug Bridge (ADB)4.Observation Space.The observation space consists solely of the current system screen for bothenvironments, captured in image format at each step of the agent’s interaction. We employ theSet-of-Marks visual prompt method [51] to label each interactive element on the screen. Interactiveelements are identiﬁed using the GroundingDINO [25] withicon.logo.text prompt to locate all1https://developer.android.com/studio/run/emulator2https://github.com/asweigart/pyautogui3https://github.com/BoboTiG/python-mss4https://developer.android.com/tools/adb7', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='fca332d9-58f9-46fa-aae8-4615fe84b1b0', embedding=None, metadata={'page_label': '8', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='interactive icons. Additionally, Optical Character Recognition (OCR) is utilized through EasyOCR5to detect and label interactive text elements. Each detected item is assigned a unique integer ID,facilitating reference within the action space.Action Space.The action spaces for Ubuntu and Android are distinct and designed to be close to thecommon interactions in the real devices. For Ubuntu, we deﬁne the following actions: mouse-basedactions, keyboard-based actions and a shortcut action to search for applications. For Android, theaction set includes tapping actions, a text action, a physical button action, and an action to open theapp drawer. Additionally, we introduce two environment-irrelevant actions: completing the task andsubmitting an answer. Detailed descriptions for all actions are shown in Table 2.Table 2:Action space ofCrab Benchmark-v0.The actions at the top of the table apply to theUbuntu environment, those in the middle to the Android environment, and those at the bottom arerelevant across all environments.Action Name (Parameters) Descriptionclick(elem)Click onelem.right_click(elem)Right-click onelem.write_text(text)Typing the speciﬁedtext.press(key)Press a keyboardkey.hotkey(keys)Press keyboardkeysat the same time.scroll(direction)Scrolls page up or down.search_app(name)Search for application withnamein the system.tap(elem)Tap onelem.long_tap(elem)Press and holdelem.swipe(elem,dire,dist)Swipe fromelemin a speciﬁeddirectionanddistance.write_text(text)Typing the speciﬁedtext.press(key)Press akey, can behomeorback.show_all_drawer()Show the app drawer to list installed applications.submit(answer)Submitanswerif needed.complete()Tell system a task is completed.5.1 Task ConstructionInCrab Benchmark-v0, we meticulously construct 16 sub-task templates for the Android environmentand 19 sub-task templates for the Ubuntu environment. The Ubuntu templates encompass a varietyof tasks such as Command Line Interface (CLI) operations, ﬁle system management, search engineusage, desktop conﬁgurations, and map navigation. Conversely, the Android sub-task templates areprimarily focused on the storage and transmission of messages via various applications. Each sub-tasktemplate is linked to a graph evaluator consisting of one to four nodes. Each sub-task is veriﬁed by atleast two related ﬁeld experts. To ensure the accuracy of the evaluators, the benchmark includes ahuman mode, which periodically activates the graph evaluators during manual operation by a user.We ascertain the correctness of each task by ensuring that all evaluators in the graph can be passedin human mode. The task dataset is consisted by 35 single tasks generated by a single sub-task, 53simple tasks and 12 challenging tasks obtained by sub-task composition aiming to verify the differentlevels of challenges in daily usage scenarios. The dataset has 29 android tasks, 53 Ubuntu tasks and18 cross-platform tasks. Besides, he sub-task pool has 19 in Ubuntu and 17 in Android.Evaluator Design.To assess the intermediate states of sub-tasks as described in Sec. 4.2, we haveimplemented a comprehensive suite of execution-based reward functions (evaluators) [49]. Theseevaluators retrieve and assess speciﬁc current states, such as the edited content of a ﬁle or a modiﬁedsetting, thereby determining the successful completion of a sub-task. For each evaluator, inputattributes are carefully selected to interpret software information or system settings relevant to thescenario deﬁned for the sub-task. For instance, evaluators use ﬁle paths before and after edits as inputparameters to verify the completion of ﬁle editing sub-tasks.5https://github.com/JaidedAI/EasyOCR8', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='687858e3-6e78-4788-b21f-590d1f8dc37f', embedding=None, metadata={'page_label': '9', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Speciﬁcally, for sub-tasks on the Android platform, we incorporate XML-based evaluators [50].We dump UI layout as XML path and verify whether the UI content matches the expected state.For the Ubuntu platform, we employ image matching techniques [35, 14, 7] and OCR to handlescenarios where acquiring necessary state information through conventional APIs is challenging.Image matching offers ﬁne-grained visual correspondences by comparing keypoint features betweenimages, allowing us to assess spatial relationships among visual elements. Using OCR and imagematching, we can accurately evaluate tasks such as verifying whether an agent has successfullycreated a slide with speciﬁed images, text content, and layouts—tasks for which trivial evaluationmethods are lacking. We utilize EasyOCR5and XFeat6as our primary tools for OCR and imagematching. For tasks with real-time characteristics that may change over time, we implement crawlerscripts to capture dynamic values at the moment of evaluation. These values are then compared withthe results achieved by the agent upon task completion. We have a total of 59 evaluator functions.6 Experiments6.1 Baseline Agent SystemAt the core of MLM Agents are back-end Multimodal Language Models that provide natural languageand image understanding, basic device knowledge, task planning, and logical reasoning abilities. Torun inCrab Benchmark-v0, the back-end model needs to support: (1) Accept multimodal mixedinput, as the system provides both screenshots and text instructions as prompts; (2) Handle multi-turnconversations, as most tasks require the agent to take multiple actions, necessitating the storage ofhistory messages in its context; (3) Generate structured output through function calling, ensuring theproper use of provided actions with type-correct parameters.We selected four MLMs that meet these criteria for our experiments: GPT-4o (gpt-4o-2024-05-13)[33], GPT-4 Turbo (gpt-4-turbo-2024-04-09) [1], Gemini 1.5 Pro (May 2024 version) [37], Claude3 Opus (claude-3-opus-20240229) [2]. These models serve as the backend models for our agents.Beyond the MLM backend, the structure of agent systems signiﬁcantly inﬂuences overall performance.To examine how different multi-agent structures impact performance, we design three agent systemstructures:single agent,multi-agent by functionality, andmulti-agent by environment. Inthesingle agentstructure, one agent manages all responsibilities, including observation analysis,planning, reasoning, and format the output action. Themulti-agent by functionalitystructure splitstasks between a main agent, responsible for analysis and planning, and a tool agent that translatesinstructions into actions without accessing environmental observations. This division allows themain agent to concentrate on high-level tasks without managing functional call formats. Meanwhile,in themulti-agent by environmentsetup, responsibilities are further distributed. A main agentprocesses all environmental observations for high-level planning, while each environment-speciﬁcsub-agent executes actions based on the main agent’s instructions, incorporating observations fromtheir respective environments.For all models, we utilized the default API parameters and retained only two turns of historicalmessages to ensure messages do not exceed the context window. The interaction turns are limitedto 15 and the task will terminated because reaching max turns. The agent can also terminate thetask ahead if it thinks the task is completed. The screenshots do not descale and passed throughPNG format with the highest quality that the APIs provide. Detailed agent and prompt designs areshown in Appendix B. In the experiment, we deployed four cloud machines cloned from the samedisk image to ensure a consistent environment for all agents. Running a single agent setting in thebenchmark requires at least 30 hours to complete on one machine. This duration depends on the APIcall times and the necessity for manual resets in certain tasks.6.2 ResultsThe primary outcomes are detailed in Table 3. The GPT-4o and GPT-4 Turbo models, developed byOpenAI, achieve the highest average success rates and completion ratios among the tested models.Speciﬁcally, GPT-4o slightly outperforms GPT-4 Turbo. This result suggests a tiny difference intheir underlying architectures or training data, but GPT-4o possibly be trained on more GUI data.Claude 3 outperforms Gemini 1.5 according to CR. The multi-agent structures’ performances on all6https://github.com/verlab/accelerated_features9', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='f21f09b3-591e-4ca0-b18c-8d37c6555c24', embedding=None, metadata={'page_label': '10', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 3:Evaluation results onCrab Benchmark-v0.TheModelcolumn identiﬁes the backendmasked language models (MLMs) used. TheStructurecolumn describes the conﬁguration of theagent system:Singlemeanssingle agent;By Funcismulti-agent by functionality;By Envindicatesmulti-agent by environment. We provide traditional metric ofSuccess Rate(SR) alongside newlyintroduced metrics:Completion Ratio(CR),Execution Efﬁciency(EE), andCost Efﬁciency(CE).Note that Gemini 1.5 Pro has an invalid CE because the Gemini API does not support retrievingtoken counts at the start time of experiments. TheTermination Reasonshows the ratio of reasonswhy the agent stops when it does not complete the task.False Completion(FC) indicates that theagent believes it has completed the task, but it actually has not;Reach Step Limit(RSL) means theagent has reached the step limit but has not completed the task;Invalid Action(IA) refers to the agentproducing outputs that do not follow instructions, which may include invalid formats, nonexistentactions, or invalid action parameters.Agent systemMetricsTermination ReasonModel StructureSR(%)\\x00CR(%)\\x00EE(%)\\x00CE(%)\\x00FC(%) RSL(%) IA(%)GPT-4OSingle14.0035.26 3.665.26✓10\\x0047.00 59.00 20.00GPT-4OBy Func13.0032.48 3.29 5.20✓10\\x00412.00 54.00 21.00GPT-4OBy Env14.0033.74 3.40 2.71✓10\\x0048.00 49.00 29.00GPT-4 TURBOSingle11.0031.52 3.606.45✓10-47.00 64.00 18.00GPT-4 TURBOBy Func13.0029.99 3.53 4.79✓10\\x00411.00 41.00 35.00GEMINI1.5 PROSingle6.0017.19 1.69\\\\3.00 55.00 36.00GEMINI1.5 PROBy Func6.0014.53 1.50\\\\10.00 33.00 51.00CLAUDE3OPUSSingle6.0021.39 2.66 4.51✓10\\x0047.00 53.00 34.00CLAUDE3OPUSBy Func5.0018.79 1.90 3.31✓10\\x00429.00 32.00 34.00backend MLMs are slightly lower than the single agent, indicating that current autonomous agentsmainly rely on backend model performance. Regarding termination reason, multi-agent structureshave higher possibility to take invalid action and incorrectly complete the task, this can caused bythe hallucination when main agent generating the instruction messages or misunderstanding of thesub-agents when receiving these messages. In terms of execution efﬁciency, the GPT-4 series showstrong performance. However, when evaluating cost efﬁciency, GPT-4 Turbo exhibited a lower CEvalue compared to GPT-4o, suggesting that GPT-4 Turbo is more cost-effective.The completion ratio metric reveals a notable performance difference between models. For instance,even though Claude (single agent) and Gemini (multi-agent by functionality) have the same successrates, their completion ratios differ by up to 6.86%. This highlights the value of the completionratio in assessing the effectiveness of different methods. We provide more detailed analyses andcomparisons of agent conﬁgurations in the supplementary materials of this paper.7 ConclusionWe propose theCrabframework introducing cross-environment automatic task performing problem,featuring advanced graph-based task generation and evaluation methods, which reduce the manualeffort in task step and provide a more dynamic and accurate agent assessments. Based on theframework, we proposeCrab Benchmark-v0, including a set of high quality cross-environment tasksfor a smart phone and desktop, equipped with visual prompting strategy. We test various backendmodels and agent system structures on the dataset. The result reﬂects preference of different agentsettings. Despite our work contributing to better cross-environment agent research, there are still somelimitations. We build sub-tasks upon the original apps in the Ubuntu system and the Android systemon Pixel, which cannot cover a wider range of applications. Moreover, the visual information is notused in the evaluation on the sub-tasks in Android System. Future works can focus on expandingdatasets and environments and testing more models, prompts, structure of agents upon the benchmark.10', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='2ea07fdb-3c24-430f-a40e-f9a4fad78ac6', embedding=None, metadata={'page_label': '11', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='AcknowledgementWe express our gratitude to Yuhui Wang for reﬁning the expressions in our paper and providinginvaluable advice on writing. We would like to also thank Beichen Huang for the helpful discussionson solving virtualization technology issues.References[1]Josh Achiam et al.GPT-4 Technical Report. Mar. 4, 2024.URL:http://arxiv.org/abs/2303.08774. preprint.[2]Anthropic.The Claude 3 Model Family: Opus, Sonnet, Haiku.https : / / www - cdn .anthropic . com / de8ba9b01c9ab7cbabf5c33b80b7bbc618857627 / Model _ Card _Claude_3.pdf. Year.[3]Fabrice Bellard. “QEMU, a fast and portable dynamic translator.” In:USENIX annual technicalconference, FREENIX Track. Vol. 41. 46. California, USA. 2005, pp. 10–5555.[4]Yongchao Chen et al. “Autotamp: Autoregressive task and motion planning with llms astranslators and checkers”. In:arXiv preprint arXiv:2306.06531(2023).[5]Kanzhi Cheng et al.SeeClick: Harnessing GUI Grounding for Advanced Visual GUI Agents.Jan. 17, 2024.URL:http://arxiv.org/abs/2401.10935. preprint.[6]Xiang Deng et al.Mind2Web: Towards a Generalist Agent for the Web. 2023. arXiv:2306.06070 [cs.CL].[7]Johan Edstedt et al. “RoMa: Robust Dense Feature Matching”. In: 2024.[8]Hiroki Furuta et al. “Multimodal Web Navigation with Instruction-Finetuned FoundationModels”. In:The Twelfth International Conference on Learning Representations. 2024.URL:https://openreview.net/forum?id=efFmBWioSc.[9]Aric A. Hagberg, Daniel A. Schult, and Pieter J. Swart. “Exploring Network Structure, Dynam-ics, and Function Using NetworkX”. In:Proceedings of the 7th Python in Science Conference.Ed. by Gaël Varoquaux, Travis Vaught, and Jarrod Millman. Pasadena, CA USA, 2008, pp. 11–15.[10]Hongliang He et al.WebVoyager: Building an End-to-End Web Agent with Large MultimodalModels. 2024. arXiv:2401.13919 [cs.CL].[11]Sirui Hong et al. “Metagpt: Meta programming for multi-agent collaborative framework”. In:arXiv preprint arXiv:2308.00352(2023).[12]Wenyi Hong et al.CogAgent: A Visual Language Model for GUI Agents. arXiv.org. Dec. 14,2023.URL:https://arxiv.org/abs/2312.08914v2.[13]Wenlong Huang et al. “Language models as zero-shot planners: Extracting actionable knowl-edge for embodied agents”. In:International Conference on Machine Learning. PMLR. 2022,pp. 9118–9147.[14]Hanwen Jiang et al. “OmniGlue: Generalizable Feature Matching with Foundation ModelGuidance”. In:Proceedings of the IEEE/CVF Conference on Computer Vision and PatternRecognition (CVPR). 2024.[15]Carlos E. Jimenez et al.SWE-Bench: Can Language Models Resolve Real-World GitHubIssues?Oct. 10, 2023.URL:http://arxiv.org/abs/2310.06770. preprint.[16]Raghav Kapoor et al.OmniACT: A Dataset and Benchmark for Enabling Multimodal GeneralistAutonomous Agents for Desktop and Web. Feb. 28, 2024.URL:http://arxiv.org/abs/2402.17553. preprint.[17]Tushar Khot et al. “Decomposed Prompting: A Modular Approach for Solving ComplexTasks”. In:The Eleventh International Conference on Learning Representations. 2023.URL:https://openreview.net/forum?id=_nGgzQjzaRy.[18]Avi Kivity et al. “kvm: the Linux virtual machine monitor”. In:Proceedings of the Linuxsymposium. Vol. 1. 8. Dttawa, Dntorio, Canada. 2007, pp. 225–230.[19]Jing Yu Koh et al.VisualWebArena: Evaluating Multimodal Agents on Realistic Visual WebTasks. Jan. 24, 2024.URL:http://arxiv.org/abs/2401.13649. preprint.[20]Yihuai Lan et al. “Llm-based agent society investigation: Collaboration and confrontation inavalon gameplay”. In:arXiv preprint arXiv:2310.14985(2023).11', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='9c212a1f-a514-4889-92ed-4c9d108fff2d', embedding=None, metadata={'page_label': '12', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='[21]Guohao Li et al. “Camel: Communicative agents for\" mind\" exploration of large scale languagemodel society”. In: (2023).[22]Hongxin Li et al. “SheetCopilot: Bringing Software Productivity to the Next Level throughLarge Language Models”. In:Advances in Neural Information Processing Systems36 (2024).[23]Fangru Lin et al.Graph-Enhanced Large Language Models in Asynchronous Plan Reasoning.Feb. 5, 2024.URL:http://arxiv.org/abs/2402.02805. preprint.[24]Jiaju Lin et al. “Agentsims: An open-source sandbox for large language model evaluation”. In:arXiv preprint arXiv:2308.04026(2023).[25]Shilong Liu et al.Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-SetObject Detection. arXiv.org. Mar. 9, 2023.URL:https://arxiv.org/abs/2303.05499v4.[26]Shilong Liu et al.LLaVA-Plus: Learning to Use Tools for Creating Multimodal Agents.arXiv.org. Nov. 9, 2023.URL:https://arxiv.org/abs/2311.05437v1.[27]Xiao Liu et al. “AgentBench: Evaluating LLMs as Agents”. In:The Twelfth InternationalConference on Learning Representations. 2024.URL:https://openreview.net/forum?id=zAdUB0aCTQ.[28]Xing Han Lù, Zdenˇek Kasner, and Siva Reddy.WebLINX: Real-World Website Navigationwith Multi-Turn Dialogue. arXiv.org. Feb. 8, 2024.URL:https://arxiv.org/abs/2402.05930v1.[29]Michael M. McKerns et al.Building a Framework for Predictive Science. Feb. 6, 2012.URL:http://arxiv.org/abs/1202.1056. preprint.[30]Grégoire Mialon et al.GAIA: A Benchmark for General AI Assistants. Nov. 21, 2023.URL:http://arxiv.org/abs/2311.12983. preprint.[31]Volodymyr Mnih et al. “Human-Level Control through Deep Reinforcement Learning”. In:Nature518.7540 (Feb. 2015), pp. 529–533.ISSN: 1476-4687.URL:https://www.nature.com/articles/nature14236.[32]Runliang Niu et al.ScreenAgent: A Vision Language Model-Driven Computer Control Agent.Feb. 8, 2024.URL:http://arxiv.org/abs/2402.07945. preprint.[33]OpenAI.GPT-4 omni.https://openai.com/index/hello-gpt-4o/. 2024.[34]Joon Sung Park et al. “Generative agents: Interactive simulacra of human behavior”. In:Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology.2023, pp. 1–22.[35]Guilherme Potje et al. “XFeat: Accelerated Features for Lightweight Image Matching”. In:IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR). 2024.[36]Christopher Rawles et al.Android in the Wild: A Large-Scale Dataset for Android DeviceControl. 2023. arXiv:2307.10088 [cs.LG].[37]Machel Reid et al.Gemini 1.5: Unlocking Multimodal Understanding across Millions ofTokens of Context. Apr. 25, 2024.URL:http://arxiv.org/abs/2403.05530. preprint.[38]Tianlin Shi et al. “World of Bits: An Open-Domain Platform for Web-Based Agents”. In:Proceedings of the 34th International Conference on Machine Learning. Ed. by Doina Precupand Yee Whye Teh. Vol. 70. Proceedings of Machine Learning Research. PMLR, June 2017,pp. 3135–3144.URL:https://proceedings.mlr.press/v70/shi17a.html.[39]Chan Hee Song et al. “Llm-planner: Few-shot grounded planning for embodied agents withlarge language models”. In:Proceedings of the IEEE/CVF International Conference on Com-puter Vision. 2023, pp. 2998–3009.[40]Liangtai Sun et al.META-GUI: Towards Multi-Modal Conversational Agents on Mobile GUI.Nov. 24, 2022.URL:http://arxiv.org/abs/2205.11029. preprint.[41]Weihao Tan et al.Towards General Computer Control: A Multimodal Agent for Red DeadRedemption II as a Case Study. 2024. arXiv:2403.03186 [cs.AI].[42]Oriol Vinyals et al. “Grandmaster Level in StarCraft II Using Multi-Agent ReinforcementLearning”. In:Nature575.7782 (Nov. 2019), pp. 350–354.ISSN: 1476-4687.URL:https://www.nature.com/articles/s41586-019-1724-z.[43]Guanzhi Wang et al.Voyager: An Open-Ended Embodied Agent with Large Language Models.May 25, 2023.URL:http://arxiv.org/abs/2305.16291. preprint.[44]Junyang Wang et al.Mobile-Agent: Autonomous Multi-Modal Mobile Device Agent with VisualPerception. arXiv.org. Jan. 29, 2024.URL:https://arxiv.org/abs/2401.16158v2.12', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='6a4fc2ed-1972-4970-8fc0-165581704f4b', embedding=None, metadata={'page_label': '13', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='[45]Lei Wang et al. “A survey on large language model based autonomous agents”. In:Frontiersof Computer Science18.6 (2024), pp. 1–26.[46]Qingyun Wu et al. “Autogen: Enabling next-gen llm applications via multi-agent conversationframework”. In:arXiv preprint arXiv:2308.08155(2023).[47]Zhiyong Wu et al.OS-Copilot: Towards Generalist Computer Agents with Self-Improvement.arXiv.org. Feb. 12, 2024.URL:https://arxiv.org/abs/2402.07456v2.[48]Zhiheng Xi et al.The Rise and Potential of Large Language Model Based Agents: A Survey.arXiv.org. Sept. 14, 2023.URL:https://arxiv.org/abs/2309.07864v3.[49]Tianbao Xie et al.OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks inReal Computer Environments. Apr. 11, 2024.URL:http://arxiv.org/abs/2404.07972.preprint.[50]Mingzhe Xing et al.Understanding the Weakness of Large Language Model Agents within aComplex Android Environment. Feb. 9, 2024.URL:http://arxiv.org/abs/2402.06596.preprint.[51]Jianwei Yang et al.Set-of-Mark Prompting Unleashes Extraordinary Visual Grounding inGPT-4V. Nov. 6, 2023.URL:http://arxiv.org/abs/2310.11441. preprint.[52]John Yang et al.InterCode: Standardizing and Benchmarking Interactive Coding with Execu-tion Feedback. Oct. 30, 2023.URL:http://arxiv.org/abs/2306.14898. preprint.[53]Shunyu Yao et al. “Webshop: Towards scalable real-world web interaction with grounded lan-guage agents”. In:Advances in Neural Information Processing Systems35 (2022), pp. 20744–20757.[54]Chaoyun Zhang et al.UFO: A UI-Focused Agent for Windows OS Interaction. Mar. 1, 2024.URL:http://arxiv.org/abs/2402.07939. preprint.[55]Chi Zhang et al.AppAgent: Multimodal Agents as Smartphone Users. Dec. 21, 2023.URL:http://arxiv.org/abs/2312.13771. preprint.[56]Jiwen Zhang et al.Android in the Zoo: Chain-of-Action-Thought for GUI Agents. 2024. arXiv:2403.02713 [cs.CL].[57]Shun Zhang et al. “Planning with large language models for code generation”. In:arXivpreprint arXiv:2303.05510(2023).[58]Longtao Zheng et al. “Synapse: Trajectory-as-Exemplar Prompting with Memory for ComputerControl”. In:The Twelfth International Conference on Learning Representations. 2024.URL:https://openreview.net/forum?id=Pc8AU1aF5e.[59]Shuyan Zhou et al.WebArena: A Realistic Web Environment for Building Autonomous Agents.Oct. 24, 2023.URL:http://arxiv.org/abs/2307.13854. preprint.[60]Kaijie Zhu et al. “DyVal: Dynamic Evaluation of Large Language Models for ReasoningTasks”. In:The Twelfth International Conference on Learning Representations. 2024.URL:https://openreview.net/forum?id=gjfOL9z5Xr.\\n13', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='88327131-01e8-4441-bbcf-15e7c24edd5e', embedding=None, metadata={'page_label': '14', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='A Benchmark DetailThe framework code and task dataset are open source7under Apache-2.0 licences. Section A.2describes the our experiment settings in detail. Section A.3 describes the speciﬁc format deﬁned inour framework that ease data extension and how to use them. We provides a detailed document tosetup experiment environments and reproduce our results.8Fig. 3 shows the structure of modulesinsideCrab Benchmark-v0.Crab Benchmark v0Android Environment\\nUbuntu EnvironmentTasksName: \"android\"Description: \"A Google Pixel smartphoneruns on the Android operating system...\"Observation SpaceScreenshotPrompt SpaceVisual PromptAction SpaceTapPress KeyWrite TextSwipeOpen App Drawer...\\nName: \"ubuntu\"Description: \"An Ubuntu 22.04 Linux desktopoperating system...\"Observation SpaceScreenshotPrompt SpaceVisual PromptAction SpaceClickPress KeyWrite TextRight ClickSeach Application...Sub-tasksSub-task Template 1Sub-task 1Evaluator Generator 1UbuntuSub-task Template 2Sub-task 2Evaluator Generator 2UbuntuSub-task Template 3Sub-task 3Evaluator Generator 3Android...Cross-platform TaskDescriptionAttributesGraph EvaluatorAndroid TaskDescriptionAttributesGraph EvaluatorUbuntu TaskDescriptionAttributesGraph Evaluator...\\nFigure 3:Module Structure ofCrab Benchmark-v0.The benchmark is divided into two primarysections: the left section, highlighted with warm hues, features two environments, while the rightsection, accentuated with cool hues, outlines various tasks. Each environment is deﬁned by attributesincluding name, description, observation space, prompt method, and action space. Blocks markedin red denote actions. As for the tasks, they are composed of multiple sub-tasks and formulatedby combine multiple evaluator sub-graphs derived from the sub-task evaluator generators. Arrowsillustrate the compositional relationships between tasks and sub-tasks.A.1 Framework DesignCraboffers a modular and extensible framework for evaluating agent performance in diverse tasks.At the heart of the framework lies theaction, a unit operation representing the fundamental operationwithin the benchmark. Theactionis essentially an executable Python function that can be deﬁnedwith explicit typed parameters and a clear description.actionsserve not only as building blocks butalso as interfaces through which agents interact with the environment. Theevaluatoris a specializedactionrestricted to returning boolean values, signifying the success or failure of an agent’s task. Itenhances theactionsby analyzing the state of the environment and the sequence ofactionsexecutedby the agent, providing a decisive metric of task accomplishment. Additionally, multipleevaluatorscan be interconnected to form a graph evaluator for complex tasks (Sec. 4.2).Thebenchmarkis a key deﬁnition in the framework. A benchmark includes multipleenvironmentsand cross-environmenttasks. Theenvironmentis formed by an action space and an observationspace, which are both deﬁned by a list ofactions, and other essential parameters necessary for its7https://github.com/camel-ai/crab8https://github.com/camel-ai/crab/blob/main/crab-benchmark-v0/README.md14', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='76ba3ef8-64a1-48d9-8307-925ab2e81aee', embedding=None, metadata={'page_label': '15', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='conﬁguration. This composite structure facilitates the execution and monitoring ofactions, whetheron local machines, remote servers, virtual machines, or physical devices networked together. Ataskencapsulates a natural language description and a graph evaluator.Crabutilizes Python functions to deﬁne all actions and evaluators, embodying a \"code as conﬁgura-tion\" philosophy. Each function’s docstring outlines its description and parameter deﬁnitions, whichare then presented to the agent as structured prompts. Compared to traditional methods using datainterchange formats like JSON or YAML, Python code conﬁgurations provide a more structuredapproach and ﬁts in modern IDE.By decoupling actions, environments, tasks, and evaluations,Crabfacilitates a plug-and-play archi-tecture that can adapt to various scenarios. Such a system is scalable, maintainable and expandable,allowing researchers and developers to introduce new tasks and environments without restructuringthe entire framework. Our implementation usesnetworkx[9] for building graph anddill[29] forfunction serialization in our implementation.A.2 Conﬁguration by ModulesBuilding on the declarative and modular design of our framework, this section explains the conﬁgura-tion and potential extensibility of each module.EnvironmentThe environments inCrabare a combination of multiple different uses of actions withsome environment metadata, such as name and natural language description. InCrab Benchmark-v0,we use a computer desktop environment and a smartphone environment both based on virtual machinetechnology. The computer desktop environment, namedUbuntu, is installed from an ISO image ofUbuntu 22.04.4 LTS (Jammy Jellyﬁsh) downloaded from the Ubuntu Ofﬁcial website9. Necessaryapplications such as the LibreOfﬁce suite (Writer, Calc, and Impress) and Slack are installed latervia snap and apt, according to the task dataset requirements. The smartphone environment, namedAndroid, is installed using pre-deﬁned devices (Google Pixel 8 Pro with release nameR) provided inGoogle Android Studio10. We install additional required applications such asKeep Notes,Tasks, andDocsfrom Google Play. The descriptions of the two environments inCrab Benchmark-v0, which areinserted in the agent prompts, are as follows:•Ubuntu: An Ubuntu 22.04 Linux desktop operating system. The interface displays a currentscreenshot at each step and primarily supports interaction via mouse and keyboard. Youmust use searching functionality to open any application in the system. This device includessystem-related applications including Terminal, Files, Text Editor, Vim, and Settings. It alsofeatures Firefox as the web browser, and the LibreOfﬁce suite—Writer, Calc, and Impress.For communication, Slack is available. The Google account is pre-logged in on Firefox,synchronized with the same account used in the Android environment.•Android: A Google Pixel smartphone runs on the Android operating system. The interfacedisplays a current screenshot at each step and primarily supports interaction through tappingand typing. This device offers a suite of standard applications including Phone, Photos,Camera, Chrome, and Calendar, among others. Access the app drawer to view all installedapplications on the device. The Google account is pre-logged in, synchronized with thesame account used in the Ubuntu environment.ActionAction implementation inCrab Benchmark-v0utilize the dynamic feature of Python. Itprovides an intuitive method to deﬁne actions through Python function. Here is an example of actionsearch_applicationin the Ubuntu environment:@actiondefsearch_application(name:str)- >N o n e :\"\"\"Searchanapplicationname.Forexmaple,ifyouwanttoopenanapplicationnamed\"slack\",youcancallsearch_application(name=\"slack\").YouMUSTusethis9https://releases.ubuntu.com/jammy/ubuntu-22.04.4-desktop-amd64.iso10https://developer.android.com/studio15', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='7c299ce5-e88f-4a20-b5f4-f0a4a43fa240', embedding=None, metadata={'page_label': '16', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='actiontosearchforapplications.Args:name:theapplicationname.\"\"\"pyautogui.hotkey(\"win\",\"a\")time.sleep(0.5)pyautogui.write(name)time.sleep(0.5)Listing 1: Deﬁne \"search_application\" action.We extract key information from the function through the@actiondecorator as following:•Name: The action name serves as the identiﬁer for backend models. It should semanticallymatch the action’s behavior to improve the accuracy of the agent in executing the action.The function name is extracted as the action name. In this example,search_applicationis the assigned name.•Description: The description provides a natural language explanation of the action to assistthe agent in understanding how to use it. The main body of the function’s docstring is usedas the description. For example, in this instance, the description outlines the basic usage ofthe action:Search an application name, along with an example of its usage.•Parameters: The parameters are the arguments that the functions accept, offering ﬂexibilityfor the agent to control the environment. Typically, a set of parameters is deﬁned, eachconsisting of a name, type, and a natural language description. Parameters are extractedfrom the function’s parameters along with their type annotations. Additionally, parameter de-scriptions are extracted from theArgssection in the docstring. In this example, there is onlyone parameter namedname, with a type ofstr, and its description isthe applicationname.•Entry: The entry represents the implementation of the function, deﬁned within the functionbody to specify how the action is executed. When the agent invokes the function, the entryis executed with the provided parameters. In this example, we utilize thepyautoguipackagefor keyboard control. Initially, it presses a hotkey to enter the application search panel inUbuntu, then proceeds to type the application name provided by the parameters, ﬁnallydisplaying the search results.ObservationThe observation space is represented by a set of actions. These observation actionsare designed to be parameter-free and return an observation result. For instance, within the Ubuntuenvironment, the sole observation action available is thescreenshotfunction, deﬁned as follows:@actiondefscreenshot() ->str:\"\"\"Capturethecurrentscreenasascreenshot.\"\"\"with mss() as sct:#Capturerawpixelsfromthescreensct_img = sct.grab(sct.monitors[1])#ConverttoPNGformatpng = tools.to_png(sct_img.rgb, sct_img.size)#EncodetoBase64formatforeasiertransmissionbase64_img = base64.b64encode(png).decode(\"utf-8\")returnbase64_imgListing 2: Deﬁne the \"screenshot\" observation action.This action captures the screen’s current view and encodes it in Base64 format. Additionally, visualprompts are also deﬁned by actions that utilize the output from an observation action as their input,further processing it to generate a visual prompt for the agent.EvaluatorThe evaluator inCrab Benchmark-v0is crafted to assess the outcome of actionsperformed by the agent within the environment. The evaluator is deﬁned as an action thatoutputs a boolean value. An example of an evaluator in the Ubuntu environment is thecheck_text_in_current_window_namefunction, outlined below:16', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='9234616f-5e19-4c11-ae65-3e2196f3e385', embedding=None, metadata={'page_label': '17', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='@evaluator(env_name=\"ubuntu\")defcheck_text_in_current_window_name(text:str)- >bool:try:out = subprocess.check_output([\"xdotool\",\"getwindowfocus\",\"getwindowname\"], text=True).strip()exceptsubprocess.CalledProcessError:returnFalsereturntextinoutListing 3: Deﬁne \"check_text_in_current_window_name\" evaluator.The evaluator function is denoted with an@evaluatordecorator and speciﬁes its operating envi-ronment. The function’s primary role is to execute a check within the system and return a booleanvalue indicating success or failure based on the condition being evaluated. Here, the function aims toverify whether a speciﬁed text appears in the title of the currently focused window. This is achievedthrough the use of thesubprocessmodule to execute system commands that fetch the window’stitle, checking if the provided text parameter is contained within it.TaskFollowing a declarative programming paradigm, the task is deﬁned as a data model. Here isan example of a cross-platform task in the dataset:Task(id=\"a3476778-e512-40ca-b1c0-d7aab0c7f18b\",description=\"Open\\\\\"Tasks\\\\\"apponAndroid,checkthe...\",evaluator=path_graph(check_current_package_name(\"com.google.android.apps.tasks\"),check_current_window_process(\"gnome-control-center\"),check_color_scheme(\"prefer-dark\"),),)Listing 4: Deﬁne a task.In this model, each task is represented as an instance of theTaskclass, which is a subclass ofBaseModelinPydantic11package. Each task is uniquely identiﬁed by an ID and described by adetailed description. The evaluator component is structured as a graph evaluator, which integratesmultiple evaluative functions into a directed graph using thenetworkx12package. Each evaluatorwithin this graph must be appropriately parameterized to assess speciﬁc conditions relevant to thetask. For example, the task demonstrated aims to open the \"Tasks\" app on Android and performa series of veriﬁcations: it checks whether the correct Android app is opened, whether the currentfocused window’s process name isgnome-control-center, and whether the color scheme is set todark.Sub-taskThe sub-task inCrabis the unit component of in task construction. The followingexample is a sub-task template that we used to easily generate sub-tasks:SubTask(id=\"0f589bf9-9b26-4581-8b78-2961b115ab49\",description=\"Open\\\\\"{file_path}\\\\\"usingviminaterminal,write\\\\\"{content}\\\\\",thensaveandexitvim.\",attribute_dict={\"file_path\":\"file_path\",\"content\":\"message\"},output_type=\"file_path\",evaluator_generator=lambdafile_path , content: path_graph(check_current_window_process(\"gnome-terminal-server\"),is_process_open(\"vim\"),is_process_close(\"vim\"),check_file_content(file_path, content),),11https://pydantic.dev/12https://networkx.org/17', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='3215604e-9585-4d25-8a9d-f55c1302f6ac', embedding=None, metadata={'page_label': '18', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='),Listing 5: Deﬁne a task.In this sub-task model, each sub-task is deﬁned using a similar approach to the main task. Theattributes of the sub-task are outlined in anattribute_dict, which details the types and roles ofeach attribute used in the sub-task’s operations. Theoutput_typeﬁeld speciﬁes the expected typeof output from the sub-task. The types reﬂected inattribute_dictandoutput_type, play acritical role in determining the compatibility and sequential logic of compose multiple sub-tasks.The evaluator for the sub-task is dynamically generated using a lambda function, which crafts anevaluator sub-graph based on the sub-task’s attributes.A.3 Task DatasetWe use a JSON format to save the composed tasks, which includes the task ID, overall task description,sub-tasks with their attribute values, and a graph structure represented in an adjacency list. The entiretask dataset is deﬁned by the sub-task pool in Python code and the task composition JSON ﬁlescategorized by task platform.B Agent systemB.1 Agent ImplementationIn this section, we outline the implementation of the agents used in our experiments, which leverageadvanced multimodal language models from OpenAI, Anthropic, and Google. Each agent is designedto function in multi-environment setups, interacting with various action spaces deﬁned by differentenvironments.General FrameworkAll agents share a common architecture but are tailored to the speciﬁc APIsand capabilities of each language model provider.InitializationEach agent is initialized with several key parameters, including a description, anaction space, the model type, maximum tokens, history message length, and an optional environmentdescription. The initialization process involves:•Action Space Conversion: Actions deﬁned for each environment are converted into aschema compatible with the respective API. This ensures that the actions can be correctlyinterpreted and executed by the language models.•System Message Setup: Depending on whether the agent is conﬁgured for single or multipleenvironments, a system message is formatted to provide the model with context about thetasks and environments.Interaction (Chat Method)The core functionality of each agent is encapsulated in its ability tointeract with users through a chat method. This involves:•Content Parsing: Input content is parsed and formatted to match the requirements ofthe respective API. This includes structuring user messages and any necessary contextualinformation.•Request Construction: The request payload is constructed, incorporating the systemmessage, chat history, and the newly parsed user input.•API Communication: The constructed request is sent to the appropriate API, whichgenerates a response. The agents handle API-speciﬁc constraints such as rate limits andresponse formats.•Response Handling: The response from the API is processed to extract any tool callssuggested by the model. These are then appended to the chat history, maintaining a coherentconversation state.18', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='f3f439b7-b2af-4ec7-8b42-7c33b5cce126', embedding=None, metadata={'page_label': '19', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Multi-Environment SupportFor agents conﬁgured to operate in multiple environments, additionallogic ensures that actions are correctly associated with their respective environments. This involvesmodifying action names and descriptions to reﬂect their environmental context and handling responsesaccordingly.Utilities and Shared FunctionsSeveral utility functions support the operation of these agents, facil-itating tasks such as content parsing, action prompt generation, and schema conversion. These sharedfunctions ensure consistency and reduce redundancy across the different agent implementations.B.2 Inter-agent Communication StrategiesIn this section we introduce the details of two multi-agent communications methods, which areintroduced in 6.1.Multi-agent Communication by FunctionalityThis setting involves two agents: a main agentprompted with the task description and a tool agent with the entire action space. The main agentgenerates the instruction for the next step and sends it to the tool agent. The tool agent chooses theproper action with parameters and a target environment, then feeds it back to the system.Multi-agent Communication by EnvironmentThis setting involves four agents in our benchmarksetting: a main agent prompted with the task description and three tool agents, each corresponding tothe environments of Android, Ubuntu, and Root, with the respective action spaces. The main agentgenerates the instruction for the next step and sends it to the tool agents. Each sub-environmentagent receives the message containing the instruction and environment observation information.The environment agents process the message using their specialized models and action schemas,performing the required actions within their environments.B.3 Agent PromptB.3.1 Single AgentPromptYou are a helpful assistant. Now you have to do a task as described below:{task_description}. And this is the description of each given environment:{env_description}. A unit operation you can perform is called action in a given envi-ronment. For each environment, you are given a limited action space as function calls:{action_descriptions}You may receive a screenshot of the current system. The interactive UI elements on thescreenshot are labeled with numeric tags starting from 1. For each step, You must state whatactions to take, what the parameters are, and you MUST provide in which environment toperform these actions. Your answer must be a least one function call. please do not output anyother information. You must make sure all function calls get their required parameters.B.3.2 Multi-Agent by FunctionalityMain Agent PromptYou are a helpful assistant. Now you have to do a task as described below:{task_description}. And this is the description of each given environment:{env_description}. A unit operation you can perform is called action in a given envi-ronment. For each environment, you are given a limited action space as function calls:{action_descriptions}You may receive a screenshot of the current system. The interactive UI elements on thescreenshot are labeled with numeric tags starting from 1. For each step, You must state whatactions to take, what the parameters are, and you MUST provide in which environment toperform these actions.19', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='39a2561f-60bb-445f-ba9c-3246d23e5bb0', embedding=None, metadata={'page_label': '20', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Tool Agent PromptYou are a helpful assistant in generating function calls. I will give you a detailed descriptionof what actions to take next, you should translate it into function calls. please do not outputany other information.B.3.3 Multi-Agent by EnvironmentMain Agent PromptYou are a main agent, and your goal is to plan and give instructions to sub-agents in eachenvironment to complete the ﬁnal task. Now you have to do a task as described below:{description}. The description of each given environment:{env_description}. Foreach step, you are required to provide high-level instructions detailing the next actions to betaken. Additionally, you must specify which sub-agent in the designated environment shouldexecute these instructions. If a sub-agent is not needed for a particular step, you may instructit to skip that step.Root Environment Agent PromptYou are a sub-agent responsible for the crab benchmark root environment. Your goal isto assist the main agent in completing the whole task:\"{description}\". You can onlycomplete the task or submit the result when the main agent tells you the whole task has beencompleted. Otherwise, you can only call SKIP.Sub-environment Agent PromptYou are a sub-agent responsible for the{environmentenvironment. The description ofthe{environment}environment is:{env_description}. Your goal is to assist the mainagent in completing the ﬁnal task by performing actions in the{environment}environ-ment according to the instructions from the main agent. The ﬁnal task is described be-low:{task_description}. A unit operation you can perform is called action in a givenenvironment. You can only execute action in the{environment}environment. For the{environment}environment, you are given a limited action space as function calls:{action_descriptions}The interactive UI elements on the screenshot are labeled with numeric tags starting from 1.For each step, You will receive an instruction telling you what you need to do next. Afteranalyzing the instruction you received and the current{environmentsystem, if you thinkyou don’t need to do anything in the current{environment}system, you should chooseSKIP action. Otherwise, you must state what actions to take, what the parameters are, and youMUST provide in which environment to perform these actions. Your answer must be functioncalls. Please do not output any other information. You must make sure all function calls gettheir required parameters.C Further Result AnalysisThis section further discusses our experimental results in detail. Section C.1 categorizes the results intothree types of tasks: Ubuntu, Android, and cross-platform, and provides further analysis. Section C.2examines three speciﬁc tasks and analyzes the performance of different agent settings on each.C.1 Result by PlatformsTable 4, 5 and 6 show the experiment results on Ubuntu Tasks, Android Tasks, and cross-platformTasks, respectively.We ﬁnd that certain models demonstrate a distinct preference or better alignment with speciﬁcplatforms. The GPT-4o, Gemini, and Claude models, for instance, show notably better outcomes on20', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='b58300d9-5e23-42f3-b84b-56ec4f28651d', embedding=None, metadata={'page_label': '21', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 4:Evaluation results on Ubuntu tasks.TheModelcolumn identiﬁes the backend maskedlanguage models (MLMs) used. TheStructurecolumn describes the conﬁguration of the agent system:Singlemeanssingle agent;By Funcismulti-agent by functionality;By Envindicatesmulti-agent byenvironment. We provide traditional metric ofSuccess Rate(SR) alongside newly introduced metrics:Completion Ratio(CR),Execution Efﬁciency(EE), andCost Efﬁciency(CE). Note that Gemini 1.5Pro has an invalid CE because the Gemini API does not support retrieving token counts at the starttime of experiments. TheTermination Reasonshows the ratio of reasons why the agent stops when itdoes not complete the task.False Completion(FC) indicates that the agent believes it has completedthe task, but it actually has not;Reach Step Limit(RSL) means the agent has reached the step limitbut has not completed the task;Invalid Action(IA) refers to the agent producing outputs that donot follow instructions, which may include invalid formats, nonexistent actions, or invalid actionparameters.Agent systemMetricsTermination ReasonModel StructureSR(%)\\x00CR(%)\\x00EE(%)\\x00CE(%)\\x00FC(%) RSL(%) IA(%)GPT-4OSingle10.3426.64 2.72 4.68✓10\\x0045.17 60.34 24.14GPT-4OBy Func6.9021.90 2.07 3.86✓10\\x0046.90 60.34 25.86GPT-4OBy Env8.6220.60 2.06 2.01✓10\\x0043.45 48.28 39.66GPT-4 TURBOSingle12.0728.36 3.82 8.79✓10\\x0041.72 63.79 22.41GPT-4 TURBOBy Func10.3424.45 3.10 4.74✓10\\x0048.62 34.48 46.55GEMINI1.5 PROSingle1.727.61 0.54\\\\0.00 46.55 51.72GEMINI1.5 PROBy Func1.723.30 0.30\\\\0.00 20.69 77.59CLAUDE3OPUSSingle1.729.54 1.41 3.42✓10\\x0045.17 56.90 36.21CLAUDE3OPUSBy Func1.726.75 0.65 2.81✓10\\x00427.59 31.03 39.66Table 5:Evaluation results on Android tasks.Agent systemMetricsTermination ReasonModel StructureSR(%)\\x00CR(%)\\x00EE(%)\\x00CE(%)\\x00FC(%) RSL(%) IA(%)GPT-4OSingle24.1447.91 5.12 7.17✓10\\x00413.79 58.62 3.45GPT-4OBy Func24.1448.74 5.779.19✓10\\x00424.14 37.93 13.79GPT-4OBy Env27.5953.34 5.934.58✓10\\x00413.79 44.83 13.79GPT-4 TURBOSingle10.3430.53 2.84 3.36✓10\\x00420.69 62.07 6.90GPT-4 TURBOBy Func20.6937.01 4.32 5.92✓10\\x00413.79 51.72 13.79GEMINI1.5 PROSingle17.2434.52 4.09\\\\10.34 65.52 6.90GEMINI1.5 PROBy Func17.2435.99 3.88\\\\31.03 41.38 10.34CLAUDE3OPUSSingle17.2443.62 5.30 7.78✓10\\x00413.79 51.72 17.24CLAUDE3OPUSBy Func13.7942.30 4.20 5.07✓10\\x00444.83 31.03 10.34Android platforms. This suggests potential optimizations or intrinsic features within these modelsthat cater effectively to the Android environment’s requirements. Conversely, the GPT-4 Turbo modelexhibits superior performance on Ubuntu tasks, hinting at possible architectural or training aspectsthat are better suited for that speciﬁc environment.In multi-agent system organized by environment, consistently yields better results in both Androidand cross-platform tasks. This conﬁguration appears to enhance the agents’ ability to manage andadapt to diverse tasks more effectively, leveraging environmental speciﬁcs to optimize performance.This suggests that employing multiple agents that are either specialized or speciﬁcally conﬁgured tooperate within the same environment can signiﬁcantly improve task handling and overall adaptability.Cross-platform tasks present a greater challenge for all models, as evidenced by lower SuccessRates and Completion Ratios. These tasks, which necessitate functionality across different operatingsystems or platforms, demand a broader capability range and more sophisticated agent coordination.The importance of CR is especially critical in such environments, where it serves as a more reliablemetric for distinguishing between agent models than SR. Given the presence of all Gemini andClaude agents’ SR is 0.0, indicating that Completion Ratio more effectively captures an agent model’scapability, thereby better reﬂecting its robustness and adaptability to complex requirements.21', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='c9968447-f8e9-476b-bc15-9d25eb7d8389', embedding=None, metadata={'page_label': '22', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 6:Evaluation results on cross-platform tasks.Agent systemMetricsTermination ReasonModel StructureSR(%)\\x00CR(%)\\x00EE(%)\\x00CE(%)\\x00FC(%) RSL(%) IA(%)GPT-4OSingle7.6945.534.54 3.57✓10\\x0040.00 53.85 38.46GPT-4OBy Func15.3843.41 3.19 2.25✓10\\x0047.69 61.54 15.38GPT-4OBy Env7.6948.613.69 1.68✓10\\x00415.38 61.54 15.38GPT-4 TURBOSingle7.6947.84 4.31 2.89✓10\\x0040.00 69.23 23.08GPT-4 TURBOBy Func7.6939.05 3.73 2.51✓10\\x00415.38 46.15 30.77GEMINI1.5 PROSingle0.0021.25 1.49\\\\0.00 69.23 30.77GEMINI1.5 PROBy Func0.0016.70 1.52\\\\7.69 69.23 23.08CLAUDE3OPUSSingle0.0024.69 2.33 1.62✓10\\x0040.00 38.46 61.54CLAUDE3OPUSBy Func0.0020.07 2.32 1.49✓10\\x0040.00 38.46 61.54Furthermore, analyzing the reasons for task termination offers additional insights into the operationalchallenges these models encounter. False Completion is notably prevalent in Android tasks. ReachStep Limit remains the most frequent cause of termination, particularly in cross-platform tasks. TheClaude model exhibits a signiﬁcantly high Invalid Action ratio in cross-platform tasks, indicating itsdifﬁculties in managing multi-environment scenarios effectively.Overall, these ﬁndings underscore the necessity of selecting the appropriate agent model and con-ﬁguration based on speciﬁc platform and task needs. The variability in model performance acrossdifferent setups also highlights the ongoing need for development and reﬁnement of multi-agentsystems to enhance their versatility and efﬁcacy in increasingly diverse and complex operationalenvironments.C.2 Case StudyTo better understand how different agents perform the same task and exhibit varied properties, wepresent visual results along with detailed metrics and logs for three cases by platform. The screenshotsillustrate the progress of agents executing tasks according to speciﬁc natural language instructions.C.2.1 Cross-platform TaskTask: Open the \"Tasks\" app on an Android device, check the ﬁrst incomplete task, and thenexecute it as described.The ﬁrst task, found incomplete in the \"Tasks\" app, involvesswitching thesystem to dark mode in Ubuntu via the \"Settings\" application.This task exempliﬁes message passing across different environments, where the \"incomplete task\"serves as the critical information that the agent must relay and apply in the Ubuntu setting. Thesetwo phases—retrieving the task details via the phone and executing the task on a computer—areinseparably linked and cannot be treated as distinct tasks. The agent can only proceed to the secondstage after successfully acquiring information from the ﬁrst.In this task, GPT-4o (single agent), GPT-4 Turbo (single agent), and GPT-4 Turbo (multi-agent byfunctionality) all successfully complete the task using the minimal steps necessary to locate and exe-cute the task, demonstrating their efﬁciency in managing multiple environments simultaneously. Onthe other hand, both GPT-4o (multi-agent by functionality) and GPT-4o (multi-agent by environment)also perform commendably, completing the task up until the ﬁnal step. However, after incorrectlyperforming the last step, they both erroneously conclude the task is completed and exit. This indicatesa communication breakdown, where the sub-agents misinterpret the instructions from the main agent.The remaining four agents fail to complete the task. Agents equipped with the Gemini model donot even manage to open the \"Tasks\" app within the allocated step limit, whereas agents with theClaude model quickly open the \"Tasks\" app to complete the ﬁrst step but fail at the task execution.The performance disparity between single-agent and multi-agent conﬁgurations in both the Geminiand Claude models highlights the variance in capability across different models and devices.22', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='c64593ab-6553-464e-a7e7-f0100359da61', embedding=None, metadata={'page_label': '23', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='C.2.2 Ubuntu TaskTask: Create a new directory \"/home/crab/assets_copy\" and copy all ﬁles with the speciﬁed\"txt\" extension from \"/home/crab/assets\" to the directory \"/home/crab/assets_copy\".This task can be approached through multiple methods. An agent may opt for a straightforwardstrategy ﬁrst using thesearch_applicationcommand to ﬁnd the Terminal, then using Linuxcommands to create the directory and copy the necessary ﬁles. Alternatively, the agent could employa GUI-based approach, manually creating the folder and selecting ﬁles through actions likeclickandright_click. We evaluate various agent systems in a single-agent setting for this task. Asillustrated in Table 7–10 , both GPT-4o and GPT-4 Turbo from OpenAI successfully interpret the taskinstructions and employ a simpler solution using Terminal commands. These agents also demonstratesuperior capability in understanding the UI, selecting the correct commands, and accurately using theTerminal application to fulﬁll the task requirements.Conversely, the Gemini and Claude agents, despite attempting to solve the task with Terminal,ultimately fail in different ways. Both agents struggle with precise clicking and selecting the correcticons for the intended actions, even though they share the same visual prompting mechanism asGPT-4o and GPT-4 Turbo. For instance, the Claude agent mistakenly opens the Ubuntu DesktopGuide instead of the Terminal and continues executing commands in the wrong application withoutrealizing the error. The Gemini agent, on the other hand, unexpectedly opens the Firefox browserbefore correctly navigating to the Terminal but still interacts incorrectly with unrelated applicationsand icons. Unlike Claude, Gemini does not type in commands in the wrong applications but persistsin exploring alternative methods using the Files application’s UI. Despite taking signiﬁcantly moresteps than the GPT-4o and GPT-4 Turbo agents, neither the Claude nor the Gemini agents achieve thetask’s goC.2.3 Android TaskTask: In Android, using the \"Contacts\" app, ﬁnd the email of the contact named John Lauphin,then using the \"Gmail\" app, send an email to that contact with the subject \"Hello John.\"This task consists of sub-tasks across two different applications. Agents must sequentially open thetwo apps, retrieve the email address from the ﬁrst app, and use it in the second app to send an email.This straightforward yet formal task can be completed using various methods. Agents may need tolocate the contact in the Contacts app and then use the retrieved email address to send a message. Wereports the performance of agents in a multi-agent setting for this challenging task. Following is thedetails of agents in operating the task.GPT-4o multi-agent by functionalityIn steps 1-11, the agent tries to open the Contacts app butmistakenly opens Google Assistant multiple times. In steps 12-14, the agent successfully enters theContacts app and ﬁnds the contact information. The agent then returns to the home page, and theprocess is terminated due to the limitation of operation steps.GPT-4 Turbo multi-agent by functionalityIn steps 1-2, the agent tries to open the Contacts appbut mistakenly opens Google Messages. In steps 3-5, the agent opens the Contacts app and obtains thecorresponding information. In steps 6-14, the agent repeatedly opens Google Chrome and Messagesapps, failing to ﬁnd the Gmail app as planned.Gemini 1.5 Pro multi-agent by functionalityIn steps 1-2, the agent ﬁnds the Contacts app andenters it. However, the agent misunderstands the instruction, gets lost in creating a new contact withthe given name, and cannot obtain the corresponding information.Claude 3 Opus multi-agent by functionalityIn steps 1-7, the agent tries to open the Contactsapp but mistakenly opens Google Messages multiple times. In steps 7-11, the agent tries to open theContacts app but mistakenly opens Google Assistant. In steps 12-14, the agent successfully entersthe Contacts app and ﬁnds the contact information. The agent then returns to the home page, plans toopen the Gmail app, and the process is terminated due to the limitation of operation steps.GPT-4o multi-agent by environmentIn steps 1-7, the agent plans to open the Contacts app, butthe operation fails due to an error in opening the app drawer, which prevents the agent from ﬁnding23', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='f89da38e-bdb9-46c8-8d5e-813fba34247e', embedding=None, metadata={'page_label': '24', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='and tapping the Contacts app. In steps 8-11, the agent successfully enters the Contacts app andobtains the information. In steps 12-14, the agent opens the Gmail app, navigates to the sending page,and tries to input the retrieved email address as the recipient.AnalysisFor the agents which are organized by functionality, Gemini 1.5 Pro struggles to completethe ﬁrst operation. Although it recognizes and opens the Contacts app as instructed, it fails to proceedfurther. In contrast, Claude 3 Opus and GPT-4o successfully obtain the necessary information. Inthe initial phase, the multiple agents agree that opening the Contacts app is the ﬁrst step. However,they often fail to ﬁnd the correct position to tap, frequently opening incorrect apps such as GoogleAssistant and Messages. Once the agents do open the correct app, they usually ﬁnd the email addressof the contact quickly. Even when agents plan to go back home and open the Gmail app to send themessage, due to the limitation of operations, the system ended. As shown in steps 3-5, GPT-4 Turboquickly ﬁnishes the corresponding task after opening the correct app. However, similar to GPT-4o,GPT4-Turbo agents get stuck as they can not open the correct apps in the following steps. Besides,GPT-4o (multi-agent by environment) overcomes the issue encountered by GPT-4o (multi-agent byfunctionality). Even affected by not being able to access the app drawer, the system could still ﬁndand copy the corresponding information and change to the Gmail app for further operations.\\n24', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='534d07c8-1e69-4716-a3bf-1ee7e18dd83c', embedding=None, metadata={'page_label': '25', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 7:Ubuntu task case with Gemini (Single):Create a new directory\"/home/crab/assets_copy\" and copy all ﬁles with the speciﬁed \"txt\" exten-sion from \"/home/crab/assets\" to the directory \"/home/crab/assets_copy\".StepAgent Observation and Action\\n0\\nclick(5)\\n1\\nclick(1)Continued on next page\\n25', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='c99c0006-2547-4ea2-80c2-177765aa8f46', embedding=None, metadata={'page_label': '26', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 7 continued from previous page\\n2\\nclick(4)\\n3\\nsearch_application(\"terminal\")\\n4\\nclick(9)Continued on next page26', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='5a72e49c-d227-4531-9656-0097ad244727', embedding=None, metadata={'page_label': '27', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 7 continued from previous page\\n5\\nclick(3)\\n6\\nclick(13)\\n7\\nsearch_application(\"terminal\")Continued on next page27', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='2cbad613-961e-4f79-a411-468819abe9b7', embedding=None, metadata={'page_label': '28', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 7 continued from previous page\\n8\\nclick(10)\\n9\\nclick(6)10-14The agent is stuck at this stage and keeps clicking useless elements.Completed Nodes / Total Nodes:0/2Termination Reason: Reach Step Limit\\n28', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='14657c3d-ca38-4995-8f64-bc93976d2af9', embedding=None, metadata={'page_label': '29', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 8:Ubuntu task case with Claude (Single):Create a new directory\"/home/crab/assets_copy\" and copy all ﬁles with the speciﬁed \"txt\" exten-sion from \"/home/crab/assets\" to the directory \"/home/crab/assets_copy\".StepAgent Observation and Action\\n0\\nsearch_application(\"terminal\")\\n1\\nclick(13)Continued on next page\\n29', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='e7afe781-e08d-4e87-8534-93c7881f793e', embedding=None, metadata={'page_label': '30', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 8 continued from previous page\\n2\\nwrite_text(\"mkdir /home/crab/assets_copy\")\\n3\\nkey_press(\"enter\")\\n4\\nwrite_text(\"cd /home/crab/assets\")Continued on next page30', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='77b658be-67c0-4ff7-bbd1-ca28193497fe', embedding=None, metadata={'page_label': '31', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 8 continued from previous page\\n5\\nkey_press(\"enter\")\\n6\\nwrite_text(\"mkdir assets_copy\")7-14The agent is stuck at this stage and keeps pressing keys.Completed Nodes / Total Nodes:0/2Termination Reason: Reach Step Limit\\n31', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='079c08e4-a6b3-46b4-96ec-12cb16e5f43e', embedding=None, metadata={'page_label': '32', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 9:Ubuntu task case with GPT-4o (Single):Create a new di-rectory \"/home/crab/assets_copy\" and copy all ﬁles with the speciﬁed\"txt\" extension from \"/home/crab/assets\" to the directory \"/home/crab/assets_copy\".StepAgent Observation and Action\\n0\\nclick(2)\\n1-2\\nclick(2)right_click(2)Continued on next page\\n32', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='d4942481-2292-4b2e-b47c-9d371aa8de9b', embedding=None, metadata={'page_label': '33', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 9 continued from previous page\\n3\\nclick(6)\\n4\\nwrite_text(\"mkdir -p /home/crab/assets_copy\")\\n5-6\\nContinued on next page33', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='62cc9345-430b-4423-9c6d-6ff2b5e2be38', embedding=None, metadata={'page_label': '34', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 9 continued from previous page5-6key_press(\"enter\")write_text(\"cp /home/crab/assets/*.txt /home/crab/assets_copy/\")\\n7\\nkey_press(\"enter\")Completed Nodes / Total Nodes:2/2Termination Reason: Success\\n34', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='944c046e-7b97-4e2c-b054-90093ab1b7c9', embedding=None, metadata={'page_label': '35', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 10:Ubuntu task case with GPT-4 Turbo (Single):Create a newdirectory \"/home/crab/assets_copy\" and copy all ﬁles with the speciﬁed\"txt\" extension from \"/home/crab/assets\" to the directory \"/home/crab/assets_copy\".StepAgent Observation and Action\\n0\\nsearch_application(\"terminal\")\\n1\\nclick(20)Continued on next page\\n35', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='e386568c-9a95-4d0a-b960-1f8088b6f924', embedding=None, metadata={'page_label': '36', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 10 continued from previous page\\n2\\nwrite_text(\"mkdir -p /home/crab/assets_copy\")\\n3\\nkey_press(\"enter\")\\n4\\nwrite_text(\"cp /home/crab/assets/*.txt /home/crab/assets_copy\")Continued on next page36', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
       " Document(id_='8027660b-e43a-4c40-a498-a8eb1c7e40b4', embedding=None, metadata={'page_label': '37', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={}, text='Table 10 continued from previous page\\n5\\nkey_press(\"enter\")Completed Nodes / Total Nodes:2/2Termination Reason: Success\\n37', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents # see documents - pages in the pdf/s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = VectorStoreIndex.from_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = index.as_query_engine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = query_engine.query(\"What does the crab framework demonstrate? \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Response(response='The Crab Framework demonstrates a benchmarking workflow for a multi-agent system. It involves initializing a task by assigning instructions to the main agent and a graph evaluator, progressing through a cycle where the main agent observes, plans, and instructs sub-agents to execute actions within their respective environments. The graph evaluator continuously monitors task status, updates, and outputs task completion metrics throughout the workflow.', source_nodes=[NodeWithScore(node=TextNode(id_='11a931ca-47e2-4207-ba60-faa7753e4af2', embedding=None, metadata={'page_label': '2', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='30c6f75d-f221-42bc-bd57-592d65731d3f', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '2', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, hash='4ed16016668e22a30e1b02f2e06c1b80a38715a8388402bb348e3f8091982786')}, text='Crab FrameworkEnvironment\\nAgent SystemGraph Evaluator\\nBenchmarkDatasetUbuntu TaskInstructionEvaluatorAndroid TaskInstructionEvaluatorEnvironment\\nCross-platform TaskInstructionEvaluator\\nAndroidAgent\\nUbuntuAgent\\nMain Agent\\nInitialize Task①Observe②Plan & Instruct ③Take Action \\n④Update Evaluator CompletedActiveInactive⑤Output Metrics \\nFigure 1:Architecture of the Crab Framework demonstrating a benchmarking workﬂow for amulti-agent system.A task is initialized by assigning instructions to the main agent and a graphevaluator inside the benchmark system. The workﬂow progresses through a cycle where the mainagent observes, plans, and instructs the sub-agents, who then execute actions within their respectiveenvironments. The graph evaluator monitors the status of tasks within the environments, continuouslyupdating and outputting the task completion metrics throughout the workﬂow.typically evaluated on a single platform, either Web, Android, or Desktop OS [38, 50, 49]. However,the practical applications usually involve tasks that span multiple platforms. For example, using asmartphone to take a photo and sending it to a desktop for editing with a graphics editor is a commonreal-world task across multiple platforms. Third, existing evaluation methods are generally eithergoal-based or trajectory-based [38, 50]. Goal-based methods typically employ a coarse-grainedbinary reward, solely evaluating whether the ﬁnal system state aligns with the task’s objectives. Incontrast, trajectory-based methods can offer more nuanced metrics by assessing the agent’s actionsagainst a gold trajectory yet ignore the possibility of multiple valid pathways to complete a task,making the evaluation results less fair. Lastly, task creation within these complex systems are notstatic and extensible with ﬁxed templates [40, 49], which limits the diversity and scope of tasks.We propose a benchmark that closely mirrors real-world situations and an evaluation method thatmore accurately reﬂects an agent’s performance on complex tasks. To this end, we introduceCrab, a novelCRoss-environmentAgentBenchmark framework.Crabprovides a comprehensiveframework for evaluating cross-environment tasks in interactive environments, where the agentneeds to operate simultaneously across various devices and platforms, adapting to varied systemconditions to complete tasks efﬁciently. To the best of our knowledge,Crabis the ﬁrst autonomousagent benchmark framework that incorporates thecross-environment tasks. Moreover, we proposea novel evaluation method calledgraph evaluator. Unlike traditional goal-based and trajectory-based evaluation, our graph evaluator checks the intermediate procedures of completing a task bydecomposing the task into multiple sub-goals. Each sub-goal is assigned a judge function to verify itscompleteness, and each is considered a node in the graph evaluator. The graph structure describes thesequential and parallel relationships between the sub-goals. Therefore, it offers ﬁne-grained metricssimilar to trajectory-based evaluations while accommodating multiple valid pathways to a solution,making it more suitable for evaluating tasks that involve various correct approaches. To solve theincreasing complexity in cross-environment task construction. We also propose a highly extensiblegraph-based task construction method calledsub-task composition. Combining multiple sub-tasksin a graph with task targets allows for efﬁcient construction of various cross-environment tasks withcorresponding graph evaluators. Table 1 comparesCrabwith existing frameworks.Based onCrabframework, we develop a benchmarkCrab Benchmark-v0with two cooperatedenvironments that include an Android emulator and an Ubuntu desktop virtual machine. We have de-2', mimetype='text/plain', start_char_idx=0, end_char_idx=3747, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.8360984645114197), NodeWithScore(node=TextNode(id_='33c81820-fc8b-47f6-9da1-182bf9455f94', embedding=None, metadata={'page_label': '14', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='25507fd8-5420-4ccc-b80c-bb9a0a4d7f96', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '14', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, hash='45a7340a2bd718b03d485c9a73e7b83eaa4c4132ca4ddc86412f2b7d65b01f92')}, text='A Benchmark DetailThe framework code and task dataset are open source7under Apache-2.0 licences. Section A.2describes the our experiment settings in detail. Section A.3 describes the speciﬁc format deﬁned inour framework that ease data extension and how to use them. We provides a detailed document tosetup experiment environments and reproduce our results.8Fig. 3 shows the structure of modulesinsideCrab Benchmark-v0.Crab Benchmark v0Android Environment\\nUbuntu EnvironmentTasksName: \"android\"Description: \"A Google Pixel smartphoneruns on the Android operating system...\"Observation SpaceScreenshotPrompt SpaceVisual PromptAction SpaceTapPress KeyWrite TextSwipeOpen App Drawer...\\nName: \"ubuntu\"Description: \"An Ubuntu 22.04 Linux desktopoperating system...\"Observation SpaceScreenshotPrompt SpaceVisual PromptAction SpaceClickPress KeyWrite TextRight ClickSeach Application...Sub-tasksSub-task Template 1Sub-task 1Evaluator Generator 1UbuntuSub-task Template 2Sub-task 2Evaluator Generator 2UbuntuSub-task Template 3Sub-task 3Evaluator Generator 3Android...Cross-platform TaskDescriptionAttributesGraph EvaluatorAndroid TaskDescriptionAttributesGraph EvaluatorUbuntu TaskDescriptionAttributesGraph Evaluator...\\nFigure 3:Module Structure ofCrab Benchmark-v0.The benchmark is divided into two primarysections: the left section, highlighted with warm hues, features two environments, while the rightsection, accentuated with cool hues, outlines various tasks. Each environment is deﬁned by attributesincluding name, description, observation space, prompt method, and action space. Blocks markedin red denote actions. As for the tasks, they are composed of multiple sub-tasks and formulatedby combine multiple evaluator sub-graphs derived from the sub-task evaluator generators. Arrowsillustrate the compositional relationships between tasks and sub-tasks.A.1 Framework DesignCraboffers a modular and extensible framework for evaluating agent performance in diverse tasks.At the heart of the framework lies theaction, a unit operation representing the fundamental operationwithin the benchmark. Theactionis essentially an executable Python function that can be deﬁnedwith explicit typed parameters and a clear description.actionsserve not only as building blocks butalso as interfaces through which agents interact with the environment. Theevaluatoris a specializedactionrestricted to returning boolean values, signifying the success or failure of an agent’s task. Itenhances theactionsby analyzing the state of the environment and the sequence ofactionsexecutedby the agent, providing a decisive metric of task accomplishment. Additionally, multipleevaluatorscan be interconnected to form a graph evaluator for complex tasks (Sec. 4.2).Thebenchmarkis a key deﬁnition in the framework. A benchmark includes multipleenvironmentsand cross-environmenttasks. Theenvironmentis formed by an action space and an observationspace, which are both deﬁned by a list ofactions, and other essential parameters necessary for its7https://github.com/camel-ai/crab8https://github.com/camel-ai/crab/blob/main/crab-benchmark-v0/README.md14', mimetype='text/plain', start_char_idx=0, end_char_idx=3119, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.8291105055260253)], metadata={'11a931ca-47e2-4207-ba60-faa7753e4af2': {'page_label': '2', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}, '33c81820-fc8b-47f6-9da1-182bf9455f94': {'page_label': '14', 'file_name': 'research.pdf', 'file_path': '/Users/barry/Documents/code/llamaindex-101/data/research.pdf', 'file_type': 'application/pdf', 'file_size': 34686809, 'creation_date': '2024-07-03', 'last_modified_date': '2024-07-03'}})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "index.storage_context.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import StorageContext, load_index_from_storage\n",
    "\n",
    "storage_context = StorageContext.from_defaults(persist_dir=\"./storage\")\n",
    "index = load_index_from_storage(storage_context=storage_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import ServiceContext, set_global_service_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting llama-index-llms-ollama\n",
      "  Downloading llama_index_llms_ollama-0.1.5-py3-none-any.whl.metadata (585 bytes)\n",
      "Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in ./.venv/lib/python3.12/site-packages (from llama-index-llms-ollama) (0.10.51)\n",
      "Requirement already satisfied: PyYAML>=6.0.1 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy>=1.4.49 in ./.venv/lib/python3.12/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.0.31)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.9.5)\n",
      "Requirement already satisfied: dataclasses-json in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.6.7)\n",
      "Requirement already satisfied: deprecated>=1.2.9.3 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.2.14)\n",
      "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.0.8)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2024.6.1)\n",
      "Requirement already satisfied: httpx in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.27.0)\n",
      "Requirement already satisfied: llama-cloud<0.0.7,>=0.0.6 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.0.6)\n",
      "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.6.0)\n",
      "Requirement already satisfied: networkx>=3.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.3)\n",
      "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.8.1)\n",
      "Requirement already satisfied: numpy<2.0.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.26.4)\n",
      "Requirement already satisfied: openai>=1.1.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.35.9)\n",
      "Requirement already satisfied: pandas in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.2.2)\n",
      "Requirement already satisfied: pillow>=9.0.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (10.4.0)\n",
      "Requirement already satisfied: requests>=2.31.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.32.3)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (8.4.2)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.7.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (4.12.2)\n",
      "Requirement already satisfied: typing-inspect>=0.8.0 in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.9.0)\n",
      "Requirement already satisfied: wrapt in ./.venv/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.16.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in ./.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in ./.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in ./.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in ./.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in ./.venv/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.9.4)\n",
      "Requirement already satisfied: pydantic>=1.10 in ./.venv/lib/python3.12/site-packages (from llama-cloud<0.0.7,>=0.0.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.8.0)\n",
      "Requirement already satisfied: anyio in ./.venv/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (4.4.0)\n",
      "Requirement already satisfied: certifi in ./.venv/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2024.6.2)\n",
      "Requirement already satisfied: httpcore==1.* in ./.venv/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.0.5)\n",
      "Requirement already satisfied: idna in ./.venv/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.7)\n",
      "Requirement already satisfied: sniffio in ./.venv/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in ./.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.14.0)\n",
      "Requirement already satisfied: click in ./.venv/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (8.1.7)\n",
      "Requirement already satisfied: joblib in ./.venv/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.4.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in ./.venv/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2024.5.15)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in ./.venv/lib/python3.12/site-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in ./.venv/lib/python3.12/site-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.12/site-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.2.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in ./.venv/lib/python3.12/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.0.3)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in ./.venv/lib/python3.12/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.0.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in ./.venv/lib/python3.12/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (3.21.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2024.1)\n",
      "Requirement already satisfied: packaging>=17.0 in ./.venv/lib/python3.12/site-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (24.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in ./.venv/lib/python3.12/site-packages (from pydantic>=1.10->llama-cloud<0.0.7,>=0.0.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.0 in ./.venv/lib/python3.12/site-packages (from pydantic>=1.10->llama-cloud<0.0.7,>=0.0.6->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (2.20.0)\n",
      "Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-llms-ollama) (1.16.0)\n",
      "Downloading llama_index_llms_ollama-0.1.5-py3-none-any.whl (3.6 kB)\n",
      "Installing collected packages: llama-index-llms-ollama\n",
      "Successfully installed llama-index-llms-ollama-0.1.5\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install llama-index-llms-ollama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.llms.ollama import Ollama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = Ollama(model=\"llama3\", request_timeout=120.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ds/4wljlmm12jsd9ng511ttmn6h0000gn/T/ipykernel_46024/1651316493.py:5: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
      "  service_context = ServiceContext.from_defaults(llm=llm, chunk_size=800, chunk_overlap=20)\n"
     ]
    }
   ],
   "source": [
    "# configure service context\n",
    "service_context = ServiceContext.from_defaults(llm=llm, chunk_size=800, chunk_overlap=20)\n",
    "\n",
    "# set_global_service_context(service_context)\n",
    "index = VectorStoreIndex.from_documents(documents, service_context=service_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Crab Framework demonstrates a benchmarking workflow for a multi-agent system. It involves initializing a task by assigning instructions to the main agent and a graph evaluator, progressing through a cycle where the main agent observes, plans, and instructs sub-agents to execute actions within their respective environments. The graph evaluator continuously monitors task status, updates, and outputs task completion metrics throughout the workflow.'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.response"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}