From 4c8837565b5ab8d71b6b9ca55b97c8116828bbaf Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 16:22:39 -0500 Subject: [PATCH 01/32] set use_docker to default to true --- autogen/agentchat/conversable_agent.py | 15 +++++- autogen/code_utils.py | 65 +++++++++++++++++--------- setup.py | 1 + website/docs/FAQ.md | 2 +- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 08f71160057..f672a51894b 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -8,7 +8,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Tuple, Type, TypeVar, Union from .. import OpenAIWrapper -from ..code_utils import DEFAULT_MODEL, UNKNOWN, content_str, execute_code, extract_code, infer_lang +from ..code_utils import DEFAULT_MODEL, UNKNOWN, content_str, execute_code, extract_code, infer_lang, is_docker_running, in_docker_container from ..function_utils import get_function_schema, load_basemodels_if_needed, serialize_to_str from .agent import Agent from .._pydantic import model_dump @@ -129,6 +129,19 @@ def __init__( self._code_execution_config: Union[Dict, Literal[False]] = ( {} if code_execution_config is None else code_execution_config ) + + if isinstance(self._code_execution_config, Dict): + if "use_docker" in self._code_execution_config: + use_docker = self._code_execution_config["use_docker"] + else: + use_docker = True + if use_docker is not None: + inside_docker = in_docker_container() + docker_installed_and_running = is_docker_running() + if use_docker and not inside_docker and not docker_installed_and_running: + raise RuntimeError("Docker is not running, please make sure docker is running (advised approach for code execution) or set \"use_docker\":False.") + if not use_docker: + logger.warning("use_docker was set to False. Any code execution will be run natively but we strongly advise to set \"use_docker\":True. Set \"use_docker\":None to silence this message.") self.human_input_mode = human_input_mode self._max_consecutive_auto_reply = ( max_consecutive_auto_reply if max_consecutive_auto_reply is not None else self.MAX_CONSECUTIVE_AUTO_REPLY diff --git a/autogen/code_utils.py b/autogen/code_utils.py index dbb8637162d..4feb337ab0e 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -223,13 +223,35 @@ def _cmd(lang): return "powershell" raise NotImplementedError(f"{lang} not recognized in code execution") +def is_docker_running(): + """Check if docker is running. + + Returns: + bool: True if docker is running; False otherwise. + """ + if docker is None: + return False + try: + client = docker.from_env() + client.ping() + return True + except docker.errors.DockerException: + return False + +def in_docker_container(): + """Check if the code is running in a docker container. + + Returns: + bool: True if the code is running in a docker container; False otherwise. + """ + return os.path.exists("/.dockerenv") def execute_code( code: Optional[str] = None, timeout: Optional[int] = None, filename: Optional[str] = None, work_dir: Optional[str] = None, - use_docker: Optional[Union[List[str], str, bool]] = None, + use_docker: Union[List[str], str, bool] = True, lang: Optional[str] = "python", ) -> Tuple[int, str, str]: """Execute code in a docker container. @@ -249,15 +271,15 @@ def execute_code( If None, a default working directory will be used. The default working directory is the "extensions" directory under "path_to_autogen". - use_docker (Optional, list, str or bool): The docker image to use for code execution. + use_docker (list, str or bool): The docker image to use for code execution. If a list or a str of image name(s) is provided, the code will be executed in a docker container with the first image successfully pulled. If None, False or empty, the code will be executed in the current environment. - Default is None, which will be converted into an empty list when docker package is available. + Default is True. Expected behaviour: - - If `use_docker` is explicitly set to True and the docker package is available, the code will run in a Docker container. - - If `use_docker` is explicitly set to True but the Docker package is missing, an error will be raised. - - If `use_docker` is not set (i.e., left default to None) and the Docker package is not available, a warning will be displayed, but the code will run natively. + - If `use_docker` is not set (i.e. left default to True) or is explicitly set to True and the docker package is available, the code will run in a Docker container. + - If `use_docker` is not set (i.e. left default to True) or is explicitly set to True but the Docker package is missing, an error will be raised. + - If `use_docker` is explicitly set to False, a warning will be displayed, but the code will run natively. If the code is executed in the current environment, the code must be trusted. lang (Optional, str): The language of the code. Default is "python". @@ -272,18 +294,16 @@ def execute_code( logger.error(error_msg) raise AssertionError(error_msg) - # Warn if use_docker was unspecified (or None), and cannot be provided (the default). - # In this case the current behavior is to fall back to run natively, but this behavior - # is subject to change. - if use_docker is None: - if docker is None: - use_docker = False - logger.warning( - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change" - ) - else: - # Default to true - use_docker = True + # # Warn if use_docker was set to False + # # In this case the current behavior is to fall back to run natively, but this behavior + # # is subject to change. + running_inside_docker = in_docker_container() + docker_running = is_docker_running() + + if use_docker is not None and not use_docker and not running_inside_docker: + logger.warning( + "execute_code was called with use_docker set to False. Code will be run natively but we strongly advise to set use_docker=True. Set use_docker=None to silence this message." + ) timeout = timeout or DEFAULT_TIMEOUT original_filename = filename @@ -301,9 +321,8 @@ def execute_code( if code is not None: with open(filepath, "w", encoding="utf-8") as fout: fout.write(code) - # check if already running in a docker container - in_docker_container = os.path.exists("/.dockerenv") - if not use_docker or in_docker_container: + + if not use_docker or running_inside_docker: # already running in a docker container cmd = [ sys.executable if lang.startswith("python") else _cmd(lang), @@ -347,7 +366,11 @@ def execute_code( return result.returncode, logs, None # create a docker client + if use_docker and not docker_running: + raise RuntimeError("Docker package is missing or docker is not running. Please make sure docker is running or set use_docker=False.") + client = docker.from_env() + image_list = ( ["python:3-alpine", "python:3", "python:3-windowsservercore"] if use_docker is True diff --git a/setup.py b/setup.py index ea8f83be279..1789fdb4fef 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ "python-dotenv", "tiktoken", "pydantic>=1.10,<3", # could be both V1 and V2 + "docker", ] setuptools.setup( diff --git a/website/docs/FAQ.md b/website/docs/FAQ.md index dcd4dab6f54..37b699027a5 100644 --- a/website/docs/FAQ.md +++ b/website/docs/FAQ.md @@ -72,7 +72,7 @@ The `AssistantAgent` doesn't save all the code by default, because there are cas We strongly recommend using docker to execute code. There are two ways to use docker: 1. Run autogen in a docker container. For example, when developing in GitHub codespace, the autogen runs in a docker container. -2. Run autogen outside of a docker, while perform code execution with a docker container. For this option, make sure the python package `docker` is installed. When it is not installed and `use_docker` is omitted in `code_execution_config`, the code will be executed locally (this behavior is subject to change in future). +2. Run autogen outside of a docker container, while performing code execution with a docker container. For this option, make sure the python package `docker` is installed. When it is not installed and `use_docker` is explicitly set to False in `code_execution_config`, the code will be executed locally (this behavior is subject to change in future). ### Enable Python 3 docker image From a21e800efc6d8a4a9483555bcaab3c1a5f04caaa Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 16:31:33 -0500 Subject: [PATCH 02/32] black formatting --- autogen/agentchat/conversable_agent.py | 19 ++++++++++++++++--- autogen/code_utils.py | 7 ++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index f672a51894b..012215b5a01 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -8,7 +8,16 @@ from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Tuple, Type, TypeVar, Union from .. import OpenAIWrapper -from ..code_utils import DEFAULT_MODEL, UNKNOWN, content_str, execute_code, extract_code, infer_lang, is_docker_running, in_docker_container +from ..code_utils import ( + DEFAULT_MODEL, + UNKNOWN, + content_str, + execute_code, + extract_code, + infer_lang, + is_docker_running, + in_docker_container, +) from ..function_utils import get_function_schema, load_basemodels_if_needed, serialize_to_str from .agent import Agent from .._pydantic import model_dump @@ -139,9 +148,13 @@ def __init__( inside_docker = in_docker_container() docker_installed_and_running = is_docker_running() if use_docker and not inside_docker and not docker_installed_and_running: - raise RuntimeError("Docker is not running, please make sure docker is running (advised approach for code execution) or set \"use_docker\":False.") + raise RuntimeError( + 'Docker is not running, please make sure docker is running (advised approach for code execution) or set "use_docker":False.' + ) if not use_docker: - logger.warning("use_docker was set to False. Any code execution will be run natively but we strongly advise to set \"use_docker\":True. Set \"use_docker\":None to silence this message.") + logger.warning( + 'use_docker was set to False. Any code execution will be run natively but we strongly advise to set "use_docker":True. Set "use_docker":None to silence this message.' + ) self.human_input_mode = human_input_mode self._max_consecutive_auto_reply = ( max_consecutive_auto_reply if max_consecutive_auto_reply is not None else self.MAX_CONSECUTIVE_AUTO_REPLY diff --git a/autogen/code_utils.py b/autogen/code_utils.py index 4feb337ab0e..5ef1adf03ad 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -223,6 +223,7 @@ def _cmd(lang): return "powershell" raise NotImplementedError(f"{lang} not recognized in code execution") + def is_docker_running(): """Check if docker is running. @@ -238,6 +239,7 @@ def is_docker_running(): except docker.errors.DockerException: return False + def in_docker_container(): """Check if the code is running in a docker container. @@ -246,6 +248,7 @@ def in_docker_container(): """ return os.path.exists("/.dockerenv") + def execute_code( code: Optional[str] = None, timeout: Optional[int] = None, @@ -367,7 +370,9 @@ def execute_code( # create a docker client if use_docker and not docker_running: - raise RuntimeError("Docker package is missing or docker is not running. Please make sure docker is running or set use_docker=False.") + raise RuntimeError( + "Docker package is missing or docker is not running. Please make sure docker is running or set use_docker=False." + ) client = docker.from_env() From 9ebb4fac85db868286d915ecf4229b9815afcf20 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 19:11:52 -0500 Subject: [PATCH 03/32] test --- autogen/code_utils.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index 5ef1adf03ad..d0a4e2618bf 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -1,5 +1,6 @@ import logging import os +import platform import pathlib import re import subprocess @@ -231,12 +232,15 @@ def is_docker_running(): bool: True if docker is running; False otherwise. """ if docker is None: + print("******** docker not installed ********") return False try: client = docker.from_env() client.ping() return True - except docker.errors.DockerException: + except docker.errors.DockerException as e: + print("******** docker not running ********") + print(e) return False @@ -399,6 +403,13 @@ def execute_code( # get a randomized str based on current time to wrap the exit code exit_code_str = f"exitcode{time.time()}" abs_path = pathlib.Path(work_dir).absolute() + host_os = platform.system() + if host_os == "Windows": + volume_path = str(abs_path).replace("\\", "/") + volume_path = volume_path[0].lower() + volume_path[1:] + else: + volume_path = str(abs_path) + cmd = [ "sh", "-c", @@ -411,7 +422,7 @@ def execute_code( working_dir="/workspace", detach=True, # get absolute path to the working directory - volumes={abs_path: {"bind": "/workspace", "mode": "rw"}}, + volumes={volume_path: {"bind": "/workspace", "mode": "rw"}}, ) start_time = time.time() while container.status != "exited" and time.time() - start_time < timeout: From 6a82fae057faa49d3e9789377ffe4440c866df09 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 19:38:54 -0500 Subject: [PATCH 04/32] test --- autogen/code_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index d0a4e2618bf..b325dbf6f8e 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -237,10 +237,11 @@ def is_docker_running(): try: client = docker.from_env() client.ping() + print("******** docker is running ********") return True except docker.errors.DockerException as e: print("******** docker not running ********") - print(e) + print(f"******** {e} ********") return False @@ -374,6 +375,7 @@ def execute_code( # create a docker client if use_docker and not docker_running: + print("******** docker not running raising runtime error ********") raise RuntimeError( "Docker package is missing or docker is not running. Please make sure docker is running or set use_docker=False." ) @@ -410,6 +412,8 @@ def execute_code( else: volume_path = str(abs_path) + print(f"***** volume_path: {volume_path} *****") + cmd = [ "sh", "-c", From a2b05fd65a498e5244ad0c280cdac43f3031f182 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 21:23:57 -0500 Subject: [PATCH 05/32] test test --- autogen/code_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index b325dbf6f8e..c83b0efcbf6 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -325,6 +325,19 @@ def execute_code( work_dir = WORKING_DIR filepath = os.path.join(work_dir, filename) file_dir = os.path.dirname(filepath) + + path = pathlib.Path(work_dir) + print(f"printing contents of {path}") + for item in os.listdir(path): + print(item) + pparent = path.parent + print(f"printing contents of {pparent}") + for item in os.listdir(pparent): + print(item) + ppparent = pparent.parent + print(f"printing contents of {ppparent}") + for item in os.listdir(ppparent): + print(item) os.makedirs(file_dir, exist_ok=True) if code is not None: with open(filepath, "w", encoding="utf-8") as fout: From 69e99af9921463ea29866958fb632a53f944ea82 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 21:28:12 -0500 Subject: [PATCH 06/32] .. --- autogen/code_utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index c83b0efcbf6..0e8a8e7fc3a 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -326,6 +326,8 @@ def execute_code( filepath = os.path.join(work_dir, filename) file_dir = os.path.dirname(filepath) + os.makedirs(file_dir, exist_ok=True) + path = pathlib.Path(work_dir) print(f"printing contents of {path}") for item in os.listdir(path): @@ -334,11 +336,7 @@ def execute_code( print(f"printing contents of {pparent}") for item in os.listdir(pparent): print(item) - ppparent = pparent.parent - print(f"printing contents of {ppparent}") - for item in os.listdir(ppparent): - print(item) - os.makedirs(file_dir, exist_ok=True) + if code is not None: with open(filepath, "w", encoding="utf-8") as fout: fout.write(code) From 6ca7a255867f7a7f4668fc0b6e6447ac24a09d76 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 23:16:52 -0500 Subject: [PATCH 07/32] .. --- test/test_code.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_code.py b/test/test_code.py index ec8ada2c906..c0432b4e98b 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -305,10 +305,16 @@ def test_execute_code(use_docker=None): docker = None if use_docker is None: use_docker = docker is not None - exit_code, msg, image = execute_code("print('hello world')", filename="tmp/codetest.py", use_docker=use_docker) + exit_code, msg, image = execute_code( + "print('hello world')", filename="tmp/codetest.py", use_docker=use_docker, work_dir=f"{here}/my_tmp" + ) assert exit_code == 0 and msg == "hello world\n", msg # read a file - print(execute_code("with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker)) + print( + execute_code( + "with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker, work_dir=f"{here}/my_tmp" + ) + ) # create a file exit_code, msg, image = execute_code( "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", From 59dbe7e0f21b21c0fe592b0404de48366972ff32 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 23:25:45 -0500 Subject: [PATCH 08/32] .. --- test/test_code.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/test/test_code.py b/test/test_code.py index c0432b4e98b..8383259a4be 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -305,35 +305,28 @@ def test_execute_code(use_docker=None): docker = None if use_docker is None: use_docker = docker is not None - exit_code, msg, image = execute_code( - "print('hello world')", filename="tmp/codetest.py", use_docker=use_docker, work_dir=f"{here}/my_tmp" - ) + exit_code, msg, image = execute_code("print('hello world')", filename="codetest.py", use_docker=use_docker) assert exit_code == 0 and msg == "hello world\n", msg # read a file - print( - execute_code( - "with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker, work_dir=f"{here}/my_tmp" - ) - ) + print(execute_code("with open('codetest.py', 'r') as f: a=f.read()", use_docker=use_docker)) # create a file exit_code, msg, image = execute_code( - "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", + "with open('codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp", - filename="tmp2/codetest.py", + filename="codetest2.py", use_docker=use_docker, ) assert exit_code and ( - 'File "tmp2/codetest.py"'.replace("/", PATH_SEPARATOR) in msg - or 'File ".\\tmp2/codetest.py' in msg # py3.8 + win32 + 'File "codetest2.py"'.replace("/", PATH_SEPARATOR) in msg or 'File ".\\codetest2.py' in msg # py3.8 + win32 ), msg print( execute_code( - "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp", use_docker=use_docker + "with open('codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp", use_docker=use_docker ) ) # execute code in a file - print(execute_code(filename="tmp/codetest.py", use_docker=use_docker)) - print(execute_code("python tmp/codetest.py", lang="sh", use_docker=use_docker)) + print(execute_code(filename="codetest.py", use_docker=use_docker)) + print(execute_code("python codetest.py", lang="sh", use_docker=use_docker)) # execute code for assertion error exit_code, msg, image = execute_code("assert 1==2", use_docker=use_docker) assert exit_code, msg From 5ce8e9ea54d68a2421da2eb8fecb43f4a65ea07a Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 23:37:41 -0500 Subject: [PATCH 09/32] .. --- autogen/code_utils.py | 14 +++----------- test/test_code.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index 0e8a8e7fc3a..e39f17bc39b 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -323,20 +323,11 @@ def execute_code( filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}" if work_dir is None: work_dir = WORKING_DIR + filepath = os.path.join(work_dir, filename) file_dir = os.path.dirname(filepath) - os.makedirs(file_dir, exist_ok=True) - path = pathlib.Path(work_dir) - print(f"printing contents of {path}") - for item in os.listdir(path): - print(item) - pparent = path.parent - print(f"printing contents of {pparent}") - for item in os.listdir(pparent): - print(item) - if code is not None: with open(filepath, "w", encoding="utf-8") as fout: fout.write(code) @@ -419,7 +410,8 @@ def execute_code( host_os = platform.system() if host_os == "Windows": volume_path = str(abs_path).replace("\\", "/") - volume_path = volume_path[0].lower() + volume_path[1:] + volume_path = volume_path.replace(":", "").lower() + volume_path = "//" + volume_path[0].lower() + volume_path[1:] else: volume_path = str(abs_path) diff --git a/test/test_code.py b/test/test_code.py index 8383259a4be..ec8ada2c906 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -305,28 +305,29 @@ def test_execute_code(use_docker=None): docker = None if use_docker is None: use_docker = docker is not None - exit_code, msg, image = execute_code("print('hello world')", filename="codetest.py", use_docker=use_docker) + exit_code, msg, image = execute_code("print('hello world')", filename="tmp/codetest.py", use_docker=use_docker) assert exit_code == 0 and msg == "hello world\n", msg # read a file - print(execute_code("with open('codetest.py', 'r') as f: a=f.read()", use_docker=use_docker)) + print(execute_code("with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker)) # create a file exit_code, msg, image = execute_code( - "with open('codetest.py', 'w') as f: f.write('b=1')", + "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp", - filename="codetest2.py", + filename="tmp2/codetest.py", use_docker=use_docker, ) assert exit_code and ( - 'File "codetest2.py"'.replace("/", PATH_SEPARATOR) in msg or 'File ".\\codetest2.py' in msg # py3.8 + win32 + 'File "tmp2/codetest.py"'.replace("/", PATH_SEPARATOR) in msg + or 'File ".\\tmp2/codetest.py' in msg # py3.8 + win32 ), msg print( execute_code( - "with open('codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp", use_docker=use_docker + "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp", use_docker=use_docker ) ) # execute code in a file - print(execute_code(filename="codetest.py", use_docker=use_docker)) - print(execute_code("python codetest.py", lang="sh", use_docker=use_docker)) + print(execute_code(filename="tmp/codetest.py", use_docker=use_docker)) + print(execute_code("python tmp/codetest.py", lang="sh", use_docker=use_docker)) # execute code for assertion error exit_code, msg, image = execute_code("assert 1==2", use_docker=use_docker) assert exit_code, msg From 35fced0218e99250bd91bc52877590889a94dea9 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 23:38:32 -0500 Subject: [PATCH 10/32] .. --- autogen/code_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index e39f17bc39b..cef4a6543a6 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -407,6 +407,9 @@ def execute_code( # get a randomized str based on current time to wrap the exit code exit_code_str = f"exitcode{time.time()}" abs_path = pathlib.Path(work_dir).absolute() + non_abs_path = pathlib.Path(work_dir) + print(f"***** abs_path: {abs_path} *****") + print(f"***** non_abs_path: {non_abs_path} *****") host_os = platform.system() if host_os == "Windows": volume_path = str(abs_path).replace("\\", "/") From 0342f0f848791985b68ccd32b5eba0f66adfa043 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 23:46:42 -0500 Subject: [PATCH 11/32] .. --- autogen/code_utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index cef4a6543a6..589b799e4fd 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -407,14 +407,9 @@ def execute_code( # get a randomized str based on current time to wrap the exit code exit_code_str = f"exitcode{time.time()}" abs_path = pathlib.Path(work_dir).absolute() - non_abs_path = pathlib.Path(work_dir) - print(f"***** abs_path: {abs_path} *****") - print(f"***** non_abs_path: {non_abs_path} *****") host_os = platform.system() if host_os == "Windows": volume_path = str(abs_path).replace("\\", "/") - volume_path = volume_path.replace(":", "").lower() - volume_path = "//" + volume_path[0].lower() + volume_path[1:] else: volume_path = str(abs_path) From 1f684c13fd12ef6fa57685abd7aaddcf38ce3f28 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Thu, 4 Jan 2024 23:51:28 -0500 Subject: [PATCH 12/32] .. --- test/test_code.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_code.py b/test/test_code.py index ec8ada2c906..92028eb1cce 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -305,10 +305,16 @@ def test_execute_code(use_docker=None): docker = None if use_docker is None: use_docker = docker is not None - exit_code, msg, image = execute_code("print('hello world')", filename="tmp/codetest.py", use_docker=use_docker) + exit_code, msg, image = execute_code( + "print('hello world')", filename="tmp/codetest.py", use_docker=use_docker, work_dir=f"{here}/my_tmp1" + ) assert exit_code == 0 and msg == "hello world\n", msg # read a file - print(execute_code("with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker)) + print( + execute_code( + "with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker, work_dir=f"{here}/my_tmp1" + ) + ) # create a file exit_code, msg, image = execute_code( "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", From 3c563c8e8012cce85eca1770f1874c1c5a849081 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:00:36 -0500 Subject: [PATCH 13/32] .. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f1af9ab744..da09770f80f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,9 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 + - run: | + echo ${{ github.workspace }} + echo $GITHUB_WORKSPACE - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From d8928925724ce6295cf83c70a026757b3ee2b309 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:00:54 -0500 Subject: [PATCH 14/32] .. --- .github/workflows/build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da09770f80f..2f1af9ab744 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,9 +31,6 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - run: | - echo ${{ github.workspace }} - echo $GITHUB_WORKSPACE - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 22ca0a4e25d496e878e361424ec11d07ddec30fa Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:11:08 -0500 Subject: [PATCH 15/32] .. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f1af9ab744..d2edd804cf6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,8 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 + - run: | + docker ps - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From ecb23a496ecb2d09447f92b077a97b957780db0e Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:19:44 -0500 Subject: [PATCH 16/32] .. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2edd804cf6..f21e48bbf40 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,8 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - run: | docker ps - name: Set up Python ${{ matrix.python-version }} From 0f4426c45a5b61010cf0f2921cabd6b723f46c02 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:22:46 -0500 Subject: [PATCH 17/32] .. --- .github/workflows/build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f21e48bbf40..cd0a2be8e1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,10 +31,13 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - run: | - docker ps + - name: Run a Python container and execute script + run: | + docker run \ + --name py-container \ + -v ${{ github.workspace }}:/workspace \ + python:3.8 \ + python print("hello world") - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 7f273588f8a2ae0991b9b8bb5fb2f52513d85332 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:25:03 -0500 Subject: [PATCH 18/32] .. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd0a2be8e1f..57a73fa010a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,7 @@ jobs: --name py-container \ -v ${{ github.workspace }}:/workspace \ python:3.8 \ - python print("hello world") + python -c "print('Hello, World!')" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 3a6e35be2e79340c1916a207b714b1bdd0038e27 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:29:40 -0500 Subject: [PATCH 19/32] .. --- .github/workflows/build.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57a73fa010a..3d9642899d6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,12 +32,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Run a Python container and execute script - run: | - docker run \ - --name py-container \ - -v ${{ github.workspace }}:/workspace \ - python:3.8 \ - python -c "print('Hello, World!')" + run: docker run --name py-container -v ${{ github.workspace }}:/workspace python:3.8 python -c "print('Hello, World!')" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 1663f8c14f88abbe96eff0604709d8b791ef353e Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:31:45 -0500 Subject: [PATCH 20/32] .. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d9642899d6..2b3ceb522e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Run a Python container and execute script - run: docker run --name py-container -v ${{ github.workspace }}:/workspace python:3.8 python -c "print('Hello, World!')" + run: docker run --name py-container -v ${{ github.workspace }}:/workspace python:3 python -c "print('Hello, World!')" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From d083897031662491aeeb9917ad1bfd9ea6bef7ea Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:46:27 -0500 Subject: [PATCH 21/32] .. --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b3ceb522e9..b1c686d3214 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,10 @@ jobs: steps: - uses: actions/checkout@v3 - name: Run a Python container and execute script - run: docker run --name py-container -v ${{ github.workspace }}:/workspace python:3 python -c "print('Hello, World!')" + run: | + $path = "${{ github.workspace }}".Replace("\", "/") + docker run --name py-container -v $path:/workspace python:3.8 python -c "print('Hello, World!')" + shell: pwsh - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 4eec002c8257a5a57e77e5e604ab1df3b1b5c48b Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:49:04 -0500 Subject: [PATCH 22/32] .. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1c686d3214..c8654ea309b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - name: Run a Python container and execute script run: | $path = "${{ github.workspace }}".Replace("\", "/") - docker run --name py-container -v $path:/workspace python:3.8 python -c "print('Hello, World!')" + docker run --name py-container -v ${path}:/workspace python:3.8 python -c "print('Hello, World!')" shell: pwsh - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From b6712d19a887f58c1d079eff939496217cdb9b8f Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 00:50:34 -0500 Subject: [PATCH 23/32] .. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8654ea309b..07bd096e561 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - name: Run a Python container and execute script run: | $path = "${{ github.workspace }}".Replace("\", "/") - docker run --name py-container -v ${path}:/workspace python:3.8 python -c "print('Hello, World!')" + docker run --name py-container -v ${path}:/workspace python:3 python -c "print('Hello, World!')" shell: pwsh - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 8bef15b6a4d5802e146b119ad2a274f5a10851af Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 01:00:14 -0500 Subject: [PATCH 24/32] .. --- .github/workflows/build.yml | 2 +- autogen/code_utils.py | 5 +++-- test/test_code.py | 7 +++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07bd096e561..f76195ac29c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - name: Run a Python container and execute script run: | $path = "${{ github.workspace }}".Replace("\", "/") - docker run --name py-container -v ${path}:/workspace python:3 python -c "print('Hello, World!')" + docker run --name py-container -v ${path}:c:/workspace python:3 python -c "print('Hello, World!')" shell: pwsh - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/autogen/code_utils.py b/autogen/code_utils.py index 589b799e4fd..fff6235fef2 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -421,13 +421,14 @@ def execute_code( f"{_cmd(lang)} {filename}; exit_code=$?; echo -n {exit_code_str}; echo -n $exit_code; echo {exit_code_str}", ] # create a docker container + workspace = "c:/workspace" if host_os == "Windows" else "/workspace" container = client.containers.run( image, command=cmd, - working_dir="/workspace", + working_dir=workspace, detach=True, # get absolute path to the working directory - volumes={volume_path: {"bind": "/workspace", "mode": "rw"}}, + volumes={volume_path: {"bind": workspace, "mode": "rw"}}, ) start_time = time.time() while container.status != "exited" and time.time() - start_time < timeout: diff --git a/test/test_code.py b/test/test_code.py index 92028eb1cce..2dbbf36bc1b 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -303,6 +303,13 @@ def test_execute_code(use_docker=None): except ImportError as exc: print(exc) docker = None + + import platform + + host_os = platform.system() + if host_os == "Darwin": + print("Skip test_execute_code on MacOS") + use_docker = False if use_docker is None: use_docker = docker is not None exit_code, msg, image = execute_code( From 12e2a66f93727721cefae76b15ccc3260037caf6 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 01:09:48 -0500 Subject: [PATCH 25/32] .. --- autogen/code_utils.py | 2 +- test/test_code.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index fff6235fef2..ac2b81b65b6 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -402,7 +402,7 @@ def execute_code( try: client.images.pull(image) break - except docker.errors.DockerException: + except (docker.errors.DockerException, docker.errors.APIError): print("Failed to pull image", image) # get a randomized str based on current time to wrap the exit code exit_code_str = f"exitcode{time.time()}" diff --git a/test/test_code.py b/test/test_code.py index 2dbbf36bc1b..e68d78fb3ff 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -312,16 +312,10 @@ def test_execute_code(use_docker=None): use_docker = False if use_docker is None: use_docker = docker is not None - exit_code, msg, image = execute_code( - "print('hello world')", filename="tmp/codetest.py", use_docker=use_docker, work_dir=f"{here}/my_tmp1" - ) + exit_code, msg, image = execute_code("print('hello world')", filename="tmp/codetest.py", use_docker=use_docker) assert exit_code == 0 and msg == "hello world\n", msg # read a file - print( - execute_code( - "with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker, work_dir=f"{here}/my_tmp1" - ) - ) + print(execute_code("with open('tmp/codetest.py', 'r') as f: a=f.read()", use_docker=use_docker)) # create a file exit_code, msg, image = execute_code( "with open('tmp/codetest.py', 'w') as f: f.write('b=1')", From c4f49bbb6f9311cdfdc117aeb959fe46e572ec87 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 01:10:40 -0500 Subject: [PATCH 26/32] .. --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f76195ac29c..2f1af9ab744 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,11 +31,6 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - name: Run a Python container and execute script - run: | - $path = "${{ github.workspace }}".Replace("\", "/") - docker run --name py-container -v ${path}:c:/workspace python:3 python -c "print('Hello, World!')" - shell: pwsh - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 2f89091a3b9a1f3a77a6b0d6daee44756a7d9393 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 02:39:43 -0500 Subject: [PATCH 27/32] .. --- .github/workflows/build.yml | 6 +++ autogen/agentchat/conversable_agent.py | 59 +++++++++++++++++++------- test/test_code.py | 6 --- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f1af9ab744..488d6d8b0dd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,12 @@ jobs: pip install -e . python -c "import autogen" pip install -e. pytest mock + - name: Set environment variable based on OS + run: | + if [ "${{ matrix.os }}" = "macos-latest" ]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + elif [ "${{ matrix.os }}" = "windows-latest" ]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV - name: Test with pytest if: matrix.python-version != '3.10' run: | diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 012215b5a01..7881acee85b 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -1,3 +1,4 @@ +import os import asyncio import copy import functools @@ -37,6 +38,45 @@ def colored(x, *args, **kwargs): F = TypeVar("F", bound=Callable[..., Any]) +def check_and_decide_use_docker(code_execution_config: Dict) -> bool: + if "use_docker" in code_execution_config: + use_docker = code_execution_config["use_docker"] + else: + env_var_use_docker = os.environ.get("AUTOGEN_USE_DOCKER", "True") + + truthy_values = {"1", "true", "yes", "t"} + falsy_values = {"0", "false", "no", "f"} + + # Convert the value to lowercase for case-insensitive comparison + env_var_use_docker_lower = env_var_use_docker.lower() + + # Determine the boolean value based on the environment variable + if env_var_use_docker_lower in truthy_values: + use_docker = True + elif env_var_use_docker_lower in falsy_values: + use_docker = False + elif env_var_use_docker_lower == "none": # Special case for 'None' as a string + use_docker = None + else: + # Raise an error for any unrecognized value + raise ValueError( + f'Invalid value for AUTOGEN_USE_DOCKER: {env_var_use_docker}. Please set AUTOGEN_USE_DOCKER to "1/True/yes", "0/False/no", or "None".' + ) + + if use_docker is not None: + inside_docker = in_docker_container() + docker_installed_and_running = is_docker_running() + if use_docker and not inside_docker and not docker_installed_and_running: + raise RuntimeError( + 'Docker is not running, please make sure docker is running (advised approach for code execution) or set "use_docker":False.' + ) + if not use_docker: + logger.warning( + 'use_docker was set to False. Any code execution will be run natively but we strongly advise to set "use_docker":True. Set "use_docker":None to silence this message.' + ) + return use_docker + + class ConversableAgent(Agent): """(In preview) A class for generic conversable agents which can be configured as assistant or user proxy. @@ -140,21 +180,10 @@ def __init__( ) if isinstance(self._code_execution_config, Dict): - if "use_docker" in self._code_execution_config: - use_docker = self._code_execution_config["use_docker"] - else: - use_docker = True - if use_docker is not None: - inside_docker = in_docker_container() - docker_installed_and_running = is_docker_running() - if use_docker and not inside_docker and not docker_installed_and_running: - raise RuntimeError( - 'Docker is not running, please make sure docker is running (advised approach for code execution) or set "use_docker":False.' - ) - if not use_docker: - logger.warning( - 'use_docker was set to False. Any code execution will be run natively but we strongly advise to set "use_docker":True. Set "use_docker":None to silence this message.' - ) + use_docker = check_and_decide_use_docker(self._code_execution_config) + self._code_execution_config["use_docker"] = use_docker + print(f"Code execution is set to use docker: {use_docker}") + self.human_input_mode = human_input_mode self._max_consecutive_auto_reply = ( max_consecutive_auto_reply if max_consecutive_auto_reply is not None else self.MAX_CONSECUTIVE_AUTO_REPLY diff --git a/test/test_code.py b/test/test_code.py index e68d78fb3ff..6b02a3b0656 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -304,12 +304,6 @@ def test_execute_code(use_docker=None): print(exc) docker = None - import platform - - host_os = platform.system() - if host_os == "Darwin": - print("Skip test_execute_code on MacOS") - use_docker = False if use_docker is None: use_docker = docker is not None exit_code, msg, image = execute_code("print('hello world')", filename="tmp/codetest.py", use_docker=use_docker) From 06c18f3d893b2adbbef1402951cd09cf9e92dbd5 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 02:42:56 -0500 Subject: [PATCH 28/32] .. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 488d6d8b0dd..99b111edb70 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,6 +47,7 @@ jobs: echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV elif [ "${{ matrix.os }}" = "windows-latest" ]; then echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi - name: Test with pytest if: matrix.python-version != '3.10' run: | From b082678ee6104d64dc764607dd44462283bd0bbc Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 02:47:42 -0500 Subject: [PATCH 29/32] .. --- .github/workflows/build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99b111edb70..9a1fb33fc37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,13 +41,12 @@ jobs: pip install -e . python -c "import autogen" pip install -e. pytest mock - - name: Set environment variable based on OS + - name: Set AUTOGEN_USE_DOCKER based on OS run: | - if [ "${{ matrix.os }}" = "macos-latest" ]; then - echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV - elif [ "${{ matrix.os }}" = "windows-latest" ]; then - echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV - fi + if ($env:matrix_os -eq "macos-latest" -or $env:matrix_os -eq "windows-latest") { + "AUTOGEN_USE_DOCKER=False" | Out-File -Append -FilePath $env:GITHUB_ENV + } # Add other conditions or an else clause as needed + shell: pwsh # Specify pwsh as the shell for this step - name: Test with pytest if: matrix.python-version != '3.10' run: | From dad98543ee5b3db33a8b678c06fd4c70eb24c075 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 03:33:03 -0500 Subject: [PATCH 30/32] ... --- .github/workflows/build.yml | 2 +- autogen/agentchat/conversable_agent.py | 48 +++------------------- autogen/code_utils.py | 55 +++++++++++++++++++++----- 3 files changed, 53 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a1fb33fc37..41d6b7e432e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: pip install -e. pytest mock - name: Set AUTOGEN_USE_DOCKER based on OS run: | - if ($env:matrix_os -eq "macos-latest" -or $env:matrix_os -eq "windows-latest") { + if ($env:matrix_os -eq "macos-latest" -or $env:matrix_os -eq "windows-2019") { "AUTOGEN_USE_DOCKER=False" | Out-File -Append -FilePath $env:GITHUB_ENV } # Add other conditions or an else clause as needed shell: pwsh # Specify pwsh as the shell for this step diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 7881acee85b..8a3b7467748 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -13,12 +13,13 @@ DEFAULT_MODEL, UNKNOWN, content_str, + check_use_docker, + decide_use_docker, execute_code, extract_code, infer_lang, - is_docker_running, - in_docker_container, ) + from ..function_utils import get_function_schema, load_basemodels_if_needed, serialize_to_str from .agent import Agent from .._pydantic import model_dump @@ -38,45 +39,6 @@ def colored(x, *args, **kwargs): F = TypeVar("F", bound=Callable[..., Any]) -def check_and_decide_use_docker(code_execution_config: Dict) -> bool: - if "use_docker" in code_execution_config: - use_docker = code_execution_config["use_docker"] - else: - env_var_use_docker = os.environ.get("AUTOGEN_USE_DOCKER", "True") - - truthy_values = {"1", "true", "yes", "t"} - falsy_values = {"0", "false", "no", "f"} - - # Convert the value to lowercase for case-insensitive comparison - env_var_use_docker_lower = env_var_use_docker.lower() - - # Determine the boolean value based on the environment variable - if env_var_use_docker_lower in truthy_values: - use_docker = True - elif env_var_use_docker_lower in falsy_values: - use_docker = False - elif env_var_use_docker_lower == "none": # Special case for 'None' as a string - use_docker = None - else: - # Raise an error for any unrecognized value - raise ValueError( - f'Invalid value for AUTOGEN_USE_DOCKER: {env_var_use_docker}. Please set AUTOGEN_USE_DOCKER to "1/True/yes", "0/False/no", or "None".' - ) - - if use_docker is not None: - inside_docker = in_docker_container() - docker_installed_and_running = is_docker_running() - if use_docker and not inside_docker and not docker_installed_and_running: - raise RuntimeError( - 'Docker is not running, please make sure docker is running (advised approach for code execution) or set "use_docker":False.' - ) - if not use_docker: - logger.warning( - 'use_docker was set to False. Any code execution will be run natively but we strongly advise to set "use_docker":True. Set "use_docker":None to silence this message.' - ) - return use_docker - - class ConversableAgent(Agent): """(In preview) A class for generic conversable agents which can be configured as assistant or user proxy. @@ -180,7 +142,9 @@ def __init__( ) if isinstance(self._code_execution_config, Dict): - use_docker = check_and_decide_use_docker(self._code_execution_config) + use_docker = self._code_execution_config.get("use_docker", None) + use_docker = decide_use_docker(use_docker) + check_use_docker(use_docker) self._code_execution_config["use_docker"] = use_docker print(f"Code execution is set to use docker: {use_docker}") diff --git a/autogen/code_utils.py b/autogen/code_utils.py index ac2b81b65b6..ae5edff8e02 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -17,6 +17,7 @@ except ImportError: docker = None +SENTINEL = object() DEFAULT_MODEL = "gpt-4" FAST_MODEL = "gpt-3.5-turbo" # Regular expression for finding a code block @@ -254,12 +255,51 @@ def in_docker_container(): return os.path.exists("/.dockerenv") +def decide_use_docker(use_docker) -> bool: + if use_docker is None: + env_var_use_docker = os.environ.get("AUTOGEN_USE_DOCKER", "True") + + truthy_values = {"1", "true", "yes", "t"} + falsy_values = {"0", "false", "no", "f"} + + # Convert the value to lowercase for case-insensitive comparison + env_var_use_docker_lower = env_var_use_docker.lower() + + # Determine the boolean value based on the environment variable + if env_var_use_docker_lower in truthy_values: + use_docker = True + elif env_var_use_docker_lower in falsy_values: + use_docker = False + elif env_var_use_docker_lower == "none": # Special case for 'None' as a string + use_docker = None + else: + # Raise an error for any unrecognized value + raise ValueError( + f'Invalid value for AUTOGEN_USE_DOCKER: {env_var_use_docker}. Please set AUTOGEN_USE_DOCKER to "1/True/yes", "0/False/no", or "None".' + ) + return use_docker + + +def check_use_docker(use_docker) -> None: + if use_docker is not None: + inside_docker = in_docker_container() + docker_installed_and_running = is_docker_running() + if use_docker and not inside_docker and not docker_installed_and_running: + raise RuntimeError( + 'Docker is not running, please make sure docker is running (advised approach for code execution) or set "use_docker":False.' + ) + if not use_docker: + logger.warning( + 'use_docker was set to False. Any code execution will be run natively but we strongly advise to set "use_docker":True. Set "use_docker":None to silence this message.' + ) + + def execute_code( code: Optional[str] = None, timeout: Optional[int] = None, filename: Optional[str] = None, work_dir: Optional[str] = None, - use_docker: Union[List[str], str, bool] = True, + use_docker: Union[List[str], str, bool] = SENTINEL, lang: Optional[str] = "python", ) -> Tuple[int, str, str]: """Execute code in a docker container. @@ -302,16 +342,14 @@ def execute_code( logger.error(error_msg) raise AssertionError(error_msg) - # # Warn if use_docker was set to False - # # In this case the current behavior is to fall back to run natively, but this behavior - # # is subject to change. running_inside_docker = in_docker_container() docker_running = is_docker_running() - if use_docker is not None and not use_docker and not running_inside_docker: - logger.warning( - "execute_code was called with use_docker set to False. Code will be run natively but we strongly advise to set use_docker=True. Set use_docker=None to silence this message." - ) + print(f"BEFORE check_and_decide_use_docker: {use_docker}") + if use_docker is SENTINEL: + use_docker = decide_use_docker(use_docker=None) + print(f"AFTER check_and_decide_use_docker: {use_docker}") + check_use_docker(use_docker) timeout = timeout or DEFAULT_TIMEOUT original_filename = filename @@ -377,7 +415,6 @@ def execute_code( # create a docker client if use_docker and not docker_running: - print("******** docker not running raising runtime error ********") raise RuntimeError( "Docker package is missing or docker is not running. Please make sure docker is running or set use_docker=False." ) From 52db2b0f6871d87476a06495d0496d623cae223e Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 03:43:07 -0500 Subject: [PATCH 31/32] .. --- autogen/code_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index ae5edff8e02..471ebbfaff5 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -256,8 +256,10 @@ def in_docker_container(): def decide_use_docker(use_docker) -> bool: + print(f"inside decide_use_docker: {use_docker}") if use_docker is None: env_var_use_docker = os.environ.get("AUTOGEN_USE_DOCKER", "True") + print(f"env_var_use_docker: {env_var_use_docker}") truthy_values = {"1", "true", "yes", "t"} falsy_values = {"0", "false", "no", "f"} From 355d11532a62f74cc9d72d8bb05a49f1f610779a Mon Sep 17 00:00:00 2001 From: olgavrou Date: Fri, 5 Jan 2024 03:53:08 -0500 Subject: [PATCH 32/32] .. --- .github/workflows/build.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41d6b7e432e..dde7594fcda 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,11 +42,16 @@ jobs: python -c "import autogen" pip install -e. pytest mock - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash run: | - if ($env:matrix_os -eq "macos-latest" -or $env:matrix_os -eq "windows-2019") { - "AUTOGEN_USE_DOCKER=False" | Out-File -Append -FilePath $env:GITHUB_ENV - } # Add other conditions or an else clause as needed - shell: pwsh # Specify pwsh as the shell for this step + if [[ ${{ matrix.os }} == windows-2019 ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + elif [[ ${{ matrix.os }} == macos-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + - name: Use the Environment Variable + run: echo "The value of AUTOGEN_USE_DOCKER is $AUTOGEN_USE_DOCKER" + shell: bash - name: Test with pytest if: matrix.python-version != '3.10' run: |