From 78795b0b1d82a4e472ad213739db182fcb045ea5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 15 Sep 2025 12:39:33 +0530 Subject: [PATCH 01/17] Support mlc_run_cmd with quotes --- mlc/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 6a1813a20..0ed1d70e1 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -3,7 +3,7 @@ import sys from pathlib import Path import inspect - +import shlex from . import utils from .action import Action, default_parent @@ -255,7 +255,7 @@ def main(): run_args = res['args_dict'] - run_args['mlc_run_cmd'] = " ".join(sys.argv) + run_args['mlc_run_cmd'] = " ".join(shlex.quote(arg) for arg in sys.argv) if hasattr(args, 'repo') and args.repo: run_args['repo'] = args.repo From 926f76aac9e2dc84f5cc8613cf928dd0e7dbba6c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 15 Sep 2025 13:27:32 +0530 Subject: [PATCH 02/17] Fix an issue with --help for individual scripts --- mlc/main.py | 69 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 0ed1d70e1..97272f3a6 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -162,40 +162,6 @@ def main(): pre_parser.add_argument("-h", "--help", action="store_true") pre_args, remaining_args = pre_parser.parse_known_args() - if pre_args.help and not any("--tags" in arg for arg in remaining_args): - help_text = "" - if pre_args.target == "run": - if pre_args.action == "docker": - pre_args.target = "script" - else: - logger.error(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") - raise Exception(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") - if not pre_args.action and not pre_args.target: - help_text += main.__doc__ - elif pre_args.action and not pre_args.target: - if pre_args.action not in ['script', 'cache', 'repo']: - logger.error(f"Invalid target {pre_args.action}") - raise Exception(f"""Invalid target {pre_args.action}""") - else: - pre_args.target, pre_args.action = pre_args.action, None - actions = get_action(pre_args.target, default_parent) - help_text += actions.__doc__ - # iterate through every method - for method_name, method in inspect.getmembers(actions.__class__, inspect.isfunction): - method = getattr(actions, method_name) - if method.__doc__ and not method.__doc__.startswith("_"): - help_text += method.__doc__ - elif pre_args.action and pre_args.target: - actions = get_action(pre_args.target, default_parent) - try: - method = getattr(actions, pre_args.action) - help_text += actions.__doc__ - help_text += method.__doc__ - except: - logger.error(f"Error: '{pre_args.action}' is not supported for {pre_args.target}.") - if help_text != "": - print(help_text) - sys.exit(0) # parser for execution of the automation scripts parser = argparse.ArgumentParser(prog='mlc', description='A CLI tool for managing repos, scripts, and caches.', add_help=False) @@ -282,6 +248,41 @@ def main(): if hasattr(args, 'extra') and args.extra: run_args['dest'] = args.extra[0] + if pre_args.help and not "tags" in run_args: + help_text = "" + if pre_args.target == "run": + if pre_args.action == "docker": + pre_args.target = "script" + else: + logger.error(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") + raise Exception(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") + if not pre_args.action and not pre_args.target: + help_text += main.__doc__ + elif pre_args.action and not pre_args.target: + if pre_args.action not in ['script', 'cache', 'repo']: + logger.error(f"Invalid target {pre_args.action}") + raise Exception(f"""Invalid target {pre_args.action}""") + else: + pre_args.target, pre_args.action = pre_args.action, None + actions = get_action(pre_args.target, default_parent) + help_text += actions.__doc__ + # iterate through every method + for method_name, method in inspect.getmembers(actions.__class__, inspect.isfunction): + method = getattr(actions, method_name) + if method.__doc__ and not method.__doc__.startswith("_"): + help_text += method.__doc__ + elif pre_args.action and pre_args.target: + actions = get_action(pre_args.target, default_parent) + try: + method = getattr(actions, pre_args.action) + help_text += actions.__doc__ + help_text += method.__doc__ + except: + logger.error(f"Error: '{pre_args.action}' is not supported for {pre_args.target}.") + if help_text != "": + print(help_text) + sys.exit(0) + # Get the action handler for other commands action = get_action(args.target, default_parent) # Dynamically call the method (e.g., run, list, show) diff --git a/pyproject.toml b/pyproject.toml index cd7cd9644..b2251c285 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.0.23" +version = "1.1" From 568afaa1add684b50f3f2f20756113a5877e7733 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 15 Sep 2025 14:41:52 +0530 Subject: [PATCH 03/17] Refactor main --- mlc/main.py | 196 +++++++++++++++++++++++++--------------------------- 1 file changed, 94 insertions(+), 102 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 97272f3a6..159da5242 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -115,7 +115,83 @@ def process_console_output(res, target, action, run_args): if default_parent is None: default_parent = Action() -# Main CLI function + +log_flag_aliases = {'-v': '--verbose', '-s': '--silent'} +log_levels = {'--verbose': logging.DEBUG, '--silent': logging.WARNING} + + +def build_pre_parser(): + pre_parser = argparse.ArgumentParser(add_help=False) + pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)") + pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo'], nargs="?", help="Target (repo, script, cache, ...)") + pre_parser.add_argument("-h", "--help", action="store_true") + return pre_parser + + +def build_parser(pre_args): + parser = argparse.ArgumentParser(prog="mlc", description="Manage repos, scripts, and caches.", add_help=False) + subparsers = parser.add_subparsers(dest="command", required=not pre_args.help) + + # General commands + for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv', 'help']: + p = subparsers.add_parser(action, add_help=False) + p.add_argument('target', choices=['repo', 'script', 'cache']) + p.add_argument('details', nargs='?', help='Details or identifier (optional)') + p.add_argument('extra', nargs=argparse.REMAINDER) + + # Script-only + for action in ['docker', 'experiment', 'doc', 'lint']: + p = subparsers.add_parser(action, add_help=False) + p.add_argument('target', choices=['script', 'run']) + p.add_argument('details', nargs='?', help='Details or identifier (optional)') + p.add_argument('extra', nargs=argparse.REMAINDER) + + # Load cfg + load_parser = subparsers.add_parser("load", add_help=False) + load_parser.add_argument("target", choices=["cfg"]) + return parser + + +def configure_logging(args): + if hasattr(args, 'extra') and args.extra: + args.extra[:] = [log_flag_aliases.get(a, a) for a in args.extra] + for flag, level in log_levels.items(): + if flag in args.extra: + logging.getLogger().setLevel(level) + args.extra.remove(flag) + + +def build_run_args(args): + res = utils.convert_args_to_dictionary(getattr(args, 'extra', [])) + if res['return'] > 0: + return res + + run_args = res['args_dict'] + run_args['mlc_run_cmd'] = shlex.join(sys.argv) + + if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": + run_args['repo'] = args.details + + if args.command in ['docker', 'experiment', 'doc', 'lint'] and args.target == "run": + run_args['target'] = 'script' + args.target = "script" + + if args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]: + run_args['tags'] = args.details + + if not run_args.get('details') and args.details: + run_args['details'] = args.details + + if args.command in ["cp", "mv"]: + run_args['target'] = args.target + if args.details: + run_args['src'] = args.details + if args.extra: + run_args['dest'] = args.extra[0] + + return run_args + + def main(): """ MLCFlow is a CLI tool for managing repos, scripts, and caches. @@ -154,99 +230,14 @@ def main(): mlc run script --help mlc pull repo -h """ - - # First level parser for showing help - pre_parser = argparse.ArgumentParser(add_help=False) - pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)") - pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo'], nargs="?", help="Potential target (repo, script, cache, ...)") - pre_parser.add_argument("-h", "--help", action="store_true") + pre_parser = build_pre_parser() pre_args, remaining_args = pre_parser.parse_known_args() - - # parser for execution of the automation scripts - parser = argparse.ArgumentParser(prog='mlc', description='A CLI tool for managing repos, scripts, and caches.', add_help=False) - - # Subparsers are added to main parser, allowing for different commands (subcommands) to be defined. - # The chosen subcommand will be stored in the "command" attribute of the parsed arguments. - subparsers = parser.add_subparsers(dest='command', required=True) - - # Script and Cache-specific subcommands - for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv']: - action_parser = subparsers.add_parser(action, add_help=False) - action_parser.add_argument('target', choices=['repo', 'script', 'cache'], help='Target type (repo, script, cache).') - # the argument given after target and before any extra options like --tags will be stored in "details" - action_parser.add_argument('details', nargs='?', help='Details or identifier (optional for list).') - action_parser.add_argument('extra', nargs=argparse.REMAINDER, help='Extra options (e.g., -v)') - - # Script specific subcommands - for action in ['docker', 'experiment', 'doc', 'lint']: - action_parser = subparsers.add_parser(action, add_help=False) - action_parser.add_argument('target', choices=['script', 'run'], help='Target type (script).') - # the argument given after target and before any extra options like --tags will be stored in "details" - action_parser.add_argument('details', nargs='?', help='Details or identifier (optional for list).') - action_parser.add_argument('extra', nargs=argparse.REMAINDER, help='Extra options (e.g., -v)') - - for action in ['load']: - load_parser = subparsers.add_parser(action, add_help=False) - load_parser.add_argument('target', choices=['cfg'], help='Target type (cfg).') - - # Parse arguments - args = parser.parse_args() + parser = build_parser(pre_args) + args = parser.parse_args() if remaining_args or pre_args.action else pre_args - #logger.info(f"Args = {args}") - - # set log level for MLCFlow if -v/--verbose or -s/--silent is specified - log_flag_aliases = { - '-v': '--verbose', - '-s': '--silent' - } - - log_levels = { - '--verbose': logging.DEBUG, - '--silent': logging.WARNING - } - - # Modify args.extra in place - args.extra[:] = [log_flag_aliases.get(arg, arg) for arg in args.extra] - - # Set log level based on the first matching flag - for flag, level in log_levels.items(): - if flag in args.extra: - logger.setLevel(level) - args.extra.remove(flag) - - res = utils.convert_args_to_dictionary(args.extra) - if res['return'] > 0: - return res - - run_args = res['args_dict'] - - run_args['mlc_run_cmd'] = " ".join(shlex.quote(arg) for arg in sys.argv) - - if hasattr(args, 'repo') and args.repo: - run_args['repo'] = args.repo - - if args.command in ['pull', 'rm', 'add', 'find']: - if args.target == "repo": - run_args['repo'] = args.details - - if args.command in ['docker', 'experiment', 'doc', 'lint']: - if args.target == "run": - run_args['target'] = 'script' #allowing run to be used for docker run instead of docker script - args.target = "script" - - if hasattr(args, 'details') and args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]: - run_args['tags'] = args.details - - if not run_args.get('details') and args.details: - run_args['details'] = args.details - - if args.command in ["cp", "mv"]: - run_args['target'] = args.target - if hasattr(args, 'details') and args.details: - run_args['src'] = args.details - if hasattr(args, 'extra') and args.extra: - run_args['dest'] = args.extra[0] + configure_logging(args) + run_args = build_run_args(args) if hasattr(args, "command") else {} if pre_args.help and not "tags" in run_args: help_text = "" @@ -283,18 +274,19 @@ def main(): print(help_text) sys.exit(0) - # Get the action handler for other commands action = get_action(args.target, default_parent) - # Dynamically call the method (e.g., run, list, show) - if action and hasattr(action, args.command): - method = getattr(action, args.command) - res = method(run_args) - if res['return'] > 0: - logger.error(res.get('error', f"Error in {action}")) - raise Exception(f"""An error occurred {res}""") - process_console_output(res, args.target, args.command, run_args) - else: - logger.error(f"Error: '{args.command}' is not supported for {args.target}.") + + if not action or not hasattr(action, args.command): + logging.error("Error: '%s' is not supported for %s.", args.command, args.target) + sys.exit(1) + + method = getattr(action, args.command) + res = method(run_args) + if res['return'] > 0: + logging.error(res.get('error', f"Error in {action}")) + raise Exception(f"An error occurred {res}") + + process_console_output(res, args.target, args.command, run_args) if __name__ == '__main__': main() From 18d5af1f34462c39de340cfbad78921027c11ca1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 15 Sep 2025 14:44:32 +0530 Subject: [PATCH 04/17] Refactor main --- mlc/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlc/main.py b/mlc/main.py index 159da5242..aa6455816 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -234,7 +234,7 @@ def main(): pre_args, remaining_args = pre_parser.parse_known_args() parser = build_parser(pre_args) - args = parser.parse_args() if remaining_args or pre_args.action else pre_args + args = parser.parse_args() if remaining_args or pre_args.target else pre_args configure_logging(args) run_args = build_run_args(args) if hasattr(args, "command") else {} From 7ccaf7e8472e8afffd4b2ced5c7360eaddec761d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 15 Sep 2025 14:53:19 +0530 Subject: [PATCH 05/17] Version upgrade --- VERSION | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 26aaba0e8..524cb5524 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.1.1 diff --git a/pyproject.toml b/pyproject.toml index b2251c285..3c5fe0218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1" +version = "1.1.1" From 66ed83620d8d97dbd49aa487853577d7f9c0ad5a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 2 Oct 2025 17:57:02 +0100 Subject: [PATCH 06/17] Fixes for local repo support (#186) --- .github/workflows/test-mlc-docker-core.yml | 8 ++++---- mlc/action.py | 2 ++ mlc/main.py | 11 +++++++---- mlc/repo_action.py | 11 ++++++----- mlc/script_action.py | 3 +++ 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-mlc-docker-core.yml b/.github/workflows/test-mlc-docker-core.yml index 47f902d4c..eced01d05 100644 --- a/.github/workflows/test-mlc-docker-core.yml +++ b/.github/workflows/test-mlc-docker-core.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.8"] + python-version: ["3.13", "3.8"] os: ["ubuntu-latest", "windows-latest", "macos-latest"] exclude: - os: windows-latest @@ -46,12 +46,12 @@ jobs: - name: Test --docker_dt for running in detached mode run: | - mlc docker run --tags=detect,os --docker_dt + mlc docker-run script --tags=detect,os --docker_dt docker stop $(docker ps -aq) - name: Test --docker_detached for running in detached mode run: | - mlc docker run --tags=detect,os --docker_detached + mlcd detect,os --docker_detached docker stop $(docker ps -aq) - name: Test --docker_cache @@ -64,4 +64,4 @@ jobs: - name: Test --dockerfile_recreate run: | - mlc docker run --tags=detect,os --docker_dt --docker_cache=no --docker_rebuild --dockerfile_recreate + mlc docker-run script --tags=detect,os --docker_dt --docker_cache=no --docker_rebuild --dockerfile_recreate diff --git a/mlc/action.py b/mlc/action.py index d662760b8..3debb9d98 100644 --- a/mlc/action.py +++ b/mlc/action.py @@ -42,6 +42,8 @@ def access(self, options): return {'return': 1, 'error': "'action' key is required in options"} #logger.info(f"options = {options}") + action_name = action_name.replace("-", "_") + action_target = options.get('target') if not action_target: action_target = options.get('automation', 'script') # Default to script if not provided diff --git a/mlc/main.py b/mlc/main.py index aa6455816..f9051ca87 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -140,7 +140,7 @@ def build_parser(pre_args): p.add_argument('extra', nargs=argparse.REMAINDER) # Script-only - for action in ['docker', 'experiment', 'doc', 'lint']: + for action in ['docker', 'docker-run', 'experiment', 'doc', 'lint']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['script', 'run']) p.add_argument('details', nargs='?', help='Details or identifier (optional)') @@ -172,7 +172,7 @@ def build_run_args(args): if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": run_args['repo'] = args.details - if args.command in ['docker', 'experiment', 'doc', 'lint'] and args.target == "run": + if args.command in ['docker', 'docker-run', 'experiment', 'doc', 'lint'] and args.target == "run": run_args['target'] = 'script' args.target = "script" @@ -208,7 +208,7 @@ def main(): | Target | Actions | |---------|-------------------------------------------------------| - | script | run, find/search, rm, mv, cp, add, test, docker, show | + | script | run, find/search, rm, mv, cp, add, test, docker-run, show | | cache | find/search, rm, show | | repo | pull, search, rm, list, find/search | @@ -235,6 +235,9 @@ def main(): parser = build_parser(pre_args) args = parser.parse_args() if remaining_args or pre_args.target else pre_args + + if hasattr(args, 'command') and args.command: + args.command = args.command.replace("-", "_") configure_logging(args) run_args = build_run_args(args) if hasattr(args, "command") else {} @@ -242,7 +245,7 @@ def main(): if pre_args.help and not "tags" in run_args: help_text = "" if pre_args.target == "run": - if pre_args.action == "docker": + if pre_args.action.startswith("docker"): pre_args.target = "script" else: logger.error(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") diff --git a/mlc/repo_action.py b/mlc/repo_action.py index 1fd6fe78e..a7dc67375 100644 --- a/mlc/repo_action.py +++ b/mlc/repo_action.py @@ -81,7 +81,7 @@ def add(self, run_args): return {'return': 1, "error": f"""Repo {run_args['repo']} already exists at {repo_path}"""} for repo in self.repos: if repo.path == i_repo_path: - return {'return': 1, "error": f"""Repo {run_args['repo']} already exists at {repo_path}"""} + return {'return': 1, "error": f"""Repo already exists at {repo.path}"""} if not os.path.exists(i_repo_path): #check if its an URL @@ -256,7 +256,7 @@ def github_url_to_user_repo_format(self, url): else: return {"return": 0, "value": os.path.basename(url).replace(".git", "")} - def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = None, ssh = None, ignore_on_conflict = False): + def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = None, ssh = None, ignore_on_conflict = False, repo_path = None): # Determine the checkout path from environment or default repo_base_path = self.repos_path # either the value will be from 'MLC_REPOS' @@ -287,7 +287,8 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No return res else: repo_download_name = res["value"] - repo_path = os.path.join(repo_base_path, repo_download_name) + if not repo_path: + repo_path = os.path.join(repo_base_path, repo_download_name) try: # If the directory doesn't exist, clone it @@ -420,9 +421,9 @@ def pull(self, run_args): repo_url = run_args.get('repo', run_args.get('url', 'repo')) if not repo_url or repo_url == "repo": for repo_object in self.repos: - if os.path.exists(os.path.join(repo_object.path, ".git")): + if os.path.exists(os.path.join(repo_object.path, ".git")) and os.access(repo_object.path, os.W_OK): repo_folder_name = os.path.basename(repo_object.path) - res = self.pull_repo(repo_folder_name) + res = self.pull_repo(repo_folder_name, repo_path = repo_object.path) if res['return'] > 0: return res else: diff --git a/mlc/script_action.py b/mlc/script_action.py index 245fd12c3..ee37722e3 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -253,6 +253,9 @@ def call_script_module_function(self, function_name, run_args): return {'return': 1, 'error': 'ScriptAutomation class not found in the script.'} def docker(self, run_args): + return self.docker_run(run_args) + + def docker_run(self, run_args): """ #################################################################################################################### Target: Script From 66597dc4212cb4877dd29be9f50e755b6ee88c01 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 12 Oct 2025 22:43:43 +0100 Subject: [PATCH 07/17] Support remote-run script action (#187) * Fix error message for repo add * Fixes #155, improve support for local repo * Use docker-run instead of docker for script action * Support - usage in mlc action * Support remote-run for mlc --- mlc/main.py | 4 ++++ mlc/script_action.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mlc/main.py b/mlc/main.py index f9051ca87..70c6ce71b 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -88,6 +88,10 @@ def mlcr(): mlc_expand_short("run") def mlcd(): mlc_expand_short("docker") +def mlcdr(): + mlc_expand_short("docker") +def mlcrr(): + mlc_expand_short("remote-run") def mlce(): mlc_expand_short("experiment") def mlct(): diff --git a/mlc/script_action.py b/mlc/script_action.py index ee37722e3..d5c0a2fb4 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -293,6 +293,30 @@ def docker_run(self, run_args): """ return self.call_script_module_function("docker", run_args) + def remote_run(self, run_args): + """ + #################################################################################################################### + Target: Script + Action: remote-run + #################################################################################################################### + + The `remote-run` action runs a shell command on a remote machine via ssh connection. + + + Flags Available: + + 1. --remote_host: + IP or hostnanme for the remote machine + 2. --remote_port: + ssh port for the remote machineIP or hostnanme for the remote machine + + Example Command: + + mlc remote-run script --tags=detect,os -j + + """ + return self.call_script_module_function("remote_run", run_args) + def run(self, run_args): """ From 49c3b1d1602c3396a8e723558e1db259b0e25188 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 20 Oct 2025 23:59:44 +0100 Subject: [PATCH 08/17] Support remote-run for mlc, fix for logging (#190) --- mlc/main.py | 6 +++--- mlc/script_action.py | 2 ++ pyproject.toml | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 70c6ce71b..60ee4cd88 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -144,7 +144,7 @@ def build_parser(pre_args): p.add_argument('extra', nargs=argparse.REMAINDER) # Script-only - for action in ['docker', 'docker-run', 'experiment', 'doc', 'lint']: + for action in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['script', 'run']) p.add_argument('details', nargs='?', help='Details or identifier (optional)') @@ -161,7 +161,7 @@ def configure_logging(args): args.extra[:] = [log_flag_aliases.get(a, a) for a in args.extra] for flag, level in log_levels.items(): if flag in args.extra: - logging.getLogger().setLevel(level) + logger.setLevel(level) args.extra.remove(flag) @@ -176,7 +176,7 @@ def build_run_args(args): if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": run_args['repo'] = args.details - if args.command in ['docker', 'docker-run', 'experiment', 'doc', 'lint'] and args.target == "run": + if args.command in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint'] and args.target == "run": run_args['target'] = 'script' args.target = "script" diff --git a/mlc/script_action.py b/mlc/script_action.py index d5c0a2fb4..7debbb784 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -235,6 +235,8 @@ def call_script_module_function(self, function_name, run_args): result = automation_instance.test(run_args) # Pass args to the run method elif function_name == "experiment": result = automation_instance.experiment(run_args) # Pass args to the experiment method + elif function_name == "remote_run": + result = automation_instance.remote_run(run_args) # Pass args to the experiment method elif function_name == "help": result = automation_instance.help(run_args) # Pass args to the help method elif function_name == "doc": diff --git a/pyproject.toml b/pyproject.toml index 3c5fe0218..3fe73f1b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.1" +version = "1.1.6" @@ -47,6 +47,7 @@ py-modules = [] [project.scripts] mlc = "mlc.main:main" mlcr = "mlc.main:mlcr" +mlcrr = "mlc.main:mlcrr" mlcd = "mlc.main:mlcd" mlce = "mlc.main:mlce" mlct = "mlc.main:mlct" From 1ea7ac0473de9e9407cf2b2eece833128b8f6c46 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 21 Oct 2025 02:18:24 +0100 Subject: [PATCH 09/17] Fix debug print (#191) * Prevent debug output for index saving --- mlc/index.py | 2 +- pyproject.toml | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mlc/index.py b/mlc/index.py index 6ee5b1fb5..64526ceb1 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -178,6 +178,6 @@ def _save_indices(self): try: with open(output_file, "w") as f: json.dump(index_data, f, indent=4, cls=CustomJSONEncoder) - logger.debug(f"Shared index for {folder_type} saved to {output_file}.") + #logger.debug(f"Shared index for {folder_type} saved to {output_file}.") except Exception as e: logger.error(f"Error saving shared index for {folder_type}: {e}") diff --git a/pyproject.toml b/pyproject.toml index 3fe73f1b1..6f1583d89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,9 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.6" +version = "1.1.7" - - -description = "An automation interface for ML applications" +description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ { name = "MLCommons", email = "systems@mlcommons.org" } ] From 744ba8d4d795ce8a03f19c62e468b3930653fac6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 21 Oct 2025 02:27:10 +0100 Subject: [PATCH 10/17] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 524cb5524..2bf1ca5f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.1 +1.1.7 From ff4d970750cfa0c1fa66e04a16338684dcfdf300 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 21 Oct 2025 02:45:45 +0100 Subject: [PATCH 11/17] Update README.md --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 814a7539e..b41df20d7 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Each target has its own set of specific actions to tailor automation workflows a | Target | Action | |--------|-----------------| -| script | run, find/search, rm, mv, cp, add, test, docker, show | +| script | run, find/search, rm, mv, cp, add, test, docker-run, show, experiment, doc | | cache | find/search, rm, show | | repo | pull, search, rm, list, find/search , add | @@ -115,9 +115,6 @@ classDiagram +show(args) +list(args) } - class CfgAction { - +load(args) - } class Index { +add(meta, folder_type, path, repo) +get_index(folder_type, uid) @@ -148,8 +145,6 @@ classDiagram Action <|-- RepoAction Action <|-- ScriptAction Action <|-- CacheAction - Action <|-- ExperimentAction - Action <|-- CfgAction RepoAction o-- Repo ScriptAction o-- Automation CacheAction o-- Index From a74ccf9c85c836f65958d74494f008847f5748f4 Mon Sep 17 00:00:00 2001 From: MUHAMMED SINAN D Date: Sat, 25 Oct 2025 18:41:45 +0530 Subject: [PATCH 12/17] implementation done, successfully added auto pulling mechanism (#193) Thank you for the PR. LGTM --- mlc/script_action.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mlc/script_action.py b/mlc/script_action.py index 7debbb784..aebe08825 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -219,7 +219,25 @@ def call_script_module_function(self, function_name, run_args): # Import script submodule script_path = self.find_target_folder("script") if not script_path: - return {'return': 1, 'error': f"""Script automation not found. Have you done "mlc pull repo mlcommons@mlperf-automations --branch=dev"?"""} + logger.warning("Script automation not found. Automatically pulling mlcommons@mlperf-automations repository...") + + # Use the access method to pull the required repository + result = self.access({ + "automation": "repo", + "action": "pull", + "repo": "mlcommons@mlperf-automations", + "branch": "dev" + }) + + if result['return'] == 0: + # Try to find the script path again after pulling + script_path = self.find_target_folder("script") + if not script_path: + return {'return': 1, 'error': f"""Script automation still not found after pulling mlcommons@mlperf-automations --branch=dev."""} + else: + # If pull failed, return the original error with additional info + logger.error(f"Failed to pull mlcommons@mlperf-automations repository: {result.get('error', 'Unknown error')}") + return {'return': 1, 'error': f"""Script automation not found and failed to automatically pull mlcommons@mlperf-automations --branch=dev. Please run "mlc pull repo mlcommons@mlperf-automations --branch=dev" manually: {result.get('error', 'Unknown error')}"""} module_path = os.path.join(script_path, "module.py") module = self.dynamic_import_module(module_path) From 8d59ea72fff3cfbe882e61c64fd704253b8f66c3 Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Sat, 25 Oct 2025 16:16:05 +0100 Subject: [PATCH 13/17] Fix mlc_run_cmd (#194) --- mlc/main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 60ee4cd88..8ec3604fb 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -74,9 +74,11 @@ def search(self, i): return {'return': 0, 'list': result} #indices +mlc_run_cmd = None def mlc_expand_short(action, target = "script"): - + global mlc_run_cmd + mlc_run_cmd = shlex.join(sys.argv) # Insert the positional argument into sys.argv for the main function sys.argv.insert(1, action) sys.argv.insert(2, target) @@ -166,18 +168,21 @@ def configure_logging(args): def build_run_args(args): + global mlc_run_cmd res = utils.convert_args_to_dictionary(getattr(args, 'extra', [])) if res['return'] > 0: return res run_args = res['args_dict'] - run_args['mlc_run_cmd'] = shlex.join(sys.argv) + if not mlc_run_cmd: + mlc_run_cmd = shlex.join(sys.argv) + run_args['mlc_run_cmd'] = mlc_run_cmd if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": run_args['repo'] = args.details if args.command in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint'] and args.target == "run": - run_args['target'] = 'script' + #run_args['target'] = 'script' #dont modify this as script might have target as in input args.target = "script" if args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]: From 74360c06488a8256077e213c3d0cb41f84ad4aea Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 25 Oct 2025 16:19:17 +0100 Subject: [PATCH 14/17] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2bf1ca5f5..18efdb9ae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.7 +1.1.8 From afbea54ded22c90098fb3cb8e2df825b74bc6493 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 28 Oct 2025 23:16:33 +0000 Subject: [PATCH 15/17] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6f1583d89..be24a4d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.7" +version = "1.1.8" description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ From 0f2e8f63b5f7e4d39682172e969b2fd90ff812c7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 28 Oct 2025 23:17:02 +0000 Subject: [PATCH 16/17] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 18efdb9ae..512a1faa6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.8 +1.1.9 From e34b924ea25afcd4338a4dae2c76efaec64cd099 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Sat, 8 Nov 2025 19:38:10 +0530 Subject: [PATCH 17/17] Added support for show repos command (#196) --- mlc/main.py | 18 ++++++++++++------ mlc/repo_action.py | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 8ec3604fb..0dbbe9d05 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -129,7 +129,7 @@ def process_console_output(res, target, action, run_args): def build_pre_parser(): pre_parser = argparse.ArgumentParser(add_help=False) pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)") - pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo'], nargs="?", help="Target (repo, script, cache, ...)") + pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo', 'repos'], nargs="?", help="Target (repo, script, cache, ...)") pre_parser.add_argument("-h", "--help", action="store_true") return pre_parser @@ -141,7 +141,7 @@ def build_parser(pre_args): # General commands for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv', 'help']: p = subparsers.add_parser(action, add_help=False) - p.add_argument('target', choices=['repo', 'script', 'cache']) + p.add_argument('target', choices=['repo', 'repos', 'script', 'cache']) p.add_argument('details', nargs='?', help='Details or identifier (optional)') p.add_argument('extra', nargs=argparse.REMAINDER) @@ -215,11 +215,11 @@ def main(): Each target has a specific set of actions to tailor automation workflows, as shown below: - | Target | Actions | - |---------|-------------------------------------------------------| + | Target | Actions | + |---------|-----------------------------------------------------------| | script | run, find/search, rm, mv, cp, add, test, docker-run, show | - | cache | find/search, rm, show | - | repo | pull, search, rm, list, find/search | + | cache | find/search, rm, show | + | repo | pull, search, rm, list, find/search | Example: mlc run script detect-os @@ -286,6 +286,12 @@ def main(): print(help_text) sys.exit(0) + # show repos alias list repo + if args.command in ("show"): + args.command = "list" + if args.target == "repos": + args.target = "repo" + action = get_action(args.target, default_parent) if not action or not hasattr(action, args.command): diff --git a/mlc/repo_action.py b/mlc/repo_action.py index a7dc67375..7d6908302 100644 --- a/mlc/repo_action.py +++ b/mlc/repo_action.py @@ -447,8 +447,8 @@ def pull(self, run_args): def list(self, run_args): """ #################################################################################################################### - Target: Repo - Action: List + Target: Repo/Repos + Action: List/Show #################################################################################################################### The `list` action displays all registered MLC repositories along with their aliases and paths.