From 438d8529069c6bb8c0171ca8e27a9cb8b0310c53 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 13:39:58 +0100 Subject: [PATCH 01/24] hopla/ccc: deal with multiple backends. --- ..._tasks.py => plot_ccc_flux_multi_tasks.py} | 1 + examples/plot_ccc_joblib_multi_tasks.py | 76 +++++++++++++++++++ hopla/ccc.py | 56 +++++++++++--- hopla/executor.py | 10 ++- hopla/utils.py | 6 ++ 5 files changed, 137 insertions(+), 12 deletions(-) rename examples/{plot_ccc_multi_tasks.py => plot_ccc_flux_multi_tasks.py} (98%) create mode 100644 examples/plot_ccc_joblib_multi_tasks.py diff --git a/examples/plot_ccc_multi_tasks.py b/examples/plot_ccc_flux_multi_tasks.py similarity index 98% rename from examples/plot_ccc_multi_tasks.py rename to examples/plot_ccc_flux_multi_tasks.py index 90cca08..57a75e9 100644 --- a/examples/plot_ccc_multi_tasks.py +++ b/examples/plot_ccc_flux_multi_tasks.py @@ -32,6 +32,7 @@ image="/tmp/hopla/my-docker-img.tar", walltime=1, project_id="genXXX", + backend="flux", ) diff --git a/examples/plot_ccc_joblib_multi_tasks.py b/examples/plot_ccc_joblib_multi_tasks.py new file mode 100644 index 0000000..2574d8e --- /dev/null +++ b/examples/plot_ccc_joblib_multi_tasks.py @@ -0,0 +1,76 @@ +""" +Basic example on how to use the CCC cluster using multi-tasks +============================================================= + +Basic example + +When you're running hundreds or thousands of jobs, automation is a necessity. +This is where ``hopla`` can help you. + +A simple example of how to use ``hopla`` on a CCC cluster. Please check +the :ref:`user guide ` for a more in depth presentation of all +functionalities. + + +Imports +------- +""" + +import hopla +import numpy as np +from pprint import pprint + + +# %% +# Executor Context +# ---------------- + +executor = hopla.Executor( + cluster="ccc", + folder="/tmp/hopla", + queue="rome", + image="/tmp/hopla/my-docker-img.tar", + walltime=1, + project_id="genXXX", + backend="joblib", +) + + +# %% +# Submit Jobs +# ----------- + +chunks = np.array_split(range(1, 11), 3) +jobs = [ + executor.submit([hopla.DelayedSubmission("sleep", k) for k in c]) + for c in chunks +] +pprint(jobs) +print(jobs[0].delayed_submission) + + +# %% +# Generate a batch +# ---------------- + +jobs[0].generate_batch() +print(jobs[0].paths) +batch = jobs[0].paths.submission_file +with open(batch) as of: + print(of.read()) +script = jobs[0].paths.joblib_file +with open(script) as of: + print(of.read()) + + +# %% +# Start Jobs +# ---------- +# +# We can't execute the code on the CI since the CCC infrastructure is not +# available. + +from hopla.config import Config + +with Config(dryrun=True, delay_s=3): + executor(max_jobs=2) diff --git a/hopla/ccc.py b/hopla/ccc.py index 0ac0e89..b7076c3 100644 --- a/hopla/ccc.py +++ b/hopla/ccc.py @@ -74,18 +74,24 @@ class DelayedCCCJob(DelayedJob): base job executor. job_id: str the job identifier. + backend: str, default 'flux' + the multi-taks backend to use: 'flux' or 'joblib'. """ _hub = "n4h00001rs" _submission_cmd = "ccc_msub" _container_cmd = "pcocc-rs run {hub}:{image_name} {params} -- {command}" - def __init__(self, delayed_submission, executor, job_id): + def __init__(self, delayed_submission, executor, job_id, backend="flux"): super().__init__(delayed_submission, executor, job_id) self.multi_task = isinstance(delayed_submission, (list, tuple)) + self.backend = backend resource_dir = Path(__file__).parent / "resources" - if self.multi_task: + if self.multi_task and self.backend == "flux": path = resource_dir / "ccc_multi_batch_template.txt" self.worker_file = resource_dir / "worker.sh" + elif self.multi_task and self.backend == "joblib": + path = resource_dir / "ccc_batch_template.txt" + self.worker_file = resource_dir / "joblib_script_template.txt" else: path = resource_dir / "ccc_batch_template.txt" with open(path) as of: @@ -118,7 +124,11 @@ def generate_batch(self): params = copy.deepcopy(self._executor.parameters) params["walltime"] *= 3600 params["memory"] *= 1000 - print(params["modules"]) + if self.backend == "joblib": + if params["modules"] != "": + params["modules"] = f"python3/3.12,{params['modules']}" + else: + params["modules"] = "python3/3.12" if params["modules"] != "": params["modules"] = f"module load {params['modules']}" try: @@ -130,7 +140,7 @@ def generate_batch(self): stacklevel=2 ) print(err) - if self.multi_task: + if self.multi_task and self.backend == "flux": n_multi_cpus = self._executor.parameters["nmulticpus"] shutil.copy(self.worker_file, self.paths.worker_file) subcmds = [ @@ -150,6 +160,31 @@ def generate_batch(self): with open(self.paths.task_file, "w") as of: of.write("\n".join(subcmds)) cmd = self.paths.task_file + elif self.multi_task and self.backend == "joblib": + n_cpus = self._executor.parameters["ncpus"] + with open(self.worker_file) as of: + joblib_template = of.read() + subcmds = [ + self._container_cmd.format( + hub=self._hub, + image_name=self.image_name, + params=submission.execution_parameters, + command=submission.command + ) + for submission in self.delayed_submission + ] + subcmds = [ + f"'{command}'," + for command in subcmds + ] + with open(self.paths.joblib_file, "w") as of: + of.write( + joblib_template.format( + commands="\n".join(subcmds), + njobs=n_cpus, + ) + ) + cmd = f"python {self.paths.joblib_file}" else: cmd = self._container_cmd.format( hub=self._hub, @@ -162,11 +197,14 @@ def generate_batch(self): os.remove(self.paths.stdout) if self.paths.stderr.exists(): os.remove(self.paths.stderr) - of.write(self.template.format( - command=cmd, - stdout=self.paths.stdout, - stderr=self.paths.stderr, - **params)) + of.write( + self.template.format( + command=cmd, + stdout=self.paths.stdout, + stderr=self.paths.stderr, + **params + ) + ) def import_image(self): """ Load the docker image if not available. diff --git a/hopla/executor.py b/hopla/executor.py index 00ed62e..91b168e 100644 --- a/hopla/executor.py +++ b/hopla/executor.py @@ -59,11 +59,13 @@ class Executor: n_gpus: int, default 0 the number of GPUs allocated for each job. n_multi_cpus: int, default 1 - the number of cores reserved fir each multi-tasks job. + the number of cores reserved for each multi-tasks job. modules: list of str, default None the environment modules to be loaded. project_id: str, default None the project ID where you have computing hours. + backend: str, default 'flux' + the multi-taks backend to use: 'flux' or 'jobli Examples -------- @@ -89,7 +91,7 @@ class Executor: def __init__(self, cluster, folder, queue, image, name="hopla", memory=2, walltime=72, n_cpus=1, n_gpus=0, n_multi_cpus=1, modules=None, - project_id=None): + project_id=None, backend="flux"): if cluster == "pbs": self._job_class = DelayedPbsJob self._watcher_class = PbsInfoWatcher @@ -103,6 +105,7 @@ def __init__(self, cluster, folder, queue, image, name="hopla", memory=2, raise ValueError( f"Unsupported cluster type: {cluster}" ) + self.backend = backend self.watcher = self._watcher_class(self._delay_s) self.folder = Path(folder).expanduser().absolute() modules = modules or [] @@ -180,7 +183,8 @@ def submit(self, script, *args, execution_parameters=None, **kwargs): job = self._job_class( script, self, - self._counter + self._counter, + backend=self.backend, ) else: job = self._job_class( diff --git a/hopla/utils.py b/hopla/utils.py index c140f6c..afdbf86 100644 --- a/hopla/utils.py +++ b/hopla/utils.py @@ -102,6 +102,12 @@ def worker_file(self): """ return self.submission_folder / "worker.sh" + @property + def joblib_file(self): + """ Generate the joblib file location. + """ + return self.submission_folder / f"{self.job_id}_joblib_script.py" + @property def flux_dir(self): """ Generate the flux output dir. From e445fadc262b1e4ae13cf65572ed630dd944697c Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 13:42:25 +0100 Subject: [PATCH 02/24] hopla/resources/joblib_script_template: add resource. --- hopla/resources/joblib_script_template.txt | 63 ++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 hopla/resources/joblib_script_template.txt diff --git a/hopla/resources/joblib_script_template.txt b/hopla/resources/joblib_script_template.txt new file mode 100644 index 0000000..df8291c --- /dev/null +++ b/hopla/resources/joblib_script_template.txt @@ -0,0 +1,63 @@ +########################################################################## +# Hopla - Copyright (C) AGrigis, 2015 - 2025 +# Distributed under the terms of the CeCILL-B license, as published by +# the CEA-CNRS-INRIA. Refer to the LICENSE file or to +# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html +# for details. +########################################################################## + +import sys +import subprocess +from joblib import Parallel, delayed + + +def run_command(cmd): + """ + Run a single command line string. + Returns a dictionary with command, status code, stdout, stderr. + """ + try: + result = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True + ) + return {{ + "command": cmd, + "returncode": result.returncode, + "stdout": result.stdout.strip(), + "stderr": result.stderr.strip() + }} + except Exception as e: + return {{ + "command": cmd, + "returncode": -1, + "stdout": "", + "stderr": str(e) + }} + + +if __name__ == "__main__": + + commands = [ + {commands} + ] + n_jobs = {njobs} + + results = Parallel(n_jobs=n_jobs)( + delayed(run_command)(cmd) for cmd in commands + ) + + for item in results: + print("="*40) + print(f"Command : {{item['command']}}") + print(f"Returncode: {{item['returncode']}}") + print(f"Stdout : {{item['stdout']}}") + print(f"Stderr : {{item['stderr']}}") + + if any(item["returncode"] != 0 for item in results): + sys.exit(1) + else: + sys.exit(0) + From fcbb61c0494c99bb3d95cbaf7deea050dd5486e2 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 15:38:57 +0100 Subject: [PATCH 03/24] hopla: fix CI. --- hopla/executor.py | 15 +++++++++++---- hopla/tests/test_examples.py | 8 ++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hopla/executor.py b/hopla/executor.py index 91b168e..92fdfcf 100644 --- a/hopla/executor.py +++ b/hopla/executor.py @@ -111,10 +111,15 @@ def __init__(self, cluster, folder, queue, image, name="hopla", memory=2, modules = modules or [] self.parameters = { "name": name, - "queue": queue, "memory": memory, "walltime": walltime, - "ncpus": n_cpus, "nmulticpus": n_multi_cpus, "ngpus": n_gpus, + "queue": queue, + "memory": memory, + "walltime": walltime, + "ncpus": n_cpus, + "nmulticpus": n_multi_cpus, + "ngpus": n_gpus, "modules": ",".join(modules), - "image": image, "project_id": project_id + "image": Path(image).expanduser().absolute(), + "project_id": project_id } self._delayed_jobs = [] @@ -146,9 +151,11 @@ def __call__(self, max_jobs=300): for job in self._delayed_jobs[_start:_stop]: assert job.status == "NOTSTARTED" job.start(dryrun=dryrun) + pbar.update(1) + pbar.refresh() _start = _stop - pbar.update(_delta) time.sleep(self._delay_s) + pbar.close() self.watcher.update() def submit(self, script, *args, execution_parameters=None, **kwargs): diff --git a/hopla/tests/test_examples.py b/hopla/tests/test_examples.py index 5d2fc7c..35f23a0 100644 --- a/hopla/tests/test_examples.py +++ b/hopla/tests/test_examples.py @@ -29,8 +29,12 @@ def test_ccc(self): script_path = self.examples_dir / "plot_ccc.py" runpy.run_path(str(script_path)) - def test_ccc_multi_tasks(self): - script_path = self.examples_dir / "plot_ccc_multi_tasks.py" + def test_ccc_flux_multi_tasks(self): + script_path = self.examples_dir / "plot_ccc_flux_multi_tasks.py" + runpy.run_path(str(script_path)) + + def test_ccc_joblib_multi_tasks(self): + script_path = self.examples_dir / "plot_ccc_joblib_multi_tasks.py" runpy.run_path(str(script_path)) From 493c2538ec005e8663c90e3dfb13c35b5d9cd138 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 15:45:22 +0100 Subject: [PATCH 04/24] doc/highlights: fix CI. --- doc/highlights.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/highlights.csv b/doc/highlights.csv index ee27f34..02176d6 100644 --- a/doc/highlights.csv +++ b/doc/highlights.csv @@ -1,3 +1,4 @@ +sphx_glr_auto_examples_plot_slurm.py,/auto_examples/images/thumb/sphx_glr_plot_slurm_thumb.png,Basic example on how to use the SLURM cluster...,Basic example sphx_glr_auto_examples_plot_pbs.py,/auto_examples/images/thumb/sphx_glr_plot_pbs_thumb.png,Basic example on how to use the PBS cluster...,Basic example sphx_glr_auto_examples_plot_ccc.py,/auto_examples/images/thumb/sphx_glr_plot_ccc_thumb.png,Basic example on how to use the CCC cluster...,Basic example -sphx_glr_auto_examples_plot_ccc_multi_tasks.py,/auto_examples/images/thumb/sphx_glr_plot_ccc_multi_tasks_thumb.png,Basic example on how to use the CCC cluster using multi-tasks...,Basic example +sphx_glr_auto_examples_plot_ccc_flux_multi_tasks.py,/auto_examples/images/thumb/sphx_glr_plot_ccc_flux_multi_tasks_thumb.png,Basic example on how to use the CCC cluster using multi-tasks...,Basic example From c9653cebe4f1f8204dd9a72098875fd376630ec1 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 16:20:03 +0100 Subject: [PATCH 05/24] hopla/cli: add CLI interface. --- examples/data.tsv | 4 + examples/experiment.toml | 23 ++++++ examples/plot_hoplactl.py | 19 +++++ hopla/cli.py | 163 ++++++++++++++++++++++++++++++++++++++ hopla/executor.py | 2 +- pyproject.toml | 6 ++ 6 files changed, 216 insertions(+), 1 deletion(-) create mode 100755 examples/data.tsv create mode 100644 examples/experiment.toml create mode 100644 examples/plot_hoplactl.py create mode 100644 hopla/cli.py diff --git a/examples/data.tsv b/examples/data.tsv new file mode 100755 index 0000000..cfe3879 --- /dev/null +++ b/examples/data.tsv @@ -0,0 +1,4 @@ +participant_id k +10 1 +11 2 +12 3 diff --git a/examples/experiment.toml b/examples/experiment.toml new file mode 100644 index 0000000..454004e --- /dev/null +++ b/examples/experiment.toml @@ -0,0 +1,23 @@ +[project] +name = "SleepTest" +operator = "MyName" +date = "12/02/2020" + +[inputs] +commands = "sleep {k}s" +# commands = ["sleep 1", "sleep 2", "sleep 3"] +data = "./examples/data.tsv" +parameters = "--cleanenv" + +[environment] +cluster = "slurm" +folder = "/tmp/hopla" +queue = "Nspin_short" +walltime = 1 +n_cpus = 1 +image = "./ubuntu-jammy.sif" + +[config] +dryrun = true +delay_s = 1 +verbose = false diff --git a/examples/plot_hoplactl.py b/examples/plot_hoplactl.py new file mode 100644 index 0000000..661170b --- /dev/null +++ b/examples/plot_hoplactl.py @@ -0,0 +1,19 @@ +""" +Basic example on how to use the CLI +=================================== + +Basic example + +When you're running hundreds or thousands of jobs, automation is a necessity. +This is where ``hopla`` can help you. + +A simple example of how to use ``hopla`` on a cluster using the CLI interface. +Please check the :ref:`user guide ` for a more in depth +presentation of all functionalities. +""" + +import subprocess + + +command = ["hoplacli", "--config", "./examples/experiment.toml", "--njobs 2"] +subprocess.check_call(command) diff --git a/hopla/cli.py b/hopla/cli.py new file mode 100644 index 0000000..31a5490 --- /dev/null +++ b/hopla/cli.py @@ -0,0 +1,163 @@ +########################################################################## +# Hopla - Copyright (C) AGrigis, 2015 - 2025 +# Distributed under the terms of the CeCILL-B license, as published by +# the CEA-CNRS-INRIA. Refer to the LICENSE file or to +# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html +# for details. +########################################################################## + +import argparse + +import numpy as np +import pandas as pd +import tomllib + +import hopla +from hopla.config import Config + + +def main(): + """ + Command-line interface for automated job execution with hopla. + + This function parses command-line arguments, loads a TOML configuration + file, initializes a hopla executor, and submits jobs either individually + or in chunks depending on the configuration. It then runs the executor + with a specified maximum number of jobs and writes a report to disk. + + Parameters + ---------- + --config : str, required + Path to an experiment TOML configuration file. The file must contain + sections for `project`, `environment`, `inputs`, and `config`. + Optionally, a `multi` section can be provided to split commands into + chunks. + --njobs : int, required + The maximum number of job submissions to execute concurrently. + + Workflow + -------- + 1. Parse CLI arguments using argparse. + 2. Load the TOML configuration file with `tomllib`. + 3. Initialize a `hopla.Executor` with environment parameters. + 4. Extract commands from the configuration: + - If `multi` is defined, split commands into chunks and submit them + as delayed submissions. + - Otherwise, submit commands directly. + 5. Run the executor with the specified maximum number of jobs. + 6. Write a textual report to `report.txt` inside the executor's folder. + + TOML Configuration + ------------------ + The configuration file is structured into sections: + + [project] + name : str + Name of the project. + operator : str + Person responsible for running the analysis. + date : str + Date of the experiment in DD/MM/YYYY format. + + [inputs] + commands : str or list + Commands to execute. Can be a Python expression string (e.g., + "sleep {k}") or a list of commands. + data : str + A TSV file used to fill the previous Python expression string. + Column names must match expression (e.g., "k" in the previous + example). + parameters : str + Additional parameters passed to the container execution command + (e.g., "--cleanenv"). + + [environment] + cluster : str + Cluster type (e.g., "pbs"). + folder : str + Working directory for job execution (e.g., "/tmp/hopla"). + queue : str + Queue or partition name (e.g., "Nspin_short"). + walltime : int + Maximum walltime in hours for each job. + n_cpus : int + Number of CPUs allocated per job. + image : str + Path to container image used for execution. + + [config] + dryrun : bool + If true, simulate job submission without executing. + delay_s : int + Delay in seconds between submissions. + verbose : bool + If true, enable verbose logging. + + Examples + -------- + >>> hoplactl --config experiment.toml --njobs 5 + + Notes + ----- + - The `multi` section should define `n_splits` to control chunking. + - The `Config` context manager is used to apply configuration settings + during execution. + """ + parser = argparse.ArgumentParser( + prog="hoplactl", + description=( + "Automate job execution with hopla using a configuration file." + ), + ) + parser.add_argument( + "--config", + type=str, + required=True, + help="An experiment TOML configuration file." + ) + parser.add_argument( + "--njobs", + type=int, + required=True, + help="The number of submissions." + ) + args = parser.parse_args() + + with open(args.config, "rb") as of: + config = tomllib.load(of) + + executor = hopla.Executor( + **config["environment"] + ) + + commands = config["inputs"]["commands"] + if not isinstance(commands, (list, tuple)): + df = pd.read_csv(config["inputs"]["data"], sep="\t") + commands = [commands.format(**dict(row)) for _, row in df.iterrows()] + if config.get("multi") is not None: + chunks = np.array_split(commands, config["multi"]["n_splits"]) + jobs = [ + executor.submit( + [hopla.DelayedSubmission(cmd) for cmd in subcmds], + execution_parameters=config["inputs"].get("parameters"), + ) for subcmds in chunks + ] + else: + jobs = [ + executor.submit( + cmd, + execution_parameters=config["inputs"].get("parameters"), + ) for cmd in commands + ] + print(jobs) + + with Config(**config["config"]): + executor(max_jobs=args.njobs) + + report_file = executor.folder / "report.txt" + with open(report_file, "w") as of: + of.write(executor.report) + + +if __name__ == "__main__": + main() diff --git a/hopla/executor.py b/hopla/executor.py index 92fdfcf..1b94c8d 100644 --- a/hopla/executor.py +++ b/hopla/executor.py @@ -166,7 +166,7 @@ def submit(self, script, *args, execution_parameters=None, **kwargs): script: Path/str or list of DelayedSubmission script(s) to execute. *args: any positional argument of the script. - execution_parameters: str or list of str + execution_parameters: str parameters passed to the container during execution. **kwargs: any named argument of the script. diff --git a/pyproject.toml b/pyproject.toml index 9f3e0e5..4a1b862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ ] dynamic = ["version"] +[project.scripts] +hoplacli = "hopla.cli:main" + [project.urls] Source = "https://github.com/AGrigis/hopla" Tracker = "https://github.com/AGrigis/hopla/issues" @@ -64,6 +67,9 @@ extend-select = [ "UP", # pyupgrade "FURB", # refurb "RUF", + "W293", # blank-line-with-whitespace + "W291", # trailing whitespace + ] ignore = [ ] From 4f2fedb0f86d3270d2cd2af21126b7b6d6cd13d0 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 16:34:47 +0100 Subject: [PATCH 06/24] hopla/cli: fix CI. --- hopla/cli.py | 30 ++++++++++++++++-------------- pyproject.toml | 3 ++- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/hopla/cli.py b/hopla/cli.py index 31a5490..028e812 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -25,16 +25,6 @@ def main(): or in chunks depending on the configuration. It then runs the executor with a specified maximum number of jobs and writes a report to disk. - Parameters - ---------- - --config : str, required - Path to an experiment TOML configuration file. The file must contain - sections for `project`, `environment`, `inputs`, and `config`. - Optionally, a `multi` section can be provided to split commands into - chunks. - --njobs : int, required - The maximum number of job submissions to execute concurrently. - Workflow -------- 1. Parse CLI arguments using argparse. @@ -95,7 +85,7 @@ def main(): Examples -------- - >>> hoplactl --config experiment.toml --njobs 5 + >>> hoplactl --config experiment.toml --njobs 5 # doctest: +SKIP Notes ----- @@ -106,20 +96,32 @@ def main(): parser = argparse.ArgumentParser( prog="hoplactl", description=( - "Automate job execution with hopla using a configuration file." + "Automate job execution with hopla using a configuration file.\n\n" + "This function parses command-line arguments, loads a TOML " + "configuration file, initializes a hopla executor, and submits " + "jobs either individually or in chunks depending on the " + "configuration. It then runs the executor with a specified " + "maximum number of jobs and writes a report to disk." ), + epilog="Notes:\n- Use a valid TOML file.\n- See docs for examples.", + formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument( "--config", type=str, required=True, - help="An experiment TOML configuration file." + help=( + "Path to an experiment TOML configuration file. The file must " + "contain sections for `project`, `environment`, `inputs`, and " + "`config`. Optionally, a `multi` section can be provided to split " + "commands into chunks." + ) ) parser.add_argument( "--njobs", type=int, required=True, - help="The number of submissions." + help="The maximum number of job submissions to execute concurrently." ) args = parser.parse_args() diff --git a/pyproject.toml b/pyproject.toml index 4a1b862..2ef0053 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "hopla" readme = "README.rst" -requires-python = ">=3.9" +requires-python = ">=3.11" authors = [ {name = "hopla developers", email = "antoine.grigis@cea.fr"}, ] @@ -19,6 +19,7 @@ classifiers = [ ] dependencies = [ "tqdm", + "pandas", ] dynamic = ["version"] From 3d3c4408e09f4cf34b867e982560d1e5de2dadb2 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 16:52:20 +0100 Subject: [PATCH 07/24] hopla/cli: fix CI. --- doc/user_guide/index.rst | 12 ++++++++++++ examples/plot_hoplactl.py | 19 ------------------- hopla/cli.py | 10 +++++----- 3 files changed, 17 insertions(+), 24 deletions(-) delete mode 100644 examples/plot_hoplactl.py diff --git a/doc/user_guide/index.rst b/doc/user_guide/index.rst index d33cda9..1b6bcec 100644 --- a/doc/user_guide/index.rst +++ b/doc/user_guide/index.rst @@ -20,3 +20,15 @@ Table of contents introduction.rst how_it_works.rst clusters.rst + +CLI +=== + +``hopla`` has a CLI interface. The `hoplacli` command loads a TOML +configuration file, initializes a hopla executor, and submits jobs either +individually or in chunks depending on the configuration. It then runs the +executor with a specified maximum number of jobs and writes a report to disk. + +.. code-block:: bash + + hoplacli --config ./examples/experiment.toml --njobs 2 diff --git a/examples/plot_hoplactl.py b/examples/plot_hoplactl.py deleted file mode 100644 index 661170b..0000000 --- a/examples/plot_hoplactl.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Basic example on how to use the CLI -=================================== - -Basic example - -When you're running hundreds or thousands of jobs, automation is a necessity. -This is where ``hopla`` can help you. - -A simple example of how to use ``hopla`` on a cluster using the CLI interface. -Please check the :ref:`user guide ` for a more in depth -presentation of all functionalities. -""" - -import subprocess - - -command = ["hoplacli", "--config", "./examples/experiment.toml", "--njobs 2"] -subprocess.check_call(command) diff --git a/hopla/cli.py b/hopla/cli.py index 028e812..3113486 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -7,10 +7,10 @@ ########################################################################## import argparse +import tomllib import numpy as np import pandas as pd -import tomllib import hopla from hopla.config import Config @@ -20,10 +20,10 @@ def main(): """ Command-line interface for automated job execution with hopla. - This function parses command-line arguments, loads a TOML configuration - file, initializes a hopla executor, and submits jobs either individually - or in chunks depending on the configuration. It then runs the executor - with a specified maximum number of jobs and writes a report to disk. + This function loads a TOML configuration file, initializes a hopla + executor, and submits jobs either individually or in chunks depending + on the configuration. It then runs the executor with a specified maximum + number of jobs and writes a report to disk. Workflow -------- From 4a41708b85c58b7ab312c60f34be025c9ddd0105 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Thu, 11 Dec 2025 16:58:31 +0100 Subject: [PATCH 08/24] doc/user_guide: reorganize sections. --- doc/user_guide/cli.rst | 13 +++++++++++++ doc/user_guide/index.rst | 13 +------------ 2 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 doc/user_guide/cli.rst diff --git a/doc/user_guide/cli.rst b/doc/user_guide/cli.rst new file mode 100644 index 0000000..4889b18 --- /dev/null +++ b/doc/user_guide/cli.rst @@ -0,0 +1,13 @@ +.. _cli: + +CLI +=== + +``hopla`` has a CLI interface. The `hoplacli` command loads a TOML +configuration file, initializes a hopla executor, and submits jobs either +individually or in chunks depending on the configuration. It then runs the +executor with a specified maximum number of jobs and writes a report to disk. + +.. code-block:: bash + + hoplacli --config ./examples/experiment.toml --njobs 2 diff --git a/doc/user_guide/index.rst b/doc/user_guide/index.rst index 1b6bcec..902d2aa 100644 --- a/doc/user_guide/index.rst +++ b/doc/user_guide/index.rst @@ -20,15 +20,4 @@ Table of contents introduction.rst how_it_works.rst clusters.rst - -CLI -=== - -``hopla`` has a CLI interface. The `hoplacli` command loads a TOML -configuration file, initializes a hopla executor, and submits jobs either -individually or in chunks depending on the configuration. It then runs the -executor with a specified maximum number of jobs and writes a report to disk. - -.. code-block:: bash - - hoplacli --config ./examples/experiment.toml --njobs 2 + cli.rst From 7ce38395f8063de82af32efa05fdfa8d1263065e Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Fri, 12 Dec 2025 10:28:44 +0100 Subject: [PATCH 09/24] hopla/cli: add display. --- examples/experiment.toml | 1 - hopla/cli.py | 165 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 156 insertions(+), 10 deletions(-) diff --git a/examples/experiment.toml b/examples/experiment.toml index 454004e..f56314e 100644 --- a/examples/experiment.toml +++ b/examples/experiment.toml @@ -6,7 +6,6 @@ date = "12/02/2020" [inputs] commands = "sleep {k}s" # commands = ["sleep 1", "sleep 2", "sleep 3"] -data = "./examples/data.tsv" parameters = "--cleanenv" [environment] diff --git a/hopla/cli.py b/hopla/cli.py index 3113486..51227f7 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -7,7 +7,11 @@ ########################################################################## import argparse +import datetime +import re +import shutil import tomllib +from pathlib import Path import numpy as np import pandas as pd @@ -15,6 +19,134 @@ import hopla from hopla.config import Config +# Colors (ANSI) +RESET = "\033[0m" +BOLD = "\033[1m" +CYAN = "\033[36m" +MAGENTA = "\033[35m" +DIM = "\033[2m" + + +def print_hoplacli_header( + license="CeCILL-B", + subtitle="Fast, friendly CLI to submit you jobs"): + """ + Print a styled header banner for the hoplacli tool. + + Parameters + ---------- + license : str, optional + The license displayed in the header. Default is "CeCILL-B". + subtitle : str, optional + A secondary line of text displayed below the title. + Default is "Fast, friendly CLI to submit you jobs". + + Notes + ----- + - The header is styled with ANSI escape codes for colors and bold text. + - The width of the banner adapts to the terminal size (up to 100 + characters). + - Includes a timestamp of when the session started. + - Uses box-drawing characters for a clean framed look. + """ + # Terminal width + width = shutil.get_terminal_size((80, 20)).columns + max_content = min(width - 6, 100) # keep borders tidy + + # Banner text + banner = f"{BOLD}{MAGENTA}HOPLA{RESET} {BOLD}{CYAN}CLI{RESET}" + meta = f"{DIM}{datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}{RESET}" + + # Compose lines (truncate if needed) + def clip(s): return (s[:max_content] + "…" if len(s) > max_content else s) + line1 = clip(banner) + line2 = clip(f"{BOLD}License: {license}{RESET}") + line3 = clip(subtitle) + line4 = clip(f"Session started • {meta}") + + # Box drawing + top = "╭" + "─" * (max_content + 2) + "╮" + bottom = "╰" + "─" * (max_content + 2) + "╯" + + def center(s): + pad = max_content - len(_strip_ansi(s)) + left = pad // 2 + right = pad - left + return "│ " + (" " * left) + s + (" " * right) + " │" + + print(top) + print(center(line1)) + print(center(line2)) + print(center(line3)) + print(center(line4)) + print(bottom) + print() # spacing + + +def display_toml( + data, + title="TOML Configuration"): + """ + Pretty-print TOML content inside a styled box with colors. + + Parameters + ---------- + data : dict + TOML data content. + title : str, optional + Title displayed at the top of the box. Default is "TOML Configuration". + + Notes + ----- + - Uses ANSI escape codes for colors. + - Automatically adapts to terminal width (up to 100 characters). + - Displays keys in cyan and values in magenta for readability. + """ + # Terminal width + width = shutil.get_terminal_size((80, 20)).columns + max_content = min(width - 6, 100) + + # Box drawing + top = "╭" + "─" * (max_content + 2) + "╮" + bottom = "╰" + "─" * (max_content + 2) + "╯" + + def center(s): + pad = max_content - len(_strip_ansi(s)) + left = pad // 2 + right = pad - left + return "│ " + (" " * left) + s + (" " * right) + " │" + + def format_line(key, value=None): + if value is None: + line = f"{CYAN}{key}{RESET}" + else: + line = f"{CYAN}{key}{RESET} = {MAGENTA}{value}{RESET}" + visible_len = len(_strip_ansi(line)) + if visible_len > max_content: + line = line[:max_content-1] + "…" + visible_len = len(_strip_ansi(line)) + return "│ " + line + " " * (max_content - visible_len) + " │" + + # Print box + print(top) + print(center(f"{BOLD}{title}{RESET}")) + print("│ " + " " * max_content + " │") + for section, content in data.items(): + if isinstance(content, dict): + print(format_line(f"[{section}]")) + for k, v in content.items(): + print(format_line(k, v)) + else: + print(format_line(section, content)) + print(format_line("")) + + print(bottom) + + +def _strip_ansi(s: str) -> str: + """Remove ANSI escape codes from a string.""" + return re.sub(r"\x1b\[[0-9;]*m", "", s) + def main(): """ @@ -53,10 +185,6 @@ def main(): commands : str or list Commands to execute. Can be a Python expression string (e.g., "sleep {k}") or a list of commands. - data : str - A TSV file used to fill the previous Python expression string. - Column names must match expression (e.g., "k" in the previous - example). parameters : str Additional parameters passed to the container execution command (e.g., "--cleanenv"). @@ -92,7 +220,14 @@ def main(): - The `multi` section should define `n_splits` to control chunking. - The `Config` context manager is used to apply configuration settings during execution. + + Raises + ------ + ValueError + If 'data.tsv' is missing. """ + print_hoplacli_header() + parser = argparse.ArgumentParser( prog="hoplactl", description=( @@ -103,7 +238,10 @@ def main(): "configuration. It then runs the executor with a specified " "maximum number of jobs and writes a report to disk." ), - epilog="Notes:\n- Use a valid TOML file.\n- See docs for examples.", + epilog=( + "Notes:\n- Use a valid TOML file.\n- Add a 'data.tsv' file next " + "to the configuration file if needed.\n- See docs for examples." + ), formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument( @@ -127,6 +265,7 @@ def main(): with open(args.config, "rb") as of: config = tomllib.load(of) + display_toml(config) executor = hopla.Executor( **config["environment"] @@ -134,24 +273,32 @@ def main(): commands = config["inputs"]["commands"] if not isinstance(commands, (list, tuple)): - df = pd.read_csv(config["inputs"]["data"], sep="\t") + data_file = Path(args.config).parent / "data.tsv" + if not data_file.is_file(): + raise ValueError( + "A TSV file named 'data.tsv' must be located next to the " + "configuration file. It is needed to fill the 'commands' " + "Python expression string in TOML configuration. Column " + "names must match expression keys." + ) + df = pd.read_csv(data_file, sep="\t") commands = [commands.format(**dict(row)) for _, row in df.iterrows()] + if config.get("multi") is not None: chunks = np.array_split(commands, config["multi"]["n_splits"]) - jobs = [ + _ = [ executor.submit( [hopla.DelayedSubmission(cmd) for cmd in subcmds], execution_parameters=config["inputs"].get("parameters"), ) for subcmds in chunks ] else: - jobs = [ + _ = [ executor.submit( cmd, execution_parameters=config["inputs"].get("parameters"), ) for cmd in commands ] - print(jobs) with Config(**config["config"]): executor(max_jobs=args.njobs) From d4c6bb870f245d5779f036fc9a89cf6109822ce4 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Fri, 12 Dec 2025 10:44:29 +0100 Subject: [PATCH 10/24] hopla/cli: fix CI. --- hopla/cli.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hopla/cli.py b/hopla/cli.py index 51227f7..ee9acd1 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -27,7 +27,7 @@ DIM = "\033[2m" -def print_hoplacli_header( +def print_header( license="CeCILL-B", subtitle="Fast, friendly CLI to submit you jobs"): """ @@ -41,6 +41,10 @@ def print_hoplacli_header( A secondary line of text displayed below the title. Default is "Fast, friendly CLI to submit you jobs". + Returns + ------- + None + Notes ----- - The header is styled with ANSI escape codes for colors and bold text. @@ -80,10 +84,10 @@ def center(s): print(center(line3)) print(center(line4)) print(bottom) - print() # spacing + print() -def display_toml( +def print_toml( data, title="TOML Configuration"): """ @@ -226,7 +230,7 @@ def main(): ValueError If 'data.tsv' is missing. """ - print_hoplacli_header() + print_header() parser = argparse.ArgumentParser( prog="hoplactl", @@ -265,7 +269,7 @@ def main(): with open(args.config, "rb") as of: config = tomllib.load(of) - display_toml(config) + print_toml(config) executor = hopla.Executor( **config["environment"] From 43d7f483582690c1e440e5cd25e547da7a25e045 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Fri, 12 Dec 2025 11:34:56 +0100 Subject: [PATCH 11/24] hopla/cli: fix typo. --- hopla/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hopla/cli.py b/hopla/cli.py index ee9acd1..89c116b 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -29,7 +29,7 @@ def print_header( license="CeCILL-B", - subtitle="Fast, friendly CLI to submit you jobs"): + subtitle="Fast, friendly CLI to submit your jobs"): """ Print a styled header banner for the hoplacli tool. From 611614c958a85ddbfc84577dccba9f9884077011 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Mon, 15 Dec 2025 13:54:48 +0100 Subject: [PATCH 12/24] hopla/ccc: add 'oneshot' backend. --- examples/plot_ccc.py | 1 + examples/plot_ccc_flux_multi_tasks.py | 1 + examples/plot_ccc_joblib_multi_tasks.py | 1 + examples/plot_ccc_oneshot_multi_tasks.py | 77 +++++++++++++++++++++ examples/plot_pbs.py | 1 + examples/plot_slurm.py | 1 + hopla/ccc.py | 60 +++++++++++++++- hopla/executor.py | 3 +- hopla/resources/oneshot_script_template.txt | 35 ++++++++++ hopla/utils.py | 14 +++- 10 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 examples/plot_ccc_oneshot_multi_tasks.py create mode 100644 hopla/resources/oneshot_script_template.txt diff --git a/examples/plot_ccc.py b/examples/plot_ccc.py index 180e17b..074a4aa 100644 --- a/examples/plot_ccc.py +++ b/examples/plot_ccc.py @@ -67,3 +67,4 @@ with Config(dryrun=True, delay_s=3): executor(max_jobs=2) + print(executor.report) diff --git a/examples/plot_ccc_flux_multi_tasks.py b/examples/plot_ccc_flux_multi_tasks.py index 57a75e9..d538f04 100644 --- a/examples/plot_ccc_flux_multi_tasks.py +++ b/examples/plot_ccc_flux_multi_tasks.py @@ -74,3 +74,4 @@ with Config(dryrun=True, delay_s=3): executor(max_jobs=2) + print(executor.report) diff --git a/examples/plot_ccc_joblib_multi_tasks.py b/examples/plot_ccc_joblib_multi_tasks.py index 2574d8e..fd6a6c2 100644 --- a/examples/plot_ccc_joblib_multi_tasks.py +++ b/examples/plot_ccc_joblib_multi_tasks.py @@ -74,3 +74,4 @@ with Config(dryrun=True, delay_s=3): executor(max_jobs=2) + print(executor.report) diff --git a/examples/plot_ccc_oneshot_multi_tasks.py b/examples/plot_ccc_oneshot_multi_tasks.py new file mode 100644 index 0000000..372e8d2 --- /dev/null +++ b/examples/plot_ccc_oneshot_multi_tasks.py @@ -0,0 +1,77 @@ +""" +Basic example on how to use the CCC cluster using multi-tasks +============================================================= + +Basic example + +When you're running hundreds or thousands of jobs, automation is a necessity. +This is where ``hopla`` can help you. + +A simple example of how to use ``hopla`` on a CCC cluster. Please check +the :ref:`user guide ` for a more in depth presentation of all +functionalities. + + +Imports +------- +""" + +import hopla +import numpy as np +from pprint import pprint + + +# %% +# Executor Context +# ---------------- + +executor = hopla.Executor( + cluster="ccc", + folder="/tmp/hopla", + queue="rome", + image="/tmp/hopla/my-docker-img.tar", + walltime=1, + project_id="genXXX", + backend="oneshot", +) + + +# %% +# Submit Jobs +# ----------- + +chunks = np.array_split(range(1, 11), 3) +jobs = [ + executor.submit([hopla.DelayedSubmission("sleep", k) for k in c]) + for c in chunks +] +pprint(jobs) +print(jobs[0].delayed_submission) + + +# %% +# Generate a batch +# ---------------- + +jobs[0].generate_batch() +print(jobs[0].paths) +batch = jobs[0].paths.submission_file +with open(batch) as of: + print(of.read()) +script = jobs[0].paths.submission_file +with open(script) as of: + print(of.read()) + + +# %% +# Start Jobs +# ---------- +# +# We can't execute the code on the CI since the CCC infrastructure is not +# available. + +from hopla.config import Config + +with Config(dryrun=True, delay_s=3): + executor(max_jobs=2) + print(executor.report) diff --git a/examples/plot_pbs.py b/examples/plot_pbs.py index 6390955..0416c58 100644 --- a/examples/plot_pbs.py +++ b/examples/plot_pbs.py @@ -66,3 +66,4 @@ with Config(dryrun=True, delay_s=3): executor(max_jobs=2) + print(executor.report) diff --git a/examples/plot_slurm.py b/examples/plot_slurm.py index b694d15..248990c 100644 --- a/examples/plot_slurm.py +++ b/examples/plot_slurm.py @@ -66,3 +66,4 @@ with Config(dryrun=True, delay_s=3): executor(max_jobs=2) + print(executor.report) diff --git a/hopla/ccc.py b/hopla/ccc.py index b7076c3..78eadef 100644 --- a/hopla/ccc.py +++ b/hopla/ccc.py @@ -75,15 +75,28 @@ class DelayedCCCJob(DelayedJob): job_id: str the job identifier. backend: str, default 'flux' - the multi-taks backend to use: 'flux' or 'joblib'. + the multi-taks backend to use: 'flux', 'joblib' or 'oneshot'. + + Raises + ------ + ValueError + If an invalid backend is specified. """ _hub = "n4h00001rs" _submission_cmd = "ccc_msub" _container_cmd = "pcocc-rs run {hub}:{image_name} {params} -- {command}" + _container_onshot_cmd = ( + "pcocc-rs run {hub}:{image_name} {params} /bin/bash -- -c '{command}'" + ) def __init__(self, delayed_submission, executor, job_id, backend="flux"): super().__init__(delayed_submission, executor, job_id) self.multi_task = isinstance(delayed_submission, (list, tuple)) + if backend not in ("flux", "joblib", "oneshot"): + raise ValueError( + "Invalid backend. Valid multi-taks backends are: 'flux', " + "'joblib' or 'oneshot'." + ) self.backend = backend resource_dir = Path(__file__).parent / "resources" if self.multi_task and self.backend == "flux": @@ -92,6 +105,9 @@ def __init__(self, delayed_submission, executor, job_id, backend="flux"): elif self.multi_task and self.backend == "joblib": path = resource_dir / "ccc_batch_template.txt" self.worker_file = resource_dir / "joblib_script_template.txt" + elif self.multi_task and self.backend == "oneshot": + path = resource_dir / "ccc_batch_template.txt" + self.worker_file = resource_dir / "oneshot_script_template.txt" else: path = resource_dir / "ccc_batch_template.txt" with open(path) as of: @@ -157,6 +173,7 @@ def generate_batch(self): for command in subcmds ] params["logdir"] = self.paths.flux_dir + self.paths.flux_dir.mkdir(parents=True, exist_ok=True) with open(self.paths.task_file, "w") as of: of.write("\n".join(subcmds)) cmd = self.paths.task_file @@ -185,6 +202,31 @@ def generate_batch(self): ) ) cmd = f"python {self.paths.joblib_file}" + elif self.multi_task and self.backend == "oneshot": + with open(self.worker_file) as of: + oneshot_template = of.read() + subcmds = [ + submission.command + for submission in self.delayed_submission + ] + subcmds = [ + f"'{command}'" + for command in subcmds + ] + with open(self.paths.oneshot_file, "w") as of: + of.write( + oneshot_template.format( + logdir=self.paths.oneshot_dir, + commands="\n".join(subcmds), + ) + ) + self.paths.oneshot_dir.mkdir(parents=True, exist_ok=True) + cmd = self._container_onshot_cmd.format( + hub=self._hub, + image_name=self.image_name, + params=self.delayed_submission[0].execution_parameters, + command=self.paths.oneshot_file + ) else: cmd = self._container_cmd.format( hub=self._hub, @@ -243,7 +285,7 @@ def read_jobid(self, string): def sub_report(self): report = [] prefix = f"{self.__class__.__name__}" - if self.multi_task: + if self.multi_task and self.backend == "flux": log_files = list(self.paths.flux_dir.glob("bulk_*")) tasks_ids = [ path.name.split("_")[1] @@ -254,6 +296,20 @@ def sub_report(self): f"{prefix}number_of_tasks: {len(self.delayed_submission)}") report.append(f"{prefix}failed_tasks: {tasks_ids}") report.append(f"{prefix}running_tasks: {len(log_files)}") + report.append(f"{prefix}logdir: {self.paths.flux_dir}") + elif self.multi_task and self.backend == "oneshot": + log_files = list(self.paths.flux_dir.glob("job_*.exitcode")) + exitcodes = [] + for path in log_files: + with open(path) as of: + exitcodes.append(int(of.read().strip())) + n_fail = sum(1 for code in exitcodes if code != 0) + n_submissions = len(self.delayed_submission) + n_tasks = len(log_files) + report.append(f"{prefix}number_of_tasks: {n_submissions}") + report.append(f"{prefix}failed_tasks: {n_fail}") + report.append(f"{prefix}running_tasks: {n_tasks}") + report.append(f"{prefix}logdir: {self.paths.oneshot_dir}") return report @property diff --git a/hopla/executor.py b/hopla/executor.py index 1b94c8d..4db5523 100644 --- a/hopla/executor.py +++ b/hopla/executor.py @@ -65,7 +65,8 @@ class Executor: project_id: str, default None the project ID where you have computing hours. backend: str, default 'flux' - the multi-taks backend to use: 'flux' or 'jobli + the multi-taks backend to use: 'flux', 'joblib or 'oneshot'. This + option is only used with CCC cluster type. Examples -------- diff --git a/hopla/resources/oneshot_script_template.txt b/hopla/resources/oneshot_script_template.txt new file mode 100644 index 0000000..2d4ad1e --- /dev/null +++ b/hopla/resources/oneshot_script_template.txt @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + + +# First parameter = log directory +LOG_DIR="{logdir}" +echo "Logs written to $LOG_DIR/job_N.out, job_N.err, and job_N.exitcode" + +# List of jobs +jobs=( +{commands} +) + +# Run jobs in parallel and redirect stdout/stderr +pids=() +for i in "${{!jobs[@]}}"; do + ( + bash -c "${{jobs[$i]}}" \ + >"$LOG_DIR/job_${{i}}.out" \ + 2>"$LOG_DIR/job_${{i}}.err" + echo $? >"$LOG_DIR/job_${{i}}.exitcode" + ) & + pids+=($!) +done + +# Wait for all jobs and collect exit codes +exit_code=0 +for pid in "${{pids[@]}}"; do + if ! wait "$pid"; then + exit_code=$? + fi +done + +# Return proper exit code (0 if all succeeded, non‑zero if any failed) +exit $exit_code diff --git a/hopla/utils.py b/hopla/utils.py index afdbf86..dbb7409 100644 --- a/hopla/utils.py +++ b/hopla/utils.py @@ -108,12 +108,24 @@ def joblib_file(self): """ return self.submission_folder / f"{self.job_id}_joblib_script.py" + @property + def oneshot_file(self): + """ Generate the oneshot file location. + """ + return self.submission_folder / f"{self.job_id}_oneshot_script.sh" + @property def flux_dir(self): """ Generate the flux output dir. """ path = self.log_folder / f"{self.job_id}_flux" - path.mkdir(parents=True, exist_ok=True) + return path + + @property + def oneshot_dir(self): + """ Generate the oneshot output dir. + """ + path = self.log_folder / f"{self.job_id}_oneshot" return path def __repr__(self): From e01de6276254d6adbb2455030f669a46b80f0908 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Mon, 15 Dec 2025 16:07:37 +0100 Subject: [PATCH 13/24] hopla/ccc: fix oneshot command. --- hopla/executor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hopla/executor.py b/hopla/executor.py index 4db5523..f4bd3cb 100644 --- a/hopla/executor.py +++ b/hopla/executor.py @@ -119,7 +119,11 @@ def __init__(self, cluster, folder, queue, image, name="hopla", memory=2, "nmulticpus": n_multi_cpus, "ngpus": n_gpus, "modules": ",".join(modules), - "image": Path(image).expanduser().absolute(), + "image": ( + Path(image).expanduser().absolute() + if Path(image).is_file() + else image + ), "project_id": project_id } self._delayed_jobs = [] From 79905240616263d54ea7bf776cf05dbdd870dee7 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Mon, 15 Dec 2025 16:08:41 +0100 Subject: [PATCH 14/24] hopla/ccc: fix oneshot command. --- hopla/ccc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hopla/ccc.py b/hopla/ccc.py index 78eadef..42c5acc 100644 --- a/hopla/ccc.py +++ b/hopla/ccc.py @@ -86,7 +86,8 @@ class DelayedCCCJob(DelayedJob): _submission_cmd = "ccc_msub" _container_cmd = "pcocc-rs run {hub}:{image_name} {params} -- {command}" _container_onshot_cmd = ( - "pcocc-rs run {hub}:{image_name} {params} /bin/bash -- -c '{command}'" + "pcocc-rs run {hub}:{image_name} {params} /bin/bash -- " + "-c '/bin/bash {command}'" ) def __init__(self, delayed_submission, executor, job_id, backend="flux"): From 3c8505aabbc92208eb2b06ff9de5f95bbbea4ba5 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Mon, 15 Dec 2025 16:26:15 +0100 Subject: [PATCH 15/24] hopla/resources/oneshot_script_template: add tic/toc information in logs. --- hopla/resources/oneshot_script_template.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hopla/resources/oneshot_script_template.txt b/hopla/resources/oneshot_script_template.txt index 2d4ad1e..d53a2a4 100644 --- a/hopla/resources/oneshot_script_template.txt +++ b/hopla/resources/oneshot_script_template.txt @@ -15,9 +15,12 @@ jobs=( pids=() for i in "${{!jobs[@]}}"; do ( + echo "=== Job $i started at $(date '+%Y-%m-%d %H:%M:%S') ===" >>"$LOG_DIR/job_${{i}}.out" + echo "Command: ${{jobs[$i]}}" >>"$LOG_DIR/job_${{i}}.out" bash -c "${{jobs[$i]}}" \ - >"$LOG_DIR/job_${{i}}.out" \ + >>"$LOG_DIR/job_${{i}}.out" \ 2>"$LOG_DIR/job_${{i}}.err" + echo "=== Job $i ended at $(date '+%Y-%m-%d %H:%M:%S') ===" >>"$LOG_DIR/job_${{i}}.out" echo $? >"$LOG_DIR/job_${{i}}.exitcode" ) & pids+=($!) From a098a5d4186f7eb94f0f039b4db28587918190cc Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Tue, 16 Dec 2025 08:57:13 +0100 Subject: [PATCH 16/24] github/workflows/testing: update coveralls. --- .github/workflows/testing.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 433ac98..f3ef537 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -44,7 +44,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install numpy - python -m pip install pynose coverage coveralls + python -m pip install pynose coverage[toml] python -m pip install --progress-bar off . - name: Run unit tests run: | @@ -55,3 +55,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | coveralls --service=github + - name: Coveralls + uses: coverallsapp/github-action@v2 + with: + debug: true + fail-on-error: true From 58c2f4ab8e8f77bc23d757b80b362678a46644f1 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Tue, 16 Dec 2025 13:33:20 +0100 Subject: [PATCH 17/24] github/workflows/testing: fix action. --- .github/workflows/testing.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f3ef537..89f1f4e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -49,12 +49,6 @@ jobs: - name: Run unit tests run: | nosetests --with-coverage --cover-package=hopla --verbosity=2 --with-doctest --doctest-options='+ELLIPSIS,+NORMALIZE_WHITESPACE' - - name: Coveralls - if: matrix.python-version == 3.12 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - coveralls --service=github - name: Coveralls uses: coverallsapp/github-action@v2 with: From 9cacfefe6cfed30e956a0ea4e8579cc115ab1080 Mon Sep 17 00:00:00 2001 From: AGrigis Date: Tue, 20 Jan 2026 14:10:19 +0100 Subject: [PATCH 18/24] hopla/resources/slurm_batch_template: deal properly with exit status. --- hopla/resources/slurm_batch_template.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/hopla/resources/slurm_batch_template.txt b/hopla/resources/slurm_batch_template.txt index 539b45b..0ea443f 100644 --- a/hopla/resources/slurm_batch_template.txt +++ b/hopla/resources/slurm_batch_template.txt @@ -17,4 +17,11 @@ unset LD_PRELOAD # Command {command} -echo "HOPLASAY-DONE" +exitcode=$? +echo "Exit code was: $exitcode" + +# Exit +if [ "$exitcode" -ne 1 ]; then + echo "HOPLASAY-DONE" +fi + From 785b7314662ce19ab9fbf2b9a537b9f89c49f1b5 Mon Sep 17 00:00:00 2001 From: AGrigis Date: Tue, 20 Jan 2026 14:11:02 +0100 Subject: [PATCH 19/24] hopla/cli: fix command submit + compat with python <3.11. --- hopla/cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hopla/cli.py b/hopla/cli.py index 89c116b..96427b3 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -10,7 +10,10 @@ import datetime import re import shutil -import tomllib +try: + import tomllib # Python 3.11+ +except ModuleNotFoundError: + import tomli as tomllib # Python 3.8–3.10 from pathlib import Path import numpy as np @@ -292,14 +295,14 @@ def main(): chunks = np.array_split(commands, config["multi"]["n_splits"]) _ = [ executor.submit( - [hopla.DelayedSubmission(cmd) for cmd in subcmds], + [hopla.DelayedSubmission(*cmd) for cmd in subcmds], execution_parameters=config["inputs"].get("parameters"), ) for subcmds in chunks ] else: _ = [ executor.submit( - cmd, + *cmd, execution_parameters=config["inputs"].get("parameters"), ) for cmd in commands ] From 54aa9dd73cc85dc18a3406ed3cf90214a569865a Mon Sep 17 00:00:00 2001 From: AGrigis Date: Tue, 20 Jan 2026 14:13:22 +0100 Subject: [PATCH 20/24] doc/user_guide/cli: extend CLI doc. --- doc/user_guide/cli.rst | 94 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/doc/user_guide/cli.rst b/doc/user_guide/cli.rst index 4889b18..77f1cb6 100644 --- a/doc/user_guide/cli.rst +++ b/doc/user_guide/cli.rst @@ -3,11 +3,95 @@ CLI === -``hopla`` has a CLI interface. The `hoplacli` command loads a TOML -configuration file, initializes a hopla executor, and submits jobs either -individually or in chunks depending on the configuration. It then runs the -executor with a specified maximum number of jobs and writes a report to disk. +`hoplacli` is a command-line interface designed to automate job submission and +execution using the ``hopla`` framework. +It loads a TOML configuration file, prepares an execution environment, submits +jobs (individually or in chunks), and produces a final execution report. +This guide explains how to use the CLI, how to structure the TOML +configuration file, and how the workflow operates. + +Usage +----- .. code-block:: bash - hoplacli --config ./examples/experiment.toml --njobs 2 + hoplacli --config --config --njobs + +An ``experiment.toml`` demonstration configuration file can be found in the +project examples folder. + +Workflow +-------- + +1. Parse CLI arguments using ``argparse``. +2. Load the TOML configuration file with ``tomllib``: expect four mandatory + sections (``[project]``, ``[inputs]``, ``[environment]``, and ``[config]``) + and one optional (``[multi]``). +3. Initialize a `:class:`~hopla.executor.Executor`` with ``[environment]`` + settings. +4. Extract and submit commands from the ``[inputs]`` settings: + - If a ``[multi]`` section is present, split commands into chunks and + submit them as delayed submissions. + - Otherwise, submit commands directly. +5. Run the executor with the specified maximum number of jobs using the + ``[config]`` settings. +6. Write a textual report to ``report.txt`` inside the executor's working + directory. + +TOML Configuration +------------------ + +The configuration file is divided into several sections. + +``[project]`` +~~~~~~~~~~~~~ + +``name`` (str) + Project name. + +``operator`` (str) + Person responsible for running the analysis. + +``date`` (str) + Date of the experiment in ``DD/MM/YYYY`` format. + +``[inputs]`` +~~~~~~~~~~~~ + +``commands`` (str or list) + Commands to execute. May be a Python expression string (e.g. + ``"sleep {k}"``) or a list of explicit commands. + +``parameters`` (str) + Additional parameters passed to the container execution command + (e.g. ``"--cleanenv"``). + +``[environment]`` +~~~~~~~~~~~~~~~~~ + +See the :class:`~hopla.executor.Executor` parameters. + +``[config]`` +~~~~~~~~~~~~ + +``dryrun`` (bool) + Simulate job submission without executing. + +``delay_s`` (int) + Delay (seconds) between submissions. + +``verbose`` (bool) + Enable verbose logging. + +``[multi]`` (optional) +~~~~~~~~~~~~~~~~~~~~~~ + +``n_splits`` (int) + Number of chunks to split commands into. + +Notes +----- + +- The ``multi`` section is optional but required for chunked submissions. +- The ``Config`` context manager is used internally to apply configuration + settings during execution. From 4e31809d838ba4293f59cd55e2308c350838888d Mon Sep 17 00:00:00 2001 From: AGrigis Date: Tue, 20 Jan 2026 14:13:55 +0100 Subject: [PATCH 21/24] doc/user_guide/clusters: new recommendation. --- doc/user_guide/clusters.rst | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/doc/user_guide/clusters.rst b/doc/user_guide/clusters.rst index 3051dd8..5cf6423 100644 --- a/doc/user_guide/clusters.rst +++ b/doc/user_guide/clusters.rst @@ -166,6 +166,28 @@ multi tasks strategy (3 chunks here). ] +.. important:: + + Don't forget to decalre the `n4h00001` hub by copying the + **.../n4h00001/n4h00001/config/repositories.yaml** file in your home directory + **$HOME/.config/pcocc/repositories.yaml**.To list images don't forget + also to export the **CCCWORKDIR** env variable to **n4h00001/n4h00001/gaia**. + +.. important:: + + Don't forget to load the **gcc/11.1.0** module to launch multi tasks jobs, + and the **python3/3.12** (or another compatible version of Python) module. + Dont't forget also to switch to the appropriate data + **dfldatadir/XXX** module. + +.. important:: + + Use the appropriate backend when performing chunked submissions: + + - flux by default. + - oneshot when using MCR: you need to reserve one full node and it will + ran the chuncked commands in a single container call. + .. tip:: You need to adapt the the `n_multi_cpus` parameter of the `Executor` in @@ -184,20 +206,6 @@ multi tasks strategy (3 chunks here). brainprep -.. important:: - - Don't forget to decalre the `n4h00001` hub by copying the - **.../n4h00001/n4h00001/config/repositories.yaml** file in your home directory - **$HOME/.config/pcocc/repositories.yaml**.To list images don't forget - also to export the **CCCWORKDIR** env variable to **n4h00001/n4h00001/gaia**. - -.. important:: - - Don't forget to load the **gcc/11.1.0** module to launch multi tasks jobs, - and the **python3/3.12** (or another compatible version of Python) module. - Dont't forget also to switch to the appropriate data - **dfldatadir/XXX** module. - .. tip:: You can export your docker image in a `.tar` file as follows: From 019cbfd65592a36fa92448ad0bf976bc2d9c2f9e Mon Sep 17 00:00:00 2001 From: AGrigis Date: Tue, 20 Jan 2026 14:25:51 +0100 Subject: [PATCH 22/24] hopla: fix CI. --- doc/user_guide/cli.rst | 1 + hopla/cli.py | 5 +++-- pyproject.toml | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/user_guide/cli.rst b/doc/user_guide/cli.rst index 77f1cb6..844bc26 100644 --- a/doc/user_guide/cli.rst +++ b/doc/user_guide/cli.rst @@ -30,6 +30,7 @@ Workflow 3. Initialize a `:class:`~hopla.executor.Executor`` with ``[environment]`` settings. 4. Extract and submit commands from the ``[inputs]`` settings: + - If a ``[multi]`` section is present, split commands into chunks and submit them as delayed submissions. - Otherwise, submit commands directly. diff --git a/hopla/cli.py b/hopla/cli.py index 96427b3..fae2c25 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -10,10 +10,11 @@ import datetime import re import shutil + try: - import tomllib # Python 3.11+ + import tomllib # Python 3.11+ except ModuleNotFoundError: - import tomli as tomllib # Python 3.8–3.10 + import tomli as tomllib from pathlib import Path import numpy as np diff --git a/pyproject.toml b/pyproject.toml index 2ef0053..4d85254 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,8 @@ extend-select = [ ] ignore = [ + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` + ] [tool.ruff] From da106ff1c695aabbfff7fc02dbff6f129b770e23 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Tue, 3 Mar 2026 16:02:41 +0100 Subject: [PATCH 23/24] hopla/cli: add venv option. --- examples/experiment.toml | 2 +- hopla/cli.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/examples/experiment.toml b/examples/experiment.toml index f56314e..a068485 100644 --- a/examples/experiment.toml +++ b/examples/experiment.toml @@ -5,7 +5,7 @@ date = "12/02/2020" [inputs] commands = "sleep {k}s" -# commands = ["sleep 1", "sleep 2", "sleep 3"] +# commands = [["sleep", "1"], ["sleep", "2"], ["sleep", "3"]] parameters = "--cleanenv" [environment] diff --git a/hopla/cli.py b/hopla/cli.py index fae2c25..9af2b17 100644 --- a/hopla/cli.py +++ b/hopla/cli.py @@ -269,15 +269,30 @@ def main(): required=True, help="The maximum number of job submissions to execute concurrently." ) + parser.add_argument( + "--venv", + action="store_true", + help=( + "Enable this option to run the command outside of a container. " + "In this case, the image environment parameter is automatically " + "set to None, so providing it is optional." + ) + ) args = parser.parse_args() with open(args.config, "rb") as of: config = tomllib.load(of) print_toml(config) + if args.venv: + config["environment"]["image"] = "" executor = hopla.Executor( **config["environment"] ) + if args.venv: + executor._job_class._container_cmd = ( + "{command}" + ) commands = config["inputs"]["commands"] if not isinstance(commands, (list, tuple)): @@ -290,7 +305,10 @@ def main(): "names must match expression keys." ) df = pd.read_csv(data_file, sep="\t") - commands = [commands.format(**dict(row)) for _, row in df.iterrows()] + commands = [ + commands.format(**dict(row)).split(" ") + for _, row in df.iterrows() + ] if config.get("multi") is not None: chunks = np.array_split(commands, config["multi"]["n_splits"]) From af79f2116176f13456f6345c4126b18fdf003903 Mon Sep 17 00:00:00 2001 From: Antoine Grigis Date: Wed, 4 Mar 2026 09:01:13 +0100 Subject: [PATCH 24/24] hopla: update doc. --- .github/workflows/documentation.yml | 2 +- .github/workflows/testing.yml | 2 +- LICENSE.rst | 518 ++++++++++++++++++++++- README.rst | 2 +- doc/user_guide/cli.rst | 12 +- examples/plot_ccc.py | 2 +- examples/plot_ccc_flux_multi_tasks.py | 2 +- examples/plot_ccc_joblib_multi_tasks.py | 2 +- examples/plot_ccc_oneshot_multi_tasks.py | 2 +- examples/plot_pbs.py | 2 +- examples/plot_slurm.py | 2 +- pyproject.toml | 16 +- 12 files changed, 539 insertions(+), 25 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3ab9210..e594e50 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -9,7 +9,7 @@ # $ cd $MODULE_DIR/doc # $ make html-strict ### -name: "DocumentationBuilder" +name: "DocumentationBuilder[sphinxdoc]" on: push: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 89f1f4e..fc0424b 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -7,7 +7,7 @@ # $ nosetests --with-coverage --cover-package=hopla --verbosity=2 # --with-doctest --doctest-options='+ELLIPSIS,+NORMALIZE_WHITESPACE' ### -name: "TESTING" +name: "Testing[nosetests]" on: push: diff --git a/LICENSE.rst b/LICENSE.rst index e71dcbd..da41897 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -1,5 +1,515 @@ -.. -*- mode: rst -*- -The code is distributed under the terms of the -`CeCILL-B `_ -license, as published by the CEA-CNRS-INRIA. +CeCILL-B FREE SOFTWARE LICENSE AGREEMENT + + + Notice + +This Agreement is a Free Software license agreement that is the result +of discussions between its authors in order to ensure compliance with +the two main principles guiding its drafting: + + * firstly, compliance with the principles governing the distribution + of Free Software: access to source code, broad rights granted to + users, + * secondly, the election of a governing law, French law, with which + it is conformant, both as regards the law of torts and + intellectual property law, and the protection that it offers to + both authors and holders of the economic rights over software. + +The authors of the CeCILL-B (for Ce[a] C[nrs] I[nria] L[ogiciel] L[ibre]) +license are: + +Commissariat à l'Energie Atomique - CEA, a public scientific, technical +and industrial research establishment, having its principal place of +business at 25 rue Leblanc, immeuble Le Ponant D, 75015 Paris, France. + +Centre National de la Recherche Scientifique - CNRS, a public scientific +and technological establishment, having its principal place of business +at 3 rue Michel-Ange, 75794 Paris cedex 16, France. + +Institut National de Recherche en Informatique et en Automatique - +INRIA, a public scientific and technological establishment, having its +principal place of business at Domaine de Voluceau, Rocquencourt, BP +105, 78153 Le Chesnay cedex, France. + + + Preamble + +This Agreement is an open source software license intended to give users +significant freedom to modify and redistribute the software licensed +hereunder. + +The exercising of this freedom is conditional upon a strong obligation +of giving credits for everybody that distributes a software +incorporating a software ruled by the current license so as all +contributions to be properly identified and acknowledged. + +In consideration of access to the source code and the rights to copy, +modify and redistribute granted by the license, users are provided only +with a limited warranty and the software's author, the holder of the +economic rights, and the successive licensors only have limited liability. + +In this respect, the risks associated with loading, using, modifying +and/or developing or reproducing the software by the user are brought to +the user's attention, given its Free Software status, which may make it +complicated to use, with the result that its use is reserved for +developers and experienced professionals having in-depth computer +knowledge. Users are therefore encouraged to load and test the +suitability of the software as regards their requirements in conditions +enabling the security of their systems and/or data to be ensured and, +more generally, to use and operate it in the same conditions of +security. This Agreement may be freely reproduced and published, +provided it is not altered, and that no provisions are either added or +removed herefrom. + +This Agreement may apply to any or all software for which the holder of +the economic rights decides to submit the use thereof to its provisions. + + + Article 1 - DEFINITIONS + +For the purpose of this Agreement, when the following expressions +commence with a capital letter, they shall have the following meaning: + +Agreement: means this license agreement, and its possible subsequent +versions and annexes. + +Software: means the software in its Object Code and/or Source Code form +and, where applicable, its documentation, "as is" when the Licensee +accepts the Agreement. + +Initial Software: means the Software in its Source Code and possibly its +Object Code form and, where applicable, its documentation, "as is" when +it is first distributed under the terms and conditions of the Agreement. + +Modified Software: means the Software modified by at least one +Contribution. + +Source Code: means all the Software's instructions and program lines to +which access is required so as to modify the Software. + +Object Code: means the binary files originating from the compilation of +the Source Code. + +Holder: means the holder(s) of the economic rights over the Initial +Software. + +Licensee: means the Software user(s) having accepted the Agreement. + +Contributor: means a Licensee having made at least one Contribution. + +Licensor: means the Holder, or any other individual or legal entity, who +distributes the Software under the Agreement. + +Contribution: means any or all modifications, corrections, translations, +adaptations and/or new functions integrated into the Software by any or +all Contributors, as well as any or all Internal Modules. + +Module: means a set of sources files including their documentation that +enables supplementary functions or services in addition to those offered +by the Software. + +External Module: means any or all Modules, not derived from the +Software, so that this Module and the Software run in separate address +spaces, with one calling the other when they are run. + +Internal Module: means any or all Module, connected to the Software so +that they both execute in the same address space. + +Parties: mean both the Licensee and the Licensor. + +These expressions may be used both in singular and plural form. + + + Article 2 - PURPOSE + +The purpose of the Agreement is the grant by the Licensor to the +Licensee of a non-exclusive, transferable and worldwide license for the +Software as set forth in Article 5 hereinafter for the whole term of the +protection granted by the rights over said Software. + + + Article 3 - ACCEPTANCE + +3.1 The Licensee shall be deemed as having accepted the terms and +conditions of this Agreement upon the occurrence of the first of the +following events: + + * (i) loading the Software by any or all means, notably, by + downloading from a remote server, or by loading from a physical + medium; + * (ii) the first time the Licensee exercises any of the rights + granted hereunder. + +3.2 One copy of the Agreement, containing a notice relating to the +characteristics of the Software, to the limited warranty, and to the +fact that its use is restricted to experienced users has been provided +to the Licensee prior to its acceptance as set forth in Article 3.1 +hereinabove, and the Licensee hereby acknowledges that it has read and +understood it. + + + Article 4 - EFFECTIVE DATE AND TERM + + + 4.1 EFFECTIVE DATE + +The Agreement shall become effective on the date when it is accepted by +the Licensee as set forth in Article 3.1. + + + 4.2 TERM + +The Agreement shall remain in force for the entire legal term of +protection of the economic rights over the Software. + + + Article 5 - SCOPE OF RIGHTS GRANTED + +The Licensor hereby grants to the Licensee, who accepts, the following +rights over the Software for any or all use, and for the term of the +Agreement, on the basis of the terms and conditions set forth hereinafter. + +Besides, if the Licensor owns or comes to own one or more patents +protecting all or part of the functions of the Software or of its +components, the Licensor undertakes not to enforce the rights granted by +these patents against successive Licensees using, exploiting or +modifying the Software. If these patents are transferred, the Licensor +undertakes to have the transferees subscribe to the obligations set +forth in this paragraph. + + + 5.1 RIGHT OF USE + +The Licensee is authorized to use the Software, without any limitation +as to its fields of application, with it being hereinafter specified +that this comprises: + + 1. permanent or temporary reproduction of all or part of the Software + by any or all means and in any or all form. + + 2. loading, displaying, running, or storing the Software on any or + all medium. + + 3. entitlement to observe, study or test its operation so as to + determine the ideas and principles behind any or all constituent + elements of said Software. This shall apply when the Licensee + carries out any or all loading, displaying, running, transmission + or storage operation as regards the Software, that it is entitled + to carry out hereunder. + + + 5.2 ENTITLEMENT TO MAKE CONTRIBUTIONS + +The right to make Contributions includes the right to translate, adapt, +arrange, or make any or all modifications to the Software, and the right +to reproduce the resulting software. + +The Licensee is authorized to make any or all Contributions to the +Software provided that it includes an explicit notice that it is the +author of said Contribution and indicates the date of the creation thereof. + + + 5.3 RIGHT OF DISTRIBUTION + +In particular, the right of distribution includes the right to publish, +transmit and communicate the Software to the general public on any or +all medium, and by any or all means, and the right to market, either in +consideration of a fee, or free of charge, one or more copies of the +Software by any means. + +The Licensee is further authorized to distribute copies of the modified +or unmodified Software to third parties according to the terms and +conditions set forth hereinafter. + + + 5.3.1 DISTRIBUTION OF SOFTWARE WITHOUT MODIFICATION + +The Licensee is authorized to distribute true copies of the Software in +Source Code or Object Code form, provided that said distribution +complies with all the provisions of the Agreement and is accompanied by: + + 1. a copy of the Agreement, + + 2. a notice relating to the limitation of both the Licensor's + warranty and liability as set forth in Articles 8 and 9, + +and that, in the event that only the Object Code of the Software is +redistributed, the Licensee allows effective access to the full Source +Code of the Software at a minimum during the entire period of its +distribution of the Software, it being understood that the additional +cost of acquiring the Source Code shall not exceed the cost of +transferring the data. + + + 5.3.2 DISTRIBUTION OF MODIFIED SOFTWARE + +If the Licensee makes any Contribution to the Software, the resulting +Modified Software may be distributed under a license agreement other +than this Agreement subject to compliance with the provisions of Article +5.3.4. + + + 5.3.3 DISTRIBUTION OF EXTERNAL MODULES + +When the Licensee has developed an External Module, the terms and +conditions of this Agreement do not apply to said External Module, that +may be distributed under a separate license agreement. + + + 5.3.4 CREDITS + +Any Licensee who may distribute a Modified Software hereby expressly +agrees to: + + 1. indicate in the related documentation that it is based on the + Software licensed hereunder, and reproduce the intellectual + property notice for the Software, + + 2. ensure that written indications of the Software intended use, + intellectual property notice and license hereunder are included in + easily accessible format from the Modified Software interface, + + 3. mention, on a freely accessible website describing the Modified + Software, at least throughout the distribution term thereof, that + it is based on the Software licensed hereunder, and reproduce the + Software intellectual property notice, + + 4. where it is distributed to a third party that may distribute a + Modified Software without having to make its source code + available, make its best efforts to ensure that said third party + agrees to comply with the obligations set forth in this Article . + +If the Software, whether or not modified, is distributed with an +External Module designed for use in connection with the Software, the +Licensee shall submit said External Module to the foregoing obligations. + + + 5.3.5 COMPATIBILITY WITH THE CeCILL AND CeCILL-C LICENSES + +Where a Modified Software contains a Contribution subject to the CeCILL +license, the provisions set forth in Article 5.3.4 shall be optional. + +A Modified Software may be distributed under the CeCILL-C license. In +such a case the provisions set forth in Article 5.3.4 shall be optional. + + + Article 6 - INTELLECTUAL PROPERTY + + + 6.1 OVER THE INITIAL SOFTWARE + +The Holder owns the economic rights over the Initial Software. Any or +all use of the Initial Software is subject to compliance with the terms +and conditions under which the Holder has elected to distribute its work +and no one shall be entitled to modify the terms and conditions for the +distribution of said Initial Software. + +The Holder undertakes that the Initial Software will remain ruled at +least by this Agreement, for the duration set forth in Article 4.2. + + + 6.2 OVER THE CONTRIBUTIONS + +The Licensee who develops a Contribution is the owner of the +intellectual property rights over this Contribution as defined by +applicable law. + + + 6.3 OVER THE EXTERNAL MODULES + +The Licensee who develops an External Module is the owner of the +intellectual property rights over this External Module as defined by +applicable law and is free to choose the type of agreement that shall +govern its distribution. + + + 6.4 JOINT PROVISIONS + +The Licensee expressly undertakes: + + 1. not to remove, or modify, in any manner, the intellectual property + notices attached to the Software; + + 2. to reproduce said notices, in an identical manner, in the copies + of the Software modified or not. + +The Licensee undertakes not to directly or indirectly infringe the +intellectual property rights of the Holder and/or Contributors on the +Software and to take, where applicable, vis-à-vis its staff, any and all +measures required to ensure respect of said intellectual property rights +of the Holder and/or Contributors. + + + Article 7 - RELATED SERVICES + +7.1 Under no circumstances shall the Agreement oblige the Licensor to +provide technical assistance or maintenance services for the Software. + +However, the Licensor is entitled to offer this type of services. The +terms and conditions of such technical assistance, and/or such +maintenance, shall be set forth in a separate instrument. Only the +Licensor offering said maintenance and/or technical assistance services +shall incur liability therefor. + +7.2 Similarly, any Licensor is entitled to offer to its licensees, under +its sole responsibility, a warranty, that shall only be binding upon +itself, for the redistribution of the Software and/or the Modified +Software, under terms and conditions that it is free to decide. Said +warranty, and the financial terms and conditions of its application, +shall be subject of a separate instrument executed between the Licensor +and the Licensee. + + + Article 8 - LIABILITY + +8.1 Subject to the provisions of Article 8.2, the Licensee shall be +entitled to claim compensation for any direct loss it may have suffered +from the Software as a result of a fault on the part of the relevant +Licensor, subject to providing evidence thereof. + +8.2 The Licensor's liability is limited to the commitments made under +this Agreement and shall not be incurred as a result of in particular: +(i) loss due the Licensee's total or partial failure to fulfill its +obligations, (ii) direct or consequential loss that is suffered by the +Licensee due to the use or performance of the Software, and (iii) more +generally, any consequential loss. In particular the Parties expressly +agree that any or all pecuniary or business loss (i.e. loss of data, +loss of profits, operating loss, loss of customers or orders, +opportunity cost, any disturbance to business activities) or any or all +legal proceedings instituted against the Licensee by a third party, +shall constitute consequential loss and shall not provide entitlement to +any or all compensation from the Licensor. + + + Article 9 - WARRANTY + +9.1 The Licensee acknowledges that the scientific and technical +state-of-the-art when the Software was distributed did not enable all +possible uses to be tested and verified, nor for the presence of +possible defects to be detected. In this respect, the Licensee's +attention has been drawn to the risks associated with loading, using, +modifying and/or developing and reproducing the Software which are +reserved for experienced users. + +The Licensee shall be responsible for verifying, by any or all means, +the suitability of the product for its requirements, its good working +order, and for ensuring that it shall not cause damage to either persons +or properties. + +9.2 The Licensor hereby represents, in good faith, that it is entitled +to grant all the rights over the Software (including in particular the +rights set forth in Article 5). + +9.3 The Licensee acknowledges that the Software is supplied "as is" by +the Licensor without any other express or tacit warranty, other than +that provided for in Article 9.2 and, in particular, without any warranty +as to its commercial value, its secured, safe, innovative or relevant +nature. + +Specifically, the Licensor does not warrant that the Software is free +from any error, that it will operate without interruption, that it will +be compatible with the Licensee's own equipment and software +configuration, nor that it will meet the Licensee's requirements. + +9.4 The Licensor does not either expressly or tacitly warrant that the +Software does not infringe any third party intellectual property right +relating to a patent, software or any other property right. Therefore, +the Licensor disclaims any and all liability towards the Licensee +arising out of any or all proceedings for infringement that may be +instituted in respect of the use, modification and redistribution of the +Software. Nevertheless, should such proceedings be instituted against +the Licensee, the Licensor shall provide it with technical and legal +assistance for its defense. Such technical and legal assistance shall be +decided on a case-by-case basis between the relevant Licensor and the +Licensee pursuant to a memorandum of understanding. The Licensor +disclaims any and all liability as regards the Licensee's use of the +name of the Software. No warranty is given as regards the existence of +prior rights over the name of the Software or as regards the existence +of a trademark. + + + Article 10 - TERMINATION + +10.1 In the event of a breach by the Licensee of its obligations +hereunder, the Licensor may automatically terminate this Agreement +thirty (30) days after notice has been sent to the Licensee and has +remained ineffective. + +10.2 A Licensee whose Agreement is terminated shall no longer be +authorized to use, modify or distribute the Software. However, any +licenses that it may have granted prior to termination of the Agreement +shall remain valid subject to their having been granted in compliance +with the terms and conditions hereof. + + + Article 11 - MISCELLANEOUS + + + 11.1 EXCUSABLE EVENTS + +Neither Party shall be liable for any or all delay, or failure to +perform the Agreement, that may be attributable to an event of force +majeure, an act of God or an outside cause, such as defective +functioning or interruptions of the electricity or telecommunications +networks, network paralysis following a virus attack, intervention by +government authorities, natural disasters, water damage, earthquakes, +fire, explosions, strikes and labor unrest, war, etc. + +11.2 Any failure by either Party, on one or more occasions, to invoke +one or more of the provisions hereof, shall under no circumstances be +interpreted as being a waiver by the interested Party of its right to +invoke said provision(s) subsequently. + +11.3 The Agreement cancels and replaces any or all previous agreements, +whether written or oral, between the Parties and having the same +purpose, and constitutes the entirety of the agreement between said +Parties concerning said purpose. No supplement or modification to the +terms and conditions hereof shall be effective as between the Parties +unless it is made in writing and signed by their duly authorized +representatives. + +11.4 In the event that one or more of the provisions hereof were to +conflict with a current or future applicable act or legislative text, +said act or legislative text shall prevail, and the Parties shall make +the necessary amendments so as to comply with said act or legislative +text. All other provisions shall remain effective. Similarly, invalidity +of a provision of the Agreement, for any reason whatsoever, shall not +cause the Agreement as a whole to be invalid. + + + 11.5 LANGUAGE + +The Agreement is drafted in both French and English and both versions +are deemed authentic. + + + Article 12 - NEW VERSIONS OF THE AGREEMENT + +12.1 Any person is authorized to duplicate and distribute copies of this +Agreement. + +12.2 So as to ensure coherence, the wording of this Agreement is +protected and may only be modified by the authors of the License, who +reserve the right to periodically publish updates or new versions of the +Agreement, each with a separate number. These subsequent versions may +address new issues encountered by Free Software. + +12.3 Any Software distributed under a given version of the Agreement may +only be subsequently distributed under the same version of the Agreement +or a subsequent version. + + + Article 13 - GOVERNING LAW AND JURISDICTION + +13.1 The Agreement is governed by French law. The Parties agree to +endeavor to seek an amicable solution to any disagreements or disputes +that may arise during the performance of the Agreement. + +13.2 Failing an amicable solution within two (2) months as from their +occurrence, and unless emergency proceedings are necessary, the +disagreements or disputes shall be referred to the Paris Courts having +jurisdiction, by the more diligent Party. + + +Version 1.0 dated 2006-09-05. diff --git a/README.rst b/README.rst index 90c88ef..00445f7 100644 --- a/README.rst +++ b/README.rst @@ -115,4 +115,4 @@ Dependencies ============ The required dependencies to use the software are listed -in the file `pyproject.toml `_. +in the file `pyproject.toml `_. diff --git a/doc/user_guide/cli.rst b/doc/user_guide/cli.rst index 844bc26..60ffc24 100644 --- a/doc/user_guide/cli.rst +++ b/doc/user_guide/cli.rst @@ -15,10 +15,13 @@ Usage .. code-block:: bash - hoplacli --config --config --njobs + hoplacli --config --njobs [--venv] An ``experiment.toml`` demonstration configuration file can be found in the project examples folder. +If the ``venv`` option is enabled, run the command outside of a container. +In this case, the image environment parameter is automatically set to None, +so providing it is optional. Workflow -------- @@ -27,7 +30,7 @@ Workflow 2. Load the TOML configuration file with ``tomllib``: expect four mandatory sections (``[project]``, ``[inputs]``, ``[environment]``, and ``[config]``) and one optional (``[multi]``). -3. Initialize a `:class:`~hopla.executor.Executor`` with ``[environment]`` +3. Initialize a :class:`~hopla.executor.Executor` with ``[environment]`` settings. 4. Extract and submit commands from the ``[inputs]`` settings: @@ -61,7 +64,8 @@ The configuration file is divided into several sections. ``commands`` (str or list) Commands to execute. May be a Python expression string (e.g. - ``"sleep {k}"``) or a list of explicit commands. + ``"sleep {k}"``) or a list of explicit commands. In the first case, + a ``data.tsv`` file is expected for the mapping. ``parameters`` (str) Additional parameters passed to the container execution command @@ -79,7 +83,7 @@ See the :class:`~hopla.executor.Executor` parameters. Simulate job submission without executing. ``delay_s`` (int) - Delay (seconds) between submissions. + Delay (seconds) between refresh. ``verbose`` (bool) Enable verbose logging. diff --git a/examples/plot_ccc.py b/examples/plot_ccc.py index 074a4aa..f7167f5 100644 --- a/examples/plot_ccc.py +++ b/examples/plot_ccc.py @@ -2,7 +2,7 @@ Basic example on how to use the CCC cluster =========================================== -Basic example +CCC-based cluster When you're running hundreds or thousands of jobs, automation is a necessity. This is where ``hopla`` can help you. diff --git a/examples/plot_ccc_flux_multi_tasks.py b/examples/plot_ccc_flux_multi_tasks.py index d538f04..8e09464 100644 --- a/examples/plot_ccc_flux_multi_tasks.py +++ b/examples/plot_ccc_flux_multi_tasks.py @@ -2,7 +2,7 @@ Basic example on how to use the CCC cluster using multi-tasks ============================================================= -Basic example +CCC-based cluster - flux When you're running hundreds or thousands of jobs, automation is a necessity. This is where ``hopla`` can help you. diff --git a/examples/plot_ccc_joblib_multi_tasks.py b/examples/plot_ccc_joblib_multi_tasks.py index fd6a6c2..1298775 100644 --- a/examples/plot_ccc_joblib_multi_tasks.py +++ b/examples/plot_ccc_joblib_multi_tasks.py @@ -2,7 +2,7 @@ Basic example on how to use the CCC cluster using multi-tasks ============================================================= -Basic example +CCC-based cluster - joblib When you're running hundreds or thousands of jobs, automation is a necessity. This is where ``hopla`` can help you. diff --git a/examples/plot_ccc_oneshot_multi_tasks.py b/examples/plot_ccc_oneshot_multi_tasks.py index 372e8d2..bd8202b 100644 --- a/examples/plot_ccc_oneshot_multi_tasks.py +++ b/examples/plot_ccc_oneshot_multi_tasks.py @@ -2,7 +2,7 @@ Basic example on how to use the CCC cluster using multi-tasks ============================================================= -Basic example +CCC-based cluster - oneshot When you're running hundreds or thousands of jobs, automation is a necessity. This is where ``hopla`` can help you. diff --git a/examples/plot_pbs.py b/examples/plot_pbs.py index 0416c58..563439b 100644 --- a/examples/plot_pbs.py +++ b/examples/plot_pbs.py @@ -2,7 +2,7 @@ Basic example on how to use the PBS cluster =========================================== -Basic example +CCC-based cluster - PBS When you're running hundreds or thousands of jobs, automation is a necessity. This is where ``hopla`` can help you. diff --git a/examples/plot_slurm.py b/examples/plot_slurm.py index 248990c..523fe1e 100644 --- a/examples/plot_slurm.py +++ b/examples/plot_slurm.py @@ -2,7 +2,7 @@ Basic example on how to use the SLURM cluster ============================================= -Basic example +CCC-based cluster - SLURM When you're running hundreds or thousands of jobs, automation is a necessity. This is where ``hopla`` can help you. diff --git a/pyproject.toml b/pyproject.toml index 4d85254..2c4a29d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,19 @@ [build-system] -requires = ["setuptools>=61.2"] +requires = ["setuptools>=77.0.3", "setuptools_scm"] build-backend = "setuptools.build_meta" [project] name = "hopla" readme = "README.rst" requires-python = ">=3.11" -authors = [ - {name = "hopla developers", email = "antoine.grigis@cea.fr"}, -] +authors = [{name = "hopla developers"}] license = "CECILL-B" +license-files = ["LICENSE"] classifiers = [ "Development Status :: 1 - Planning", "Environment :: Console", - "Operating System :: OS Independent", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Topic :: Scientific/Engineering", ] @@ -27,11 +27,11 @@ dynamic = ["version"] hoplacli = "hopla.cli:main" [project.urls] -Source = "https://github.com/AGrigis/hopla" -Tracker = "https://github.com/AGrigis/hopla/issues" +Development = "https://github.com/AGrigis/hopla" +issues = "https://github.com/AGrigis/hopla/issues" +homepage = "https://agrigis.github.io/hopla/stable/" [tool.setuptools] -platforms = ["Linux", "OSX"] include-package-data = true [tool.setuptools.dynamic]