From cb07e9d233b56d868837ab8b14f59a670c27a7fc Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Tue, 2 Dec 2025 12:11:35 +0100 Subject: [PATCH 01/43] WIP: Start working on wave commands: Add command dummies. --- nf_core/__main__.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index de682be24f..e1902c936d 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -862,7 +862,7 @@ def command_pipelines_schema_docs(directory, schema_file, output, format, force, help="Do not pull in latest changes to local clone of modules repository.", ) @click.command_panel("For pipeline development", commands=["list", "info", "install", "update", "remove", "patch"]) -@click.command_panel("For module development", commands=["create", "lint", "test", "bump-versions"]) +@click.command_panel("For module development", commands=["create", "lint", "test", "bump-versions", "containers"]) @click.pass_context def modules(ctx, git_remote, branch, no_pull): """ @@ -1377,6 +1377,39 @@ def command_modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): modules_bump_versions(ctx, tool, directory, all, show_all, dry_run) +@modules.group("containers") +@click.pass_context +def modules_containers(ctx): + """ """ + pass + + +@modules_containers.command("create") +def command_modules_containers_create(ctx): + """ + Build the docker and singularity container files for linux/arm64 linux/amd64 with wave and create container config file. + """ + pass + + +@modules_containers.command("conda-lock") +def command_modules_containers_conda_lock(ctx): + """ """ + pass + + +@modules_containers.command("lint") +def command_modules_containers_lint(ctx): + """ """ + pass + + +@modules_containers.command("list") +def command_modules_containers_list(ctx): + """ """ + pass + + # nf-core subworkflows click command @nf_core_cli.group(aliases=["s", "swf", "subworkflow"]) @click.option( From aa40d8b6d15559fc2d7cf1ca4fb96d0dfd5294cc Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 2 Dec 2025 11:13:35 +0000 Subject: [PATCH 02/43] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb9aed7bcf..c700ece59c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Pin j178/prek-action action to 91fd7d7 ([#3931](https://github.com/nf-core/tools/pull/3931)) - add pre-commit hook to keep uv.lock in sync ([#3933](https://github.com/nf-core/tools/pull/3933)) - Update mcr.microsoft.com/devcontainers/miniconda Docker digest to 2be0f5a ([#3946](https://github.com/nf-core/tools/pull/3946)) +- switch pre-commit to prek for development ([#3954](https://github.com/nf-core/tools/pull/3954)) ### Template From 6b73c3c7d434bbf2698614f19508c3b5f466ec4c Mon Sep 17 00:00:00 2001 From: yuxinNing Date: Tue, 2 Dec 2025 13:30:52 +0100 Subject: [PATCH 03/43] added architecture --- nf_core/__main__.py | 72 ++++++++++++++++++++++++++++++----- nf_core/commands_modules.py | 12 ++++++ nf_core/modules/containers.py | 1 + 3 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 nf_core/modules/containers.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e1902c936d..3f3ff30dbd 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -27,6 +27,10 @@ modules_remove, modules_test, modules_update, + modules_containers_create, + modules_containers_conda_lock, + modules_containers_list, + modules_containers_lint, ) from nf_core.commands_pipelines import ( pipelines_bump_version, @@ -1380,33 +1384,83 @@ def command_modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): @modules.group("containers") @click.pass_context def modules_containers(ctx): - """ """ + """Manage module container builds and metadata.""" pass @modules_containers.command("create") -def command_modules_containers_create(ctx): +@click.pass_context +@click.option( + "-await", + "--await", + "await_", + is_flag=True, + default=False, + help="Wait for the container build to finish.", +) +@click.argument( + "module", + type=str, + required=False, + callback=normalize_case, + metavar=" or ", + shell_complete=autocomplete_modules, +) +def command_modules_containers_create(ctx, await_, module): """ - Build the docker and singularity container files for linux/arm64 linux/amd64 with wave and create container config file. + Build docker and singularity container files for linux/arm64 and linux/amd64 with wave from environment.yml and create container config file. """ pass @modules_containers.command("conda-lock") -def command_modules_containers_conda_lock(ctx): - """ """ +@click.pass_context +@click.argument( + "module", + type=str, + required=False, + callback=normalize_case, + metavar=" or ", + shell_complete=autocomplete_modules, +) +def command_modules_containers_conda_lock(ctx, module): + """ + Build a Docker linux/arm64 container and fetch the conda lock file for a module. + """ pass @modules_containers.command("lint") -def command_modules_containers_lint(ctx): - """ """ +@click.pass_context +@click.argument( + "module", + type=str, + required=False, + callback=normalize_case, + metavar=" or ", + shell_complete=autocomplete_modules, +) +def command_modules_containers_lint(ctx, module): + """ + Confirm that container images for a module exist. + """ pass @modules_containers.command("list") -def command_modules_containers_list(ctx): - """ """ +@click.pass_context +@click.argument( + "module", + type=str, + required=False, + callback=normalize_case, + metavar=" or ", + shell_complete=autocomplete_modules, +) +def command_modules_containers_list(ctx, module): + """ + Print containers defined in a module meta.yml. + """ pass diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index e8b7341827..9f6a43412f 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -356,3 +356,15 @@ def modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) + +def modules_containers_create(ctx,): + pass + +def modules_containers_conda_lock(): + pass + +def modules_containers_list(): + pass + +def modules_containers_lint(): + pass \ No newline at end of file diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py new file mode 100644 index 0000000000..eb6bb1d12b --- /dev/null +++ b/nf_core/modules/containers.py @@ -0,0 +1 @@ +import logging \ No newline at end of file From e689754277e960c64fc3341f26c49cdd52686b18 Mon Sep 17 00:00:00 2001 From: yuxinNing Date: Tue, 2 Dec 2025 13:59:35 +0100 Subject: [PATCH 04/43] wire commands with functions --- nf_core/__main__.py | 8 ++-- nf_core/commands_modules.py | 94 ++++++++++++++++++++++++++++++++----- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3f3ff30dbd..9252238b6d 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1410,7 +1410,7 @@ def command_modules_containers_create(ctx, await_, module): """ Build docker and singularity container files for linux/arm64 and linux/amd64 with wave from environment.yml and create container config file. """ - pass + modules_containers_create(ctx, module, await_) @modules_containers.command("conda-lock") @@ -1427,7 +1427,7 @@ def command_modules_containers_conda_lock(ctx, module): """ Build a Docker linux/arm64 container and fetch the conda lock file for a module. """ - pass + modules_containers_conda_lock(ctx, module) @modules_containers.command("lint") @@ -1444,7 +1444,7 @@ def command_modules_containers_lint(ctx, module): """ Confirm that container images for a module exist. """ - pass + modules_containers_lint(ctx, module) @modules_containers.command("list") @@ -1461,7 +1461,7 @@ def command_modules_containers_list(ctx, module): """ Print containers defined in a module meta.yml. """ - pass + modules_containers_list(ctx, module) # nf-core subworkflows click command diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 9f6a43412f..364fe46dd7 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -357,14 +357,86 @@ def modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): log.critical(e) sys.exit(1) -def modules_containers_create(ctx,): - pass - -def modules_containers_conda_lock(): - pass - -def modules_containers_list(): - pass - -def modules_containers_lint(): - pass \ No newline at end of file + +def modules_containers_create(ctx, module, await_: bool): + """ + Build docker and singularity containers for linux/arm64 and linux/amd64 using wave. + """ + from nf_core.modules.containers import ModuleContainers + + try: + manager = ModuleContainers( + ".", + ctx.obj.get("modules_repo_url"), + ctx.obj.get("modules_repo_branch"), + ctx.obj.get("modules_repo_no_pull"), + ctx.obj.get("hide_progress"), + ) + commands = manager.create(module, await_) + for cmd in commands: + stdout.print(" ".join(cmd)) + except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: + log.error(e) + sys.exit(1) + + +def modules_containers_conda_lock(ctx, module,): + """ + Build a Docker linux/arm64 container and fetch the conda lock file using wave. + """ + from nf_core.modules.containers import ModuleContainers + + try: + manager = ModuleContainers( + ".", + ctx.obj.get("modules_repo_url"), + ctx.obj.get("modules_repo_branch"), + ctx.obj.get("modules_repo_no_pull"), + ctx.obj.get("hide_progress"), + ) + cmd = manager.conda_lock(module) + stdout.print(" ".join(cmd)) + except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: + log.error(e) + sys.exit(1) + + +def modules_containers_list(ctx, module,): + """ + Print containers defined in a module meta.yml. + """ + from nf_core.modules.containers import ModuleContainers + + try: + manager = ModuleContainers( + ".", + ctx.obj.get("modules_repo_url"), + ctx.obj.get("modules_repo_branch"), + ctx.obj.get("modules_repo_no_pull"), + ctx.obj.get("hide_progress"), + ) + containers = manager.list_containers(module) + stdout.print("\n".join(containers)) + except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: + log.error(e) + sys.exit(1) + +def modules_containers_lint(ctx, module,): + """ + Confirm containers are defined for the module. + """ + from nf_core.modules.containers import ModuleContainers + + try: + manager = ModuleContainers( + ".", + ctx.obj.get("modules_repo_url"), + ctx.obj.get("modules_repo_branch"), + ctx.obj.get("modules_repo_no_pull"), + ctx.obj.get("hide_progress"), + ) + containers = manager.lint(module) + stdout.print(f"Found {len(containers)} container(s) for {module}.") + except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: + log.error(e) + sys.exit(1) \ No newline at end of file From 8328f6fdb7980156945438a10d3e722a7d025ffb Mon Sep 17 00:00:00 2001 From: yuxinNing Date: Tue, 2 Dec 2025 15:13:19 +0100 Subject: [PATCH 05/43] containers obj --- nf_core/modules/containers.py | 109 +++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index eb6bb1d12b..1142ed4df4 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1 +1,108 @@ -import logging \ No newline at end of file +import logging +from pathlib import Path + +import yaml + +log = logging.getLogger(__name__) + + +class ModuleContainers: + """ + Helpers for building, linting and listing module containers. + """ + + def __init__( # not sure how accurate this is... + self, + directory: str | Path = ".", + remote_url: str | None = None, + branch: str | None = None, + no_pull: bool = False, + hide_progress: bool | None = None, + ): + self.directory = Path(directory) + self.remote_url = remote_url + self.branch = branch + self.no_pull = no_pull + self.hide_progress = hide_progress + + def create(self, module: str, await_: bool = False) -> list[list[str]]: + """ + Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. + """ + # module_dir = self._resolve_module_dir(module) + # env_path = self._environment_path(module_dir) + + commands: list[list[str]] = [] + for profile in ["docker", "singularity"]: + for platform in ["linux/amd64", "linux/arm64"]: + cmd = ["wave", "--conda-file", str(env_path), "--freeze", "--platform", platform] + # here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}] + if profile == "singularity": + cmd.append("--singularity") + if await_: + cmd.append("--await") + commands.append(cmd) + return commands + + # def conda_lock(self, module: str) -> list[str]: + # """ + # Build a Docker linux/arm64 container and fetch the conda lock file using wave. + # """ + # module_dir = self._resolve_module_dir(module) + # env_path = self._environment_path(module_dir) + # return ["wave", "--conda-file", str(env_path), "--freeze", "--platform", "linux/arm64", "--await"] + + # def lint(self, module: str) -> list[str]: + # """ + # Confirm containers are defined for the module. + # """ + # return self._containers_from_meta(self._resolve_module_dir(module)) + + # def list_containers(self, module: str) -> list[str]: + # """ + # Return containers defined in the module meta.yml. + # """ + # return self._containers_from_meta(self._resolve_module_dir(module)) + + def _resolve_module_dir(self, module: str) -> Path: + if module is None: + raise UserWarning("Please specify a module name.") + + module_path = Path(module) + if module_path.parts and module_path.parts[0] == "nf-core": + module_path = Path(*module_path.parts[1:]) + + module_dir = self.directory / "modules" / "nf-core" / module_path + if not module_dir.exists(): + raise LookupError(f"Module '{module}' not found at {module_dir}") + return module_dir + + @staticmethod + def _environment_path(module_dir: Path) -> Path: + env_path = module_dir / "environment.yml" + if not env_path.exists(): + raise FileNotFoundError(f"environment.yml not found for module at {module_dir}") + return env_path + + # @staticmethod + # def _containers_from_meta(module_dir: Path) -> list[str]: + # meta_path = module_dir / "meta.yml" + # if not meta_path.exists(): + # raise FileNotFoundError(f"meta.yml not found for module at {module_dir}") + + # with open(meta_path) as fh: + # meta = yaml.safe_load(fh) or {} + + # containers = meta.get("containers") + # if containers is None: + # raise UserWarning("No containers defined in meta.yml") + # if not isinstance(containers, list): + # raise ValueError("Expected 'containers' to be a list in meta.yml") + + # cleaned = [c for c in containers if c is not None and str(c).strip()] + # if len(cleaned) != len(containers): + # raise UserWarning("Empty container entries found in meta.yml") + # if len(cleaned) == 0: + # raise UserWarning("No containers defined in meta.yml") + + # return cleaned From 78358aefae92645f41c7191dba5e0afb98b9fcac Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Wed, 3 Dec 2025 10:55:00 +0100 Subject: [PATCH 06/43] Update resolve module_dir. Change to using ValueError --- nf_core/modules/containers.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 1142ed4df4..71a0bda718 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,8 +1,6 @@ import logging from pathlib import Path -import yaml - log = logging.getLogger(__name__) @@ -11,7 +9,7 @@ class ModuleContainers: Helpers for building, linting and listing module containers. """ - def __init__( # not sure how accurate this is... + def __init__( # not sure how accurate this is... self, directory: str | Path = ".", remote_url: str | None = None, @@ -31,6 +29,7 @@ def create(self, module: str, await_: bool = False) -> list[list[str]]: """ # module_dir = self._resolve_module_dir(module) # env_path = self._environment_path(module_dir) + env_path = None # TODO: remove commands: list[list[str]] = [] for profile in ["docker", "singularity"]: @@ -64,17 +63,16 @@ def create(self, module: str, await_: bool = False) -> list[list[str]]: # """ # return self._containers_from_meta(self._resolve_module_dir(module)) - def _resolve_module_dir(self, module: str) -> Path: + def _resolve_module_dir(self, module: str | Path) -> Path: if module is None: - raise UserWarning("Please specify a module name.") - - module_path = Path(module) - if module_path.parts and module_path.parts[0] == "nf-core": - module_path = Path(*module_path.parts[1:]) + raise ValueError("Please specify a module name.") - module_dir = self.directory / "modules" / "nf-core" / module_path + module_dir = Path(self.directory, "modules", "nf-core", module) if not module_dir.exists(): - raise LookupError(f"Module '{module}' not found at {module_dir}") + raise ValueError(f"Module '{module}' not found at {module_dir}") + + # TODO: Check if meta.yml and environment.yml are there + return module_dir @staticmethod From 4493cad235d5446e176ee4b8e097da1ff7852333 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Wed, 3 Dec 2025 13:41:09 +0100 Subject: [PATCH 07/43] Add TODOs --- nf_core/components/nfcore_component.py | 1 + nf_core/module-template/main.nf | 2 ++ nf_core/module-template/meta.yml | 2 ++ nf_core/modules/lint/__init__.py | 17 ++++++++++++++++- nf_core/utils.py | 2 ++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 8d2a5b9c55..1bae55e4aa 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -58,6 +58,7 @@ def __init__( self.is_patched: bool = False self.branch: str | None = None self.workflow_name: str | None = None + # TODO container-conversion: add containers (or containeR :D) if remote_component: # Initialize the important files diff --git a/nf_core/module-template/main.nf b/nf_core/module-template/main.nf index 49802b58c9..a87fce6aa5 100644 --- a/nf_core/module-template/main.nf +++ b/nf_core/module-template/main.nf @@ -25,6 +25,8 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. {% endif -%} conda "${moduleDir}/environment.yml" + + // TODO container-conversion: Update to only one line. Move the platform logic to meta.yml container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'biocontainers/YOUR-TOOL-HERE' }}' }" diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index e9e5c114bd..4884033157 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -73,3 +73,5 @@ authors: - "{{ author }}" maintainers: - "{{ author }}" + +# TODO container-conversion: Add "containers" section diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 71a79d0de5..6ca845c42e 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -236,6 +236,7 @@ def lint_module( mod.get_inputs_from_main_nf() mod.get_outputs_from_main_nf() mod.get_topics_from_main_nf() + # TODO container-conversion: get_containers from main_nf # Update meta.yml file if requested if self.fix and mod.meta_yml is not None: self.update_meta_yml_file(mod) @@ -263,6 +264,8 @@ def lint_module( mod.get_inputs_from_main_nf() mod.get_outputs_from_main_nf() mod.get_topics_from_main_nf() + # TODO container-conversion: get_containers from main_nf + # Update meta.yml file if requested if self.fix: self.update_meta_yml_file(mod) @@ -324,7 +327,13 @@ def _find_meta_info(meta_yml: dict, element_name: str, is_output=False) -> dict: return {} def _sort_meta_yml(meta_yml: dict) -> dict: - """Ensure topics comes after input/output and before authors""" + """ + Ensure topics comes after input/output and before authors. + Ensure containers comes at the end of the meta.yml. + """ + + # TODO container-conversion: Sort container section to end of meta.yml + # Early return if no topics to reorder if "topics" not in meta_yml: return meta_yml @@ -363,6 +372,9 @@ def _sort_meta_yml(meta_yml: dict) -> dict: if "output" in meta_yml: correct_outputs = self.obtain_outputs(mod.outputs) meta_outputs = self.obtain_outputs(meta_yml["output"]) + if "containers" in meta_yml: + # TODO container-conversion: Read from main.nf + pass correct_topics = self.obtain_topics(mod.topics) meta_topics = self.obtain_topics(meta_yml.get("topics", {})) @@ -543,6 +555,9 @@ def _add_edam_ontologies(section, edam_formats, desc): if hasattr(corrected_meta_yml["output"][versions_key], "yaml_set_anchor"): corrected_meta_yml["output"][versions_key].yaml_set_anchor(versions_key) + # TODO container-conversion: If containers in original meta.yml: + # - Run _add_containers + corrected_meta_yml = _sort_meta_yml(corrected_meta_yml) with open(mod.meta_yml, "w") as fh: diff --git a/nf_core/utils.py b/nf_core/utils.py index ad72559e7b..8e7a460be5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -92,6 +92,8 @@ ) NFCORE_DIR = Path(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME") or "", ".config")), "nfcore") +# TODO container-conversion: Add constants + def fetch_remote_version(source_url): response = requests.get(source_url, timeout=3) From 1915c1539c3362d801f0aebe7b71f8d8e16446ab Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Wed, 3 Dec 2025 13:55:22 +0100 Subject: [PATCH 08/43] WIP: add container listing --- nf_core/modules/containers.py | 69 +++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 71a0bda718..da36be458a 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,8 +1,14 @@ import logging from pathlib import Path +from nf_core.modules.info import ModuleInfo + log = logging.getLogger(__name__) +# TODO: Use these constants +CONTAINER_SYSTEMS = ["docker", "singularity"] +CONTAINER_PLATFORMS = ["linux/amd64", "linux/arm64"] + class ModuleContainers: """ @@ -57,11 +63,17 @@ def create(self, module: str, await_: bool = False) -> list[list[str]]: # """ # return self._containers_from_meta(self._resolve_module_dir(module)) - # def list_containers(self, module: str) -> list[str]: - # """ - # Return containers defined in the module meta.yml. - # """ - # return self._containers_from_meta(self._resolve_module_dir(module)) + def list_containers(self, module: str) -> None: + """ + Print containers defined in the module meta.yml. + """ + containers_valid = self._containers_from_meta(self, module, self.directory) + # TODO container-conversion: Print container list (as rich table?) + + # make ruff happy ... + print(containers_valid) + + pass def _resolve_module_dir(self, module: str | Path) -> Path: if module is None: @@ -82,25 +94,28 @@ def _environment_path(module_dir: Path) -> Path: raise FileNotFoundError(f"environment.yml not found for module at {module_dir}") return env_path - # @staticmethod - # def _containers_from_meta(module_dir: Path) -> list[str]: - # meta_path = module_dir / "meta.yml" - # if not meta_path.exists(): - # raise FileNotFoundError(f"meta.yml not found for module at {module_dir}") - - # with open(meta_path) as fh: - # meta = yaml.safe_load(fh) or {} - - # containers = meta.get("containers") - # if containers is None: - # raise UserWarning("No containers defined in meta.yml") - # if not isinstance(containers, list): - # raise ValueError("Expected 'containers' to be a list in meta.yml") - - # cleaned = [c for c in containers if c is not None and str(c).strip()] - # if len(cleaned) != len(containers): - # raise UserWarning("Empty container entries found in meta.yml") - # if len(cleaned) == 0: - # raise UserWarning("No containers defined in meta.yml") - - # return cleaned + @staticmethod + def _containers_from_meta(cls, module_name: str, dir: Path = Path(".")) -> dict: + """ + Return containers defined in the module meta.yml. + """ + module_info = ModuleInfo(dir, module_name) + module_info.get_component_info() + if module_info.meta is None: + raise ValueError(f"The meta.yml for module {module_name} could not be parsed or doesn't exist.") + + containers = module_info.meta.get("containers", None) + if containers is None: + raise ValueError(f"Required section 'containers' missing from meta.yaml for module '{module_name}'") + + for system in CONTAINER_SYSTEMS: + cs = containers.get(system, None) + if cs is None: + raise ValueError(f"Container missing for {cs}") + + for pf in CONTAINER_PLATFORMS: + spec = containers.get(pf, None) + if spec is None: + raise ValueError(f"Platform build {pf} missing for {cs} container for module {module_name}") + + return containers From 109d0b817cd2f9416e25c5d5949936e1ac78950c Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Wed, 3 Dec 2025 14:56:36 +0100 Subject: [PATCH 09/43] Move container constants to global utils. --- nf_core/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 8e7a460be5..b717bb7b36 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -92,7 +92,8 @@ ) NFCORE_DIR = Path(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME") or "", ".config")), "nfcore") -# TODO container-conversion: Add constants +CONTAINER_SYSTEMS = ["docker", "singularity"] +CONTAINER_PLATFORMS = ["linux/amd64", "linux/arm64"] def fetch_remote_version(source_url): From e723d5b2b5c1e1911d5cde6b94579551ec24ea10 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Wed, 3 Dec 2025 15:00:09 +0100 Subject: [PATCH 10/43] Use constants from global utils. Change list_containers to return list instead of printing. simplify checks --- nf_core/modules/containers.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index da36be458a..d2b7ad63d5 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -2,13 +2,10 @@ from pathlib import Path from nf_core.modules.info import ModuleInfo +from nf_core.utils import CONTAINER_PLATFORMS, CONTAINER_SYSTEMS log = logging.getLogger(__name__) -# TODO: Use these constants -CONTAINER_SYSTEMS = ["docker", "singularity"] -CONTAINER_PLATFORMS = ["linux/amd64", "linux/arm64"] - class ModuleContainers: """ @@ -63,17 +60,15 @@ def create(self, module: str, await_: bool = False) -> list[list[str]]: # """ # return self._containers_from_meta(self._resolve_module_dir(module)) - def list_containers(self, module: str) -> None: + def list_containers(self, module: str) -> list[tuple[str, str, str]]: """ - Print containers defined in the module meta.yml. + Return containers defined in the module meta.yml as a list of (, , ). """ containers_valid = self._containers_from_meta(self, module, self.directory) - # TODO container-conversion: Print container list (as rich table?) - - # make ruff happy ... - print(containers_valid) - - pass + containers_flat = [ + (cs, p, containers_valid[cs][p]["name"]) for cs in CONTAINER_SYSTEMS for p in CONTAINER_PLATFORMS + ] + return containers_flat def _resolve_module_dir(self, module: str | Path) -> Path: if module is None: @@ -104,18 +99,18 @@ def _containers_from_meta(cls, module_name: str, dir: Path = Path(".")) -> dict: if module_info.meta is None: raise ValueError(f"The meta.yml for module {module_name} could not be parsed or doesn't exist.") - containers = module_info.meta.get("containers", None) - if containers is None: + containers = module_info.meta.get("containers") + if not containers: raise ValueError(f"Required section 'containers' missing from meta.yaml for module '{module_name}'") for system in CONTAINER_SYSTEMS: - cs = containers.get(system, None) - if cs is None: + cs = containers.get(system) + if not cs: raise ValueError(f"Container missing for {cs}") for pf in CONTAINER_PLATFORMS: - spec = containers.get(pf, None) - if spec is None: + spec = containers.get(pf) + if not spec: raise ValueError(f"Platform build {pf} missing for {cs} container for module {module_name}") return containers From f44757be06275ef0380e5b54f7f894760cc8b7dc Mon Sep 17 00:00:00 2001 From: yuxinNing Date: Wed, 3 Dec 2025 15:25:42 +0100 Subject: [PATCH 11/43] add dry-run to the containers create, fixed the uitls import and module path --- nf_core/__main__.py | 11 +++++++++-- nf_core/commands_modules.py | 12 ++++++++---- nf_core/modules/containers.py | 9 ++++----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9252238b6d..9eec7b7e81 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1406,11 +1406,18 @@ def modules_containers(ctx): metavar=" or ", shell_complete=autocomplete_modules, ) -def command_modules_containers_create(ctx, await_, module): +@click.option( + "--dry-run/--run", + "dry_run", + is_flag=True, + default=False, + help="Print the wave commands instead of executing them.", +) +def command_modules_containers_create(ctx, await_, dry_run, module): """ Build docker and singularity container files for linux/arm64 and linux/amd64 with wave from environment.yml and create container config file. """ - modules_containers_create(ctx, module, await_) + modules_containers_create(ctx, module, await_, dry_run) @modules_containers.command("conda-lock") diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 364fe46dd7..256ba271be 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -3,7 +3,7 @@ import rich -from nf_core.utils import rich_force_colors +from nf_core.utils import rich_force_colors, run_cmd log = logging.getLogger(__name__) stdout = rich.console.Console(force_terminal=rich_force_colors()) @@ -358,7 +358,7 @@ def modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): sys.exit(1) -def modules_containers_create(ctx, module, await_: bool): +def modules_containers_create(ctx, module, await_: bool, dry_run: bool=False): """ Build docker and singularity containers for linux/arm64 and linux/amd64 using wave. """ @@ -373,8 +373,12 @@ def modules_containers_create(ctx, module, await_: bool): ctx.obj.get("hide_progress"), ) commands = manager.create(module, await_) - for cmd in commands: - stdout.print(" ".join(cmd)) + if dry_run: + for cmd in commands: + stdout.print(" ".join(cmd)) + else: + for cmd in commands: + run_cmd("wave", cmd) except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index d2b7ad63d5..da30d1726e 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -30,13 +30,12 @@ def create(self, module: str, await_: bool = False) -> list[list[str]]: """ Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. """ - # module_dir = self._resolve_module_dir(module) - # env_path = self._environment_path(module_dir) - env_path = None # TODO: remove + module_dir = self._resolve_module_dir(module) + env_path = self._environment_path(module_dir) commands: list[list[str]] = [] - for profile in ["docker", "singularity"]: - for platform in ["linux/amd64", "linux/arm64"]: + for profile in CONTAINER_SYSTEMS: + for platform in CONTAINER_PLATFORMS: cmd = ["wave", "--conda-file", str(env_path), "--freeze", "--platform", platform] # here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}] if profile == "singularity": From 7c369a420ad40dea0fe759339d4d5895067a6688 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 12:55:06 +0100 Subject: [PATCH 12/43] Add contextmanager that extends set_wd but changes into a tempdir --- nf_core/utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nf_core/utils.py b/nf_core/utils.py index b717bb7b36..522b626fc0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -17,6 +17,7 @@ import shlex import subprocess import sys +import tempfile import time from collections.abc import Callable, Generator from contextlib import contextmanager @@ -1646,6 +1647,17 @@ def set_wd(path: Path) -> Generator[None, None, None]: os.chdir(start_wd) +@contextmanager +def set_wd_tempdir() -> Generator[None, None, None]: + """ + Context manager to provide and change into a tempdir and ensure its removal and return to the + original_dir upon exceptions. + """ + with tempfile.TemporaryDirectory() as tmp: + with set_wd(Path(tmp)): + yield + + def get_wf_files(wf_path: Path): """Return a list of all files in a directory (ignores .gitigore files)""" From 81ddaf8fd11d250ccb492bfc5a206cd2004ed5de Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 12:57:11 +0100 Subject: [PATCH 13/43] Add extracting container string from a module main.nf --- nf_core/components/nfcore_component.py | 65 +++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 1bae55e4aa..68e50044b3 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -2,11 +2,14 @@ The NFCoreComponent class holds information and utility functions for a single module or subworkflow """ +import json import logging import re from pathlib import Path from typing import Any +from nf_core.utils import NF_INSPECT_MIN_NF_VERSION, check_nextflow_version, run_cmd, set_wd_tempdir + log = logging.getLogger(__name__) @@ -58,7 +61,7 @@ def __init__( self.is_patched: bool = False self.branch: str | None = None self.workflow_name: str | None = None - # TODO container-conversion: add containers (or containeR :D) + self.container: str if remote_component: # Initialize the important files @@ -342,3 +345,63 @@ def get_topics_from_main_nf(self) -> None: log.debug(f"Found {len(list(topics.keys()))} topics in {self.main_nf}") log.debug(f"Topics: {topics}") self.topics = topics + + def get_containers_from_main_nf(self) -> None: + if self.component_type == "module": + if check_nextflow_version(NF_INSPECT_MIN_NF_VERSION): + self.container = self._get_container_with_inspect() + else: + self.container = self._get_container_with_regex() + + if not self.container: + log.warning(f"No container was extracted for {self.component_name} from {self.main_nf}") + + def _get_container_with_inspect(self): + with set_wd_tempdir(): + self.component_dir.absolute() + + executable = "nextflow" + cmd_params = f"inspect -format json {self.main_nf}" + cmd_out = run_cmd(executable, cmd_params) + if cmd_out is None: + log.debug("Failed to run `nextflow inspect`") + log.debug("Falling back to regex method") + return self._get_container_with_regex() + + out, _ = cmd_out + out_json = json.loads(out) + container = out_json.get("processes", [{}])[0].get("container", None) + if container is None: + log.debug( + f"Container for {self.component_name} could not be extracted from the output of nextflow inspect" + ) + log.debug(f"Output of nextflow inspect: {out}") + log.debug("Falling back to regex method.") + return self._get_container_with_regex() + + return container + + def _get_container_with_regex(self): + with open(self.main_nf) as f: + data = f.read() + + if "container:" not in data: + log.debug(f"Could not find a container directive for {self.component_name} in {self.main_nf}") + return "" + + # Regex explained: + # 1. negative lookahead for "container" and arbitrary white spaces. + # 2. Capturing group 1: Match a quote char " or ' + # 3. Match any characters + # 4. Match whatever was most recently captured in capturing group 1 + regex_container = r"(?<=container\s+)([\"']).+?(\1)" + match = re.search(regex_container, data) + if not match: + log.warning( + f"Container for {self.component_name} could not be extracted from {self.main_nf} with regex" + ) + return "" + + # quotes " or ' were matched as well and are clipped + container = data[match.start()[0] + 1 : match.end()[0] - 1] + return container From 882ed67114e965acf471218697ece959eb788e2d Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 12:58:12 +0100 Subject: [PATCH 14/43] Rename method to match field name --- nf_core/components/nfcore_component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 68e50044b3..08addc11d4 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -346,7 +346,7 @@ def get_topics_from_main_nf(self) -> None: log.debug(f"Topics: {topics}") self.topics = topics - def get_containers_from_main_nf(self) -> None: + def get_container_from_main_nf(self) -> None: if self.component_type == "module": if check_nextflow_version(NF_INSPECT_MIN_NF_VERSION): self.container = self._get_container_with_inspect() From aa9fa01b6ad7310fccb382faf5ab7900089af260 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 13:26:02 +0100 Subject: [PATCH 15/43] Print container info as richt table --- nf_core/commands_modules.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 364fe46dd7..cae52cf781 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -380,11 +380,11 @@ def modules_containers_create(ctx, module, await_: bool): sys.exit(1) -def modules_containers_conda_lock(ctx, module,): +def modules_containers_conda_lock(ctx, module): """ Build a Docker linux/arm64 container and fetch the conda lock file using wave. """ - from nf_core.modules.containers import ModuleContainers + from nf_core.modules.containers import ModuleContainers try: manager = ModuleContainers( @@ -401,7 +401,7 @@ def modules_containers_conda_lock(ctx, module,): sys.exit(1) -def modules_containers_list(ctx, module,): +def modules_containers_list(ctx, module): """ Print containers defined in a module meta.yml. """ @@ -416,12 +416,16 @@ def modules_containers_list(ctx, module,): ctx.obj.get("hide_progress"), ) containers = manager.list_containers(module) - stdout.print("\n".join(containers)) + t = rich.table.Table("Container System", "Platform", "Image") + for cs, p, img in containers: + t.add_row(cs, p, img) + stdout.print(t) except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) sys.exit(1) -def modules_containers_lint(ctx, module,): + +def modules_containers_lint(ctx, module): """ Confirm containers are defined for the module. """ @@ -439,4 +443,4 @@ def modules_containers_lint(ctx, module,): stdout.print(f"Found {len(containers)} container(s) for {module}.") except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) - sys.exit(1) \ No newline at end of file + sys.exit(1) From 11228d44947b7e652986fbac3ec54d5e15bc0891 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 15:38:22 +0100 Subject: [PATCH 16/43] Move calling wave to ModuleContainers class. Finish create method and add dry_run arg --- nf_core/commands_modules.py | 14 ++++------- nf_core/modules/containers.py | 44 +++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index b044a725a2..f9922bcd07 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -3,7 +3,7 @@ import rich -from nf_core.utils import rich_force_colors, run_cmd +from nf_core.utils import rich_force_colors log = logging.getLogger(__name__) stdout = rich.console.Console(force_terminal=rich_force_colors()) @@ -358,7 +358,7 @@ def modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): sys.exit(1) -def modules_containers_create(ctx, module, await_: bool, dry_run: bool=False): +def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False): """ Build docker and singularity containers for linux/arm64 and linux/amd64 using wave. """ @@ -372,13 +372,9 @@ def modules_containers_create(ctx, module, await_: bool, dry_run: bool=False): ctx.obj.get("modules_repo_no_pull"), ctx.obj.get("hide_progress"), ) - commands = manager.create(module, await_) - if dry_run: - for cmd in commands: - stdout.print(" ".join(cmd)) - else: - for cmd in commands: - run_cmd("wave", cmd) + containers = manager.create(module, await_, dry_run) + # make ruff happy + print(containers) except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index da30d1726e..fbee6adf65 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,8 +1,10 @@ import logging from pathlib import Path +import regex as re + from nf_core.modules.info import ModuleInfo -from nf_core.utils import CONTAINER_PLATFORMS, CONTAINER_SYSTEMS +from nf_core.utils import CONTAINER_PLATFORMS, CONTAINER_SYSTEMS, run_cmd log = logging.getLogger(__name__) @@ -26,24 +28,46 @@ def __init__( # not sure how accurate this is... self.no_pull = no_pull self.hide_progress = hide_progress - def create(self, module: str, await_: bool = False) -> list[list[str]]: + def create(self, module: str, await_: bool = False, dry_run: bool = False) -> dict[str, dict[str, dict[str, str]]]: """ Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. """ module_dir = self._resolve_module_dir(module) env_path = self._environment_path(module_dir) - commands: list[list[str]] = [] - for profile in CONTAINER_SYSTEMS: + containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} + for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: - cmd = ["wave", "--conda-file", str(env_path), "--freeze", "--platform", platform] + exectuable = "wave" + args = ["--conda-file", str(env_path), "--freeze", "--platform", platform] # here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}] - if profile == "singularity": - cmd.append("--singularity") + if cs == "singularity": + args.append("--singularity") if await_: - cmd.append("--await") - commands.append(cmd) - return commands + args.append("--await") + + args_str = " ".join(args) + log.debug(f"Wave command to request container build for {module} ({cs} {platform}): `wave {args_str}`") + if not dry_run: + out = run_cmd(exectuable, args_str) + + if out is None: + raise RuntimeError("Wave command did not return any output") + + wave_out, wave_err = out + # Match singularity and docker container image names from seqera containers + # to validate wave return value + regex_container = r"(oras://)?.*wave\.seqera\.io.+$" + match = re.match(regex_container, wave_out) + if not match: + raise RuntimeError( + f"Returned output from wave build for {module} ({cs} {platform}) could not be parsed: {str(wave_out)}" + ) + + container = match.string + containers[cs][platform]["name"] = container + + return containers # def conda_lock(self, module: str) -> list[str]: # """ From c08a6dd8d21c3077ad810bff176b5d3127fde42b Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 15:46:53 +0100 Subject: [PATCH 17/43] Fix output encoding --- nf_core/modules/containers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index fbee6adf65..77f7a4297b 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -54,14 +54,20 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di if out is None: raise RuntimeError("Wave command did not return any output") - wave_out, wave_err = out + try: + wave_out = out[0].decode() + except AttributeError: + wave_out = str(out[0]) + finally: + wave_out = wave_out.strip() + # Match singularity and docker container image names from seqera containers # to validate wave return value regex_container = r"(oras://)?.*wave\.seqera\.io.+$" match = re.match(regex_container, wave_out) if not match: raise RuntimeError( - f"Returned output from wave build for {module} ({cs} {platform}) could not be parsed: {str(wave_out)}" + f"Returned output from wave build for {module} ({cs} {platform}) could not be parsed: {wave_out}" ) container = match.string From c6e83605ded9ab497c642d343c17796bfd6f368e Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 16:16:50 +0100 Subject: [PATCH 18/43] Remove comment --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 77f7a4297b..2e13fd6c95 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -14,7 +14,7 @@ class ModuleContainers: Helpers for building, linting and listing module containers. """ - def __init__( # not sure how accurate this is... + def __init__( self, directory: str | Path = ".", remote_url: str | None = None, From 43179777d45b997b39e001090d684f7ca728b2f7 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Thu, 4 Dec 2025 18:14:25 +0100 Subject: [PATCH 19/43] Add TODOs --- nf_core/modules/containers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 2e13fd6c95..00e80897be 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -27,6 +27,7 @@ def __init__( self.branch = branch self.no_pull = no_pull self.hide_progress = hide_progress + # TODO: save the created containers in this instance def create(self, module: str, await_: bool = False, dry_run: bool = False) -> dict[str, dict[str, dict[str, str]]]: """ @@ -39,7 +40,9 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: exectuable = "wave" + # TODO: add -o yaml or -o json flag! args = ["--conda-file", str(env_path), "--freeze", "--platform", platform] + # TODO: use access tokens # here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}] if cs == "singularity": args.append("--singularity") @@ -61,6 +64,8 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di finally: wave_out = wave_out.strip() + # TODO: change to reading container, build_id, url (?) from json or yaml output! + # Match singularity and docker container image names from seqera containers # to validate wave return value regex_container = r"(oras://)?.*wave\.seqera\.io.+$" From e1d7f7d7cda720a940060a02cc509e4c899fd0a7 Mon Sep 17 00:00:00 2001 From: yuxinNing Date: Fri, 5 Dec 2025 09:55:17 +0100 Subject: [PATCH 20/43] buildid and scanid saved to the dict --- nf_core/modules/containers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 2e13fd6c95..5a5e9b3ee8 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,3 +1,4 @@ +import json import logging from pathlib import Path @@ -69,10 +70,19 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di raise RuntimeError( f"Returned output from wave build for {module} ({cs} {platform}) could not be parsed: {wave_out}" ) + + meta_out = run_cmd("wave", f'--conda-package "{module}" --platform {platform} -o json') + if meta_out is None or not meta_out[0]: + raise RuntimeError("Wave command did not return any metadata JSON") + try: + meta_data = json.loads(meta_out[0].decode()) + except (AttributeError, json.JSONDecodeError) as e: + raise RuntimeError(f"Could not parse wave JSON metadata for {module} ({cs} {platform})") from e container = match.string containers[cs][platform]["name"] = container - + containers[cs][platform]["buildId"] = meta_data.get("buildId", "") + containers[cs][platform]["scanId"] = meta_data.get("scanId", "") return containers # def conda_lock(self, module: str) -> list[str]: From 180287e0e7a9b2971e443ed89a79ec32b264f946 Mon Sep 17 00:00:00 2001 From: yuxinNing Date: Fri, 5 Dec 2025 10:44:13 +0100 Subject: [PATCH 21/43] meta yaml --- nf_core/modules/containers.py | 41 +++++++++-------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index f00c727557..86bb091c54 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,8 +1,7 @@ -import json import logging from pathlib import Path -import regex as re +import yaml from nf_core.modules.info import ModuleInfo from nf_core.utils import CONTAINER_PLATFORMS, CONTAINER_SYSTEMS, run_cmd @@ -41,9 +40,7 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: exectuable = "wave" - # TODO: add -o yaml or -o json flag! - args = ["--conda-file", str(env_path), "--freeze", "--platform", platform] - # TODO: use access tokens + args = ["--conda-file", str(env_path), "--freeze", "--platform", platform, "-o yaml"] # here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}] if cs == "singularity": args.append("--singularity") @@ -58,34 +55,16 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di if out is None: raise RuntimeError("Wave command did not return any output") + if not out[0]: + raise RuntimeError("Wave command did not return any metadata output") + try: - wave_out = out[0].decode() - except AttributeError: - wave_out = str(out[0]) - finally: - wave_out = wave_out.strip() - - # TODO: change to reading container, build_id, url (?) from json or yaml output! - - # Match singularity and docker container image names from seqera containers - # to validate wave return value - regex_container = r"(oras://)?.*wave\.seqera\.io.+$" - match = re.match(regex_container, wave_out) - if not match: - raise RuntimeError( - f"Returned output from wave build for {module} ({cs} {platform}) could not be parsed: {wave_out}" - ) - - meta_out = run_cmd("wave", f'--conda-package "{module}" --platform {platform} -o json') - if meta_out is None or not meta_out[0]: - raise RuntimeError("Wave command did not return any metadata JSON") - try: - meta_data = json.loads(meta_out[0].decode()) - except (AttributeError, json.JSONDecodeError) as e: - raise RuntimeError(f"Could not parse wave JSON metadata for {module} ({cs} {platform})") from e + meta_data = yaml.safe_load(out[0].decode()) or {} + except (AttributeError, yaml.YAMLError) as e: + raise RuntimeError(f"Could not parse wave YAML metadata for {module} ({cs} {platform})") from e - container = match.string - containers[cs][platform]["name"] = container + # container = meta_data.get("targetImage") or meta_data.get("containerImage") or "" + containers[cs][platform]["name"] = meta_data.get("targetImage") or meta_data.get("containerImage") or "" containers[cs][platform]["buildId"] = meta_data.get("buildId", "") containers[cs][platform]["scanId"] = meta_data.get("scanId", "") return containers From d6e744f07841227988300c7db9b4c31650077690 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 11:37:56 +0100 Subject: [PATCH 22/43] Update calling wave command to get buildId/scanId. Only add to outpu if they exist. --- nf_core/modules/containers.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 86bb091c54..4c8e99ef4e 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -27,7 +27,7 @@ def __init__( self.branch = branch self.no_pull = no_pull self.hide_progress = hide_progress - # TODO: save the created containers in this instance + self.containers: dict | None = None def create(self, module: str, await_: bool = False, dry_run: bool = False) -> dict[str, dict[str, dict[str, str]]]: """ @@ -41,7 +41,6 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di for platform in CONTAINER_PLATFORMS: exectuable = "wave" args = ["--conda-file", str(env_path), "--freeze", "--platform", platform, "-o yaml"] - # here "--tower-token" ${{ secrets.TOWER_ACCESS_TOKEN }} --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}] if cs == "singularity": args.append("--singularity") if await_: @@ -63,10 +62,21 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di except (AttributeError, yaml.YAMLError) as e: raise RuntimeError(f"Could not parse wave YAML metadata for {module} ({cs} {platform})") from e - # container = meta_data.get("targetImage") or meta_data.get("containerImage") or "" - containers[cs][platform]["name"] = meta_data.get("targetImage") or meta_data.get("containerImage") or "" - containers[cs][platform]["buildId"] = meta_data.get("buildId", "") - containers[cs][platform]["scanId"] = meta_data.get("scanId", "") + image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" + if not image: + raise RuntimeError(f"Wave build for {module} ({cs} {platform}) did not return a image name") + + containers[cs][platform]["name"] = image + + build_id = meta_data.get("buildId", "") + if build_id: + containers[cs][platform]["buildId"] = build_id + + scan_id = meta_data.get("scanId", "") + if scan_id: + containers[cs][platform]["scanId"] = scan_id + + self.containers = containers return containers # def conda_lock(self, module: str) -> list[str]: From ad5463cb3f584d4d61fb2a3f45a3f9098965cf4f Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 12:05:21 +0100 Subject: [PATCH 23/43] Simplify create method --- nf_core/modules/containers.py | 83 ++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 4c8e99ef4e..284eea7b1d 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -39,46 +39,57 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: - exectuable = "wave" - args = ["--conda-file", str(env_path), "--freeze", "--platform", platform, "-o yaml"] - if cs == "singularity": - args.append("--singularity") - if await_: - args.append("--await") - - args_str = " ".join(args) - log.debug(f"Wave command to request container build for {module} ({cs} {platform}): `wave {args_str}`") - if not dry_run: - out = run_cmd(exectuable, args_str) - - if out is None: - raise RuntimeError("Wave command did not return any output") - - if not out[0]: - raise RuntimeError("Wave command did not return any metadata output") - - try: - meta_data = yaml.safe_load(out[0].decode()) or {} - except (AttributeError, yaml.YAMLError) as e: - raise RuntimeError(f"Could not parse wave YAML metadata for {module} ({cs} {platform})") from e - - image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" - if not image: - raise RuntimeError(f"Wave build for {module} ({cs} {platform}) did not return a image name") - - containers[cs][platform]["name"] = image - - build_id = meta_data.get("buildId", "") - if build_id: - containers[cs][platform]["buildId"] = build_id - - scan_id = meta_data.get("scanId", "") - if scan_id: - containers[cs][platform]["scanId"] = scan_id + containers[cs][platform] = self.request_container(cs, platform, env_path, await_, dry_run) self.containers = containers return containers + @staticmethod + def request_container( + container_system: str, platform: str, conda_file: Path, await_build=False, dry_run=False + ) -> dict: + assert conda_file.exists() + assert container_system in CONTAINER_SYSTEMS + assert platform in CONTAINER_PLATFORMS + + container: dict[str, str] = dict() + exectuable = "wave" + args = ["--conda-file", str(conda_file.absolute()), "--freeze", "--platform", str(platform), "-o yaml"] + if container_system == "singularity": + args.append("--singularity") + if await_build: + args.append("--await") + + args_str = " ".join(args) + log.debug(f"Wave command to request container ({container_system} {platform}): `wave {args_str}`") + if not dry_run: + out = run_cmd(exectuable, args_str) + + if out is None: + raise RuntimeError("Wave command did not return any output") + + try: + meta_data = yaml.safe_load(out[0].decode()) or dict() + except (KeyError, AttributeError, yaml.YAMLError) as e: + log.debug(f"Output yaml from wave build command: {out}") + raise RuntimeError(f"Could not parse wave YAML metadata ({container_system} {platform})") from e + + image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" + if not image: + raise RuntimeError(f"Wave build ({container_system} {platform}) did not return a image name") + + container["name"] = image + + build_id = meta_data.get("buildId", "") + if build_id: + container["buildId"] = build_id + + scan_id = meta_data.get("scanId", "") + if scan_id: + container["scanId"] = scan_id + + return container + # def conda_lock(self, module: str) -> list[str]: # """ # Build a Docker linux/arm64 container and fetch the conda lock file using wave. From e0edf977abc5052690bf79ccb54786d8e41622e3 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 12:17:07 +0100 Subject: [PATCH 24/43] Refactor to reuse key constants. --- nf_core/modules/containers.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 284eea7b1d..d8a604e991 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -14,6 +14,10 @@ class ModuleContainers: Helpers for building, linting and listing module containers. """ + IMAGE_KEY = "name" + BUILD_ID_KEY = "buildId" + SCAN_ID_KEY = "scanId" + def __init__( self, directory: str | Path = ".", @@ -41,12 +45,19 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di for platform in CONTAINER_PLATFORMS: containers[cs][platform] = self.request_container(cs, platform, env_path, await_, dry_run) + for platform in CONTAINER_PLATFORMS: + build_id = containers.get("docker", dict()).get(platform, dict()).get(self.BUILD_ID_KEY, "") + if build_id: + # TODO: Add conda-lock url for platform based on the build_id for docker and the same platform + pass + # containers["conda"].update(dict((platform, dict(())))) + self.containers = containers return containers - @staticmethod + @classmethod def request_container( - container_system: str, platform: str, conda_file: Path, await_build=False, dry_run=False + cls, container_system: str, platform: str, conda_file: Path, await_build=False, dry_run=False ) -> dict: assert conda_file.exists() assert container_system in CONTAINER_SYSTEMS @@ -78,15 +89,15 @@ def request_container( if not image: raise RuntimeError(f"Wave build ({container_system} {platform}) did not return a image name") - container["name"] = image + container[cls.IMAGE_KEY] = image - build_id = meta_data.get("buildId", "") + build_id = meta_data.get(cls.BUILD_ID_KEY, "") if build_id: - container["buildId"] = build_id + container[cls.BUILD_ID_KEY] = build_id - scan_id = meta_data.get("scanId", "") + scan_id = meta_data.get(cls.SCAN_ID_KEY, "") if scan_id: - container["scanId"] = scan_id + container[cls.SCAN_ID_KEY] = scan_id return container From 141c2b82b2b6d0d3f8465f4419a7757fc4763236 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 12:30:07 +0100 Subject: [PATCH 25/43] Add conda-lock information to containers dict --- nf_core/modules/containers.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index d8a604e991..bc0352eb86 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from urllib.parse import quote import yaml @@ -47,10 +48,13 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di for platform in CONTAINER_PLATFORMS: build_id = containers.get("docker", dict()).get(platform, dict()).get(self.BUILD_ID_KEY, "") - if build_id: - # TODO: Add conda-lock url for platform based on the build_id for docker and the same platform - pass - # containers["conda"].update(dict((platform, dict(())))) + if not build_id: + log.debug("Docker image for {platform} missing - Conda-lock skipped") + continue + + conda_data = containers.get("conda", dict()) + conda_data.update({platform: {"lock_file": self.get_conda_lock_url(build_id)}}) + containers["conda"] = conda_data self.containers = containers return containers @@ -101,6 +105,12 @@ def request_container( return container + @staticmethod + def get_conda_lock_url(build_id) -> str: + build_id_safe = quote(build_id, safe="") + url = f" https://wave.seqera.io/v1alpha1/builds/{build_id_safe}/condalock" + return url + # def conda_lock(self, module: str) -> list[str]: # """ # Build a Docker linux/arm64 container and fetch the conda lock file using wave. From 972cd5ed8e5a00225fe42fb74ca3301ecd8d436f Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 14:57:57 +0100 Subject: [PATCH 26/43] Fix space in url --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index bc0352eb86..a035f5fac3 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -108,7 +108,7 @@ def request_container( @staticmethod def get_conda_lock_url(build_id) -> str: build_id_safe = quote(build_id, safe="") - url = f" https://wave.seqera.io/v1alpha1/builds/{build_id_safe}/condalock" + url = f"https://wave.seqera.io/v1alpha1/builds/{build_id_safe}/condalock" return url # def conda_lock(self, module: str) -> list[str]: From 78bd2733c6e63a1bd2a15d0d94800805168f965b Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 15:32:44 +0100 Subject: [PATCH 27/43] Implement conda lock functions. Do not fail in containers_from_meta if containers section missing --- nf_core/modules/containers.py | 43 +++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index a035f5fac3..212256a48a 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -2,6 +2,7 @@ from pathlib import Path from urllib.parse import quote +import requests import yaml from nf_core.modules.info import ModuleInfo @@ -18,6 +19,7 @@ class ModuleContainers: IMAGE_KEY = "name" BUILD_ID_KEY = "buildId" SCAN_ID_KEY = "scanId" + LOCK_FILE_KEY = "lock_file" def __init__( self, @@ -53,7 +55,7 @@ def create(self, module: str, await_: bool = False, dry_run: bool = False) -> di continue conda_data = containers.get("conda", dict()) - conda_data.update({platform: {"lock_file": self.get_conda_lock_url(build_id)}}) + conda_data.update({platform: {self.LOCK_FILE_KEY: self.get_conda_lock_url(build_id)}}) containers["conda"] = conda_data self.containers = containers @@ -111,13 +113,30 @@ def get_conda_lock_url(build_id) -> str: url = f"https://wave.seqera.io/v1alpha1/builds/{build_id_safe}/condalock" return url - # def conda_lock(self, module: str) -> list[str]: - # """ - # Build a Docker linux/arm64 container and fetch the conda lock file using wave. - # """ - # module_dir = self._resolve_module_dir(module) - # env_path = self._environment_path(module_dir) - # return ["wave", "--conda-file", str(env_path), "--freeze", "--platform", "linux/arm64", "--await"] + def conda_lock(self, module: str, platform: str) -> str: + """ + Get the conda lock file for an existing environment. + Try (in that order): + 1. reading from meta.yml + 2. reading from cached containers + 3. recreating with wave commands + """ + assert platform in CONTAINER_PLATFORMS + + containers = ( + self.containers or self._containers_from_meta(module, self.directory) or self.create(module) or dict() + ) + + conda_lock_url = containers.get("conda", dict()).get(platform, dict()).get(self.LOCK_FILE_KEY) + if not conda_lock_url: + raise ValueError("") + + return self.request_conda_lock(conda_lock_url) + + @staticmethod + def request_conda_lock(conda_lock_url: str) -> str: + resp = requests.get(conda_lock_url) + return resp.text # def lint(self, module: str) -> list[str]: # """ @@ -129,7 +148,7 @@ def list_containers(self, module: str) -> list[tuple[str, str, str]]: """ Return containers defined in the module meta.yml as a list of (, , ). """ - containers_valid = self._containers_from_meta(self, module, self.directory) + containers_valid = self._containers_from_meta(module, self.directory) containers_flat = [ (cs, p, containers_valid[cs][p]["name"]) for cs in CONTAINER_SYSTEMS for p in CONTAINER_PLATFORMS ] @@ -155,7 +174,7 @@ def _environment_path(module_dir: Path) -> Path: return env_path @staticmethod - def _containers_from_meta(cls, module_name: str, dir: Path = Path(".")) -> dict: + def _containers_from_meta(module_name: str, dir: Path = Path(".")) -> dict: """ Return containers defined in the module meta.yml. """ @@ -164,9 +183,9 @@ def _containers_from_meta(cls, module_name: str, dir: Path = Path(".")) -> dict: if module_info.meta is None: raise ValueError(f"The meta.yml for module {module_name} could not be parsed or doesn't exist.") - containers = module_info.meta.get("containers") + containers = module_info.meta.get("containers", dict()) if not containers: - raise ValueError(f"Required section 'containers' missing from meta.yaml for module '{module_name}'") + log.warning(f"Section 'containers' missing from meta.yaml for module '{module_name}'") for system in CONTAINER_SYSTEMS: cs = containers.get(system) From ad640fa5a69214ecb96fe2b5f24474fdc38ce046 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 16:10:41 +0100 Subject: [PATCH 28/43] Simplify class --- nf_core/commands_modules.py | 12 ++------ nf_core/modules/containers.py | 56 +++++++++++++++-------------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index f9922bcd07..386a304731 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -365,16 +365,8 @@ def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False): from nf_core.modules.containers import ModuleContainers try: - manager = ModuleContainers( - ".", - ctx.obj.get("modules_repo_url"), - ctx.obj.get("modules_repo_branch"), - ctx.obj.get("modules_repo_no_pull"), - ctx.obj.get("hide_progress"), - ) - containers = manager.create(module, await_, dry_run) - # make ruff happy - print(containers) + manager = ModuleContainers(module=module, directory=".") + _ = manager.create(await_, dry_run) except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 212256a48a..0ab682765e 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -21,32 +21,22 @@ class ModuleContainers: SCAN_ID_KEY = "scanId" LOCK_FILE_KEY = "lock_file" - def __init__( - self, - directory: str | Path = ".", - remote_url: str | None = None, - branch: str | None = None, - no_pull: bool = False, - hide_progress: bool | None = None, - ): + def __init__(self, module: str, directory: str | Path = "."): self.directory = Path(directory) - self.remote_url = remote_url - self.branch = branch - self.no_pull = no_pull - self.hide_progress = hide_progress + self.module = module + self.module_directory = self.get_module_dir(module) + self.condafile = self.get_environment_path(self.module_directory) + self.metafile = self.get_metayaml_path(self.module_directory) self.containers: dict | None = None - def create(self, module: str, await_: bool = False, dry_run: bool = False) -> dict[str, dict[str, dict[str, str]]]: + def create(self, await_: bool = False, dry_run: bool = False) -> dict[str, dict[str, dict[str, str]]]: """ Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. """ - module_dir = self._resolve_module_dir(module) - env_path = self._environment_path(module_dir) - containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: - containers[cs][platform] = self.request_container(cs, platform, env_path, await_, dry_run) + containers[cs][platform] = self.request_container(cs, platform, self.condafile, await_, dry_run) for platform in CONTAINER_PLATFORMS: build_id = containers.get("docker", dict()).get(platform, dict()).get(self.BUILD_ID_KEY, "") @@ -113,7 +103,7 @@ def get_conda_lock_url(build_id) -> str: url = f"https://wave.seqera.io/v1alpha1/builds/{build_id_safe}/condalock" return url - def conda_lock(self, module: str, platform: str) -> str: + def conda_lock(self, platform: str) -> str: """ Get the conda lock file for an existing environment. Try (in that order): @@ -123,9 +113,7 @@ def conda_lock(self, module: str, platform: str) -> str: """ assert platform in CONTAINER_PLATFORMS - containers = ( - self.containers or self._containers_from_meta(module, self.directory) or self.create(module) or dict() - ) + containers = self.containers or self.get_containers_from_meta() or self.create() or dict() conda_lock_url = containers.get("conda", dict()).get(platform, dict()).get(self.LOCK_FILE_KEY) if not conda_lock_url: @@ -144,17 +132,17 @@ def request_conda_lock(conda_lock_url: str) -> str: # """ # return self._containers_from_meta(self._resolve_module_dir(module)) - def list_containers(self, module: str) -> list[tuple[str, str, str]]: + def list_containers(self) -> list[tuple[str, str, str]]: """ Return containers defined in the module meta.yml as a list of (, , ). """ - containers_valid = self._containers_from_meta(module, self.directory) + containers_valid = self.get_containers_from_meta() containers_flat = [ (cs, p, containers_valid[cs][p]["name"]) for cs in CONTAINER_SYSTEMS for p in CONTAINER_PLATFORMS ] return containers_flat - def _resolve_module_dir(self, module: str | Path) -> Path: + def get_module_dir(self, module: str | Path) -> Path: if module is None: raise ValueError("Please specify a module name.") @@ -162,30 +150,34 @@ def _resolve_module_dir(self, module: str | Path) -> Path: if not module_dir.exists(): raise ValueError(f"Module '{module}' not found at {module_dir}") - # TODO: Check if meta.yml and environment.yml are there - return module_dir @staticmethod - def _environment_path(module_dir: Path) -> Path: + def get_environment_path(module_dir: Path) -> Path: env_path = module_dir / "environment.yml" if not env_path.exists(): raise FileNotFoundError(f"environment.yml not found for module at {module_dir}") return env_path @staticmethod - def _containers_from_meta(module_name: str, dir: Path = Path(".")) -> dict: + def get_metayaml_path(module_dir: Path) -> Path: + metayaml_path = module_dir / "meta.yaml" + if not metayaml_path.exists(): + raise FileNotFoundError(f"meta.yml not found for module at {module_dir}") + return metayaml_path + + def get_containers_from_meta(self) -> dict: """ Return containers defined in the module meta.yml. """ - module_info = ModuleInfo(dir, module_name) + module_info = ModuleInfo(dir, self.module) module_info.get_component_info() if module_info.meta is None: - raise ValueError(f"The meta.yml for module {module_name} could not be parsed or doesn't exist.") + raise ValueError(f"The meta.yml for module {self.module} could not be parsed or doesn't exist.") containers = module_info.meta.get("containers", dict()) if not containers: - log.warning(f"Section 'containers' missing from meta.yaml for module '{module_name}'") + log.warning(f"Section 'containers' missing from meta.yaml for module '{self.module}'") for system in CONTAINER_SYSTEMS: cs = containers.get(system) @@ -195,6 +187,6 @@ def _containers_from_meta(module_name: str, dir: Path = Path(".")) -> dict: for pf in CONTAINER_PLATFORMS: spec = containers.get(pf) if not spec: - raise ValueError(f"Platform build {pf} missing for {cs} container for module {module_name}") + raise ValueError(f"Platform build {pf} missing for {cs} container for module {self.module}") return containers From 9b821e7d28bf57737d896111af4d54d13b9b013f Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 16:23:22 +0100 Subject: [PATCH 29/43] Implement updating meta - WIP: sorting --- nf_core/commands_modules.py | 1 + nf_core/modules/containers.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 386a304731..1c3d693681 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -367,6 +367,7 @@ def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False): try: manager = ModuleContainers(module=module, directory=".") _ = manager.create(await_, dry_run) + manager.update_containers_in_meta() except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 0ab682765e..2b98023de3 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -190,3 +190,15 @@ def get_containers_from_meta(self) -> dict: raise ValueError(f"Platform build {pf} missing for {cs} container for module {self.module}") return containers + + def update_containers_in_meta(self) -> None: + if self.containers is None: + log.debug("Containers not initialized - running `create()` ...") + self.create() + + with open(self.metafile, "rw") as f: + meta = yaml.safe_load(f.read()) + meta.get("containers").update(self.containers) + # TODO container-conversion: sort the yaml (again) -> call linting? + out = yaml.dump(meta) + f.write(out) From f117fca35eaf782cd31eb28ad63a3447716077f5 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 16:33:14 +0100 Subject: [PATCH 30/43] Fix ModuleContainers initialisation --- nf_core/commands_modules.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 1c3d693681..7e63852c19 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -380,13 +380,7 @@ def modules_containers_conda_lock(ctx, module): from nf_core.modules.containers import ModuleContainers try: - manager = ModuleContainers( - ".", - ctx.obj.get("modules_repo_url"), - ctx.obj.get("modules_repo_branch"), - ctx.obj.get("modules_repo_no_pull"), - ctx.obj.get("hide_progress"), - ) + manager = ModuleContainers(module, ".") cmd = manager.conda_lock(module) stdout.print(" ".join(cmd)) except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: @@ -401,13 +395,7 @@ def modules_containers_list(ctx, module): from nf_core.modules.containers import ModuleContainers try: - manager = ModuleContainers( - ".", - ctx.obj.get("modules_repo_url"), - ctx.obj.get("modules_repo_branch"), - ctx.obj.get("modules_repo_no_pull"), - ctx.obj.get("hide_progress"), - ) + manager = ModuleContainers(module, ".") containers = manager.list_containers(module) t = rich.table.Table("Container System", "Platform", "Image") for cs, p, img in containers: @@ -425,13 +413,7 @@ def modules_containers_lint(ctx, module): from nf_core.modules.containers import ModuleContainers try: - manager = ModuleContainers( - ".", - ctx.obj.get("modules_repo_url"), - ctx.obj.get("modules_repo_branch"), - ctx.obj.get("modules_repo_no_pull"), - ctx.obj.get("hide_progress"), - ) + manager = ModuleContainers(module, ".") containers = manager.lint(module) stdout.print(f"Found {len(containers)} container(s) for {module}.") except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: From d0f42cc14eede7a8df41b1b52aa4d5e6cb8e1bcc Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 16:37:12 +0100 Subject: [PATCH 31/43] Fix expected meta.yml path --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 2b98023de3..5cc8d27685 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -161,7 +161,7 @@ def get_environment_path(module_dir: Path) -> Path: @staticmethod def get_metayaml_path(module_dir: Path) -> Path: - metayaml_path = module_dir / "meta.yaml" + metayaml_path = module_dir / "meta.yml" if not metayaml_path.exists(): raise FileNotFoundError(f"meta.yml not found for module at {module_dir}") return metayaml_path From 3ccd190230a9909924bf5602fac6417e597e5a0e Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 17:13:47 +0100 Subject: [PATCH 32/43] Update containers in meta.yml - WIP: sorting --- nf_core/modules/containers.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 5cc8d27685..532bcf53c0 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -196,9 +196,15 @@ def update_containers_in_meta(self) -> None: log.debug("Containers not initialized - running `create()` ...") self.create() - with open(self.metafile, "rw") as f: - meta = yaml.safe_load(f.read()) - meta.get("containers").update(self.containers) - # TODO container-conversion: sort the yaml (again) -> call linting? - out = yaml.dump(meta) + with open(self.metafile) as f: + meta = yaml.safe_load(f) + + meta_containers = meta.get("containers", dict()) + meta_containers.update(self.containers) + meta["containers"] = meta_containers + + # TODO container-conversion: sort the yaml (again) -> call linting? + + out = yaml.dump(meta) + with open(self.metafile, "w") as f: f.write(out) From cf1dcfadb56d24cf91f48f8edb00df97632d1497 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Fri, 5 Dec 2025 17:19:48 +0100 Subject: [PATCH 33/43] update container from meta method --- nf_core/modules/containers.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 532bcf53c0..605d40de3f 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -5,7 +5,6 @@ import requests import yaml -from nf_core.modules.info import ModuleInfo from nf_core.utils import CONTAINER_PLATFORMS, CONTAINER_SYSTEMS, run_cmd log = logging.getLogger(__name__) @@ -166,16 +165,19 @@ def get_metayaml_path(module_dir: Path) -> Path: raise FileNotFoundError(f"meta.yml not found for module at {module_dir}") return metayaml_path + def get_meta(self) -> dict: + with open(self.metafile) as f: + meta = yaml.safe_load(f) + return meta + def get_containers_from_meta(self) -> dict: """ Return containers defined in the module meta.yml. """ - module_info = ModuleInfo(dir, self.module) - module_info.get_component_info() - if module_info.meta is None: - raise ValueError(f"The meta.yml for module {self.module} could not be parsed or doesn't exist.") + assert self.metafile.exists() - containers = module_info.meta.get("containers", dict()) + meta = self.get_meta() + containers = meta.get("containers", dict()) if not containers: log.warning(f"Section 'containers' missing from meta.yaml for module '{self.module}'") @@ -196,9 +198,7 @@ def update_containers_in_meta(self) -> None: log.debug("Containers not initialized - running `create()` ...") self.create() - with open(self.metafile) as f: - meta = yaml.safe_load(f) - + meta = self.get_meta() meta_containers = meta.get("containers", dict()) meta_containers.update(self.containers) meta["containers"] = meta_containers From 76c64f298972fd5b0a0f8949a0c34fd9ca7ab95e Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 09:46:08 +0100 Subject: [PATCH 34/43] Review: Rename conda lock methods --- nf_core/commands_modules.py | 8 ++++---- nf_core/modules/containers.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 7e63852c19..de02cb49a2 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -3,7 +3,7 @@ import rich -from nf_core.utils import rich_force_colors +from nf_core.utils import CONTAINER_PLATFORMS, rich_force_colors log = logging.getLogger(__name__) stdout = rich.console.Console(force_terminal=rich_force_colors()) @@ -373,7 +373,7 @@ def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False): sys.exit(1) -def modules_containers_conda_lock(ctx, module): +def modules_containers_conda_lock(ctx, module, platform=CONTAINER_PLATFORMS[0]): """ Build a Docker linux/arm64 container and fetch the conda lock file using wave. """ @@ -381,8 +381,8 @@ def modules_containers_conda_lock(ctx, module): try: manager = ModuleContainers(module, ".") - cmd = manager.conda_lock(module) - stdout.print(" ".join(cmd)) + lock_file = manager.get_conda_lock_file(platform) + stdout.print(lock_file) except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 605d40de3f..c6ae9a0127 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -102,7 +102,7 @@ def get_conda_lock_url(build_id) -> str: url = f"https://wave.seqera.io/v1alpha1/builds/{build_id_safe}/condalock" return url - def conda_lock(self, platform: str) -> str: + def get_conda_lock_file(self, platform: str) -> str: """ Get the conda lock file for an existing environment. Try (in that order): @@ -118,10 +118,10 @@ def conda_lock(self, platform: str) -> str: if not conda_lock_url: raise ValueError("") - return self.request_conda_lock(conda_lock_url) + return self.request_conda_lock_file(conda_lock_url) @staticmethod - def request_conda_lock(conda_lock_url: str) -> str: + def request_conda_lock_file(conda_lock_url: str) -> str: resp = requests.get(conda_lock_url) return resp.text From 6a69fb25fcd3de3b6d8bc9300bb673ba185169a4 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 09:49:13 +0100 Subject: [PATCH 35/43] Fix typo --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index c6ae9a0127..a51177a675 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -82,7 +82,7 @@ def request_container( image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" if not image: - raise RuntimeError(f"Wave build ({container_system} {platform}) did not return a image name") + raise RuntimeError(f"Wave build ({container_system} {platform}) did not return an image name") container[cls.IMAGE_KEY] = image From ff23c7d1ee40f2a23028c84ff57e5055b0469a83 Mon Sep 17 00:00:00 2001 From: JulianFlesch Date: Mon, 8 Dec 2025 09:49:39 +0100 Subject: [PATCH 36/43] Update nf_core/modules/containers.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix typo Co-authored-by: Matthias Hörtenhuber --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 605d40de3f..8feedfa31b 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -82,7 +82,7 @@ def request_container( image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" if not image: - raise RuntimeError(f"Wave build ({container_system} {platform}) did not return a image name") + raise RuntimeError(f"Wave build ({container_system} {platform}) did not return an image name") container[cls.IMAGE_KEY] = image From b7e5102ea7e08708bab690f9acdb28d9672ce7df Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 09:54:25 +0100 Subject: [PATCH 37/43] Review command: Remove dry_run flag for module container creation# --- nf_core/__main__.py | 19 +++++---------- nf_core/commands_modules.py | 4 ++-- nf_core/modules/containers.py | 45 ++++++++++++++++------------------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9eec7b7e81..a500da6ed3 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -17,6 +17,10 @@ from nf_core import __version__ from nf_core.commands_modules import ( modules_bump_versions, + modules_containers_conda_lock, + modules_containers_create, + modules_containers_lint, + modules_containers_list, modules_create, modules_info, modules_install, @@ -27,10 +31,6 @@ modules_remove, modules_test, modules_update, - modules_containers_create, - modules_containers_conda_lock, - modules_containers_list, - modules_containers_lint, ) from nf_core.commands_pipelines import ( pipelines_bump_version, @@ -1406,18 +1406,11 @@ def modules_containers(ctx): metavar=" or ", shell_complete=autocomplete_modules, ) -@click.option( - "--dry-run/--run", - "dry_run", - is_flag=True, - default=False, - help="Print the wave commands instead of executing them.", -) -def command_modules_containers_create(ctx, await_, dry_run, module): +def command_modules_containers_create(ctx, await_, module): """ Build docker and singularity container files for linux/arm64 and linux/amd64 with wave from environment.yml and create container config file. """ - modules_containers_create(ctx, module, await_, dry_run) + modules_containers_create(ctx, module, await_) @modules_containers.command("conda-lock") diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index de02cb49a2..64596c294d 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -358,7 +358,7 @@ def modules_bump_versions(ctx, tool, directory, all, show_all, dry_run): sys.exit(1) -def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False): +def modules_containers_create(ctx, module, await_: bool): """ Build docker and singularity containers for linux/arm64 and linux/amd64 using wave. """ @@ -366,7 +366,7 @@ def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False): try: manager = ModuleContainers(module=module, directory=".") - _ = manager.create(await_, dry_run) + _ = manager.create(await_) manager.update_containers_in_meta() except (UserWarning, LookupError, FileNotFoundError, ValueError) as e: log.error(e) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index a51177a675..2290a9fc08 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -28,14 +28,14 @@ def __init__(self, module: str, directory: str | Path = "."): self.metafile = self.get_metayaml_path(self.module_directory) self.containers: dict | None = None - def create(self, await_: bool = False, dry_run: bool = False) -> dict[str, dict[str, dict[str, str]]]: + def create(self, await_: bool = False) -> dict[str, dict[str, dict[str, str]]]: """ Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. """ containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: - containers[cs][platform] = self.request_container(cs, platform, self.condafile, await_, dry_run) + containers[cs][platform] = self.request_container(cs, platform, self.condafile, await_) for platform in CONTAINER_PLATFORMS: build_id = containers.get("docker", dict()).get(platform, dict()).get(self.BUILD_ID_KEY, "") @@ -51,9 +51,7 @@ def create(self, await_: bool = False, dry_run: bool = False) -> dict[str, dict[ return containers @classmethod - def request_container( - cls, container_system: str, platform: str, conda_file: Path, await_build=False, dry_run=False - ) -> dict: + def request_container(cls, container_system: str, platform: str, conda_file: Path, await_build=False) -> dict: assert conda_file.exists() assert container_system in CONTAINER_SYSTEMS assert platform in CONTAINER_PLATFORMS @@ -68,31 +66,30 @@ def request_container( args_str = " ".join(args) log.debug(f"Wave command to request container ({container_system} {platform}): `wave {args_str}`") - if not dry_run: - out = run_cmd(exectuable, args_str) + out = run_cmd(exectuable, args_str) - if out is None: - raise RuntimeError("Wave command did not return any output") + if out is None: + raise RuntimeError("Wave command did not return any output") - try: - meta_data = yaml.safe_load(out[0].decode()) or dict() - except (KeyError, AttributeError, yaml.YAMLError) as e: - log.debug(f"Output yaml from wave build command: {out}") - raise RuntimeError(f"Could not parse wave YAML metadata ({container_system} {platform})") from e + try: + meta_data = yaml.safe_load(out[0].decode()) or dict() + except (KeyError, AttributeError, yaml.YAMLError) as e: + log.debug(f"Output yaml from wave build command: {out}") + raise RuntimeError(f"Could not parse wave YAML metadata ({container_system} {platform})") from e - image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" - if not image: - raise RuntimeError(f"Wave build ({container_system} {platform}) did not return an image name") + image = meta_data.get("targetImage") or meta_data.get("containerImage") or "" + if not image: + raise RuntimeError(f"Wave build ({container_system} {platform}) did not return an image name") - container[cls.IMAGE_KEY] = image + container[cls.IMAGE_KEY] = image - build_id = meta_data.get(cls.BUILD_ID_KEY, "") - if build_id: - container[cls.BUILD_ID_KEY] = build_id + build_id = meta_data.get(cls.BUILD_ID_KEY, "") + if build_id: + container[cls.BUILD_ID_KEY] = build_id - scan_id = meta_data.get(cls.SCAN_ID_KEY, "") - if scan_id: - container[cls.SCAN_ID_KEY] = scan_id + scan_id = meta_data.get(cls.SCAN_ID_KEY, "") + if scan_id: + container[cls.SCAN_ID_KEY] = scan_id return container From 2fa3538d9e8bcc6da36078a096a6b87e2231ebb9 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 09:57:42 +0100 Subject: [PATCH 38/43] remove unnecessary string casting --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 2290a9fc08..4fa0ca74d8 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -58,7 +58,7 @@ def request_container(cls, container_system: str, platform: str, conda_file: Pat container: dict[str, str] = dict() exectuable = "wave" - args = ["--conda-file", str(conda_file.absolute()), "--freeze", "--platform", str(platform), "-o yaml"] + args = ["--conda-file", str(conda_file.absolute()), "--freeze", "--platform", platform, "-o yaml"] if container_system == "singularity": args.append("--singularity") if await_build: From af97d90e5bfc5ad6412d99ca62e84e32bc89586a Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 10:04:48 +0100 Subject: [PATCH 39/43] Review comment: Remove extra loop for conda-lock files --- nf_core/modules/containers.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 4fa0ca74d8..e884d4f7fc 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -35,17 +35,19 @@ def create(self, await_: bool = False) -> dict[str, dict[str, dict[str, str]]]: containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: + # Add container info for all container systems containers[cs][platform] = self.request_container(cs, platform, self.condafile, await_) - for platform in CONTAINER_PLATFORMS: - build_id = containers.get("docker", dict()).get(platform, dict()).get(self.BUILD_ID_KEY, "") - if not build_id: - log.debug("Docker image for {platform} missing - Conda-lock skipped") - continue + # Add conda lock information based on info for docker container + if platform == "docker": + build_id = containers[cs][platform].get(self.BUILD_ID_KEY, "") + if not build_id: + log.debug("Docker image for {platform} missing - Conda-lock skipped") + continue - conda_data = containers.get("conda", dict()) - conda_data.update({platform: {self.LOCK_FILE_KEY: self.get_conda_lock_url(build_id)}}) - containers["conda"] = conda_data + conda_data = containers.get("conda", dict()) + conda_data.update({platform: {self.LOCK_FILE_KEY: self.get_conda_lock_url(build_id)}}) + containers["conda"] = conda_data self.containers = containers return containers From 8f3bfb1e5ac00b4107730aa95651dcef2d6d6f1e Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 12:21:46 +0100 Subject: [PATCH 40/43] Make wave container requests run concurrently --- nf_core/modules/containers.py | 43 ++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index e884d4f7fc..5a67f7a046 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -1,4 +1,5 @@ import logging +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from urllib.parse import quote @@ -28,26 +29,36 @@ def __init__(self, module: str, directory: str | Path = "."): self.metafile = self.get_metayaml_path(self.module_directory) self.containers: dict | None = None - def create(self, await_: bool = False) -> dict[str, dict[str, dict[str, str]]]: + def create(self, await_: bool = False, max_threads: int = 4) -> dict[str, dict[str, dict[str, str]]]: """ Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. """ containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} - for cs in CONTAINER_SYSTEMS: - for platform in CONTAINER_PLATFORMS: - # Add container info for all container systems - containers[cs][platform] = self.request_container(cs, platform, self.condafile, await_) - - # Add conda lock information based on info for docker container - if platform == "docker": - build_id = containers[cs][platform].get(self.BUILD_ID_KEY, "") - if not build_id: - log.debug("Docker image for {platform} missing - Conda-lock skipped") - continue - - conda_data = containers.get("conda", dict()) - conda_data.update({platform: {self.LOCK_FILE_KEY: self.get_conda_lock_url(build_id)}}) - containers["conda"] = conda_data + tasks = dict() + threads = min(len(CONTAINER_SYSTEMS) * len(CONTAINER_PLATFORMS), max_threads) + with ThreadPoolExecutor(max_workers=threads) as pool: + for cs in CONTAINER_SYSTEMS: + for platform in CONTAINER_PLATFORMS: + fut = pool.submit(self.request_container, cs, platform, self.condafile, await_) + tasks[fut] = (cs, platform) + + for fut in as_completed(tasks): + cs, platform = tasks[fut] + # Add container info for all container systems + containers[cs][platform] = tasks[fut] + + # Add conda lock information based on info for docker container + if platform != "docker": + continue + + build_id = containers[cs][platform].get(self.BUILD_ID_KEY, "") + if not build_id: + log.debug("Docker image for {platform} missing - Conda-lock skipped") + continue + + conda_data = containers.get("conda", dict()) + conda_data.update({platform: {self.LOCK_FILE_KEY: self.get_conda_lock_url(build_id)}}) + containers["conda"] = conda_data self.containers = containers return containers From 3221030c03ea03973f591da018d609b6cecf7226 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 13:39:12 +0100 Subject: [PATCH 41/43] fix gathering concurrent results --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 5a67f7a046..fea1792ef9 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -45,7 +45,7 @@ def create(self, await_: bool = False, max_threads: int = 4) -> dict[str, dict[s for fut in as_completed(tasks): cs, platform = tasks[fut] # Add container info for all container systems - containers[cs][platform] = tasks[fut] + containers[cs][platform] = fut.result() # Add conda lock information based on info for docker container if platform != "docker": From 6aca629ac39f5c1dca71c4af6c17b9452e6165ad Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 13:39:26 +0100 Subject: [PATCH 42/43] fix building conda section --- nf_core/modules/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index fea1792ef9..123558c979 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -48,7 +48,7 @@ def create(self, await_: bool = False, max_threads: int = 4) -> dict[str, dict[s containers[cs][platform] = fut.result() # Add conda lock information based on info for docker container - if platform != "docker": + if cs != "docker": continue build_id = containers[cs][platform].get(self.BUILD_ID_KEY, "") From 69b64240ffde50815df04c350881a2aa3f939edd Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Mon, 8 Dec 2025 13:41:38 +0100 Subject: [PATCH 43/43] Remove threads option. --- nf_core/modules/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/containers.py b/nf_core/modules/containers.py index 123558c979..f21d092108 100644 --- a/nf_core/modules/containers.py +++ b/nf_core/modules/containers.py @@ -29,13 +29,13 @@ def __init__(self, module: str, directory: str | Path = "."): self.metafile = self.get_metayaml_path(self.module_directory) self.containers: dict | None = None - def create(self, await_: bool = False, max_threads: int = 4) -> dict[str, dict[str, dict[str, str]]]: + def create(self, await_: bool = False) -> dict[str, dict[str, dict[str, str]]]: """ Build docker and singularity containers for linux/amd64 and linux/arm64 using wave. """ containers: dict = {cs: {p: dict() for p in CONTAINER_PLATFORMS} for cs in CONTAINER_SYSTEMS} tasks = dict() - threads = min(len(CONTAINER_SYSTEMS) * len(CONTAINER_PLATFORMS), max_threads) + threads = max(len(CONTAINER_SYSTEMS) * len(CONTAINER_PLATFORMS), 1) with ThreadPoolExecutor(max_workers=threads) as pool: for cs in CONTAINER_SYSTEMS: for platform in CONTAINER_PLATFORMS: