Implement wave container commands

CHANGELOG.md

@@ -11,6 +11,7 @@
 - Pin j178/prek-action action to 91fd7d7 ([#3931](https://github.com/nf-core/tools/pull/3931))
 - add pre-commit hook to keep uv.lock in sync ([#3933](https://github.com/nf-core/tools/pull/3933))
 - Update mcr.microsoft.com/devcontainers/miniconda Docker digest to 2be0f5a ([#3946](https://github.com/nf-core/tools/pull/3946))
+- Implement wave container commands ([#3954](https://github.com/nf-core/tools/pull/3954))
 - Fix docker errors in test ([#3924](https://github.com/nf-core/tools/pull/3924))
 - Update actions/checkout digest to 8e8c483 ([#3956](https://github.com/nf-core/tools/pull/3956))
 

nf_core/__main__.py

@@ -27,6 +27,10 @@
     modules_remove,
     modules_test,
     modules_update,
+    modules_containers_create,
+    modules_containers_conda_lock,
+    modules_containers_list,
+    modules_containers_lint,
 )
 from nf_core.commands_pipelines import (
     pipelines_bump_version,
@@ -862,7 +866,7 @@ def command_pipelines_schema_docs(directory, schema_file, output, format, force,
     help="Do not pull in latest changes to local clone of modules repository.",
 )
 @click.command_panel("For pipeline development", commands=["list", "info", "install", "update", "remove", "patch"])
-@click.command_panel("For module development", commands=["create", "lint", "test", "bump-versions"])
+@click.command_panel("For module development", commands=["create", "lint", "test", "bump-versions", "containers"])
 @click.pass_context
 def modules(ctx, git_remote, branch, no_pull):
     """
@@ -1377,6 +1381,96 @@ def command_modules_bump_versions(ctx, tool, directory, all, show_all, dry_run):
     modules_bump_versions(ctx, tool, directory, all, show_all, dry_run)
 
 
+@modules.group("containers")
+@click.pass_context
+def modules_containers(ctx):
+    """Manage module container builds and metadata."""
+    pass
+
+
+@modules_containers.command("create")
+@click.pass_context
+@click.option(
+    "-await",
+    "--await",
+    "await_",
+    is_flag=True,
+    default=False,
+    help="Wait for the container build to finish.",
+)
+@click.argument(
+    "module",
+    type=str,
+    required=False,
+    callback=normalize_case,
+    metavar="<module> or <module/submodule>",
+    shell_complete=autocomplete_modules,
+)
+@click.option(
+    "--dry-run/--run",
+    "dry_run",
+    is_flag=True,
+    default=False,
+    help="Print the wave commands instead of executing them.",
+)
+def command_modules_containers_create(ctx, await_, dry_run, module):
+    """
+    Build docker and singularity container files for linux/arm64 and linux/amd64 with wave from environment.yml and create container config file.
+    """
+    modules_containers_create(ctx, module, await_, dry_run)
+
+
+@modules_containers.command("conda-lock")
+@click.pass_context
+@click.argument(
+    "module",
+    type=str,
+    required=False,
+    callback=normalize_case,
+    metavar="<module> or <module/submodule>",
+    shell_complete=autocomplete_modules,
+)
+def command_modules_containers_conda_lock(ctx, module):
+    """
+    Build a Docker linux/arm64 container and fetch the conda lock file for a module.
+    """
+    modules_containers_conda_lock(ctx, module)
+
+
+@modules_containers.command("lint")
+@click.pass_context
+@click.argument(
+    "module",
+    type=str,
+    required=False,
+    callback=normalize_case,
+    metavar="<module> or <module/submodule>",
+    shell_complete=autocomplete_modules,
+)
+def command_modules_containers_lint(ctx, module):
+    """
+    Confirm that container images for a module exist.
+    """
+    modules_containers_lint(ctx, module)
+
+
+@modules_containers.command("list")
+@click.pass_context
+@click.argument(
+    "module",
+    type=str,
+    required=False,
+    callback=normalize_case,
+    metavar="<module> or <module/submodule>",
+    shell_complete=autocomplete_modules,
+)
+def command_modules_containers_list(ctx, module):
+    """
+    Print containers defined in a module meta.yml.
+    """
+    modules_containers_list(ctx, module)
+
+
 # nf-core subworkflows click command
 @nf_core_cli.group(aliases=["s", "swf", "subworkflow"])
 @click.option(

nf_core/commands_modules.py

@@ -356,3 +356,91 @@ def modules_bump_versions(ctx, tool, directory, all, show_all, dry_run):
     except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
+
+
+def modules_containers_create(ctx, module, await_: bool, dry_run: bool = False):
+    """
+    Build docker and singularity containers for linux/arm64 and linux/amd64 using wave.
+    """
+    from nf_core.modules.containers import ModuleContainers
+
+    try:
+        manager = ModuleContainers(
+            ".",
+            ctx.obj.get("modules_repo_url"),
+            ctx.obj.get("modules_repo_branch"),
+            ctx.obj.get("modules_repo_no_pull"),
+            ctx.obj.get("hide_progress"),
+        )
+        containers = manager.create(module, await_, dry_run)
+        # make ruff happy
+        print(containers)
+    except (UserWarning, LookupError, FileNotFoundError, ValueError) as e:
+        log.error(e)
+        sys.exit(1)
+
+
+def modules_containers_conda_lock(ctx, module):
+    """
+    Build a Docker linux/arm64 container and fetch the conda lock file using wave.
+    """
+    from nf_core.modules.containers import ModuleContainers
+
+    try:
+        manager = ModuleContainers(
+            ".",
+            ctx.obj.get("modules_repo_url"),
+            ctx.obj.get("modules_repo_branch"),
+            ctx.obj.get("modules_repo_no_pull"),
+            ctx.obj.get("hide_progress"),
+        )
+        cmd = manager.conda_lock(module)
+        stdout.print(" ".join(cmd))
+    except (UserWarning, LookupError, FileNotFoundError, ValueError) as e:
+        log.error(e)
+        sys.exit(1)
+
+
+def modules_containers_list(ctx, module):
+    """
+    Print containers defined in a module meta.yml.
+    """
+    from nf_core.modules.containers import ModuleContainers
+
+    try:
+        manager = ModuleContainers(
+            ".",
+            ctx.obj.get("modules_repo_url"),
+            ctx.obj.get("modules_repo_branch"),
+            ctx.obj.get("modules_repo_no_pull"),
+            ctx.obj.get("hide_progress"),
+        )
+        containers = manager.list_containers(module)
+        t = rich.table.Table("Container System", "Platform", "Image")
+        for cs, p, img in containers:
+            t.add_row(cs, p, img)
+        stdout.print(t)
+    except (UserWarning, LookupError, FileNotFoundError, ValueError) as e:
+        log.error(e)
+        sys.exit(1)
+
+
+def modules_containers_lint(ctx, module):
+    """
+    Confirm containers are defined for the module.
+    """
+    from nf_core.modules.containers import ModuleContainers
+
+    try:
+        manager = ModuleContainers(
+            ".",
+            ctx.obj.get("modules_repo_url"),
+            ctx.obj.get("modules_repo_branch"),
+            ctx.obj.get("modules_repo_no_pull"),
+            ctx.obj.get("hide_progress"),
+        )
+        containers = manager.lint(module)
+        stdout.print(f"Found {len(containers)} container(s) for {module}.")
+    except (UserWarning, LookupError, FileNotFoundError, ValueError) as e:
+        log.error(e)
+        sys.exit(1)

nf_core/components/nfcore_component.py

@@ -2,11 +2,14 @@
 The NFCoreComponent class holds information and utility functions for a single module or subworkflow
 """
 
+import json
 import logging
 import re
 from pathlib import Path
 from typing import Any
 
+from nf_core.utils import NF_INSPECT_MIN_NF_VERSION, check_nextflow_version, run_cmd, set_wd_tempdir
+
 log = logging.getLogger(__name__)
 
 
@@ -58,6 +61,7 @@ def __init__(
         self.is_patched: bool = False
         self.branch: str | None = None
         self.workflow_name: str | None = None
+        self.container: str
 
         if remote_component:
             # Initialize the important files
@@ -341,3 +345,63 @@ def get_topics_from_main_nf(self) -> None:
             log.debug(f"Found {len(list(topics.keys()))} topics in {self.main_nf}")
             log.debug(f"Topics: {topics}")
             self.topics = topics
+
+    def get_container_from_main_nf(self) -> None:
+        if self.component_type == "module":
+            if check_nextflow_version(NF_INSPECT_MIN_NF_VERSION):
+                self.container = self._get_container_with_inspect()
+            else:
+                self.container = self._get_container_with_regex()
+
+            if not self.container:
+                log.warning(f"No container was extracted for {self.component_name} from {self.main_nf}")
+
+    def _get_container_with_inspect(self):
+        with set_wd_tempdir():
+            self.component_dir.absolute()
+
+            executable = "nextflow"
+            cmd_params = f"inspect -format json {self.main_nf}"
+            cmd_out = run_cmd(executable, cmd_params)
+            if cmd_out is None:
+                log.debug("Failed to run `nextflow inspect`")
+                log.debug("Falling back to regex method")
+                return self._get_container_with_regex()
+
+            out, _ = cmd_out
+            out_json = json.loads(out)
+            container = out_json.get("processes", [{}])[0].get("container", None)
+            if container is None:
+                log.debug(
+                    f"Container for {self.component_name} could not be extracted from the output of nextflow inspect"
+                )
+                log.debug(f"Output of nextflow inspect: {out}")
+                log.debug("Falling back to regex method.")
+                return self._get_container_with_regex()
+
+            return container
+
+    def _get_container_with_regex(self):
+        with open(self.main_nf) as f:
+            data = f.read()
+
+            if "container:" not in data:
+                log.debug(f"Could not find a container directive for {self.component_name} in {self.main_nf}")
+                return ""
+
+            # Regex explained:
+            #  1. negative lookahead for "container" and arbitrary white spaces.
+            #  2. Capturing group 1: Match a quote char " or '
+            #  3. Match any characters
+            #  4. Match whatever was most recently captured in capturing group 1
+            regex_container = r"(?<=container\s+)([\"']).+?(\1)"
+            match = re.search(regex_container, data)
+            if not match:
+                log.warning(
+                    f"Container for {self.component_name} could not be extracted from {self.main_nf} with regex"
+                )
+                return ""
+
+            # quotes " or ' were matched as well and are clipped
+            container = data[match.start()[0] + 1 : match.end()[0] - 1]
+            return container

nf_core/module-template/main.nf

@@ -25,6 +25,8 @@ process {{ component_name_underscore|upper }} {
     // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below.
     {% endif -%}
     conda "${moduleDir}/environment.yml"
+
+    // TODO container-conversion: Update to only one line. Move the platform logic to meta.yml
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}':
         '{{ docker_container if docker_container else 'biocontainers/YOUR-TOOL-HERE' }}' }"

nf_core/module-template/meta.yml

@@ -73,3 +73,5 @@ authors:
   - "{{ author }}"
 maintainers:
   - "{{ author }}"
+
+# TODO container-conversion: Add "containers" section