[WIP] Command to generate pipeline container config files

CHANGELOG.md

@@ -12,6 +12,7 @@
 - add pre-commit hook to keep uv.lock in sync ([#3933](https://github.com/nf-core/tools/pull/3933))
 - Update mcr.microsoft.com/devcontainers/miniconda Docker digest to 2be0f5a ([#3946](https://github.com/nf-core/tools/pull/3946))
 - Fix docker errors in test ([#3924](https://github.com/nf-core/tools/pull/3924))
+- Add new command to generate pipeline container config files ([#3955](https://github.com/nf-core/tools/pull/3955))
 
 ### Template
 

nf_core/components/components_utils.py

@@ -9,6 +9,7 @@
 
 import nf_core.utils
 from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.pipelines.containers_utils import ContainerConfigs
 
 log = logging.getLogger(__name__)
 
@@ -279,3 +280,10 @@ def _iterate_input_output(type) -> DictWithStrAndTuple:
     # If the tool name was not found in the response
     log.warning(f"Could not find an EDAM ontology term for '{tool_name}'")
     return None
+
+
+def try_generate_container_configs(directory: str | Path, path: str):
+    try:
+        ContainerConfigs(directory, path).generate_container_configs()
+    except UserWarning as e:
+        log.warning(f"Could not regenerate container configuration files: {e}")

nf_core/components/install.py

@@ -16,6 +16,7 @@
 from nf_core.components.components_utils import (
     get_components_to_install,
     prompt_component_version_sha,
+    try_generate_container_configs,
 )
 from nf_core.components.constants import (
     NF_CORE_MODULES_NAME,
@@ -157,7 +158,7 @@ def install(self, component: str | dict[str, str], silent: bool = False) -> bool
         if not self.install_component_files(component, version, self.modules_repo, install_folder):
             return False
 
-        # Update module.json with newly installed subworkflow
+        # Update module.json with newly installed component
         modules_json.load()
         modules_json.update(
             self.component_type, self.modules_repo, component, version, self.installed_by, install_track
@@ -167,6 +168,10 @@ def install(self, component: str | dict[str, str], silent: bool = False) -> bool
             # Install included modules and subworkflows
             self.install_included_components(component_dir)
 
+        # Regenerate container configuration files for the pipeline when modules are installed
+        if self.component_type == "modules":
+            try_generate_container_configs(self.directory, self.modules_repo.repo_path)
+
         if not silent:
             modules_json.load()
             modules_json.dump(run_prettier=True)

nf_core/components/remove.py

@@ -8,6 +8,7 @@
 
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
+from nf_core.components.components_utils import try_generate_container_configs
 from nf_core.modules.modules_json import ModulesJson
 
 from .install import ComponentInstall
@@ -172,6 +173,10 @@ def remove(self, component, repo_url=None, repo_path=None, removed_by=None, remo
                     # remember removed dependencies
                     if dependency_removed:
                         removed_components.append(component_name.replace("/", "_"))
+            # Regenerate container configuration files for the pipeline when modules are removed
+            if self.component_type == "modules":
+                try_generate_container_configs(self.directory, repo_path)
+
             # print removed dependencies
             if removed_components:
                 log.info(f"Removed files for '{component}' and its dependencies '{', '.join(removed_components)}'.")

nf_core/components/update.py

@@ -13,6 +13,7 @@
 from nf_core.components.components_utils import (
     get_components_to_install,
     prompt_component_version_sha,
+    try_generate_container_configs,
 )
 from nf_core.components.install import ComponentInstall
 from nf_core.components.remove import ComponentRemove
@@ -298,6 +299,10 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 # Update modules.json with newly installed component
                 self.modules_json.update(self.component_type, modules_repo, component, version, installed_by=None)
                 updated.append(component)
+
+                # Regenerate container configuration files for the pipeline when modules are updated
+                if self.component_type == "modules":
+                    try_generate_container_configs(self.directory, modules_repo.repo_path)
                 recursive_update = True
                 modules_to_update, subworkflows_to_update = self.get_components_to_update(component)
                 if not silent and len(modules_to_update + subworkflows_to_update) > 0:

nf_core/pipelines/containers_utils.py

@@ -0,0 +1,133 @@
+import logging
+import re
+from pathlib import Path
+
+import yaml
+
+from nf_core.utils import NF_INSPECT_MIN_NF_VERSION, check_nextflow_version, pretty_nf_version, run_cmd
+
+log = logging.getLogger(__name__)
+
+
+class ContainerConfigs:
+    """Generates the container configuration files for a pipeline.
+
+    Args:
+        workflow_directory (str | Path): The directory containing the workflow files.
+        org (str): Organisation path.
+    """
+
+    def __init__(
+        self,
+        workflow_directory: str | Path = ".",
+        org: str = "nf-core",
+    ):
+        self.workflow_directory = Path(workflow_directory)
+        self.org: str = org
+
+    def generate_container_configs(self) -> None:
+        """Generate the container configuration files for a pipeline."""
+        self.check_nextflow_version_sufficient()
+        default_config = self.generate_default_container_config()
+        self.generate_all_container_configs(default_config)
+
+    def check_nextflow_version_sufficient(self) -> None:
+        """Check if the Nextflow version is sufficient to run `nextflow inspect`."""
+        if not check_nextflow_version(NF_INSPECT_MIN_NF_VERSION):
+            raise UserWarning(
+                f"To use Seqera containers Nextflow version >= {pretty_nf_version(NF_INSPECT_MIN_NF_VERSION)} is required.\n"
+                f"Please update your Nextflow version with [magenta]'nextflow self-update'[/]\n"
+            )
+
+    def generate_default_container_config(self) -> str:
+        """
+        Generate the default container configuration file for a pipeline.
+        Requires Nextflow >= 25.04.4
+        """
+        log.debug("Generating container config file with [magenta bold]nextflow inspect[/].")
+        try:
+            # Run nextflow inspect
+            executable = "nextflow"
+            cmd_params = f"inspect -format config {self.workflow_directory}"
+            cmd_out = run_cmd(executable, cmd_params)
+            if cmd_out is None:
+                raise UserWarning("Failed to run `nextflow inspect`. Please check your Nextflow installation.")
+
+            out, _ = cmd_out
+            out_str = str(out, encoding="utf-8")
+            with open(self.workflow_directory / "conf" / "containers_docker_amd64.config", "w") as fh:
+                fh.write(out_str)
+            log.info(
+                f"Generated container config file for Docker AMD64: {self.workflow_directory / 'conf' / 'containers_docker_amd64.config'}"
+            )
+            return out_str
+
+        except RuntimeError as e:
+            log.error("Running 'nextflow inspect' failed with the following error:")
+            raise UserWarning(e)
+
+    def generate_all_container_configs(self, default_config: str) -> None:
+        """Generate the container configuration files for all platforms."""
+        containers: dict[str, dict[str, str]] = {
+            "docker_amd64": {},
+            "docker_arm64": {},
+            "singularity_oras_amd64": {},
+            "singularity_oras_arm64": {},
+            "singularity_https_amd64": {},
+            "singularity_https_arm64": {},
+            "conda_amd64_lockfile": {},
+            "conda_arm64_lockfile": {},
+        }
+        for line in default_config.split("\n"):
+            if line.startswith("process"):
+                pattern = r"process { withName: \'(.*)\' { container = \'(.*)\' } }"
+                match = re.search(pattern, line)
+                if match:
+                    try:
+                        module_name = match.group(1)
+                        container = match.group(2)
+                    except AttributeError:
+                        log.warning(f"Could not parse container for process {line}")
+                        continue
+                else:
+                    continue
+                containers["docker_amd64"][module_name] = container
+        for module_name in containers["docker_amd64"].keys():
+            # Find module containers in meta.yml
+            if "_" in module_name:
+                module_path = Path(module_name.split("_")[0].lower()) / module_name.split("_")[1].lower()
+            else:
+                module_path = Path(module_name.lower())
+
+            try:
+                with open(self.workflow_directory / "modules" / self.org / module_path / "meta.yml") as fh:
+                    meta = yaml.safe_load(fh)
+            except FileNotFoundError:
+                log.warning(f"Could not find meta.yml for {module_name}")
+                continue
+
+            platforms: dict[str, list[str]] = {
+                "docker_amd64": ["docker", "linux_amd64", "name"],
+                "docker_arm64": ["docker", "linux_arm64", "name"],
+                "singularity_oras_amd64": ["singularity", "linux_amd64", "name"],
+                "singularity_oras_arm64": ["singularity", "linux_arm64", "name"],
+                "singularity_https_amd64": ["singularity", "linux_amd64", "https"],
+                "singularity_https_arm64": ["singularity", "linux_arm64", "https"],
+                "conda_amd64_lockfile": ["conda", "linux_amd64", "lock_file"],
+                "conda_arm64_lockfile": ["conda", "linux_arm64", "lock_file"],
+            }
+
+            for p_name, (runtime, arch, protocol) in platforms.items():
+                try:
+                    containers[p_name][module_name] = meta["containers"][runtime][arch][protocol]
+                except KeyError:
+                    log.warning(f"Could not find {p_name} container for {module_name}")
+                    continue
+
+        # write config files
+        for platform in containers.keys():
+            with open(self.workflow_directory / "conf" / f"containers_{platform}.config", "w") as fh:
+                for module_name in containers[platform].keys():
+                    fh.write(
+                        f"process {{ withName: '{module_name}' {{ container = '{containers[platform][module_name]}' }} }}\n"
+                    )

tests/pipelines/test_container_configs.py

@@ -0,0 +1,159 @@
+"""Tests for the ContainerConfigs helper used by pipelines."""
+
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import yaml
+
+from nf_core.pipelines.containers_utils import ContainerConfigs
+from nf_core.utils import NF_INSPECT_MIN_NF_VERSION, pretty_nf_version
+
+from ..test_pipelines import TestPipelines
+
+
+class TestContainerConfigs(TestPipelines):
+    """Tests for ContainerConfigs using a test pipeline."""
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.container_configs = ContainerConfigs(self.pipeline_dir, "nf-core")
+
+    def test_check_nextflow_version_sufficient_ok(self) -> None:
+        """check_nextflow_version should return silently when version is sufficient."""
+        with patch(
+            "nf_core.pipelines.containers_utils.check_nextflow_version",
+            return_value=True,
+        ) as mocked_check:
+            self.container_configs.check_nextflow_version_sufficient()
+
+        mocked_check.assert_called_once_with(NF_INSPECT_MIN_NF_VERSION)
+
+    def test_check_nextflow_version_sufficient_too_low(self) -> None:
+        """check_nextflow_version should raise UserWarning when version is too low."""
+        with patch(
+            "nf_core.pipelines.containers_utils.check_nextflow_version",
+            return_value=False,
+        ):
+            with pytest.raises(UserWarning) as excinfo:
+                self.container_configs.check_nextflow_version_sufficient()
+
+        # Error message should mention the minimal required version
+        assert pretty_nf_version(NF_INSPECT_MIN_NF_VERSION) in str(excinfo.value)
+
+    def test_generate_default_container_config(self) -> None:
+        """Run generate_default_container_config with mocking."""
+        mock_config_bytes = b"process { withName: 'FOO_BAR' { container = 'docker://foo/bar:amd64' } }\n"
+
+        with patch(
+            "nf_core.pipelines.containers_utils.run_cmd",
+            return_value=(mock_config_bytes, b""),
+        ) as mocked_run_cmd:
+            out = self.container_configs.generate_default_container_config()
+
+        expected_cmd_params = f"inspect -format config {self.pipeline_dir}"
+        mocked_run_cmd.assert_called_once_with("nextflow", expected_cmd_params)
+
+        conf_path = Path(self.pipeline_dir / "conf" / "containers_docker_amd64.config")
+        assert conf_path.exists()
+        conf_path_content = conf_path.read_text(encoding="utf-8")
+        assert conf_path_content == mock_config_bytes.decode("utf-8")
+        assert out == conf_path_content
+
+    def test_generate_default_container_config_in_pipeline(self) -> None:
+        """Run generate_default_container_config in a pipeline."""
+        out = self.container_configs.generate_default_container_config()
+        conf_path = Path(self.pipeline_dir / "conf" / "containers_docker_amd64.config")
+        assert conf_path.exists()
+        conf_path_content = conf_path.read_text(encoding="utf-8")
+        # FASTQC and MULTIQC should be present in the config file
+        # Don't check for the exact version
+        assert "process { withName: 'FASTQC' { container = 'quay.io/biocontainers/fastqc" in conf_path_content
+        assert "process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc" in out
+
+    def test_generate_all_container_configs(self) -> None:
+        """Run generate_all_container_configs in a pipeline."""
+        # Mock generate_default_container_config() output
+        default_config = (
+            "process { withName: 'FASTQC' { container = 'quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0' } }\n"
+            "process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.32--d58f60e4deb769bf' } }\n"
+        )
+
+        # TODO: Test with real meata.yml files once they are available in the template
+        # Update meta.yml files
+        fastqc_dir = self.pipeline_dir / "modules" / "nf-core" / "fastqc"
+        meta = {
+            "containers": {
+                "docker": {
+                    "linux_amd64": {
+                        "name": "quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0",
+                    },
+                    "linux_arm64": {
+                        "name": "community.wave.seqera.io/library/fastqc:0.12.1--d3caca66b4f3d3b0",
+                    },
+                },
+                "singularity": {
+                    "linux_amd64": {
+                        "name": "oras://community.wave.seqera.io/library/fastqc:0.12.1--0827550dd72a3745",
+                        "https": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b2/b280a35770a70ed67008c1d6b6db118409bc3adbb3a98edcd55991189e5116f6/data",
+                    },
+                    "linux_arm64": {
+                        "name": "oras://community.wave.seqera.io/library/fastqc:0.12.1--b2ccdee5305e5859",
+                        "https": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/76/76e744b425a6b4c7eb8f12e03fa15daf7054de36557d2f0c4eb53ad952f9b0e3/data",
+                    },
+                },
+                "conda": {
+                    "linux_amd64": {
+                        "lock_file": "https://wave.seqera.io/v1alpha1/builds/5cfd0f3cb6760c42_1/condalock",
+                    },
+                    "linux_arm64": {
+                        "lock_file": "https://wave.seqera.io/v1alpha1/builds/d3caca66b4f3d3b0_1/condalock",
+                    },
+                },
+            },
+        }
+        with (fastqc_dir / "meta.yml").open("r") as fh:
+            current_meta = yaml.safe_load(fh)
+            current_meta.update(meta)
+        with (fastqc_dir / "meta.yml").open("w") as fh:
+            yaml.safe_dump(current_meta, fh)
+
+        self.container_configs.generate_all_container_configs(default_config)
+
+        conf_dir = self.pipeline_dir / "conf"
+        # Expected platforms and one expected container
+        expected_platforms = {
+            "docker_arm64": {
+                "FASTQC": "community.wave.seqera.io/library/fastqc:0.12.1--d3caca66b4f3d3b0",
+            },
+            "singularity_oras_amd64": {
+                "FASTQC": "oras://community.wave.seqera.io/library/fastqc:0.12.1--0827550dd72a3745",
+            },
+            "singularity_oras_arm64": {
+                "FASTQC": "oras://community.wave.seqera.io/library/fastqc:0.12.1--b2ccdee5305e5859",
+            },
+            "singularity_https_amd64": {
+                "FASTQC": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b2/b280a35770a70ed67008c1d6b6db118409bc3adbb3a98edcd55991189e5116f6/data",
+            },
+            "singularity_https_arm64": {
+                "FASTQC": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/76/76e744b425a6b4c7eb8f12e03fa15daf7054de36557d2f0c4eb53ad952f9b0e3/data",
+            },
+            "conda_amd64_lockfile": {
+                "FASTQC": "https://wave.seqera.io/v1alpha1/builds/5cfd0f3cb6760c42_1/condalock",
+            },
+            "conda_arm64_lockfile": {
+                "FASTQC": "https://wave.seqera.io/v1alpha1/builds/d3caca66b4f3d3b0_1/condalock",
+            },
+        }
+
+        for platform in expected_platforms.keys():
+            cfg_path = conf_dir / f"containers_{platform}.config"
+            print(cfg_path)
+            assert cfg_path.exists()
+            with cfg_path.open("r") as fh:
+                content = fh.readlines()
+                print(content)
+                assert (
+                    f"process {{ withName: 'FASTQC' {{ container = '{expected_platforms[platform]['FASTQC']}' }} }}\n"
+                    in content
+                )