diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c45c8ffde..e13f32cb07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - add pre-commit hook to keep uv.lock in sync ([#3933](https://github.com/nf-core/tools/pull/3933)) - Update mcr.microsoft.com/devcontainers/miniconda Docker digest to 2be0f5a ([#3946](https://github.com/nf-core/tools/pull/3946)) - Fix docker errors in test ([#3924](https://github.com/nf-core/tools/pull/3924)) +- Add new command to generate pipeline container config files ([#3955](https://github.com/nf-core/tools/pull/3955)) ### Template diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 7423de7f2b..55c1cd84e0 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -9,6 +9,7 @@ import nf_core.utils from nf_core.modules.modules_repo import ModulesRepo +from nf_core.pipelines.containers_utils import ContainerConfigs log = logging.getLogger(__name__) @@ -279,3 +280,10 @@ def _iterate_input_output(type) -> DictWithStrAndTuple: # If the tool name was not found in the response log.warning(f"Could not find an EDAM ontology term for '{tool_name}'") return None + + +def try_generate_container_configs(directory: str | Path, path: str): + try: + ContainerConfigs(directory, path).generate_container_configs() + except UserWarning as e: + log.warning(f"Could not regenerate container configuration files: {e}") diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 61e088ef16..c5392c364a 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -16,6 +16,7 @@ from nf_core.components.components_utils import ( get_components_to_install, prompt_component_version_sha, + try_generate_container_configs, ) from nf_core.components.constants import ( NF_CORE_MODULES_NAME, @@ -157,7 +158,7 @@ def install(self, component: str | dict[str, str], silent: bool = False) -> bool if not self.install_component_files(component, version, self.modules_repo, install_folder): return False - # Update module.json with newly installed subworkflow + # Update module.json with newly installed component modules_json.load() modules_json.update( self.component_type, self.modules_repo, component, version, self.installed_by, install_track @@ -167,6 +168,10 @@ def install(self, component: str | dict[str, str], silent: bool = False) -> bool # Install included modules and subworkflows self.install_included_components(component_dir) + # Regenerate container configuration files for the pipeline when modules are installed + if self.component_type == "modules": + try_generate_container_configs(self.directory, self.modules_repo.repo_path) + if not silent: modules_json.load() modules_json.dump(run_prettier=True) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index afe12c88a9..ef0668ddd8 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -8,6 +8,7 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.components.components_utils import try_generate_container_configs from nf_core.modules.modules_json import ModulesJson from .install import ComponentInstall @@ -172,6 +173,10 @@ def remove(self, component, repo_url=None, repo_path=None, removed_by=None, remo # remember removed dependencies if dependency_removed: removed_components.append(component_name.replace("/", "_")) + # Regenerate container configuration files for the pipeline when modules are removed + if self.component_type == "modules": + try_generate_container_configs(self.directory, repo_path) + # print removed dependencies if removed_components: log.info(f"Removed files for '{component}' and its dependencies '{', '.join(removed_components)}'.") diff --git a/nf_core/components/update.py b/nf_core/components/update.py index fcdb005e9f..af97625730 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -13,6 +13,7 @@ from nf_core.components.components_utils import ( get_components_to_install, prompt_component_version_sha, + try_generate_container_configs, ) from nf_core.components.install import ComponentInstall from nf_core.components.remove import ComponentRemove @@ -298,6 +299,10 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr # Update modules.json with newly installed component self.modules_json.update(self.component_type, modules_repo, component, version, installed_by=None) updated.append(component) + + # Regenerate container configuration files for the pipeline when modules are updated + if self.component_type == "modules": + try_generate_container_configs(self.directory, modules_repo.repo_path) recursive_update = True modules_to_update, subworkflows_to_update = self.get_components_to_update(component) if not silent and len(modules_to_update + subworkflows_to_update) > 0: diff --git a/nf_core/pipelines/containers_utils.py b/nf_core/pipelines/containers_utils.py new file mode 100644 index 0000000000..07169efdd2 --- /dev/null +++ b/nf_core/pipelines/containers_utils.py @@ -0,0 +1,133 @@ +import logging +import re +from pathlib import Path + +import yaml + +from nf_core.utils import NF_INSPECT_MIN_NF_VERSION, check_nextflow_version, pretty_nf_version, run_cmd + +log = logging.getLogger(__name__) + + +class ContainerConfigs: + """Generates the container configuration files for a pipeline. + + Args: + workflow_directory (str | Path): The directory containing the workflow files. + org (str): Organisation path. + """ + + def __init__( + self, + workflow_directory: str | Path = ".", + org: str = "nf-core", + ): + self.workflow_directory = Path(workflow_directory) + self.org: str = org + + def generate_container_configs(self) -> None: + """Generate the container configuration files for a pipeline.""" + self.check_nextflow_version_sufficient() + default_config = self.generate_default_container_config() + self.generate_all_container_configs(default_config) + + def check_nextflow_version_sufficient(self) -> None: + """Check if the Nextflow version is sufficient to run `nextflow inspect`.""" + if not check_nextflow_version(NF_INSPECT_MIN_NF_VERSION): + raise UserWarning( + f"To use Seqera containers Nextflow version >= {pretty_nf_version(NF_INSPECT_MIN_NF_VERSION)} is required.\n" + f"Please update your Nextflow version with [magenta]'nextflow self-update'[/]\n" + ) + + def generate_default_container_config(self) -> str: + """ + Generate the default container configuration file for a pipeline. + Requires Nextflow >= 25.04.4 + """ + log.debug("Generating container config file with [magenta bold]nextflow inspect[/].") + try: + # Run nextflow inspect + executable = "nextflow" + cmd_params = f"inspect -format config {self.workflow_directory}" + cmd_out = run_cmd(executable, cmd_params) + if cmd_out is None: + raise UserWarning("Failed to run `nextflow inspect`. Please check your Nextflow installation.") + + out, _ = cmd_out + out_str = str(out, encoding="utf-8") + with open(self.workflow_directory / "conf" / "containers_docker_amd64.config", "w") as fh: + fh.write(out_str) + log.info( + f"Generated container config file for Docker AMD64: {self.workflow_directory / 'conf' / 'containers_docker_amd64.config'}" + ) + return out_str + + except RuntimeError as e: + log.error("Running 'nextflow inspect' failed with the following error:") + raise UserWarning(e) + + def generate_all_container_configs(self, default_config: str) -> None: + """Generate the container configuration files for all platforms.""" + containers: dict[str, dict[str, str]] = { + "docker_amd64": {}, + "docker_arm64": {}, + "singularity_oras_amd64": {}, + "singularity_oras_arm64": {}, + "singularity_https_amd64": {}, + "singularity_https_arm64": {}, + "conda_amd64_lockfile": {}, + "conda_arm64_lockfile": {}, + } + for line in default_config.split("\n"): + if line.startswith("process"): + pattern = r"process { withName: \'(.*)\' { container = \'(.*)\' } }" + match = re.search(pattern, line) + if match: + try: + module_name = match.group(1) + container = match.group(2) + except AttributeError: + log.warning(f"Could not parse container for process {line}") + continue + else: + continue + containers["docker_amd64"][module_name] = container + for module_name in containers["docker_amd64"].keys(): + # Find module containers in meta.yml + if "_" in module_name: + module_path = Path(module_name.split("_")[0].lower()) / module_name.split("_")[1].lower() + else: + module_path = Path(module_name.lower()) + + try: + with open(self.workflow_directory / "modules" / self.org / module_path / "meta.yml") as fh: + meta = yaml.safe_load(fh) + except FileNotFoundError: + log.warning(f"Could not find meta.yml for {module_name}") + continue + + platforms: dict[str, list[str]] = { + "docker_amd64": ["docker", "linux_amd64", "name"], + "docker_arm64": ["docker", "linux_arm64", "name"], + "singularity_oras_amd64": ["singularity", "linux_amd64", "name"], + "singularity_oras_arm64": ["singularity", "linux_arm64", "name"], + "singularity_https_amd64": ["singularity", "linux_amd64", "https"], + "singularity_https_arm64": ["singularity", "linux_arm64", "https"], + "conda_amd64_lockfile": ["conda", "linux_amd64", "lock_file"], + "conda_arm64_lockfile": ["conda", "linux_arm64", "lock_file"], + } + + for p_name, (runtime, arch, protocol) in platforms.items(): + try: + containers[p_name][module_name] = meta["containers"][runtime][arch][protocol] + except KeyError: + log.warning(f"Could not find {p_name} container for {module_name}") + continue + + # write config files + for platform in containers.keys(): + with open(self.workflow_directory / "conf" / f"containers_{platform}.config", "w") as fh: + for module_name in containers[platform].keys(): + fh.write( + f"process {{ withName: '{module_name}' {{ container = '{containers[platform][module_name]}' }} }}\n" + ) diff --git a/tests/pipelines/test_container_configs.py b/tests/pipelines/test_container_configs.py new file mode 100644 index 0000000000..285444a66d --- /dev/null +++ b/tests/pipelines/test_container_configs.py @@ -0,0 +1,159 @@ +"""Tests for the ContainerConfigs helper used by pipelines.""" + +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from nf_core.pipelines.containers_utils import ContainerConfigs +from nf_core.utils import NF_INSPECT_MIN_NF_VERSION, pretty_nf_version + +from ..test_pipelines import TestPipelines + + +class TestContainerConfigs(TestPipelines): + """Tests for ContainerConfigs using a test pipeline.""" + + def setUp(self) -> None: + super().setUp() + self.container_configs = ContainerConfigs(self.pipeline_dir, "nf-core") + + def test_check_nextflow_version_sufficient_ok(self) -> None: + """check_nextflow_version should return silently when version is sufficient.""" + with patch( + "nf_core.pipelines.containers_utils.check_nextflow_version", + return_value=True, + ) as mocked_check: + self.container_configs.check_nextflow_version_sufficient() + + mocked_check.assert_called_once_with(NF_INSPECT_MIN_NF_VERSION) + + def test_check_nextflow_version_sufficient_too_low(self) -> None: + """check_nextflow_version should raise UserWarning when version is too low.""" + with patch( + "nf_core.pipelines.containers_utils.check_nextflow_version", + return_value=False, + ): + with pytest.raises(UserWarning) as excinfo: + self.container_configs.check_nextflow_version_sufficient() + + # Error message should mention the minimal required version + assert pretty_nf_version(NF_INSPECT_MIN_NF_VERSION) in str(excinfo.value) + + def test_generate_default_container_config(self) -> None: + """Run generate_default_container_config with mocking.""" + mock_config_bytes = b"process { withName: 'FOO_BAR' { container = 'docker://foo/bar:amd64' } }\n" + + with patch( + "nf_core.pipelines.containers_utils.run_cmd", + return_value=(mock_config_bytes, b""), + ) as mocked_run_cmd: + out = self.container_configs.generate_default_container_config() + + expected_cmd_params = f"inspect -format config {self.pipeline_dir}" + mocked_run_cmd.assert_called_once_with("nextflow", expected_cmd_params) + + conf_path = Path(self.pipeline_dir / "conf" / "containers_docker_amd64.config") + assert conf_path.exists() + conf_path_content = conf_path.read_text(encoding="utf-8") + assert conf_path_content == mock_config_bytes.decode("utf-8") + assert out == conf_path_content + + def test_generate_default_container_config_in_pipeline(self) -> None: + """Run generate_default_container_config in a pipeline.""" + out = self.container_configs.generate_default_container_config() + conf_path = Path(self.pipeline_dir / "conf" / "containers_docker_amd64.config") + assert conf_path.exists() + conf_path_content = conf_path.read_text(encoding="utf-8") + # FASTQC and MULTIQC should be present in the config file + # Don't check for the exact version + assert "process { withName: 'FASTQC' { container = 'quay.io/biocontainers/fastqc" in conf_path_content + assert "process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc" in out + + def test_generate_all_container_configs(self) -> None: + """Run generate_all_container_configs in a pipeline.""" + # Mock generate_default_container_config() output + default_config = ( + "process { withName: 'FASTQC' { container = 'quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0' } }\n" + "process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.32--d58f60e4deb769bf' } }\n" + ) + + # TODO: Test with real meata.yml files once they are available in the template + # Update meta.yml files + fastqc_dir = self.pipeline_dir / "modules" / "nf-core" / "fastqc" + meta = { + "containers": { + "docker": { + "linux_amd64": { + "name": "quay.io/biocontainers/fastqc:0.12.1--hdfd78af_0", + }, + "linux_arm64": { + "name": "community.wave.seqera.io/library/fastqc:0.12.1--d3caca66b4f3d3b0", + }, + }, + "singularity": { + "linux_amd64": { + "name": "oras://community.wave.seqera.io/library/fastqc:0.12.1--0827550dd72a3745", + "https": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b2/b280a35770a70ed67008c1d6b6db118409bc3adbb3a98edcd55991189e5116f6/data", + }, + "linux_arm64": { + "name": "oras://community.wave.seqera.io/library/fastqc:0.12.1--b2ccdee5305e5859", + "https": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/76/76e744b425a6b4c7eb8f12e03fa15daf7054de36557d2f0c4eb53ad952f9b0e3/data", + }, + }, + "conda": { + "linux_amd64": { + "lock_file": "https://wave.seqera.io/v1alpha1/builds/5cfd0f3cb6760c42_1/condalock", + }, + "linux_arm64": { + "lock_file": "https://wave.seqera.io/v1alpha1/builds/d3caca66b4f3d3b0_1/condalock", + }, + }, + }, + } + with (fastqc_dir / "meta.yml").open("r") as fh: + current_meta = yaml.safe_load(fh) + current_meta.update(meta) + with (fastqc_dir / "meta.yml").open("w") as fh: + yaml.safe_dump(current_meta, fh) + + self.container_configs.generate_all_container_configs(default_config) + + conf_dir = self.pipeline_dir / "conf" + # Expected platforms and one expected container + expected_platforms = { + "docker_arm64": { + "FASTQC": "community.wave.seqera.io/library/fastqc:0.12.1--d3caca66b4f3d3b0", + }, + "singularity_oras_amd64": { + "FASTQC": "oras://community.wave.seqera.io/library/fastqc:0.12.1--0827550dd72a3745", + }, + "singularity_oras_arm64": { + "FASTQC": "oras://community.wave.seqera.io/library/fastqc:0.12.1--b2ccdee5305e5859", + }, + "singularity_https_amd64": { + "FASTQC": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/b2/b280a35770a70ed67008c1d6b6db118409bc3adbb3a98edcd55991189e5116f6/data", + }, + "singularity_https_arm64": { + "FASTQC": "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/76/76e744b425a6b4c7eb8f12e03fa15daf7054de36557d2f0c4eb53ad952f9b0e3/data", + }, + "conda_amd64_lockfile": { + "FASTQC": "https://wave.seqera.io/v1alpha1/builds/5cfd0f3cb6760c42_1/condalock", + }, + "conda_arm64_lockfile": { + "FASTQC": "https://wave.seqera.io/v1alpha1/builds/d3caca66b4f3d3b0_1/condalock", + }, + } + + for platform in expected_platforms.keys(): + cfg_path = conf_dir / f"containers_{platform}.config" + print(cfg_path) + assert cfg_path.exists() + with cfg_path.open("r") as fh: + content = fh.readlines() + print(content) + assert ( + f"process {{ withName: 'FASTQC' {{ container = '{expected_platforms[platform]['FASTQC']}' }} }}\n" + in content + )